ldc.c 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* ldc.c: Logical Domain Channel link-layer protocol driver.
  3. *
  4. * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/export.h>
  8. #include <linux/slab.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/delay.h>
  11. #include <linux/errno.h>
  12. #include <linux/string.h>
  13. #include <linux/scatterlist.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/list.h>
  16. #include <linux/init.h>
  17. #include <linux/bitmap.h>
  18. #include <asm/iommu-common.h>
  19. #include <asm/hypervisor.h>
  20. #include <asm/iommu.h>
  21. #include <asm/page.h>
  22. #include <asm/ldc.h>
  23. #include <asm/mdesc.h>
  24. #define DRV_MODULE_NAME "ldc"
  25. #define PFX DRV_MODULE_NAME ": "
  26. #define DRV_MODULE_VERSION "1.1"
  27. #define DRV_MODULE_RELDATE "July 22, 2008"
  28. #define COOKIE_PGSZ_CODE 0xf000000000000000ULL
  29. #define COOKIE_PGSZ_CODE_SHIFT 60ULL
  30. static char version[] =
  31. DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  32. /* Packet header layout for unreliable and reliable mode frames.
  33. * When in RAW mode, packets are simply straight 64-byte payloads
  34. * with no headers.
  35. */
  36. struct ldc_packet {
  37. u8 type;
  38. #define LDC_CTRL 0x01
  39. #define LDC_DATA 0x02
  40. #define LDC_ERR 0x10
  41. u8 stype;
  42. #define LDC_INFO 0x01
  43. #define LDC_ACK 0x02
  44. #define LDC_NACK 0x04
  45. u8 ctrl;
  46. #define LDC_VERS 0x01 /* Link Version */
  47. #define LDC_RTS 0x02 /* Request To Send */
  48. #define LDC_RTR 0x03 /* Ready To Receive */
  49. #define LDC_RDX 0x04 /* Ready for Data eXchange */
  50. #define LDC_CTRL_MSK 0x0f
  51. u8 env;
  52. #define LDC_LEN 0x3f
  53. #define LDC_FRAG_MASK 0xc0
  54. #define LDC_START 0x40
  55. #define LDC_STOP 0x80
  56. u32 seqid;
  57. union {
  58. u8 u_data[LDC_PACKET_SIZE - 8];
  59. struct {
  60. u32 pad;
  61. u32 ackid;
  62. u8 r_data[LDC_PACKET_SIZE - 8 - 8];
  63. } r;
  64. } u;
  65. };
  66. struct ldc_version {
  67. u16 major;
  68. u16 minor;
  69. };
  70. /* Ordered from largest major to lowest. */
  71. static struct ldc_version ver_arr[] = {
  72. { .major = 1, .minor = 0 },
  73. };
  74. #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE)
  75. #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE)
  76. struct ldc_channel;
  77. struct ldc_mode_ops {
  78. int (*write)(struct ldc_channel *, const void *, unsigned int);
  79. int (*read)(struct ldc_channel *, void *, unsigned int);
  80. };
  81. static const struct ldc_mode_ops raw_ops;
  82. static const struct ldc_mode_ops nonraw_ops;
  83. static const struct ldc_mode_ops stream_ops;
  84. int ldom_domaining_enabled;
  85. struct ldc_iommu {
  86. /* Protects ldc_unmap. */
  87. spinlock_t lock;
  88. struct ldc_mtable_entry *page_table;
  89. struct iommu_map_table iommu_map_table;
  90. };
  91. struct ldc_channel {
  92. /* Protects all operations that depend upon channel state. */
  93. spinlock_t lock;
  94. unsigned long id;
  95. u8 *mssbuf;
  96. u32 mssbuf_len;
  97. u32 mssbuf_off;
  98. struct ldc_packet *tx_base;
  99. unsigned long tx_head;
  100. unsigned long tx_tail;
  101. unsigned long tx_num_entries;
  102. unsigned long tx_ra;
  103. unsigned long tx_acked;
  104. struct ldc_packet *rx_base;
  105. unsigned long rx_head;
  106. unsigned long rx_tail;
  107. unsigned long rx_num_entries;
  108. unsigned long rx_ra;
  109. u32 rcv_nxt;
  110. u32 snd_nxt;
  111. unsigned long chan_state;
  112. struct ldc_channel_config cfg;
  113. void *event_arg;
  114. const struct ldc_mode_ops *mops;
  115. struct ldc_iommu iommu;
  116. struct ldc_version ver;
  117. u8 hs_state;
  118. #define LDC_HS_CLOSED 0x00
  119. #define LDC_HS_OPEN 0x01
  120. #define LDC_HS_GOTVERS 0x02
  121. #define LDC_HS_SENTRTR 0x03
  122. #define LDC_HS_GOTRTR 0x04
  123. #define LDC_HS_COMPLETE 0x10
  124. u8 flags;
  125. #define LDC_FLAG_ALLOCED_QUEUES 0x01
  126. #define LDC_FLAG_REGISTERED_QUEUES 0x02
  127. #define LDC_FLAG_REGISTERED_IRQS 0x04
  128. #define LDC_FLAG_RESET 0x10
  129. u8 mss;
  130. u8 state;
  131. #define LDC_IRQ_NAME_MAX 32
  132. char rx_irq_name[LDC_IRQ_NAME_MAX];
  133. char tx_irq_name[LDC_IRQ_NAME_MAX];
  134. struct hlist_head mh_list;
  135. struct hlist_node list;
  136. };
  137. #define ldcdbg(TYPE, f, a...) \
  138. do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
  139. printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
  140. } while (0)
  141. #define LDC_ABORT(lp) ldc_abort((lp), __func__)
  142. static const char *state_to_str(u8 state)
  143. {
  144. switch (state) {
  145. case LDC_STATE_INVALID:
  146. return "INVALID";
  147. case LDC_STATE_INIT:
  148. return "INIT";
  149. case LDC_STATE_BOUND:
  150. return "BOUND";
  151. case LDC_STATE_READY:
  152. return "READY";
  153. case LDC_STATE_CONNECTED:
  154. return "CONNECTED";
  155. default:
  156. return "<UNKNOWN>";
  157. }
  158. }
  159. static unsigned long __advance(unsigned long off, unsigned long num_entries)
  160. {
  161. off += LDC_PACKET_SIZE;
  162. if (off == (num_entries * LDC_PACKET_SIZE))
  163. off = 0;
  164. return off;
  165. }
  166. static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
  167. {
  168. return __advance(off, lp->rx_num_entries);
  169. }
  170. static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
  171. {
  172. return __advance(off, lp->tx_num_entries);
  173. }
  174. static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
  175. unsigned long *new_tail)
  176. {
  177. struct ldc_packet *p;
  178. unsigned long t;
  179. t = tx_advance(lp, lp->tx_tail);
  180. if (t == lp->tx_head)
  181. return NULL;
  182. *new_tail = t;
  183. p = lp->tx_base;
  184. return p + (lp->tx_tail / LDC_PACKET_SIZE);
  185. }
  186. /* When we are in reliable or stream mode, have to track the next packet
  187. * we haven't gotten an ACK for in the TX queue using tx_acked. We have
  188. * to be careful not to stomp over the queue past that point. During
  189. * the handshake, we don't have TX data packets pending in the queue
  190. * and that's why handshake_get_tx_packet() need not be mindful of
  191. * lp->tx_acked.
  192. */
  193. static unsigned long head_for_data(struct ldc_channel *lp)
  194. {
  195. if (lp->cfg.mode == LDC_MODE_STREAM)
  196. return lp->tx_acked;
  197. return lp->tx_head;
  198. }
  199. static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
  200. {
  201. unsigned long limit, tail, new_tail, diff;
  202. unsigned int mss;
  203. limit = head_for_data(lp);
  204. tail = lp->tx_tail;
  205. new_tail = tx_advance(lp, tail);
  206. if (new_tail == limit)
  207. return 0;
  208. if (limit > new_tail)
  209. diff = limit - new_tail;
  210. else
  211. diff = (limit +
  212. ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
  213. diff /= LDC_PACKET_SIZE;
  214. mss = lp->mss;
  215. if (diff * mss < size)
  216. return 0;
  217. return 1;
  218. }
  219. static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
  220. unsigned long *new_tail)
  221. {
  222. struct ldc_packet *p;
  223. unsigned long h, t;
  224. h = head_for_data(lp);
  225. t = tx_advance(lp, lp->tx_tail);
  226. if (t == h)
  227. return NULL;
  228. *new_tail = t;
  229. p = lp->tx_base;
  230. return p + (lp->tx_tail / LDC_PACKET_SIZE);
  231. }
  232. static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
  233. {
  234. unsigned long orig_tail = lp->tx_tail;
  235. int limit = 1000;
  236. lp->tx_tail = tail;
  237. while (limit-- > 0) {
  238. unsigned long err;
  239. err = sun4v_ldc_tx_set_qtail(lp->id, tail);
  240. if (!err)
  241. return 0;
  242. if (err != HV_EWOULDBLOCK) {
  243. lp->tx_tail = orig_tail;
  244. return -EINVAL;
  245. }
  246. udelay(1);
  247. }
  248. lp->tx_tail = orig_tail;
  249. return -EBUSY;
  250. }
  251. /* This just updates the head value in the hypervisor using
  252. * a polling loop with a timeout. The caller takes care of
  253. * upating software state representing the head change, if any.
  254. */
  255. static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
  256. {
  257. int limit = 1000;
  258. while (limit-- > 0) {
  259. unsigned long err;
  260. err = sun4v_ldc_rx_set_qhead(lp->id, head);
  261. if (!err)
  262. return 0;
  263. if (err != HV_EWOULDBLOCK)
  264. return -EINVAL;
  265. udelay(1);
  266. }
  267. return -EBUSY;
  268. }
  269. static int send_tx_packet(struct ldc_channel *lp,
  270. struct ldc_packet *p,
  271. unsigned long new_tail)
  272. {
  273. BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
  274. return set_tx_tail(lp, new_tail);
  275. }
  276. static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
  277. u8 stype, u8 ctrl,
  278. void *data, int dlen,
  279. unsigned long *new_tail)
  280. {
  281. struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
  282. if (p) {
  283. memset(p, 0, sizeof(*p));
  284. p->type = LDC_CTRL;
  285. p->stype = stype;
  286. p->ctrl = ctrl;
  287. if (data)
  288. memcpy(p->u.u_data, data, dlen);
  289. }
  290. return p;
  291. }
  292. static int start_handshake(struct ldc_channel *lp)
  293. {
  294. struct ldc_packet *p;
  295. struct ldc_version *ver;
  296. unsigned long new_tail;
  297. ver = &ver_arr[0];
  298. ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
  299. ver->major, ver->minor);
  300. p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
  301. ver, sizeof(*ver), &new_tail);
  302. if (p) {
  303. int err = send_tx_packet(lp, p, new_tail);
  304. if (!err)
  305. lp->flags &= ~LDC_FLAG_RESET;
  306. return err;
  307. }
  308. return -EBUSY;
  309. }
  310. static int send_version_nack(struct ldc_channel *lp,
  311. u16 major, u16 minor)
  312. {
  313. struct ldc_packet *p;
  314. struct ldc_version ver;
  315. unsigned long new_tail;
  316. ver.major = major;
  317. ver.minor = minor;
  318. p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
  319. &ver, sizeof(ver), &new_tail);
  320. if (p) {
  321. ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
  322. ver.major, ver.minor);
  323. return send_tx_packet(lp, p, new_tail);
  324. }
  325. return -EBUSY;
  326. }
  327. static int send_version_ack(struct ldc_channel *lp,
  328. struct ldc_version *vp)
  329. {
  330. struct ldc_packet *p;
  331. unsigned long new_tail;
  332. p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
  333. vp, sizeof(*vp), &new_tail);
  334. if (p) {
  335. ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
  336. vp->major, vp->minor);
  337. return send_tx_packet(lp, p, new_tail);
  338. }
  339. return -EBUSY;
  340. }
  341. static int send_rts(struct ldc_channel *lp)
  342. {
  343. struct ldc_packet *p;
  344. unsigned long new_tail;
  345. p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
  346. &new_tail);
  347. if (p) {
  348. p->env = lp->cfg.mode;
  349. p->seqid = 0;
  350. lp->rcv_nxt = 0;
  351. ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
  352. p->env, p->seqid);
  353. return send_tx_packet(lp, p, new_tail);
  354. }
  355. return -EBUSY;
  356. }
  357. static int send_rtr(struct ldc_channel *lp)
  358. {
  359. struct ldc_packet *p;
  360. unsigned long new_tail;
  361. p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
  362. &new_tail);
  363. if (p) {
  364. p->env = lp->cfg.mode;
  365. p->seqid = 0;
  366. ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
  367. p->env, p->seqid);
  368. return send_tx_packet(lp, p, new_tail);
  369. }
  370. return -EBUSY;
  371. }
  372. static int send_rdx(struct ldc_channel *lp)
  373. {
  374. struct ldc_packet *p;
  375. unsigned long new_tail;
  376. p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
  377. &new_tail);
  378. if (p) {
  379. p->env = 0;
  380. p->seqid = ++lp->snd_nxt;
  381. p->u.r.ackid = lp->rcv_nxt;
  382. ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
  383. p->env, p->seqid, p->u.r.ackid);
  384. return send_tx_packet(lp, p, new_tail);
  385. }
  386. return -EBUSY;
  387. }
  388. static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
  389. {
  390. struct ldc_packet *p;
  391. unsigned long new_tail;
  392. int err;
  393. p = data_get_tx_packet(lp, &new_tail);
  394. if (!p)
  395. return -EBUSY;
  396. memset(p, 0, sizeof(*p));
  397. p->type = data_pkt->type;
  398. p->stype = LDC_NACK;
  399. p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
  400. p->seqid = lp->snd_nxt + 1;
  401. p->u.r.ackid = lp->rcv_nxt;
  402. ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
  403. p->type, p->ctrl, p->seqid, p->u.r.ackid);
  404. err = send_tx_packet(lp, p, new_tail);
  405. if (!err)
  406. lp->snd_nxt++;
  407. return err;
  408. }
  409. static int ldc_abort(struct ldc_channel *lp, const char *msg)
  410. {
  411. unsigned long hv_err;
  412. ldcdbg(STATE, "ABORT[%s]\n", msg);
  413. ldc_print(lp);
  414. /* We report but do not act upon the hypervisor errors because
  415. * there really isn't much we can do if they fail at this point.
  416. */
  417. hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
  418. if (hv_err)
  419. printk(KERN_ERR PFX "ldc_abort: "
  420. "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
  421. lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
  422. hv_err = sun4v_ldc_tx_get_state(lp->id,
  423. &lp->tx_head,
  424. &lp->tx_tail,
  425. &lp->chan_state);
  426. if (hv_err)
  427. printk(KERN_ERR PFX "ldc_abort: "
  428. "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
  429. lp->id, hv_err);
  430. hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
  431. if (hv_err)
  432. printk(KERN_ERR PFX "ldc_abort: "
  433. "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
  434. lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
  435. /* Refetch the RX queue state as well, because we could be invoked
  436. * here in the queue processing context.
  437. */
  438. hv_err = sun4v_ldc_rx_get_state(lp->id,
  439. &lp->rx_head,
  440. &lp->rx_tail,
  441. &lp->chan_state);
  442. if (hv_err)
  443. printk(KERN_ERR PFX "ldc_abort: "
  444. "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
  445. lp->id, hv_err);
  446. return -ECONNRESET;
  447. }
  448. static struct ldc_version *find_by_major(u16 major)
  449. {
  450. struct ldc_version *ret = NULL;
  451. int i;
  452. for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
  453. struct ldc_version *v = &ver_arr[i];
  454. if (v->major <= major) {
  455. ret = v;
  456. break;
  457. }
  458. }
  459. return ret;
  460. }
  461. static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
  462. {
  463. struct ldc_version *vap;
  464. int err;
  465. ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
  466. vp->major, vp->minor);
  467. if (lp->hs_state == LDC_HS_GOTVERS) {
  468. lp->hs_state = LDC_HS_OPEN;
  469. memset(&lp->ver, 0, sizeof(lp->ver));
  470. }
  471. vap = find_by_major(vp->major);
  472. if (!vap) {
  473. err = send_version_nack(lp, 0, 0);
  474. } else if (vap->major != vp->major) {
  475. err = send_version_nack(lp, vap->major, vap->minor);
  476. } else {
  477. struct ldc_version ver = *vp;
  478. if (ver.minor > vap->minor)
  479. ver.minor = vap->minor;
  480. err = send_version_ack(lp, &ver);
  481. if (!err) {
  482. lp->ver = ver;
  483. lp->hs_state = LDC_HS_GOTVERS;
  484. }
  485. }
  486. if (err)
  487. return LDC_ABORT(lp);
  488. return 0;
  489. }
  490. static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
  491. {
  492. ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
  493. vp->major, vp->minor);
  494. if (lp->hs_state == LDC_HS_GOTVERS) {
  495. if (lp->ver.major != vp->major ||
  496. lp->ver.minor != vp->minor)
  497. return LDC_ABORT(lp);
  498. } else {
  499. lp->ver = *vp;
  500. lp->hs_state = LDC_HS_GOTVERS;
  501. }
  502. if (send_rts(lp))
  503. return LDC_ABORT(lp);
  504. return 0;
  505. }
  506. static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
  507. {
  508. struct ldc_version *vap;
  509. struct ldc_packet *p;
  510. unsigned long new_tail;
  511. if (vp->major == 0 && vp->minor == 0)
  512. return LDC_ABORT(lp);
  513. vap = find_by_major(vp->major);
  514. if (!vap)
  515. return LDC_ABORT(lp);
  516. p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
  517. vap, sizeof(*vap),
  518. &new_tail);
  519. if (!p)
  520. return LDC_ABORT(lp);
  521. return send_tx_packet(lp, p, new_tail);
  522. }
  523. static int process_version(struct ldc_channel *lp,
  524. struct ldc_packet *p)
  525. {
  526. struct ldc_version *vp;
  527. vp = (struct ldc_version *) p->u.u_data;
  528. switch (p->stype) {
  529. case LDC_INFO:
  530. return process_ver_info(lp, vp);
  531. case LDC_ACK:
  532. return process_ver_ack(lp, vp);
  533. case LDC_NACK:
  534. return process_ver_nack(lp, vp);
  535. default:
  536. return LDC_ABORT(lp);
  537. }
  538. }
  539. static int process_rts(struct ldc_channel *lp,
  540. struct ldc_packet *p)
  541. {
  542. ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
  543. p->stype, p->seqid, p->env);
  544. if (p->stype != LDC_INFO ||
  545. lp->hs_state != LDC_HS_GOTVERS ||
  546. p->env != lp->cfg.mode)
  547. return LDC_ABORT(lp);
  548. lp->snd_nxt = p->seqid;
  549. lp->rcv_nxt = p->seqid;
  550. lp->hs_state = LDC_HS_SENTRTR;
  551. if (send_rtr(lp))
  552. return LDC_ABORT(lp);
  553. return 0;
  554. }
  555. static int process_rtr(struct ldc_channel *lp,
  556. struct ldc_packet *p)
  557. {
  558. ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
  559. p->stype, p->seqid, p->env);
  560. if (p->stype != LDC_INFO ||
  561. p->env != lp->cfg.mode)
  562. return LDC_ABORT(lp);
  563. lp->snd_nxt = p->seqid;
  564. lp->hs_state = LDC_HS_COMPLETE;
  565. ldc_set_state(lp, LDC_STATE_CONNECTED);
  566. send_rdx(lp);
  567. return LDC_EVENT_UP;
  568. }
  569. static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
  570. {
  571. return lp->rcv_nxt + 1 == seqid;
  572. }
  573. static int process_rdx(struct ldc_channel *lp,
  574. struct ldc_packet *p)
  575. {
  576. ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
  577. p->stype, p->seqid, p->env, p->u.r.ackid);
  578. if (p->stype != LDC_INFO ||
  579. !(rx_seq_ok(lp, p->seqid)))
  580. return LDC_ABORT(lp);
  581. lp->rcv_nxt = p->seqid;
  582. lp->hs_state = LDC_HS_COMPLETE;
  583. ldc_set_state(lp, LDC_STATE_CONNECTED);
  584. return LDC_EVENT_UP;
  585. }
  586. static int process_control_frame(struct ldc_channel *lp,
  587. struct ldc_packet *p)
  588. {
  589. switch (p->ctrl) {
  590. case LDC_VERS:
  591. return process_version(lp, p);
  592. case LDC_RTS:
  593. return process_rts(lp, p);
  594. case LDC_RTR:
  595. return process_rtr(lp, p);
  596. case LDC_RDX:
  597. return process_rdx(lp, p);
  598. default:
  599. return LDC_ABORT(lp);
  600. }
  601. }
  602. static int process_error_frame(struct ldc_channel *lp,
  603. struct ldc_packet *p)
  604. {
  605. return LDC_ABORT(lp);
  606. }
  607. static int process_data_ack(struct ldc_channel *lp,
  608. struct ldc_packet *ack)
  609. {
  610. unsigned long head = lp->tx_acked;
  611. u32 ackid = ack->u.r.ackid;
  612. while (1) {
  613. struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
  614. head = tx_advance(lp, head);
  615. if (p->seqid == ackid) {
  616. lp->tx_acked = head;
  617. return 0;
  618. }
  619. if (head == lp->tx_tail)
  620. return LDC_ABORT(lp);
  621. }
  622. return 0;
  623. }
  624. static void send_events(struct ldc_channel *lp, unsigned int event_mask)
  625. {
  626. if (event_mask & LDC_EVENT_RESET)
  627. lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
  628. if (event_mask & LDC_EVENT_UP)
  629. lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
  630. if (event_mask & LDC_EVENT_DATA_READY)
  631. lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
  632. }
  633. static irqreturn_t ldc_rx(int irq, void *dev_id)
  634. {
  635. struct ldc_channel *lp = dev_id;
  636. unsigned long orig_state, flags;
  637. unsigned int event_mask;
  638. spin_lock_irqsave(&lp->lock, flags);
  639. orig_state = lp->chan_state;
  640. /* We should probably check for hypervisor errors here and
  641. * reset the LDC channel if we get one.
  642. */
  643. sun4v_ldc_rx_get_state(lp->id,
  644. &lp->rx_head,
  645. &lp->rx_tail,
  646. &lp->chan_state);
  647. ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
  648. orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
  649. event_mask = 0;
  650. if (lp->cfg.mode == LDC_MODE_RAW &&
  651. lp->chan_state == LDC_CHANNEL_UP) {
  652. lp->hs_state = LDC_HS_COMPLETE;
  653. ldc_set_state(lp, LDC_STATE_CONNECTED);
  654. /*
  655. * Generate an LDC_EVENT_UP event if the channel
  656. * was not already up.
  657. */
  658. if (orig_state != LDC_CHANNEL_UP) {
  659. event_mask |= LDC_EVENT_UP;
  660. orig_state = lp->chan_state;
  661. }
  662. }
  663. /* If we are in reset state, flush the RX queue and ignore
  664. * everything.
  665. */
  666. if (lp->flags & LDC_FLAG_RESET) {
  667. (void) ldc_rx_reset(lp);
  668. goto out;
  669. }
  670. /* Once we finish the handshake, we let the ldc_read()
  671. * paths do all of the control frame and state management.
  672. * Just trigger the callback.
  673. */
  674. if (lp->hs_state == LDC_HS_COMPLETE) {
  675. handshake_complete:
  676. if (lp->chan_state != orig_state) {
  677. unsigned int event = LDC_EVENT_RESET;
  678. if (lp->chan_state == LDC_CHANNEL_UP)
  679. event = LDC_EVENT_UP;
  680. event_mask |= event;
  681. }
  682. if (lp->rx_head != lp->rx_tail)
  683. event_mask |= LDC_EVENT_DATA_READY;
  684. goto out;
  685. }
  686. if (lp->chan_state != orig_state)
  687. goto out;
  688. while (lp->rx_head != lp->rx_tail) {
  689. struct ldc_packet *p;
  690. unsigned long new;
  691. int err;
  692. p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
  693. switch (p->type) {
  694. case LDC_CTRL:
  695. err = process_control_frame(lp, p);
  696. if (err > 0)
  697. event_mask |= err;
  698. break;
  699. case LDC_DATA:
  700. event_mask |= LDC_EVENT_DATA_READY;
  701. err = 0;
  702. break;
  703. case LDC_ERR:
  704. err = process_error_frame(lp, p);
  705. break;
  706. default:
  707. err = LDC_ABORT(lp);
  708. break;
  709. }
  710. if (err < 0)
  711. break;
  712. new = lp->rx_head;
  713. new += LDC_PACKET_SIZE;
  714. if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
  715. new = 0;
  716. lp->rx_head = new;
  717. err = __set_rx_head(lp, new);
  718. if (err < 0) {
  719. (void) LDC_ABORT(lp);
  720. break;
  721. }
  722. if (lp->hs_state == LDC_HS_COMPLETE)
  723. goto handshake_complete;
  724. }
  725. out:
  726. spin_unlock_irqrestore(&lp->lock, flags);
  727. send_events(lp, event_mask);
  728. return IRQ_HANDLED;
  729. }
  730. static irqreturn_t ldc_tx(int irq, void *dev_id)
  731. {
  732. struct ldc_channel *lp = dev_id;
  733. unsigned long flags, orig_state;
  734. unsigned int event_mask = 0;
  735. spin_lock_irqsave(&lp->lock, flags);
  736. orig_state = lp->chan_state;
  737. /* We should probably check for hypervisor errors here and
  738. * reset the LDC channel if we get one.
  739. */
  740. sun4v_ldc_tx_get_state(lp->id,
  741. &lp->tx_head,
  742. &lp->tx_tail,
  743. &lp->chan_state);
  744. ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
  745. orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
  746. if (lp->cfg.mode == LDC_MODE_RAW &&
  747. lp->chan_state == LDC_CHANNEL_UP) {
  748. lp->hs_state = LDC_HS_COMPLETE;
  749. ldc_set_state(lp, LDC_STATE_CONNECTED);
  750. /*
  751. * Generate an LDC_EVENT_UP event if the channel
  752. * was not already up.
  753. */
  754. if (orig_state != LDC_CHANNEL_UP) {
  755. event_mask |= LDC_EVENT_UP;
  756. orig_state = lp->chan_state;
  757. }
  758. }
  759. spin_unlock_irqrestore(&lp->lock, flags);
  760. send_events(lp, event_mask);
  761. return IRQ_HANDLED;
  762. }
  763. /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
  764. * XXX that addition and removal from the ldc_channel_list has
  765. * XXX atomicity, otherwise the __ldc_channel_exists() check is
  766. * XXX totally pointless as another thread can slip into ldc_alloc()
  767. * XXX and add a channel with the same ID. There also needs to be
  768. * XXX a spinlock for ldc_channel_list.
  769. */
  770. static HLIST_HEAD(ldc_channel_list);
  771. static int __ldc_channel_exists(unsigned long id)
  772. {
  773. struct ldc_channel *lp;
  774. hlist_for_each_entry(lp, &ldc_channel_list, list) {
  775. if (lp->id == id)
  776. return 1;
  777. }
  778. return 0;
  779. }
  780. static int alloc_queue(const char *name, unsigned long num_entries,
  781. struct ldc_packet **base, unsigned long *ra)
  782. {
  783. unsigned long size, order;
  784. void *q;
  785. size = num_entries * LDC_PACKET_SIZE;
  786. order = get_order(size);
  787. q = (void *) __get_free_pages(GFP_KERNEL, order);
  788. if (!q) {
  789. printk(KERN_ERR PFX "Alloc of %s queue failed with "
  790. "size=%lu order=%lu\n", name, size, order);
  791. return -ENOMEM;
  792. }
  793. memset(q, 0, PAGE_SIZE << order);
  794. *base = q;
  795. *ra = __pa(q);
  796. return 0;
  797. }
  798. static void free_queue(unsigned long num_entries, struct ldc_packet *q)
  799. {
  800. unsigned long size, order;
  801. if (!q)
  802. return;
  803. size = num_entries * LDC_PACKET_SIZE;
  804. order = get_order(size);
  805. free_pages((unsigned long)q, order);
  806. }
  807. static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
  808. {
  809. u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
  810. /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
  811. cookie &= ~COOKIE_PGSZ_CODE;
  812. return (cookie >> (13ULL + (szcode * 3ULL)));
  813. }
  814. static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
  815. unsigned long entry, unsigned long npages)
  816. {
  817. struct ldc_mtable_entry *base;
  818. unsigned long i, shift;
  819. shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
  820. base = iommu->page_table + entry;
  821. for (i = 0; i < npages; i++) {
  822. if (base->cookie)
  823. sun4v_ldc_revoke(id, cookie + (i << shift),
  824. base->cookie);
  825. base->mte = 0;
  826. }
  827. }
  828. /* XXX Make this configurable... XXX */
  829. #define LDC_IOTABLE_SIZE (8 * 1024)
  830. static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
  831. {
  832. unsigned long sz, num_tsb_entries, tsbsize, order;
  833. struct ldc_iommu *ldc_iommu = &lp->iommu;
  834. struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
  835. struct ldc_mtable_entry *table;
  836. unsigned long hv_err;
  837. int err;
  838. num_tsb_entries = LDC_IOTABLE_SIZE;
  839. tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
  840. spin_lock_init(&ldc_iommu->lock);
  841. sz = num_tsb_entries / 8;
  842. sz = (sz + 7UL) & ~7UL;
  843. iommu->map = kzalloc(sz, GFP_KERNEL);
  844. if (!iommu->map) {
  845. printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
  846. return -ENOMEM;
  847. }
  848. iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
  849. NULL, false /* no large pool */,
  850. 1 /* npools */,
  851. true /* skip span boundary check */);
  852. order = get_order(tsbsize);
  853. table = (struct ldc_mtable_entry *)
  854. __get_free_pages(GFP_KERNEL, order);
  855. err = -ENOMEM;
  856. if (!table) {
  857. printk(KERN_ERR PFX "Alloc of MTE table failed, "
  858. "size=%lu order=%lu\n", tsbsize, order);
  859. goto out_free_map;
  860. }
  861. memset(table, 0, PAGE_SIZE << order);
  862. ldc_iommu->page_table = table;
  863. hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
  864. num_tsb_entries);
  865. err = -EINVAL;
  866. if (hv_err)
  867. goto out_free_table;
  868. return 0;
  869. out_free_table:
  870. free_pages((unsigned long) table, order);
  871. ldc_iommu->page_table = NULL;
  872. out_free_map:
  873. kfree(iommu->map);
  874. iommu->map = NULL;
  875. return err;
  876. }
  877. static void ldc_iommu_release(struct ldc_channel *lp)
  878. {
  879. struct ldc_iommu *ldc_iommu = &lp->iommu;
  880. struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
  881. unsigned long num_tsb_entries, tsbsize, order;
  882. (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
  883. num_tsb_entries = iommu->poolsize * iommu->nr_pools;
  884. tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
  885. order = get_order(tsbsize);
  886. free_pages((unsigned long) ldc_iommu->page_table, order);
  887. ldc_iommu->page_table = NULL;
  888. kfree(iommu->map);
  889. iommu->map = NULL;
  890. }
  891. struct ldc_channel *ldc_alloc(unsigned long id,
  892. const struct ldc_channel_config *cfgp,
  893. void *event_arg,
  894. const char *name)
  895. {
  896. struct ldc_channel *lp;
  897. const struct ldc_mode_ops *mops;
  898. unsigned long dummy1, dummy2, hv_err;
  899. u8 mss, *mssbuf;
  900. int err;
  901. err = -ENODEV;
  902. if (!ldom_domaining_enabled)
  903. goto out_err;
  904. err = -EINVAL;
  905. if (!cfgp)
  906. goto out_err;
  907. if (!name)
  908. goto out_err;
  909. switch (cfgp->mode) {
  910. case LDC_MODE_RAW:
  911. mops = &raw_ops;
  912. mss = LDC_PACKET_SIZE;
  913. break;
  914. case LDC_MODE_UNRELIABLE:
  915. mops = &nonraw_ops;
  916. mss = LDC_PACKET_SIZE - 8;
  917. break;
  918. case LDC_MODE_STREAM:
  919. mops = &stream_ops;
  920. mss = LDC_PACKET_SIZE - 8 - 8;
  921. break;
  922. default:
  923. goto out_err;
  924. }
  925. if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
  926. goto out_err;
  927. hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
  928. err = -ENODEV;
  929. if (hv_err == HV_ECHANNEL)
  930. goto out_err;
  931. err = -EEXIST;
  932. if (__ldc_channel_exists(id))
  933. goto out_err;
  934. mssbuf = NULL;
  935. lp = kzalloc(sizeof(*lp), GFP_KERNEL);
  936. err = -ENOMEM;
  937. if (!lp)
  938. goto out_err;
  939. spin_lock_init(&lp->lock);
  940. lp->id = id;
  941. err = ldc_iommu_init(name, lp);
  942. if (err)
  943. goto out_free_ldc;
  944. lp->mops = mops;
  945. lp->mss = mss;
  946. lp->cfg = *cfgp;
  947. if (!lp->cfg.mtu)
  948. lp->cfg.mtu = LDC_DEFAULT_MTU;
  949. if (lp->cfg.mode == LDC_MODE_STREAM) {
  950. mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
  951. if (!mssbuf) {
  952. err = -ENOMEM;
  953. goto out_free_iommu;
  954. }
  955. lp->mssbuf = mssbuf;
  956. }
  957. lp->event_arg = event_arg;
  958. /* XXX allow setting via ldc_channel_config to override defaults
  959. * XXX or use some formula based upon mtu
  960. */
  961. lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
  962. lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
  963. err = alloc_queue("TX", lp->tx_num_entries,
  964. &lp->tx_base, &lp->tx_ra);
  965. if (err)
  966. goto out_free_mssbuf;
  967. err = alloc_queue("RX", lp->rx_num_entries,
  968. &lp->rx_base, &lp->rx_ra);
  969. if (err)
  970. goto out_free_txq;
  971. lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
  972. lp->hs_state = LDC_HS_CLOSED;
  973. ldc_set_state(lp, LDC_STATE_INIT);
  974. INIT_HLIST_NODE(&lp->list);
  975. hlist_add_head(&lp->list, &ldc_channel_list);
  976. INIT_HLIST_HEAD(&lp->mh_list);
  977. snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
  978. snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
  979. err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
  980. lp->rx_irq_name, lp);
  981. if (err)
  982. goto out_free_txq;
  983. err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
  984. lp->tx_irq_name, lp);
  985. if (err) {
  986. free_irq(lp->cfg.rx_irq, lp);
  987. goto out_free_txq;
  988. }
  989. return lp;
  990. out_free_txq:
  991. free_queue(lp->tx_num_entries, lp->tx_base);
  992. out_free_mssbuf:
  993. kfree(mssbuf);
  994. out_free_iommu:
  995. ldc_iommu_release(lp);
  996. out_free_ldc:
  997. kfree(lp);
  998. out_err:
  999. return ERR_PTR(err);
  1000. }
  1001. EXPORT_SYMBOL(ldc_alloc);
  1002. void ldc_unbind(struct ldc_channel *lp)
  1003. {
  1004. if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
  1005. free_irq(lp->cfg.rx_irq, lp);
  1006. free_irq(lp->cfg.tx_irq, lp);
  1007. lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
  1008. }
  1009. if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
  1010. sun4v_ldc_tx_qconf(lp->id, 0, 0);
  1011. sun4v_ldc_rx_qconf(lp->id, 0, 0);
  1012. lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
  1013. }
  1014. if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
  1015. free_queue(lp->tx_num_entries, lp->tx_base);
  1016. free_queue(lp->rx_num_entries, lp->rx_base);
  1017. lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
  1018. }
  1019. ldc_set_state(lp, LDC_STATE_INIT);
  1020. }
  1021. EXPORT_SYMBOL(ldc_unbind);
  1022. void ldc_free(struct ldc_channel *lp)
  1023. {
  1024. ldc_unbind(lp);
  1025. hlist_del(&lp->list);
  1026. kfree(lp->mssbuf);
  1027. ldc_iommu_release(lp);
  1028. kfree(lp);
  1029. }
  1030. EXPORT_SYMBOL(ldc_free);
  1031. /* Bind the channel. This registers the LDC queues with
  1032. * the hypervisor and puts the channel into a pseudo-listening
  1033. * state. This does not initiate a handshake, ldc_connect() does
  1034. * that.
  1035. */
  1036. int ldc_bind(struct ldc_channel *lp)
  1037. {
  1038. unsigned long hv_err, flags;
  1039. int err = -EINVAL;
  1040. if (lp->state != LDC_STATE_INIT)
  1041. return -EINVAL;
  1042. spin_lock_irqsave(&lp->lock, flags);
  1043. enable_irq(lp->cfg.rx_irq);
  1044. enable_irq(lp->cfg.tx_irq);
  1045. lp->flags |= LDC_FLAG_REGISTERED_IRQS;
  1046. err = -ENODEV;
  1047. hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
  1048. if (hv_err)
  1049. goto out_free_irqs;
  1050. hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
  1051. if (hv_err)
  1052. goto out_free_irqs;
  1053. hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
  1054. if (hv_err)
  1055. goto out_unmap_tx;
  1056. hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
  1057. if (hv_err)
  1058. goto out_unmap_tx;
  1059. lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
  1060. hv_err = sun4v_ldc_tx_get_state(lp->id,
  1061. &lp->tx_head,
  1062. &lp->tx_tail,
  1063. &lp->chan_state);
  1064. err = -EBUSY;
  1065. if (hv_err)
  1066. goto out_unmap_rx;
  1067. lp->tx_acked = lp->tx_head;
  1068. lp->hs_state = LDC_HS_OPEN;
  1069. ldc_set_state(lp, LDC_STATE_BOUND);
  1070. if (lp->cfg.mode == LDC_MODE_RAW) {
  1071. /*
  1072. * There is no handshake in RAW mode, so handshake
  1073. * is completed.
  1074. */
  1075. lp->hs_state = LDC_HS_COMPLETE;
  1076. }
  1077. spin_unlock_irqrestore(&lp->lock, flags);
  1078. return 0;
  1079. out_unmap_rx:
  1080. lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
  1081. sun4v_ldc_rx_qconf(lp->id, 0, 0);
  1082. out_unmap_tx:
  1083. sun4v_ldc_tx_qconf(lp->id, 0, 0);
  1084. out_free_irqs:
  1085. lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
  1086. free_irq(lp->cfg.tx_irq, lp);
  1087. free_irq(lp->cfg.rx_irq, lp);
  1088. spin_unlock_irqrestore(&lp->lock, flags);
  1089. return err;
  1090. }
  1091. EXPORT_SYMBOL(ldc_bind);
  1092. int ldc_connect(struct ldc_channel *lp)
  1093. {
  1094. unsigned long flags;
  1095. int err;
  1096. if (lp->cfg.mode == LDC_MODE_RAW)
  1097. return -EINVAL;
  1098. spin_lock_irqsave(&lp->lock, flags);
  1099. if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
  1100. !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
  1101. lp->hs_state != LDC_HS_OPEN)
  1102. err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
  1103. else
  1104. err = start_handshake(lp);
  1105. spin_unlock_irqrestore(&lp->lock, flags);
  1106. return err;
  1107. }
  1108. EXPORT_SYMBOL(ldc_connect);
  1109. int ldc_disconnect(struct ldc_channel *lp)
  1110. {
  1111. unsigned long hv_err, flags;
  1112. int err;
  1113. if (lp->cfg.mode == LDC_MODE_RAW)
  1114. return -EINVAL;
  1115. if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
  1116. !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
  1117. return -EINVAL;
  1118. spin_lock_irqsave(&lp->lock, flags);
  1119. err = -ENODEV;
  1120. hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
  1121. if (hv_err)
  1122. goto out_err;
  1123. hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
  1124. if (hv_err)
  1125. goto out_err;
  1126. hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
  1127. if (hv_err)
  1128. goto out_err;
  1129. hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
  1130. if (hv_err)
  1131. goto out_err;
  1132. ldc_set_state(lp, LDC_STATE_BOUND);
  1133. lp->hs_state = LDC_HS_OPEN;
  1134. lp->flags |= LDC_FLAG_RESET;
  1135. spin_unlock_irqrestore(&lp->lock, flags);
  1136. return 0;
  1137. out_err:
  1138. sun4v_ldc_tx_qconf(lp->id, 0, 0);
  1139. sun4v_ldc_rx_qconf(lp->id, 0, 0);
  1140. free_irq(lp->cfg.tx_irq, lp);
  1141. free_irq(lp->cfg.rx_irq, lp);
  1142. lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
  1143. LDC_FLAG_REGISTERED_QUEUES);
  1144. ldc_set_state(lp, LDC_STATE_INIT);
  1145. spin_unlock_irqrestore(&lp->lock, flags);
  1146. return err;
  1147. }
  1148. EXPORT_SYMBOL(ldc_disconnect);
  1149. int ldc_state(struct ldc_channel *lp)
  1150. {
  1151. return lp->state;
  1152. }
  1153. EXPORT_SYMBOL(ldc_state);
  1154. void ldc_set_state(struct ldc_channel *lp, u8 state)
  1155. {
  1156. ldcdbg(STATE, "STATE (%s) --> (%s)\n",
  1157. state_to_str(lp->state),
  1158. state_to_str(state));
  1159. lp->state = state;
  1160. }
  1161. EXPORT_SYMBOL(ldc_set_state);
  1162. int ldc_mode(struct ldc_channel *lp)
  1163. {
  1164. return lp->cfg.mode;
  1165. }
  1166. EXPORT_SYMBOL(ldc_mode);
  1167. int ldc_rx_reset(struct ldc_channel *lp)
  1168. {
  1169. return __set_rx_head(lp, lp->rx_tail);
  1170. }
  1171. EXPORT_SYMBOL(ldc_rx_reset);
  1172. void __ldc_print(struct ldc_channel *lp, const char *caller)
  1173. {
  1174. pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
  1175. "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
  1176. "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
  1177. "\trcv_nxt=%u snd_nxt=%u\n",
  1178. caller, lp->id, lp->flags, state_to_str(lp->state),
  1179. lp->chan_state, lp->hs_state,
  1180. lp->rx_head, lp->rx_tail, lp->rx_num_entries,
  1181. lp->tx_head, lp->tx_tail, lp->tx_num_entries,
  1182. lp->rcv_nxt, lp->snd_nxt);
  1183. }
  1184. EXPORT_SYMBOL(__ldc_print);
  1185. static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
  1186. {
  1187. struct ldc_packet *p;
  1188. unsigned long new_tail, hv_err;
  1189. int err;
  1190. hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
  1191. &lp->chan_state);
  1192. if (unlikely(hv_err))
  1193. return -EBUSY;
  1194. if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
  1195. return LDC_ABORT(lp);
  1196. if (size > LDC_PACKET_SIZE)
  1197. return -EMSGSIZE;
  1198. p = data_get_tx_packet(lp, &new_tail);
  1199. if (!p)
  1200. return -EAGAIN;
  1201. memcpy(p, buf, size);
  1202. err = send_tx_packet(lp, p, new_tail);
  1203. if (!err)
  1204. err = size;
  1205. return err;
  1206. }
  1207. static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
  1208. {
  1209. struct ldc_packet *p;
  1210. unsigned long hv_err, new;
  1211. int err;
  1212. if (size < LDC_PACKET_SIZE)
  1213. return -EINVAL;
  1214. hv_err = sun4v_ldc_rx_get_state(lp->id,
  1215. &lp->rx_head,
  1216. &lp->rx_tail,
  1217. &lp->chan_state);
  1218. if (hv_err)
  1219. return LDC_ABORT(lp);
  1220. if (lp->chan_state == LDC_CHANNEL_DOWN ||
  1221. lp->chan_state == LDC_CHANNEL_RESETTING)
  1222. return -ECONNRESET;
  1223. if (lp->rx_head == lp->rx_tail)
  1224. return 0;
  1225. p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
  1226. memcpy(buf, p, LDC_PACKET_SIZE);
  1227. new = rx_advance(lp, lp->rx_head);
  1228. lp->rx_head = new;
  1229. err = __set_rx_head(lp, new);
  1230. if (err < 0)
  1231. err = -ECONNRESET;
  1232. else
  1233. err = LDC_PACKET_SIZE;
  1234. return err;
  1235. }
  1236. static const struct ldc_mode_ops raw_ops = {
  1237. .write = write_raw,
  1238. .read = read_raw,
  1239. };
  1240. static int write_nonraw(struct ldc_channel *lp, const void *buf,
  1241. unsigned int size)
  1242. {
  1243. unsigned long hv_err, tail;
  1244. unsigned int copied;
  1245. u32 seq;
  1246. int err;
  1247. hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
  1248. &lp->chan_state);
  1249. if (unlikely(hv_err))
  1250. return -EBUSY;
  1251. if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
  1252. return LDC_ABORT(lp);
  1253. if (!tx_has_space_for(lp, size))
  1254. return -EAGAIN;
  1255. seq = lp->snd_nxt;
  1256. copied = 0;
  1257. tail = lp->tx_tail;
  1258. while (copied < size) {
  1259. struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
  1260. u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
  1261. p->u.u_data :
  1262. p->u.r.r_data);
  1263. int data_len;
  1264. p->type = LDC_DATA;
  1265. p->stype = LDC_INFO;
  1266. p->ctrl = 0;
  1267. data_len = size - copied;
  1268. if (data_len > lp->mss)
  1269. data_len = lp->mss;
  1270. BUG_ON(data_len > LDC_LEN);
  1271. p->env = (data_len |
  1272. (copied == 0 ? LDC_START : 0) |
  1273. (data_len == size - copied ? LDC_STOP : 0));
  1274. p->seqid = ++seq;
  1275. ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
  1276. p->type,
  1277. p->stype,
  1278. p->ctrl,
  1279. p->env,
  1280. p->seqid);
  1281. memcpy(data, buf, data_len);
  1282. buf += data_len;
  1283. copied += data_len;
  1284. tail = tx_advance(lp, tail);
  1285. }
  1286. err = set_tx_tail(lp, tail);
  1287. if (!err) {
  1288. lp->snd_nxt = seq;
  1289. err = size;
  1290. }
  1291. return err;
  1292. }
  1293. static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
  1294. struct ldc_packet *first_frag)
  1295. {
  1296. int err;
  1297. if (first_frag)
  1298. lp->rcv_nxt = first_frag->seqid - 1;
  1299. err = send_data_nack(lp, p);
  1300. if (err)
  1301. return err;
  1302. err = ldc_rx_reset(lp);
  1303. if (err < 0)
  1304. return LDC_ABORT(lp);
  1305. return 0;
  1306. }
  1307. static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
  1308. {
  1309. if (p->stype & LDC_ACK) {
  1310. int err = process_data_ack(lp, p);
  1311. if (err)
  1312. return err;
  1313. }
  1314. if (p->stype & LDC_NACK)
  1315. return LDC_ABORT(lp);
  1316. return 0;
  1317. }
  1318. static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
  1319. {
  1320. unsigned long dummy;
  1321. int limit = 1000;
  1322. ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
  1323. cur_head, lp->rx_head, lp->rx_tail);
  1324. while (limit-- > 0) {
  1325. unsigned long hv_err;
  1326. hv_err = sun4v_ldc_rx_get_state(lp->id,
  1327. &dummy,
  1328. &lp->rx_tail,
  1329. &lp->chan_state);
  1330. if (hv_err)
  1331. return LDC_ABORT(lp);
  1332. if (lp->chan_state == LDC_CHANNEL_DOWN ||
  1333. lp->chan_state == LDC_CHANNEL_RESETTING)
  1334. return -ECONNRESET;
  1335. if (cur_head != lp->rx_tail) {
  1336. ldcdbg(DATA, "DATA WAIT DONE "
  1337. "head[%lx] tail[%lx] chan_state[%lx]\n",
  1338. dummy, lp->rx_tail, lp->chan_state);
  1339. return 0;
  1340. }
  1341. udelay(1);
  1342. }
  1343. return -EAGAIN;
  1344. }
  1345. static int rx_set_head(struct ldc_channel *lp, unsigned long head)
  1346. {
  1347. int err = __set_rx_head(lp, head);
  1348. if (err < 0)
  1349. return LDC_ABORT(lp);
  1350. lp->rx_head = head;
  1351. return 0;
  1352. }
  1353. static void send_data_ack(struct ldc_channel *lp)
  1354. {
  1355. unsigned long new_tail;
  1356. struct ldc_packet *p;
  1357. p = data_get_tx_packet(lp, &new_tail);
  1358. if (likely(p)) {
  1359. int err;
  1360. memset(p, 0, sizeof(*p));
  1361. p->type = LDC_DATA;
  1362. p->stype = LDC_ACK;
  1363. p->ctrl = 0;
  1364. p->seqid = lp->snd_nxt + 1;
  1365. p->u.r.ackid = lp->rcv_nxt;
  1366. err = send_tx_packet(lp, p, new_tail);
  1367. if (!err)
  1368. lp->snd_nxt++;
  1369. }
  1370. }
  1371. static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
  1372. {
  1373. struct ldc_packet *first_frag;
  1374. unsigned long hv_err, new;
  1375. int err, copied;
  1376. hv_err = sun4v_ldc_rx_get_state(lp->id,
  1377. &lp->rx_head,
  1378. &lp->rx_tail,
  1379. &lp->chan_state);
  1380. if (hv_err)
  1381. return LDC_ABORT(lp);
  1382. if (lp->chan_state == LDC_CHANNEL_DOWN ||
  1383. lp->chan_state == LDC_CHANNEL_RESETTING)
  1384. return -ECONNRESET;
  1385. if (lp->rx_head == lp->rx_tail)
  1386. return 0;
  1387. first_frag = NULL;
  1388. copied = err = 0;
  1389. new = lp->rx_head;
  1390. while (1) {
  1391. struct ldc_packet *p;
  1392. int pkt_len;
  1393. BUG_ON(new == lp->rx_tail);
  1394. p = lp->rx_base + (new / LDC_PACKET_SIZE);
  1395. ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
  1396. "rcv_nxt[%08x]\n",
  1397. p->type,
  1398. p->stype,
  1399. p->ctrl,
  1400. p->env,
  1401. p->seqid,
  1402. p->u.r.ackid,
  1403. lp->rcv_nxt);
  1404. if (unlikely(!rx_seq_ok(lp, p->seqid))) {
  1405. err = rx_bad_seq(lp, p, first_frag);
  1406. copied = 0;
  1407. break;
  1408. }
  1409. if (p->type & LDC_CTRL) {
  1410. err = process_control_frame(lp, p);
  1411. if (err < 0)
  1412. break;
  1413. err = 0;
  1414. }
  1415. lp->rcv_nxt = p->seqid;
  1416. /*
  1417. * If this is a control-only packet, there is nothing
  1418. * else to do but advance the rx queue since the packet
  1419. * was already processed above.
  1420. */
  1421. if (!(p->type & LDC_DATA)) {
  1422. new = rx_advance(lp, new);
  1423. break;
  1424. }
  1425. if (p->stype & (LDC_ACK | LDC_NACK)) {
  1426. err = data_ack_nack(lp, p);
  1427. if (err)
  1428. break;
  1429. }
  1430. if (!(p->stype & LDC_INFO)) {
  1431. new = rx_advance(lp, new);
  1432. err = rx_set_head(lp, new);
  1433. if (err)
  1434. break;
  1435. goto no_data;
  1436. }
  1437. pkt_len = p->env & LDC_LEN;
  1438. /* Every initial packet starts with the START bit set.
  1439. *
  1440. * Singleton packets will have both START+STOP set.
  1441. *
  1442. * Fragments will have START set in the first frame, STOP
  1443. * set in the last frame, and neither bit set in middle
  1444. * frames of the packet.
  1445. *
  1446. * Therefore if we are at the beginning of a packet and
  1447. * we don't see START, or we are in the middle of a fragmented
  1448. * packet and do see START, we are unsynchronized and should
  1449. * flush the RX queue.
  1450. */
  1451. if ((first_frag == NULL && !(p->env & LDC_START)) ||
  1452. (first_frag != NULL && (p->env & LDC_START))) {
  1453. if (!first_frag)
  1454. new = rx_advance(lp, new);
  1455. err = rx_set_head(lp, new);
  1456. if (err)
  1457. break;
  1458. if (!first_frag)
  1459. goto no_data;
  1460. }
  1461. if (!first_frag)
  1462. first_frag = p;
  1463. if (pkt_len > size - copied) {
  1464. /* User didn't give us a big enough buffer,
  1465. * what to do? This is a pretty serious error.
  1466. *
  1467. * Since we haven't updated the RX ring head to
  1468. * consume any of the packets, signal the error
  1469. * to the user and just leave the RX ring alone.
  1470. *
  1471. * This seems the best behavior because this allows
  1472. * a user of the LDC layer to start with a small
  1473. * RX buffer for ldc_read() calls and use -EMSGSIZE
  1474. * as a cue to enlarge it's read buffer.
  1475. */
  1476. err = -EMSGSIZE;
  1477. break;
  1478. }
  1479. /* Ok, we are gonna eat this one. */
  1480. new = rx_advance(lp, new);
  1481. memcpy(buf,
  1482. (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
  1483. p->u.u_data : p->u.r.r_data), pkt_len);
  1484. buf += pkt_len;
  1485. copied += pkt_len;
  1486. if (p->env & LDC_STOP)
  1487. break;
  1488. no_data:
  1489. if (new == lp->rx_tail) {
  1490. err = rx_data_wait(lp, new);
  1491. if (err)
  1492. break;
  1493. }
  1494. }
  1495. if (!err)
  1496. err = rx_set_head(lp, new);
  1497. if (err && first_frag)
  1498. lp->rcv_nxt = first_frag->seqid - 1;
  1499. if (!err) {
  1500. err = copied;
  1501. if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
  1502. send_data_ack(lp);
  1503. }
  1504. return err;
  1505. }
  1506. static const struct ldc_mode_ops nonraw_ops = {
  1507. .write = write_nonraw,
  1508. .read = read_nonraw,
  1509. };
  1510. static int write_stream(struct ldc_channel *lp, const void *buf,
  1511. unsigned int size)
  1512. {
  1513. if (size > lp->cfg.mtu)
  1514. size = lp->cfg.mtu;
  1515. return write_nonraw(lp, buf, size);
  1516. }
  1517. static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
  1518. {
  1519. if (!lp->mssbuf_len) {
  1520. int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
  1521. if (err < 0)
  1522. return err;
  1523. lp->mssbuf_len = err;
  1524. lp->mssbuf_off = 0;
  1525. }
  1526. if (size > lp->mssbuf_len)
  1527. size = lp->mssbuf_len;
  1528. memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
  1529. lp->mssbuf_off += size;
  1530. lp->mssbuf_len -= size;
  1531. return size;
  1532. }
  1533. static const struct ldc_mode_ops stream_ops = {
  1534. .write = write_stream,
  1535. .read = read_stream,
  1536. };
  1537. int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
  1538. {
  1539. unsigned long flags;
  1540. int err;
  1541. if (!buf)
  1542. return -EINVAL;
  1543. if (!size)
  1544. return 0;
  1545. spin_lock_irqsave(&lp->lock, flags);
  1546. if (lp->hs_state != LDC_HS_COMPLETE)
  1547. err = -ENOTCONN;
  1548. else
  1549. err = lp->mops->write(lp, buf, size);
  1550. spin_unlock_irqrestore(&lp->lock, flags);
  1551. return err;
  1552. }
  1553. EXPORT_SYMBOL(ldc_write);
  1554. int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
  1555. {
  1556. unsigned long flags;
  1557. int err;
  1558. ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
  1559. if (!buf)
  1560. return -EINVAL;
  1561. if (!size)
  1562. return 0;
  1563. spin_lock_irqsave(&lp->lock, flags);
  1564. if (lp->hs_state != LDC_HS_COMPLETE)
  1565. err = -ENOTCONN;
  1566. else
  1567. err = lp->mops->read(lp, buf, size);
  1568. spin_unlock_irqrestore(&lp->lock, flags);
  1569. ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
  1570. lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
  1571. return err;
  1572. }
  1573. EXPORT_SYMBOL(ldc_read);
  1574. static u64 pagesize_code(void)
  1575. {
  1576. switch (PAGE_SIZE) {
  1577. default:
  1578. case (8ULL * 1024ULL):
  1579. return 0;
  1580. case (64ULL * 1024ULL):
  1581. return 1;
  1582. case (512ULL * 1024ULL):
  1583. return 2;
  1584. case (4ULL * 1024ULL * 1024ULL):
  1585. return 3;
  1586. case (32ULL * 1024ULL * 1024ULL):
  1587. return 4;
  1588. case (256ULL * 1024ULL * 1024ULL):
  1589. return 5;
  1590. }
  1591. }
  1592. static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
  1593. {
  1594. return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
  1595. (index << PAGE_SHIFT) |
  1596. page_offset);
  1597. }
  1598. static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
  1599. unsigned long npages)
  1600. {
  1601. long entry;
  1602. entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
  1603. npages, NULL, (unsigned long)-1, 0);
  1604. if (unlikely(entry == IOMMU_ERROR_CODE))
  1605. return NULL;
  1606. return iommu->page_table + entry;
  1607. }
  1608. static u64 perm_to_mte(unsigned int map_perm)
  1609. {
  1610. u64 mte_base;
  1611. mte_base = pagesize_code();
  1612. if (map_perm & LDC_MAP_SHADOW) {
  1613. if (map_perm & LDC_MAP_R)
  1614. mte_base |= LDC_MTE_COPY_R;
  1615. if (map_perm & LDC_MAP_W)
  1616. mte_base |= LDC_MTE_COPY_W;
  1617. }
  1618. if (map_perm & LDC_MAP_DIRECT) {
  1619. if (map_perm & LDC_MAP_R)
  1620. mte_base |= LDC_MTE_READ;
  1621. if (map_perm & LDC_MAP_W)
  1622. mte_base |= LDC_MTE_WRITE;
  1623. if (map_perm & LDC_MAP_X)
  1624. mte_base |= LDC_MTE_EXEC;
  1625. }
  1626. if (map_perm & LDC_MAP_IO) {
  1627. if (map_perm & LDC_MAP_R)
  1628. mte_base |= LDC_MTE_IOMMU_R;
  1629. if (map_perm & LDC_MAP_W)
  1630. mte_base |= LDC_MTE_IOMMU_W;
  1631. }
  1632. return mte_base;
  1633. }
  1634. static int pages_in_region(unsigned long base, long len)
  1635. {
  1636. int count = 0;
  1637. do {
  1638. unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
  1639. len -= (new - base);
  1640. base = new;
  1641. count++;
  1642. } while (len > 0);
  1643. return count;
  1644. }
  1645. struct cookie_state {
  1646. struct ldc_mtable_entry *page_table;
  1647. struct ldc_trans_cookie *cookies;
  1648. u64 mte_base;
  1649. u64 prev_cookie;
  1650. u32 pte_idx;
  1651. u32 nc;
  1652. };
  1653. static void fill_cookies(struct cookie_state *sp, unsigned long pa,
  1654. unsigned long off, unsigned long len)
  1655. {
  1656. do {
  1657. unsigned long tlen, new = pa + PAGE_SIZE;
  1658. u64 this_cookie;
  1659. sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
  1660. tlen = PAGE_SIZE;
  1661. if (off)
  1662. tlen = PAGE_SIZE - off;
  1663. if (tlen > len)
  1664. tlen = len;
  1665. this_cookie = make_cookie(sp->pte_idx,
  1666. pagesize_code(), off);
  1667. off = 0;
  1668. if (this_cookie == sp->prev_cookie) {
  1669. sp->cookies[sp->nc - 1].cookie_size += tlen;
  1670. } else {
  1671. sp->cookies[sp->nc].cookie_addr = this_cookie;
  1672. sp->cookies[sp->nc].cookie_size = tlen;
  1673. sp->nc++;
  1674. }
  1675. sp->prev_cookie = this_cookie + tlen;
  1676. sp->pte_idx++;
  1677. len -= tlen;
  1678. pa = new;
  1679. } while (len > 0);
  1680. }
  1681. static int sg_count_one(struct scatterlist *sg)
  1682. {
  1683. unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
  1684. long len = sg->length;
  1685. if ((sg->offset | len) & (8UL - 1))
  1686. return -EFAULT;
  1687. return pages_in_region(base + sg->offset, len);
  1688. }
  1689. static int sg_count_pages(struct scatterlist *sg, int num_sg)
  1690. {
  1691. int count;
  1692. int i;
  1693. count = 0;
  1694. for (i = 0; i < num_sg; i++) {
  1695. int err = sg_count_one(sg + i);
  1696. if (err < 0)
  1697. return err;
  1698. count += err;
  1699. }
  1700. return count;
  1701. }
  1702. int ldc_map_sg(struct ldc_channel *lp,
  1703. struct scatterlist *sg, int num_sg,
  1704. struct ldc_trans_cookie *cookies, int ncookies,
  1705. unsigned int map_perm)
  1706. {
  1707. unsigned long i, npages;
  1708. struct ldc_mtable_entry *base;
  1709. struct cookie_state state;
  1710. struct ldc_iommu *iommu;
  1711. int err;
  1712. struct scatterlist *s;
  1713. if (map_perm & ~LDC_MAP_ALL)
  1714. return -EINVAL;
  1715. err = sg_count_pages(sg, num_sg);
  1716. if (err < 0)
  1717. return err;
  1718. npages = err;
  1719. if (err > ncookies)
  1720. return -EMSGSIZE;
  1721. iommu = &lp->iommu;
  1722. base = alloc_npages(iommu, npages);
  1723. if (!base)
  1724. return -ENOMEM;
  1725. state.page_table = iommu->page_table;
  1726. state.cookies = cookies;
  1727. state.mte_base = perm_to_mte(map_perm);
  1728. state.prev_cookie = ~(u64)0;
  1729. state.pte_idx = (base - iommu->page_table);
  1730. state.nc = 0;
  1731. for_each_sg(sg, s, num_sg, i) {
  1732. fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
  1733. s->offset, s->length);
  1734. }
  1735. return state.nc;
  1736. }
  1737. EXPORT_SYMBOL(ldc_map_sg);
  1738. int ldc_map_single(struct ldc_channel *lp,
  1739. void *buf, unsigned int len,
  1740. struct ldc_trans_cookie *cookies, int ncookies,
  1741. unsigned int map_perm)
  1742. {
  1743. unsigned long npages, pa;
  1744. struct ldc_mtable_entry *base;
  1745. struct cookie_state state;
  1746. struct ldc_iommu *iommu;
  1747. if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
  1748. return -EINVAL;
  1749. pa = __pa(buf);
  1750. if ((pa | len) & (8UL - 1))
  1751. return -EFAULT;
  1752. npages = pages_in_region(pa, len);
  1753. iommu = &lp->iommu;
  1754. base = alloc_npages(iommu, npages);
  1755. if (!base)
  1756. return -ENOMEM;
  1757. state.page_table = iommu->page_table;
  1758. state.cookies = cookies;
  1759. state.mte_base = perm_to_mte(map_perm);
  1760. state.prev_cookie = ~(u64)0;
  1761. state.pte_idx = (base - iommu->page_table);
  1762. state.nc = 0;
  1763. fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
  1764. BUG_ON(state.nc > ncookies);
  1765. return state.nc;
  1766. }
  1767. EXPORT_SYMBOL(ldc_map_single);
  1768. static void free_npages(unsigned long id, struct ldc_iommu *iommu,
  1769. u64 cookie, u64 size)
  1770. {
  1771. unsigned long npages, entry;
  1772. npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
  1773. entry = ldc_cookie_to_index(cookie, iommu);
  1774. ldc_demap(iommu, id, cookie, entry, npages);
  1775. iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
  1776. }
  1777. void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
  1778. int ncookies)
  1779. {
  1780. struct ldc_iommu *iommu = &lp->iommu;
  1781. int i;
  1782. unsigned long flags;
  1783. spin_lock_irqsave(&iommu->lock, flags);
  1784. for (i = 0; i < ncookies; i++) {
  1785. u64 addr = cookies[i].cookie_addr;
  1786. u64 size = cookies[i].cookie_size;
  1787. free_npages(lp->id, iommu, addr, size);
  1788. }
  1789. spin_unlock_irqrestore(&iommu->lock, flags);
  1790. }
  1791. EXPORT_SYMBOL(ldc_unmap);
  1792. int ldc_copy(struct ldc_channel *lp, int copy_dir,
  1793. void *buf, unsigned int len, unsigned long offset,
  1794. struct ldc_trans_cookie *cookies, int ncookies)
  1795. {
  1796. unsigned int orig_len;
  1797. unsigned long ra;
  1798. int i;
  1799. if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
  1800. printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
  1801. lp->id, copy_dir);
  1802. return -EINVAL;
  1803. }
  1804. ra = __pa(buf);
  1805. if ((ra | len | offset) & (8UL - 1)) {
  1806. printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
  1807. "ra[%lx] len[%x] offset[%lx]\n",
  1808. lp->id, ra, len, offset);
  1809. return -EFAULT;
  1810. }
  1811. if (lp->hs_state != LDC_HS_COMPLETE ||
  1812. (lp->flags & LDC_FLAG_RESET)) {
  1813. printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
  1814. "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
  1815. return -ECONNRESET;
  1816. }
  1817. orig_len = len;
  1818. for (i = 0; i < ncookies; i++) {
  1819. unsigned long cookie_raddr = cookies[i].cookie_addr;
  1820. unsigned long this_len = cookies[i].cookie_size;
  1821. unsigned long actual_len;
  1822. if (unlikely(offset)) {
  1823. unsigned long this_off = offset;
  1824. if (this_off > this_len)
  1825. this_off = this_len;
  1826. offset -= this_off;
  1827. this_len -= this_off;
  1828. if (!this_len)
  1829. continue;
  1830. cookie_raddr += this_off;
  1831. }
  1832. if (this_len > len)
  1833. this_len = len;
  1834. while (1) {
  1835. unsigned long hv_err;
  1836. hv_err = sun4v_ldc_copy(lp->id, copy_dir,
  1837. cookie_raddr, ra,
  1838. this_len, &actual_len);
  1839. if (unlikely(hv_err)) {
  1840. printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
  1841. "HV error %lu\n",
  1842. lp->id, hv_err);
  1843. if (lp->hs_state != LDC_HS_COMPLETE ||
  1844. (lp->flags & LDC_FLAG_RESET))
  1845. return -ECONNRESET;
  1846. else
  1847. return -EFAULT;
  1848. }
  1849. cookie_raddr += actual_len;
  1850. ra += actual_len;
  1851. len -= actual_len;
  1852. if (actual_len == this_len)
  1853. break;
  1854. this_len -= actual_len;
  1855. }
  1856. if (!len)
  1857. break;
  1858. }
  1859. /* It is caller policy what to do about short copies.
  1860. * For example, a networking driver can declare the
  1861. * packet a runt and drop it.
  1862. */
  1863. return orig_len - len;
  1864. }
  1865. EXPORT_SYMBOL(ldc_copy);
  1866. void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
  1867. struct ldc_trans_cookie *cookies, int *ncookies,
  1868. unsigned int map_perm)
  1869. {
  1870. void *buf;
  1871. int err;
  1872. if (len & (8UL - 1))
  1873. return ERR_PTR(-EINVAL);
  1874. buf = kzalloc(len, GFP_ATOMIC);
  1875. if (!buf)
  1876. return ERR_PTR(-ENOMEM);
  1877. err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
  1878. if (err < 0) {
  1879. kfree(buf);
  1880. return ERR_PTR(err);
  1881. }
  1882. *ncookies = err;
  1883. return buf;
  1884. }
  1885. EXPORT_SYMBOL(ldc_alloc_exp_dring);
  1886. void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
  1887. struct ldc_trans_cookie *cookies, int ncookies)
  1888. {
  1889. ldc_unmap(lp, cookies, ncookies);
  1890. kfree(buf);
  1891. }
  1892. EXPORT_SYMBOL(ldc_free_exp_dring);
  1893. static int __init ldc_init(void)
  1894. {
  1895. unsigned long major, minor;
  1896. struct mdesc_handle *hp;
  1897. const u64 *v;
  1898. int err;
  1899. u64 mp;
  1900. hp = mdesc_grab();
  1901. if (!hp)
  1902. return -ENODEV;
  1903. mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
  1904. err = -ENODEV;
  1905. if (mp == MDESC_NODE_NULL)
  1906. goto out;
  1907. v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
  1908. if (!v)
  1909. goto out;
  1910. major = 1;
  1911. minor = 0;
  1912. if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
  1913. printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
  1914. goto out;
  1915. }
  1916. printk(KERN_INFO "%s", version);
  1917. if (!*v) {
  1918. printk(KERN_INFO PFX "Domaining disabled.\n");
  1919. goto out;
  1920. }
  1921. ldom_domaining_enabled = 1;
  1922. err = 0;
  1923. out:
  1924. mdesc_release(hp);
  1925. return err;
  1926. }
  1927. core_initcall(ldc_init);