card_ddcb.c 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * IBM Accelerator Family 'GenWQE'
  4. *
  5. * (C) Copyright IBM Corp. 2013
  6. *
  7. * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
  8. * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
  9. * Author: Michael Jung <mijung@gmx.net>
  10. * Author: Michael Ruettger <michael@ibmra.de>
  11. */
  12. /*
  13. * Device Driver Control Block (DDCB) queue support. Definition of
  14. * interrupt handlers for queue support as well as triggering the
  15. * health monitor code in case of problems. The current hardware uses
  16. * an MSI interrupt which is shared between error handling and
  17. * functional code.
  18. */
  19. #include <linux/types.h>
  20. #include <linux/sched.h>
  21. #include <linux/wait.h>
  22. #include <linux/pci.h>
  23. #include <linux/string.h>
  24. #include <linux/dma-mapping.h>
  25. #include <linux/delay.h>
  26. #include <linux/module.h>
  27. #include <linux/interrupt.h>
  28. #include <linux/crc-itu-t.h>
  29. #include "card_base.h"
  30. #include "card_ddcb.h"
  31. /*
  32. * N: next DDCB, this is where the next DDCB will be put.
  33. * A: active DDCB, this is where the code will look for the next completion.
  34. * x: DDCB is enqueued, we are waiting for its completion.
  35. * Situation (1): Empty queue
  36. * +---+---+---+---+---+---+---+---+
  37. * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  38. * | | | | | | | | |
  39. * +---+---+---+---+---+---+---+---+
  40. * A/N
  41. * enqueued_ddcbs = A - N = 2 - 2 = 0
  42. *
  43. * Situation (2): Wrapped, N > A
  44. * +---+---+---+---+---+---+---+---+
  45. * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  46. * | | | x | x | | | | |
  47. * +---+---+---+---+---+---+---+---+
  48. * A N
  49. * enqueued_ddcbs = N - A = 4 - 2 = 2
  50. *
  51. * Situation (3): Queue wrapped, A > N
  52. * +---+---+---+---+---+---+---+---+
  53. * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  54. * | x | x | | | x | x | x | x |
  55. * +---+---+---+---+---+---+---+---+
  56. * N A
  57. * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6
  58. *
  59. * Situation (4a): Queue full N > A
  60. * +---+---+---+---+---+---+---+---+
  61. * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  62. * | x | x | x | x | x | x | x | |
  63. * +---+---+---+---+---+---+---+---+
  64. * A N
  65. *
  66. * enqueued_ddcbs = N - A = 7 - 0 = 7
  67. *
  68. * Situation (4a): Queue full A > N
  69. * +---+---+---+---+---+---+---+---+
  70. * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  71. * | x | x | x | | x | x | x | x |
  72. * +---+---+---+---+---+---+---+---+
  73. * N A
  74. * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
  75. */
  76. static int queue_empty(struct ddcb_queue *queue)
  77. {
  78. return queue->ddcb_next == queue->ddcb_act;
  79. }
  80. static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
  81. {
  82. if (queue->ddcb_next >= queue->ddcb_act)
  83. return queue->ddcb_next - queue->ddcb_act;
  84. return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
  85. }
  86. static int queue_free_ddcbs(struct ddcb_queue *queue)
  87. {
  88. int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
  89. if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
  90. return 0;
  91. }
  92. return free_ddcbs;
  93. }
  94. /*
  95. * Use of the PRIV field in the DDCB for queue debugging:
  96. *
  97. * (1) Trying to get rid of a DDCB which saw a timeout:
  98. * pddcb->priv[6] = 0xcc; # cleared
  99. *
  100. * (2) Append a DDCB via NEXT bit:
  101. * pddcb->priv[7] = 0xaa; # appended
  102. *
  103. * (3) DDCB needed tapping:
  104. * pddcb->priv[7] = 0xbb; # tapped
  105. *
  106. * (4) DDCB marked as correctly finished:
  107. * pddcb->priv[6] = 0xff; # finished
  108. */
  109. static inline void ddcb_mark_tapped(struct ddcb *pddcb)
  110. {
  111. pddcb->priv[7] = 0xbb; /* tapped */
  112. }
  113. static inline void ddcb_mark_appended(struct ddcb *pddcb)
  114. {
  115. pddcb->priv[7] = 0xaa; /* appended */
  116. }
  117. static inline void ddcb_mark_cleared(struct ddcb *pddcb)
  118. {
  119. pddcb->priv[6] = 0xcc; /* cleared */
  120. }
  121. static inline void ddcb_mark_finished(struct ddcb *pddcb)
  122. {
  123. pddcb->priv[6] = 0xff; /* finished */
  124. }
  125. static inline void ddcb_mark_unused(struct ddcb *pddcb)
  126. {
  127. pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
  128. }
  129. /**
  130. * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
  131. * @buff: pointer to data buffer
  132. * @len: length of data for calculation
  133. * @init: initial crc (0xffff at start)
  134. *
  135. * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021)
  136. * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
  137. * should result in a crc16 of 0x89c3
  138. *
  139. * Return: crc16 checksum in big endian format !
  140. */
  141. static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
  142. {
  143. return crc_itu_t(init, buff, len);
  144. }
  145. static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
  146. {
  147. int i;
  148. struct ddcb *pddcb;
  149. unsigned long flags;
  150. struct pci_dev *pci_dev = cd->pci_dev;
  151. spin_lock_irqsave(&cd->print_lock, flags);
  152. dev_info(&pci_dev->dev,
  153. "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
  154. cd->card_idx, queue->ddcb_act, queue->ddcb_next);
  155. pddcb = queue->ddcb_vaddr;
  156. for (i = 0; i < queue->ddcb_max; i++) {
  157. dev_err(&pci_dev->dev,
  158. " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
  159. i == queue->ddcb_act ? '>' : ' ',
  160. i,
  161. be16_to_cpu(pddcb->retc_16),
  162. be16_to_cpu(pddcb->seqnum_16),
  163. pddcb->hsi,
  164. pddcb->shi,
  165. be64_to_cpu(pddcb->priv_64),
  166. pddcb->cmd);
  167. pddcb++;
  168. }
  169. spin_unlock_irqrestore(&cd->print_lock, flags);
  170. }
  171. struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
  172. {
  173. struct ddcb_requ *req;
  174. req = kzalloc(sizeof(*req), GFP_KERNEL);
  175. if (!req)
  176. return NULL;
  177. return &req->cmd;
  178. }
  179. void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
  180. {
  181. struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
  182. kfree(req);
  183. }
  184. static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
  185. {
  186. return req->req_state;
  187. }
  188. static inline void ddcb_requ_set_state(struct ddcb_requ *req,
  189. enum genwqe_requ_state new_state)
  190. {
  191. req->req_state = new_state;
  192. }
  193. static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
  194. {
  195. return req->cmd.ddata_addr != 0x0;
  196. }
  197. /**
  198. * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
  199. * @cd: pointer to genwqe device descriptor
  200. * @req: DDCB work request
  201. *
  202. * Status of ddcb_requ mirrors this hardware state, but is copied in
  203. * the ddcb_requ on interrupt/polling function. The lowlevel code
  204. * should check the hardware state directly, the higher level code
  205. * should check the copy.
  206. *
  207. * This function will also return true if the state of the queue is
  208. * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
  209. * shutdown case.
  210. */
  211. static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
  212. {
  213. return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
  214. (cd->card_state != GENWQE_CARD_USED);
  215. }
  216. #define RET_DDCB_APPENDED 1
  217. #define RET_DDCB_TAPPED 2
  218. /**
  219. * enqueue_ddcb() - Enqueue a DDCB
  220. * @cd: pointer to genwqe device descriptor
  221. * @queue: queue this operation should be done on
  222. * @pddcb: pointer to ddcb structure
  223. * @ddcb_no: pointer to ddcb number being tapped
  224. *
  225. * Start execution of DDCB by tapping or append to queue via NEXT
  226. * bit. This is done by an atomic 'compare and swap' instruction and
  227. * checking SHI and HSI of the previous DDCB.
  228. *
  229. * This function must only be called with ddcb_lock held.
  230. *
  231. * Return: 1 if new DDCB is appended to previous
  232. * 2 if DDCB queue is tapped via register/simulation
  233. */
  234. static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
  235. struct ddcb *pddcb, int ddcb_no)
  236. {
  237. unsigned int try;
  238. int prev_no;
  239. struct ddcb *prev_ddcb;
  240. __be32 old, new, icrc_hsi_shi;
  241. u64 num;
  242. /*
  243. * For performance checks a Dispatch Timestamp can be put into
  244. * DDCB It is supposed to use the SLU's free running counter,
  245. * but this requires PCIe cycles.
  246. */
  247. ddcb_mark_unused(pddcb);
  248. /* check previous DDCB if already fetched */
  249. prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
  250. prev_ddcb = &queue->ddcb_vaddr[prev_no];
  251. /*
  252. * It might have happened that the HSI.FETCHED bit is
  253. * set. Retry in this case. Therefore I expect maximum 2 times
  254. * trying.
  255. */
  256. ddcb_mark_appended(pddcb);
  257. for (try = 0; try < 2; try++) {
  258. old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
  259. /* try to append via NEXT bit if prev DDCB is not completed */
  260. if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
  261. break;
  262. new = (old | DDCB_NEXT_BE32);
  263. wmb(); /* need to ensure write ordering */
  264. icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
  265. if (icrc_hsi_shi == old)
  266. return RET_DDCB_APPENDED; /* appended to queue */
  267. }
  268. /* Queue must be re-started by updating QUEUE_OFFSET */
  269. ddcb_mark_tapped(pddcb);
  270. num = (u64)ddcb_no << 8;
  271. wmb(); /* need to ensure write ordering */
  272. __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
  273. return RET_DDCB_TAPPED;
  274. }
  275. /**
  276. * copy_ddcb_results() - Copy output state from real DDCB to request
  277. * @req: pointer to requsted DDCB parameters
  278. * @ddcb_no: pointer to ddcb number being tapped
  279. *
  280. * Copy DDCB ASV to request struct. There is no endian
  281. * conversion made, since data structure in ASV is still
  282. * unknown here.
  283. *
  284. * This is needed by:
  285. * - genwqe_purge_ddcb()
  286. * - genwqe_check_ddcb_queue()
  287. */
  288. static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
  289. {
  290. struct ddcb_queue *queue = req->queue;
  291. struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
  292. memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
  293. /* copy status flags of the variant part */
  294. req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16);
  295. req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
  296. req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
  297. req->cmd.attn = be16_to_cpu(pddcb->attn_16);
  298. req->cmd.progress = be32_to_cpu(pddcb->progress_32);
  299. req->cmd.retc = be16_to_cpu(pddcb->retc_16);
  300. if (ddcb_requ_collect_debug_data(req)) {
  301. int prev_no = (ddcb_no == 0) ?
  302. queue->ddcb_max - 1 : ddcb_no - 1;
  303. struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
  304. memcpy(&req->debug_data.ddcb_finished, pddcb,
  305. sizeof(req->debug_data.ddcb_finished));
  306. memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
  307. sizeof(req->debug_data.ddcb_prev));
  308. }
  309. }
  310. /**
  311. * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
  312. * @cd: pointer to genwqe device descriptor
  313. * @queue: queue to be checked
  314. *
  315. * Return: Number of DDCBs which were finished
  316. */
  317. static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
  318. struct ddcb_queue *queue)
  319. {
  320. unsigned long flags;
  321. int ddcbs_finished = 0;
  322. struct pci_dev *pci_dev = cd->pci_dev;
  323. spin_lock_irqsave(&queue->ddcb_lock, flags);
  324. /* FIXME avoid soft locking CPU */
  325. while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
  326. struct ddcb *pddcb;
  327. struct ddcb_requ *req;
  328. u16 vcrc, vcrc_16, retc_16;
  329. pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
  330. if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
  331. 0x00000000)
  332. goto go_home; /* not completed, continue waiting */
  333. wmb(); /* Add sync to decouple prev. read operations */
  334. /* Note: DDCB could be purged */
  335. req = queue->ddcb_req[queue->ddcb_act];
  336. if (req == NULL) {
  337. /* this occurs if DDCB is purged, not an error */
  338. /* Move active DDCB further; Nothing to do anymore. */
  339. goto pick_next_one;
  340. }
  341. /*
  342. * HSI=0x44 (fetched and completed), but RETC is
  343. * 0x101, or even worse 0x000.
  344. *
  345. * In case of seeing the queue in inconsistent state
  346. * we read the errcnts and the queue status to provide
  347. * a trigger for our PCIe analyzer stop capturing.
  348. */
  349. retc_16 = be16_to_cpu(pddcb->retc_16);
  350. if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
  351. u64 errcnts, status;
  352. u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
  353. errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
  354. status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
  355. dev_err(&pci_dev->dev,
  356. "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
  357. __func__, be16_to_cpu(pddcb->seqnum_16),
  358. pddcb->hsi, retc_16, errcnts, status,
  359. queue->ddcb_daddr + ddcb_offs);
  360. }
  361. copy_ddcb_results(req, queue->ddcb_act);
  362. queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
  363. dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
  364. genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
  365. ddcb_mark_finished(pddcb);
  366. /* calculate CRC_16 to see if VCRC is correct */
  367. vcrc = genwqe_crc16(pddcb->asv,
  368. VCRC_LENGTH(req->cmd.asv_length),
  369. 0xffff);
  370. vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
  371. if (vcrc != vcrc_16) {
  372. printk_ratelimited(KERN_ERR
  373. "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
  374. GENWQE_DEVNAME, dev_name(&pci_dev->dev),
  375. pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
  376. vcrc, vcrc_16);
  377. }
  378. ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
  379. queue->ddcbs_completed++;
  380. queue->ddcbs_in_flight--;
  381. /* wake up process waiting for this DDCB, and
  382. processes on the busy queue */
  383. wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
  384. wake_up_interruptible(&queue->busy_waitq);
  385. pick_next_one:
  386. queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
  387. ddcbs_finished++;
  388. }
  389. go_home:
  390. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  391. return ddcbs_finished;
  392. }
  393. /**
  394. * __genwqe_wait_ddcb(): Waits until DDCB is completed
  395. * @cd: pointer to genwqe device descriptor
  396. * @req: pointer to requsted DDCB parameters
  397. *
  398. * The Service Layer will update the RETC in DDCB when processing is
  399. * pending or done.
  400. *
  401. * Return: > 0 remaining jiffies, DDCB completed
  402. * -ETIMEDOUT when timeout
  403. * -ERESTARTSYS when ^C
  404. * -EINVAL when unknown error condition
  405. *
  406. * When an error is returned the called needs to ensure that
  407. * purge_ddcb() is being called to get the &req removed from the
  408. * queue.
  409. */
  410. int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
  411. {
  412. int rc;
  413. unsigned int ddcb_no;
  414. struct ddcb_queue *queue;
  415. struct pci_dev *pci_dev = cd->pci_dev;
  416. if (req == NULL)
  417. return -EINVAL;
  418. queue = req->queue;
  419. if (queue == NULL)
  420. return -EINVAL;
  421. ddcb_no = req->num;
  422. if (ddcb_no >= queue->ddcb_max)
  423. return -EINVAL;
  424. rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
  425. ddcb_requ_finished(cd, req),
  426. GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
  427. /*
  428. * We need to distinguish 3 cases here:
  429. * 1. rc == 0 timeout occured
  430. * 2. rc == -ERESTARTSYS signal received
  431. * 3. rc > 0 remaining jiffies condition is true
  432. */
  433. if (rc == 0) {
  434. struct ddcb_queue *queue = req->queue;
  435. struct ddcb *pddcb;
  436. /*
  437. * Timeout may be caused by long task switching time.
  438. * When timeout happens, check if the request has
  439. * meanwhile completed.
  440. */
  441. genwqe_check_ddcb_queue(cd, req->queue);
  442. if (ddcb_requ_finished(cd, req))
  443. return rc;
  444. dev_err(&pci_dev->dev,
  445. "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
  446. __func__, req->num, rc, ddcb_requ_get_state(req),
  447. req);
  448. dev_err(&pci_dev->dev,
  449. "[%s] IO_QUEUE_STATUS=0x%016llx\n", __func__,
  450. __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
  451. pddcb = &queue->ddcb_vaddr[req->num];
  452. genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
  453. print_ddcb_info(cd, req->queue);
  454. return -ETIMEDOUT;
  455. } else if (rc == -ERESTARTSYS) {
  456. return rc;
  457. /*
  458. * EINTR: Stops the application
  459. * ERESTARTSYS: Restartable systemcall; called again
  460. */
  461. } else if (rc < 0) {
  462. dev_err(&pci_dev->dev,
  463. "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
  464. __func__, req->num, rc, ddcb_requ_get_state(req));
  465. return -EINVAL;
  466. }
  467. /* Severe error occured. Driver is forced to stop operation */
  468. if (cd->card_state != GENWQE_CARD_USED) {
  469. dev_err(&pci_dev->dev,
  470. "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
  471. __func__, req->num, rc);
  472. return -EIO;
  473. }
  474. return rc;
  475. }
  476. /**
  477. * get_next_ddcb() - Get next available DDCB
  478. * @cd: pointer to genwqe device descriptor
  479. * @queue: DDCB queue
  480. * @num: internal DDCB number
  481. *
  482. * DDCB's content is completely cleared but presets for PRE and
  483. * SEQNUM. This function must only be called when ddcb_lock is held.
  484. *
  485. * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
  486. */
  487. static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
  488. struct ddcb_queue *queue,
  489. int *num)
  490. {
  491. u64 *pu64;
  492. struct ddcb *pddcb;
  493. if (queue_free_ddcbs(queue) == 0) /* queue is full */
  494. return NULL;
  495. /* find new ddcb */
  496. pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
  497. /* if it is not completed, we are not allowed to use it */
  498. /* barrier(); */
  499. if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
  500. return NULL;
  501. *num = queue->ddcb_next; /* internal DDCB number */
  502. queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
  503. /* clear important DDCB fields */
  504. pu64 = (u64 *)pddcb;
  505. pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */
  506. pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */
  507. /* destroy previous results in ASV */
  508. pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */
  509. pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */
  510. pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */
  511. pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */
  512. pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */
  513. pddcb->pre = DDCB_PRESET_PRE; /* 128 */
  514. pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
  515. return pddcb;
  516. }
  517. /**
  518. * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
  519. * @cd: genwqe device descriptor
  520. * @req: DDCB request
  521. *
  522. * This will fail when the request was already FETCHED. In this case
  523. * we need to wait until it is finished. Else the DDCB can be
  524. * reused. This function also ensures that the request data structure
  525. * is removed from ddcb_req[].
  526. *
  527. * Do not forget to call this function when genwqe_wait_ddcb() fails,
  528. * such that the request gets really removed from ddcb_req[].
  529. *
  530. * Return: 0 success
  531. */
  532. int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
  533. {
  534. struct ddcb *pddcb = NULL;
  535. unsigned int t;
  536. unsigned long flags;
  537. struct ddcb_queue *queue = req->queue;
  538. struct pci_dev *pci_dev = cd->pci_dev;
  539. u64 queue_status;
  540. __be32 icrc_hsi_shi = 0x0000;
  541. __be32 old, new;
  542. /* unsigned long flags; */
  543. if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
  544. dev_err(&pci_dev->dev,
  545. "[%s] err: software timeout is not set!\n", __func__);
  546. return -EFAULT;
  547. }
  548. pddcb = &queue->ddcb_vaddr[req->num];
  549. for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
  550. spin_lock_irqsave(&queue->ddcb_lock, flags);
  551. /* Check if req was meanwhile finished */
  552. if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
  553. goto go_home;
  554. /* try to set PURGE bit if FETCHED/COMPLETED are not set */
  555. old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
  556. if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
  557. new = (old | DDCB_PURGE_BE32);
  558. icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
  559. old, new);
  560. if (icrc_hsi_shi == old)
  561. goto finish_ddcb;
  562. }
  563. /* normal finish with HSI bit */
  564. barrier();
  565. icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
  566. if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
  567. goto finish_ddcb;
  568. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  569. /*
  570. * Here the check_ddcb() function will most likely
  571. * discover this DDCB to be finished some point in
  572. * time. It will mark the req finished and free it up
  573. * in the list.
  574. */
  575. copy_ddcb_results(req, req->num); /* for the failing case */
  576. msleep(100); /* sleep for 1/10 second and try again */
  577. continue;
  578. finish_ddcb:
  579. copy_ddcb_results(req, req->num);
  580. ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
  581. queue->ddcbs_in_flight--;
  582. queue->ddcb_req[req->num] = NULL; /* delete from array */
  583. ddcb_mark_cleared(pddcb);
  584. /* Move active DDCB further; Nothing to do here anymore. */
  585. /*
  586. * We need to ensure that there is at least one free
  587. * DDCB in the queue. To do that, we must update
  588. * ddcb_act only if the COMPLETED bit is set for the
  589. * DDCB we are working on else we treat that DDCB even
  590. * if we PURGED it as occupied (hardware is supposed
  591. * to set the COMPLETED bit yet!).
  592. */
  593. icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
  594. if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
  595. (queue->ddcb_act == req->num)) {
  596. queue->ddcb_act = ((queue->ddcb_act + 1) %
  597. queue->ddcb_max);
  598. }
  599. go_home:
  600. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  601. return 0;
  602. }
  603. /*
  604. * If the card is dead and the queue is forced to stop, we
  605. * might see this in the queue status register.
  606. */
  607. queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
  608. dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
  609. genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
  610. dev_err(&pci_dev->dev,
  611. "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
  612. __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
  613. queue_status);
  614. print_ddcb_info(cd, req->queue);
  615. return -EFAULT;
  616. }
  617. int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
  618. {
  619. int len;
  620. struct pci_dev *pci_dev = cd->pci_dev;
  621. if (d == NULL) {
  622. dev_err(&pci_dev->dev,
  623. "[%s] err: invalid memory for debug data!\n",
  624. __func__);
  625. return -EFAULT;
  626. }
  627. len = sizeof(d->driver_version);
  628. snprintf(d->driver_version, len, "%s", DRV_VERSION);
  629. d->slu_unitcfg = cd->slu_unitcfg;
  630. d->app_unitcfg = cd->app_unitcfg;
  631. return 0;
  632. }
  633. /**
  634. * __genwqe_enqueue_ddcb() - Enqueue a DDCB
  635. * @cd: pointer to genwqe device descriptor
  636. * @req: pointer to DDCB execution request
  637. * @f_flags: file mode: blocking, non-blocking
  638. *
  639. * Return: 0 if enqueuing succeeded
  640. * -EIO if card is unusable/PCIe problems
  641. * -EBUSY if enqueuing failed
  642. */
  643. int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
  644. unsigned int f_flags)
  645. {
  646. struct ddcb *pddcb;
  647. unsigned long flags;
  648. struct ddcb_queue *queue;
  649. struct pci_dev *pci_dev = cd->pci_dev;
  650. u16 icrc;
  651. retry:
  652. if (cd->card_state != GENWQE_CARD_USED) {
  653. printk_ratelimited(KERN_ERR
  654. "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
  655. GENWQE_DEVNAME, dev_name(&pci_dev->dev),
  656. __func__, req->num);
  657. return -EIO;
  658. }
  659. queue = req->queue = &cd->queue;
  660. /* FIXME circumvention to improve performance when no irq is
  661. * there.
  662. */
  663. if (GENWQE_POLLING_ENABLED)
  664. genwqe_check_ddcb_queue(cd, queue);
  665. /*
  666. * It must be ensured to process all DDCBs in successive
  667. * order. Use a lock here in order to prevent nested DDCB
  668. * enqueuing.
  669. */
  670. spin_lock_irqsave(&queue->ddcb_lock, flags);
  671. pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */
  672. if (pddcb == NULL) {
  673. int rc;
  674. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  675. if (f_flags & O_NONBLOCK) {
  676. queue->return_on_busy++;
  677. return -EBUSY;
  678. }
  679. queue->wait_on_busy++;
  680. rc = wait_event_interruptible(queue->busy_waitq,
  681. queue_free_ddcbs(queue) != 0);
  682. dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
  683. __func__, rc);
  684. if (rc == -ERESTARTSYS)
  685. return rc; /* interrupted by a signal */
  686. goto retry;
  687. }
  688. if (queue->ddcb_req[req->num] != NULL) {
  689. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  690. dev_err(&pci_dev->dev,
  691. "[%s] picked DDCB %d with req=%p still in use!!\n",
  692. __func__, req->num, req);
  693. return -EFAULT;
  694. }
  695. ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
  696. queue->ddcb_req[req->num] = req;
  697. pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
  698. pddcb->cmd = req->cmd.cmd;
  699. pddcb->acfunc = req->cmd.acfunc; /* functional unit */
  700. /*
  701. * We know that we can get retc 0x104 with CRC error, do not
  702. * stop the queue in those cases for this command. XDIR = 1
  703. * does not work for old SLU versions.
  704. *
  705. * Last bitstream with the old XDIR behavior had SLU_ID
  706. * 0x34199.
  707. */
  708. if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
  709. pddcb->xdir = 0x1;
  710. else
  711. pddcb->xdir = 0x0;
  712. pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
  713. ((req->cmd.asv_length / 8)));
  714. pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
  715. /*
  716. * If copying the whole DDCB_ASIV_LENGTH is impacting
  717. * performance we need to change it to
  718. * req->cmd.asiv_length. But simulation benefits from some
  719. * non-architectured bits behind the architectured content.
  720. *
  721. * How much data is copied depends on the availability of the
  722. * ATS field, which was introduced late. If the ATS field is
  723. * supported ASIV is 8 bytes shorter than it used to be. Since
  724. * the ATS field is copied too, the code should do exactly
  725. * what it did before, but I wanted to make copying of the ATS
  726. * field very explicit.
  727. */
  728. if (genwqe_get_slu_id(cd) <= 0x2) {
  729. memcpy(&pddcb->__asiv[0], /* destination */
  730. &req->cmd.__asiv[0], /* source */
  731. DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */
  732. } else {
  733. pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
  734. memcpy(&pddcb->n.asiv[0], /* destination */
  735. &req->cmd.asiv[0], /* source */
  736. DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */
  737. }
  738. pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
  739. /*
  740. * Calculate CRC_16 for corresponding range PSP(7:4). Include
  741. * empty 4 bytes prior to the data.
  742. */
  743. icrc = genwqe_crc16((const u8 *)pddcb,
  744. ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
  745. pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
  746. /* enable DDCB completion irq */
  747. if (!GENWQE_POLLING_ENABLED)
  748. pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
  749. dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
  750. genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
  751. if (ddcb_requ_collect_debug_data(req)) {
  752. /* use the kernel copy of debug data. copying back to
  753. user buffer happens later */
  754. genwqe_init_debug_data(cd, &req->debug_data);
  755. memcpy(&req->debug_data.ddcb_before, pddcb,
  756. sizeof(req->debug_data.ddcb_before));
  757. }
  758. enqueue_ddcb(cd, queue, pddcb, req->num);
  759. queue->ddcbs_in_flight++;
  760. if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
  761. queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
  762. ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
  763. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  764. wake_up_interruptible(&cd->queue_waitq);
  765. return 0;
  766. }
  767. /**
  768. * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
  769. * @cd: pointer to genwqe device descriptor
  770. * @cmd: user provided DDCB command
  771. * @f_flags: file mode: blocking, non-blocking
  772. */
  773. int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
  774. struct genwqe_ddcb_cmd *cmd,
  775. unsigned int f_flags)
  776. {
  777. int rc = 0;
  778. struct pci_dev *pci_dev = cd->pci_dev;
  779. struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
  780. if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
  781. dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
  782. __func__, cmd->asiv_length);
  783. return -EINVAL;
  784. }
  785. if (cmd->asv_length > DDCB_ASV_LENGTH) {
  786. dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
  787. __func__, cmd->asiv_length);
  788. return -EINVAL;
  789. }
  790. rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
  791. if (rc != 0)
  792. return rc;
  793. rc = __genwqe_wait_ddcb(cd, req);
  794. if (rc < 0) /* error or signal interrupt */
  795. goto err_exit;
  796. if (ddcb_requ_collect_debug_data(req)) {
  797. if (copy_to_user((struct genwqe_debug_data __user *)
  798. (unsigned long)cmd->ddata_addr,
  799. &req->debug_data,
  800. sizeof(struct genwqe_debug_data)))
  801. return -EFAULT;
  802. }
  803. /*
  804. * Higher values than 0x102 indicate completion with faults,
  805. * lower values than 0x102 indicate processing faults. Note
  806. * that DDCB might have been purged. E.g. Cntl+C.
  807. */
  808. if (cmd->retc != DDCB_RETC_COMPLETE) {
  809. /* This might happen e.g. flash read, and needs to be
  810. handled by the upper layer code. */
  811. rc = -EBADMSG; /* not processed/error retc */
  812. }
  813. return rc;
  814. err_exit:
  815. __genwqe_purge_ddcb(cd, req);
  816. if (ddcb_requ_collect_debug_data(req)) {
  817. if (copy_to_user((struct genwqe_debug_data __user *)
  818. (unsigned long)cmd->ddata_addr,
  819. &req->debug_data,
  820. sizeof(struct genwqe_debug_data)))
  821. return -EFAULT;
  822. }
  823. return rc;
  824. }
  825. /**
  826. * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
  827. * @cd: pointer to genwqe device descriptor
  828. *
  829. * We use this as condition for our wait-queue code.
  830. */
  831. static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
  832. {
  833. unsigned long flags;
  834. struct ddcb *pddcb;
  835. struct ddcb_queue *queue = &cd->queue;
  836. spin_lock_irqsave(&queue->ddcb_lock, flags);
  837. if (queue_empty(queue)) { /* emtpy queue */
  838. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  839. return 0;
  840. }
  841. pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
  842. if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
  843. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  844. return 1;
  845. }
  846. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  847. return 0;
  848. }
  849. /**
  850. * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
  851. * @cd: pointer to genwqe device descriptor
  852. *
  853. * Keep track on the number of DDCBs which ware currently in the
  854. * queue. This is needed for statistics as well as conditon if we want
  855. * to wait or better do polling in case of no interrupts available.
  856. */
  857. int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
  858. {
  859. unsigned long flags;
  860. int ddcbs_in_flight = 0;
  861. struct ddcb_queue *queue = &cd->queue;
  862. spin_lock_irqsave(&queue->ddcb_lock, flags);
  863. ddcbs_in_flight += queue->ddcbs_in_flight;
  864. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  865. return ddcbs_in_flight;
  866. }
  867. static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
  868. {
  869. int rc, i;
  870. struct ddcb *pddcb;
  871. u64 val64;
  872. unsigned int queue_size;
  873. struct pci_dev *pci_dev = cd->pci_dev;
  874. if (GENWQE_DDCB_MAX < 2)
  875. return -EINVAL;
  876. queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
  877. queue->ddcbs_in_flight = 0; /* statistics */
  878. queue->ddcbs_max_in_flight = 0;
  879. queue->ddcbs_completed = 0;
  880. queue->return_on_busy = 0;
  881. queue->wait_on_busy = 0;
  882. queue->ddcb_seq = 0x100; /* start sequence number */
  883. queue->ddcb_max = GENWQE_DDCB_MAX;
  884. queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
  885. &queue->ddcb_daddr);
  886. if (queue->ddcb_vaddr == NULL) {
  887. dev_err(&pci_dev->dev,
  888. "[%s] **err: could not allocate DDCB **\n", __func__);
  889. return -ENOMEM;
  890. }
  891. queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
  892. GFP_KERNEL);
  893. if (!queue->ddcb_req) {
  894. rc = -ENOMEM;
  895. goto free_ddcbs;
  896. }
  897. queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
  898. sizeof(wait_queue_head_t),
  899. GFP_KERNEL);
  900. if (!queue->ddcb_waitqs) {
  901. rc = -ENOMEM;
  902. goto free_requs;
  903. }
  904. for (i = 0; i < queue->ddcb_max; i++) {
  905. pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */
  906. pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
  907. pddcb->retc_16 = cpu_to_be16(0xfff);
  908. queue->ddcb_req[i] = NULL; /* requests */
  909. init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
  910. }
  911. queue->ddcb_act = 0;
  912. queue->ddcb_next = 0; /* queue is empty */
  913. spin_lock_init(&queue->ddcb_lock);
  914. init_waitqueue_head(&queue->busy_waitq);
  915. val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */
  916. __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */
  917. __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
  918. __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
  919. __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64);
  920. return 0;
  921. free_requs:
  922. kfree(queue->ddcb_req);
  923. queue->ddcb_req = NULL;
  924. free_ddcbs:
  925. __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
  926. queue->ddcb_daddr);
  927. queue->ddcb_vaddr = NULL;
  928. queue->ddcb_daddr = 0ull;
  929. return rc;
  930. }
  931. static int ddcb_queue_initialized(struct ddcb_queue *queue)
  932. {
  933. return queue->ddcb_vaddr != NULL;
  934. }
  935. static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
  936. {
  937. unsigned int queue_size;
  938. queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
  939. kfree(queue->ddcb_req);
  940. queue->ddcb_req = NULL;
  941. if (queue->ddcb_vaddr) {
  942. __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
  943. queue->ddcb_daddr);
  944. queue->ddcb_vaddr = NULL;
  945. queue->ddcb_daddr = 0ull;
  946. }
  947. }
  948. static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
  949. {
  950. u64 gfir;
  951. struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
  952. struct pci_dev *pci_dev = cd->pci_dev;
  953. /*
  954. * In case of fatal FIR error the queue is stopped, such that
  955. * we can safely check it without risking anything.
  956. */
  957. cd->irqs_processed++;
  958. wake_up_interruptible(&cd->queue_waitq);
  959. /*
  960. * Checking for errors before kicking the queue might be
  961. * safer, but slower for the good-case ... See above.
  962. */
  963. gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
  964. if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
  965. !pci_channel_offline(pci_dev)) {
  966. if (cd->use_platform_recovery) {
  967. /*
  968. * Since we use raw accessors, EEH errors won't be
  969. * detected by the platform until we do a non-raw
  970. * MMIO or config space read
  971. */
  972. readq(cd->mmio + IO_SLC_CFGREG_GFIR);
  973. /* Don't do anything if the PCI channel is frozen */
  974. if (pci_channel_offline(pci_dev))
  975. goto exit;
  976. }
  977. wake_up_interruptible(&cd->health_waitq);
  978. /*
  979. * By default GFIRs causes recovery actions. This
  980. * count is just for debug when recovery is masked.
  981. */
  982. dev_err_ratelimited(&pci_dev->dev,
  983. "[%s] GFIR=%016llx\n",
  984. __func__, gfir);
  985. }
  986. exit:
  987. return IRQ_HANDLED;
  988. }
  989. static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
  990. {
  991. struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
  992. cd->irqs_processed++;
  993. wake_up_interruptible(&cd->queue_waitq);
  994. return IRQ_HANDLED;
  995. }
  996. /**
  997. * genwqe_card_thread() - Work thread for the DDCB queue
  998. * @data: pointer to genwqe device descriptor
  999. *
  1000. * The idea is to check if there are DDCBs in processing. If there are
  1001. * some finished DDCBs, we process them and wakeup the
  1002. * requestors. Otherwise we give other processes time using
  1003. * cond_resched().
  1004. */
  1005. static int genwqe_card_thread(void *data)
  1006. {
  1007. int should_stop = 0;
  1008. struct genwqe_dev *cd = (struct genwqe_dev *)data;
  1009. while (!kthread_should_stop()) {
  1010. genwqe_check_ddcb_queue(cd, &cd->queue);
  1011. if (GENWQE_POLLING_ENABLED) {
  1012. wait_event_interruptible_timeout(
  1013. cd->queue_waitq,
  1014. genwqe_ddcbs_in_flight(cd) ||
  1015. (should_stop = kthread_should_stop()), 1);
  1016. } else {
  1017. wait_event_interruptible_timeout(
  1018. cd->queue_waitq,
  1019. genwqe_next_ddcb_ready(cd) ||
  1020. (should_stop = kthread_should_stop()), HZ);
  1021. }
  1022. if (should_stop)
  1023. break;
  1024. /*
  1025. * Avoid soft lockups on heavy loads; we do not want
  1026. * to disable our interrupts.
  1027. */
  1028. cond_resched();
  1029. }
  1030. return 0;
  1031. }
  1032. /**
  1033. * genwqe_setup_service_layer() - Setup DDCB queue
  1034. * @cd: pointer to genwqe device descriptor
  1035. *
  1036. * Allocate DDCBs. Configure Service Layer Controller (SLC).
  1037. *
  1038. * Return: 0 success
  1039. */
  1040. int genwqe_setup_service_layer(struct genwqe_dev *cd)
  1041. {
  1042. int rc;
  1043. struct ddcb_queue *queue;
  1044. struct pci_dev *pci_dev = cd->pci_dev;
  1045. if (genwqe_is_privileged(cd)) {
  1046. rc = genwqe_card_reset(cd);
  1047. if (rc < 0) {
  1048. dev_err(&pci_dev->dev,
  1049. "[%s] err: reset failed.\n", __func__);
  1050. return rc;
  1051. }
  1052. genwqe_read_softreset(cd);
  1053. }
  1054. queue = &cd->queue;
  1055. queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG;
  1056. queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS;
  1057. queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
  1058. queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
  1059. queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET;
  1060. queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP;
  1061. queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME;
  1062. queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
  1063. queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW;
  1064. rc = setup_ddcb_queue(cd, queue);
  1065. if (rc != 0) {
  1066. rc = -ENODEV;
  1067. goto err_out;
  1068. }
  1069. init_waitqueue_head(&cd->queue_waitq);
  1070. cd->card_thread = kthread_run(genwqe_card_thread, cd,
  1071. GENWQE_DEVNAME "%d_thread",
  1072. cd->card_idx);
  1073. if (IS_ERR(cd->card_thread)) {
  1074. rc = PTR_ERR(cd->card_thread);
  1075. cd->card_thread = NULL;
  1076. goto stop_free_queue;
  1077. }
  1078. rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
  1079. if (rc)
  1080. goto stop_kthread;
  1081. /*
  1082. * We must have all wait-queues initialized when we enable the
  1083. * interrupts. Otherwise we might crash if we get an early
  1084. * irq.
  1085. */
  1086. init_waitqueue_head(&cd->health_waitq);
  1087. if (genwqe_is_privileged(cd)) {
  1088. rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
  1089. GENWQE_DEVNAME, cd);
  1090. } else {
  1091. rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
  1092. GENWQE_DEVNAME, cd);
  1093. }
  1094. if (rc < 0) {
  1095. dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
  1096. goto stop_irq_cap;
  1097. }
  1098. cd->card_state = GENWQE_CARD_USED;
  1099. return 0;
  1100. stop_irq_cap:
  1101. genwqe_reset_interrupt_capability(cd);
  1102. stop_kthread:
  1103. kthread_stop(cd->card_thread);
  1104. cd->card_thread = NULL;
  1105. stop_free_queue:
  1106. free_ddcb_queue(cd, queue);
  1107. err_out:
  1108. return rc;
  1109. }
  1110. /**
  1111. * queue_wake_up_all() - Handles fatal error case
  1112. * @cd: pointer to genwqe device descriptor
  1113. *
  1114. * The PCI device got unusable and we have to stop all pending
  1115. * requests as fast as we can. The code after this must purge the
  1116. * DDCBs in question and ensure that all mappings are freed.
  1117. */
  1118. static int queue_wake_up_all(struct genwqe_dev *cd)
  1119. {
  1120. unsigned int i;
  1121. unsigned long flags;
  1122. struct ddcb_queue *queue = &cd->queue;
  1123. spin_lock_irqsave(&queue->ddcb_lock, flags);
  1124. for (i = 0; i < queue->ddcb_max; i++)
  1125. wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
  1126. wake_up_interruptible(&queue->busy_waitq);
  1127. spin_unlock_irqrestore(&queue->ddcb_lock, flags);
  1128. return 0;
  1129. }
  1130. /**
  1131. * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
  1132. * @cd: pointer to genwqe device descriptor
  1133. *
  1134. * Relies on the pre-condition that there are no users of the card
  1135. * device anymore e.g. with open file-descriptors.
  1136. *
  1137. * This function must be robust enough to be called twice.
  1138. */
  1139. int genwqe_finish_queue(struct genwqe_dev *cd)
  1140. {
  1141. int i, rc = 0, in_flight;
  1142. int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
  1143. struct pci_dev *pci_dev = cd->pci_dev;
  1144. struct ddcb_queue *queue = &cd->queue;
  1145. if (!ddcb_queue_initialized(queue))
  1146. return 0;
  1147. /* Do not wipe out the error state. */
  1148. if (cd->card_state == GENWQE_CARD_USED)
  1149. cd->card_state = GENWQE_CARD_UNUSED;
  1150. /* Wake up all requests in the DDCB queue such that they
  1151. should be removed nicely. */
  1152. queue_wake_up_all(cd);
  1153. /* We must wait to get rid of the DDCBs in flight */
  1154. for (i = 0; i < waitmax; i++) {
  1155. in_flight = genwqe_ddcbs_in_flight(cd);
  1156. if (in_flight == 0)
  1157. break;
  1158. dev_dbg(&pci_dev->dev,
  1159. " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
  1160. i, waitmax, in_flight);
  1161. /*
  1162. * Severe severe error situation: The card itself has
  1163. * 16 DDCB queues, each queue has e.g. 32 entries,
  1164. * each DDBC has a hardware timeout of currently 250
  1165. * msec but the PFs have a hardware timeout of 8 sec
  1166. * ... so I take something large.
  1167. */
  1168. msleep(1000);
  1169. }
  1170. if (i == waitmax) {
  1171. dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n",
  1172. __func__);
  1173. rc = -EIO;
  1174. }
  1175. return rc;
  1176. }
  1177. /**
  1178. * genwqe_release_service_layer() - Shutdown DDCB queue
  1179. * @cd: genwqe device descriptor
  1180. *
  1181. * This function must be robust enough to be called twice.
  1182. */
  1183. int genwqe_release_service_layer(struct genwqe_dev *cd)
  1184. {
  1185. struct pci_dev *pci_dev = cd->pci_dev;
  1186. if (!ddcb_queue_initialized(&cd->queue))
  1187. return 1;
  1188. free_irq(pci_dev->irq, cd);
  1189. genwqe_reset_interrupt_capability(cd);
  1190. if (cd->card_thread != NULL) {
  1191. kthread_stop(cd->card_thread);
  1192. cd->card_thread = NULL;
  1193. }
  1194. free_ddcb_queue(cd, &cd->queue);
  1195. return 0;
  1196. }