card_utils.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * IBM Accelerator Family 'GenWQE'
  4. *
  5. * (C) Copyright IBM Corp. 2013
  6. *
  7. * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
  8. * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
  9. * Author: Michael Jung <mijung@gmx.net>
  10. * Author: Michael Ruettger <michael@ibmra.de>
  11. */
  12. /*
  13. * Miscelanous functionality used in the other GenWQE driver parts.
  14. */
  15. #include <linux/kernel.h>
  16. #include <linux/sched.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/page-flags.h>
  19. #include <linux/scatterlist.h>
  20. #include <linux/hugetlb.h>
  21. #include <linux/iommu.h>
  22. #include <linux/pci.h>
  23. #include <linux/dma-mapping.h>
  24. #include <linux/ctype.h>
  25. #include <linux/module.h>
  26. #include <linux/platform_device.h>
  27. #include <linux/delay.h>
  28. #include <linux/pgtable.h>
  29. #include "genwqe_driver.h"
  30. #include "card_base.h"
  31. #include "card_ddcb.h"
  32. /**
  33. * __genwqe_writeq() - Write 64-bit register
  34. * @cd: genwqe device descriptor
  35. * @byte_offs: byte offset within BAR
  36. * @val: 64-bit value
  37. *
  38. * Return: 0 if success; < 0 if error
  39. */
  40. int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val)
  41. {
  42. struct pci_dev *pci_dev = cd->pci_dev;
  43. if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
  44. return -EIO;
  45. if (cd->mmio == NULL)
  46. return -EIO;
  47. if (pci_channel_offline(pci_dev))
  48. return -EIO;
  49. __raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs);
  50. return 0;
  51. }
  52. /**
  53. * __genwqe_readq() - Read 64-bit register
  54. * @cd: genwqe device descriptor
  55. * @byte_offs: offset within BAR
  56. *
  57. * Return: value from register
  58. */
  59. u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs)
  60. {
  61. if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
  62. return 0xffffffffffffffffull;
  63. if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) &&
  64. (byte_offs == IO_SLC_CFGREG_GFIR))
  65. return 0x000000000000ffffull;
  66. if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) &&
  67. (byte_offs == IO_SLC_CFGREG_GFIR))
  68. return 0x00000000ffff0000ull;
  69. if (cd->mmio == NULL)
  70. return 0xffffffffffffffffull;
  71. return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs));
  72. }
  73. /**
  74. * __genwqe_writel() - Write 32-bit register
  75. * @cd: genwqe device descriptor
  76. * @byte_offs: byte offset within BAR
  77. * @val: 32-bit value
  78. *
  79. * Return: 0 if success; < 0 if error
  80. */
  81. int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val)
  82. {
  83. struct pci_dev *pci_dev = cd->pci_dev;
  84. if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
  85. return -EIO;
  86. if (cd->mmio == NULL)
  87. return -EIO;
  88. if (pci_channel_offline(pci_dev))
  89. return -EIO;
  90. __raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs);
  91. return 0;
  92. }
  93. /**
  94. * __genwqe_readl() - Read 32-bit register
  95. * @cd: genwqe device descriptor
  96. * @byte_offs: offset within BAR
  97. *
  98. * Return: Value from register
  99. */
  100. u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs)
  101. {
  102. if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
  103. return 0xffffffff;
  104. if (cd->mmio == NULL)
  105. return 0xffffffff;
  106. return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs));
  107. }
  108. /**
  109. * genwqe_read_app_id() - Extract app_id
  110. * @cd: genwqe device descriptor
  111. * @app_name: carrier used to pass-back name
  112. * @len: length of data for name
  113. *
  114. * app_unitcfg need to be filled with valid data first
  115. */
  116. int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len)
  117. {
  118. int i, j;
  119. u32 app_id = (u32)cd->app_unitcfg;
  120. memset(app_name, 0, len);
  121. for (i = 0, j = 0; j < min(len, 4); j++) {
  122. char ch = (char)((app_id >> (24 - j*8)) & 0xff);
  123. if (ch == ' ')
  124. continue;
  125. app_name[i++] = isprint(ch) ? ch : 'X';
  126. }
  127. return i;
  128. }
  129. /**
  130. * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations
  131. *
  132. * Existing kernel functions seem to use a different polynom,
  133. * therefore we could not use them here.
  134. *
  135. * Genwqe's Polynomial = 0x20044009
  136. */
  137. #define CRC32_POLYNOMIAL 0x20044009
  138. static u32 crc32_tab[256]; /* crc32 lookup table */
  139. void genwqe_init_crc32(void)
  140. {
  141. int i, j;
  142. u32 crc;
  143. for (i = 0; i < 256; i++) {
  144. crc = i << 24;
  145. for (j = 0; j < 8; j++) {
  146. if (crc & 0x80000000)
  147. crc = (crc << 1) ^ CRC32_POLYNOMIAL;
  148. else
  149. crc = (crc << 1);
  150. }
  151. crc32_tab[i] = crc;
  152. }
  153. }
  154. /**
  155. * genwqe_crc32() - Generate 32-bit crc as required for DDCBs
  156. * @buff: pointer to data buffer
  157. * @len: length of data for calculation
  158. * @init: initial crc (0xffffffff at start)
  159. *
  160. * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009)
  161. *
  162. * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should
  163. * result in a crc32 of 0xf33cb7d3.
  164. *
  165. * The existing kernel crc functions did not cover this polynom yet.
  166. *
  167. * Return: crc32 checksum.
  168. */
  169. u32 genwqe_crc32(u8 *buff, size_t len, u32 init)
  170. {
  171. int i;
  172. u32 crc;
  173. crc = init;
  174. while (len--) {
  175. i = ((crc >> 24) ^ *buff++) & 0xFF;
  176. crc = (crc << 8) ^ crc32_tab[i];
  177. }
  178. return crc;
  179. }
  180. void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
  181. dma_addr_t *dma_handle)
  182. {
  183. if (get_order(size) >= MAX_ORDER)
  184. return NULL;
  185. return dma_alloc_coherent(&cd->pci_dev->dev, size, dma_handle,
  186. GFP_KERNEL);
  187. }
  188. void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
  189. void *vaddr, dma_addr_t dma_handle)
  190. {
  191. if (vaddr == NULL)
  192. return;
  193. dma_free_coherent(&cd->pci_dev->dev, size, vaddr, dma_handle);
  194. }
  195. static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list,
  196. int num_pages)
  197. {
  198. int i;
  199. struct pci_dev *pci_dev = cd->pci_dev;
  200. for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) {
  201. pci_unmap_page(pci_dev, dma_list[i],
  202. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  203. dma_list[i] = 0x0;
  204. }
  205. }
  206. static int genwqe_map_pages(struct genwqe_dev *cd,
  207. struct page **page_list, int num_pages,
  208. dma_addr_t *dma_list)
  209. {
  210. int i;
  211. struct pci_dev *pci_dev = cd->pci_dev;
  212. /* establish DMA mapping for requested pages */
  213. for (i = 0; i < num_pages; i++) {
  214. dma_addr_t daddr;
  215. dma_list[i] = 0x0;
  216. daddr = pci_map_page(pci_dev, page_list[i],
  217. 0, /* map_offs */
  218. PAGE_SIZE,
  219. PCI_DMA_BIDIRECTIONAL); /* FIXME rd/rw */
  220. if (pci_dma_mapping_error(pci_dev, daddr)) {
  221. dev_err(&pci_dev->dev,
  222. "[%s] err: no dma addr daddr=%016llx!\n",
  223. __func__, (long long)daddr);
  224. goto err;
  225. }
  226. dma_list[i] = daddr;
  227. }
  228. return 0;
  229. err:
  230. genwqe_unmap_pages(cd, dma_list, num_pages);
  231. return -EIO;
  232. }
  233. static int genwqe_sgl_size(int num_pages)
  234. {
  235. int len, num_tlb = num_pages / 7;
  236. len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1);
  237. return roundup(len, PAGE_SIZE);
  238. }
  239. /*
  240. * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages
  241. *
  242. * Allocates memory for sgl and overlapping pages. Pages which might
  243. * overlap other user-space memory blocks are being cached for DMAs,
  244. * such that we do not run into syncronization issues. Data is copied
  245. * from user-space into the cached pages.
  246. */
  247. int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
  248. void __user *user_addr, size_t user_size, int write)
  249. {
  250. int ret = -ENOMEM;
  251. struct pci_dev *pci_dev = cd->pci_dev;
  252. sgl->fpage_offs = offset_in_page((unsigned long)user_addr);
  253. sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size);
  254. sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE);
  255. sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE;
  256. dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n",
  257. __func__, user_addr, user_size, sgl->nr_pages,
  258. sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size);
  259. sgl->user_addr = user_addr;
  260. sgl->user_size = user_size;
  261. sgl->write = write;
  262. sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages);
  263. if (get_order(sgl->sgl_size) > MAX_ORDER) {
  264. dev_err(&pci_dev->dev,
  265. "[%s] err: too much memory requested!\n", __func__);
  266. return ret;
  267. }
  268. sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size,
  269. &sgl->sgl_dma_addr);
  270. if (sgl->sgl == NULL) {
  271. dev_err(&pci_dev->dev,
  272. "[%s] err: no memory available!\n", __func__);
  273. return ret;
  274. }
  275. /* Only use buffering on incomplete pages */
  276. if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) {
  277. sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
  278. &sgl->fpage_dma_addr);
  279. if (sgl->fpage == NULL)
  280. goto err_out;
  281. /* Sync with user memory */
  282. if (copy_from_user(sgl->fpage + sgl->fpage_offs,
  283. user_addr, sgl->fpage_size)) {
  284. ret = -EFAULT;
  285. goto err_out;
  286. }
  287. }
  288. if (sgl->lpage_size != 0) {
  289. sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
  290. &sgl->lpage_dma_addr);
  291. if (sgl->lpage == NULL)
  292. goto err_out1;
  293. /* Sync with user memory */
  294. if (copy_from_user(sgl->lpage, user_addr + user_size -
  295. sgl->lpage_size, sgl->lpage_size)) {
  296. ret = -EFAULT;
  297. goto err_out2;
  298. }
  299. }
  300. return 0;
  301. err_out2:
  302. __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
  303. sgl->lpage_dma_addr);
  304. sgl->lpage = NULL;
  305. sgl->lpage_dma_addr = 0;
  306. err_out1:
  307. __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
  308. sgl->fpage_dma_addr);
  309. sgl->fpage = NULL;
  310. sgl->fpage_dma_addr = 0;
  311. err_out:
  312. __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
  313. sgl->sgl_dma_addr);
  314. sgl->sgl = NULL;
  315. sgl->sgl_dma_addr = 0;
  316. sgl->sgl_size = 0;
  317. return ret;
  318. }
  319. int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
  320. dma_addr_t *dma_list)
  321. {
  322. int i = 0, j = 0, p;
  323. unsigned long dma_offs, map_offs;
  324. dma_addr_t prev_daddr = 0;
  325. struct sg_entry *s, *last_s = NULL;
  326. size_t size = sgl->user_size;
  327. dma_offs = 128; /* next block if needed/dma_offset */
  328. map_offs = sgl->fpage_offs; /* offset in first page */
  329. s = &sgl->sgl[0]; /* first set of 8 entries */
  330. p = 0; /* page */
  331. while (p < sgl->nr_pages) {
  332. dma_addr_t daddr;
  333. unsigned int size_to_map;
  334. /* always write the chaining entry, cleanup is done later */
  335. j = 0;
  336. s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs);
  337. s[j].len = cpu_to_be32(128);
  338. s[j].flags = cpu_to_be32(SG_CHAINED);
  339. j++;
  340. while (j < 8) {
  341. /* DMA mapping for requested page, offs, size */
  342. size_to_map = min(size, PAGE_SIZE - map_offs);
  343. if ((p == 0) && (sgl->fpage != NULL)) {
  344. daddr = sgl->fpage_dma_addr + map_offs;
  345. } else if ((p == sgl->nr_pages - 1) &&
  346. (sgl->lpage != NULL)) {
  347. daddr = sgl->lpage_dma_addr;
  348. } else {
  349. daddr = dma_list[p] + map_offs;
  350. }
  351. size -= size_to_map;
  352. map_offs = 0;
  353. if (prev_daddr == daddr) {
  354. u32 prev_len = be32_to_cpu(last_s->len);
  355. /* pr_info("daddr combining: "
  356. "%016llx/%08x -> %016llx\n",
  357. prev_daddr, prev_len, daddr); */
  358. last_s->len = cpu_to_be32(prev_len +
  359. size_to_map);
  360. p++; /* process next page */
  361. if (p == sgl->nr_pages)
  362. goto fixup; /* nothing to do */
  363. prev_daddr = daddr + size_to_map;
  364. continue;
  365. }
  366. /* start new entry */
  367. s[j].target_addr = cpu_to_be64(daddr);
  368. s[j].len = cpu_to_be32(size_to_map);
  369. s[j].flags = cpu_to_be32(SG_DATA);
  370. prev_daddr = daddr + size_to_map;
  371. last_s = &s[j];
  372. j++;
  373. p++; /* process next page */
  374. if (p == sgl->nr_pages)
  375. goto fixup; /* nothing to do */
  376. }
  377. dma_offs += 128;
  378. s += 8; /* continue 8 elements further */
  379. }
  380. fixup:
  381. if (j == 1) { /* combining happened on last entry! */
  382. s -= 8; /* full shift needed on previous sgl block */
  383. j = 7; /* shift all elements */
  384. }
  385. for (i = 0; i < j; i++) /* move elements 1 up */
  386. s[i] = s[i + 1];
  387. s[i].target_addr = cpu_to_be64(0);
  388. s[i].len = cpu_to_be32(0);
  389. s[i].flags = cpu_to_be32(SG_END_LIST);
  390. return 0;
  391. }
  392. /**
  393. * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages
  394. * @cd: genwqe device descriptor
  395. * @sgl: scatter gather list describing user-space memory
  396. *
  397. * After the DMA transfer has been completed we free the memory for
  398. * the sgl and the cached pages. Data is being transferred from cached
  399. * pages into user-space buffers.
  400. */
  401. int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
  402. {
  403. int rc = 0;
  404. size_t offset;
  405. unsigned long res;
  406. struct pci_dev *pci_dev = cd->pci_dev;
  407. if (sgl->fpage) {
  408. if (sgl->write) {
  409. res = copy_to_user(sgl->user_addr,
  410. sgl->fpage + sgl->fpage_offs, sgl->fpage_size);
  411. if (res) {
  412. dev_err(&pci_dev->dev,
  413. "[%s] err: copying fpage! (res=%lu)\n",
  414. __func__, res);
  415. rc = -EFAULT;
  416. }
  417. }
  418. __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
  419. sgl->fpage_dma_addr);
  420. sgl->fpage = NULL;
  421. sgl->fpage_dma_addr = 0;
  422. }
  423. if (sgl->lpage) {
  424. if (sgl->write) {
  425. offset = sgl->user_size - sgl->lpage_size;
  426. res = copy_to_user(sgl->user_addr + offset, sgl->lpage,
  427. sgl->lpage_size);
  428. if (res) {
  429. dev_err(&pci_dev->dev,
  430. "[%s] err: copying lpage! (res=%lu)\n",
  431. __func__, res);
  432. rc = -EFAULT;
  433. }
  434. }
  435. __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
  436. sgl->lpage_dma_addr);
  437. sgl->lpage = NULL;
  438. sgl->lpage_dma_addr = 0;
  439. }
  440. __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
  441. sgl->sgl_dma_addr);
  442. sgl->sgl = NULL;
  443. sgl->sgl_dma_addr = 0x0;
  444. sgl->sgl_size = 0;
  445. return rc;
  446. }
  447. /**
  448. * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
  449. * @cd: pointer to genwqe device
  450. * @m: mapping params
  451. * @uaddr: user virtual address
  452. * @size: size of memory to be mapped
  453. *
  454. * We need to think about how we could speed this up. Of course it is
  455. * not a good idea to do this over and over again, like we are
  456. * currently doing it. Nevertheless, I am curious where on the path
  457. * the performance is spend. Most probably within the memory
  458. * allocation functions, but maybe also in the DMA mapping code.
  459. *
  460. * Restrictions: The maximum size of the possible mapping currently depends
  461. * on the amount of memory we can get using kzalloc() for the
  462. * page_list and pci_alloc_consistent for the sg_list.
  463. * The sg_list is currently itself not scattered, which could
  464. * be fixed with some effort. The page_list must be split into
  465. * PAGE_SIZE chunks too. All that will make the complicated
  466. * code more complicated.
  467. *
  468. * Return: 0 if success
  469. */
  470. int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
  471. unsigned long size)
  472. {
  473. int rc = -EINVAL;
  474. unsigned long data, offs;
  475. struct pci_dev *pci_dev = cd->pci_dev;
  476. if ((uaddr == NULL) || (size == 0)) {
  477. m->size = 0; /* mark unused and not added */
  478. return -EINVAL;
  479. }
  480. m->u_vaddr = uaddr;
  481. m->size = size;
  482. /* determine space needed for page_list. */
  483. data = (unsigned long)uaddr;
  484. offs = offset_in_page(data);
  485. if (size > ULONG_MAX - PAGE_SIZE - offs) {
  486. m->size = 0; /* mark unused and not added */
  487. return -EINVAL;
  488. }
  489. m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE);
  490. m->page_list = kcalloc(m->nr_pages,
  491. sizeof(struct page *) + sizeof(dma_addr_t),
  492. GFP_KERNEL);
  493. if (!m->page_list) {
  494. dev_err(&pci_dev->dev, "err: alloc page_list failed\n");
  495. m->nr_pages = 0;
  496. m->u_vaddr = NULL;
  497. m->size = 0; /* mark unused and not added */
  498. return -ENOMEM;
  499. }
  500. m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
  501. /* pin user pages in memory */
  502. rc = pin_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
  503. m->nr_pages,
  504. m->write ? FOLL_WRITE : 0, /* readable/writable */
  505. m->page_list); /* ptrs to pages */
  506. if (rc < 0)
  507. goto fail_pin_user_pages;
  508. /* assumption: pin_user_pages can be killed by signals. */
  509. if (rc < m->nr_pages) {
  510. unpin_user_pages_dirty_lock(m->page_list, rc, m->write);
  511. rc = -EFAULT;
  512. goto fail_pin_user_pages;
  513. }
  514. rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
  515. if (rc != 0)
  516. goto fail_free_user_pages;
  517. return 0;
  518. fail_free_user_pages:
  519. unpin_user_pages_dirty_lock(m->page_list, m->nr_pages, m->write);
  520. fail_pin_user_pages:
  521. kfree(m->page_list);
  522. m->page_list = NULL;
  523. m->dma_list = NULL;
  524. m->nr_pages = 0;
  525. m->u_vaddr = NULL;
  526. m->size = 0; /* mark unused and not added */
  527. return rc;
  528. }
  529. /**
  530. * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel
  531. * memory
  532. * @cd: pointer to genwqe device
  533. * @m: mapping params
  534. */
  535. int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
  536. {
  537. struct pci_dev *pci_dev = cd->pci_dev;
  538. if (!dma_mapping_used(m)) {
  539. dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n",
  540. __func__, m);
  541. return -EINVAL;
  542. }
  543. if (m->dma_list)
  544. genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
  545. if (m->page_list) {
  546. unpin_user_pages_dirty_lock(m->page_list, m->nr_pages,
  547. m->write);
  548. kfree(m->page_list);
  549. m->page_list = NULL;
  550. m->dma_list = NULL;
  551. m->nr_pages = 0;
  552. }
  553. m->u_vaddr = NULL;
  554. m->size = 0; /* mark as unused and not added */
  555. return 0;
  556. }
  557. /**
  558. * genwqe_card_type() - Get chip type SLU Configuration Register
  559. * @cd: pointer to the genwqe device descriptor
  560. * Return: 0: Altera Stratix-IV 230
  561. * 1: Altera Stratix-IV 530
  562. * 2: Altera Stratix-V A4
  563. * 3: Altera Stratix-V A7
  564. */
  565. u8 genwqe_card_type(struct genwqe_dev *cd)
  566. {
  567. u64 card_type = cd->slu_unitcfg;
  568. return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20);
  569. }
  570. /**
  571. * genwqe_card_reset() - Reset the card
  572. * @cd: pointer to the genwqe device descriptor
  573. */
  574. int genwqe_card_reset(struct genwqe_dev *cd)
  575. {
  576. u64 softrst;
  577. struct pci_dev *pci_dev = cd->pci_dev;
  578. if (!genwqe_is_privileged(cd))
  579. return -ENODEV;
  580. /* new SL */
  581. __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull);
  582. msleep(1000);
  583. __genwqe_readq(cd, IO_HSU_FIR_CLR);
  584. __genwqe_readq(cd, IO_APP_FIR_CLR);
  585. __genwqe_readq(cd, IO_SLU_FIR_CLR);
  586. /*
  587. * Read-modify-write to preserve the stealth bits
  588. *
  589. * For SL >= 039, Stealth WE bit allows removing
  590. * the read-modify-wrote.
  591. * r-m-w may require a mask 0x3C to avoid hitting hard
  592. * reset again for error reset (should be 0, chicken).
  593. */
  594. softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull;
  595. __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull);
  596. /* give ERRORRESET some time to finish */
  597. msleep(50);
  598. if (genwqe_need_err_masking(cd)) {
  599. dev_info(&pci_dev->dev,
  600. "[%s] masking errors for old bitstreams\n", __func__);
  601. __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
  602. }
  603. return 0;
  604. }
  605. int genwqe_read_softreset(struct genwqe_dev *cd)
  606. {
  607. u64 bitstream;
  608. if (!genwqe_is_privileged(cd))
  609. return -ENODEV;
  610. bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1;
  611. cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull;
  612. return 0;
  613. }
  614. /**
  615. * genwqe_set_interrupt_capability() - Configure MSI capability structure
  616. * @cd: pointer to the device
  617. * @count: number of vectors to allocate
  618. * Return: 0 if no error
  619. */
  620. int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
  621. {
  622. int rc;
  623. rc = pci_alloc_irq_vectors(cd->pci_dev, 1, count, PCI_IRQ_MSI);
  624. if (rc < 0)
  625. return rc;
  626. return 0;
  627. }
  628. /**
  629. * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability()
  630. * @cd: pointer to the device
  631. */
  632. void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
  633. {
  634. pci_free_irq_vectors(cd->pci_dev);
  635. }
  636. /**
  637. * set_reg_idx() - Fill array with data. Ignore illegal offsets.
  638. * @cd: card device
  639. * @r: debug register array
  640. * @i: index to desired entry
  641. * @m: maximum possible entries
  642. * @addr: addr which is read
  643. * @idx: index in debug array
  644. * @val: read value
  645. */
  646. static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r,
  647. unsigned int *i, unsigned int m, u32 addr, u32 idx,
  648. u64 val)
  649. {
  650. if (WARN_ON_ONCE(*i >= m))
  651. return -EFAULT;
  652. r[*i].addr = addr;
  653. r[*i].idx = idx;
  654. r[*i].val = val;
  655. ++*i;
  656. return 0;
  657. }
  658. static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r,
  659. unsigned int *i, unsigned int m, u32 addr, u64 val)
  660. {
  661. return set_reg_idx(cd, r, i, m, addr, 0, val);
  662. }
  663. int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs,
  664. unsigned int max_regs, int all)
  665. {
  666. unsigned int i, j, idx = 0;
  667. u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr;
  668. u64 gfir, sluid, appid, ufir, ufec, sfir, sfec;
  669. /* Global FIR */
  670. gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
  671. set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir);
  672. /* UnitCfg for SLU */
  673. sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */
  674. set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid);
  675. /* UnitCfg for APP */
  676. appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */
  677. set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid);
  678. /* Check all chip Units */
  679. for (i = 0; i < GENWQE_MAX_UNITS; i++) {
  680. /* Unit FIR */
  681. ufir_addr = (i << 24) | 0x008;
  682. ufir = __genwqe_readq(cd, ufir_addr);
  683. set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir);
  684. /* Unit FEC */
  685. ufec_addr = (i << 24) | 0x018;
  686. ufec = __genwqe_readq(cd, ufec_addr);
  687. set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec);
  688. for (j = 0; j < 64; j++) {
  689. /* wherever there is a primary 1, read the 2ndary */
  690. if (!all && (!(ufir & (1ull << j))))
  691. continue;
  692. sfir_addr = (i << 24) | (0x100 + 8 * j);
  693. sfir = __genwqe_readq(cd, sfir_addr);
  694. set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir);
  695. sfec_addr = (i << 24) | (0x300 + 8 * j);
  696. sfec = __genwqe_readq(cd, sfec_addr);
  697. set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec);
  698. }
  699. }
  700. /* fill with invalid data until end */
  701. for (i = idx; i < max_regs; i++) {
  702. regs[i].addr = 0xffffffff;
  703. regs[i].val = 0xffffffffffffffffull;
  704. }
  705. return idx;
  706. }
  707. /**
  708. * genwqe_ffdc_buff_size() - Calculates the number of dump registers
  709. * @cd: genwqe device descriptor
  710. * @uid: unit ID
  711. */
  712. int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid)
  713. {
  714. int entries = 0, ring, traps, traces, trace_entries;
  715. u32 eevptr_addr, l_addr, d_len, d_type;
  716. u64 eevptr, val, addr;
  717. eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
  718. eevptr = __genwqe_readq(cd, eevptr_addr);
  719. if ((eevptr != 0x0) && (eevptr != -1ull)) {
  720. l_addr = GENWQE_UID_OFFS(uid) | eevptr;
  721. while (1) {
  722. val = __genwqe_readq(cd, l_addr);
  723. if ((val == 0x0) || (val == -1ull))
  724. break;
  725. /* 38:24 */
  726. d_len = (val & 0x0000007fff000000ull) >> 24;
  727. /* 39 */
  728. d_type = (val & 0x0000008000000000ull) >> 36;
  729. if (d_type) { /* repeat */
  730. entries += d_len;
  731. } else { /* size in bytes! */
  732. entries += d_len >> 3;
  733. }
  734. l_addr += 8;
  735. }
  736. }
  737. for (ring = 0; ring < 8; ring++) {
  738. addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
  739. val = __genwqe_readq(cd, addr);
  740. if ((val == 0x0ull) || (val == -1ull))
  741. continue;
  742. traps = (val >> 24) & 0xff;
  743. traces = (val >> 16) & 0xff;
  744. trace_entries = val & 0xffff;
  745. entries += traps + (traces * trace_entries);
  746. }
  747. return entries;
  748. }
  749. /**
  750. * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure
  751. * @cd: genwqe device descriptor
  752. * @uid: unit ID
  753. * @regs: register information
  754. * @max_regs: number of register entries
  755. */
  756. int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid,
  757. struct genwqe_reg *regs, unsigned int max_regs)
  758. {
  759. int i, traps, traces, trace, trace_entries, trace_entry, ring;
  760. unsigned int idx = 0;
  761. u32 eevptr_addr, l_addr, d_addr, d_len, d_type;
  762. u64 eevptr, e, val, addr;
  763. eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
  764. eevptr = __genwqe_readq(cd, eevptr_addr);
  765. if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) {
  766. l_addr = GENWQE_UID_OFFS(uid) | eevptr;
  767. while (1) {
  768. e = __genwqe_readq(cd, l_addr);
  769. if ((e == 0x0) || (e == 0xffffffffffffffffull))
  770. break;
  771. d_addr = (e & 0x0000000000ffffffull); /* 23:0 */
  772. d_len = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */
  773. d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */
  774. d_addr |= GENWQE_UID_OFFS(uid);
  775. if (d_type) {
  776. for (i = 0; i < (int)d_len; i++) {
  777. val = __genwqe_readq(cd, d_addr);
  778. set_reg_idx(cd, regs, &idx, max_regs,
  779. d_addr, i, val);
  780. }
  781. } else {
  782. d_len >>= 3; /* Size in bytes! */
  783. for (i = 0; i < (int)d_len; i++, d_addr += 8) {
  784. val = __genwqe_readq(cd, d_addr);
  785. set_reg_idx(cd, regs, &idx, max_regs,
  786. d_addr, 0, val);
  787. }
  788. }
  789. l_addr += 8;
  790. }
  791. }
  792. /*
  793. * To save time, there are only 6 traces poplulated on Uid=2,
  794. * Ring=1. each with iters=512.
  795. */
  796. for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds,
  797. 2...7 are ASI rings */
  798. addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
  799. val = __genwqe_readq(cd, addr);
  800. if ((val == 0x0ull) || (val == -1ull))
  801. continue;
  802. traps = (val >> 24) & 0xff; /* Number of Traps */
  803. traces = (val >> 16) & 0xff; /* Number of Traces */
  804. trace_entries = val & 0xffff; /* Entries per trace */
  805. /* Note: This is a combined loop that dumps both the traps */
  806. /* (for the trace == 0 case) as well as the traces 1 to */
  807. /* 'traces'. */
  808. for (trace = 0; trace <= traces; trace++) {
  809. u32 diag_sel =
  810. GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace);
  811. addr = (GENWQE_UID_OFFS(uid) |
  812. IO_EXTENDED_DIAG_SELECTOR);
  813. __genwqe_writeq(cd, addr, diag_sel);
  814. for (trace_entry = 0;
  815. trace_entry < (trace ? trace_entries : traps);
  816. trace_entry++) {
  817. addr = (GENWQE_UID_OFFS(uid) |
  818. IO_EXTENDED_DIAG_READ_MBX);
  819. val = __genwqe_readq(cd, addr);
  820. set_reg_idx(cd, regs, &idx, max_regs, addr,
  821. (diag_sel<<16) | trace_entry, val);
  822. }
  823. }
  824. }
  825. return 0;
  826. }
  827. /**
  828. * genwqe_write_vreg() - Write register in virtual window
  829. * @cd: genwqe device descriptor
  830. * @reg: register (byte) offset within BAR
  831. * @val: value to write
  832. * @func: PCI virtual function
  833. *
  834. * Note, these registers are only accessible to the PF through the
  835. * VF-window. It is not intended for the VF to access.
  836. */
  837. int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func)
  838. {
  839. __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
  840. __genwqe_writeq(cd, reg, val);
  841. return 0;
  842. }
  843. /**
  844. * genwqe_read_vreg() - Read register in virtual window
  845. * @cd: genwqe device descriptor
  846. * @reg: register (byte) offset within BAR
  847. * @func: PCI virtual function
  848. *
  849. * Note, these registers are only accessible to the PF through the
  850. * VF-window. It is not intended for the VF to access.
  851. */
  852. u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func)
  853. {
  854. __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
  855. return __genwqe_readq(cd, reg);
  856. }
  857. /**
  858. * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card
  859. * @cd: genwqe device descriptor
  860. *
  861. * Note: From a design perspective it turned out to be a bad idea to
  862. * use codes here to specifiy the frequency/speed values. An old
  863. * driver cannot understand new codes and is therefore always a
  864. * problem. Better is to measure out the value or put the
  865. * speed/frequency directly into a register which is always a valid
  866. * value for old as well as for new software.
  867. *
  868. * Return: Card clock in MHz
  869. */
  870. int genwqe_base_clock_frequency(struct genwqe_dev *cd)
  871. {
  872. u16 speed; /* MHz MHz MHz MHz */
  873. static const int speed_grade[] = { 250, 200, 166, 175 };
  874. speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
  875. if (speed >= ARRAY_SIZE(speed_grade))
  876. return 0; /* illegal value */
  877. return speed_grade[speed];
  878. }
  879. /**
  880. * genwqe_stop_traps() - Stop traps
  881. * @cd: genwqe device descriptor
  882. *
  883. * Before reading out the analysis data, we need to stop the traps.
  884. */
  885. void genwqe_stop_traps(struct genwqe_dev *cd)
  886. {
  887. __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull);
  888. }
  889. /**
  890. * genwqe_start_traps() - Start traps
  891. * @cd: genwqe device descriptor
  892. *
  893. * After having read the data, we can/must enable the traps again.
  894. */
  895. void genwqe_start_traps(struct genwqe_dev *cd)
  896. {
  897. __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull);
  898. if (genwqe_need_err_masking(cd))
  899. __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
  900. }