vha_plat_nexef.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491
  1. /*!
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/delay.h>
  42. #include <linux/interrupt.h>
  43. #include <linux/module.h>
  44. #include <linux/device.h>
  45. #include <linux/gfp.h>
  46. #include <linux/dma-mapping.h>
  47. #include <linux/pm.h>
  48. #include <linux/mod_devicetable.h>
  49. #include <linux/workqueue.h>
  50. #include <linux/version.h>
  51. #include <linux/of_platform.h>
  52. #include "vha_common.h"
  53. #include "vha_plat.h"
  54. #include <nexef_plat.h>
  55. /* NNPU TC exports we need*/
  56. #include <tc_drv.h>
  57. #define DEVICE_NAME "vha"
  58. /*
  59. * Special handling (not implemented) is required for the VHA device
  60. * to be able to access both carveout buffers (internal memory) and
  61. * dmabuf buffers (system memory).The latter have to go through
  62. * the system bus to be accessed whereas the former do not.
  63. */
  64. static struct heap_config vha_plat_fpga_heap_configs[] = {
  65. /* Primary heap used for internal allocations */
  66. #ifdef FPGA_BUS_MASTERING
  67. #error Bus mastering not supported on this platform
  68. #elif CONFIG_GENERIC_ALLOCATOR
  69. {
  70. .type = IMG_MEM_HEAP_TYPE_CARVEOUT,
  71. /* .options.carveout to be filled at run time */
  72. /* .to_dev_addr to be filled at run time */
  73. },
  74. #else
  75. #error Neither FPGA_BUS_MASTERING or CONFIG_GENERIC_ALLOCATOR was defined
  76. #endif
  77. /* Secondary heap used for importing an external memory */
  78. #ifdef CONFIG_DMA_SHARED_BUFFER
  79. {
  80. .type = IMG_MEM_HEAP_TYPE_DMABUF,
  81. .to_dev_addr = NULL,
  82. },
  83. #else
  84. #warning "Memory importing not supported!"
  85. #endif
  86. };
  87. /* Number of core cycles used to measure the core clock frequency */
  88. #define FREQ_MEASURE_CYCLES 0x7fffff
  89. static const int vha_plat_fpga_heaps = sizeof(vha_plat_fpga_heap_configs)/
  90. sizeof(*vha_plat_fpga_heap_configs);
  91. static int vha_plat_probe(struct platform_device *pdev);
  92. static int vha_plat_remove(struct platform_device *pdev);
  93. static int vha_plat_suspend(struct platform_device *pdev, pm_message_t state);
  94. static int vha_plat_resume(struct platform_device *pdev);
  95. enum {
  96. PLATFORM_IS_NEXEF = 1,
  97. };
  98. static struct platform_device_id nna_platform_device_id_table[] = {
  99. { .name = NEXEF_NNA_DEVICE_NAME, .driver_data = PLATFORM_IS_NEXEF },
  100. { },
  101. };
  102. static struct platform_driver vha_platform_drv = {
  103. .probe = vha_plat_probe,
  104. .remove = vha_plat_remove,
  105. .suspend = vha_plat_suspend,
  106. .resume = vha_plat_resume,
  107. .driver = {
  108. .owner = THIS_MODULE,
  109. .name = DEVICE_NAME,
  110. },
  111. .id_table = nna_platform_device_id_table,
  112. };
  113. struct nna_driver_priv {
  114. struct platform_device *pdev;
  115. void __iomem *nna_regs;
  116. uint32_t nna_size;
  117. /* Work for the threaded interrupt. */
  118. struct work_struct work;
  119. };
  120. /*
  121. * reset_dut - Reset the Device Under Test
  122. */
  123. static void reset_dut(struct device *dev)
  124. {
  125. /* Nothing yet until Odin baseboard is updated to support that */
  126. //tc_dut2_reset(dev);
  127. }
  128. /*
  129. * pci_thread_irq - High latency interrupt handler
  130. */
  131. static void nna_soft_isr_cb(struct work_struct *work)
  132. {
  133. struct nna_driver_priv *priv = container_of(work, struct nna_driver_priv, work);
  134. struct platform_device *pdev = priv->pdev;
  135. struct device *dev = &pdev->dev;
  136. vha_handle_thread_irq(dev);
  137. }
  138. /*
  139. * pci_isr_cb - Low latency interrupt handler
  140. */
  141. static void nna_hard_isr_cb(void *pdev_id)
  142. {
  143. struct platform_device *pdev = (struct platform_device *)pdev_id;
  144. struct device *dev = &pdev->dev;
  145. struct nna_driver_priv *priv = (struct nna_driver_priv *)vha_get_plat_data(dev);
  146. irqreturn_t ret = IRQ_NONE;
  147. ret = vha_handle_irq(dev);
  148. if (ret == IRQ_WAKE_THREAD) {
  149. schedule_work(&priv->work);
  150. }
  151. }
  152. #ifdef CONFIG_GENERIC_ALLOCATOR
  153. static phys_addr_t carveout_to_dev_addr(union heap_options *options,
  154. phys_addr_t addr)
  155. {
  156. phys_addr_t base = options->carveout.phys;
  157. size_t size = options->carveout.size;
  158. unsigned long offset = options->carveout.offs;
  159. if (addr - offset >= base && addr < base + size - offset)
  160. return addr - base;
  161. pr_err("%s: unexpected addr! base 0x%llx size %zu offs %zu addr 0x%llx\n",
  162. __func__, base, size, offset, addr);
  163. WARN_ON(1);
  164. return addr;
  165. }
  166. static phys_addr_t carveout_to_host_addr(union heap_options *options,
  167. phys_addr_t addr)
  168. {
  169. phys_addr_t base = options->carveout.phys;
  170. size_t size = options->carveout.size;
  171. unsigned long offset = options->carveout.offs;
  172. if (addr < size - offset)
  173. return base + addr;
  174. pr_err("%s: unexpected addr! base %llx size %zu offs %zu addr %#llx\n",
  175. __func__, base, size, offset, addr);
  176. WARN_ON(1);
  177. return addr;
  178. }
  179. static void *carveout_get_kptr(phys_addr_t addr,
  180. size_t size, enum img_mem_attr mattr)
  181. {
  182. /*
  183. * Device memory is I/O memory and as a rule, it cannot
  184. * be dereferenced safely without memory barriers, that
  185. * is why it is guarded by __iomem (return of ioremap)
  186. * and checked by sparse. It is accessed only through
  187. * ioread32(), iowrit32(), etc.
  188. *
  189. * In x86 this memory can be dereferenced and safely
  190. * accessed, i.e. a * __iomem pointer can be casted to
  191. * a regular void* * pointer. We cast this here
  192. * assuming FPGA is x86 and add __force to silence the
  193. * sparse warning
  194. *
  195. * Note: System memory carveout can be used with cached turned on.
  196. * */
  197. void *kptr = NULL;
  198. if (mattr & IMG_MEM_ATTR_UNCACHED)
  199. #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
  200. kptr = (void * __force *)ioremap_nocache(addr, size);
  201. #else
  202. kptr = (void * __force *)ioremap(addr, size);
  203. #endif
  204. else if (mattr & IMG_MEM_ATTR_CACHED)
  205. kptr = (void * __force *)ioremap_cache(addr, size);
  206. else if (mattr & IMG_MEM_ATTR_WRITECOMBINE)
  207. kptr = (void * __force *)ioremap_wc(addr, size);
  208. /*pr_debug(
  209. "Mapping %zu bytes into kernel memory (Phys:%08llX, Kptr:%p)\n",
  210. size, addr, kptr);
  211. pr_debug("[%c%c%c]\n",
  212. (mattr & IMG_MEM_ATTR_UNCACHED) ? 'U' : '.',
  213. (mattr & IMG_MEM_ATTR_CACHED) ? 'C' : '.',
  214. (mattr & IMG_MEM_ATTR_WRITECOMBINE) ? 'W' : '.');*/
  215. return kptr;
  216. }
  217. static int carveout_put_kptr(void *addr)
  218. {
  219. /* pr_debug("Unmapping kernel memory (Phys: %p)\n", addr);*/
  220. iounmap(addr);
  221. return 0;
  222. }
  223. #endif
  224. static int vha_plat_probe(struct platform_device *pdev)
  225. {
  226. int ret = 0;
  227. struct nna_driver_priv *priv;
  228. struct device *dev = &pdev->dev;
  229. struct nexef_nna_platform_data *platdata;
  230. struct resource *nna_registers;
  231. uint64_t vha_mem_base, vha_mem_size;
  232. uint64_t vha_mem_phys_offset = 0;
  233. int heap;
  234. dev_info(dev, "%s dma_get_mask : %#llx\n", __func__, dma_get_mask(dev));
  235. priv = devm_kzalloc(dev, sizeof(struct nna_driver_priv), GFP_KERNEL);
  236. if (!priv) {
  237. ret = -ENOMEM;
  238. goto out_no_free;
  239. }
  240. priv->pdev = pdev;
  241. if (dev->dma_mask) {
  242. dev_info(dev, "%s dev->dma_mask : %p : %#llx\n",
  243. __func__, dev->dma_mask, *dev->dma_mask);
  244. } else {
  245. dev_info(dev, "%s mask unset, setting coherent\n", __func__);
  246. dev->dma_mask = &dev->coherent_dma_mask;
  247. }
  248. dev_info(dev, "%s dma_set_mask %#llx\n", __func__, dma_get_mask(dev));
  249. ret = dma_set_mask(dev, dma_get_mask(dev));
  250. if (ret) {
  251. dev_err(dev, "%s failed to set dma mask\n", __func__);
  252. goto out_dma_free;
  253. }
  254. /* Allocate dut2 registers */
  255. nna_registers = platform_get_resource_byname(pdev,
  256. IORESOURCE_MEM, "nna-regs");
  257. if (!nna_registers) {
  258. ret = -EIO;
  259. goto out_dma_free;
  260. }
  261. priv->nna_regs = devm_ioremap_resource(dev, nna_registers);
  262. if (!priv->nna_regs) {
  263. ret = -EIO;
  264. goto out_dma_free;
  265. }
  266. priv->nna_size = nna_registers->end - nna_registers->start;
  267. /* Get infos for DUT memory */
  268. platdata = dev_get_platdata(dev);
  269. /* Get out mem specs */
  270. vha_mem_size = platdata->nna_memory_size;
  271. vha_mem_base = platdata->nna_memory_base;
  272. vha_mem_phys_offset = platdata->nna_memory_offset;
  273. dev_dbg(dev, "PCI memory: base: %#llX - size: %#llX - offset: %#llX",
  274. vha_mem_base, vha_mem_size, vha_mem_phys_offset);
  275. /* patch heap config with PCI memory addresses */
  276. for (heap = 0; heap < vha_plat_fpga_heaps; heap++) {
  277. struct heap_config *cfg = &vha_plat_fpga_heap_configs[heap];
  278. switch(cfg->type) {
  279. case IMG_MEM_HEAP_TYPE_CARVEOUT:
  280. cfg->options.carveout.phys = vha_mem_base;
  281. cfg->options.carveout.size = vha_mem_size;
  282. cfg->options.carveout.offs = vha_mem_phys_offset;
  283. cfg->to_dev_addr = carveout_to_dev_addr;
  284. cfg->to_host_addr = carveout_to_host_addr;
  285. /* IO memory access callbacks */
  286. cfg->options.carveout.get_kptr = carveout_get_kptr;
  287. cfg->options.carveout.put_kptr = carveout_put_kptr;
  288. break;
  289. case IMG_MEM_HEAP_TYPE_DMABUF: /* Nothing to do here */
  290. break;
  291. default:
  292. dev_err(dev, "Unsupported heap type %d!\n", cfg->type);
  293. break;
  294. }
  295. }
  296. reset_dut(dev->parent);
  297. ret = vha_add_dev(dev,
  298. vha_plat_fpga_heap_configs,
  299. vha_plat_fpga_heaps,
  300. priv,
  301. priv->nna_regs,
  302. priv->nna_size);
  303. if (ret) {
  304. dev_err(dev, "failed to initialize driver core!\n");
  305. goto out_dma_free;
  306. }
  307. /*
  308. * Reset FPGA DUT only after disabling clocks in
  309. * vha_add_dev()-> get properties.
  310. * This workaround is required to ensure that
  311. * clocks (on daughter board) are enabled for test slave scripts to
  312. * read FPGA build version register.
  313. */
  314. reset_dut(dev->parent);
  315. /* Install the ISR callback...*/
  316. INIT_WORK(&priv->work, nna_soft_isr_cb);
  317. ret = tc_set_interrupt_handler(dev->parent, TC_INTERRUPT_NNA, nna_hard_isr_cb, pdev);
  318. ret |= tc_enable_interrupt(dev->parent, TC_INTERRUPT_NNA);
  319. if (ret) {
  320. dev_err(dev, "failed to request irq!\n");
  321. goto out_rm_dev;
  322. }
  323. /* Try to calibrate the core if needed */
  324. ret = vha_dev_calibrate(dev, FREQ_MEASURE_CYCLES);
  325. if (ret) {
  326. dev_err(dev, "%s: Failed to start clock calibration!\n", __func__);
  327. goto out_rm_dev;
  328. }
  329. return ret;
  330. out_rm_dev:
  331. /* Disable interrupt handler just in case it is enable which fail */
  332. tc_set_interrupt_handler(dev->parent, TC_INTERRUPT_NNA, NULL, NULL);
  333. vha_rm_dev(dev);
  334. out_dma_free:
  335. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
  336. /* Release any declared mem regions */
  337. dma_release_declared_memory(dev);
  338. #endif
  339. out_no_free:
  340. return ret;
  341. }
  342. static int vha_plat_remove(struct platform_device *pdev)
  343. {
  344. struct device *dev = &pdev->dev;
  345. struct nna_driver_priv *priv =
  346. (struct nna_driver_priv *)vha_get_plat_data(dev);
  347. dev_dbg(dev, "removing device\n");
  348. /* Disable interrupts */
  349. tc_disable_interrupt(dev->parent, TC_INTERRUPT_NNA);
  350. tc_set_interrupt_handler(dev->parent, TC_INTERRUPT_NNA, NULL, NULL);
  351. /* Make sure there is no work in the queue */
  352. if (priv)
  353. cancel_work_sync(&priv->work);
  354. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
  355. /* Release any declared mem regions */
  356. dma_release_declared_memory(dev);
  357. #endif
  358. vha_rm_dev(dev);
  359. return 0;
  360. }
  361. #ifdef CONFIG_PM
  362. static int vha_plat_suspend(struct platform_device *pdev, pm_message_t state)
  363. {
  364. struct device *dev = &pdev->dev;
  365. return vha_suspend_dev(dev);
  366. }
  367. static int vha_plat_resume(struct platform_device *pdev)
  368. {
  369. struct device *dev = &pdev->dev;
  370. return vha_resume_dev(dev);
  371. }
  372. #endif
  373. /* Functions called by vha_core */
  374. int vha_plat_init(void)
  375. {
  376. int ret;
  377. ret = platform_driver_register(&vha_platform_drv);
  378. if (ret) {
  379. pr_err("failed to register platform driver!\n");
  380. return ret;
  381. }
  382. return 0;
  383. }
  384. int vha_plat_deinit(void)
  385. {
  386. int ret;
  387. //reset_dut();
  388. platform_driver_unregister(&vha_platform_drv);
  389. ret = vha_deinit();
  390. if (ret)
  391. pr_err("VHA driver deinit failed\n");
  392. return ret;
  393. }
  394. /*
  395. * NOTE: customer may want to use spinlock to avoid
  396. * problems with multi threaded IO access.
  397. *
  398. */
  399. uint64_t vha_plat_read64(void *addr)
  400. {
  401. return (uint64_t)readl(addr) | ((uint64_t)readl(addr + 4) << 32);
  402. }
  403. void vha_plat_write64(void *addr, uint64_t val)
  404. {
  405. writel(val & 0xffffffff, addr);
  406. writel(((uint64_t)val >> 32), addr + 4);
  407. }