vha_plat_odin.c 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152
  1. /*!
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/delay.h>
  42. #include <linux/interrupt.h>
  43. #include <linux/module.h>
  44. #include <linux/device.h>
  45. #include <linux/gfp.h>
  46. #include <linux/version.h>
  47. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
  48. #include <linux/dma-mapping.h>
  49. #else
  50. #include <linux/dma-map-ops.h>
  51. #endif
  52. #include <linux/pci.h>
  53. #include <linux/pm.h>
  54. #include <linux/mod_devicetable.h>
  55. #include <linux/workqueue.h>
  56. #include "uapi/version.h"
  57. #include "vha_common.h"
  58. #include "vha_plat.h"
  59. #if defined(CFG_SYS_VAGUS)
  60. #include <hwdefs/nn_sys_cr_vagus.h>
  61. #endif
  62. #if defined(CFG_SYS_VAGUS)
  63. #include <hwdefs/vagus_system.h>
  64. #elif defined(CFG_SYS_AURA)
  65. #include <hwdefs/aura_system.h>
  66. #elif defined(CFG_SYS_MIRAGE)
  67. #include <hwdefs/mirage_system.h>
  68. #endif
  69. #define DEVICE_NAME "vha"
  70. #define IS_ODIN_DEVICE(devid) ((devid) == PCI_ODIN_DEVICE_ID)
  71. /*
  72. * from Odin Lite TRM rev 1.0.88
  73. */
  74. #define PCI_ODIN_VENDOR_ID (0x1AEE)
  75. #define PCI_ODIN_DEVICE_ID (0x1010)
  76. /* Odin - System control register bar */
  77. #define PCI_ODIN_SYS_CTRL_REGS_BAR (0)
  78. #define PCI_ODIN_SYS_CTRL_BASE_OFFSET (0x0000)
  79. /* srs_core */
  80. #define PCI_ODIN_CORE_ID (0x0000)
  81. #define PCI_ODIN_CORE_REVISION (0x0004)
  82. #define PCI_ODIN_CORE_CHANGE_SET (0x0008)
  83. #define PCI_ODIN_CORE_USER_ID (0x000C)
  84. #define PCI_ODIN_CORE_USER_BUILD (0x0010)
  85. /* Resets */
  86. #define PCI_ODIN_CORE_INTERNAL_RESETN (0x0080)
  87. #define PCI_ODIN_CORE_EXTERNAL_RESETN (0x0084)
  88. #define PCI_ODIN_CORE_EXTERNAL_RESET (0x0088)
  89. #define PCI_ODIN_CORE_INTERNAL_AUTO_RESETN (0x008C)
  90. /* Clock */
  91. #define PCI_ODIN_CORE_CLK_GEN_RESET (0x0090)
  92. /* Interrupts */
  93. #define PCI_ODIN_CORE_INTERRUPT_STATUS (0x0100)
  94. #define PCI_ODIN_CORE_INTERRUPT_ENABLE (0x0104)
  95. #define PCI_ODIN_CORE_INTERRUPT_CLR (0x010C)
  96. #define PCI_ODIN_CORE_INTERRUPT_TEST (0x0110)
  97. /* GPIOs */
  98. #define PCI_ODIN_CORE_NUM_GPIO (0x0180)
  99. #define PCI_ODIN_CORE_GPIO_EN (0x0184)
  100. #define PCI_ODIN_CORE_GPIO (0x0188)
  101. /* DUT Ctrl */
  102. #define PCI_ODIN_CORE_NUM_DUT_CTRL (0x0190)
  103. #define PCI_ODIN_CORE_DUT_CTRL1 (0x0194)
  104. #define PCI_ODIN_CORE_DUT_CTRL2 (0x0198)
  105. #define PCI_ODIN_CORE_NUM_DUT_STAT (0x019C)
  106. #define PCI_ODIN_CORE_DUT_STAT1 (0x01A0)
  107. #define PCI_ODIN_CORE_DUT_STAT2 (0x01A4)
  108. /* LEDs! */
  109. #define PCI_ODIN_CORE_DASH_LEDS (0x01A8)
  110. /* Core stuff */
  111. #define PCI_ODIN_CORE_CORE_STATUS (0x0200)
  112. #define PCI_ODIN_CORE_CORE_CONTROL (0x0204)
  113. #define PCI_ODIN_CORE_REG_BANK_STATUS (0x0208)
  114. #define PCI_ODIN_CORE_MMCM_LOCK_STATUS (0x020C)
  115. #define PCI_ODIN_CORE_GIST_STATUS (0x0210)
  116. /* core bits definitions */
  117. #define INTERNAL_RESET_INTERNAL_RESETN_PIKE (1 << 7)
  118. #define EXTERNAL_RESET_EXTERNAL_RESETN_DUT (1 << 0)
  119. #define DUT_CTRL1_DUT_MST_OFFSET (1 << 31)
  120. #define ODIN_CORE_CONTROL_DUT_OFFSET_SHIFT (24)
  121. #define ODIN_CORE_CONTROL_DUT_OFFSET_MASK (0x7 << ODIN_CORE_CONTROL_DUT_OFFSET_SHIFT)
  122. /* interrupt bits definitions */
  123. #define INT_INTERRUPT_MASTER_ENABLE (1 << 31)
  124. #define INT_INTERRUPT_DUT0 (1 << 0)
  125. #define INT_INTERRUPT_DUT1 (1 << 9)
  126. /* srs_clk_blk */
  127. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_OUT_DIV1 (0x0020)
  128. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_OUT_DIV2 (0x0024)
  129. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_OUT_DIV3 (0x001C)
  130. #define PCI_ODIN_CLK_BLK_DUT_REG_CLK_OUT_DIV1 (0x0028)
  131. #define PCI_ODIN_CLK_BLK_DUT_REG_CLK_OUT_DIV2 (0x002C)
  132. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_MULT1 (0x0050)
  133. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_MULT2 (0x0054)
  134. #define PCI_ODIN_CLK_BLK_DUT_CORE_CLK_MULT3 (0x004C)
  135. #define PCI_ODIN_CLK_BLK_DUT_CORE_VLK_IN_DIV (0x0058)
  136. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_OUT_DIV1 (0x0220)
  137. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_OUT_DIV2 (0x0224)
  138. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_OUT_DIV3 (0x021C)
  139. #define PCI_ODIN_CLK_BLK_DUT_MEM_CLK_OUT_DIV1 (0x0228)
  140. #define PCI_ODIN_CLK_BLK_DUT_MEM_CLK_OUT_DIV2 (0x022C)
  141. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_MULT1 (0x0250)
  142. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_MULT2 (0x0254)
  143. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_MULT3 (0x024C)
  144. #define PCI_ODIN_CLK_BLK_DUT_SYS_CLK_IN_DIV (0x0258)
  145. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_OUT_DIV1 (0x0620)
  146. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_OUT_DIV2 (0x0624)
  147. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_OUT_DIV3 (0x061C)
  148. #define PCI_ODIN_CLK_BLK_PDP_MEM_CLK_OUT_DIV1 (0x0628)
  149. #define PCI_ODIN_CLK_BLK_PDP_MEM_CLK_OUT_DIV2 (0x062C)
  150. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_MULT1 (0x0650)
  151. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_MULT2 (0x0654)
  152. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_MULT3 (0x064C)
  153. #define PCI_ODIN_CLK_BLK_PDP_PIXEL_CLK_IN_DIV (0x0658)
  154. #define PCI_ODIN_CORE_REG_SIZE (0x1000)
  155. /* Odin - Device Under Test (DUT) register bar */
  156. #define PCI_ODIN_DUT_REGS_BAR (2)
  157. #define PCI_ODIN_DUT_MEM_BAR (4)
  158. /* Number of core cycles used to measure the core clock frequency */
  159. #define FREQ_MEASURE_CYCLES 0x7fffff
  160. /* Parameters applicable when using bus master mode */
  161. static unsigned long contig_phys_start;
  162. module_param(contig_phys_start, ulong, 0444);
  163. MODULE_PARM_DESC(contig_phys_start, "Physical address of start of contiguous region");
  164. static uint32_t contig_size;
  165. module_param(contig_size, uint, 0444);
  166. MODULE_PARM_DESC(contig_size, "Size of contiguous region: takes precedence over any PCI based memory");
  167. static uint32_t fpga_heap_type = IMG_MEM_HEAP_TYPE_UNIFIED;
  168. module_param(fpga_heap_type, uint, 0444);
  169. MODULE_PARM_DESC(fpga_heap_type, "Fpga primary heap type");
  170. static unsigned long pci_size;
  171. module_param(pci_size, ulong, 0444);
  172. MODULE_PARM_DESC(pci_size, "physical size in bytes. when 0 (the default), use all memory in the PCI bar");
  173. static unsigned long pci_offset;
  174. module_param(pci_offset, ulong, 0444);
  175. MODULE_PARM_DESC(pci_offset, "offset from PCI bar start. (default: 0)");
  176. static unsigned short pool_alloc_order;
  177. module_param(pool_alloc_order, ushort, 0444);
  178. MODULE_PARM_DESC(pool_alloc_order,
  179. "Carveout pool allocation order, depends on PAGE_SIZE, \
  180. for CPU PAGE_SIZE=4kB, 0-4kB, 1-8kB, 2-16kB, 3-32kB, 4-64kB");
  181. /* Newest version of ODIN allow for dual DUT devices, this parameter allow at load time to select which DUT to use */
  182. static unsigned long dut_id = 0;
  183. module_param(dut_id, ulong, 0444);
  184. MODULE_PARM_DESC(dut_id, "DUT the driver try to address. valid: {0, 1}, (default: 0)");
  185. static bool mem_static_kptr = true;
  186. module_param(mem_static_kptr, bool, 0444);
  187. MODULE_PARM_DESC(mem_static_kptr,
  188. "Creates static kernel mapping for fpga memory");
  189. /* Maximum DUT_ID allowed */
  190. #define MAX_DUT_ID (1)
  191. static uint32_t odin_dut_register_offset[] = {
  192. 0x00000000, /* DUT 0 */
  193. 0x02000000, /* DUT 1 */
  194. };
  195. static uint32_t odin_dut_interrupt_bit[] = {
  196. INT_INTERRUPT_DUT0, /* DUT 0 */
  197. INT_INTERRUPT_DUT1, /* DUT 1 */
  198. };
  199. /*
  200. * Special handling (not implemented) is required for the VHA device
  201. * to be able to access both carveout buffers (internal memory) and
  202. * dmabuf buffers (system memory).The latter have to go through
  203. * the system bus to be accessed whereas the former do not.
  204. */
  205. #if !defined(FPGA_BUS_MASTERING) && !defined(CONFIG_GENERIC_ALLOCATOR)
  206. #error Neither FPGA_BUS_MASTERING or GENERIC_ALLOCATOR is defined
  207. #endif
  208. static struct heap_config vha_dev_fpga_heap_configs[] = {
  209. /* Primary heap used for internal allocations */
  210. #if CONFIG_GENERIC_ALLOCATOR
  211. {
  212. .type = IMG_MEM_HEAP_TYPE_CARVEOUT,
  213. /* .options.carveout to be filled at run time */
  214. /* .to_dev_addr to be filled at run time */
  215. },
  216. #endif
  217. };
  218. static const int vha_dev_fpga_heaps = sizeof(vha_dev_fpga_heap_configs)/
  219. sizeof(*vha_dev_fpga_heap_configs);
  220. static struct heap_config vha_plat_fpga_heap_configs[] = {
  221. /* Secondary heap used for importing an external memory */
  222. #if defined(FPGA_BUS_MASTERING)
  223. #error Bus mastering not supported for now.
  224. {
  225. .type = -1, /* selected with fpga_heap_type */
  226. .options = {
  227. .unified.gfp_type = GFP_DMA32 | __GFP_ZERO,
  228. .coherent.gfp_flags = GFP_DMA32 | __GFP_ZERO,
  229. },
  230. .to_dev_addr = NULL,
  231. },
  232. {
  233. .type = IMG_MEM_HEAP_TYPE_ANONYMOUS,
  234. },
  235. #endif
  236. #if CONFIG_DMA_SHARED_BUFFER
  237. {
  238. .type = IMG_MEM_HEAP_TYPE_DMABUF,
  239. .to_dev_addr = NULL,
  240. #if !defined(FPGA_BUS_MASTERING)
  241. .options.dmabuf = {
  242. .use_sg_dma = true,
  243. },
  244. #endif
  245. },
  246. #else
  247. #warning "Memory importing not supported!"
  248. #endif
  249. };
  250. static const int vha_plat_fpga_heaps = sizeof(vha_plat_fpga_heap_configs)/
  251. sizeof(*vha_plat_fpga_heap_configs);
  252. static const struct pci_device_id pci_pci_ids[] = {
  253. { PCI_DEVICE(PCI_ODIN_VENDOR_ID, PCI_ODIN_DEVICE_ID), },
  254. { 0, }
  255. };
  256. MODULE_DEVICE_TABLE(pci, pci_pci_ids);
  257. enum { CORE_REG_BANK = 0,
  258. NNA_REG_BANK, MEM_REG_BANK,
  259. REG_BANK_COUNT /* Must be the last */};
  260. struct imgpci_prvdata {
  261. int irq;
  262. struct {
  263. int bar;
  264. unsigned long addr;
  265. unsigned long size;
  266. void __iomem *km_addr;
  267. } reg_bank[REG_BANK_COUNT];
  268. struct pci_dev *pci_dev;
  269. };
  270. struct img_pci_driver {
  271. struct pci_dev *pci_dev;
  272. struct pci_driver pci_driver;
  273. struct delayed_work irq_work;
  274. };
  275. static int vha_plat_probe(struct pci_dev *pci_dev,
  276. const struct pci_device_id *id);
  277. static void vha_plat_remove(struct pci_dev *dev);
  278. static int vha_plat_suspend(struct device *dev);
  279. static int vha_plat_resume(struct device *dev);
  280. static SIMPLE_DEV_PM_OPS(vha_pm_plat_ops,
  281. vha_plat_suspend, vha_plat_resume);
  282. static ssize_t info_show(struct device_driver *drv, char *buf)
  283. {
  284. return sprintf(buf, "VHA Odin driver version : " VERSION_STRING "\n");
  285. }
  286. static inline uint64_t __readreg64(struct imgpci_prvdata *data,
  287. int bank, unsigned long offset) __maybe_unused;
  288. static inline void __writereg64(struct imgpci_prvdata *data,
  289. int bank, unsigned long offset, uint64_t val) __maybe_unused;
  290. static DRIVER_ATTR_RO(info);
  291. static struct attribute *drv_attrs[] = {
  292. &driver_attr_info.attr,
  293. NULL
  294. };
  295. ATTRIBUTE_GROUPS(drv);
  296. static struct img_pci_driver vha_pci_drv = {
  297. .pci_driver = {
  298. .name = "vha_pci",
  299. .id_table = pci_pci_ids,
  300. .probe = vha_plat_probe,
  301. .remove = vha_plat_remove,
  302. .driver = {
  303. .groups = drv_groups,
  304. .pm = &vha_pm_plat_ops,
  305. }
  306. },
  307. };
  308. static ulong maxmapsizeMB = (sizeof(void *) == 4) ? 400 : 4096;
  309. /**
  310. * __readreg32 - Generic PCI bar read functions
  311. * @data: pointer to the data
  312. * @bank: register bank
  313. * @offset: offset within bank
  314. */
  315. static inline unsigned int __readreg32(struct imgpci_prvdata *data,
  316. int bank, unsigned long offset)
  317. {
  318. void __iomem *reg = (void __iomem *)(data->reg_bank[bank].km_addr +
  319. offset);
  320. return ioread32(reg);
  321. }
  322. /**
  323. * __writereg32 - Generic PCI bar write functions
  324. * @data: pointer to the data
  325. * @bank: register bank
  326. * @offset: offset within bank
  327. * @val: value to be written
  328. */
  329. static inline void __writereg32(struct imgpci_prvdata *data,
  330. int bank, unsigned long offset, int val)
  331. {
  332. void __iomem *reg = (void __iomem *)(data->reg_bank[bank].km_addr +
  333. offset);
  334. iowrite32(val, reg);
  335. }
  336. /*
  337. * __readreg64 - Generic PCI bar read functions
  338. * @data: pointer to the data
  339. * @bank: register bank
  340. * @offset: offset within bank
  341. */
  342. static inline uint64_t __readreg64(struct imgpci_prvdata *data,
  343. int bank, unsigned long offset)
  344. {
  345. void __iomem *reg = (void __iomem *)(data->reg_bank[bank].km_addr +
  346. offset);
  347. return (uint64_t)ioread32(reg) | ((uint64_t)ioread32(reg + 4) << 32);
  348. }
  349. /*
  350. * __writereg64 - Generic PCI bar write functions
  351. * @data: pointer to the data
  352. * @bank: register bank
  353. * @offset: offset within bank
  354. * @val: value to be written
  355. */
  356. static inline void __writereg64(struct imgpci_prvdata *data,
  357. int bank, unsigned long offset, uint64_t val)
  358. {
  359. void __iomem *reg = (void __iomem *)(data->reg_bank[bank].km_addr +
  360. offset);
  361. iowrite32(val & 0xFFFFFFFF, reg);
  362. iowrite32(val >> 32, reg + 4);
  363. }
  364. /**
  365. * odin_core_writereg32 - Write to Odin control registers
  366. * @data: pointer to the data
  367. * @offset: offset within bank
  368. * @val: value to be written
  369. */
  370. static inline void odin_core_writereg32(struct imgpci_prvdata *data,
  371. unsigned long offset, int val)
  372. {
  373. __writereg32(data, CORE_REG_BANK, offset, val);
  374. }
  375. /**
  376. * odin_core_readreg32 - Read Odin control registers
  377. * @data: pointer to the data
  378. * @offset: offset within bank
  379. */
  380. static inline unsigned int odin_core_readreg32(struct imgpci_prvdata *data,
  381. unsigned long offset)
  382. {
  383. return __readreg32(data, CORE_REG_BANK, offset);
  384. }
  385. /**
  386. * reset_dut - Reset the Device Under Test
  387. * @data: pointer to the data
  388. */
  389. static void reset_dut(struct imgpci_prvdata *data)
  390. {
  391. uint32_t internal_rst = odin_core_readreg32(data, PCI_ODIN_CORE_INTERNAL_RESETN);
  392. uint32_t external_rst = odin_core_readreg32(data, PCI_ODIN_CORE_EXTERNAL_RESETN);
  393. dev_dbg(&data->pci_dev->dev, "going to reset DUT fpga!\n");
  394. odin_core_writereg32(data, PCI_ODIN_CORE_INTERNAL_RESETN,
  395. internal_rst & ~(INTERNAL_RESET_INTERNAL_RESETN_PIKE));
  396. odin_core_writereg32(data, PCI_ODIN_CORE_EXTERNAL_RESETN,
  397. external_rst & ~(EXTERNAL_RESET_EXTERNAL_RESETN_DUT));
  398. udelay(100); /* arbitrary delays, just in case! */
  399. odin_core_writereg32(data, PCI_ODIN_CORE_INTERNAL_RESETN, internal_rst);
  400. odin_core_writereg32(data, PCI_ODIN_CORE_EXTERNAL_RESETN, external_rst);
  401. msleep(100);
  402. dev_dbg(&data->pci_dev->dev, "DUT fpga reset done!\n");
  403. }
  404. /**
  405. * pci_thread_irq - High latency interrupt handler
  406. * @irq: irq number
  407. * @dev_id: pointer to private data
  408. */
  409. static irqreturn_t pci_thread_irq(int irq, void *dev_id)
  410. {
  411. struct pci_dev *dev = (struct pci_dev *)dev_id;
  412. return vha_handle_thread_irq(&dev->dev);
  413. }
  414. /**
  415. * odin_isr_clear - Clear an interrupt
  416. * @data: pointer to the data
  417. * @intstatus: interrupt status
  418. *
  419. * note: the reason of that function is unclear, it is taken from Apollo/Atlas code that have
  420. * the same interrupt handler as Odin, is it because of a bug?
  421. */
  422. static void odin_isr_clear(struct imgpci_prvdata *data, unsigned int intstatus)
  423. {
  424. unsigned int max_retries = 1000;
  425. while ((odin_core_readreg32(data, PCI_ODIN_CORE_INTERRUPT_STATUS) & intstatus) && max_retries--)
  426. odin_core_writereg32(data, PCI_ODIN_CORE_INTERRUPT_CLR,
  427. (INT_INTERRUPT_MASTER_ENABLE | intstatus));
  428. }
  429. /**
  430. * pci_isr_cb - Low latency interrupt handler
  431. * @irq: irq number
  432. * @dev_id: pointer to private data
  433. */
  434. static irqreturn_t pci_isr_cb(int irq, void *dev_id)
  435. {
  436. uint32_t intstatus;
  437. struct pci_dev *dev = (struct pci_dev *)dev_id;
  438. struct imgpci_prvdata *data;
  439. irqreturn_t ret = IRQ_NONE;
  440. if (dev_id == NULL) {
  441. /* Spurious interrupt: not yet initialised. */
  442. pr_warn("Spurious interrupt data/dev_id not initialised!\n");
  443. goto exit;
  444. }
  445. data = vha_get_plat_data(&dev->dev);
  446. if (data == NULL) {
  447. /* Spurious interrupt: not yet initialised. */
  448. pr_warn("Invalid driver private data!\n");
  449. goto exit;
  450. }
  451. /* Read interrupt status register */
  452. intstatus = odin_core_readreg32(data, PCI_ODIN_CORE_INTERRUPT_STATUS);
  453. /* Now handle the ints */
  454. if (intstatus & odin_dut_interrupt_bit[dut_id]) {
  455. /* call real irq handler */
  456. ret = vha_handle_irq(&dev->dev);
  457. } else {
  458. /* most likely this is a shared interrupt line */
  459. dev_dbg(&dev->dev,
  460. "%s: unexpected or spurious interrupt [%x] (shared IRQ?)!\n",
  461. __func__, intstatus);
  462. /* WARN_ON(1); */
  463. goto exit;
  464. }
  465. /* Ack the ints */
  466. odin_isr_clear(data, intstatus);
  467. exit:
  468. return ret;
  469. }
  470. static inline void odin_reset_int(struct imgpci_prvdata *data) {
  471. odin_core_writereg32(data, PCI_ODIN_CORE_INTERRUPT_ENABLE, 0);
  472. odin_core_writereg32(data, PCI_ODIN_CORE_INTERRUPT_CLR, 0xFFFFFFFF);
  473. }
  474. /**
  475. * odin_enable_int - Enable an interrupt
  476. * @data: pointer to the data
  477. * @intmask: interrupt mask
  478. */
  479. static inline void odin_enable_int(struct imgpci_prvdata *data,
  480. uint32_t intmask)
  481. {
  482. uint32_t irq_enabled = odin_core_readreg32(data, PCI_ODIN_CORE_INTERRUPT_ENABLE);
  483. intmask &= odin_dut_interrupt_bit[dut_id];
  484. odin_core_writereg32(data, PCI_ODIN_CORE_INTERRUPT_ENABLE, irq_enabled | intmask | INT_INTERRUPT_MASTER_ENABLE);
  485. }
  486. /**
  487. * odin_disable_int - Disable an interrupt
  488. * @data: pointer to the data
  489. * @intmask: interrupt mask
  490. */
  491. static inline void odin_disable_int(struct imgpci_prvdata *data,
  492. uint32_t intmask)
  493. {
  494. uint32_t irq_enabled = odin_core_readreg32(data, PCI_ODIN_CORE_INTERRUPT_ENABLE);
  495. intmask &= odin_dut_interrupt_bit[dut_id];
  496. odin_core_writereg32(data, PCI_ODIN_CORE_INTERRUPT_ENABLE,
  497. irq_enabled & ~intmask);
  498. }
  499. /**
  500. * odin_allocate_registers - Allocate memory for a register (or memory) bank
  501. * @pci_dev: pointer to pci device
  502. * @data: pointer to the data
  503. * @bank: bank to set
  504. * @bar: BAR where the register are
  505. * @base: base address in the BAR
  506. * @size: size of the register set
  507. */
  508. static inline int odin_allocate_registers(struct pci_dev *pci_dev,
  509. struct imgpci_prvdata *data, int bank,
  510. int bar, unsigned long base, unsigned long size)
  511. {
  512. unsigned long bar_size = pci_resource_len(pci_dev, bar);
  513. unsigned long bar_addr = pci_resource_start(pci_dev, bar);
  514. unsigned long bar_max_size = bar_size - base;
  515. BUG_ON((base > bar_size) || ((base+size) > bar_size));
  516. data->reg_bank[bank].bar = bar;
  517. data->reg_bank[bank].addr = bar_addr + base;
  518. data->reg_bank[bank].size = min(size, bar_max_size);
  519. #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
  520. data->reg_bank[bank].km_addr = devm_ioremap_nocache(
  521. &pci_dev->dev, data->reg_bank[bank].addr,
  522. data->reg_bank[bank].size);
  523. #else
  524. data->reg_bank[bank].km_addr = devm_ioremap(
  525. &pci_dev->dev, data->reg_bank[bank].addr,
  526. data->reg_bank[bank].size);
  527. #endif
  528. pr_debug("[bank %u] bar:%d addr:0x%lx size:0x%lx km:0x%px\n",
  529. bank, bar, data->reg_bank[bank].addr,
  530. data->reg_bank[bank].size,
  531. data->reg_bank[bank].km_addr);
  532. return data->reg_bank[bank].km_addr == NULL;
  533. }
  534. int vha_plat_deinit(void)
  535. {
  536. struct pci_dev *dev = vha_pci_drv.pci_dev;
  537. int ret;
  538. if (dev) {
  539. struct imgpci_prvdata *data = vha_get_plat_data(&dev->dev);
  540. if (data) {
  541. /* reset the hardware */
  542. reset_dut(data);
  543. } else {
  544. dev_dbg(&dev->dev,
  545. "%s: prv data not found, HW reset omitted\n",
  546. __func__);
  547. }
  548. } else {
  549. /*pr_debug("%s: dev missing, HW reset omitted\n", __func__);*/
  550. }
  551. /* Unregister the driver from the OS */
  552. pci_unregister_driver(&(vha_pci_drv.pci_driver));
  553. ret = vha_deinit();
  554. if (ret)
  555. pr_err("VHA driver deinit failed\n");
  556. return ret;
  557. }
  558. #define NNA_REG_BAR (PCI_ODIN_DUT_REGS_BAR)
  559. #ifdef CFG_SYS_VAGUS
  560. #define NNA_REG_SIZE (_REG_SIZE + _REG_NNSYS_SIZE)
  561. #else
  562. #define NNA_REG_SIZE (_REG_SIZE)
  563. #endif
  564. #define NNA_REG_OFFSET (_REG_START)
  565. #ifdef CONFIG_GENERIC_ALLOCATOR
  566. static phys_addr_t carveout_to_dev_addr(union heap_options *options,
  567. phys_addr_t addr)
  568. {
  569. phys_addr_t base = options->carveout.phys;
  570. size_t size = options->carveout.size;
  571. unsigned long offset = options->carveout.offs;
  572. if (addr - offset >= base && addr < base + size - offset)
  573. return addr - base;
  574. pr_err("%s: unexpected addr! base 0x%llx size %zu offs %zu addr 0x%llx\n",
  575. __func__, base, size, offset, addr);
  576. WARN_ON(1);
  577. return addr;
  578. }
  579. static phys_addr_t carveout_to_host_addr(union heap_options *options,
  580. phys_addr_t addr)
  581. {
  582. phys_addr_t base = options->carveout.phys;
  583. size_t size = options->carveout.size;
  584. unsigned long offset = options->carveout.offs;
  585. if (addr < size - offset)
  586. return base + addr;
  587. pr_err("%s: unexpected addr! base %llx size %zu offs %zu addr %#llx\n",
  588. __func__, base, size, offset, addr);
  589. WARN_ON(1);
  590. return addr;
  591. }
  592. static void *carveout_get_kptr(phys_addr_t addr,
  593. size_t size, enum img_mem_attr mattr)
  594. {
  595. /*
  596. * Device memory is I/O memory and as a rule, it cannot
  597. * be dereferenced safely without memory barriers, that
  598. * is why it is guarded by __iomem (return of ioremap)
  599. * and checked by sparse. It is accessed only through
  600. * ioread32(), iowrit32(), etc.
  601. *
  602. * In x86 this memory can be dereferenced and safely
  603. * accessed, i.e. a * __iomem pointer can be casted to
  604. * a regular void* * pointer. We cast this here
  605. * assuming FPGA is x86 and add __force to silence the
  606. * sparse warning
  607. *
  608. * Note: System memory carveout can be used with cached turned on.
  609. * */
  610. void *kptr = NULL;
  611. if (mattr & IMG_MEM_ATTR_UNCACHED)
  612. #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
  613. kptr = (void * __force *)ioremap_nocache(addr, size);
  614. #else
  615. kptr = (void * __force *)ioremap(addr, size);
  616. #endif
  617. else if (mattr & IMG_MEM_ATTR_CACHED)
  618. kptr = (void * __force *)ioremap_cache(addr, size);
  619. else if (mattr & IMG_MEM_ATTR_WRITECOMBINE)
  620. kptr = (void * __force *)ioremap_wc(addr, size);
  621. /*pr_debug(
  622. "Mapping %zu bytes into kernel memory (Phys:%08llX, Kptr:%p)\n",
  623. size, addr, kptr);
  624. pr_debug("[%c%c%c]\n",
  625. (mattr & IMG_MEM_ATTR_UNCACHED) ? 'U' : '.',
  626. (mattr & IMG_MEM_ATTR_CACHED) ? 'C' : '.',
  627. (mattr & IMG_MEM_ATTR_WRITECOMBINE) ? 'W' : '.');*/
  628. return kptr;
  629. }
  630. static int carveout_put_kptr(void *addr)
  631. {
  632. /* pr_debug("Unmapping kernel memory (Phys: %p)\n", addr);*/
  633. iounmap(addr);
  634. return 0;
  635. }
  636. #endif
  637. /*
  638. * IO hooks.
  639. * NOTE: using spinlock to avoid
  640. * problems with multi threaded IO access
  641. */
  642. static DEFINE_SPINLOCK(io_irq_lock);
  643. static unsigned long io_irq_flags;
  644. uint64_t vha_plat_read64(void *addr)
  645. {
  646. u64 val;
  647. spin_lock_irqsave(&io_irq_lock, io_irq_flags);
  648. val =(uint64_t)readl((const volatile void __iomem *)addr) |
  649. ((uint64_t)readl((const volatile void __iomem *)addr + 4) << 32);
  650. spin_unlock_irqrestore(&io_irq_lock, io_irq_flags);
  651. return val;
  652. }
  653. void vha_plat_write64(void *addr, uint64_t val)
  654. {
  655. spin_lock_irqsave(&io_irq_lock, io_irq_flags);
  656. writel((uint32_t)(val & 0xffffffff), (volatile void __iomem *)addr);
  657. writel((uint32_t)(val >> 32), (volatile void __iomem *)addr + 4);
  658. spin_unlock_irqrestore(&io_irq_lock, io_irq_flags);
  659. }
  660. static int vha_plat_probe(struct pci_dev *pci_dev,
  661. const struct pci_device_id *id)
  662. {
  663. int ret = 0;
  664. struct imgpci_prvdata *data;
  665. size_t maxmapsize = maxmapsizeMB * 1024 * 1024;
  666. unsigned long vha_base_mem, vha_mem_size;
  667. struct device *dev = &pci_dev->dev;
  668. int heap;
  669. uint32_t tmp;
  670. dev_dbg(dev, "probing device, pci_dev: %p\n", dev);
  671. /* Enable the device */
  672. if (pci_enable_device(pci_dev))
  673. goto out_free;
  674. dev_info(dev, "%s dma_get_mask : %#llx\n", __func__, dma_get_mask(dev));
  675. if (dev->dma_mask) {
  676. dev_info(dev, "%s dev->dma_mask : %p : %#llx\n",
  677. __func__, dev->dma_mask, *dev->dma_mask);
  678. } else {
  679. dev_info(dev, "%s mask unset, setting coherent\n", __func__);
  680. dev->dma_mask = &dev->coherent_dma_mask;
  681. }
  682. ret = dma_set_mask(dev, DMA_BIT_MASK(36));
  683. if (ret) {
  684. dev_err(dev, "%s failed to set dma mask\n", __func__);
  685. goto out_disable;
  686. }
  687. dev_info(dev, "%s dma_set_mask %#llx\n", __func__, dma_get_mask(dev));
  688. /* Reserve PCI I/O and memory resources */
  689. if (pci_request_regions(pci_dev, "imgpci"))
  690. goto out_disable;
  691. /* Create a kernel space mapping for each of the bars */
  692. data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
  693. if (!data) {
  694. pr_err("Memory allocation error, aborting.\n");
  695. ret = -ENOMEM;
  696. goto out_release;
  697. }
  698. dev_dbg(dev, "allocated imgpci_prvdata @ %p\n", data);
  699. memset(data, 0, sizeof(*data));
  700. /* Allocate odin core registers */
  701. ret = odin_allocate_registers(pci_dev, data,
  702. CORE_REG_BANK, PCI_ODIN_SYS_CTRL_REGS_BAR,
  703. PCI_ODIN_SYS_CTRL_BASE_OFFSET,
  704. PCI_ODIN_CORE_REG_SIZE);
  705. if (ret) {
  706. dev_err(dev, "Can't allocate memory for odin regs!");
  707. ret = -ENOMEM;
  708. goto out_release;
  709. }
  710. /* Display some infos */
  711. {
  712. uint32_t odin_id = odin_core_readreg32(data, PCI_ODIN_CORE_ID);
  713. uint32_t odin_rev = odin_core_readreg32(data, PCI_ODIN_CORE_REVISION);
  714. uint32_t odin_cs = odin_core_readreg32(data, PCI_ODIN_CORE_CHANGE_SET);
  715. uint32_t odin_ui = odin_core_readreg32(data, PCI_ODIN_CORE_USER_ID);
  716. uint32_t odin_ub = odin_core_readreg32(data, PCI_ODIN_CORE_USER_BUILD);
  717. pr_info("Found Odin lite board v%d.%d (ID:%X CS:%X UI:%X UB:%X)",
  718. (odin_rev >> 8) & 0xF, odin_rev & 0xF, odin_id & 0x7, odin_cs, odin_ui, odin_ub);
  719. }
  720. if (dut_id > MAX_DUT_ID) {
  721. dev_err(dev, "Invalid DUT number (%lu), setting it to 0\n", dut_id);
  722. dut_id = 0;
  723. }
  724. /* Allocate NNA register space */
  725. ret = odin_allocate_registers(pci_dev, data,
  726. NNA_REG_BANK, NNA_REG_BAR,
  727. NNA_REG_OFFSET + odin_dut_register_offset[dut_id],
  728. NNA_REG_SIZE);
  729. if (ret) {
  730. dev_err(dev, "Can't allocate memory for vha regs!");
  731. ret = -ENOMEM;
  732. goto out_release;
  733. }
  734. /* Allocate DUT memory space */
  735. vha_mem_size = pci_resource_len(pci_dev, PCI_ODIN_DUT_MEM_BAR);
  736. if (vha_mem_size > maxmapsize)
  737. vha_mem_size = maxmapsize;
  738. vha_base_mem = pci_resource_start(pci_dev, PCI_ODIN_DUT_MEM_BAR);
  739. /* change alloc size according to module parameter */
  740. if (pci_size)
  741. vha_mem_size = pci_size;
  742. /* allocating memory only when static kernel mapping is requested,
  743. * so hand set values here: */
  744. data->reg_bank[MEM_REG_BANK].bar = PCI_ODIN_DUT_MEM_BAR;
  745. data->reg_bank[MEM_REG_BANK].addr = vha_base_mem;
  746. data->reg_bank[MEM_REG_BANK].size = vha_mem_size;
  747. if (mem_static_kptr) {
  748. data->reg_bank[MEM_REG_BANK].km_addr = devm_ioremap(
  749. &pci_dev->dev, data->reg_bank[MEM_REG_BANK].addr,
  750. data->reg_bank[MEM_REG_BANK].size);
  751. if (data->reg_bank[MEM_REG_BANK].km_addr == NULL) {
  752. dev_err(dev, "Can't allocate memory for vha regs!");
  753. ret = -ENOMEM;
  754. goto out_release;
  755. }
  756. }
  757. pr_debug("[bank %u] bar: %d addr: 0x%lx (kptr:%p) size: 0x%lx\n",
  758. MEM_REG_BANK, PCI_ODIN_DUT_MEM_BAR,
  759. data->reg_bank[MEM_REG_BANK].addr,
  760. data->reg_bank[MEM_REG_BANK].km_addr,
  761. data->reg_bank[MEM_REG_BANK].size);
  762. #ifdef FPGA_BUS_MASTERING
  763. tmp = odin_core_readreg32(data, PCI_ODIN_CORE_DUT_CTRL1);
  764. tmp &= ~DUT_CTRL1_DUT_MST_OFFSET;
  765. odin_core_writereg32(data, PCI_ODIN_CORE_DUT_CTRL1, tmp);
  766. tmp = odin_core_readreg32(data, PCI_ODIN_CORE_CORE_CONTROL);
  767. tmp &= ODIN_CORE_CONTROL_DUT_OFFSET_MASK;
  768. odin_core_writereg32(data, PCI_ODIN_CORE_CORE_CONTROL, tmp);
  769. #else
  770. /* Set the Odin board in a similar way as the Apollo is,
  771. * DUT memory starting at 0x0 instead of 0x4_0000_0000
  772. */
  773. tmp = odin_core_readreg32(data, PCI_ODIN_CORE_DUT_CTRL1);
  774. tmp |= DUT_CTRL1_DUT_MST_OFFSET;
  775. odin_core_writereg32(data, PCI_ODIN_CORE_DUT_CTRL1, tmp);
  776. tmp = odin_core_readreg32(data, PCI_ODIN_CORE_CORE_CONTROL);
  777. tmp &= ODIN_CORE_CONTROL_DUT_OFFSET_MASK;
  778. tmp |= (4 << ODIN_CORE_CONTROL_DUT_OFFSET_SHIFT);
  779. odin_core_writereg32(data, PCI_ODIN_CORE_CORE_CONTROL, tmp);
  780. #endif
  781. /* Get the IRQ...*/
  782. data->irq = pci_dev->irq;
  783. data->pci_dev = pci_dev;
  784. vha_pci_drv.pci_dev = pci_dev;
  785. reset_dut(data);
  786. odin_reset_int(data);
  787. odin_enable_int(data, odin_dut_interrupt_bit[dut_id]);
  788. for (heap = 0; heap < vha_dev_fpga_heaps; heap++) {
  789. struct heap_config *cfg = &vha_dev_fpga_heap_configs[heap];
  790. #ifdef CONFIG_GENERIC_ALLOCATOR
  791. if (cfg->type == IMG_MEM_HEAP_TYPE_CARVEOUT) {
  792. if (contig_size && contig_phys_start) {
  793. /*
  794. * 2 types of carveout memory are supported:
  795. * memory carved out of the main DDR
  796. * memory region.
  797. * eg: linux boot option memmap=512M$0x5CAFFFFF
  798. * This is configured using module parameters:
  799. * contig_phys_start and size
  800. * DDR populated in the actual PCI card,
  801. * in BAR 4.
  802. * The module parameters take precedence
  803. * over PCI memory.
  804. */
  805. cfg->options.carveout.phys = contig_phys_start;
  806. cfg->options.carveout.size = contig_size;
  807. cfg->to_dev_addr = NULL;
  808. cfg->to_host_addr = NULL;
  809. /*dev_info(dev, "using %dMB CARVEOUT at x%lx\n",
  810. contig_size/1024/1024,
  811. contig_phys_start);*/
  812. } else {
  813. cfg->options.carveout.phys =
  814. data->reg_bank[MEM_REG_BANK].addr;
  815. if (mem_static_kptr)
  816. cfg->options.carveout.kptr =
  817. data->reg_bank[MEM_REG_BANK].km_addr;
  818. cfg->options.carveout.size =
  819. data->reg_bank[MEM_REG_BANK].size;
  820. cfg->options.carveout.offs = pci_offset;
  821. cfg->to_dev_addr = carveout_to_dev_addr;
  822. cfg->to_host_addr = carveout_to_host_addr;
  823. /* dev_info(dev,
  824. "using %zuMB CARVEOUT from PCI at 0x%x\n",
  825. cfg->options.carveout.size/1024/1024,
  826. cfg->options.carveout.phys);*/
  827. }
  828. /* IO memory access callbacks */
  829. if (!mem_static_kptr) {
  830. /* Dynamic kernel memory mapping */
  831. cfg->options.carveout.get_kptr = carveout_get_kptr;
  832. cfg->options.carveout.put_kptr = carveout_put_kptr;
  833. }
  834. /* Allocation order */
  835. cfg->options.carveout.pool_order = pool_alloc_order;
  836. break;
  837. }
  838. #endif
  839. if (cfg->type == IMG_MEM_HEAP_TYPE_COHERENT) {
  840. ret = dma_declare_coherent_memory(dev,
  841. contig_phys_start,
  842. contig_phys_start,
  843. contig_size
  844. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,1,0)
  845. #if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
  846. , DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE
  847. #else
  848. , DMA_MEMORY_EXCLUSIVE
  849. #endif
  850. #endif
  851. );
  852. if (ret == 0) {
  853. dev_err(dev, "failed to initialize coherent memory!\n");
  854. /* We will fallback to the
  855. * default pool anyway
  856. goto out_release; */
  857. }
  858. break;
  859. }
  860. }
  861. ret = vha_add_dev(dev,
  862. vha_dev_fpga_heap_configs,
  863. vha_dev_fpga_heaps,
  864. data,
  865. data->reg_bank[NNA_REG_BANK].km_addr,
  866. data->reg_bank[NNA_REG_BANK].size);
  867. if (ret) {
  868. dev_err(dev, "failed to initialize driver core!\n");
  869. goto out_heap_deinit;
  870. }
  871. /*
  872. * Reset FPGA DUT only after disabling clocks in
  873. * vha_add_dev()-> get properties.
  874. * This workaround is required to ensure that
  875. * clocks (on daughter board) are enabled for test slave scripts to
  876. * read FPGA build version register.
  877. * NOTE: Asserting other bits like DDR reset bit cause problems
  878. * with bus mastering feature, thus results in memory failures.
  879. */
  880. reset_dut(data);
  881. {
  882. /*uint32_t fpga_rev = odin_readreg32(data, 1,
  883. FPGA_IMAGE_REV_OFFSET) & FPGA_IMAGE_REV_MASK;
  884. dev_dbg(dev, "fpga image revision: 0x%x\n", fpga_rev);
  885. if (!fpga_rev || fpga_rev == 0xdead1) {
  886. dev_err(dev, "fpga revision incorrect (0x%x)!\n",
  887. fpga_rev);
  888. goto out_rm_dev;
  889. }*/
  890. }
  891. /* Install the ISR callback...*/
  892. ret = devm_request_threaded_irq(dev, data->irq, &pci_isr_cb,
  893. &pci_thread_irq, IRQF_SHARED, DEVICE_NAME,
  894. (void *)pci_dev);
  895. if (ret) {
  896. dev_err(dev, "failed to request irq!\n");
  897. goto out_rm_dev;
  898. }
  899. dev_dbg(dev, "registered irq %d\n", data->irq);
  900. /* Try to calibrate the core if needed */
  901. ret = vha_dev_calibrate(dev, FREQ_MEASURE_CYCLES);
  902. if (ret) {
  903. dev_err(dev, "%s: Failed to start clock calibration!\n", __func__);
  904. goto out_rm_dev;
  905. }
  906. return ret;
  907. out_rm_dev:
  908. vha_rm_dev(dev);
  909. out_heap_deinit:
  910. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
  911. /* Release any declared mem regions */
  912. dma_release_declared_memory(dev);
  913. #endif
  914. /* Make sure int are no longer enabled */
  915. odin_disable_int(data, odin_dut_interrupt_bit[dut_id]);
  916. out_release:
  917. pci_release_regions(pci_dev);
  918. out_disable:
  919. pci_disable_device(pci_dev);
  920. out_free:
  921. return ret;
  922. }
  923. static void vha_plat_remove(struct pci_dev *dev)
  924. {
  925. struct imgpci_prvdata *data = vha_get_plat_data(&dev->dev);
  926. dev_dbg(&dev->dev, "removing device\n");
  927. if (data == NULL) {
  928. dev_err(&dev->dev, "PCI priv data missing!\n");
  929. } else {
  930. /*
  931. * We need to disable interrupts for the
  932. * embedded device via the fpga interrupt controller...
  933. */
  934. odin_disable_int(data, odin_dut_interrupt_bit[dut_id]);
  935. /* Unregister int */
  936. devm_free_irq(&dev->dev, data->irq, dev);
  937. }
  938. #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
  939. /* Release any declared mem regions */
  940. dma_release_declared_memory(&dev->dev);
  941. #endif
  942. pci_release_regions(dev);
  943. pci_disable_device(dev);
  944. vha_rm_dev(&dev->dev);
  945. }
  946. #ifdef CONFIG_PM
  947. static int vha_plat_suspend(struct device *dev)
  948. {
  949. return vha_suspend_dev(dev);
  950. }
  951. static int vha_plat_resume(struct device *dev)
  952. {
  953. return vha_resume_dev(dev);
  954. }
  955. #endif
  956. int vha_plat_init(void)
  957. {
  958. int ret;
  959. #if 0
  960. #ifdef FPGA_BUS_MASTERING
  961. vha_plat_fpga_heap_configs[0].type = fpga_heap_type;
  962. #endif
  963. #endif
  964. ret = vha_init_plat_heaps(vha_plat_fpga_heap_configs, vha_plat_fpga_heaps);
  965. if(ret) {
  966. pr_err("failed to initialize global heaps\n");
  967. return -ENOMEM;
  968. }
  969. ret = pci_register_driver(&vha_pci_drv.pci_driver);
  970. if (ret) {
  971. pr_err("failed to register PCI driver!\n");
  972. return ret;
  973. }
  974. /* pci_dev should be set in probe */
  975. if (!vha_pci_drv.pci_dev) {
  976. pr_err("failed to find VHA PCI dev!\n");
  977. pci_unregister_driver(&vha_pci_drv.pci_driver);
  978. return -ENODEV;
  979. }
  980. return 0;
  981. }