rockchip-iommu.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * IOMMU API for Rockchip
  4. *
  5. * Module Authors: Simon Xue <xxm@rock-chips.com>
  6. * Daniel Kurtz <djkurtz@chromium.org>
  7. */
  8. #include <linux/clk.h>
  9. #include <linux/compiler.h>
  10. #include <linux/delay.h>
  11. #include <linux/device.h>
  12. #include <linux/dma-iommu.h>
  13. #include <linux/dma-mapping.h>
  14. #include <linux/errno.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/io.h>
  17. #include <linux/iommu.h>
  18. #include <linux/iopoll.h>
  19. #include <linux/list.h>
  20. #include <linux/mm.h>
  21. #include <linux/init.h>
  22. #include <linux/of.h>
  23. #include <linux/of_iommu.h>
  24. #include <linux/of_platform.h>
  25. #include <linux/platform_device.h>
  26. #include <linux/pm_runtime.h>
  27. #include <linux/slab.h>
  28. #include <linux/spinlock.h>
  29. /** MMU register offsets */
  30. #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */
  31. #define RK_MMU_STATUS 0x04
  32. #define RK_MMU_COMMAND 0x08
  33. #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */
  34. #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */
  35. #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */
  36. #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */
  37. #define RK_MMU_INT_MASK 0x1C /* IRQ enable */
  38. #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */
  39. #define RK_MMU_AUTO_GATING 0x24
  40. #define DTE_ADDR_DUMMY 0xCAFEBABE
  41. #define RK_MMU_POLL_PERIOD_US 100
  42. #define RK_MMU_FORCE_RESET_TIMEOUT_US 100000
  43. #define RK_MMU_POLL_TIMEOUT_US 1000
  44. /* RK_MMU_STATUS fields */
  45. #define RK_MMU_STATUS_PAGING_ENABLED BIT(0)
  46. #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1)
  47. #define RK_MMU_STATUS_STALL_ACTIVE BIT(2)
  48. #define RK_MMU_STATUS_IDLE BIT(3)
  49. #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4)
  50. #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5)
  51. #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31)
  52. /* RK_MMU_COMMAND command values */
  53. #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */
  54. #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */
  55. #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */
  56. #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */
  57. #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */
  58. #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */
  59. #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */
  60. /* RK_MMU_INT_* register fields */
  61. #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */
  62. #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */
  63. #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR)
  64. #define NUM_DT_ENTRIES 1024
  65. #define NUM_PT_ENTRIES 1024
  66. #define SPAGE_ORDER 12
  67. #define SPAGE_SIZE (1 << SPAGE_ORDER)
  68. /*
  69. * Support mapping any size that fits in one page table:
  70. * 4 KiB to 4 MiB
  71. */
  72. #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000
  73. struct rk_iommu_domain {
  74. struct list_head iommus;
  75. u32 *dt; /* page directory table */
  76. dma_addr_t dt_dma;
  77. spinlock_t iommus_lock; /* lock for iommus list */
  78. spinlock_t dt_lock; /* lock for modifying page directory table */
  79. struct iommu_domain domain;
  80. };
  81. /* list of clocks required by IOMMU */
  82. static const char * const rk_iommu_clocks[] = {
  83. "aclk", "iface",
  84. };
  85. struct rk_iommu {
  86. struct device *dev;
  87. void __iomem **bases;
  88. int num_mmu;
  89. int num_irq;
  90. struct clk_bulk_data *clocks;
  91. int num_clocks;
  92. bool reset_disabled;
  93. struct iommu_device iommu;
  94. struct list_head node; /* entry in rk_iommu_domain.iommus */
  95. struct iommu_domain *domain; /* domain to which iommu is attached */
  96. struct iommu_group *group;
  97. };
  98. struct rk_iommudata {
  99. struct device_link *link; /* runtime PM link from IOMMU to master */
  100. struct rk_iommu *iommu;
  101. };
  102. static struct device *dma_dev;
  103. static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma,
  104. unsigned int count)
  105. {
  106. size_t size = count * sizeof(u32); /* count of u32 entry */
  107. dma_sync_single_for_device(dma_dev, dma, size, DMA_TO_DEVICE);
  108. }
  109. static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
  110. {
  111. return container_of(dom, struct rk_iommu_domain, domain);
  112. }
  113. /*
  114. * The Rockchip rk3288 iommu uses a 2-level page table.
  115. * The first level is the "Directory Table" (DT).
  116. * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing
  117. * to a "Page Table".
  118. * The second level is the 1024 Page Tables (PT).
  119. * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to
  120. * a 4 KB page of physical memory.
  121. *
  122. * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries).
  123. * Each iommu device has a MMU_DTE_ADDR register that contains the physical
  124. * address of the start of the DT page.
  125. *
  126. * The structure of the page table is as follows:
  127. *
  128. * DT
  129. * MMU_DTE_ADDR -> +-----+
  130. * | |
  131. * +-----+ PT
  132. * | DTE | -> +-----+
  133. * +-----+ | | Memory
  134. * | | +-----+ Page
  135. * | | | PTE | -> +-----+
  136. * +-----+ +-----+ | |
  137. * | | | |
  138. * | | | |
  139. * +-----+ | |
  140. * | |
  141. * | |
  142. * +-----+
  143. */
  144. /*
  145. * Each DTE has a PT address and a valid bit:
  146. * +---------------------+-----------+-+
  147. * | PT address | Reserved |V|
  148. * +---------------------+-----------+-+
  149. * 31:12 - PT address (PTs always starts on a 4 KB boundary)
  150. * 11: 1 - Reserved
  151. * 0 - 1 if PT @ PT address is valid
  152. */
  153. #define RK_DTE_PT_ADDRESS_MASK 0xfffff000
  154. #define RK_DTE_PT_VALID BIT(0)
  155. static inline phys_addr_t rk_dte_pt_address(u32 dte)
  156. {
  157. return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK;
  158. }
  159. static inline bool rk_dte_is_pt_valid(u32 dte)
  160. {
  161. return dte & RK_DTE_PT_VALID;
  162. }
  163. static inline u32 rk_mk_dte(dma_addr_t pt_dma)
  164. {
  165. return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID;
  166. }
  167. /*
  168. * Each PTE has a Page address, some flags and a valid bit:
  169. * +---------------------+---+-------+-+
  170. * | Page address |Rsv| Flags |V|
  171. * +---------------------+---+-------+-+
  172. * 31:12 - Page address (Pages always start on a 4 KB boundary)
  173. * 11: 9 - Reserved
  174. * 8: 1 - Flags
  175. * 8 - Read allocate - allocate cache space on read misses
  176. * 7 - Read cache - enable cache & prefetch of data
  177. * 6 - Write buffer - enable delaying writes on their way to memory
  178. * 5 - Write allocate - allocate cache space on write misses
  179. * 4 - Write cache - different writes can be merged together
  180. * 3 - Override cache attributes
  181. * if 1, bits 4-8 control cache attributes
  182. * if 0, the system bus defaults are used
  183. * 2 - Writable
  184. * 1 - Readable
  185. * 0 - 1 if Page @ Page address is valid
  186. */
  187. #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000
  188. #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe
  189. #define RK_PTE_PAGE_WRITABLE BIT(2)
  190. #define RK_PTE_PAGE_READABLE BIT(1)
  191. #define RK_PTE_PAGE_VALID BIT(0)
  192. static inline phys_addr_t rk_pte_page_address(u32 pte)
  193. {
  194. return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK;
  195. }
  196. static inline bool rk_pte_is_page_valid(u32 pte)
  197. {
  198. return pte & RK_PTE_PAGE_VALID;
  199. }
  200. /* TODO: set cache flags per prot IOMMU_CACHE */
  201. static u32 rk_mk_pte(phys_addr_t page, int prot)
  202. {
  203. u32 flags = 0;
  204. flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
  205. flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
  206. page &= RK_PTE_PAGE_ADDRESS_MASK;
  207. return page | flags | RK_PTE_PAGE_VALID;
  208. }
  209. static u32 rk_mk_pte_invalid(u32 pte)
  210. {
  211. return pte & ~RK_PTE_PAGE_VALID;
  212. }
  213. /*
  214. * rk3288 iova (IOMMU Virtual Address) format
  215. * 31 22.21 12.11 0
  216. * +-----------+-----------+-------------+
  217. * | DTE index | PTE index | Page offset |
  218. * +-----------+-----------+-------------+
  219. * 31:22 - DTE index - index of DTE in DT
  220. * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address
  221. * 11: 0 - Page offset - offset into page @ PTE.page_address
  222. */
  223. #define RK_IOVA_DTE_MASK 0xffc00000
  224. #define RK_IOVA_DTE_SHIFT 22
  225. #define RK_IOVA_PTE_MASK 0x003ff000
  226. #define RK_IOVA_PTE_SHIFT 12
  227. #define RK_IOVA_PAGE_MASK 0x00000fff
  228. #define RK_IOVA_PAGE_SHIFT 0
  229. static u32 rk_iova_dte_index(dma_addr_t iova)
  230. {
  231. return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT;
  232. }
  233. static u32 rk_iova_pte_index(dma_addr_t iova)
  234. {
  235. return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT;
  236. }
  237. static u32 rk_iova_page_offset(dma_addr_t iova)
  238. {
  239. return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT;
  240. }
  241. static u32 rk_iommu_read(void __iomem *base, u32 offset)
  242. {
  243. return readl(base + offset);
  244. }
  245. static void rk_iommu_write(void __iomem *base, u32 offset, u32 value)
  246. {
  247. writel(value, base + offset);
  248. }
  249. static void rk_iommu_command(struct rk_iommu *iommu, u32 command)
  250. {
  251. int i;
  252. for (i = 0; i < iommu->num_mmu; i++)
  253. writel(command, iommu->bases[i] + RK_MMU_COMMAND);
  254. }
  255. static void rk_iommu_base_command(void __iomem *base, u32 command)
  256. {
  257. writel(command, base + RK_MMU_COMMAND);
  258. }
  259. static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start,
  260. size_t size)
  261. {
  262. int i;
  263. dma_addr_t iova_end = iova_start + size;
  264. /*
  265. * TODO(djkurtz): Figure out when it is more efficient to shootdown the
  266. * entire iotlb rather than iterate over individual iovas.
  267. */
  268. for (i = 0; i < iommu->num_mmu; i++) {
  269. dma_addr_t iova;
  270. for (iova = iova_start; iova < iova_end; iova += SPAGE_SIZE)
  271. rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova);
  272. }
  273. }
  274. static bool rk_iommu_is_stall_active(struct rk_iommu *iommu)
  275. {
  276. bool active = true;
  277. int i;
  278. for (i = 0; i < iommu->num_mmu; i++)
  279. active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &
  280. RK_MMU_STATUS_STALL_ACTIVE);
  281. return active;
  282. }
  283. static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu)
  284. {
  285. bool enable = true;
  286. int i;
  287. for (i = 0; i < iommu->num_mmu; i++)
  288. enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &
  289. RK_MMU_STATUS_PAGING_ENABLED);
  290. return enable;
  291. }
  292. static bool rk_iommu_is_reset_done(struct rk_iommu *iommu)
  293. {
  294. bool done = true;
  295. int i;
  296. for (i = 0; i < iommu->num_mmu; i++)
  297. done &= rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0;
  298. return done;
  299. }
  300. static int rk_iommu_enable_stall(struct rk_iommu *iommu)
  301. {
  302. int ret, i;
  303. bool val;
  304. if (rk_iommu_is_stall_active(iommu))
  305. return 0;
  306. /* Stall can only be enabled if paging is enabled */
  307. if (!rk_iommu_is_paging_enabled(iommu))
  308. return 0;
  309. rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL);
  310. ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,
  311. val, RK_MMU_POLL_PERIOD_US,
  312. RK_MMU_POLL_TIMEOUT_US);
  313. if (ret)
  314. for (i = 0; i < iommu->num_mmu; i++)
  315. dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n",
  316. rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
  317. return ret;
  318. }
  319. static int rk_iommu_disable_stall(struct rk_iommu *iommu)
  320. {
  321. int ret, i;
  322. bool val;
  323. if (!rk_iommu_is_stall_active(iommu))
  324. return 0;
  325. rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL);
  326. ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val,
  327. !val, RK_MMU_POLL_PERIOD_US,
  328. RK_MMU_POLL_TIMEOUT_US);
  329. if (ret)
  330. for (i = 0; i < iommu->num_mmu; i++)
  331. dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n",
  332. rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
  333. return ret;
  334. }
  335. static int rk_iommu_enable_paging(struct rk_iommu *iommu)
  336. {
  337. int ret, i;
  338. bool val;
  339. if (rk_iommu_is_paging_enabled(iommu))
  340. return 0;
  341. rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING);
  342. ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,
  343. val, RK_MMU_POLL_PERIOD_US,
  344. RK_MMU_POLL_TIMEOUT_US);
  345. if (ret)
  346. for (i = 0; i < iommu->num_mmu; i++)
  347. dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n",
  348. rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
  349. return ret;
  350. }
  351. static int rk_iommu_disable_paging(struct rk_iommu *iommu)
  352. {
  353. int ret, i;
  354. bool val;
  355. if (!rk_iommu_is_paging_enabled(iommu))
  356. return 0;
  357. rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING);
  358. ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val,
  359. !val, RK_MMU_POLL_PERIOD_US,
  360. RK_MMU_POLL_TIMEOUT_US);
  361. if (ret)
  362. for (i = 0; i < iommu->num_mmu; i++)
  363. dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n",
  364. rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
  365. return ret;
  366. }
  367. static int rk_iommu_force_reset(struct rk_iommu *iommu)
  368. {
  369. int ret, i;
  370. u32 dte_addr;
  371. bool val;
  372. if (iommu->reset_disabled)
  373. return 0;
  374. /*
  375. * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY
  376. * and verifying that upper 5 nybbles are read back.
  377. */
  378. for (i = 0; i < iommu->num_mmu; i++) {
  379. rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY);
  380. dte_addr = rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR);
  381. if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) {
  382. dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n");
  383. return -EFAULT;
  384. }
  385. }
  386. rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET);
  387. ret = readx_poll_timeout(rk_iommu_is_reset_done, iommu, val,
  388. val, RK_MMU_FORCE_RESET_TIMEOUT_US,
  389. RK_MMU_POLL_TIMEOUT_US);
  390. if (ret) {
  391. dev_err(iommu->dev, "FORCE_RESET command timed out\n");
  392. return ret;
  393. }
  394. return 0;
  395. }
  396. static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
  397. {
  398. void __iomem *base = iommu->bases[index];
  399. u32 dte_index, pte_index, page_offset;
  400. u32 mmu_dte_addr;
  401. phys_addr_t mmu_dte_addr_phys, dte_addr_phys;
  402. u32 *dte_addr;
  403. u32 dte;
  404. phys_addr_t pte_addr_phys = 0;
  405. u32 *pte_addr = NULL;
  406. u32 pte = 0;
  407. phys_addr_t page_addr_phys = 0;
  408. u32 page_flags = 0;
  409. dte_index = rk_iova_dte_index(iova);
  410. pte_index = rk_iova_pte_index(iova);
  411. page_offset = rk_iova_page_offset(iova);
  412. mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR);
  413. mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr;
  414. dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index);
  415. dte_addr = phys_to_virt(dte_addr_phys);
  416. dte = *dte_addr;
  417. if (!rk_dte_is_pt_valid(dte))
  418. goto print_it;
  419. pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4);
  420. pte_addr = phys_to_virt(pte_addr_phys);
  421. pte = *pte_addr;
  422. if (!rk_pte_is_page_valid(pte))
  423. goto print_it;
  424. page_addr_phys = rk_pte_page_address(pte) + page_offset;
  425. page_flags = pte & RK_PTE_PAGE_FLAGS_MASK;
  426. print_it:
  427. dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n",
  428. &iova, dte_index, pte_index, page_offset);
  429. dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n",
  430. &mmu_dte_addr_phys, &dte_addr_phys, dte,
  431. rk_dte_is_pt_valid(dte), &pte_addr_phys, pte,
  432. rk_pte_is_page_valid(pte), &page_addr_phys, page_flags);
  433. }
  434. static irqreturn_t rk_iommu_irq(int irq, void *dev_id)
  435. {
  436. struct rk_iommu *iommu = dev_id;
  437. u32 status;
  438. u32 int_status;
  439. dma_addr_t iova;
  440. irqreturn_t ret = IRQ_NONE;
  441. int i, err;
  442. err = pm_runtime_get_if_in_use(iommu->dev);
  443. if (!err || WARN_ON_ONCE(err < 0))
  444. return ret;
  445. if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)))
  446. goto out;
  447. for (i = 0; i < iommu->num_mmu; i++) {
  448. int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS);
  449. if (int_status == 0)
  450. continue;
  451. ret = IRQ_HANDLED;
  452. iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR);
  453. if (int_status & RK_MMU_IRQ_PAGE_FAULT) {
  454. int flags;
  455. status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS);
  456. flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ?
  457. IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
  458. dev_err(iommu->dev, "Page fault at %pad of type %s\n",
  459. &iova,
  460. (flags == IOMMU_FAULT_WRITE) ? "write" : "read");
  461. log_iova(iommu, i, iova);
  462. /*
  463. * Report page fault to any installed handlers.
  464. * Ignore the return code, though, since we always zap cache
  465. * and clear the page fault anyway.
  466. */
  467. if (iommu->domain)
  468. report_iommu_fault(iommu->domain, iommu->dev, iova,
  469. flags);
  470. else
  471. dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n");
  472. rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
  473. rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE);
  474. }
  475. if (int_status & RK_MMU_IRQ_BUS_ERROR)
  476. dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova);
  477. if (int_status & ~RK_MMU_IRQ_MASK)
  478. dev_err(iommu->dev, "unexpected int_status: %#08x\n",
  479. int_status);
  480. rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status);
  481. }
  482. clk_bulk_disable(iommu->num_clocks, iommu->clocks);
  483. out:
  484. pm_runtime_put(iommu->dev);
  485. return ret;
  486. }
  487. static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain,
  488. dma_addr_t iova)
  489. {
  490. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  491. unsigned long flags;
  492. phys_addr_t pt_phys, phys = 0;
  493. u32 dte, pte;
  494. u32 *page_table;
  495. spin_lock_irqsave(&rk_domain->dt_lock, flags);
  496. dte = rk_domain->dt[rk_iova_dte_index(iova)];
  497. if (!rk_dte_is_pt_valid(dte))
  498. goto out;
  499. pt_phys = rk_dte_pt_address(dte);
  500. page_table = (u32 *)phys_to_virt(pt_phys);
  501. pte = page_table[rk_iova_pte_index(iova)];
  502. if (!rk_pte_is_page_valid(pte))
  503. goto out;
  504. phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova);
  505. out:
  506. spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
  507. return phys;
  508. }
  509. static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain,
  510. dma_addr_t iova, size_t size)
  511. {
  512. struct list_head *pos;
  513. unsigned long flags;
  514. /* shootdown these iova from all iommus using this domain */
  515. spin_lock_irqsave(&rk_domain->iommus_lock, flags);
  516. list_for_each(pos, &rk_domain->iommus) {
  517. struct rk_iommu *iommu;
  518. int ret;
  519. iommu = list_entry(pos, struct rk_iommu, node);
  520. /* Only zap TLBs of IOMMUs that are powered on. */
  521. ret = pm_runtime_get_if_in_use(iommu->dev);
  522. if (WARN_ON_ONCE(ret < 0))
  523. continue;
  524. if (ret) {
  525. WARN_ON(clk_bulk_enable(iommu->num_clocks,
  526. iommu->clocks));
  527. rk_iommu_zap_lines(iommu, iova, size);
  528. clk_bulk_disable(iommu->num_clocks, iommu->clocks);
  529. pm_runtime_put(iommu->dev);
  530. }
  531. }
  532. spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
  533. }
  534. static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain,
  535. dma_addr_t iova, size_t size)
  536. {
  537. rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);
  538. if (size > SPAGE_SIZE)
  539. rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE,
  540. SPAGE_SIZE);
  541. }
  542. static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
  543. dma_addr_t iova)
  544. {
  545. u32 *page_table, *dte_addr;
  546. u32 dte_index, dte;
  547. phys_addr_t pt_phys;
  548. dma_addr_t pt_dma;
  549. assert_spin_locked(&rk_domain->dt_lock);
  550. dte_index = rk_iova_dte_index(iova);
  551. dte_addr = &rk_domain->dt[dte_index];
  552. dte = *dte_addr;
  553. if (rk_dte_is_pt_valid(dte))
  554. goto done;
  555. page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
  556. if (!page_table)
  557. return ERR_PTR(-ENOMEM);
  558. pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE);
  559. if (dma_mapping_error(dma_dev, pt_dma)) {
  560. dev_err(dma_dev, "DMA mapping error while allocating page table\n");
  561. free_page((unsigned long)page_table);
  562. return ERR_PTR(-ENOMEM);
  563. }
  564. dte = rk_mk_dte(pt_dma);
  565. *dte_addr = dte;
  566. rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES);
  567. rk_table_flush(rk_domain,
  568. rk_domain->dt_dma + dte_index * sizeof(u32), 1);
  569. done:
  570. pt_phys = rk_dte_pt_address(dte);
  571. return (u32 *)phys_to_virt(pt_phys);
  572. }
  573. static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,
  574. u32 *pte_addr, dma_addr_t pte_dma,
  575. size_t size)
  576. {
  577. unsigned int pte_count;
  578. unsigned int pte_total = size / SPAGE_SIZE;
  579. assert_spin_locked(&rk_domain->dt_lock);
  580. for (pte_count = 0; pte_count < pte_total; pte_count++) {
  581. u32 pte = pte_addr[pte_count];
  582. if (!rk_pte_is_page_valid(pte))
  583. break;
  584. pte_addr[pte_count] = rk_mk_pte_invalid(pte);
  585. }
  586. rk_table_flush(rk_domain, pte_dma, pte_count);
  587. return pte_count * SPAGE_SIZE;
  588. }
  589. static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
  590. dma_addr_t pte_dma, dma_addr_t iova,
  591. phys_addr_t paddr, size_t size, int prot)
  592. {
  593. unsigned int pte_count;
  594. unsigned int pte_total = size / SPAGE_SIZE;
  595. phys_addr_t page_phys;
  596. assert_spin_locked(&rk_domain->dt_lock);
  597. for (pte_count = 0; pte_count < pte_total; pte_count++) {
  598. u32 pte = pte_addr[pte_count];
  599. if (rk_pte_is_page_valid(pte))
  600. goto unwind;
  601. pte_addr[pte_count] = rk_mk_pte(paddr, prot);
  602. paddr += SPAGE_SIZE;
  603. }
  604. rk_table_flush(rk_domain, pte_dma, pte_total);
  605. /*
  606. * Zap the first and last iova to evict from iotlb any previously
  607. * mapped cachelines holding stale values for its dte and pte.
  608. * We only zap the first and last iova, since only they could have
  609. * dte or pte shared with an existing mapping.
  610. */
  611. rk_iommu_zap_iova_first_last(rk_domain, iova, size);
  612. return 0;
  613. unwind:
  614. /* Unmap the range of iovas that we just mapped */
  615. rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma,
  616. pte_count * SPAGE_SIZE);
  617. iova += pte_count * SPAGE_SIZE;
  618. page_phys = rk_pte_page_address(pte_addr[pte_count]);
  619. pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
  620. &iova, &page_phys, &paddr, prot);
  621. return -EADDRINUSE;
  622. }
  623. static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
  624. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  625. {
  626. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  627. unsigned long flags;
  628. dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
  629. u32 *page_table, *pte_addr;
  630. u32 dte_index, pte_index;
  631. int ret;
  632. spin_lock_irqsave(&rk_domain->dt_lock, flags);
  633. /*
  634. * pgsize_bitmap specifies iova sizes that fit in one page table
  635. * (1024 4-KiB pages = 4 MiB).
  636. * So, size will always be 4096 <= size <= 4194304.
  637. * Since iommu_map() guarantees that both iova and size will be
  638. * aligned, we will always only be mapping from a single dte here.
  639. */
  640. page_table = rk_dte_get_page_table(rk_domain, iova);
  641. if (IS_ERR(page_table)) {
  642. spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
  643. return PTR_ERR(page_table);
  644. }
  645. dte_index = rk_domain->dt[rk_iova_dte_index(iova)];
  646. pte_index = rk_iova_pte_index(iova);
  647. pte_addr = &page_table[pte_index];
  648. pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32);
  649. ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova,
  650. paddr, size, prot);
  651. spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
  652. return ret;
  653. }
  654. static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
  655. size_t size, struct iommu_iotlb_gather *gather)
  656. {
  657. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  658. unsigned long flags;
  659. dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
  660. phys_addr_t pt_phys;
  661. u32 dte;
  662. u32 *pte_addr;
  663. size_t unmap_size;
  664. spin_lock_irqsave(&rk_domain->dt_lock, flags);
  665. /*
  666. * pgsize_bitmap specifies iova sizes that fit in one page table
  667. * (1024 4-KiB pages = 4 MiB).
  668. * So, size will always be 4096 <= size <= 4194304.
  669. * Since iommu_unmap() guarantees that both iova and size will be
  670. * aligned, we will always only be unmapping from a single dte here.
  671. */
  672. dte = rk_domain->dt[rk_iova_dte_index(iova)];
  673. /* Just return 0 if iova is unmapped */
  674. if (!rk_dte_is_pt_valid(dte)) {
  675. spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
  676. return 0;
  677. }
  678. pt_phys = rk_dte_pt_address(dte);
  679. pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
  680. pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
  681. unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size);
  682. spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
  683. /* Shootdown iotlb entries for iova range that was just unmapped */
  684. rk_iommu_zap_iova(rk_domain, iova, unmap_size);
  685. return unmap_size;
  686. }
  687. static struct rk_iommu *rk_iommu_from_dev(struct device *dev)
  688. {
  689. struct rk_iommudata *data = dev_iommu_priv_get(dev);
  690. return data ? data->iommu : NULL;
  691. }
  692. /* Must be called with iommu powered on and attached */
  693. static void rk_iommu_disable(struct rk_iommu *iommu)
  694. {
  695. int i;
  696. /* Ignore error while disabling, just keep going */
  697. WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks));
  698. rk_iommu_enable_stall(iommu);
  699. rk_iommu_disable_paging(iommu);
  700. for (i = 0; i < iommu->num_mmu; i++) {
  701. rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0);
  702. rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0);
  703. }
  704. rk_iommu_disable_stall(iommu);
  705. clk_bulk_disable(iommu->num_clocks, iommu->clocks);
  706. }
  707. /* Must be called with iommu powered on and attached */
  708. static int rk_iommu_enable(struct rk_iommu *iommu)
  709. {
  710. struct iommu_domain *domain = iommu->domain;
  711. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  712. int ret, i;
  713. ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
  714. if (ret)
  715. return ret;
  716. ret = rk_iommu_enable_stall(iommu);
  717. if (ret)
  718. goto out_disable_clocks;
  719. ret = rk_iommu_force_reset(iommu);
  720. if (ret)
  721. goto out_disable_stall;
  722. for (i = 0; i < iommu->num_mmu; i++) {
  723. rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
  724. rk_domain->dt_dma);
  725. rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
  726. rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
  727. }
  728. ret = rk_iommu_enable_paging(iommu);
  729. out_disable_stall:
  730. rk_iommu_disable_stall(iommu);
  731. out_disable_clocks:
  732. clk_bulk_disable(iommu->num_clocks, iommu->clocks);
  733. return ret;
  734. }
  735. static void rk_iommu_detach_device(struct iommu_domain *domain,
  736. struct device *dev)
  737. {
  738. struct rk_iommu *iommu;
  739. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  740. unsigned long flags;
  741. int ret;
  742. /* Allow 'virtual devices' (eg drm) to detach from domain */
  743. iommu = rk_iommu_from_dev(dev);
  744. if (!iommu)
  745. return;
  746. dev_dbg(dev, "Detaching from iommu domain\n");
  747. /* iommu already detached */
  748. if (iommu->domain != domain)
  749. return;
  750. iommu->domain = NULL;
  751. spin_lock_irqsave(&rk_domain->iommus_lock, flags);
  752. list_del_init(&iommu->node);
  753. spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
  754. ret = pm_runtime_get_if_in_use(iommu->dev);
  755. WARN_ON_ONCE(ret < 0);
  756. if (ret > 0) {
  757. rk_iommu_disable(iommu);
  758. pm_runtime_put(iommu->dev);
  759. }
  760. }
  761. static int rk_iommu_attach_device(struct iommu_domain *domain,
  762. struct device *dev)
  763. {
  764. struct rk_iommu *iommu;
  765. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  766. unsigned long flags;
  767. int ret;
  768. /*
  769. * Allow 'virtual devices' (e.g., drm) to attach to domain.
  770. * Such a device does not belong to an iommu group.
  771. */
  772. iommu = rk_iommu_from_dev(dev);
  773. if (!iommu)
  774. return 0;
  775. dev_dbg(dev, "Attaching to iommu domain\n");
  776. /* iommu already attached */
  777. if (iommu->domain == domain)
  778. return 0;
  779. if (iommu->domain)
  780. rk_iommu_detach_device(iommu->domain, dev);
  781. iommu->domain = domain;
  782. spin_lock_irqsave(&rk_domain->iommus_lock, flags);
  783. list_add_tail(&iommu->node, &rk_domain->iommus);
  784. spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
  785. ret = pm_runtime_get_if_in_use(iommu->dev);
  786. if (!ret || WARN_ON_ONCE(ret < 0))
  787. return 0;
  788. ret = rk_iommu_enable(iommu);
  789. if (ret)
  790. rk_iommu_detach_device(iommu->domain, dev);
  791. pm_runtime_put(iommu->dev);
  792. return ret;
  793. }
  794. static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
  795. {
  796. struct rk_iommu_domain *rk_domain;
  797. if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
  798. return NULL;
  799. if (!dma_dev)
  800. return NULL;
  801. rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL);
  802. if (!rk_domain)
  803. return NULL;
  804. if (type == IOMMU_DOMAIN_DMA &&
  805. iommu_get_dma_cookie(&rk_domain->domain))
  806. goto err_free_domain;
  807. /*
  808. * rk32xx iommus use a 2 level pagetable.
  809. * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
  810. * Allocate one 4 KiB page for each table.
  811. */
  812. rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
  813. if (!rk_domain->dt)
  814. goto err_put_cookie;
  815. rk_domain->dt_dma = dma_map_single(dma_dev, rk_domain->dt,
  816. SPAGE_SIZE, DMA_TO_DEVICE);
  817. if (dma_mapping_error(dma_dev, rk_domain->dt_dma)) {
  818. dev_err(dma_dev, "DMA map error for DT\n");
  819. goto err_free_dt;
  820. }
  821. rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES);
  822. spin_lock_init(&rk_domain->iommus_lock);
  823. spin_lock_init(&rk_domain->dt_lock);
  824. INIT_LIST_HEAD(&rk_domain->iommus);
  825. rk_domain->domain.geometry.aperture_start = 0;
  826. rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
  827. rk_domain->domain.geometry.force_aperture = true;
  828. return &rk_domain->domain;
  829. err_free_dt:
  830. free_page((unsigned long)rk_domain->dt);
  831. err_put_cookie:
  832. if (type == IOMMU_DOMAIN_DMA)
  833. iommu_put_dma_cookie(&rk_domain->domain);
  834. err_free_domain:
  835. kfree(rk_domain);
  836. return NULL;
  837. }
  838. static void rk_iommu_domain_free(struct iommu_domain *domain)
  839. {
  840. struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
  841. int i;
  842. WARN_ON(!list_empty(&rk_domain->iommus));
  843. for (i = 0; i < NUM_DT_ENTRIES; i++) {
  844. u32 dte = rk_domain->dt[i];
  845. if (rk_dte_is_pt_valid(dte)) {
  846. phys_addr_t pt_phys = rk_dte_pt_address(dte);
  847. u32 *page_table = phys_to_virt(pt_phys);
  848. dma_unmap_single(dma_dev, pt_phys,
  849. SPAGE_SIZE, DMA_TO_DEVICE);
  850. free_page((unsigned long)page_table);
  851. }
  852. }
  853. dma_unmap_single(dma_dev, rk_domain->dt_dma,
  854. SPAGE_SIZE, DMA_TO_DEVICE);
  855. free_page((unsigned long)rk_domain->dt);
  856. if (domain->type == IOMMU_DOMAIN_DMA)
  857. iommu_put_dma_cookie(&rk_domain->domain);
  858. kfree(rk_domain);
  859. }
  860. static struct iommu_device *rk_iommu_probe_device(struct device *dev)
  861. {
  862. struct rk_iommudata *data;
  863. struct rk_iommu *iommu;
  864. data = dev_iommu_priv_get(dev);
  865. if (!data)
  866. return ERR_PTR(-ENODEV);
  867. iommu = rk_iommu_from_dev(dev);
  868. data->link = device_link_add(dev, iommu->dev,
  869. DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
  870. return &iommu->iommu;
  871. }
  872. static void rk_iommu_release_device(struct device *dev)
  873. {
  874. struct rk_iommudata *data = dev_iommu_priv_get(dev);
  875. device_link_del(data->link);
  876. }
  877. static struct iommu_group *rk_iommu_device_group(struct device *dev)
  878. {
  879. struct rk_iommu *iommu;
  880. iommu = rk_iommu_from_dev(dev);
  881. return iommu_group_ref_get(iommu->group);
  882. }
  883. static int rk_iommu_of_xlate(struct device *dev,
  884. struct of_phandle_args *args)
  885. {
  886. struct platform_device *iommu_dev;
  887. struct rk_iommudata *data;
  888. data = devm_kzalloc(dma_dev, sizeof(*data), GFP_KERNEL);
  889. if (!data)
  890. return -ENOMEM;
  891. iommu_dev = of_find_device_by_node(args->np);
  892. data->iommu = platform_get_drvdata(iommu_dev);
  893. dev_iommu_priv_set(dev, data);
  894. platform_device_put(iommu_dev);
  895. return 0;
  896. }
  897. static const struct iommu_ops rk_iommu_ops = {
  898. .domain_alloc = rk_iommu_domain_alloc,
  899. .domain_free = rk_iommu_domain_free,
  900. .attach_dev = rk_iommu_attach_device,
  901. .detach_dev = rk_iommu_detach_device,
  902. .map = rk_iommu_map,
  903. .unmap = rk_iommu_unmap,
  904. .probe_device = rk_iommu_probe_device,
  905. .release_device = rk_iommu_release_device,
  906. .iova_to_phys = rk_iommu_iova_to_phys,
  907. .device_group = rk_iommu_device_group,
  908. .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
  909. .of_xlate = rk_iommu_of_xlate,
  910. };
  911. static int rk_iommu_probe(struct platform_device *pdev)
  912. {
  913. struct device *dev = &pdev->dev;
  914. struct rk_iommu *iommu;
  915. struct resource *res;
  916. int num_res = pdev->num_resources;
  917. int err, i;
  918. iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
  919. if (!iommu)
  920. return -ENOMEM;
  921. platform_set_drvdata(pdev, iommu);
  922. iommu->dev = dev;
  923. iommu->num_mmu = 0;
  924. iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),
  925. GFP_KERNEL);
  926. if (!iommu->bases)
  927. return -ENOMEM;
  928. for (i = 0; i < num_res; i++) {
  929. res = platform_get_resource(pdev, IORESOURCE_MEM, i);
  930. if (!res)
  931. continue;
  932. iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res);
  933. if (IS_ERR(iommu->bases[i]))
  934. continue;
  935. iommu->num_mmu++;
  936. }
  937. if (iommu->num_mmu == 0)
  938. return PTR_ERR(iommu->bases[0]);
  939. iommu->num_irq = platform_irq_count(pdev);
  940. if (iommu->num_irq < 0)
  941. return iommu->num_irq;
  942. iommu->reset_disabled = device_property_read_bool(dev,
  943. "rockchip,disable-mmu-reset");
  944. iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks);
  945. iommu->clocks = devm_kcalloc(iommu->dev, iommu->num_clocks,
  946. sizeof(*iommu->clocks), GFP_KERNEL);
  947. if (!iommu->clocks)
  948. return -ENOMEM;
  949. for (i = 0; i < iommu->num_clocks; ++i)
  950. iommu->clocks[i].id = rk_iommu_clocks[i];
  951. /*
  952. * iommu clocks should be present for all new devices and devicetrees
  953. * but there are older devicetrees without clocks out in the wild.
  954. * So clocks as optional for the time being.
  955. */
  956. err = devm_clk_bulk_get(iommu->dev, iommu->num_clocks, iommu->clocks);
  957. if (err == -ENOENT)
  958. iommu->num_clocks = 0;
  959. else if (err)
  960. return err;
  961. err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
  962. if (err)
  963. return err;
  964. iommu->group = iommu_group_alloc();
  965. if (IS_ERR(iommu->group)) {
  966. err = PTR_ERR(iommu->group);
  967. goto err_unprepare_clocks;
  968. }
  969. err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
  970. if (err)
  971. goto err_put_group;
  972. iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops);
  973. iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode);
  974. err = iommu_device_register(&iommu->iommu);
  975. if (err)
  976. goto err_remove_sysfs;
  977. /*
  978. * Use the first registered IOMMU device for domain to use with DMA
  979. * API, since a domain might not physically correspond to a single
  980. * IOMMU device..
  981. */
  982. if (!dma_dev)
  983. dma_dev = &pdev->dev;
  984. bus_set_iommu(&platform_bus_type, &rk_iommu_ops);
  985. pm_runtime_enable(dev);
  986. for (i = 0; i < iommu->num_irq; i++) {
  987. int irq = platform_get_irq(pdev, i);
  988. if (irq < 0)
  989. return irq;
  990. err = devm_request_irq(iommu->dev, irq, rk_iommu_irq,
  991. IRQF_SHARED, dev_name(dev), iommu);
  992. if (err) {
  993. pm_runtime_disable(dev);
  994. goto err_remove_sysfs;
  995. }
  996. }
  997. return 0;
  998. err_remove_sysfs:
  999. iommu_device_sysfs_remove(&iommu->iommu);
  1000. err_put_group:
  1001. iommu_group_put(iommu->group);
  1002. err_unprepare_clocks:
  1003. clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
  1004. return err;
  1005. }
  1006. static void rk_iommu_shutdown(struct platform_device *pdev)
  1007. {
  1008. struct rk_iommu *iommu = platform_get_drvdata(pdev);
  1009. int i;
  1010. for (i = 0; i < iommu->num_irq; i++) {
  1011. int irq = platform_get_irq(pdev, i);
  1012. devm_free_irq(iommu->dev, irq, iommu);
  1013. }
  1014. pm_runtime_force_suspend(&pdev->dev);
  1015. }
  1016. static int __maybe_unused rk_iommu_suspend(struct device *dev)
  1017. {
  1018. struct rk_iommu *iommu = dev_get_drvdata(dev);
  1019. if (!iommu->domain)
  1020. return 0;
  1021. rk_iommu_disable(iommu);
  1022. return 0;
  1023. }
  1024. static int __maybe_unused rk_iommu_resume(struct device *dev)
  1025. {
  1026. struct rk_iommu *iommu = dev_get_drvdata(dev);
  1027. if (!iommu->domain)
  1028. return 0;
  1029. return rk_iommu_enable(iommu);
  1030. }
  1031. static const struct dev_pm_ops rk_iommu_pm_ops = {
  1032. SET_RUNTIME_PM_OPS(rk_iommu_suspend, rk_iommu_resume, NULL)
  1033. SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
  1034. pm_runtime_force_resume)
  1035. };
  1036. static const struct of_device_id rk_iommu_dt_ids[] = {
  1037. { .compatible = "rockchip,iommu" },
  1038. { /* sentinel */ }
  1039. };
  1040. static struct platform_driver rk_iommu_driver = {
  1041. .probe = rk_iommu_probe,
  1042. .shutdown = rk_iommu_shutdown,
  1043. .driver = {
  1044. .name = "rk_iommu",
  1045. .of_match_table = rk_iommu_dt_ids,
  1046. .pm = &rk_iommu_pm_ops,
  1047. .suppress_bind_attrs = true,
  1048. },
  1049. };
  1050. static int __init rk_iommu_init(void)
  1051. {
  1052. return platform_driver_register(&rk_iommu_driver);
  1053. }
  1054. subsys_initcall(rk_iommu_init);