sun50i-iommu.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020
  1. // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2. // Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
  3. // Copyright (C) 2019-2020, Cerno
  4. #include <linux/bitfield.h>
  5. #include <linux/bug.h>
  6. #include <linux/clk.h>
  7. #include <linux/device.h>
  8. #include <linux/dma-direction.h>
  9. #include <linux/dma-iommu.h>
  10. #include <linux/dma-mapping.h>
  11. #include <linux/err.h>
  12. #include <linux/errno.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/iommu.h>
  15. #include <linux/iopoll.h>
  16. #include <linux/ioport.h>
  17. #include <linux/log2.h>
  18. #include <linux/module.h>
  19. #include <linux/of_platform.h>
  20. #include <linux/platform_device.h>
  21. #include <linux/pm.h>
  22. #include <linux/pm_runtime.h>
  23. #include <linux/reset.h>
  24. #include <linux/sizes.h>
  25. #include <linux/slab.h>
  26. #include <linux/spinlock.h>
  27. #include <linux/types.h>
  28. #define IOMMU_RESET_REG 0x010
  29. #define IOMMU_ENABLE_REG 0x020
  30. #define IOMMU_ENABLE_ENABLE BIT(0)
  31. #define IOMMU_BYPASS_REG 0x030
  32. #define IOMMU_AUTO_GATING_REG 0x040
  33. #define IOMMU_AUTO_GATING_ENABLE BIT(0)
  34. #define IOMMU_WBUF_CTRL_REG 0x044
  35. #define IOMMU_OOO_CTRL_REG 0x048
  36. #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c
  37. #define IOMMU_TTB_REG 0x050
  38. #define IOMMU_TLB_ENABLE_REG 0x060
  39. #define IOMMU_TLB_PREFETCH_REG 0x070
  40. #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m)
  41. #define IOMMU_TLB_FLUSH_REG 0x080
  42. #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17)
  43. #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16)
  44. #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0))
  45. #define IOMMU_TLB_IVLD_ADDR_REG 0x090
  46. #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094
  47. #define IOMMU_TLB_IVLD_ENABLE_REG 0x098
  48. #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0)
  49. #define IOMMU_PC_IVLD_ADDR_REG 0x0a0
  50. #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8
  51. #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0)
  52. #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4)
  53. #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2)))
  54. #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1))
  55. #define IOMMU_DM_AUT_OVWT_REG 0x0d0
  56. #define IOMMU_INT_ENABLE_REG 0x100
  57. #define IOMMU_INT_CLR_REG 0x104
  58. #define IOMMU_INT_STA_REG 0x108
  59. #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4)
  60. #define IOMMU_INT_ERR_ADDR_L1_REG 0x130
  61. #define IOMMU_INT_ERR_ADDR_L2_REG 0x134
  62. #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4)
  63. #define IOMMU_L1PG_INT_REG 0x0180
  64. #define IOMMU_L2PG_INT_REG 0x0184
  65. #define IOMMU_INT_INVALID_L2PG BIT(17)
  66. #define IOMMU_INT_INVALID_L1PG BIT(16)
  67. #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m)
  68. #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \
  69. IOMMU_INT_MASTER_PERMISSION(1) | \
  70. IOMMU_INT_MASTER_PERMISSION(2) | \
  71. IOMMU_INT_MASTER_PERMISSION(3) | \
  72. IOMMU_INT_MASTER_PERMISSION(4) | \
  73. IOMMU_INT_MASTER_PERMISSION(5))
  74. #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \
  75. IOMMU_INT_INVALID_L2PG | \
  76. IOMMU_INT_MASTER_MASK)
  77. #define PT_ENTRY_SIZE sizeof(u32)
  78. #define NUM_DT_ENTRIES 4096
  79. #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
  80. #define NUM_PT_ENTRIES 256
  81. #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
  82. struct sun50i_iommu {
  83. struct iommu_device iommu;
  84. /* Lock to modify the IOMMU registers */
  85. spinlock_t iommu_lock;
  86. struct device *dev;
  87. void __iomem *base;
  88. struct reset_control *reset;
  89. struct clk *clk;
  90. struct iommu_domain *domain;
  91. struct iommu_group *group;
  92. struct kmem_cache *pt_pool;
  93. };
  94. struct sun50i_iommu_domain {
  95. struct iommu_domain domain;
  96. /* Number of devices attached to the domain */
  97. refcount_t refcnt;
  98. /* L1 Page Table */
  99. u32 *dt;
  100. dma_addr_t dt_dma;
  101. struct sun50i_iommu *iommu;
  102. };
  103. static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
  104. {
  105. return container_of(domain, struct sun50i_iommu_domain, domain);
  106. }
  107. static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
  108. {
  109. return dev_iommu_priv_get(dev);
  110. }
  111. static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
  112. {
  113. return readl(iommu->base + offset);
  114. }
  115. static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
  116. {
  117. writel(value, iommu->base + offset);
  118. }
  119. /*
  120. * The Allwinner H6 IOMMU uses a 2-level page table.
  121. *
  122. * The first level is the usual Directory Table (DT), that consists of
  123. * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
  124. * Table (PT).
  125. *
  126. * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
  127. * pointing to a 4kB page of physical memory.
  128. *
  129. * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
  130. * register that contains its physical address.
  131. */
  132. #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20)
  133. #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12)
  134. #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0)
  135. static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
  136. {
  137. return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
  138. }
  139. static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
  140. {
  141. return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
  142. }
  143. static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
  144. {
  145. return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
  146. }
  147. /*
  148. * Each Directory Table Entry has a Page Table address and a valid
  149. * bit:
  150. * +---------------------+-----------+-+
  151. * | PT address | Reserved |V|
  152. * +---------------------+-----------+-+
  153. * 31:10 - Page Table address
  154. * 9:2 - Reserved
  155. * 1:0 - 1 if the entry is valid
  156. */
  157. #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10)
  158. #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0)
  159. #define SUN50I_DTE_PT_VALID 1
  160. static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
  161. {
  162. return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
  163. }
  164. static bool sun50i_dte_is_pt_valid(u32 dte)
  165. {
  166. return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
  167. }
  168. static u32 sun50i_mk_dte(dma_addr_t pt_dma)
  169. {
  170. return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
  171. }
  172. /*
  173. * Each PTE has a Page address, an authority index and a valid bit:
  174. *
  175. * +----------------+-----+-----+-----+---+-----+
  176. * | Page address | Rsv | ACI | Rsv | V | Rsv |
  177. * +----------------+-----+-----+-----+---+-----+
  178. * 31:12 - Page address
  179. * 11:8 - Reserved
  180. * 7:4 - Authority Control Index
  181. * 3:2 - Reserved
  182. * 1 - 1 if the entry is valid
  183. * 0 - Reserved
  184. *
  185. * The way permissions work is that the IOMMU has 16 "domains" that
  186. * can be configured to give each masters either read or write
  187. * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
  188. * 0 seems like the default domain, and its permissions in the
  189. * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
  190. * useful to enforce any particular permission.
  191. *
  192. * Each page entry will then have a reference to the domain they are
  193. * affected to, so that we can actually enforce them on a per-page
  194. * basis.
  195. *
  196. * In order to make it work with the IOMMU framework, we will be using
  197. * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
  198. * depending on the permission we want to enforce. Each domain will
  199. * have each master setup in the same way, since the IOMMU framework
  200. * doesn't seem to restrict page access on a per-device basis. And
  201. * then we will use the relevant domain index when generating the page
  202. * table entry depending on the permissions we want to be enforced.
  203. */
  204. enum sun50i_iommu_aci {
  205. SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
  206. SUN50I_IOMMU_ACI_NONE,
  207. SUN50I_IOMMU_ACI_RD,
  208. SUN50I_IOMMU_ACI_WR,
  209. SUN50I_IOMMU_ACI_RD_WR,
  210. };
  211. #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12)
  212. #define SUN50I_PTE_ACI_MASK GENMASK(7, 4)
  213. #define SUN50I_PTE_PAGE_VALID BIT(1)
  214. static phys_addr_t sun50i_pte_get_page_address(u32 pte)
  215. {
  216. return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
  217. }
  218. static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
  219. {
  220. return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
  221. }
  222. static bool sun50i_pte_is_page_valid(u32 pte)
  223. {
  224. return pte & SUN50I_PTE_PAGE_VALID;
  225. }
  226. static u32 sun50i_mk_pte(phys_addr_t page, int prot)
  227. {
  228. enum sun50i_iommu_aci aci;
  229. u32 flags = 0;
  230. if (prot & (IOMMU_READ | IOMMU_WRITE))
  231. aci = SUN50I_IOMMU_ACI_RD_WR;
  232. else if (prot & IOMMU_READ)
  233. aci = SUN50I_IOMMU_ACI_RD;
  234. else if (prot & IOMMU_WRITE)
  235. aci = SUN50I_IOMMU_ACI_WR;
  236. else
  237. aci = SUN50I_IOMMU_ACI_NONE;
  238. flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
  239. page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
  240. return page | flags | SUN50I_PTE_PAGE_VALID;
  241. }
  242. static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
  243. void *vaddr, unsigned int count)
  244. {
  245. struct sun50i_iommu *iommu = sun50i_domain->iommu;
  246. dma_addr_t dma = virt_to_phys(vaddr);
  247. size_t size = count * PT_ENTRY_SIZE;
  248. dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
  249. }
  250. static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
  251. {
  252. u32 reg;
  253. int ret;
  254. assert_spin_locked(&iommu->iommu_lock);
  255. iommu_write(iommu,
  256. IOMMU_TLB_FLUSH_REG,
  257. IOMMU_TLB_FLUSH_PTW_CACHE |
  258. IOMMU_TLB_FLUSH_MACRO_TLB |
  259. IOMMU_TLB_FLUSH_MICRO_TLB(5) |
  260. IOMMU_TLB_FLUSH_MICRO_TLB(4) |
  261. IOMMU_TLB_FLUSH_MICRO_TLB(3) |
  262. IOMMU_TLB_FLUSH_MICRO_TLB(2) |
  263. IOMMU_TLB_FLUSH_MICRO_TLB(1) |
  264. IOMMU_TLB_FLUSH_MICRO_TLB(0));
  265. ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
  266. reg, !reg,
  267. 1, 2000);
  268. if (ret)
  269. dev_warn(iommu->dev, "TLB Flush timed out!\n");
  270. return ret;
  271. }
  272. static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
  273. {
  274. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  275. struct sun50i_iommu *iommu = sun50i_domain->iommu;
  276. unsigned long flags;
  277. /*
  278. * At boot, we'll have a first call into .flush_iotlb_all right after
  279. * .probe_device, and since we link our (single) domain to our iommu in
  280. * the .attach_device callback, we don't have that pointer set.
  281. *
  282. * It shouldn't really be any trouble to ignore it though since we flush
  283. * all caches as part of the device powerup.
  284. */
  285. if (!iommu)
  286. return;
  287. spin_lock_irqsave(&iommu->iommu_lock, flags);
  288. sun50i_iommu_flush_all_tlb(iommu);
  289. spin_unlock_irqrestore(&iommu->iommu_lock, flags);
  290. }
  291. static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
  292. struct iommu_iotlb_gather *gather)
  293. {
  294. sun50i_iommu_flush_iotlb_all(domain);
  295. }
  296. static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
  297. {
  298. struct sun50i_iommu_domain *sun50i_domain;
  299. unsigned long flags;
  300. int ret;
  301. if (!iommu->domain)
  302. return 0;
  303. sun50i_domain = to_sun50i_domain(iommu->domain);
  304. ret = reset_control_deassert(iommu->reset);
  305. if (ret)
  306. return ret;
  307. ret = clk_prepare_enable(iommu->clk);
  308. if (ret)
  309. goto err_reset_assert;
  310. spin_lock_irqsave(&iommu->iommu_lock, flags);
  311. iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
  312. iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
  313. IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
  314. IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
  315. IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
  316. IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
  317. IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
  318. IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
  319. iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
  320. iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
  321. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
  322. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
  323. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
  324. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
  325. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
  326. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
  327. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
  328. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
  329. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
  330. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
  331. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
  332. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
  333. iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
  334. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
  335. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
  336. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
  337. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
  338. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
  339. IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
  340. iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
  341. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
  342. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
  343. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
  344. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
  345. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
  346. IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
  347. ret = sun50i_iommu_flush_all_tlb(iommu);
  348. if (ret) {
  349. spin_unlock_irqrestore(&iommu->iommu_lock, flags);
  350. goto err_clk_disable;
  351. }
  352. iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
  353. iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
  354. spin_unlock_irqrestore(&iommu->iommu_lock, flags);
  355. return 0;
  356. err_clk_disable:
  357. clk_disable_unprepare(iommu->clk);
  358. err_reset_assert:
  359. reset_control_assert(iommu->reset);
  360. return ret;
  361. }
  362. static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
  363. {
  364. unsigned long flags;
  365. spin_lock_irqsave(&iommu->iommu_lock, flags);
  366. iommu_write(iommu, IOMMU_ENABLE_REG, 0);
  367. iommu_write(iommu, IOMMU_TTB_REG, 0);
  368. spin_unlock_irqrestore(&iommu->iommu_lock, flags);
  369. clk_disable_unprepare(iommu->clk);
  370. reset_control_assert(iommu->reset);
  371. }
  372. static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
  373. gfp_t gfp)
  374. {
  375. dma_addr_t pt_dma;
  376. u32 *page_table;
  377. page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
  378. if (!page_table)
  379. return ERR_PTR(-ENOMEM);
  380. pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
  381. if (dma_mapping_error(iommu->dev, pt_dma)) {
  382. dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
  383. kmem_cache_free(iommu->pt_pool, page_table);
  384. return ERR_PTR(-ENOMEM);
  385. }
  386. /* We rely on the physical address and DMA address being the same */
  387. WARN_ON(pt_dma != virt_to_phys(page_table));
  388. return page_table;
  389. }
  390. static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
  391. u32 *page_table)
  392. {
  393. phys_addr_t pt_phys = virt_to_phys(page_table);
  394. dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
  395. kmem_cache_free(iommu->pt_pool, page_table);
  396. }
  397. static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
  398. dma_addr_t iova, gfp_t gfp)
  399. {
  400. struct sun50i_iommu *iommu = sun50i_domain->iommu;
  401. u32 *page_table;
  402. u32 *dte_addr;
  403. u32 old_dte;
  404. u32 dte;
  405. dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
  406. dte = *dte_addr;
  407. if (sun50i_dte_is_pt_valid(dte)) {
  408. phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
  409. return (u32 *)phys_to_virt(pt_phys);
  410. }
  411. page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
  412. if (IS_ERR(page_table))
  413. return page_table;
  414. dte = sun50i_mk_dte(virt_to_phys(page_table));
  415. old_dte = cmpxchg(dte_addr, 0, dte);
  416. if (old_dte) {
  417. phys_addr_t installed_pt_phys =
  418. sun50i_dte_get_pt_address(old_dte);
  419. u32 *installed_pt = phys_to_virt(installed_pt_phys);
  420. u32 *drop_pt = page_table;
  421. page_table = installed_pt;
  422. dte = old_dte;
  423. sun50i_iommu_free_page_table(iommu, drop_pt);
  424. }
  425. sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
  426. sun50i_table_flush(sun50i_domain, dte_addr, 1);
  427. return page_table;
  428. }
  429. static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
  430. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  431. {
  432. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  433. struct sun50i_iommu *iommu = sun50i_domain->iommu;
  434. u32 pte_index;
  435. u32 *page_table, *pte_addr;
  436. int ret = 0;
  437. page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
  438. if (IS_ERR(page_table)) {
  439. ret = PTR_ERR(page_table);
  440. goto out;
  441. }
  442. pte_index = sun50i_iova_get_pte_index(iova);
  443. pte_addr = &page_table[pte_index];
  444. if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
  445. phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
  446. dev_err(iommu->dev,
  447. "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
  448. &iova, &page_phys, &paddr, prot);
  449. ret = -EBUSY;
  450. goto out;
  451. }
  452. *pte_addr = sun50i_mk_pte(paddr, prot);
  453. sun50i_table_flush(sun50i_domain, pte_addr, 1);
  454. out:
  455. return ret;
  456. }
  457. static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
  458. size_t size, struct iommu_iotlb_gather *gather)
  459. {
  460. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  461. phys_addr_t pt_phys;
  462. u32 *pte_addr;
  463. u32 dte;
  464. dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
  465. if (!sun50i_dte_is_pt_valid(dte))
  466. return 0;
  467. pt_phys = sun50i_dte_get_pt_address(dte);
  468. pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
  469. if (!sun50i_pte_is_page_valid(*pte_addr))
  470. return 0;
  471. memset(pte_addr, 0, sizeof(*pte_addr));
  472. sun50i_table_flush(sun50i_domain, pte_addr, 1);
  473. return SZ_4K;
  474. }
  475. static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
  476. dma_addr_t iova)
  477. {
  478. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  479. phys_addr_t pt_phys;
  480. u32 *page_table;
  481. u32 dte, pte;
  482. dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
  483. if (!sun50i_dte_is_pt_valid(dte))
  484. return 0;
  485. pt_phys = sun50i_dte_get_pt_address(dte);
  486. page_table = (u32 *)phys_to_virt(pt_phys);
  487. pte = page_table[sun50i_iova_get_pte_index(iova)];
  488. if (!sun50i_pte_is_page_valid(pte))
  489. return 0;
  490. return sun50i_pte_get_page_address(pte) +
  491. sun50i_iova_get_page_offset(iova);
  492. }
  493. static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
  494. {
  495. struct sun50i_iommu_domain *sun50i_domain;
  496. if (type != IOMMU_DOMAIN_DMA &&
  497. type != IOMMU_DOMAIN_IDENTITY &&
  498. type != IOMMU_DOMAIN_UNMANAGED)
  499. return NULL;
  500. sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
  501. if (!sun50i_domain)
  502. return NULL;
  503. if (type == IOMMU_DOMAIN_DMA &&
  504. iommu_get_dma_cookie(&sun50i_domain->domain))
  505. goto err_free_domain;
  506. sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
  507. get_order(DT_SIZE));
  508. if (!sun50i_domain->dt)
  509. goto err_put_cookie;
  510. refcount_set(&sun50i_domain->refcnt, 1);
  511. sun50i_domain->domain.geometry.aperture_start = 0;
  512. sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
  513. sun50i_domain->domain.geometry.force_aperture = true;
  514. return &sun50i_domain->domain;
  515. err_put_cookie:
  516. if (type == IOMMU_DOMAIN_DMA)
  517. iommu_put_dma_cookie(&sun50i_domain->domain);
  518. err_free_domain:
  519. kfree(sun50i_domain);
  520. return NULL;
  521. }
  522. static void sun50i_iommu_domain_free(struct iommu_domain *domain)
  523. {
  524. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  525. free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
  526. sun50i_domain->dt = NULL;
  527. iommu_put_dma_cookie(domain);
  528. kfree(sun50i_domain);
  529. }
  530. static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
  531. struct sun50i_iommu_domain *sun50i_domain)
  532. {
  533. iommu->domain = &sun50i_domain->domain;
  534. sun50i_domain->iommu = iommu;
  535. sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
  536. DT_SIZE, DMA_TO_DEVICE);
  537. if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
  538. dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
  539. return -ENOMEM;
  540. }
  541. return sun50i_iommu_enable(iommu);
  542. }
  543. static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
  544. struct sun50i_iommu_domain *sun50i_domain)
  545. {
  546. unsigned int i;
  547. for (i = 0; i < NUM_DT_ENTRIES; i++) {
  548. phys_addr_t pt_phys;
  549. u32 *page_table;
  550. u32 *dte_addr;
  551. u32 dte;
  552. dte_addr = &sun50i_domain->dt[i];
  553. dte = *dte_addr;
  554. if (!sun50i_dte_is_pt_valid(dte))
  555. continue;
  556. memset(dte_addr, 0, sizeof(*dte_addr));
  557. sun50i_table_flush(sun50i_domain, dte_addr, 1);
  558. pt_phys = sun50i_dte_get_pt_address(dte);
  559. page_table = phys_to_virt(pt_phys);
  560. sun50i_iommu_free_page_table(iommu, page_table);
  561. }
  562. sun50i_iommu_disable(iommu);
  563. dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
  564. DT_SIZE, DMA_TO_DEVICE);
  565. iommu->domain = NULL;
  566. }
  567. static void sun50i_iommu_detach_device(struct iommu_domain *domain,
  568. struct device *dev)
  569. {
  570. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  571. struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
  572. dev_dbg(dev, "Detaching from IOMMU domain\n");
  573. if (iommu->domain != domain)
  574. return;
  575. if (refcount_dec_and_test(&sun50i_domain->refcnt))
  576. sun50i_iommu_detach_domain(iommu, sun50i_domain);
  577. }
  578. static int sun50i_iommu_attach_device(struct iommu_domain *domain,
  579. struct device *dev)
  580. {
  581. struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
  582. struct sun50i_iommu *iommu;
  583. iommu = sun50i_iommu_from_dev(dev);
  584. if (!iommu)
  585. return -ENODEV;
  586. dev_dbg(dev, "Attaching to IOMMU domain\n");
  587. refcount_inc(&sun50i_domain->refcnt);
  588. if (iommu->domain == domain)
  589. return 0;
  590. if (iommu->domain)
  591. sun50i_iommu_detach_device(iommu->domain, dev);
  592. sun50i_iommu_attach_domain(iommu, sun50i_domain);
  593. return 0;
  594. }
  595. static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
  596. {
  597. struct sun50i_iommu *iommu;
  598. iommu = sun50i_iommu_from_dev(dev);
  599. if (!iommu)
  600. return ERR_PTR(-ENODEV);
  601. return &iommu->iommu;
  602. }
  603. static void sun50i_iommu_release_device(struct device *dev) {}
  604. static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
  605. {
  606. struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
  607. return iommu_group_ref_get(iommu->group);
  608. }
  609. static int sun50i_iommu_of_xlate(struct device *dev,
  610. struct of_phandle_args *args)
  611. {
  612. struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
  613. unsigned id = args->args[0];
  614. dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
  615. return iommu_fwspec_add_ids(dev, &id, 1);
  616. }
  617. static const struct iommu_ops sun50i_iommu_ops = {
  618. .pgsize_bitmap = SZ_4K,
  619. .attach_dev = sun50i_iommu_attach_device,
  620. .detach_dev = sun50i_iommu_detach_device,
  621. .device_group = sun50i_iommu_device_group,
  622. .domain_alloc = sun50i_iommu_domain_alloc,
  623. .domain_free = sun50i_iommu_domain_free,
  624. .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
  625. .iotlb_sync = sun50i_iommu_iotlb_sync,
  626. .iova_to_phys = sun50i_iommu_iova_to_phys,
  627. .map = sun50i_iommu_map,
  628. .of_xlate = sun50i_iommu_of_xlate,
  629. .probe_device = sun50i_iommu_probe_device,
  630. .release_device = sun50i_iommu_release_device,
  631. .unmap = sun50i_iommu_unmap,
  632. };
  633. static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
  634. unsigned master, phys_addr_t iova,
  635. unsigned prot)
  636. {
  637. dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
  638. &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
  639. if (iommu->domain)
  640. report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
  641. else
  642. dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
  643. }
  644. static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
  645. unsigned addr_reg,
  646. unsigned blame_reg)
  647. {
  648. phys_addr_t iova;
  649. unsigned master;
  650. u32 blame;
  651. assert_spin_locked(&iommu->iommu_lock);
  652. iova = iommu_read(iommu, addr_reg);
  653. blame = iommu_read(iommu, blame_reg);
  654. master = ilog2(blame & IOMMU_INT_MASTER_MASK);
  655. /*
  656. * If the address is not in the page table, we can't get what
  657. * operation triggered the fault. Assume it's a read
  658. * operation.
  659. */
  660. sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
  661. return iova;
  662. }
  663. static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
  664. {
  665. enum sun50i_iommu_aci aci;
  666. phys_addr_t iova;
  667. unsigned master;
  668. unsigned dir;
  669. u32 blame;
  670. assert_spin_locked(&iommu->iommu_lock);
  671. blame = iommu_read(iommu, IOMMU_INT_STA_REG);
  672. master = ilog2(blame & IOMMU_INT_MASTER_MASK);
  673. iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
  674. aci = sun50i_get_pte_aci(iommu_read(iommu,
  675. IOMMU_INT_ERR_DATA_REG(master)));
  676. switch (aci) {
  677. /*
  678. * If we are in the read-only domain, then it means we
  679. * tried to write.
  680. */
  681. case SUN50I_IOMMU_ACI_RD:
  682. dir = IOMMU_FAULT_WRITE;
  683. break;
  684. /*
  685. * If we are in the write-only domain, then it means
  686. * we tried to read.
  687. */
  688. case SUN50I_IOMMU_ACI_WR:
  689. /*
  690. * If we are in the domain without any permission, we
  691. * can't really tell. Let's default to a read
  692. * operation.
  693. */
  694. case SUN50I_IOMMU_ACI_NONE:
  695. /* WTF? */
  696. case SUN50I_IOMMU_ACI_RD_WR:
  697. default:
  698. dir = IOMMU_FAULT_READ;
  699. break;
  700. }
  701. /*
  702. * If the address is not in the page table, we can't get what
  703. * operation triggered the fault. Assume it's a read
  704. * operation.
  705. */
  706. sun50i_iommu_report_fault(iommu, master, iova, dir);
  707. return iova;
  708. }
  709. static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
  710. {
  711. struct sun50i_iommu *iommu = dev_id;
  712. u32 status;
  713. spin_lock(&iommu->iommu_lock);
  714. status = iommu_read(iommu, IOMMU_INT_STA_REG);
  715. if (!(status & IOMMU_INT_MASK)) {
  716. spin_unlock(&iommu->iommu_lock);
  717. return IRQ_NONE;
  718. }
  719. if (status & IOMMU_INT_INVALID_L2PG)
  720. sun50i_iommu_handle_pt_irq(iommu,
  721. IOMMU_INT_ERR_ADDR_L2_REG,
  722. IOMMU_L2PG_INT_REG);
  723. else if (status & IOMMU_INT_INVALID_L1PG)
  724. sun50i_iommu_handle_pt_irq(iommu,
  725. IOMMU_INT_ERR_ADDR_L1_REG,
  726. IOMMU_L1PG_INT_REG);
  727. else
  728. sun50i_iommu_handle_perm_irq(iommu);
  729. iommu_write(iommu, IOMMU_INT_CLR_REG, status);
  730. iommu_write(iommu, IOMMU_RESET_REG, ~status);
  731. iommu_write(iommu, IOMMU_RESET_REG, status);
  732. spin_unlock(&iommu->iommu_lock);
  733. return IRQ_HANDLED;
  734. }
  735. static int sun50i_iommu_probe(struct platform_device *pdev)
  736. {
  737. struct sun50i_iommu *iommu;
  738. int ret, irq;
  739. iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
  740. if (!iommu)
  741. return -ENOMEM;
  742. spin_lock_init(&iommu->iommu_lock);
  743. platform_set_drvdata(pdev, iommu);
  744. iommu->dev = &pdev->dev;
  745. iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
  746. PT_SIZE, PT_SIZE,
  747. SLAB_HWCACHE_ALIGN,
  748. NULL);
  749. if (!iommu->pt_pool)
  750. return -ENOMEM;
  751. iommu->group = iommu_group_alloc();
  752. if (IS_ERR(iommu->group)) {
  753. ret = PTR_ERR(iommu->group);
  754. goto err_free_cache;
  755. }
  756. iommu->base = devm_platform_ioremap_resource(pdev, 0);
  757. if (IS_ERR(iommu->base)) {
  758. ret = PTR_ERR(iommu->base);
  759. goto err_free_group;
  760. }
  761. irq = platform_get_irq(pdev, 0);
  762. if (irq < 0) {
  763. ret = irq;
  764. goto err_free_group;
  765. }
  766. iommu->clk = devm_clk_get(&pdev->dev, NULL);
  767. if (IS_ERR(iommu->clk)) {
  768. dev_err(&pdev->dev, "Couldn't get our clock.\n");
  769. ret = PTR_ERR(iommu->clk);
  770. goto err_free_group;
  771. }
  772. iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
  773. if (IS_ERR(iommu->reset)) {
  774. dev_err(&pdev->dev, "Couldn't get our reset line.\n");
  775. ret = PTR_ERR(iommu->reset);
  776. goto err_free_group;
  777. }
  778. ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
  779. NULL, dev_name(&pdev->dev));
  780. if (ret)
  781. goto err_free_group;
  782. iommu_device_set_ops(&iommu->iommu, &sun50i_iommu_ops);
  783. iommu_device_set_fwnode(&iommu->iommu, &pdev->dev.of_node->fwnode);
  784. ret = iommu_device_register(&iommu->iommu);
  785. if (ret)
  786. goto err_remove_sysfs;
  787. ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
  788. dev_name(&pdev->dev), iommu);
  789. if (ret < 0)
  790. goto err_unregister;
  791. bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops);
  792. return 0;
  793. err_unregister:
  794. iommu_device_unregister(&iommu->iommu);
  795. err_remove_sysfs:
  796. iommu_device_sysfs_remove(&iommu->iommu);
  797. err_free_group:
  798. iommu_group_put(iommu->group);
  799. err_free_cache:
  800. kmem_cache_destroy(iommu->pt_pool);
  801. return ret;
  802. }
  803. static const struct of_device_id sun50i_iommu_dt[] = {
  804. { .compatible = "allwinner,sun50i-h6-iommu", },
  805. { /* sentinel */ },
  806. };
  807. MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
  808. static struct platform_driver sun50i_iommu_driver = {
  809. .driver = {
  810. .name = "sun50i-iommu",
  811. .of_match_table = sun50i_iommu_dt,
  812. .suppress_bind_attrs = true,
  813. }
  814. };
  815. builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
  816. MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
  817. MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
  818. MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
  819. MODULE_LICENSE("Dual BSD/GPL");