ipmmu-vmsa.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * IOMMU API for Renesas VMSA-compatible IPMMU
  4. * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
  5. *
  6. * Copyright (C) 2014-2020 Renesas Electronics Corporation
  7. */
  8. #include <linux/bitmap.h>
  9. #include <linux/delay.h>
  10. #include <linux/dma-iommu.h>
  11. #include <linux/dma-mapping.h>
  12. #include <linux/err.h>
  13. #include <linux/export.h>
  14. #include <linux/init.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/io.h>
  17. #include <linux/io-pgtable.h>
  18. #include <linux/iommu.h>
  19. #include <linux/of.h>
  20. #include <linux/of_device.h>
  21. #include <linux/of_iommu.h>
  22. #include <linux/of_platform.h>
  23. #include <linux/platform_device.h>
  24. #include <linux/sizes.h>
  25. #include <linux/slab.h>
  26. #include <linux/sys_soc.h>
  27. #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
  28. #include <asm/dma-iommu.h>
  29. #else
  30. #define arm_iommu_create_mapping(...) NULL
  31. #define arm_iommu_attach_device(...) -ENODEV
  32. #define arm_iommu_release_mapping(...) do {} while (0)
  33. #define arm_iommu_detach_device(...) do {} while (0)
  34. #endif
  35. #define IPMMU_CTX_MAX 8U
  36. #define IPMMU_CTX_INVALID -1
  37. #define IPMMU_UTLB_MAX 48U
  38. struct ipmmu_features {
  39. bool use_ns_alias_offset;
  40. bool has_cache_leaf_nodes;
  41. unsigned int number_of_contexts;
  42. unsigned int num_utlbs;
  43. bool setup_imbuscr;
  44. bool twobit_imttbcr_sl0;
  45. bool reserved_context;
  46. bool cache_snoop;
  47. unsigned int ctx_offset_base;
  48. unsigned int ctx_offset_stride;
  49. unsigned int utlb_offset_base;
  50. };
  51. struct ipmmu_vmsa_device {
  52. struct device *dev;
  53. void __iomem *base;
  54. struct iommu_device iommu;
  55. struct ipmmu_vmsa_device *root;
  56. const struct ipmmu_features *features;
  57. unsigned int num_ctx;
  58. spinlock_t lock; /* Protects ctx and domains[] */
  59. DECLARE_BITMAP(ctx, IPMMU_CTX_MAX);
  60. struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX];
  61. s8 utlb_ctx[IPMMU_UTLB_MAX];
  62. struct iommu_group *group;
  63. struct dma_iommu_mapping *mapping;
  64. };
  65. struct ipmmu_vmsa_domain {
  66. struct ipmmu_vmsa_device *mmu;
  67. struct iommu_domain io_domain;
  68. struct io_pgtable_cfg cfg;
  69. struct io_pgtable_ops *iop;
  70. unsigned int context_id;
  71. struct mutex mutex; /* Protects mappings */
  72. };
  73. static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
  74. {
  75. return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
  76. }
  77. static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
  78. {
  79. return dev_iommu_priv_get(dev);
  80. }
  81. #define TLB_LOOP_TIMEOUT 100 /* 100us */
  82. /* -----------------------------------------------------------------------------
  83. * Registers Definition
  84. */
  85. #define IM_NS_ALIAS_OFFSET 0x800
  86. /* MMU "context" registers */
  87. #define IMCTR 0x0000 /* R-Car Gen2/3 */
  88. #define IMCTR_INTEN (1 << 2) /* R-Car Gen2/3 */
  89. #define IMCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
  90. #define IMCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
  91. #define IMTTBCR 0x0008 /* R-Car Gen2/3 */
  92. #define IMTTBCR_EAE (1 << 31) /* R-Car Gen2/3 */
  93. #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */
  94. #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */
  95. #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */
  96. #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */
  97. #define IMTTBCR_SL0_LVL_1 (1 << 4) /* R-Car Gen2 only */
  98. #define IMBUSCR 0x000c /* R-Car Gen2 only */
  99. #define IMBUSCR_DVM (1 << 2) /* R-Car Gen2 only */
  100. #define IMBUSCR_BUSSEL_MASK (3 << 0) /* R-Car Gen2 only */
  101. #define IMTTLBR0 0x0010 /* R-Car Gen2/3 */
  102. #define IMTTUBR0 0x0014 /* R-Car Gen2/3 */
  103. #define IMSTR 0x0020 /* R-Car Gen2/3 */
  104. #define IMSTR_MHIT (1 << 4) /* R-Car Gen2/3 */
  105. #define IMSTR_ABORT (1 << 2) /* R-Car Gen2/3 */
  106. #define IMSTR_PF (1 << 1) /* R-Car Gen2/3 */
  107. #define IMSTR_TF (1 << 0) /* R-Car Gen2/3 */
  108. #define IMMAIR0 0x0028 /* R-Car Gen2/3 */
  109. #define IMELAR 0x0030 /* R-Car Gen2/3, IMEAR on R-Car Gen2 */
  110. #define IMEUAR 0x0034 /* R-Car Gen3 only */
  111. /* uTLB registers */
  112. #define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n))
  113. #define IMUCTR0(n) (0x0300 + ((n) * 16)) /* R-Car Gen2/3 */
  114. #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) /* R-Car Gen3 only */
  115. #define IMUCTR_TTSEL_MMU(n) ((n) << 4) /* R-Car Gen2/3 */
  116. #define IMUCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
  117. #define IMUCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
  118. #define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n))
  119. #define IMUASID0(n) (0x0308 + ((n) * 16)) /* R-Car Gen2/3 */
  120. #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) /* R-Car Gen3 only */
  121. /* -----------------------------------------------------------------------------
  122. * Root device handling
  123. */
  124. static struct platform_driver ipmmu_driver;
  125. static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)
  126. {
  127. return mmu->root == mmu;
  128. }
  129. static int __ipmmu_check_device(struct device *dev, void *data)
  130. {
  131. struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
  132. struct ipmmu_vmsa_device **rootp = data;
  133. if (ipmmu_is_root(mmu))
  134. *rootp = mmu;
  135. return 0;
  136. }
  137. static struct ipmmu_vmsa_device *ipmmu_find_root(void)
  138. {
  139. struct ipmmu_vmsa_device *root = NULL;
  140. return driver_for_each_device(&ipmmu_driver.driver, NULL, &root,
  141. __ipmmu_check_device) == 0 ? root : NULL;
  142. }
  143. /* -----------------------------------------------------------------------------
  144. * Read/Write Access
  145. */
  146. static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
  147. {
  148. return ioread32(mmu->base + offset);
  149. }
  150. static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
  151. u32 data)
  152. {
  153. iowrite32(data, mmu->base + offset);
  154. }
  155. static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,
  156. unsigned int context_id, unsigned int reg)
  157. {
  158. return mmu->features->ctx_offset_base +
  159. context_id * mmu->features->ctx_offset_stride + reg;
  160. }
  161. static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,
  162. unsigned int context_id, unsigned int reg)
  163. {
  164. return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));
  165. }
  166. static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,
  167. unsigned int context_id, unsigned int reg, u32 data)
  168. {
  169. ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);
  170. }
  171. static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain,
  172. unsigned int reg)
  173. {
  174. return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);
  175. }
  176. static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain,
  177. unsigned int reg, u32 data)
  178. {
  179. ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
  180. }
  181. static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain,
  182. unsigned int reg, u32 data)
  183. {
  184. if (domain->mmu != domain->mmu->root)
  185. ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);
  186. ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
  187. }
  188. static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)
  189. {
  190. return mmu->features->utlb_offset_base + reg;
  191. }
  192. static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,
  193. unsigned int utlb, u32 data)
  194. {
  195. ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);
  196. }
  197. static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,
  198. unsigned int utlb, u32 data)
  199. {
  200. ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);
  201. }
  202. /* -----------------------------------------------------------------------------
  203. * TLB and microTLB Management
  204. */
  205. /* Wait for any pending TLB invalidations to complete */
  206. static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
  207. {
  208. unsigned int count = 0;
  209. while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) {
  210. cpu_relax();
  211. if (++count == TLB_LOOP_TIMEOUT) {
  212. dev_err_ratelimited(domain->mmu->dev,
  213. "TLB sync timed out -- MMU may be deadlocked\n");
  214. return;
  215. }
  216. udelay(1);
  217. }
  218. }
  219. static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
  220. {
  221. u32 reg;
  222. reg = ipmmu_ctx_read_root(domain, IMCTR);
  223. reg |= IMCTR_FLUSH;
  224. ipmmu_ctx_write_all(domain, IMCTR, reg);
  225. ipmmu_tlb_sync(domain);
  226. }
  227. /*
  228. * Enable MMU translation for the microTLB.
  229. */
  230. static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
  231. unsigned int utlb)
  232. {
  233. struct ipmmu_vmsa_device *mmu = domain->mmu;
  234. /*
  235. * TODO: Reference-count the microTLB as several bus masters can be
  236. * connected to the same microTLB.
  237. */
  238. /* TODO: What should we set the ASID to ? */
  239. ipmmu_imuasid_write(mmu, utlb, 0);
  240. /* TODO: Do we need to flush the microTLB ? */
  241. ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |
  242. IMUCTR_FLUSH | IMUCTR_MMUEN);
  243. mmu->utlb_ctx[utlb] = domain->context_id;
  244. }
  245. /*
  246. * Disable MMU translation for the microTLB.
  247. */
  248. static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
  249. unsigned int utlb)
  250. {
  251. struct ipmmu_vmsa_device *mmu = domain->mmu;
  252. ipmmu_imuctr_write(mmu, utlb, 0);
  253. mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;
  254. }
  255. static void ipmmu_tlb_flush_all(void *cookie)
  256. {
  257. struct ipmmu_vmsa_domain *domain = cookie;
  258. ipmmu_tlb_invalidate(domain);
  259. }
  260. static void ipmmu_tlb_flush(unsigned long iova, size_t size,
  261. size_t granule, void *cookie)
  262. {
  263. ipmmu_tlb_flush_all(cookie);
  264. }
  265. static const struct iommu_flush_ops ipmmu_flush_ops = {
  266. .tlb_flush_all = ipmmu_tlb_flush_all,
  267. .tlb_flush_walk = ipmmu_tlb_flush,
  268. };
  269. /* -----------------------------------------------------------------------------
  270. * Domain/Context Management
  271. */
  272. static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
  273. struct ipmmu_vmsa_domain *domain)
  274. {
  275. unsigned long flags;
  276. int ret;
  277. spin_lock_irqsave(&mmu->lock, flags);
  278. ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);
  279. if (ret != mmu->num_ctx) {
  280. mmu->domains[ret] = domain;
  281. set_bit(ret, mmu->ctx);
  282. } else
  283. ret = -EBUSY;
  284. spin_unlock_irqrestore(&mmu->lock, flags);
  285. return ret;
  286. }
  287. static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
  288. unsigned int context_id)
  289. {
  290. unsigned long flags;
  291. spin_lock_irqsave(&mmu->lock, flags);
  292. clear_bit(context_id, mmu->ctx);
  293. mmu->domains[context_id] = NULL;
  294. spin_unlock_irqrestore(&mmu->lock, flags);
  295. }
  296. static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain)
  297. {
  298. u64 ttbr;
  299. u32 tmp;
  300. /* TTBR0 */
  301. ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
  302. ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
  303. ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
  304. /*
  305. * TTBCR
  306. * We use long descriptors and allocate the whole 32-bit VA space to
  307. * TTBR0.
  308. */
  309. if (domain->mmu->features->twobit_imttbcr_sl0)
  310. tmp = IMTTBCR_SL0_TWOBIT_LVL_1;
  311. else
  312. tmp = IMTTBCR_SL0_LVL_1;
  313. if (domain->mmu->features->cache_snoop)
  314. tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
  315. IMTTBCR_IRGN0_WB_WA;
  316. ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp);
  317. /* MAIR0 */
  318. ipmmu_ctx_write_root(domain, IMMAIR0,
  319. domain->cfg.arm_lpae_s1_cfg.mair);
  320. /* IMBUSCR */
  321. if (domain->mmu->features->setup_imbuscr)
  322. ipmmu_ctx_write_root(domain, IMBUSCR,
  323. ipmmu_ctx_read_root(domain, IMBUSCR) &
  324. ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
  325. /*
  326. * IMSTR
  327. * Clear all interrupt flags.
  328. */
  329. ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR));
  330. /*
  331. * IMCTR
  332. * Enable the MMU and interrupt generation. The long-descriptor
  333. * translation table format doesn't use TEX remapping. Don't enable AF
  334. * software management as we have no use for it. Flush the TLB as
  335. * required when modifying the context registers.
  336. */
  337. ipmmu_ctx_write_all(domain, IMCTR,
  338. IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
  339. }
  340. static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
  341. {
  342. int ret;
  343. /*
  344. * Allocate the page table operations.
  345. *
  346. * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory
  347. * access, Long-descriptor format" that the NStable bit being set in a
  348. * table descriptor will result in the NStable and NS bits of all child
  349. * entries being ignored and considered as being set. The IPMMU seems
  350. * not to comply with this, as it generates a secure access page fault
  351. * if any of the NStable and NS bits isn't set when running in
  352. * non-secure mode.
  353. */
  354. domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS;
  355. domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K;
  356. domain->cfg.ias = 32;
  357. domain->cfg.oas = 40;
  358. domain->cfg.tlb = &ipmmu_flush_ops;
  359. domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32);
  360. domain->io_domain.geometry.force_aperture = true;
  361. /*
  362. * TODO: Add support for coherent walk through CCI with DVM and remove
  363. * cache handling. For now, delegate it to the io-pgtable code.
  364. */
  365. domain->cfg.coherent_walk = false;
  366. domain->cfg.iommu_dev = domain->mmu->root->dev;
  367. /*
  368. * Find an unused context.
  369. */
  370. ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);
  371. if (ret < 0)
  372. return ret;
  373. domain->context_id = ret;
  374. domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
  375. domain);
  376. if (!domain->iop) {
  377. ipmmu_domain_free_context(domain->mmu->root,
  378. domain->context_id);
  379. return -EINVAL;
  380. }
  381. ipmmu_domain_setup_context(domain);
  382. return 0;
  383. }
  384. static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
  385. {
  386. if (!domain->mmu)
  387. return;
  388. /*
  389. * Disable the context. Flush the TLB as required when modifying the
  390. * context registers.
  391. *
  392. * TODO: Is TLB flush really needed ?
  393. */
  394. ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH);
  395. ipmmu_tlb_sync(domain);
  396. ipmmu_domain_free_context(domain->mmu->root, domain->context_id);
  397. }
  398. /* -----------------------------------------------------------------------------
  399. * Fault Handling
  400. */
  401. static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
  402. {
  403. const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
  404. struct ipmmu_vmsa_device *mmu = domain->mmu;
  405. unsigned long iova;
  406. u32 status;
  407. status = ipmmu_ctx_read_root(domain, IMSTR);
  408. if (!(status & err_mask))
  409. return IRQ_NONE;
  410. iova = ipmmu_ctx_read_root(domain, IMELAR);
  411. if (IS_ENABLED(CONFIG_64BIT))
  412. iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32;
  413. /*
  414. * Clear the error status flags. Unlike traditional interrupt flag
  415. * registers that must be cleared by writing 1, this status register
  416. * seems to require 0. The error address register must be read before,
  417. * otherwise its value will be 0.
  418. */
  419. ipmmu_ctx_write_root(domain, IMSTR, 0);
  420. /* Log fatal errors. */
  421. if (status & IMSTR_MHIT)
  422. dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",
  423. iova);
  424. if (status & IMSTR_ABORT)
  425. dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",
  426. iova);
  427. if (!(status & (IMSTR_PF | IMSTR_TF)))
  428. return IRQ_NONE;
  429. /*
  430. * Try to handle page faults and translation faults.
  431. *
  432. * TODO: We need to look up the faulty device based on the I/O VA. Use
  433. * the IOMMU device for now.
  434. */
  435. if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))
  436. return IRQ_HANDLED;
  437. dev_err_ratelimited(mmu->dev,
  438. "Unhandled fault: status 0x%08x iova 0x%lx\n",
  439. status, iova);
  440. return IRQ_HANDLED;
  441. }
  442. static irqreturn_t ipmmu_irq(int irq, void *dev)
  443. {
  444. struct ipmmu_vmsa_device *mmu = dev;
  445. irqreturn_t status = IRQ_NONE;
  446. unsigned int i;
  447. unsigned long flags;
  448. spin_lock_irqsave(&mmu->lock, flags);
  449. /*
  450. * Check interrupts for all active contexts.
  451. */
  452. for (i = 0; i < mmu->num_ctx; i++) {
  453. if (!mmu->domains[i])
  454. continue;
  455. if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)
  456. status = IRQ_HANDLED;
  457. }
  458. spin_unlock_irqrestore(&mmu->lock, flags);
  459. return status;
  460. }
  461. /* -----------------------------------------------------------------------------
  462. * IOMMU Operations
  463. */
  464. static struct iommu_domain *__ipmmu_domain_alloc(unsigned type)
  465. {
  466. struct ipmmu_vmsa_domain *domain;
  467. domain = kzalloc(sizeof(*domain), GFP_KERNEL);
  468. if (!domain)
  469. return NULL;
  470. mutex_init(&domain->mutex);
  471. return &domain->io_domain;
  472. }
  473. static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
  474. {
  475. struct iommu_domain *io_domain = NULL;
  476. switch (type) {
  477. case IOMMU_DOMAIN_UNMANAGED:
  478. io_domain = __ipmmu_domain_alloc(type);
  479. break;
  480. case IOMMU_DOMAIN_DMA:
  481. io_domain = __ipmmu_domain_alloc(type);
  482. if (io_domain && iommu_get_dma_cookie(io_domain)) {
  483. kfree(io_domain);
  484. io_domain = NULL;
  485. }
  486. break;
  487. }
  488. return io_domain;
  489. }
  490. static void ipmmu_domain_free(struct iommu_domain *io_domain)
  491. {
  492. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  493. /*
  494. * Free the domain resources. We assume that all devices have already
  495. * been detached.
  496. */
  497. iommu_put_dma_cookie(io_domain);
  498. ipmmu_domain_destroy_context(domain);
  499. free_io_pgtable_ops(domain->iop);
  500. kfree(domain);
  501. }
  502. static int ipmmu_attach_device(struct iommu_domain *io_domain,
  503. struct device *dev)
  504. {
  505. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  506. struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
  507. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  508. unsigned int i;
  509. int ret = 0;
  510. if (!mmu) {
  511. dev_err(dev, "Cannot attach to IPMMU\n");
  512. return -ENXIO;
  513. }
  514. mutex_lock(&domain->mutex);
  515. if (!domain->mmu) {
  516. /* The domain hasn't been used yet, initialize it. */
  517. domain->mmu = mmu;
  518. ret = ipmmu_domain_init_context(domain);
  519. if (ret < 0) {
  520. dev_err(dev, "Unable to initialize IPMMU context\n");
  521. domain->mmu = NULL;
  522. } else {
  523. dev_info(dev, "Using IPMMU context %u\n",
  524. domain->context_id);
  525. }
  526. } else if (domain->mmu != mmu) {
  527. /*
  528. * Something is wrong, we can't attach two devices using
  529. * different IOMMUs to the same domain.
  530. */
  531. dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
  532. dev_name(mmu->dev), dev_name(domain->mmu->dev));
  533. ret = -EINVAL;
  534. } else
  535. dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
  536. mutex_unlock(&domain->mutex);
  537. if (ret < 0)
  538. return ret;
  539. for (i = 0; i < fwspec->num_ids; ++i)
  540. ipmmu_utlb_enable(domain, fwspec->ids[i]);
  541. return 0;
  542. }
  543. static void ipmmu_detach_device(struct iommu_domain *io_domain,
  544. struct device *dev)
  545. {
  546. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  547. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  548. unsigned int i;
  549. for (i = 0; i < fwspec->num_ids; ++i)
  550. ipmmu_utlb_disable(domain, fwspec->ids[i]);
  551. /*
  552. * TODO: Optimize by disabling the context when no device is attached.
  553. */
  554. }
  555. static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
  556. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  557. {
  558. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  559. if (!domain)
  560. return -ENODEV;
  561. return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp);
  562. }
  563. static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
  564. size_t size, struct iommu_iotlb_gather *gather)
  565. {
  566. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  567. return domain->iop->unmap(domain->iop, iova, size, gather);
  568. }
  569. static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
  570. {
  571. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  572. if (domain->mmu)
  573. ipmmu_tlb_flush_all(domain);
  574. }
  575. static void ipmmu_iotlb_sync(struct iommu_domain *io_domain,
  576. struct iommu_iotlb_gather *gather)
  577. {
  578. ipmmu_flush_iotlb_all(io_domain);
  579. }
  580. static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
  581. dma_addr_t iova)
  582. {
  583. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  584. /* TODO: Is locking needed ? */
  585. return domain->iop->iova_to_phys(domain->iop, iova);
  586. }
  587. static int ipmmu_init_platform_device(struct device *dev,
  588. struct of_phandle_args *args)
  589. {
  590. struct platform_device *ipmmu_pdev;
  591. ipmmu_pdev = of_find_device_by_node(args->np);
  592. if (!ipmmu_pdev)
  593. return -ENODEV;
  594. dev_iommu_priv_set(dev, platform_get_drvdata(ipmmu_pdev));
  595. return 0;
  596. }
  597. static const struct soc_device_attribute soc_rcar_gen3[] = {
  598. { .soc_id = "r8a774a1", },
  599. { .soc_id = "r8a774b1", },
  600. { .soc_id = "r8a774c0", },
  601. { .soc_id = "r8a774e1", },
  602. { .soc_id = "r8a7795", },
  603. { .soc_id = "r8a77961", },
  604. { .soc_id = "r8a7796", },
  605. { .soc_id = "r8a77965", },
  606. { .soc_id = "r8a77970", },
  607. { .soc_id = "r8a77990", },
  608. { .soc_id = "r8a77995", },
  609. { /* sentinel */ }
  610. };
  611. static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = {
  612. { .soc_id = "r8a774b1", },
  613. { .soc_id = "r8a774c0", },
  614. { .soc_id = "r8a774e1", },
  615. { .soc_id = "r8a7795", .revision = "ES3.*" },
  616. { .soc_id = "r8a77961", },
  617. { .soc_id = "r8a77965", },
  618. { .soc_id = "r8a77990", },
  619. { .soc_id = "r8a77995", },
  620. { /* sentinel */ }
  621. };
  622. static const char * const rcar_gen3_slave_whitelist[] = {
  623. };
  624. static bool ipmmu_slave_whitelist(struct device *dev)
  625. {
  626. unsigned int i;
  627. /*
  628. * For R-Car Gen3 use a white list to opt-in slave devices.
  629. * For Other SoCs, this returns true anyway.
  630. */
  631. if (!soc_device_match(soc_rcar_gen3))
  632. return true;
  633. /* Check whether this R-Car Gen3 can use the IPMMU correctly or not */
  634. if (!soc_device_match(soc_rcar_gen3_whitelist))
  635. return false;
  636. /* Check whether this slave device can work with the IPMMU */
  637. for (i = 0; i < ARRAY_SIZE(rcar_gen3_slave_whitelist); i++) {
  638. if (!strcmp(dev_name(dev), rcar_gen3_slave_whitelist[i]))
  639. return true;
  640. }
  641. /* Otherwise, do not allow use of IPMMU */
  642. return false;
  643. }
  644. static int ipmmu_of_xlate(struct device *dev,
  645. struct of_phandle_args *spec)
  646. {
  647. if (!ipmmu_slave_whitelist(dev))
  648. return -ENODEV;
  649. iommu_fwspec_add_ids(dev, spec->args, 1);
  650. /* Initialize once - xlate() will call multiple times */
  651. if (to_ipmmu(dev))
  652. return 0;
  653. return ipmmu_init_platform_device(dev, spec);
  654. }
  655. static int ipmmu_init_arm_mapping(struct device *dev)
  656. {
  657. struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
  658. int ret;
  659. /*
  660. * Create the ARM mapping, used by the ARM DMA mapping core to allocate
  661. * VAs. This will allocate a corresponding IOMMU domain.
  662. *
  663. * TODO:
  664. * - Create one mapping per context (TLB).
  665. * - Make the mapping size configurable ? We currently use a 2GB mapping
  666. * at a 1GB offset to ensure that NULL VAs will fault.
  667. */
  668. if (!mmu->mapping) {
  669. struct dma_iommu_mapping *mapping;
  670. mapping = arm_iommu_create_mapping(&platform_bus_type,
  671. SZ_1G, SZ_2G);
  672. if (IS_ERR(mapping)) {
  673. dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
  674. ret = PTR_ERR(mapping);
  675. goto error;
  676. }
  677. mmu->mapping = mapping;
  678. }
  679. /* Attach the ARM VA mapping to the device. */
  680. ret = arm_iommu_attach_device(dev, mmu->mapping);
  681. if (ret < 0) {
  682. dev_err(dev, "Failed to attach device to VA mapping\n");
  683. goto error;
  684. }
  685. return 0;
  686. error:
  687. if (mmu->mapping)
  688. arm_iommu_release_mapping(mmu->mapping);
  689. return ret;
  690. }
  691. static struct iommu_device *ipmmu_probe_device(struct device *dev)
  692. {
  693. struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
  694. /*
  695. * Only let through devices that have been verified in xlate()
  696. */
  697. if (!mmu)
  698. return ERR_PTR(-ENODEV);
  699. return &mmu->iommu;
  700. }
  701. static void ipmmu_probe_finalize(struct device *dev)
  702. {
  703. int ret = 0;
  704. if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA))
  705. ret = ipmmu_init_arm_mapping(dev);
  706. if (ret)
  707. dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");
  708. }
  709. static void ipmmu_release_device(struct device *dev)
  710. {
  711. arm_iommu_detach_device(dev);
  712. }
  713. static struct iommu_group *ipmmu_find_group(struct device *dev)
  714. {
  715. struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
  716. struct iommu_group *group;
  717. if (mmu->group)
  718. return iommu_group_ref_get(mmu->group);
  719. group = iommu_group_alloc();
  720. if (!IS_ERR(group))
  721. mmu->group = group;
  722. return group;
  723. }
  724. static const struct iommu_ops ipmmu_ops = {
  725. .domain_alloc = ipmmu_domain_alloc,
  726. .domain_free = ipmmu_domain_free,
  727. .attach_dev = ipmmu_attach_device,
  728. .detach_dev = ipmmu_detach_device,
  729. .map = ipmmu_map,
  730. .unmap = ipmmu_unmap,
  731. .flush_iotlb_all = ipmmu_flush_iotlb_all,
  732. .iotlb_sync = ipmmu_iotlb_sync,
  733. .iova_to_phys = ipmmu_iova_to_phys,
  734. .probe_device = ipmmu_probe_device,
  735. .release_device = ipmmu_release_device,
  736. .probe_finalize = ipmmu_probe_finalize,
  737. .device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)
  738. ? generic_device_group : ipmmu_find_group,
  739. .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
  740. .of_xlate = ipmmu_of_xlate,
  741. };
  742. /* -----------------------------------------------------------------------------
  743. * Probe/remove and init
  744. */
  745. static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
  746. {
  747. unsigned int i;
  748. /* Disable all contexts. */
  749. for (i = 0; i < mmu->num_ctx; ++i)
  750. ipmmu_ctx_write(mmu, i, IMCTR, 0);
  751. }
  752. static const struct ipmmu_features ipmmu_features_default = {
  753. .use_ns_alias_offset = true,
  754. .has_cache_leaf_nodes = false,
  755. .number_of_contexts = 1, /* software only tested with one context */
  756. .num_utlbs = 32,
  757. .setup_imbuscr = true,
  758. .twobit_imttbcr_sl0 = false,
  759. .reserved_context = false,
  760. .cache_snoop = true,
  761. .ctx_offset_base = 0,
  762. .ctx_offset_stride = 0x40,
  763. .utlb_offset_base = 0,
  764. };
  765. static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
  766. .use_ns_alias_offset = false,
  767. .has_cache_leaf_nodes = true,
  768. .number_of_contexts = 8,
  769. .num_utlbs = 48,
  770. .setup_imbuscr = false,
  771. .twobit_imttbcr_sl0 = true,
  772. .reserved_context = true,
  773. .cache_snoop = false,
  774. .ctx_offset_base = 0,
  775. .ctx_offset_stride = 0x40,
  776. .utlb_offset_base = 0,
  777. };
  778. static const struct of_device_id ipmmu_of_ids[] = {
  779. {
  780. .compatible = "renesas,ipmmu-vmsa",
  781. .data = &ipmmu_features_default,
  782. }, {
  783. .compatible = "renesas,ipmmu-r8a774a1",
  784. .data = &ipmmu_features_rcar_gen3,
  785. }, {
  786. .compatible = "renesas,ipmmu-r8a774b1",
  787. .data = &ipmmu_features_rcar_gen3,
  788. }, {
  789. .compatible = "renesas,ipmmu-r8a774c0",
  790. .data = &ipmmu_features_rcar_gen3,
  791. }, {
  792. .compatible = "renesas,ipmmu-r8a774e1",
  793. .data = &ipmmu_features_rcar_gen3,
  794. }, {
  795. .compatible = "renesas,ipmmu-r8a7795",
  796. .data = &ipmmu_features_rcar_gen3,
  797. }, {
  798. .compatible = "renesas,ipmmu-r8a7796",
  799. .data = &ipmmu_features_rcar_gen3,
  800. }, {
  801. .compatible = "renesas,ipmmu-r8a77961",
  802. .data = &ipmmu_features_rcar_gen3,
  803. }, {
  804. .compatible = "renesas,ipmmu-r8a77965",
  805. .data = &ipmmu_features_rcar_gen3,
  806. }, {
  807. .compatible = "renesas,ipmmu-r8a77970",
  808. .data = &ipmmu_features_rcar_gen3,
  809. }, {
  810. .compatible = "renesas,ipmmu-r8a77990",
  811. .data = &ipmmu_features_rcar_gen3,
  812. }, {
  813. .compatible = "renesas,ipmmu-r8a77995",
  814. .data = &ipmmu_features_rcar_gen3,
  815. }, {
  816. /* Terminator */
  817. },
  818. };
  819. static int ipmmu_probe(struct platform_device *pdev)
  820. {
  821. struct ipmmu_vmsa_device *mmu;
  822. struct resource *res;
  823. int irq;
  824. int ret;
  825. mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
  826. if (!mmu) {
  827. dev_err(&pdev->dev, "cannot allocate device data\n");
  828. return -ENOMEM;
  829. }
  830. mmu->dev = &pdev->dev;
  831. spin_lock_init(&mmu->lock);
  832. bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
  833. mmu->features = of_device_get_match_data(&pdev->dev);
  834. memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
  835. ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
  836. if (ret)
  837. return ret;
  838. /* Map I/O memory and request IRQ. */
  839. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  840. mmu->base = devm_ioremap_resource(&pdev->dev, res);
  841. if (IS_ERR(mmu->base))
  842. return PTR_ERR(mmu->base);
  843. /*
  844. * The IPMMU has two register banks, for secure and non-secure modes.
  845. * The bank mapped at the beginning of the IPMMU address space
  846. * corresponds to the running mode of the CPU. When running in secure
  847. * mode the non-secure register bank is also available at an offset.
  848. *
  849. * Secure mode operation isn't clearly documented and is thus currently
  850. * not implemented in the driver. Furthermore, preliminary tests of
  851. * non-secure operation with the main register bank were not successful.
  852. * Offset the registers base unconditionally to point to the non-secure
  853. * alias space for now.
  854. */
  855. if (mmu->features->use_ns_alias_offset)
  856. mmu->base += IM_NS_ALIAS_OFFSET;
  857. mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);
  858. /*
  859. * Determine if this IPMMU instance is a root device by checking for
  860. * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property.
  861. */
  862. if (!mmu->features->has_cache_leaf_nodes ||
  863. !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL))
  864. mmu->root = mmu;
  865. else
  866. mmu->root = ipmmu_find_root();
  867. /*
  868. * Wait until the root device has been registered for sure.
  869. */
  870. if (!mmu->root)
  871. return -EPROBE_DEFER;
  872. /* Root devices have mandatory IRQs */
  873. if (ipmmu_is_root(mmu)) {
  874. irq = platform_get_irq(pdev, 0);
  875. if (irq < 0)
  876. return irq;
  877. ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
  878. dev_name(&pdev->dev), mmu);
  879. if (ret < 0) {
  880. dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
  881. return ret;
  882. }
  883. ipmmu_device_reset(mmu);
  884. if (mmu->features->reserved_context) {
  885. dev_info(&pdev->dev, "IPMMU context 0 is reserved\n");
  886. set_bit(0, mmu->ctx);
  887. }
  888. }
  889. /*
  890. * Register the IPMMU to the IOMMU subsystem in the following cases:
  891. * - R-Car Gen2 IPMMU (all devices registered)
  892. * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device)
  893. */
  894. if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {
  895. ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
  896. dev_name(&pdev->dev));
  897. if (ret)
  898. return ret;
  899. iommu_device_set_ops(&mmu->iommu, &ipmmu_ops);
  900. iommu_device_set_fwnode(&mmu->iommu,
  901. &pdev->dev.of_node->fwnode);
  902. ret = iommu_device_register(&mmu->iommu);
  903. if (ret)
  904. return ret;
  905. #if defined(CONFIG_IOMMU_DMA)
  906. if (!iommu_present(&platform_bus_type))
  907. bus_set_iommu(&platform_bus_type, &ipmmu_ops);
  908. #endif
  909. }
  910. /*
  911. * We can't create the ARM mapping here as it requires the bus to have
  912. * an IOMMU, which only happens when bus_set_iommu() is called in
  913. * ipmmu_init() after the probe function returns.
  914. */
  915. platform_set_drvdata(pdev, mmu);
  916. return 0;
  917. }
  918. static int ipmmu_remove(struct platform_device *pdev)
  919. {
  920. struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
  921. iommu_device_sysfs_remove(&mmu->iommu);
  922. iommu_device_unregister(&mmu->iommu);
  923. arm_iommu_release_mapping(mmu->mapping);
  924. ipmmu_device_reset(mmu);
  925. return 0;
  926. }
  927. #ifdef CONFIG_PM_SLEEP
  928. static int ipmmu_resume_noirq(struct device *dev)
  929. {
  930. struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
  931. unsigned int i;
  932. /* Reset root MMU and restore contexts */
  933. if (ipmmu_is_root(mmu)) {
  934. ipmmu_device_reset(mmu);
  935. for (i = 0; i < mmu->num_ctx; i++) {
  936. if (!mmu->domains[i])
  937. continue;
  938. ipmmu_domain_setup_context(mmu->domains[i]);
  939. }
  940. }
  941. /* Re-enable active micro-TLBs */
  942. for (i = 0; i < mmu->features->num_utlbs; i++) {
  943. if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)
  944. continue;
  945. ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);
  946. }
  947. return 0;
  948. }
  949. static const struct dev_pm_ops ipmmu_pm = {
  950. SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq)
  951. };
  952. #define DEV_PM_OPS &ipmmu_pm
  953. #else
  954. #define DEV_PM_OPS NULL
  955. #endif /* CONFIG_PM_SLEEP */
  956. static struct platform_driver ipmmu_driver = {
  957. .driver = {
  958. .name = "ipmmu-vmsa",
  959. .of_match_table = of_match_ptr(ipmmu_of_ids),
  960. .pm = DEV_PM_OPS,
  961. },
  962. .probe = ipmmu_probe,
  963. .remove = ipmmu_remove,
  964. };
  965. static int __init ipmmu_init(void)
  966. {
  967. struct device_node *np;
  968. static bool setup_done;
  969. int ret;
  970. if (setup_done)
  971. return 0;
  972. np = of_find_matching_node(NULL, ipmmu_of_ids);
  973. if (!np)
  974. return 0;
  975. of_node_put(np);
  976. ret = platform_driver_register(&ipmmu_driver);
  977. if (ret < 0)
  978. return ret;
  979. #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
  980. if (!iommu_present(&platform_bus_type))
  981. bus_set_iommu(&platform_bus_type, &ipmmu_ops);
  982. #endif
  983. setup_done = true;
  984. return 0;
  985. }
  986. subsys_initcall(ipmmu_init);