omap-iommu.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * omap iommu: tlb and pagetable primitives
  4. *
  5. * Copyright (C) 2008-2010 Nokia Corporation
  6. * Copyright (C) 2013-2017 Texas Instruments Incorporated - https://www.ti.com/
  7. *
  8. * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
  9. * Paul Mundt and Toshihiro Kobayashi
  10. */
  11. #include <linux/dma-mapping.h>
  12. #include <linux/err.h>
  13. #include <linux/slab.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/ioport.h>
  16. #include <linux/platform_device.h>
  17. #include <linux/iommu.h>
  18. #include <linux/omap-iommu.h>
  19. #include <linux/mutex.h>
  20. #include <linux/spinlock.h>
  21. #include <linux/io.h>
  22. #include <linux/pm_runtime.h>
  23. #include <linux/of.h>
  24. #include <linux/of_iommu.h>
  25. #include <linux/of_irq.h>
  26. #include <linux/of_platform.h>
  27. #include <linux/regmap.h>
  28. #include <linux/mfd/syscon.h>
  29. #include <linux/platform_data/iommu-omap.h>
  30. #include "omap-iopgtable.h"
  31. #include "omap-iommu.h"
  32. static const struct iommu_ops omap_iommu_ops;
  33. #define to_iommu(dev) ((struct omap_iommu *)dev_get_drvdata(dev))
  34. /* bitmap of the page sizes currently supported */
  35. #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
  36. #define MMU_LOCK_BASE_SHIFT 10
  37. #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
  38. #define MMU_LOCK_BASE(x) \
  39. ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
  40. #define MMU_LOCK_VICT_SHIFT 4
  41. #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
  42. #define MMU_LOCK_VICT(x) \
  43. ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
  44. static struct platform_driver omap_iommu_driver;
  45. static struct kmem_cache *iopte_cachep;
  46. /**
  47. * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
  48. * @dom: generic iommu domain handle
  49. **/
  50. static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
  51. {
  52. return container_of(dom, struct omap_iommu_domain, domain);
  53. }
  54. /**
  55. * omap_iommu_save_ctx - Save registers for pm off-mode support
  56. * @dev: client device
  57. *
  58. * This should be treated as an deprecated API. It is preserved only
  59. * to maintain existing functionality for OMAP3 ISP driver.
  60. **/
  61. void omap_iommu_save_ctx(struct device *dev)
  62. {
  63. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  64. struct omap_iommu *obj;
  65. u32 *p;
  66. int i;
  67. if (!arch_data)
  68. return;
  69. while (arch_data->iommu_dev) {
  70. obj = arch_data->iommu_dev;
  71. p = obj->ctx;
  72. for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
  73. p[i] = iommu_read_reg(obj, i * sizeof(u32));
  74. dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i,
  75. p[i]);
  76. }
  77. arch_data++;
  78. }
  79. }
  80. EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
  81. /**
  82. * omap_iommu_restore_ctx - Restore registers for pm off-mode support
  83. * @dev: client device
  84. *
  85. * This should be treated as an deprecated API. It is preserved only
  86. * to maintain existing functionality for OMAP3 ISP driver.
  87. **/
  88. void omap_iommu_restore_ctx(struct device *dev)
  89. {
  90. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  91. struct omap_iommu *obj;
  92. u32 *p;
  93. int i;
  94. if (!arch_data)
  95. return;
  96. while (arch_data->iommu_dev) {
  97. obj = arch_data->iommu_dev;
  98. p = obj->ctx;
  99. for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
  100. iommu_write_reg(obj, p[i], i * sizeof(u32));
  101. dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i,
  102. p[i]);
  103. }
  104. arch_data++;
  105. }
  106. }
  107. EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
  108. static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable)
  109. {
  110. u32 val, mask;
  111. if (!obj->syscfg)
  112. return;
  113. mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT));
  114. val = enable ? mask : 0;
  115. regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val);
  116. }
  117. static void __iommu_set_twl(struct omap_iommu *obj, bool on)
  118. {
  119. u32 l = iommu_read_reg(obj, MMU_CNTL);
  120. if (on)
  121. iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);
  122. else
  123. iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);
  124. l &= ~MMU_CNTL_MASK;
  125. if (on)
  126. l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN);
  127. else
  128. l |= (MMU_CNTL_MMU_EN);
  129. iommu_write_reg(obj, l, MMU_CNTL);
  130. }
  131. static int omap2_iommu_enable(struct omap_iommu *obj)
  132. {
  133. u32 l, pa;
  134. if (!obj->iopgd || !IS_ALIGNED((unsigned long)obj->iopgd, SZ_16K))
  135. return -EINVAL;
  136. pa = virt_to_phys(obj->iopgd);
  137. if (!IS_ALIGNED(pa, SZ_16K))
  138. return -EINVAL;
  139. l = iommu_read_reg(obj, MMU_REVISION);
  140. dev_info(obj->dev, "%s: version %d.%d\n", obj->name,
  141. (l >> 4) & 0xf, l & 0xf);
  142. iommu_write_reg(obj, pa, MMU_TTB);
  143. dra7_cfg_dspsys_mmu(obj, true);
  144. if (obj->has_bus_err_back)
  145. iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
  146. __iommu_set_twl(obj, true);
  147. return 0;
  148. }
  149. static void omap2_iommu_disable(struct omap_iommu *obj)
  150. {
  151. u32 l = iommu_read_reg(obj, MMU_CNTL);
  152. l &= ~MMU_CNTL_MASK;
  153. iommu_write_reg(obj, l, MMU_CNTL);
  154. dra7_cfg_dspsys_mmu(obj, false);
  155. dev_dbg(obj->dev, "%s is shutting down\n", obj->name);
  156. }
  157. static int iommu_enable(struct omap_iommu *obj)
  158. {
  159. int ret;
  160. ret = pm_runtime_get_sync(obj->dev);
  161. if (ret < 0)
  162. pm_runtime_put_noidle(obj->dev);
  163. return ret < 0 ? ret : 0;
  164. }
  165. static void iommu_disable(struct omap_iommu *obj)
  166. {
  167. pm_runtime_put_sync(obj->dev);
  168. }
  169. /*
  170. * TLB operations
  171. */
  172. static u32 iotlb_cr_to_virt(struct cr_regs *cr)
  173. {
  174. u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK;
  175. u32 mask = get_cam_va_mask(cr->cam & page_size);
  176. return cr->cam & mask;
  177. }
  178. static u32 get_iopte_attr(struct iotlb_entry *e)
  179. {
  180. u32 attr;
  181. attr = e->mixed << 5;
  182. attr |= e->endian;
  183. attr |= e->elsz >> 3;
  184. attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) ||
  185. (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6);
  186. return attr;
  187. }
  188. static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
  189. {
  190. u32 status, fault_addr;
  191. status = iommu_read_reg(obj, MMU_IRQSTATUS);
  192. status &= MMU_IRQ_MASK;
  193. if (!status) {
  194. *da = 0;
  195. return 0;
  196. }
  197. fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);
  198. *da = fault_addr;
  199. iommu_write_reg(obj, status, MMU_IRQSTATUS);
  200. return status;
  201. }
  202. void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
  203. {
  204. u32 val;
  205. val = iommu_read_reg(obj, MMU_LOCK);
  206. l->base = MMU_LOCK_BASE(val);
  207. l->vict = MMU_LOCK_VICT(val);
  208. }
  209. void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
  210. {
  211. u32 val;
  212. val = (l->base << MMU_LOCK_BASE_SHIFT);
  213. val |= (l->vict << MMU_LOCK_VICT_SHIFT);
  214. iommu_write_reg(obj, val, MMU_LOCK);
  215. }
  216. static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
  217. {
  218. cr->cam = iommu_read_reg(obj, MMU_READ_CAM);
  219. cr->ram = iommu_read_reg(obj, MMU_READ_RAM);
  220. }
  221. static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
  222. {
  223. iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);
  224. iommu_write_reg(obj, cr->ram, MMU_RAM);
  225. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  226. iommu_write_reg(obj, 1, MMU_LD_TLB);
  227. }
  228. /* only used in iotlb iteration for-loop */
  229. struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
  230. {
  231. struct cr_regs cr;
  232. struct iotlb_lock l;
  233. iotlb_lock_get(obj, &l);
  234. l.vict = n;
  235. iotlb_lock_set(obj, &l);
  236. iotlb_read_cr(obj, &cr);
  237. return cr;
  238. }
  239. #ifdef PREFETCH_IOTLB
  240. static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
  241. struct iotlb_entry *e)
  242. {
  243. struct cr_regs *cr;
  244. if (!e)
  245. return NULL;
  246. if (e->da & ~(get_cam_va_mask(e->pgsz))) {
  247. dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,
  248. e->da);
  249. return ERR_PTR(-EINVAL);
  250. }
  251. cr = kmalloc(sizeof(*cr), GFP_KERNEL);
  252. if (!cr)
  253. return ERR_PTR(-ENOMEM);
  254. cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;
  255. cr->ram = e->pa | e->endian | e->elsz | e->mixed;
  256. return cr;
  257. }
  258. /**
  259. * load_iotlb_entry - Set an iommu tlb entry
  260. * @obj: target iommu
  261. * @e: an iommu tlb entry info
  262. **/
  263. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  264. {
  265. int err = 0;
  266. struct iotlb_lock l;
  267. struct cr_regs *cr;
  268. if (!obj || !obj->nr_tlb_entries || !e)
  269. return -EINVAL;
  270. pm_runtime_get_sync(obj->dev);
  271. iotlb_lock_get(obj, &l);
  272. if (l.base == obj->nr_tlb_entries) {
  273. dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
  274. err = -EBUSY;
  275. goto out;
  276. }
  277. if (!e->prsvd) {
  278. int i;
  279. struct cr_regs tmp;
  280. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
  281. if (!iotlb_cr_valid(&tmp))
  282. break;
  283. if (i == obj->nr_tlb_entries) {
  284. dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
  285. err = -EBUSY;
  286. goto out;
  287. }
  288. iotlb_lock_get(obj, &l);
  289. } else {
  290. l.vict = l.base;
  291. iotlb_lock_set(obj, &l);
  292. }
  293. cr = iotlb_alloc_cr(obj, e);
  294. if (IS_ERR(cr)) {
  295. pm_runtime_put_sync(obj->dev);
  296. return PTR_ERR(cr);
  297. }
  298. iotlb_load_cr(obj, cr);
  299. kfree(cr);
  300. if (e->prsvd)
  301. l.base++;
  302. /* increment victim for next tlb load */
  303. if (++l.vict == obj->nr_tlb_entries)
  304. l.vict = l.base;
  305. iotlb_lock_set(obj, &l);
  306. out:
  307. pm_runtime_put_sync(obj->dev);
  308. return err;
  309. }
  310. #else /* !PREFETCH_IOTLB */
  311. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  312. {
  313. return 0;
  314. }
  315. #endif /* !PREFETCH_IOTLB */
  316. static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  317. {
  318. return load_iotlb_entry(obj, e);
  319. }
  320. /**
  321. * flush_iotlb_page - Clear an iommu tlb entry
  322. * @obj: target iommu
  323. * @da: iommu device virtual address
  324. *
  325. * Clear an iommu tlb entry which includes 'da' address.
  326. **/
  327. static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
  328. {
  329. int i;
  330. struct cr_regs cr;
  331. pm_runtime_get_sync(obj->dev);
  332. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
  333. u32 start;
  334. size_t bytes;
  335. if (!iotlb_cr_valid(&cr))
  336. continue;
  337. start = iotlb_cr_to_virt(&cr);
  338. bytes = iopgsz_to_bytes(cr.cam & 3);
  339. if ((start <= da) && (da < start + bytes)) {
  340. dev_dbg(obj->dev, "%s: %08x<=%08x(%zx)\n",
  341. __func__, start, da, bytes);
  342. iotlb_load_cr(obj, &cr);
  343. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  344. break;
  345. }
  346. }
  347. pm_runtime_put_sync(obj->dev);
  348. if (i == obj->nr_tlb_entries)
  349. dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
  350. }
  351. /**
  352. * flush_iotlb_all - Clear all iommu tlb entries
  353. * @obj: target iommu
  354. **/
  355. static void flush_iotlb_all(struct omap_iommu *obj)
  356. {
  357. struct iotlb_lock l;
  358. pm_runtime_get_sync(obj->dev);
  359. l.base = 0;
  360. l.vict = 0;
  361. iotlb_lock_set(obj, &l);
  362. iommu_write_reg(obj, 1, MMU_GFLUSH);
  363. pm_runtime_put_sync(obj->dev);
  364. }
  365. /*
  366. * H/W pagetable operations
  367. */
  368. static void flush_iopte_range(struct device *dev, dma_addr_t dma,
  369. unsigned long offset, int num_entries)
  370. {
  371. size_t size = num_entries * sizeof(u32);
  372. dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE);
  373. }
  374. static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid)
  375. {
  376. dma_addr_t pt_dma;
  377. /* Note: freed iopte's must be clean ready for re-use */
  378. if (iopte) {
  379. if (dma_valid) {
  380. pt_dma = virt_to_phys(iopte);
  381. dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE,
  382. DMA_TO_DEVICE);
  383. }
  384. kmem_cache_free(iopte_cachep, iopte);
  385. }
  386. }
  387. static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
  388. dma_addr_t *pt_dma, u32 da)
  389. {
  390. u32 *iopte;
  391. unsigned long offset = iopgd_index(da) * sizeof(da);
  392. /* a table has already existed */
  393. if (*iopgd)
  394. goto pte_ready;
  395. /*
  396. * do the allocation outside the page table lock
  397. */
  398. spin_unlock(&obj->page_table_lock);
  399. iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
  400. spin_lock(&obj->page_table_lock);
  401. if (!*iopgd) {
  402. if (!iopte)
  403. return ERR_PTR(-ENOMEM);
  404. *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE,
  405. DMA_TO_DEVICE);
  406. if (dma_mapping_error(obj->dev, *pt_dma)) {
  407. dev_err(obj->dev, "DMA map error for L2 table\n");
  408. iopte_free(obj, iopte, false);
  409. return ERR_PTR(-ENOMEM);
  410. }
  411. /*
  412. * we rely on dma address and the physical address to be
  413. * the same for mapping the L2 table
  414. */
  415. if (WARN_ON(*pt_dma != virt_to_phys(iopte))) {
  416. dev_err(obj->dev, "DMA translation error for L2 table\n");
  417. dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE,
  418. DMA_TO_DEVICE);
  419. iopte_free(obj, iopte, false);
  420. return ERR_PTR(-ENOMEM);
  421. }
  422. *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
  423. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  424. dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
  425. } else {
  426. /* We raced, free the reduniovant table */
  427. iopte_free(obj, iopte, false);
  428. }
  429. pte_ready:
  430. iopte = iopte_offset(iopgd, da);
  431. *pt_dma = iopgd_page_paddr(iopgd);
  432. dev_vdbg(obj->dev,
  433. "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
  434. __func__, da, iopgd, *iopgd, iopte, *iopte);
  435. return iopte;
  436. }
  437. static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  438. {
  439. u32 *iopgd = iopgd_offset(obj, da);
  440. unsigned long offset = iopgd_index(da) * sizeof(da);
  441. if ((da | pa) & ~IOSECTION_MASK) {
  442. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  443. __func__, da, pa, IOSECTION_SIZE);
  444. return -EINVAL;
  445. }
  446. *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
  447. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  448. return 0;
  449. }
  450. static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  451. {
  452. u32 *iopgd = iopgd_offset(obj, da);
  453. unsigned long offset = iopgd_index(da) * sizeof(da);
  454. int i;
  455. if ((da | pa) & ~IOSUPER_MASK) {
  456. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  457. __func__, da, pa, IOSUPER_SIZE);
  458. return -EINVAL;
  459. }
  460. for (i = 0; i < 16; i++)
  461. *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
  462. flush_iopte_range(obj->dev, obj->pd_dma, offset, 16);
  463. return 0;
  464. }
  465. static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  466. {
  467. u32 *iopgd = iopgd_offset(obj, da);
  468. dma_addr_t pt_dma;
  469. u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
  470. unsigned long offset = iopte_index(da) * sizeof(da);
  471. if (IS_ERR(iopte))
  472. return PTR_ERR(iopte);
  473. *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
  474. flush_iopte_range(obj->dev, pt_dma, offset, 1);
  475. dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
  476. __func__, da, pa, iopte, *iopte);
  477. return 0;
  478. }
  479. static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  480. {
  481. u32 *iopgd = iopgd_offset(obj, da);
  482. dma_addr_t pt_dma;
  483. u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
  484. unsigned long offset = iopte_index(da) * sizeof(da);
  485. int i;
  486. if ((da | pa) & ~IOLARGE_MASK) {
  487. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  488. __func__, da, pa, IOLARGE_SIZE);
  489. return -EINVAL;
  490. }
  491. if (IS_ERR(iopte))
  492. return PTR_ERR(iopte);
  493. for (i = 0; i < 16; i++)
  494. *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
  495. flush_iopte_range(obj->dev, pt_dma, offset, 16);
  496. return 0;
  497. }
  498. static int
  499. iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
  500. {
  501. int (*fn)(struct omap_iommu *, u32, u32, u32);
  502. u32 prot;
  503. int err;
  504. if (!obj || !e)
  505. return -EINVAL;
  506. switch (e->pgsz) {
  507. case MMU_CAM_PGSZ_16M:
  508. fn = iopgd_alloc_super;
  509. break;
  510. case MMU_CAM_PGSZ_1M:
  511. fn = iopgd_alloc_section;
  512. break;
  513. case MMU_CAM_PGSZ_64K:
  514. fn = iopte_alloc_large;
  515. break;
  516. case MMU_CAM_PGSZ_4K:
  517. fn = iopte_alloc_page;
  518. break;
  519. default:
  520. fn = NULL;
  521. break;
  522. }
  523. if (WARN_ON(!fn))
  524. return -EINVAL;
  525. prot = get_iopte_attr(e);
  526. spin_lock(&obj->page_table_lock);
  527. err = fn(obj, e->da, e->pa, prot);
  528. spin_unlock(&obj->page_table_lock);
  529. return err;
  530. }
  531. /**
  532. * omap_iopgtable_store_entry - Make an iommu pte entry
  533. * @obj: target iommu
  534. * @e: an iommu tlb entry info
  535. **/
  536. static int
  537. omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  538. {
  539. int err;
  540. flush_iotlb_page(obj, e->da);
  541. err = iopgtable_store_entry_core(obj, e);
  542. if (!err)
  543. prefetch_iotlb_entry(obj, e);
  544. return err;
  545. }
  546. /**
  547. * iopgtable_lookup_entry - Lookup an iommu pte entry
  548. * @obj: target iommu
  549. * @da: iommu device virtual address
  550. * @ppgd: iommu pgd entry pointer to be returned
  551. * @ppte: iommu pte entry pointer to be returned
  552. **/
  553. static void
  554. iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
  555. {
  556. u32 *iopgd, *iopte = NULL;
  557. iopgd = iopgd_offset(obj, da);
  558. if (!*iopgd)
  559. goto out;
  560. if (iopgd_is_table(*iopgd))
  561. iopte = iopte_offset(iopgd, da);
  562. out:
  563. *ppgd = iopgd;
  564. *ppte = iopte;
  565. }
  566. static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
  567. {
  568. size_t bytes;
  569. u32 *iopgd = iopgd_offset(obj, da);
  570. int nent = 1;
  571. dma_addr_t pt_dma;
  572. unsigned long pd_offset = iopgd_index(da) * sizeof(da);
  573. unsigned long pt_offset = iopte_index(da) * sizeof(da);
  574. if (!*iopgd)
  575. return 0;
  576. if (iopgd_is_table(*iopgd)) {
  577. int i;
  578. u32 *iopte = iopte_offset(iopgd, da);
  579. bytes = IOPTE_SIZE;
  580. if (*iopte & IOPTE_LARGE) {
  581. nent *= 16;
  582. /* rewind to the 1st entry */
  583. iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
  584. }
  585. bytes *= nent;
  586. memset(iopte, 0, nent * sizeof(*iopte));
  587. pt_dma = iopgd_page_paddr(iopgd);
  588. flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
  589. /*
  590. * do table walk to check if this table is necessary or not
  591. */
  592. iopte = iopte_offset(iopgd, 0);
  593. for (i = 0; i < PTRS_PER_IOPTE; i++)
  594. if (iopte[i])
  595. goto out;
  596. iopte_free(obj, iopte, true);
  597. nent = 1; /* for the next L1 entry */
  598. } else {
  599. bytes = IOPGD_SIZE;
  600. if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
  601. nent *= 16;
  602. /* rewind to the 1st entry */
  603. iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
  604. }
  605. bytes *= nent;
  606. }
  607. memset(iopgd, 0, nent * sizeof(*iopgd));
  608. flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent);
  609. out:
  610. return bytes;
  611. }
  612. /**
  613. * iopgtable_clear_entry - Remove an iommu pte entry
  614. * @obj: target iommu
  615. * @da: iommu device virtual address
  616. **/
  617. static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
  618. {
  619. size_t bytes;
  620. spin_lock(&obj->page_table_lock);
  621. bytes = iopgtable_clear_entry_core(obj, da);
  622. flush_iotlb_page(obj, da);
  623. spin_unlock(&obj->page_table_lock);
  624. return bytes;
  625. }
  626. static void iopgtable_clear_entry_all(struct omap_iommu *obj)
  627. {
  628. unsigned long offset;
  629. int i;
  630. spin_lock(&obj->page_table_lock);
  631. for (i = 0; i < PTRS_PER_IOPGD; i++) {
  632. u32 da;
  633. u32 *iopgd;
  634. da = i << IOPGD_SHIFT;
  635. iopgd = iopgd_offset(obj, da);
  636. offset = iopgd_index(da) * sizeof(da);
  637. if (!*iopgd)
  638. continue;
  639. if (iopgd_is_table(*iopgd))
  640. iopte_free(obj, iopte_offset(iopgd, 0), true);
  641. *iopgd = 0;
  642. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  643. }
  644. flush_iotlb_all(obj);
  645. spin_unlock(&obj->page_table_lock);
  646. }
  647. /*
  648. * Device IOMMU generic operations
  649. */
  650. static irqreturn_t iommu_fault_handler(int irq, void *data)
  651. {
  652. u32 da, errs;
  653. u32 *iopgd, *iopte;
  654. struct omap_iommu *obj = data;
  655. struct iommu_domain *domain = obj->domain;
  656. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  657. if (!omap_domain->dev)
  658. return IRQ_NONE;
  659. errs = iommu_report_fault(obj, &da);
  660. if (errs == 0)
  661. return IRQ_HANDLED;
  662. /* Fault callback or TLB/PTE Dynamic loading */
  663. if (!report_iommu_fault(domain, obj->dev, da, 0))
  664. return IRQ_HANDLED;
  665. iommu_write_reg(obj, 0, MMU_IRQENABLE);
  666. iopgd = iopgd_offset(obj, da);
  667. if (!iopgd_is_table(*iopgd)) {
  668. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
  669. obj->name, errs, da, iopgd, *iopgd);
  670. return IRQ_NONE;
  671. }
  672. iopte = iopte_offset(iopgd, da);
  673. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
  674. obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
  675. return IRQ_NONE;
  676. }
  677. /**
  678. * omap_iommu_attach() - attach iommu device to an iommu domain
  679. * @obj: target omap iommu device
  680. * @iopgd: page table
  681. **/
  682. static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
  683. {
  684. int err;
  685. spin_lock(&obj->iommu_lock);
  686. obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE,
  687. DMA_TO_DEVICE);
  688. if (dma_mapping_error(obj->dev, obj->pd_dma)) {
  689. dev_err(obj->dev, "DMA map error for L1 table\n");
  690. err = -ENOMEM;
  691. goto out_err;
  692. }
  693. obj->iopgd = iopgd;
  694. err = iommu_enable(obj);
  695. if (err)
  696. goto out_err;
  697. flush_iotlb_all(obj);
  698. spin_unlock(&obj->iommu_lock);
  699. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  700. return 0;
  701. out_err:
  702. spin_unlock(&obj->iommu_lock);
  703. return err;
  704. }
  705. /**
  706. * omap_iommu_detach - release iommu device
  707. * @obj: target iommu
  708. **/
  709. static void omap_iommu_detach(struct omap_iommu *obj)
  710. {
  711. if (!obj || IS_ERR(obj))
  712. return;
  713. spin_lock(&obj->iommu_lock);
  714. dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
  715. DMA_TO_DEVICE);
  716. obj->pd_dma = 0;
  717. obj->iopgd = NULL;
  718. iommu_disable(obj);
  719. spin_unlock(&obj->iommu_lock);
  720. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  721. }
  722. static void omap_iommu_save_tlb_entries(struct omap_iommu *obj)
  723. {
  724. struct iotlb_lock lock;
  725. struct cr_regs cr;
  726. struct cr_regs *tmp;
  727. int i;
  728. /* check if there are any locked tlbs to save */
  729. iotlb_lock_get(obj, &lock);
  730. obj->num_cr_ctx = lock.base;
  731. if (!obj->num_cr_ctx)
  732. return;
  733. tmp = obj->cr_ctx;
  734. for_each_iotlb_cr(obj, obj->num_cr_ctx, i, cr)
  735. * tmp++ = cr;
  736. }
  737. static void omap_iommu_restore_tlb_entries(struct omap_iommu *obj)
  738. {
  739. struct iotlb_lock l;
  740. struct cr_regs *tmp;
  741. int i;
  742. /* no locked tlbs to restore */
  743. if (!obj->num_cr_ctx)
  744. return;
  745. l.base = 0;
  746. tmp = obj->cr_ctx;
  747. for (i = 0; i < obj->num_cr_ctx; i++, tmp++) {
  748. l.vict = i;
  749. iotlb_lock_set(obj, &l);
  750. iotlb_load_cr(obj, tmp);
  751. }
  752. l.base = obj->num_cr_ctx;
  753. l.vict = i;
  754. iotlb_lock_set(obj, &l);
  755. }
  756. /**
  757. * omap_iommu_domain_deactivate - deactivate attached iommu devices
  758. * @domain: iommu domain attached to the target iommu device
  759. *
  760. * This API allows the client devices of IOMMU devices to suspend
  761. * the IOMMUs they control at runtime, after they are idled and
  762. * suspended all activity. System Suspend will leverage the PM
  763. * driver late callbacks.
  764. **/
  765. int omap_iommu_domain_deactivate(struct iommu_domain *domain)
  766. {
  767. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  768. struct omap_iommu_device *iommu;
  769. struct omap_iommu *oiommu;
  770. int i;
  771. if (!omap_domain->dev)
  772. return 0;
  773. iommu = omap_domain->iommus;
  774. iommu += (omap_domain->num_iommus - 1);
  775. for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {
  776. oiommu = iommu->iommu_dev;
  777. pm_runtime_put_sync(oiommu->dev);
  778. }
  779. return 0;
  780. }
  781. EXPORT_SYMBOL_GPL(omap_iommu_domain_deactivate);
  782. /**
  783. * omap_iommu_domain_activate - activate attached iommu devices
  784. * @domain: iommu domain attached to the target iommu device
  785. *
  786. * This API allows the client devices of IOMMU devices to resume the
  787. * IOMMUs they control at runtime, before they can resume operations.
  788. * System Resume will leverage the PM driver late callbacks.
  789. **/
  790. int omap_iommu_domain_activate(struct iommu_domain *domain)
  791. {
  792. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  793. struct omap_iommu_device *iommu;
  794. struct omap_iommu *oiommu;
  795. int i;
  796. if (!omap_domain->dev)
  797. return 0;
  798. iommu = omap_domain->iommus;
  799. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  800. oiommu = iommu->iommu_dev;
  801. pm_runtime_get_sync(oiommu->dev);
  802. }
  803. return 0;
  804. }
  805. EXPORT_SYMBOL_GPL(omap_iommu_domain_activate);
  806. /**
  807. * omap_iommu_runtime_suspend - disable an iommu device
  808. * @dev: iommu device
  809. *
  810. * This function performs all that is necessary to disable an
  811. * IOMMU device, either during final detachment from a client
  812. * device, or during system/runtime suspend of the device. This
  813. * includes programming all the appropriate IOMMU registers, and
  814. * managing the associated omap_hwmod's state and the device's
  815. * reset line. This function also saves the context of any
  816. * locked TLBs if suspending.
  817. **/
  818. static __maybe_unused int omap_iommu_runtime_suspend(struct device *dev)
  819. {
  820. struct platform_device *pdev = to_platform_device(dev);
  821. struct iommu_platform_data *pdata = dev_get_platdata(dev);
  822. struct omap_iommu *obj = to_iommu(dev);
  823. int ret;
  824. /* save the TLBs only during suspend, and not for power down */
  825. if (obj->domain && obj->iopgd)
  826. omap_iommu_save_tlb_entries(obj);
  827. omap2_iommu_disable(obj);
  828. if (pdata && pdata->device_idle)
  829. pdata->device_idle(pdev);
  830. if (pdata && pdata->assert_reset)
  831. pdata->assert_reset(pdev, pdata->reset_name);
  832. if (pdata && pdata->set_pwrdm_constraint) {
  833. ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst);
  834. if (ret) {
  835. dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n",
  836. ret);
  837. }
  838. }
  839. return 0;
  840. }
  841. /**
  842. * omap_iommu_runtime_resume - enable an iommu device
  843. * @dev: iommu device
  844. *
  845. * This function performs all that is necessary to enable an
  846. * IOMMU device, either during initial attachment to a client
  847. * device, or during system/runtime resume of the device. This
  848. * includes programming all the appropriate IOMMU registers, and
  849. * managing the associated omap_hwmod's state and the device's
  850. * reset line. The function also restores any locked TLBs if
  851. * resuming after a suspend.
  852. **/
  853. static __maybe_unused int omap_iommu_runtime_resume(struct device *dev)
  854. {
  855. struct platform_device *pdev = to_platform_device(dev);
  856. struct iommu_platform_data *pdata = dev_get_platdata(dev);
  857. struct omap_iommu *obj = to_iommu(dev);
  858. int ret = 0;
  859. if (pdata && pdata->set_pwrdm_constraint) {
  860. ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst);
  861. if (ret) {
  862. dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n",
  863. ret);
  864. }
  865. }
  866. if (pdata && pdata->deassert_reset) {
  867. ret = pdata->deassert_reset(pdev, pdata->reset_name);
  868. if (ret) {
  869. dev_err(dev, "deassert_reset failed: %d\n", ret);
  870. return ret;
  871. }
  872. }
  873. if (pdata && pdata->device_enable)
  874. pdata->device_enable(pdev);
  875. /* restore the TLBs only during resume, and not for power up */
  876. if (obj->domain)
  877. omap_iommu_restore_tlb_entries(obj);
  878. ret = omap2_iommu_enable(obj);
  879. return ret;
  880. }
  881. /**
  882. * omap_iommu_suspend_prepare - prepare() dev_pm_ops implementation
  883. * @dev: iommu device
  884. *
  885. * This function performs the necessary checks to determine if the IOMMU
  886. * device needs suspending or not. The function checks if the runtime_pm
  887. * status of the device is suspended, and returns 1 in that case. This
  888. * results in the PM core to skip invoking any of the Sleep PM callbacks
  889. * (suspend, suspend_late, resume, resume_early etc).
  890. */
  891. static int omap_iommu_prepare(struct device *dev)
  892. {
  893. if (pm_runtime_status_suspended(dev))
  894. return 1;
  895. return 0;
  896. }
  897. static bool omap_iommu_can_register(struct platform_device *pdev)
  898. {
  899. struct device_node *np = pdev->dev.of_node;
  900. if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
  901. return true;
  902. /*
  903. * restrict IOMMU core registration only for processor-port MDMA MMUs
  904. * on DRA7 DSPs
  905. */
  906. if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) ||
  907. (!strcmp(dev_name(&pdev->dev), "41501000.mmu")))
  908. return true;
  909. return false;
  910. }
  911. static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev,
  912. struct omap_iommu *obj)
  913. {
  914. struct device_node *np = pdev->dev.of_node;
  915. int ret;
  916. if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
  917. return 0;
  918. if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) {
  919. dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n");
  920. return -EINVAL;
  921. }
  922. obj->syscfg =
  923. syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig");
  924. if (IS_ERR(obj->syscfg)) {
  925. /* can fail with -EPROBE_DEFER */
  926. ret = PTR_ERR(obj->syscfg);
  927. return ret;
  928. }
  929. if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1,
  930. &obj->id)) {
  931. dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n");
  932. return -EINVAL;
  933. }
  934. if (obj->id != 0 && obj->id != 1) {
  935. dev_err(&pdev->dev, "invalid IOMMU instance id\n");
  936. return -EINVAL;
  937. }
  938. return 0;
  939. }
  940. /*
  941. * OMAP Device MMU(IOMMU) detection
  942. */
  943. static int omap_iommu_probe(struct platform_device *pdev)
  944. {
  945. int err = -ENODEV;
  946. int irq;
  947. struct omap_iommu *obj;
  948. struct resource *res;
  949. struct device_node *of = pdev->dev.of_node;
  950. if (!of) {
  951. pr_err("%s: only DT-based devices are supported\n", __func__);
  952. return -ENODEV;
  953. }
  954. obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
  955. if (!obj)
  956. return -ENOMEM;
  957. /*
  958. * self-manage the ordering dependencies between omap_device_enable/idle
  959. * and omap_device_assert/deassert_hardreset API
  960. */
  961. if (pdev->dev.pm_domain) {
  962. dev_dbg(&pdev->dev, "device pm_domain is being reset\n");
  963. pdev->dev.pm_domain = NULL;
  964. }
  965. obj->name = dev_name(&pdev->dev);
  966. obj->nr_tlb_entries = 32;
  967. err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries);
  968. if (err && err != -EINVAL)
  969. return err;
  970. if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
  971. return -EINVAL;
  972. if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
  973. obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
  974. obj->dev = &pdev->dev;
  975. obj->ctx = (void *)obj + sizeof(*obj);
  976. obj->cr_ctx = devm_kzalloc(&pdev->dev,
  977. sizeof(*obj->cr_ctx) * obj->nr_tlb_entries,
  978. GFP_KERNEL);
  979. if (!obj->cr_ctx)
  980. return -ENOMEM;
  981. spin_lock_init(&obj->iommu_lock);
  982. spin_lock_init(&obj->page_table_lock);
  983. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  984. obj->regbase = devm_ioremap_resource(obj->dev, res);
  985. if (IS_ERR(obj->regbase))
  986. return PTR_ERR(obj->regbase);
  987. err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj);
  988. if (err)
  989. return err;
  990. irq = platform_get_irq(pdev, 0);
  991. if (irq < 0)
  992. return -ENODEV;
  993. err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
  994. dev_name(obj->dev), obj);
  995. if (err < 0)
  996. return err;
  997. platform_set_drvdata(pdev, obj);
  998. if (omap_iommu_can_register(pdev)) {
  999. obj->group = iommu_group_alloc();
  1000. if (IS_ERR(obj->group))
  1001. return PTR_ERR(obj->group);
  1002. err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL,
  1003. obj->name);
  1004. if (err)
  1005. goto out_group;
  1006. iommu_device_set_ops(&obj->iommu, &omap_iommu_ops);
  1007. iommu_device_set_fwnode(&obj->iommu, &of->fwnode);
  1008. err = iommu_device_register(&obj->iommu);
  1009. if (err)
  1010. goto out_sysfs;
  1011. }
  1012. pm_runtime_enable(obj->dev);
  1013. omap_iommu_debugfs_add(obj);
  1014. dev_info(&pdev->dev, "%s registered\n", obj->name);
  1015. /* Re-probe bus to probe device attached to this IOMMU */
  1016. bus_iommu_probe(&platform_bus_type);
  1017. return 0;
  1018. out_sysfs:
  1019. iommu_device_sysfs_remove(&obj->iommu);
  1020. out_group:
  1021. iommu_group_put(obj->group);
  1022. return err;
  1023. }
  1024. static int omap_iommu_remove(struct platform_device *pdev)
  1025. {
  1026. struct omap_iommu *obj = platform_get_drvdata(pdev);
  1027. if (obj->group) {
  1028. iommu_group_put(obj->group);
  1029. obj->group = NULL;
  1030. iommu_device_sysfs_remove(&obj->iommu);
  1031. iommu_device_unregister(&obj->iommu);
  1032. }
  1033. omap_iommu_debugfs_remove(obj);
  1034. pm_runtime_disable(obj->dev);
  1035. dev_info(&pdev->dev, "%s removed\n", obj->name);
  1036. return 0;
  1037. }
  1038. static const struct dev_pm_ops omap_iommu_pm_ops = {
  1039. .prepare = omap_iommu_prepare,
  1040. SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
  1041. pm_runtime_force_resume)
  1042. SET_RUNTIME_PM_OPS(omap_iommu_runtime_suspend,
  1043. omap_iommu_runtime_resume, NULL)
  1044. };
  1045. static const struct of_device_id omap_iommu_of_match[] = {
  1046. { .compatible = "ti,omap2-iommu" },
  1047. { .compatible = "ti,omap4-iommu" },
  1048. { .compatible = "ti,dra7-iommu" },
  1049. { .compatible = "ti,dra7-dsp-iommu" },
  1050. {},
  1051. };
  1052. static struct platform_driver omap_iommu_driver = {
  1053. .probe = omap_iommu_probe,
  1054. .remove = omap_iommu_remove,
  1055. .driver = {
  1056. .name = "omap-iommu",
  1057. .pm = &omap_iommu_pm_ops,
  1058. .of_match_table = of_match_ptr(omap_iommu_of_match),
  1059. },
  1060. };
  1061. static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
  1062. {
  1063. memset(e, 0, sizeof(*e));
  1064. e->da = da;
  1065. e->pa = pa;
  1066. e->valid = MMU_CAM_V;
  1067. e->pgsz = pgsz;
  1068. e->endian = MMU_RAM_ENDIAN_LITTLE;
  1069. e->elsz = MMU_RAM_ELSZ_8;
  1070. e->mixed = 0;
  1071. return iopgsz_to_bytes(e->pgsz);
  1072. }
  1073. static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
  1074. phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
  1075. {
  1076. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1077. struct device *dev = omap_domain->dev;
  1078. struct omap_iommu_device *iommu;
  1079. struct omap_iommu *oiommu;
  1080. struct iotlb_entry e;
  1081. int omap_pgsz;
  1082. u32 ret = -EINVAL;
  1083. int i;
  1084. omap_pgsz = bytes_to_iopgsz(bytes);
  1085. if (omap_pgsz < 0) {
  1086. dev_err(dev, "invalid size to map: %zu\n", bytes);
  1087. return -EINVAL;
  1088. }
  1089. dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%zx\n", da, &pa, bytes);
  1090. iotlb_init_entry(&e, da, pa, omap_pgsz);
  1091. iommu = omap_domain->iommus;
  1092. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  1093. oiommu = iommu->iommu_dev;
  1094. ret = omap_iopgtable_store_entry(oiommu, &e);
  1095. if (ret) {
  1096. dev_err(dev, "omap_iopgtable_store_entry failed: %d\n",
  1097. ret);
  1098. break;
  1099. }
  1100. }
  1101. if (ret) {
  1102. while (i--) {
  1103. iommu--;
  1104. oiommu = iommu->iommu_dev;
  1105. iopgtable_clear_entry(oiommu, da);
  1106. }
  1107. }
  1108. return ret;
  1109. }
  1110. static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
  1111. size_t size, struct iommu_iotlb_gather *gather)
  1112. {
  1113. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1114. struct device *dev = omap_domain->dev;
  1115. struct omap_iommu_device *iommu;
  1116. struct omap_iommu *oiommu;
  1117. bool error = false;
  1118. size_t bytes = 0;
  1119. int i;
  1120. dev_dbg(dev, "unmapping da 0x%lx size %zu\n", da, size);
  1121. iommu = omap_domain->iommus;
  1122. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  1123. oiommu = iommu->iommu_dev;
  1124. bytes = iopgtable_clear_entry(oiommu, da);
  1125. if (!bytes)
  1126. error = true;
  1127. }
  1128. /*
  1129. * simplify return - we are only checking if any of the iommus
  1130. * reported an error, but not if all of them are unmapping the
  1131. * same number of entries. This should not occur due to the
  1132. * mirror programming.
  1133. */
  1134. return error ? 0 : bytes;
  1135. }
  1136. static int omap_iommu_count(struct device *dev)
  1137. {
  1138. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1139. int count = 0;
  1140. while (arch_data->iommu_dev) {
  1141. count++;
  1142. arch_data++;
  1143. }
  1144. return count;
  1145. }
  1146. /* caller should call cleanup if this function fails */
  1147. static int omap_iommu_attach_init(struct device *dev,
  1148. struct omap_iommu_domain *odomain)
  1149. {
  1150. struct omap_iommu_device *iommu;
  1151. int i;
  1152. odomain->num_iommus = omap_iommu_count(dev);
  1153. if (!odomain->num_iommus)
  1154. return -EINVAL;
  1155. odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),
  1156. GFP_ATOMIC);
  1157. if (!odomain->iommus)
  1158. return -ENOMEM;
  1159. iommu = odomain->iommus;
  1160. for (i = 0; i < odomain->num_iommus; i++, iommu++) {
  1161. iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC);
  1162. if (!iommu->pgtable)
  1163. return -ENOMEM;
  1164. /*
  1165. * should never fail, but please keep this around to ensure
  1166. * we keep the hardware happy
  1167. */
  1168. if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable,
  1169. IOPGD_TABLE_SIZE)))
  1170. return -EINVAL;
  1171. }
  1172. return 0;
  1173. }
  1174. static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain)
  1175. {
  1176. int i;
  1177. struct omap_iommu_device *iommu = odomain->iommus;
  1178. for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++)
  1179. kfree(iommu->pgtable);
  1180. kfree(odomain->iommus);
  1181. odomain->num_iommus = 0;
  1182. odomain->iommus = NULL;
  1183. }
  1184. static int
  1185. omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
  1186. {
  1187. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1188. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1189. struct omap_iommu_device *iommu;
  1190. struct omap_iommu *oiommu;
  1191. int ret = 0;
  1192. int i;
  1193. if (!arch_data || !arch_data->iommu_dev) {
  1194. dev_err(dev, "device doesn't have an associated iommu\n");
  1195. return -EINVAL;
  1196. }
  1197. spin_lock(&omap_domain->lock);
  1198. /* only a single client device can be attached to a domain */
  1199. if (omap_domain->dev) {
  1200. dev_err(dev, "iommu domain is already attached\n");
  1201. ret = -EBUSY;
  1202. goto out;
  1203. }
  1204. ret = omap_iommu_attach_init(dev, omap_domain);
  1205. if (ret) {
  1206. dev_err(dev, "failed to allocate required iommu data %d\n",
  1207. ret);
  1208. goto init_fail;
  1209. }
  1210. iommu = omap_domain->iommus;
  1211. for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) {
  1212. /* configure and enable the omap iommu */
  1213. oiommu = arch_data->iommu_dev;
  1214. ret = omap_iommu_attach(oiommu, iommu->pgtable);
  1215. if (ret) {
  1216. dev_err(dev, "can't get omap iommu: %d\n", ret);
  1217. goto attach_fail;
  1218. }
  1219. oiommu->domain = domain;
  1220. iommu->iommu_dev = oiommu;
  1221. }
  1222. omap_domain->dev = dev;
  1223. goto out;
  1224. attach_fail:
  1225. while (i--) {
  1226. iommu--;
  1227. arch_data--;
  1228. oiommu = iommu->iommu_dev;
  1229. omap_iommu_detach(oiommu);
  1230. iommu->iommu_dev = NULL;
  1231. oiommu->domain = NULL;
  1232. }
  1233. init_fail:
  1234. omap_iommu_detach_fini(omap_domain);
  1235. out:
  1236. spin_unlock(&omap_domain->lock);
  1237. return ret;
  1238. }
  1239. static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
  1240. struct device *dev)
  1241. {
  1242. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1243. struct omap_iommu_device *iommu = omap_domain->iommus;
  1244. struct omap_iommu *oiommu;
  1245. int i;
  1246. if (!omap_domain->dev) {
  1247. dev_err(dev, "domain has no attached device\n");
  1248. return;
  1249. }
  1250. /* only a single device is supported per domain for now */
  1251. if (omap_domain->dev != dev) {
  1252. dev_err(dev, "invalid attached device\n");
  1253. return;
  1254. }
  1255. /*
  1256. * cleanup in the reverse order of attachment - this addresses
  1257. * any h/w dependencies between multiple instances, if any
  1258. */
  1259. iommu += (omap_domain->num_iommus - 1);
  1260. arch_data += (omap_domain->num_iommus - 1);
  1261. for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) {
  1262. oiommu = iommu->iommu_dev;
  1263. iopgtable_clear_entry_all(oiommu);
  1264. omap_iommu_detach(oiommu);
  1265. iommu->iommu_dev = NULL;
  1266. oiommu->domain = NULL;
  1267. }
  1268. omap_iommu_detach_fini(omap_domain);
  1269. omap_domain->dev = NULL;
  1270. }
  1271. static void omap_iommu_detach_dev(struct iommu_domain *domain,
  1272. struct device *dev)
  1273. {
  1274. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1275. spin_lock(&omap_domain->lock);
  1276. _omap_iommu_detach_dev(omap_domain, dev);
  1277. spin_unlock(&omap_domain->lock);
  1278. }
  1279. static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
  1280. {
  1281. struct omap_iommu_domain *omap_domain;
  1282. if (type != IOMMU_DOMAIN_UNMANAGED)
  1283. return NULL;
  1284. omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
  1285. if (!omap_domain)
  1286. return NULL;
  1287. spin_lock_init(&omap_domain->lock);
  1288. omap_domain->domain.geometry.aperture_start = 0;
  1289. omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1;
  1290. omap_domain->domain.geometry.force_aperture = true;
  1291. return &omap_domain->domain;
  1292. }
  1293. static void omap_iommu_domain_free(struct iommu_domain *domain)
  1294. {
  1295. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1296. /*
  1297. * An iommu device is still attached
  1298. * (currently, only one device can be attached) ?
  1299. */
  1300. if (omap_domain->dev)
  1301. _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
  1302. kfree(omap_domain);
  1303. }
  1304. static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
  1305. dma_addr_t da)
  1306. {
  1307. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1308. struct omap_iommu_device *iommu = omap_domain->iommus;
  1309. struct omap_iommu *oiommu = iommu->iommu_dev;
  1310. struct device *dev = oiommu->dev;
  1311. u32 *pgd, *pte;
  1312. phys_addr_t ret = 0;
  1313. /*
  1314. * all the iommus within the domain will have identical programming,
  1315. * so perform the lookup using just the first iommu
  1316. */
  1317. iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
  1318. if (pte) {
  1319. if (iopte_is_small(*pte))
  1320. ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
  1321. else if (iopte_is_large(*pte))
  1322. ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
  1323. else
  1324. dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
  1325. (unsigned long long)da);
  1326. } else {
  1327. if (iopgd_is_section(*pgd))
  1328. ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
  1329. else if (iopgd_is_super(*pgd))
  1330. ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
  1331. else
  1332. dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
  1333. (unsigned long long)da);
  1334. }
  1335. return ret;
  1336. }
  1337. static struct iommu_device *omap_iommu_probe_device(struct device *dev)
  1338. {
  1339. struct omap_iommu_arch_data *arch_data, *tmp;
  1340. struct platform_device *pdev;
  1341. struct omap_iommu *oiommu;
  1342. struct device_node *np;
  1343. int num_iommus, i;
  1344. /*
  1345. * Allocate the per-device iommu structure for DT-based devices.
  1346. *
  1347. * TODO: Simplify this when removing non-DT support completely from the
  1348. * IOMMU users.
  1349. */
  1350. if (!dev->of_node)
  1351. return ERR_PTR(-ENODEV);
  1352. /*
  1353. * retrieve the count of IOMMU nodes using phandle size as element size
  1354. * since #iommu-cells = 0 for OMAP
  1355. */
  1356. num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus",
  1357. sizeof(phandle));
  1358. if (num_iommus < 0)
  1359. return ERR_PTR(-ENODEV);
  1360. arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL);
  1361. if (!arch_data)
  1362. return ERR_PTR(-ENOMEM);
  1363. for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) {
  1364. np = of_parse_phandle(dev->of_node, "iommus", i);
  1365. if (!np) {
  1366. kfree(arch_data);
  1367. return ERR_PTR(-EINVAL);
  1368. }
  1369. pdev = of_find_device_by_node(np);
  1370. if (!pdev) {
  1371. of_node_put(np);
  1372. kfree(arch_data);
  1373. return ERR_PTR(-ENODEV);
  1374. }
  1375. oiommu = platform_get_drvdata(pdev);
  1376. if (!oiommu) {
  1377. of_node_put(np);
  1378. kfree(arch_data);
  1379. return ERR_PTR(-EINVAL);
  1380. }
  1381. tmp->iommu_dev = oiommu;
  1382. tmp->dev = &pdev->dev;
  1383. of_node_put(np);
  1384. }
  1385. dev_iommu_priv_set(dev, arch_data);
  1386. /*
  1387. * use the first IOMMU alone for the sysfs device linking.
  1388. * TODO: Evaluate if a single iommu_group needs to be
  1389. * maintained for both IOMMUs
  1390. */
  1391. oiommu = arch_data->iommu_dev;
  1392. return &oiommu->iommu;
  1393. }
  1394. static void omap_iommu_release_device(struct device *dev)
  1395. {
  1396. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1397. if (!dev->of_node || !arch_data)
  1398. return;
  1399. dev_iommu_priv_set(dev, NULL);
  1400. kfree(arch_data);
  1401. }
  1402. static struct iommu_group *omap_iommu_device_group(struct device *dev)
  1403. {
  1404. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1405. struct iommu_group *group = ERR_PTR(-EINVAL);
  1406. if (!arch_data)
  1407. return ERR_PTR(-ENODEV);
  1408. if (arch_data->iommu_dev)
  1409. group = iommu_group_ref_get(arch_data->iommu_dev->group);
  1410. return group;
  1411. }
  1412. static const struct iommu_ops omap_iommu_ops = {
  1413. .domain_alloc = omap_iommu_domain_alloc,
  1414. .domain_free = omap_iommu_domain_free,
  1415. .attach_dev = omap_iommu_attach_dev,
  1416. .detach_dev = omap_iommu_detach_dev,
  1417. .map = omap_iommu_map,
  1418. .unmap = omap_iommu_unmap,
  1419. .iova_to_phys = omap_iommu_iova_to_phys,
  1420. .probe_device = omap_iommu_probe_device,
  1421. .release_device = omap_iommu_release_device,
  1422. .device_group = omap_iommu_device_group,
  1423. .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
  1424. };
  1425. static int __init omap_iommu_init(void)
  1426. {
  1427. struct kmem_cache *p;
  1428. const slab_flags_t flags = SLAB_HWCACHE_ALIGN;
  1429. size_t align = 1 << 10; /* L2 pagetable alignement */
  1430. struct device_node *np;
  1431. int ret;
  1432. np = of_find_matching_node(NULL, omap_iommu_of_match);
  1433. if (!np)
  1434. return 0;
  1435. of_node_put(np);
  1436. p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
  1437. NULL);
  1438. if (!p)
  1439. return -ENOMEM;
  1440. iopte_cachep = p;
  1441. omap_iommu_debugfs_init();
  1442. ret = platform_driver_register(&omap_iommu_driver);
  1443. if (ret) {
  1444. pr_err("%s: failed to register driver\n", __func__);
  1445. goto fail_driver;
  1446. }
  1447. ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops);
  1448. if (ret)
  1449. goto fail_bus;
  1450. return 0;
  1451. fail_bus:
  1452. platform_driver_unregister(&omap_iommu_driver);
  1453. fail_driver:
  1454. kmem_cache_destroy(iopte_cachep);
  1455. return ret;
  1456. }
  1457. subsys_initcall(omap_iommu_init);
  1458. /* must be ready before omap3isp is probed */