tegra-smmu.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
  4. */
  5. #include <linux/bitops.h>
  6. #include <linux/debugfs.h>
  7. #include <linux/err.h>
  8. #include <linux/iommu.h>
  9. #include <linux/kernel.h>
  10. #include <linux/of.h>
  11. #include <linux/of_device.h>
  12. #include <linux/platform_device.h>
  13. #include <linux/slab.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/dma-mapping.h>
  16. #include <soc/tegra/ahb.h>
  17. #include <soc/tegra/mc.h>
  18. struct tegra_smmu_group {
  19. struct list_head list;
  20. struct tegra_smmu *smmu;
  21. const struct tegra_smmu_group_soc *soc;
  22. struct iommu_group *group;
  23. unsigned int swgroup;
  24. };
  25. struct tegra_smmu {
  26. void __iomem *regs;
  27. struct device *dev;
  28. struct tegra_mc *mc;
  29. const struct tegra_smmu_soc *soc;
  30. struct list_head groups;
  31. unsigned long pfn_mask;
  32. unsigned long tlb_mask;
  33. unsigned long *asids;
  34. struct mutex lock;
  35. struct list_head list;
  36. struct dentry *debugfs;
  37. struct iommu_device iommu; /* IOMMU Core code handle */
  38. };
  39. struct tegra_smmu_as {
  40. struct iommu_domain domain;
  41. struct tegra_smmu *smmu;
  42. unsigned int use_count;
  43. spinlock_t lock;
  44. u32 *count;
  45. struct page **pts;
  46. struct page *pd;
  47. dma_addr_t pd_dma;
  48. unsigned id;
  49. u32 attr;
  50. };
  51. static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
  52. {
  53. return container_of(dom, struct tegra_smmu_as, domain);
  54. }
  55. static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
  56. unsigned long offset)
  57. {
  58. writel(value, smmu->regs + offset);
  59. }
  60. static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
  61. {
  62. return readl(smmu->regs + offset);
  63. }
  64. #define SMMU_CONFIG 0x010
  65. #define SMMU_CONFIG_ENABLE (1 << 0)
  66. #define SMMU_TLB_CONFIG 0x14
  67. #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
  68. #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
  69. #define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
  70. ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
  71. #define SMMU_PTC_CONFIG 0x18
  72. #define SMMU_PTC_CONFIG_ENABLE (1 << 29)
  73. #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
  74. #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
  75. #define SMMU_PTB_ASID 0x01c
  76. #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
  77. #define SMMU_PTB_DATA 0x020
  78. #define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
  79. #define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
  80. #define SMMU_TLB_FLUSH 0x030
  81. #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
  82. #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
  83. #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
  84. #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
  85. SMMU_TLB_FLUSH_VA_MATCH_SECTION)
  86. #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
  87. SMMU_TLB_FLUSH_VA_MATCH_GROUP)
  88. #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
  89. #define SMMU_PTC_FLUSH 0x034
  90. #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
  91. #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
  92. #define SMMU_PTC_FLUSH_HI 0x9b8
  93. #define SMMU_PTC_FLUSH_HI_MASK 0x3
  94. /* per-SWGROUP SMMU_*_ASID register */
  95. #define SMMU_ASID_ENABLE (1 << 31)
  96. #define SMMU_ASID_MASK 0x7f
  97. #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
  98. /* page table definitions */
  99. #define SMMU_NUM_PDE 1024
  100. #define SMMU_NUM_PTE 1024
  101. #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
  102. #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
  103. #define SMMU_PDE_SHIFT 22
  104. #define SMMU_PTE_SHIFT 12
  105. #define SMMU_PAGE_MASK (~(SMMU_SIZE_PT-1))
  106. #define SMMU_OFFSET_IN_PAGE(x) ((unsigned long)(x) & ~SMMU_PAGE_MASK)
  107. #define SMMU_PFN_PHYS(x) ((phys_addr_t)(x) << SMMU_PTE_SHIFT)
  108. #define SMMU_PHYS_PFN(x) ((unsigned long)((x) >> SMMU_PTE_SHIFT))
  109. #define SMMU_PD_READABLE (1 << 31)
  110. #define SMMU_PD_WRITABLE (1 << 30)
  111. #define SMMU_PD_NONSECURE (1 << 29)
  112. #define SMMU_PDE_READABLE (1 << 31)
  113. #define SMMU_PDE_WRITABLE (1 << 30)
  114. #define SMMU_PDE_NONSECURE (1 << 29)
  115. #define SMMU_PDE_NEXT (1 << 28)
  116. #define SMMU_PTE_READABLE (1 << 31)
  117. #define SMMU_PTE_WRITABLE (1 << 30)
  118. #define SMMU_PTE_NONSECURE (1 << 29)
  119. #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
  120. SMMU_PDE_NONSECURE)
  121. static unsigned int iova_pd_index(unsigned long iova)
  122. {
  123. return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
  124. }
  125. static unsigned int iova_pt_index(unsigned long iova)
  126. {
  127. return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
  128. }
  129. static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
  130. {
  131. addr >>= 12;
  132. return (addr & smmu->pfn_mask) == addr;
  133. }
  134. static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde)
  135. {
  136. return (dma_addr_t)(pde & smmu->pfn_mask) << 12;
  137. }
  138. static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
  139. {
  140. smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
  141. }
  142. static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
  143. unsigned long offset)
  144. {
  145. u32 value;
  146. offset &= ~(smmu->mc->soc->atom_size - 1);
  147. if (smmu->mc->soc->num_address_bits > 32) {
  148. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  149. value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
  150. #else
  151. value = 0;
  152. #endif
  153. smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
  154. }
  155. value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
  156. smmu_writel(smmu, value, SMMU_PTC_FLUSH);
  157. }
  158. static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
  159. {
  160. smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
  161. }
  162. static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
  163. unsigned long asid)
  164. {
  165. u32 value;
  166. if (smmu->soc->num_asids == 4)
  167. value = (asid & 0x3) << 29;
  168. else
  169. value = (asid & 0x7f) << 24;
  170. value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL;
  171. smmu_writel(smmu, value, SMMU_TLB_FLUSH);
  172. }
  173. static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
  174. unsigned long asid,
  175. unsigned long iova)
  176. {
  177. u32 value;
  178. if (smmu->soc->num_asids == 4)
  179. value = (asid & 0x3) << 29;
  180. else
  181. value = (asid & 0x7f) << 24;
  182. value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova);
  183. smmu_writel(smmu, value, SMMU_TLB_FLUSH);
  184. }
  185. static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
  186. unsigned long asid,
  187. unsigned long iova)
  188. {
  189. u32 value;
  190. if (smmu->soc->num_asids == 4)
  191. value = (asid & 0x3) << 29;
  192. else
  193. value = (asid & 0x7f) << 24;
  194. value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova);
  195. smmu_writel(smmu, value, SMMU_TLB_FLUSH);
  196. }
  197. static inline void smmu_flush(struct tegra_smmu *smmu)
  198. {
  199. smmu_readl(smmu, SMMU_PTB_ASID);
  200. }
  201. static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
  202. {
  203. unsigned long id;
  204. mutex_lock(&smmu->lock);
  205. id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
  206. if (id >= smmu->soc->num_asids) {
  207. mutex_unlock(&smmu->lock);
  208. return -ENOSPC;
  209. }
  210. set_bit(id, smmu->asids);
  211. *idp = id;
  212. mutex_unlock(&smmu->lock);
  213. return 0;
  214. }
  215. static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
  216. {
  217. mutex_lock(&smmu->lock);
  218. clear_bit(id, smmu->asids);
  219. mutex_unlock(&smmu->lock);
  220. }
  221. static bool tegra_smmu_capable(enum iommu_cap cap)
  222. {
  223. return false;
  224. }
  225. static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
  226. {
  227. struct tegra_smmu_as *as;
  228. if (type != IOMMU_DOMAIN_UNMANAGED)
  229. return NULL;
  230. as = kzalloc(sizeof(*as), GFP_KERNEL);
  231. if (!as)
  232. return NULL;
  233. as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
  234. as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
  235. if (!as->pd) {
  236. kfree(as);
  237. return NULL;
  238. }
  239. as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
  240. if (!as->count) {
  241. __free_page(as->pd);
  242. kfree(as);
  243. return NULL;
  244. }
  245. as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
  246. if (!as->pts) {
  247. kfree(as->count);
  248. __free_page(as->pd);
  249. kfree(as);
  250. return NULL;
  251. }
  252. spin_lock_init(&as->lock);
  253. /* setup aperture */
  254. as->domain.geometry.aperture_start = 0;
  255. as->domain.geometry.aperture_end = 0xffffffff;
  256. as->domain.geometry.force_aperture = true;
  257. return &as->domain;
  258. }
  259. static void tegra_smmu_domain_free(struct iommu_domain *domain)
  260. {
  261. struct tegra_smmu_as *as = to_smmu_as(domain);
  262. /* TODO: free page directory and page tables */
  263. WARN_ON_ONCE(as->use_count);
  264. kfree(as->count);
  265. kfree(as->pts);
  266. kfree(as);
  267. }
  268. static const struct tegra_smmu_swgroup *
  269. tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
  270. {
  271. const struct tegra_smmu_swgroup *group = NULL;
  272. unsigned int i;
  273. for (i = 0; i < smmu->soc->num_swgroups; i++) {
  274. if (smmu->soc->swgroups[i].swgroup == swgroup) {
  275. group = &smmu->soc->swgroups[i];
  276. break;
  277. }
  278. }
  279. return group;
  280. }
  281. static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
  282. unsigned int asid)
  283. {
  284. const struct tegra_smmu_swgroup *group;
  285. unsigned int i;
  286. u32 value;
  287. group = tegra_smmu_find_swgroup(smmu, swgroup);
  288. if (group) {
  289. value = smmu_readl(smmu, group->reg);
  290. value &= ~SMMU_ASID_MASK;
  291. value |= SMMU_ASID_VALUE(asid);
  292. value |= SMMU_ASID_ENABLE;
  293. smmu_writel(smmu, value, group->reg);
  294. } else {
  295. pr_warn("%s group from swgroup %u not found\n", __func__,
  296. swgroup);
  297. /* No point moving ahead if group was not found */
  298. return;
  299. }
  300. for (i = 0; i < smmu->soc->num_clients; i++) {
  301. const struct tegra_mc_client *client = &smmu->soc->clients[i];
  302. if (client->swgroup != swgroup)
  303. continue;
  304. value = smmu_readl(smmu, client->smmu.reg);
  305. value |= BIT(client->smmu.bit);
  306. smmu_writel(smmu, value, client->smmu.reg);
  307. }
  308. }
  309. static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
  310. unsigned int asid)
  311. {
  312. const struct tegra_smmu_swgroup *group;
  313. unsigned int i;
  314. u32 value;
  315. group = tegra_smmu_find_swgroup(smmu, swgroup);
  316. if (group) {
  317. value = smmu_readl(smmu, group->reg);
  318. value &= ~SMMU_ASID_MASK;
  319. value |= SMMU_ASID_VALUE(asid);
  320. value &= ~SMMU_ASID_ENABLE;
  321. smmu_writel(smmu, value, group->reg);
  322. }
  323. for (i = 0; i < smmu->soc->num_clients; i++) {
  324. const struct tegra_mc_client *client = &smmu->soc->clients[i];
  325. if (client->swgroup != swgroup)
  326. continue;
  327. value = smmu_readl(smmu, client->smmu.reg);
  328. value &= ~BIT(client->smmu.bit);
  329. smmu_writel(smmu, value, client->smmu.reg);
  330. }
  331. }
  332. static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
  333. struct tegra_smmu_as *as)
  334. {
  335. u32 value;
  336. int err;
  337. if (as->use_count > 0) {
  338. as->use_count++;
  339. return 0;
  340. }
  341. as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
  342. DMA_TO_DEVICE);
  343. if (dma_mapping_error(smmu->dev, as->pd_dma))
  344. return -ENOMEM;
  345. /* We can't handle 64-bit DMA addresses */
  346. if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
  347. err = -ENOMEM;
  348. goto err_unmap;
  349. }
  350. err = tegra_smmu_alloc_asid(smmu, &as->id);
  351. if (err < 0)
  352. goto err_unmap;
  353. smmu_flush_ptc(smmu, as->pd_dma, 0);
  354. smmu_flush_tlb_asid(smmu, as->id);
  355. smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
  356. value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
  357. smmu_writel(smmu, value, SMMU_PTB_DATA);
  358. smmu_flush(smmu);
  359. as->smmu = smmu;
  360. as->use_count++;
  361. return 0;
  362. err_unmap:
  363. dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
  364. return err;
  365. }
  366. static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
  367. struct tegra_smmu_as *as)
  368. {
  369. if (--as->use_count > 0)
  370. return;
  371. tegra_smmu_free_asid(smmu, as->id);
  372. dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
  373. as->smmu = NULL;
  374. }
  375. static int tegra_smmu_attach_dev(struct iommu_domain *domain,
  376. struct device *dev)
  377. {
  378. struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
  379. struct tegra_smmu_as *as = to_smmu_as(domain);
  380. struct device_node *np = dev->of_node;
  381. struct of_phandle_args args;
  382. unsigned int index = 0;
  383. int err = 0;
  384. while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
  385. &args)) {
  386. unsigned int swgroup = args.args[0];
  387. if (args.np != smmu->dev->of_node) {
  388. of_node_put(args.np);
  389. continue;
  390. }
  391. of_node_put(args.np);
  392. err = tegra_smmu_as_prepare(smmu, as);
  393. if (err < 0)
  394. return err;
  395. tegra_smmu_enable(smmu, swgroup, as->id);
  396. index++;
  397. }
  398. if (index == 0)
  399. return -ENODEV;
  400. return 0;
  401. }
  402. static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
  403. {
  404. struct tegra_smmu_as *as = to_smmu_as(domain);
  405. struct device_node *np = dev->of_node;
  406. struct tegra_smmu *smmu = as->smmu;
  407. struct of_phandle_args args;
  408. unsigned int index = 0;
  409. while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
  410. &args)) {
  411. unsigned int swgroup = args.args[0];
  412. if (args.np != smmu->dev->of_node) {
  413. of_node_put(args.np);
  414. continue;
  415. }
  416. of_node_put(args.np);
  417. tegra_smmu_disable(smmu, swgroup, as->id);
  418. tegra_smmu_as_unprepare(smmu, as);
  419. index++;
  420. }
  421. }
  422. static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
  423. u32 value)
  424. {
  425. unsigned int pd_index = iova_pd_index(iova);
  426. struct tegra_smmu *smmu = as->smmu;
  427. u32 *pd = page_address(as->pd);
  428. unsigned long offset = pd_index * sizeof(*pd);
  429. /* Set the page directory entry first */
  430. pd[pd_index] = value;
  431. /* The flush the page directory entry from caches */
  432. dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
  433. sizeof(*pd), DMA_TO_DEVICE);
  434. /* And flush the iommu */
  435. smmu_flush_ptc(smmu, as->pd_dma, offset);
  436. smmu_flush_tlb_section(smmu, as->id, iova);
  437. smmu_flush(smmu);
  438. }
  439. static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
  440. {
  441. u32 *pt = page_address(pt_page);
  442. return pt + iova_pt_index(iova);
  443. }
  444. static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
  445. dma_addr_t *dmap)
  446. {
  447. unsigned int pd_index = iova_pd_index(iova);
  448. struct tegra_smmu *smmu = as->smmu;
  449. struct page *pt_page;
  450. u32 *pd;
  451. pt_page = as->pts[pd_index];
  452. if (!pt_page)
  453. return NULL;
  454. pd = page_address(as->pd);
  455. *dmap = smmu_pde_to_dma(smmu, pd[pd_index]);
  456. return tegra_smmu_pte_offset(pt_page, iova);
  457. }
  458. static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
  459. dma_addr_t *dmap, struct page *page)
  460. {
  461. unsigned int pde = iova_pd_index(iova);
  462. struct tegra_smmu *smmu = as->smmu;
  463. if (!as->pts[pde]) {
  464. dma_addr_t dma;
  465. dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
  466. DMA_TO_DEVICE);
  467. if (dma_mapping_error(smmu->dev, dma)) {
  468. __free_page(page);
  469. return NULL;
  470. }
  471. if (!smmu_dma_addr_valid(smmu, dma)) {
  472. dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
  473. DMA_TO_DEVICE);
  474. __free_page(page);
  475. return NULL;
  476. }
  477. as->pts[pde] = page;
  478. tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
  479. SMMU_PDE_NEXT));
  480. *dmap = dma;
  481. } else {
  482. u32 *pd = page_address(as->pd);
  483. *dmap = smmu_pde_to_dma(smmu, pd[pde]);
  484. }
  485. return tegra_smmu_pte_offset(as->pts[pde], iova);
  486. }
  487. static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
  488. {
  489. unsigned int pd_index = iova_pd_index(iova);
  490. as->count[pd_index]++;
  491. }
  492. static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
  493. {
  494. unsigned int pde = iova_pd_index(iova);
  495. struct page *page = as->pts[pde];
  496. /*
  497. * When no entries in this page table are used anymore, return the
  498. * memory page to the system.
  499. */
  500. if (--as->count[pde] == 0) {
  501. struct tegra_smmu *smmu = as->smmu;
  502. u32 *pd = page_address(as->pd);
  503. dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]);
  504. tegra_smmu_set_pde(as, iova, 0);
  505. dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
  506. __free_page(page);
  507. as->pts[pde] = NULL;
  508. }
  509. }
  510. static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
  511. u32 *pte, dma_addr_t pte_dma, u32 val)
  512. {
  513. struct tegra_smmu *smmu = as->smmu;
  514. unsigned long offset = SMMU_OFFSET_IN_PAGE(pte);
  515. *pte = val;
  516. dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
  517. 4, DMA_TO_DEVICE);
  518. smmu_flush_ptc(smmu, pte_dma, offset);
  519. smmu_flush_tlb_group(smmu, as->id, iova);
  520. smmu_flush(smmu);
  521. }
  522. static struct page *as_get_pde_page(struct tegra_smmu_as *as,
  523. unsigned long iova, gfp_t gfp,
  524. unsigned long *flags)
  525. {
  526. unsigned int pde = iova_pd_index(iova);
  527. struct page *page = as->pts[pde];
  528. /* at first check whether allocation needs to be done at all */
  529. if (page)
  530. return page;
  531. /*
  532. * In order to prevent exhaustion of the atomic memory pool, we
  533. * allocate page in a sleeping context if GFP flags permit. Hence
  534. * spinlock needs to be unlocked and re-locked after allocation.
  535. */
  536. if (!(gfp & __GFP_ATOMIC))
  537. spin_unlock_irqrestore(&as->lock, *flags);
  538. page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO);
  539. if (!(gfp & __GFP_ATOMIC))
  540. spin_lock_irqsave(&as->lock, *flags);
  541. /*
  542. * In a case of blocking allocation, a concurrent mapping may win
  543. * the PDE allocation. In this case the allocated page isn't needed
  544. * if allocation succeeded and the allocation failure isn't fatal.
  545. */
  546. if (as->pts[pde]) {
  547. if (page)
  548. __free_page(page);
  549. page = as->pts[pde];
  550. }
  551. return page;
  552. }
  553. static int
  554. __tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
  555. phys_addr_t paddr, size_t size, int prot, gfp_t gfp,
  556. unsigned long *flags)
  557. {
  558. struct tegra_smmu_as *as = to_smmu_as(domain);
  559. dma_addr_t pte_dma;
  560. struct page *page;
  561. u32 pte_attrs;
  562. u32 *pte;
  563. page = as_get_pde_page(as, iova, gfp, flags);
  564. if (!page)
  565. return -ENOMEM;
  566. pte = as_get_pte(as, iova, &pte_dma, page);
  567. if (!pte)
  568. return -ENOMEM;
  569. /* If we aren't overwriting a pre-existing entry, increment use */
  570. if (*pte == 0)
  571. tegra_smmu_pte_get_use(as, iova);
  572. pte_attrs = SMMU_PTE_NONSECURE;
  573. if (prot & IOMMU_READ)
  574. pte_attrs |= SMMU_PTE_READABLE;
  575. if (prot & IOMMU_WRITE)
  576. pte_attrs |= SMMU_PTE_WRITABLE;
  577. tegra_smmu_set_pte(as, iova, pte, pte_dma,
  578. SMMU_PHYS_PFN(paddr) | pte_attrs);
  579. return 0;
  580. }
  581. static size_t
  582. __tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
  583. size_t size, struct iommu_iotlb_gather *gather)
  584. {
  585. struct tegra_smmu_as *as = to_smmu_as(domain);
  586. dma_addr_t pte_dma;
  587. u32 *pte;
  588. pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
  589. if (!pte || !*pte)
  590. return 0;
  591. tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
  592. tegra_smmu_pte_put_use(as, iova);
  593. return size;
  594. }
  595. static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
  596. phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
  597. {
  598. struct tegra_smmu_as *as = to_smmu_as(domain);
  599. unsigned long flags;
  600. int ret;
  601. spin_lock_irqsave(&as->lock, flags);
  602. ret = __tegra_smmu_map(domain, iova, paddr, size, prot, gfp, &flags);
  603. spin_unlock_irqrestore(&as->lock, flags);
  604. return ret;
  605. }
  606. static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
  607. size_t size, struct iommu_iotlb_gather *gather)
  608. {
  609. struct tegra_smmu_as *as = to_smmu_as(domain);
  610. unsigned long flags;
  611. spin_lock_irqsave(&as->lock, flags);
  612. size = __tegra_smmu_unmap(domain, iova, size, gather);
  613. spin_unlock_irqrestore(&as->lock, flags);
  614. return size;
  615. }
  616. static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
  617. dma_addr_t iova)
  618. {
  619. struct tegra_smmu_as *as = to_smmu_as(domain);
  620. unsigned long pfn;
  621. dma_addr_t pte_dma;
  622. u32 *pte;
  623. pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
  624. if (!pte || !*pte)
  625. return 0;
  626. pfn = *pte & as->smmu->pfn_mask;
  627. return SMMU_PFN_PHYS(pfn) + SMMU_OFFSET_IN_PAGE(iova);
  628. }
  629. static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
  630. {
  631. struct platform_device *pdev;
  632. struct tegra_mc *mc;
  633. pdev = of_find_device_by_node(np);
  634. if (!pdev)
  635. return NULL;
  636. mc = platform_get_drvdata(pdev);
  637. if (!mc)
  638. return NULL;
  639. return mc->smmu;
  640. }
  641. static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
  642. struct of_phandle_args *args)
  643. {
  644. const struct iommu_ops *ops = smmu->iommu.ops;
  645. int err;
  646. err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);
  647. if (err < 0) {
  648. dev_err(dev, "failed to initialize fwspec: %d\n", err);
  649. return err;
  650. }
  651. err = ops->of_xlate(dev, args);
  652. if (err < 0) {
  653. dev_err(dev, "failed to parse SW group ID: %d\n", err);
  654. iommu_fwspec_free(dev);
  655. return err;
  656. }
  657. return 0;
  658. }
  659. static struct iommu_device *tegra_smmu_probe_device(struct device *dev)
  660. {
  661. struct device_node *np = dev->of_node;
  662. struct tegra_smmu *smmu = NULL;
  663. struct of_phandle_args args;
  664. unsigned int index = 0;
  665. int err;
  666. while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
  667. &args) == 0) {
  668. smmu = tegra_smmu_find(args.np);
  669. if (smmu) {
  670. err = tegra_smmu_configure(smmu, dev, &args);
  671. of_node_put(args.np);
  672. if (err < 0)
  673. return ERR_PTR(err);
  674. /*
  675. * Only a single IOMMU master interface is currently
  676. * supported by the Linux kernel, so abort after the
  677. * first match.
  678. */
  679. dev_iommu_priv_set(dev, smmu);
  680. break;
  681. }
  682. of_node_put(args.np);
  683. index++;
  684. }
  685. if (!smmu)
  686. return ERR_PTR(-ENODEV);
  687. return &smmu->iommu;
  688. }
  689. static void tegra_smmu_release_device(struct device *dev)
  690. {
  691. dev_iommu_priv_set(dev, NULL);
  692. }
  693. static const struct tegra_smmu_group_soc *
  694. tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)
  695. {
  696. unsigned int i, j;
  697. for (i = 0; i < smmu->soc->num_groups; i++)
  698. for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)
  699. if (smmu->soc->groups[i].swgroups[j] == swgroup)
  700. return &smmu->soc->groups[i];
  701. return NULL;
  702. }
  703. static void tegra_smmu_group_release(void *iommu_data)
  704. {
  705. struct tegra_smmu_group *group = iommu_data;
  706. struct tegra_smmu *smmu = group->smmu;
  707. mutex_lock(&smmu->lock);
  708. list_del(&group->list);
  709. mutex_unlock(&smmu->lock);
  710. }
  711. static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu,
  712. unsigned int swgroup)
  713. {
  714. const struct tegra_smmu_group_soc *soc;
  715. struct tegra_smmu_group *group;
  716. struct iommu_group *grp;
  717. /* Find group_soc associating with swgroup */
  718. soc = tegra_smmu_find_group(smmu, swgroup);
  719. mutex_lock(&smmu->lock);
  720. /* Find existing iommu_group associating with swgroup or group_soc */
  721. list_for_each_entry(group, &smmu->groups, list)
  722. if ((group->swgroup == swgroup) || (soc && group->soc == soc)) {
  723. grp = iommu_group_ref_get(group->group);
  724. mutex_unlock(&smmu->lock);
  725. return grp;
  726. }
  727. group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);
  728. if (!group) {
  729. mutex_unlock(&smmu->lock);
  730. return NULL;
  731. }
  732. INIT_LIST_HEAD(&group->list);
  733. group->swgroup = swgroup;
  734. group->smmu = smmu;
  735. group->soc = soc;
  736. group->group = iommu_group_alloc();
  737. if (IS_ERR(group->group)) {
  738. devm_kfree(smmu->dev, group);
  739. mutex_unlock(&smmu->lock);
  740. return NULL;
  741. }
  742. iommu_group_set_iommudata(group->group, group, tegra_smmu_group_release);
  743. if (soc)
  744. iommu_group_set_name(group->group, soc->name);
  745. list_add_tail(&group->list, &smmu->groups);
  746. mutex_unlock(&smmu->lock);
  747. return group->group;
  748. }
  749. static struct iommu_group *tegra_smmu_device_group(struct device *dev)
  750. {
  751. struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
  752. struct tegra_smmu *smmu = dev_iommu_priv_get(dev);
  753. struct iommu_group *group;
  754. group = tegra_smmu_group_get(smmu, fwspec->ids[0]);
  755. if (!group)
  756. group = generic_device_group(dev);
  757. return group;
  758. }
  759. static int tegra_smmu_of_xlate(struct device *dev,
  760. struct of_phandle_args *args)
  761. {
  762. u32 id = args->args[0];
  763. return iommu_fwspec_add_ids(dev, &id, 1);
  764. }
  765. static const struct iommu_ops tegra_smmu_ops = {
  766. .capable = tegra_smmu_capable,
  767. .domain_alloc = tegra_smmu_domain_alloc,
  768. .domain_free = tegra_smmu_domain_free,
  769. .attach_dev = tegra_smmu_attach_dev,
  770. .detach_dev = tegra_smmu_detach_dev,
  771. .probe_device = tegra_smmu_probe_device,
  772. .release_device = tegra_smmu_release_device,
  773. .device_group = tegra_smmu_device_group,
  774. .map = tegra_smmu_map,
  775. .unmap = tegra_smmu_unmap,
  776. .iova_to_phys = tegra_smmu_iova_to_phys,
  777. .of_xlate = tegra_smmu_of_xlate,
  778. .pgsize_bitmap = SZ_4K,
  779. };
  780. static void tegra_smmu_ahb_enable(void)
  781. {
  782. static const struct of_device_id ahb_match[] = {
  783. { .compatible = "nvidia,tegra30-ahb", },
  784. { }
  785. };
  786. struct device_node *ahb;
  787. ahb = of_find_matching_node(NULL, ahb_match);
  788. if (ahb) {
  789. tegra_ahb_enable_smmu(ahb);
  790. of_node_put(ahb);
  791. }
  792. }
  793. static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
  794. {
  795. struct tegra_smmu *smmu = s->private;
  796. unsigned int i;
  797. u32 value;
  798. seq_printf(s, "swgroup enabled ASID\n");
  799. seq_printf(s, "------------------------\n");
  800. for (i = 0; i < smmu->soc->num_swgroups; i++) {
  801. const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
  802. const char *status;
  803. unsigned int asid;
  804. value = smmu_readl(smmu, group->reg);
  805. if (value & SMMU_ASID_ENABLE)
  806. status = "yes";
  807. else
  808. status = "no";
  809. asid = value & SMMU_ASID_MASK;
  810. seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
  811. asid);
  812. }
  813. return 0;
  814. }
  815. DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups);
  816. static int tegra_smmu_clients_show(struct seq_file *s, void *data)
  817. {
  818. struct tegra_smmu *smmu = s->private;
  819. unsigned int i;
  820. u32 value;
  821. seq_printf(s, "client enabled\n");
  822. seq_printf(s, "--------------------\n");
  823. for (i = 0; i < smmu->soc->num_clients; i++) {
  824. const struct tegra_mc_client *client = &smmu->soc->clients[i];
  825. const char *status;
  826. value = smmu_readl(smmu, client->smmu.reg);
  827. if (value & BIT(client->smmu.bit))
  828. status = "yes";
  829. else
  830. status = "no";
  831. seq_printf(s, "%-12s %s\n", client->name, status);
  832. }
  833. return 0;
  834. }
  835. DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients);
  836. static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
  837. {
  838. smmu->debugfs = debugfs_create_dir("smmu", NULL);
  839. if (!smmu->debugfs)
  840. return;
  841. debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
  842. &tegra_smmu_swgroups_fops);
  843. debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
  844. &tegra_smmu_clients_fops);
  845. }
  846. static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
  847. {
  848. debugfs_remove_recursive(smmu->debugfs);
  849. }
  850. struct tegra_smmu *tegra_smmu_probe(struct device *dev,
  851. const struct tegra_smmu_soc *soc,
  852. struct tegra_mc *mc)
  853. {
  854. struct tegra_smmu *smmu;
  855. size_t size;
  856. u32 value;
  857. int err;
  858. smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
  859. if (!smmu)
  860. return ERR_PTR(-ENOMEM);
  861. /*
  862. * This is a bit of a hack. Ideally we'd want to simply return this
  863. * value. However the IOMMU registration process will attempt to add
  864. * all devices to the IOMMU when bus_set_iommu() is called. In order
  865. * not to rely on global variables to track the IOMMU instance, we
  866. * set it here so that it can be looked up from the .probe_device()
  867. * callback via the IOMMU device's .drvdata field.
  868. */
  869. mc->smmu = smmu;
  870. size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);
  871. smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
  872. if (!smmu->asids)
  873. return ERR_PTR(-ENOMEM);
  874. INIT_LIST_HEAD(&smmu->groups);
  875. mutex_init(&smmu->lock);
  876. smmu->regs = mc->regs;
  877. smmu->soc = soc;
  878. smmu->dev = dev;
  879. smmu->mc = mc;
  880. smmu->pfn_mask =
  881. BIT_MASK(mc->soc->num_address_bits - SMMU_PTE_SHIFT) - 1;
  882. dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
  883. mc->soc->num_address_bits, smmu->pfn_mask);
  884. smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;
  885. dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
  886. smmu->tlb_mask);
  887. value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
  888. if (soc->supports_request_limit)
  889. value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
  890. smmu_writel(smmu, value, SMMU_PTC_CONFIG);
  891. value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
  892. SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
  893. if (soc->supports_round_robin_arbitration)
  894. value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
  895. smmu_writel(smmu, value, SMMU_TLB_CONFIG);
  896. smmu_flush_ptc_all(smmu);
  897. smmu_flush_tlb(smmu);
  898. smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
  899. smmu_flush(smmu);
  900. tegra_smmu_ahb_enable();
  901. err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
  902. if (err)
  903. return ERR_PTR(err);
  904. iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
  905. iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
  906. err = iommu_device_register(&smmu->iommu);
  907. if (err) {
  908. iommu_device_sysfs_remove(&smmu->iommu);
  909. return ERR_PTR(err);
  910. }
  911. err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
  912. if (err < 0) {
  913. iommu_device_unregister(&smmu->iommu);
  914. iommu_device_sysfs_remove(&smmu->iommu);
  915. return ERR_PTR(err);
  916. }
  917. if (IS_ENABLED(CONFIG_DEBUG_FS))
  918. tegra_smmu_debugfs_init(smmu);
  919. return smmu;
  920. }
  921. void tegra_smmu_remove(struct tegra_smmu *smmu)
  922. {
  923. iommu_device_unregister(&smmu->iommu);
  924. iommu_device_sysfs_remove(&smmu->iommu);
  925. if (IS_ENABLED(CONFIG_DEBUG_FS))
  926. tegra_smmu_debugfs_exit(smmu);
  927. }