iommu.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * iommu.c: IOMMU specific routines for memory management.
  4. *
  5. * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
  6. * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
  7. * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
  8. * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  9. */
  10. #include <linux/kernel.h>
  11. #include <linux/init.h>
  12. #include <linux/mm.h>
  13. #include <linux/slab.h>
  14. #include <linux/dma-map-ops.h>
  15. #include <linux/of.h>
  16. #include <linux/of_device.h>
  17. #include <asm/io.h>
  18. #include <asm/mxcc.h>
  19. #include <asm/mbus.h>
  20. #include <asm/cacheflush.h>
  21. #include <asm/tlbflush.h>
  22. #include <asm/bitext.h>
  23. #include <asm/iommu.h>
  24. #include <asm/dma.h>
  25. #include "mm_32.h"
  26. /*
  27. * This can be sized dynamically, but we will do this
  28. * only when we have a guidance about actual I/O pressures.
  29. */
  30. #define IOMMU_RNGE IOMMU_RNGE_256MB
  31. #define IOMMU_START 0xF0000000
  32. #define IOMMU_WINSIZE (256*1024*1024U)
  33. #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
  34. #define IOMMU_ORDER 6 /* 4096 * (1<<6) */
  35. static int viking_flush;
  36. /* viking.S */
  37. extern void viking_flush_page(unsigned long page);
  38. extern void viking_mxcc_flush_page(unsigned long page);
  39. /*
  40. * Values precomputed according to CPU type.
  41. */
  42. static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
  43. static pgprot_t dvma_prot; /* Consistent mapping pte flags */
  44. #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
  45. #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
  46. static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
  47. static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
  48. static void __init sbus_iommu_init(struct platform_device *op)
  49. {
  50. struct iommu_struct *iommu;
  51. unsigned int impl, vers;
  52. unsigned long *bitmap;
  53. unsigned long control;
  54. unsigned long base;
  55. unsigned long tmp;
  56. iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
  57. if (!iommu) {
  58. prom_printf("Unable to allocate iommu structure\n");
  59. prom_halt();
  60. }
  61. iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
  62. "iommu_regs");
  63. if (!iommu->regs) {
  64. prom_printf("Cannot map IOMMU registers\n");
  65. prom_halt();
  66. }
  67. control = sbus_readl(&iommu->regs->control);
  68. impl = (control & IOMMU_CTRL_IMPL) >> 28;
  69. vers = (control & IOMMU_CTRL_VERS) >> 24;
  70. control &= ~(IOMMU_CTRL_RNGE);
  71. control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
  72. sbus_writel(control, &iommu->regs->control);
  73. iommu_invalidate(iommu->regs);
  74. iommu->start = IOMMU_START;
  75. iommu->end = 0xffffffff;
  76. /* Allocate IOMMU page table */
  77. /* Stupid alignment constraints give me a headache.
  78. We need 256K or 512K or 1M or 2M area aligned to
  79. its size and current gfp will fortunately give
  80. it to us. */
  81. tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
  82. if (!tmp) {
  83. prom_printf("Unable to allocate iommu table [0x%lx]\n",
  84. IOMMU_NPTES * sizeof(iopte_t));
  85. prom_halt();
  86. }
  87. iommu->page_table = (iopte_t *)tmp;
  88. /* Initialize new table. */
  89. memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
  90. flush_cache_all();
  91. flush_tlb_all();
  92. base = __pa((unsigned long)iommu->page_table) >> 4;
  93. sbus_writel(base, &iommu->regs->base);
  94. iommu_invalidate(iommu->regs);
  95. bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
  96. if (!bitmap) {
  97. prom_printf("Unable to allocate iommu bitmap [%d]\n",
  98. (int)(IOMMU_NPTES>>3));
  99. prom_halt();
  100. }
  101. bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
  102. /* To be coherent on HyperSparc, the page color of DVMA
  103. * and physical addresses must match.
  104. */
  105. if (srmmu_modtype == HyperSparc)
  106. iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
  107. else
  108. iommu->usemap.num_colors = 1;
  109. printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
  110. impl, vers, iommu->page_table,
  111. (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
  112. op->dev.archdata.iommu = iommu;
  113. if (flush_page_for_dma_global)
  114. op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
  115. else
  116. op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
  117. }
  118. static int __init iommu_init(void)
  119. {
  120. struct device_node *dp;
  121. for_each_node_by_name(dp, "iommu") {
  122. struct platform_device *op = of_find_device_by_node(dp);
  123. sbus_iommu_init(op);
  124. of_propagate_archdata(op);
  125. }
  126. return 0;
  127. }
  128. subsys_initcall(iommu_init);
  129. /* Flush the iotlb entries to ram. */
  130. /* This could be better if we didn't have to flush whole pages. */
  131. static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
  132. {
  133. unsigned long start;
  134. unsigned long end;
  135. start = (unsigned long)iopte;
  136. end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
  137. start &= PAGE_MASK;
  138. if (viking_mxcc_present) {
  139. while(start < end) {
  140. viking_mxcc_flush_page(start);
  141. start += PAGE_SIZE;
  142. }
  143. } else if (viking_flush) {
  144. while(start < end) {
  145. viking_flush_page(start);
  146. start += PAGE_SIZE;
  147. }
  148. } else {
  149. while(start < end) {
  150. __flush_page_to_ram(start);
  151. start += PAGE_SIZE;
  152. }
  153. }
  154. }
  155. static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
  156. unsigned long offset, size_t len, bool per_page_flush)
  157. {
  158. struct iommu_struct *iommu = dev->archdata.iommu;
  159. phys_addr_t paddr = page_to_phys(page) + offset;
  160. unsigned long off = paddr & ~PAGE_MASK;
  161. unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  162. unsigned long pfn = __phys_to_pfn(paddr);
  163. unsigned int busa, busa0;
  164. iopte_t *iopte, *iopte0;
  165. int ioptex, i;
  166. /* XXX So what is maxphys for us and how do drivers know it? */
  167. if (!len || len > 256 * 1024)
  168. return DMA_MAPPING_ERROR;
  169. /*
  170. * We expect unmapped highmem pages to be not in the cache.
  171. * XXX Is this a good assumption?
  172. * XXX What if someone else unmaps it here and races us?
  173. */
  174. if (per_page_flush && !PageHighMem(page)) {
  175. unsigned long vaddr, p;
  176. vaddr = (unsigned long)page_address(page) + offset;
  177. for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
  178. flush_page_for_dma(p);
  179. }
  180. /* page color = pfn of page */
  181. ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
  182. if (ioptex < 0)
  183. panic("iommu out");
  184. busa0 = iommu->start + (ioptex << PAGE_SHIFT);
  185. iopte0 = &iommu->page_table[ioptex];
  186. busa = busa0;
  187. iopte = iopte0;
  188. for (i = 0; i < npages; i++) {
  189. iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
  190. iommu_invalidate_page(iommu->regs, busa);
  191. busa += PAGE_SIZE;
  192. iopte++;
  193. pfn++;
  194. }
  195. iommu_flush_iotlb(iopte0, npages);
  196. return busa0 + off;
  197. }
  198. static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
  199. struct page *page, unsigned long offset, size_t len,
  200. enum dma_data_direction dir, unsigned long attrs)
  201. {
  202. flush_page_for_dma(0);
  203. return __sbus_iommu_map_page(dev, page, offset, len, false);
  204. }
  205. static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
  206. struct page *page, unsigned long offset, size_t len,
  207. enum dma_data_direction dir, unsigned long attrs)
  208. {
  209. return __sbus_iommu_map_page(dev, page, offset, len, true);
  210. }
  211. static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
  212. int nents, enum dma_data_direction dir, unsigned long attrs,
  213. bool per_page_flush)
  214. {
  215. struct scatterlist *sg;
  216. int j;
  217. for_each_sg(sgl, sg, nents, j) {
  218. sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
  219. sg->offset, sg->length, per_page_flush);
  220. if (sg->dma_address == DMA_MAPPING_ERROR)
  221. return 0;
  222. sg->dma_length = sg->length;
  223. }
  224. return nents;
  225. }
  226. static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
  227. int nents, enum dma_data_direction dir, unsigned long attrs)
  228. {
  229. flush_page_for_dma(0);
  230. return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
  231. }
  232. static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
  233. int nents, enum dma_data_direction dir, unsigned long attrs)
  234. {
  235. return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
  236. }
  237. static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
  238. size_t len, enum dma_data_direction dir, unsigned long attrs)
  239. {
  240. struct iommu_struct *iommu = dev->archdata.iommu;
  241. unsigned int busa = dma_addr & PAGE_MASK;
  242. unsigned long off = dma_addr & ~PAGE_MASK;
  243. unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
  244. unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
  245. unsigned int i;
  246. BUG_ON(busa < iommu->start);
  247. for (i = 0; i < npages; i++) {
  248. iopte_val(iommu->page_table[ioptex + i]) = 0;
  249. iommu_invalidate_page(iommu->regs, busa);
  250. busa += PAGE_SIZE;
  251. }
  252. bit_map_clear(&iommu->usemap, ioptex, npages);
  253. }
  254. static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
  255. int nents, enum dma_data_direction dir, unsigned long attrs)
  256. {
  257. struct scatterlist *sg;
  258. int i;
  259. for_each_sg(sgl, sg, nents, i) {
  260. sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
  261. attrs);
  262. sg->dma_address = 0x21212121;
  263. }
  264. }
  265. #ifdef CONFIG_SBUS
  266. static void *sbus_iommu_alloc(struct device *dev, size_t len,
  267. dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
  268. {
  269. struct iommu_struct *iommu = dev->archdata.iommu;
  270. unsigned long va, addr, page, end, ret;
  271. iopte_t *iopte = iommu->page_table;
  272. iopte_t *first;
  273. int ioptex;
  274. /* XXX So what is maxphys for us and how do drivers know it? */
  275. if (!len || len > 256 * 1024)
  276. return NULL;
  277. len = PAGE_ALIGN(len);
  278. va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
  279. if (va == 0)
  280. return NULL;
  281. addr = ret = sparc_dma_alloc_resource(dev, len);
  282. if (!addr)
  283. goto out_free_pages;
  284. BUG_ON((va & ~PAGE_MASK) != 0);
  285. BUG_ON((addr & ~PAGE_MASK) != 0);
  286. BUG_ON((len & ~PAGE_MASK) != 0);
  287. /* page color = physical address */
  288. ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
  289. addr >> PAGE_SHIFT);
  290. if (ioptex < 0)
  291. panic("iommu out");
  292. iopte += ioptex;
  293. first = iopte;
  294. end = addr + len;
  295. while(addr < end) {
  296. page = va;
  297. {
  298. pmd_t *pmdp;
  299. pte_t *ptep;
  300. if (viking_mxcc_present)
  301. viking_mxcc_flush_page(page);
  302. else if (viking_flush)
  303. viking_flush_page(page);
  304. else
  305. __flush_page_to_ram(page);
  306. pmdp = pmd_off_k(addr);
  307. ptep = pte_offset_map(pmdp, addr);
  308. set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
  309. }
  310. iopte_val(*iopte++) =
  311. MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
  312. addr += PAGE_SIZE;
  313. va += PAGE_SIZE;
  314. }
  315. /* P3: why do we need this?
  316. *
  317. * DAVEM: Because there are several aspects, none of which
  318. * are handled by a single interface. Some cpus are
  319. * completely not I/O DMA coherent, and some have
  320. * virtually indexed caches. The driver DMA flushing
  321. * methods handle the former case, but here during
  322. * IOMMU page table modifications, and usage of non-cacheable
  323. * cpu mappings of pages potentially in the cpu caches, we have
  324. * to handle the latter case as well.
  325. */
  326. flush_cache_all();
  327. iommu_flush_iotlb(first, len >> PAGE_SHIFT);
  328. flush_tlb_all();
  329. iommu_invalidate(iommu->regs);
  330. *dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
  331. return (void *)ret;
  332. out_free_pages:
  333. free_pages(va, get_order(len));
  334. return NULL;
  335. }
  336. static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
  337. dma_addr_t busa, unsigned long attrs)
  338. {
  339. struct iommu_struct *iommu = dev->archdata.iommu;
  340. iopte_t *iopte = iommu->page_table;
  341. struct page *page = virt_to_page(cpu_addr);
  342. int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
  343. unsigned long end;
  344. if (!sparc_dma_free_resource(cpu_addr, len))
  345. return;
  346. BUG_ON((busa & ~PAGE_MASK) != 0);
  347. BUG_ON((len & ~PAGE_MASK) != 0);
  348. iopte += ioptex;
  349. end = busa + len;
  350. while (busa < end) {
  351. iopte_val(*iopte++) = 0;
  352. busa += PAGE_SIZE;
  353. }
  354. flush_tlb_all();
  355. iommu_invalidate(iommu->regs);
  356. bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
  357. __free_pages(page, get_order(len));
  358. }
  359. #endif
  360. static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
  361. #ifdef CONFIG_SBUS
  362. .alloc = sbus_iommu_alloc,
  363. .free = sbus_iommu_free,
  364. #endif
  365. .map_page = sbus_iommu_map_page_gflush,
  366. .unmap_page = sbus_iommu_unmap_page,
  367. .map_sg = sbus_iommu_map_sg_gflush,
  368. .unmap_sg = sbus_iommu_unmap_sg,
  369. };
  370. static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
  371. #ifdef CONFIG_SBUS
  372. .alloc = sbus_iommu_alloc,
  373. .free = sbus_iommu_free,
  374. #endif
  375. .map_page = sbus_iommu_map_page_pflush,
  376. .unmap_page = sbus_iommu_unmap_page,
  377. .map_sg = sbus_iommu_map_sg_pflush,
  378. .unmap_sg = sbus_iommu_unmap_sg,
  379. };
  380. void __init ld_mmu_iommu(void)
  381. {
  382. if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
  383. dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
  384. ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
  385. } else {
  386. dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
  387. ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
  388. }
  389. }