consistent.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507
  1. /*
  2. * linux/arch/arm/mm/consistent.c
  3. *
  4. * Copyright (C) 2000-2004 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. *
  10. * DMA uncached mapping support.
  11. */
  12. #include <linux/module.h>
  13. #include <linux/mm.h>
  14. #include <linux/slab.h>
  15. #include <linux/errno.h>
  16. #include <linux/list.h>
  17. #include <linux/init.h>
  18. #include <linux/device.h>
  19. #include <linux/dma-mapping.h>
  20. #include <asm/memory.h>
  21. #include <asm/cacheflush.h>
  22. #include <asm/tlbflush.h>
  23. #include <asm/sizes.h>
  24. /* Sanity check size */
  25. #if (CONSISTENT_DMA_SIZE % SZ_2M)
  26. #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
  27. #endif
  28. #define CONSISTENT_END (0xffe00000)
  29. #define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
  30. #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
  31. #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
  32. #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
  33. /*
  34. * These are the page tables (2MB each) covering uncached, DMA consistent allocations
  35. */
  36. static pte_t *consistent_pte[NUM_CONSISTENT_PTES];
  37. static DEFINE_SPINLOCK(consistent_lock);
  38. /*
  39. * VM region handling support.
  40. *
  41. * This should become something generic, handling VM region allocations for
  42. * vmalloc and similar (ioremap, module space, etc).
  43. *
  44. * I envisage vmalloc()'s supporting vm_struct becoming:
  45. *
  46. * struct vm_struct {
  47. * struct vm_region region;
  48. * unsigned long flags;
  49. * struct page **pages;
  50. * unsigned int nr_pages;
  51. * unsigned long phys_addr;
  52. * };
  53. *
  54. * get_vm_area() would then call vm_region_alloc with an appropriate
  55. * struct vm_region head (eg):
  56. *
  57. * struct vm_region vmalloc_head = {
  58. * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
  59. * .vm_start = VMALLOC_START,
  60. * .vm_end = VMALLOC_END,
  61. * };
  62. *
  63. * However, vmalloc_head.vm_start is variable (typically, it is dependent on
  64. * the amount of RAM found at boot time.) I would imagine that get_vm_area()
  65. * would have to initialise this each time prior to calling vm_region_alloc().
  66. */
  67. struct vm_region {
  68. struct list_head vm_list;
  69. unsigned long vm_start;
  70. unsigned long vm_end;
  71. struct page *vm_pages;
  72. int vm_active;
  73. };
  74. static struct vm_region consistent_head = {
  75. .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
  76. .vm_start = CONSISTENT_BASE,
  77. .vm_end = CONSISTENT_END,
  78. };
  79. static struct vm_region *
  80. vm_region_alloc(struct vm_region *head, size_t size, gfp_t gfp)
  81. {
  82. unsigned long addr = head->vm_start, end = head->vm_end - size;
  83. unsigned long flags;
  84. struct vm_region *c, *new;
  85. new = kmalloc(sizeof(struct vm_region), gfp);
  86. if (!new)
  87. goto out;
  88. spin_lock_irqsave(&consistent_lock, flags);
  89. list_for_each_entry(c, &head->vm_list, vm_list) {
  90. if ((addr + size) < addr)
  91. goto nospc;
  92. if ((addr + size) <= c->vm_start)
  93. goto found;
  94. addr = c->vm_end;
  95. if (addr > end)
  96. goto nospc;
  97. }
  98. found:
  99. /*
  100. * Insert this entry _before_ the one we found.
  101. */
  102. list_add_tail(&new->vm_list, &c->vm_list);
  103. new->vm_start = addr;
  104. new->vm_end = addr + size;
  105. new->vm_active = 1;
  106. spin_unlock_irqrestore(&consistent_lock, flags);
  107. return new;
  108. nospc:
  109. spin_unlock_irqrestore(&consistent_lock, flags);
  110. kfree(new);
  111. out:
  112. return NULL;
  113. }
  114. static struct vm_region *vm_region_find(struct vm_region *head, unsigned long addr)
  115. {
  116. struct vm_region *c;
  117. list_for_each_entry(c, &head->vm_list, vm_list) {
  118. if (c->vm_active && c->vm_start == addr)
  119. goto out;
  120. }
  121. c = NULL;
  122. out:
  123. return c;
  124. }
  125. #ifdef CONFIG_HUGETLB_PAGE
  126. #error ARM Coherent DMA allocator does not (yet) support huge TLB
  127. #endif
  128. static void *
  129. __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
  130. pgprot_t prot)
  131. {
  132. struct page *page;
  133. struct vm_region *c;
  134. unsigned long order;
  135. u64 mask = ISA_DMA_THRESHOLD, limit;
  136. if (!consistent_pte[0]) {
  137. printk(KERN_ERR "%s: not initialised\n", __func__);
  138. dump_stack();
  139. return NULL;
  140. }
  141. if (dev) {
  142. mask = dev->coherent_dma_mask;
  143. /*
  144. * Sanity check the DMA mask - it must be non-zero, and
  145. * must be able to be satisfied by a DMA allocation.
  146. */
  147. if (mask == 0) {
  148. dev_warn(dev, "coherent DMA mask is unset\n");
  149. goto no_page;
  150. }
  151. if ((~mask) & ISA_DMA_THRESHOLD) {
  152. dev_warn(dev, "coherent DMA mask %#llx is smaller "
  153. "than system GFP_DMA mask %#llx\n",
  154. mask, (unsigned long long)ISA_DMA_THRESHOLD);
  155. goto no_page;
  156. }
  157. }
  158. /*
  159. * Sanity check the allocation size.
  160. */
  161. size = PAGE_ALIGN(size);
  162. limit = (mask + 1) & ~mask;
  163. if ((limit && size >= limit) ||
  164. size >= (CONSISTENT_END - CONSISTENT_BASE)) {
  165. printk(KERN_WARNING "coherent allocation too big "
  166. "(requested %#x mask %#llx)\n", size, mask);
  167. goto no_page;
  168. }
  169. order = get_order(size);
  170. if (mask != 0xffffffff)
  171. gfp |= GFP_DMA;
  172. page = alloc_pages(gfp, order);
  173. if (!page)
  174. goto no_page;
  175. /*
  176. * Invalidate any data that might be lurking in the
  177. * kernel direct-mapped region for device DMA.
  178. */
  179. {
  180. void *ptr = page_address(page);
  181. memset(ptr, 0, size);
  182. dmac_flush_range(ptr, ptr + size);
  183. outer_flush_range(__pa(ptr), __pa(ptr) + size);
  184. }
  185. /*
  186. * Allocate a virtual address in the consistent mapping region.
  187. */
  188. c = vm_region_alloc(&consistent_head, size,
  189. gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
  190. if (c) {
  191. pte_t *pte;
  192. struct page *end = page + (1 << order);
  193. int idx = CONSISTENT_PTE_INDEX(c->vm_start);
  194. u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
  195. pte = consistent_pte[idx] + off;
  196. c->vm_pages = page;
  197. split_page(page, order);
  198. /*
  199. * Set the "dma handle"
  200. */
  201. *handle = page_to_dma(dev, page);
  202. do {
  203. BUG_ON(!pte_none(*pte));
  204. /*
  205. * x86 does not mark the pages reserved...
  206. */
  207. SetPageReserved(page);
  208. set_pte_ext(pte, mk_pte(page, prot), 0);
  209. page++;
  210. pte++;
  211. off++;
  212. if (off >= PTRS_PER_PTE) {
  213. off = 0;
  214. pte = consistent_pte[++idx];
  215. }
  216. } while (size -= PAGE_SIZE);
  217. /*
  218. * Free the otherwise unused pages.
  219. */
  220. while (page < end) {
  221. __free_page(page);
  222. page++;
  223. }
  224. return (void *)c->vm_start;
  225. }
  226. if (page)
  227. __free_pages(page, order);
  228. no_page:
  229. *handle = ~0;
  230. return NULL;
  231. }
  232. /*
  233. * Allocate DMA-coherent memory space and return both the kernel remapped
  234. * virtual and bus address for that space.
  235. */
  236. void *
  237. dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
  238. {
  239. if (arch_is_coherent()) {
  240. void *virt;
  241. virt = kmalloc(size, gfp);
  242. if (!virt)
  243. return NULL;
  244. *handle = virt_to_dma(dev, virt);
  245. return virt;
  246. }
  247. return __dma_alloc(dev, size, handle, gfp,
  248. pgprot_noncached(pgprot_kernel));
  249. }
  250. EXPORT_SYMBOL(dma_alloc_coherent);
  251. /*
  252. * Allocate a writecombining region, in much the same way as
  253. * dma_alloc_coherent above.
  254. */
  255. void *
  256. dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
  257. {
  258. return __dma_alloc(dev, size, handle, gfp,
  259. pgprot_writecombine(pgprot_kernel));
  260. }
  261. EXPORT_SYMBOL(dma_alloc_writecombine);
  262. static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
  263. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  264. {
  265. unsigned long flags, user_size, kern_size;
  266. struct vm_region *c;
  267. int ret = -ENXIO;
  268. user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  269. spin_lock_irqsave(&consistent_lock, flags);
  270. c = vm_region_find(&consistent_head, (unsigned long)cpu_addr);
  271. spin_unlock_irqrestore(&consistent_lock, flags);
  272. if (c) {
  273. unsigned long off = vma->vm_pgoff;
  274. kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
  275. if (off < kern_size &&
  276. user_size <= (kern_size - off)) {
  277. vma->vm_flags |= VM_RESERVED;
  278. ret = remap_pfn_range(vma, vma->vm_start,
  279. page_to_pfn(c->vm_pages) + off,
  280. user_size << PAGE_SHIFT,
  281. vma->vm_page_prot);
  282. }
  283. }
  284. return ret;
  285. }
  286. int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
  287. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  288. {
  289. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  290. return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
  291. }
  292. EXPORT_SYMBOL(dma_mmap_coherent);
  293. int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
  294. void *cpu_addr, dma_addr_t dma_addr, size_t size)
  295. {
  296. vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
  297. return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
  298. }
  299. EXPORT_SYMBOL(dma_mmap_writecombine);
  300. /*
  301. * free a page as defined by the above mapping.
  302. * Must not be called with IRQs disabled.
  303. */
  304. void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
  305. {
  306. struct vm_region *c;
  307. unsigned long flags, addr;
  308. pte_t *ptep;
  309. int idx;
  310. u32 off;
  311. WARN_ON(irqs_disabled());
  312. if (arch_is_coherent()) {
  313. kfree(cpu_addr);
  314. return;
  315. }
  316. size = PAGE_ALIGN(size);
  317. spin_lock_irqsave(&consistent_lock, flags);
  318. c = vm_region_find(&consistent_head, (unsigned long)cpu_addr);
  319. if (!c)
  320. goto no_area;
  321. c->vm_active = 0;
  322. spin_unlock_irqrestore(&consistent_lock, flags);
  323. if ((c->vm_end - c->vm_start) != size) {
  324. printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
  325. __func__, c->vm_end - c->vm_start, size);
  326. dump_stack();
  327. size = c->vm_end - c->vm_start;
  328. }
  329. idx = CONSISTENT_PTE_INDEX(c->vm_start);
  330. off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
  331. ptep = consistent_pte[idx] + off;
  332. addr = c->vm_start;
  333. do {
  334. pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
  335. unsigned long pfn;
  336. ptep++;
  337. addr += PAGE_SIZE;
  338. off++;
  339. if (off >= PTRS_PER_PTE) {
  340. off = 0;
  341. ptep = consistent_pte[++idx];
  342. }
  343. if (!pte_none(pte) && pte_present(pte)) {
  344. pfn = pte_pfn(pte);
  345. if (pfn_valid(pfn)) {
  346. struct page *page = pfn_to_page(pfn);
  347. /*
  348. * x86 does not mark the pages reserved...
  349. */
  350. ClearPageReserved(page);
  351. __free_page(page);
  352. continue;
  353. }
  354. }
  355. printk(KERN_CRIT "%s: bad page in kernel page table\n",
  356. __func__);
  357. } while (size -= PAGE_SIZE);
  358. flush_tlb_kernel_range(c->vm_start, c->vm_end);
  359. spin_lock_irqsave(&consistent_lock, flags);
  360. list_del(&c->vm_list);
  361. spin_unlock_irqrestore(&consistent_lock, flags);
  362. kfree(c);
  363. return;
  364. no_area:
  365. spin_unlock_irqrestore(&consistent_lock, flags);
  366. printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
  367. __func__, cpu_addr);
  368. dump_stack();
  369. }
  370. EXPORT_SYMBOL(dma_free_coherent);
  371. /*
  372. * Initialise the consistent memory allocation.
  373. */
  374. static int __init consistent_init(void)
  375. {
  376. pgd_t *pgd;
  377. pmd_t *pmd;
  378. pte_t *pte;
  379. int ret = 0, i = 0;
  380. u32 base = CONSISTENT_BASE;
  381. do {
  382. pgd = pgd_offset(&init_mm, base);
  383. pmd = pmd_alloc(&init_mm, pgd, base);
  384. if (!pmd) {
  385. printk(KERN_ERR "%s: no pmd tables\n", __func__);
  386. ret = -ENOMEM;
  387. break;
  388. }
  389. WARN_ON(!pmd_none(*pmd));
  390. pte = pte_alloc_kernel(pmd, base);
  391. if (!pte) {
  392. printk(KERN_ERR "%s: no pte tables\n", __func__);
  393. ret = -ENOMEM;
  394. break;
  395. }
  396. consistent_pte[i++] = pte;
  397. base += (1 << PGDIR_SHIFT);
  398. } while (base < CONSISTENT_END);
  399. return ret;
  400. }
  401. core_initcall(consistent_init);
  402. /*
  403. * Make an area consistent for devices.
  404. * Note: Drivers should NOT use this function directly, as it will break
  405. * platforms with CONFIG_DMABOUNCE.
  406. * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
  407. */
  408. void consistent_sync(const void *start, size_t size, int direction)
  409. {
  410. const void *end = start + size;
  411. BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(end - 1));
  412. switch (direction) {
  413. case DMA_FROM_DEVICE: /* invalidate only */
  414. dmac_inv_range(start, end);
  415. outer_inv_range(__pa(start), __pa(end));
  416. break;
  417. case DMA_TO_DEVICE: /* writeback only */
  418. dmac_clean_range(start, end);
  419. outer_clean_range(__pa(start), __pa(end));
  420. break;
  421. case DMA_BIDIRECTIONAL: /* writeback and invalidate */
  422. dmac_flush_range(start, end);
  423. outer_flush_range(__pa(start), __pa(end));
  424. break;
  425. default:
  426. BUG();
  427. }
  428. }
  429. EXPORT_SYMBOL(consistent_sync);