ioremap.c 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. /*
  2. * linux/arch/arm/mm/ioremap.c
  3. *
  4. * Re-map IO memory to kernel address space so that we can access it.
  5. *
  6. * (C) Copyright 1995 1996 Linus Torvalds
  7. *
  8. * Hacked for ARM by Phil Blundell <philb@gnu.org>
  9. * Hacked to allow all architectures to build, and various cleanups
  10. * by Russell King
  11. *
  12. * This allows a driver to remap an arbitrary region of bus memory into
  13. * virtual space. One should *only* use readl, writel, memcpy_toio and
  14. * so on with such remapped areas.
  15. *
  16. * Because the ARM only has a 32-bit address space we can't address the
  17. * whole of the (physical) PCI space at once. PCI huge-mode addressing
  18. * allows us to circumvent this restriction by splitting PCI space into
  19. * two 2GB chunks and mapping only one at a time into processor memory.
  20. * We use MMU protection domains to trap any attempt to access the bank
  21. * that is not currently mapped. (This isn't fully implemented yet.)
  22. */
  23. #include <linux/module.h>
  24. #include <linux/errno.h>
  25. #include <linux/mm.h>
  26. #include <linux/vmalloc.h>
  27. #include <asm/cacheflush.h>
  28. #include <asm/io.h>
  29. #include <asm/mmu_context.h>
  30. #include <asm/pgalloc.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/sizes.h>
  33. /*
  34. * Used by ioremap() and iounmap() code to mark (super)section-mapped
  35. * I/O regions in vm_struct->flags field.
  36. */
  37. #define VM_ARM_SECTION_MAPPING 0x80000000
  38. static int remap_area_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
  39. unsigned long phys_addr, pgprot_t prot)
  40. {
  41. pte_t *pte;
  42. pte = pte_alloc_kernel(pmd, addr);
  43. if (!pte)
  44. return -ENOMEM;
  45. do {
  46. if (!pte_none(*pte))
  47. goto bad;
  48. set_pte_ext(pte, pfn_pte(phys_addr >> PAGE_SHIFT, prot), 0);
  49. phys_addr += PAGE_SIZE;
  50. } while (pte++, addr += PAGE_SIZE, addr != end);
  51. return 0;
  52. bad:
  53. printk(KERN_CRIT "remap_area_pte: page already exists\n");
  54. BUG();
  55. }
  56. static inline int remap_area_pmd(pgd_t *pgd, unsigned long addr,
  57. unsigned long end, unsigned long phys_addr,
  58. pgprot_t prot)
  59. {
  60. unsigned long next;
  61. pmd_t *pmd;
  62. int ret = 0;
  63. pmd = pmd_alloc(&init_mm, pgd, addr);
  64. if (!pmd)
  65. return -ENOMEM;
  66. do {
  67. next = pmd_addr_end(addr, end);
  68. ret = remap_area_pte(pmd, addr, next, phys_addr, prot);
  69. if (ret)
  70. return ret;
  71. phys_addr += next - addr;
  72. } while (pmd++, addr = next, addr != end);
  73. return ret;
  74. }
  75. static int remap_area_pages(unsigned long start, unsigned long pfn,
  76. unsigned long size, unsigned long flags)
  77. {
  78. unsigned long addr = start;
  79. unsigned long next, end = start + size;
  80. unsigned long phys_addr = __pfn_to_phys(pfn);
  81. pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
  82. L_PTE_DIRTY | L_PTE_WRITE | flags);
  83. pgd_t *pgd;
  84. int err = 0;
  85. BUG_ON(addr >= end);
  86. pgd = pgd_offset_k(addr);
  87. do {
  88. next = pgd_addr_end(addr, end);
  89. err = remap_area_pmd(pgd, addr, next, phys_addr, prot);
  90. if (err)
  91. break;
  92. phys_addr += next - addr;
  93. } while (pgd++, addr = next, addr != end);
  94. return err;
  95. }
  96. void __check_kvm_seq(struct mm_struct *mm)
  97. {
  98. unsigned int seq;
  99. do {
  100. seq = init_mm.context.kvm_seq;
  101. memcpy(pgd_offset(mm, VMALLOC_START),
  102. pgd_offset_k(VMALLOC_START),
  103. sizeof(pgd_t) * (pgd_index(VMALLOC_END) -
  104. pgd_index(VMALLOC_START)));
  105. mm->context.kvm_seq = seq;
  106. } while (seq != init_mm.context.kvm_seq);
  107. }
  108. #ifndef CONFIG_SMP
  109. /*
  110. * Section support is unsafe on SMP - If you iounmap and ioremap a region,
  111. * the other CPUs will not see this change until their next context switch.
  112. * Meanwhile, (eg) if an interrupt comes in on one of those other CPUs
  113. * which requires the new ioremap'd region to be referenced, the CPU will
  114. * reference the _old_ region.
  115. *
  116. * Note that get_vm_area() allocates a guard 4K page, so we need to mask
  117. * the size back to 1MB aligned or we will overflow in the loop below.
  118. */
  119. static void unmap_area_sections(unsigned long virt, unsigned long size)
  120. {
  121. unsigned long addr = virt, end = virt + (size & ~SZ_1M);
  122. pgd_t *pgd;
  123. flush_cache_vunmap(addr, end);
  124. pgd = pgd_offset_k(addr);
  125. do {
  126. pmd_t pmd, *pmdp = pmd_offset(pgd, addr);
  127. pmd = *pmdp;
  128. if (!pmd_none(pmd)) {
  129. /*
  130. * Clear the PMD from the page table, and
  131. * increment the kvm sequence so others
  132. * notice this change.
  133. *
  134. * Note: this is still racy on SMP machines.
  135. */
  136. pmd_clear(pmdp);
  137. init_mm.context.kvm_seq++;
  138. /*
  139. * Free the page table, if there was one.
  140. */
  141. if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE)
  142. pte_free_kernel(pmd_page_vaddr(pmd));
  143. }
  144. addr += PGDIR_SIZE;
  145. pgd++;
  146. } while (addr < end);
  147. /*
  148. * Ensure that the active_mm is up to date - we want to
  149. * catch any use-after-iounmap cases.
  150. */
  151. if (current->active_mm->context.kvm_seq != init_mm.context.kvm_seq)
  152. __check_kvm_seq(current->active_mm);
  153. flush_tlb_kernel_range(virt, end);
  154. }
  155. static int
  156. remap_area_sections(unsigned long virt, unsigned long pfn,
  157. unsigned long size, unsigned long flags)
  158. {
  159. unsigned long prot, addr = virt, end = virt + size;
  160. pgd_t *pgd;
  161. /*
  162. * Remove and free any PTE-based mapping, and
  163. * sync the current kernel mapping.
  164. */
  165. unmap_area_sections(virt, size);
  166. prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO) |
  167. (flags & (L_PTE_CACHEABLE | L_PTE_BUFFERABLE));
  168. /*
  169. * ARMv6 and above need XN set to prevent speculative prefetches
  170. * hitting IO.
  171. */
  172. if (cpu_architecture() >= CPU_ARCH_ARMv6)
  173. prot |= PMD_SECT_XN;
  174. pgd = pgd_offset_k(addr);
  175. do {
  176. pmd_t *pmd = pmd_offset(pgd, addr);
  177. pmd[0] = __pmd(__pfn_to_phys(pfn) | prot);
  178. pfn += SZ_1M >> PAGE_SHIFT;
  179. pmd[1] = __pmd(__pfn_to_phys(pfn) | prot);
  180. pfn += SZ_1M >> PAGE_SHIFT;
  181. flush_pmd_entry(pmd);
  182. addr += PGDIR_SIZE;
  183. pgd++;
  184. } while (addr < end);
  185. return 0;
  186. }
  187. static int
  188. remap_area_supersections(unsigned long virt, unsigned long pfn,
  189. unsigned long size, unsigned long flags)
  190. {
  191. unsigned long prot, addr = virt, end = virt + size;
  192. pgd_t *pgd;
  193. /*
  194. * Remove and free any PTE-based mapping, and
  195. * sync the current kernel mapping.
  196. */
  197. unmap_area_sections(virt, size);
  198. prot = PMD_TYPE_SECT | PMD_SECT_SUPER | PMD_SECT_AP_WRITE |
  199. PMD_DOMAIN(DOMAIN_IO) |
  200. (flags & (L_PTE_CACHEABLE | L_PTE_BUFFERABLE));
  201. /*
  202. * ARMv6 and above need XN set to prevent speculative prefetches
  203. * hitting IO.
  204. */
  205. if (cpu_architecture() >= CPU_ARCH_ARMv6)
  206. prot |= PMD_SECT_XN;
  207. pgd = pgd_offset_k(virt);
  208. do {
  209. unsigned long super_pmd_val, i;
  210. super_pmd_val = __pfn_to_phys(pfn) | prot;
  211. super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20;
  212. for (i = 0; i < 8; i++) {
  213. pmd_t *pmd = pmd_offset(pgd, addr);
  214. pmd[0] = __pmd(super_pmd_val);
  215. pmd[1] = __pmd(super_pmd_val);
  216. flush_pmd_entry(pmd);
  217. addr += PGDIR_SIZE;
  218. pgd++;
  219. }
  220. pfn += SUPERSECTION_SIZE >> PAGE_SHIFT;
  221. } while (addr < end);
  222. return 0;
  223. }
  224. #endif
  225. /*
  226. * Remap an arbitrary physical address space into the kernel virtual
  227. * address space. Needed when the kernel wants to access high addresses
  228. * directly.
  229. *
  230. * NOTE! We need to allow non-page-aligned mappings too: we will obviously
  231. * have to convert them into an offset in a page-aligned mapping, but the
  232. * caller shouldn't need to know that small detail.
  233. *
  234. * 'flags' are the extra L_PTE_ flags that you want to specify for this
  235. * mapping. See include/asm-arm/proc-armv/pgtable.h for more information.
  236. */
  237. void __iomem *
  238. __ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
  239. unsigned long flags)
  240. {
  241. int err;
  242. unsigned long addr;
  243. struct vm_struct * area;
  244. /*
  245. * High mappings must be supersection aligned
  246. */
  247. if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
  248. return NULL;
  249. size = PAGE_ALIGN(size);
  250. area = get_vm_area(size, VM_IOREMAP);
  251. if (!area)
  252. return NULL;
  253. addr = (unsigned long)area->addr;
  254. #ifndef CONFIG_SMP
  255. if (DOMAIN_IO == 0 &&
  256. (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
  257. cpu_is_xsc3()) &&
  258. !((__pfn_to_phys(pfn) | size | addr) & ~SUPERSECTION_MASK)) {
  259. area->flags |= VM_ARM_SECTION_MAPPING;
  260. err = remap_area_supersections(addr, pfn, size, flags);
  261. } else if (!((__pfn_to_phys(pfn) | size | addr) & ~PMD_MASK)) {
  262. area->flags |= VM_ARM_SECTION_MAPPING;
  263. err = remap_area_sections(addr, pfn, size, flags);
  264. } else
  265. #endif
  266. err = remap_area_pages(addr, pfn, size, flags);
  267. if (err) {
  268. vunmap((void *)addr);
  269. return NULL;
  270. }
  271. flush_cache_vmap(addr, addr + size);
  272. return (void __iomem *) (offset + addr);
  273. }
  274. EXPORT_SYMBOL(__ioremap_pfn);
  275. void __iomem *
  276. __ioremap(unsigned long phys_addr, size_t size, unsigned long flags)
  277. {
  278. unsigned long last_addr;
  279. unsigned long offset = phys_addr & ~PAGE_MASK;
  280. unsigned long pfn = __phys_to_pfn(phys_addr);
  281. /*
  282. * Don't allow wraparound or zero size
  283. */
  284. last_addr = phys_addr + size - 1;
  285. if (!size || last_addr < phys_addr)
  286. return NULL;
  287. /*
  288. * Page align the mapping size
  289. */
  290. size = PAGE_ALIGN(last_addr + 1) - phys_addr;
  291. return __ioremap_pfn(pfn, offset, size, flags);
  292. }
  293. EXPORT_SYMBOL(__ioremap);
  294. void __iounmap(volatile void __iomem *addr)
  295. {
  296. #ifndef CONFIG_SMP
  297. struct vm_struct **p, *tmp;
  298. #endif
  299. unsigned int section_mapping = 0;
  300. addr = (volatile void __iomem *)(PAGE_MASK & (unsigned long)addr);
  301. #ifndef CONFIG_SMP
  302. /*
  303. * If this is a section based mapping we need to handle it
  304. * specially as the VM subysystem does not know how to handle
  305. * such a beast. We need the lock here b/c we need to clear
  306. * all the mappings before the area can be reclaimed
  307. * by someone else.
  308. */
  309. write_lock(&vmlist_lock);
  310. for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) {
  311. if((tmp->flags & VM_IOREMAP) && (tmp->addr == addr)) {
  312. if (tmp->flags & VM_ARM_SECTION_MAPPING) {
  313. *p = tmp->next;
  314. unmap_area_sections((unsigned long)tmp->addr,
  315. tmp->size);
  316. kfree(tmp);
  317. section_mapping = 1;
  318. }
  319. break;
  320. }
  321. }
  322. write_unlock(&vmlist_lock);
  323. #endif
  324. if (!section_mapping)
  325. vunmap((void __force *)addr);
  326. }
  327. EXPORT_SYMBOL(__iounmap);