mremap.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. /*
  2. * mm/mremap.c
  3. *
  4. * (C) Copyright 1996 Linus Torvalds
  5. *
  6. * Address space accounting code <alan@redhat.com>
  7. * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
  8. */
  9. #include <linux/mm.h>
  10. #include <linux/hugetlb.h>
  11. #include <linux/slab.h>
  12. #include <linux/shm.h>
  13. #include <linux/mman.h>
  14. #include <linux/swap.h>
  15. #include <linux/capability.h>
  16. #include <linux/fs.h>
  17. #include <linux/highmem.h>
  18. #include <linux/security.h>
  19. #include <linux/syscalls.h>
  20. #include <asm/uaccess.h>
  21. #include <asm/cacheflush.h>
  22. #include <asm/tlbflush.h>
  23. static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
  24. {
  25. pgd_t *pgd;
  26. pud_t *pud;
  27. pmd_t *pmd;
  28. pgd = pgd_offset(mm, addr);
  29. if (pgd_none_or_clear_bad(pgd))
  30. return NULL;
  31. pud = pud_offset(pgd, addr);
  32. if (pud_none_or_clear_bad(pud))
  33. return NULL;
  34. pmd = pmd_offset(pud, addr);
  35. if (pmd_none_or_clear_bad(pmd))
  36. return NULL;
  37. return pmd;
  38. }
  39. static pmd_t *alloc_new_pmd(struct mm_struct *mm, unsigned long addr)
  40. {
  41. pgd_t *pgd;
  42. pud_t *pud;
  43. pmd_t *pmd;
  44. pgd = pgd_offset(mm, addr);
  45. pud = pud_alloc(mm, pgd, addr);
  46. if (!pud)
  47. return NULL;
  48. pmd = pmd_alloc(mm, pud, addr);
  49. if (!pmd)
  50. return NULL;
  51. if (!pmd_present(*pmd) && __pte_alloc(mm, pmd, addr))
  52. return NULL;
  53. return pmd;
  54. }
  55. static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
  56. unsigned long old_addr, unsigned long old_end,
  57. struct vm_area_struct *new_vma, pmd_t *new_pmd,
  58. unsigned long new_addr)
  59. {
  60. struct address_space *mapping = NULL;
  61. struct mm_struct *mm = vma->vm_mm;
  62. pte_t *old_pte, *new_pte, pte;
  63. spinlock_t *old_ptl, *new_ptl;
  64. if (vma->vm_file) {
  65. /*
  66. * Subtle point from Rajesh Venkatasubramanian: before
  67. * moving file-based ptes, we must lock vmtruncate out,
  68. * since it might clean the dst vma before the src vma,
  69. * and we propagate stale pages into the dst afterward.
  70. */
  71. mapping = vma->vm_file->f_mapping;
  72. spin_lock(&mapping->i_mmap_lock);
  73. if (new_vma->vm_truncate_count &&
  74. new_vma->vm_truncate_count != vma->vm_truncate_count)
  75. new_vma->vm_truncate_count = 0;
  76. }
  77. /*
  78. * We don't have to worry about the ordering of src and dst
  79. * pte locks because exclusive mmap_sem prevents deadlock.
  80. */
  81. old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
  82. new_pte = pte_offset_map_nested(new_pmd, new_addr);
  83. new_ptl = pte_lockptr(mm, new_pmd);
  84. if (new_ptl != old_ptl)
  85. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  86. arch_enter_lazy_mmu_mode();
  87. for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
  88. new_pte++, new_addr += PAGE_SIZE) {
  89. if (pte_none(*old_pte))
  90. continue;
  91. pte = ptep_clear_flush(vma, old_addr, old_pte);
  92. pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
  93. set_pte_at(mm, new_addr, new_pte, pte);
  94. }
  95. arch_leave_lazy_mmu_mode();
  96. if (new_ptl != old_ptl)
  97. spin_unlock(new_ptl);
  98. pte_unmap_nested(new_pte - 1);
  99. pte_unmap_unlock(old_pte - 1, old_ptl);
  100. if (mapping)
  101. spin_unlock(&mapping->i_mmap_lock);
  102. }
  103. #define LATENCY_LIMIT (64 * PAGE_SIZE)
  104. static unsigned long move_page_tables(struct vm_area_struct *vma,
  105. unsigned long old_addr, struct vm_area_struct *new_vma,
  106. unsigned long new_addr, unsigned long len)
  107. {
  108. unsigned long extent, next, old_end;
  109. pmd_t *old_pmd, *new_pmd;
  110. old_end = old_addr + len;
  111. flush_cache_range(vma, old_addr, old_end);
  112. for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
  113. cond_resched();
  114. next = (old_addr + PMD_SIZE) & PMD_MASK;
  115. if (next - 1 > old_end)
  116. next = old_end;
  117. extent = next - old_addr;
  118. old_pmd = get_old_pmd(vma->vm_mm, old_addr);
  119. if (!old_pmd)
  120. continue;
  121. new_pmd = alloc_new_pmd(vma->vm_mm, new_addr);
  122. if (!new_pmd)
  123. break;
  124. next = (new_addr + PMD_SIZE) & PMD_MASK;
  125. if (extent > next - new_addr)
  126. extent = next - new_addr;
  127. if (extent > LATENCY_LIMIT)
  128. extent = LATENCY_LIMIT;
  129. move_ptes(vma, old_pmd, old_addr, old_addr + extent,
  130. new_vma, new_pmd, new_addr);
  131. }
  132. return len + old_addr - old_end; /* how much done */
  133. }
  134. static unsigned long move_vma(struct vm_area_struct *vma,
  135. unsigned long old_addr, unsigned long old_len,
  136. unsigned long new_len, unsigned long new_addr)
  137. {
  138. struct mm_struct *mm = vma->vm_mm;
  139. struct vm_area_struct *new_vma;
  140. unsigned long vm_flags = vma->vm_flags;
  141. unsigned long new_pgoff;
  142. unsigned long moved_len;
  143. unsigned long excess = 0;
  144. unsigned long hiwater_vm;
  145. int split = 0;
  146. /*
  147. * We'd prefer to avoid failure later on in do_munmap:
  148. * which may split one vma into three before unmapping.
  149. */
  150. if (mm->map_count >= sysctl_max_map_count - 3)
  151. return -ENOMEM;
  152. new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
  153. new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
  154. if (!new_vma)
  155. return -ENOMEM;
  156. moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
  157. if (moved_len < old_len) {
  158. /*
  159. * On error, move entries back from new area to old,
  160. * which will succeed since page tables still there,
  161. * and then proceed to unmap new area instead of old.
  162. */
  163. move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
  164. vma = new_vma;
  165. old_len = new_len;
  166. old_addr = new_addr;
  167. new_addr = -ENOMEM;
  168. }
  169. /* Conceal VM_ACCOUNT so old reservation is not undone */
  170. if (vm_flags & VM_ACCOUNT) {
  171. vma->vm_flags &= ~VM_ACCOUNT;
  172. excess = vma->vm_end - vma->vm_start - old_len;
  173. if (old_addr > vma->vm_start &&
  174. old_addr + old_len < vma->vm_end)
  175. split = 1;
  176. }
  177. /*
  178. * If we failed to move page tables we still do total_vm increment
  179. * since do_munmap() will decrement it by old_len == new_len.
  180. *
  181. * Since total_vm is about to be raised artificially high for a
  182. * moment, we need to restore high watermark afterwards: if stats
  183. * are taken meanwhile, total_vm and hiwater_vm appear too high.
  184. * If this were a serious issue, we'd add a flag to do_munmap().
  185. */
  186. hiwater_vm = mm->hiwater_vm;
  187. mm->total_vm += new_len >> PAGE_SHIFT;
  188. vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
  189. if (do_munmap(mm, old_addr, old_len) < 0) {
  190. /* OOM: unable to split vma, just get accounts right */
  191. vm_unacct_memory(excess >> PAGE_SHIFT);
  192. excess = 0;
  193. }
  194. mm->hiwater_vm = hiwater_vm;
  195. /* Restore VM_ACCOUNT if one or two pieces of vma left */
  196. if (excess) {
  197. vma->vm_flags |= VM_ACCOUNT;
  198. if (split)
  199. vma->vm_next->vm_flags |= VM_ACCOUNT;
  200. }
  201. if (vm_flags & VM_LOCKED) {
  202. mm->locked_vm += new_len >> PAGE_SHIFT;
  203. if (new_len > old_len)
  204. make_pages_present(new_addr + old_len,
  205. new_addr + new_len);
  206. }
  207. return new_addr;
  208. }
  209. /*
  210. * Expand (or shrink) an existing mapping, potentially moving it at the
  211. * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
  212. *
  213. * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
  214. * This option implies MREMAP_MAYMOVE.
  215. */
  216. unsigned long do_mremap(unsigned long addr,
  217. unsigned long old_len, unsigned long new_len,
  218. unsigned long flags, unsigned long new_addr)
  219. {
  220. struct mm_struct *mm = current->mm;
  221. struct vm_area_struct *vma;
  222. unsigned long ret = -EINVAL;
  223. unsigned long charged = 0;
  224. if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
  225. goto out;
  226. if (addr & ~PAGE_MASK)
  227. goto out;
  228. old_len = PAGE_ALIGN(old_len);
  229. new_len = PAGE_ALIGN(new_len);
  230. /*
  231. * We allow a zero old-len as a special case
  232. * for DOS-emu "duplicate shm area" thing. But
  233. * a zero new-len is nonsensical.
  234. */
  235. if (!new_len)
  236. goto out;
  237. /* new_addr is only valid if MREMAP_FIXED is specified */
  238. if (flags & MREMAP_FIXED) {
  239. if (new_addr & ~PAGE_MASK)
  240. goto out;
  241. if (!(flags & MREMAP_MAYMOVE))
  242. goto out;
  243. if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
  244. goto out;
  245. /* Check if the location we're moving into overlaps the
  246. * old location at all, and fail if it does.
  247. */
  248. if ((new_addr <= addr) && (new_addr+new_len) > addr)
  249. goto out;
  250. if ((addr <= new_addr) && (addr+old_len) > new_addr)
  251. goto out;
  252. ret = do_munmap(mm, new_addr, new_len);
  253. if (ret)
  254. goto out;
  255. }
  256. /*
  257. * Always allow a shrinking remap: that just unmaps
  258. * the unnecessary pages..
  259. * do_munmap does all the needed commit accounting
  260. */
  261. if (old_len >= new_len) {
  262. ret = do_munmap(mm, addr+new_len, old_len - new_len);
  263. if (ret && old_len != new_len)
  264. goto out;
  265. ret = addr;
  266. if (!(flags & MREMAP_FIXED) || (new_addr == addr))
  267. goto out;
  268. old_len = new_len;
  269. }
  270. /*
  271. * Ok, we need to grow.. or relocate.
  272. */
  273. ret = -EFAULT;
  274. vma = find_vma(mm, addr);
  275. if (!vma || vma->vm_start > addr)
  276. goto out;
  277. if (is_vm_hugetlb_page(vma)) {
  278. ret = -EINVAL;
  279. goto out;
  280. }
  281. /* We can't remap across vm area boundaries */
  282. if (old_len > vma->vm_end - addr)
  283. goto out;
  284. if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) {
  285. if (new_len > old_len)
  286. goto out;
  287. }
  288. if (vma->vm_flags & VM_LOCKED) {
  289. unsigned long locked, lock_limit;
  290. locked = mm->locked_vm << PAGE_SHIFT;
  291. lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
  292. locked += new_len - old_len;
  293. ret = -EAGAIN;
  294. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  295. goto out;
  296. }
  297. if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) {
  298. ret = -ENOMEM;
  299. goto out;
  300. }
  301. if (vma->vm_flags & VM_ACCOUNT) {
  302. charged = (new_len - old_len) >> PAGE_SHIFT;
  303. if (security_vm_enough_memory(charged))
  304. goto out_nc;
  305. }
  306. /* old_len exactly to the end of the area..
  307. * And we're not relocating the area.
  308. */
  309. if (old_len == vma->vm_end - addr &&
  310. !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
  311. (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
  312. unsigned long max_addr = TASK_SIZE;
  313. if (vma->vm_next)
  314. max_addr = vma->vm_next->vm_start;
  315. /* can we just expand the current mapping? */
  316. if (max_addr - addr >= new_len) {
  317. int pages = (new_len - old_len) >> PAGE_SHIFT;
  318. vma_adjust(vma, vma->vm_start,
  319. addr + new_len, vma->vm_pgoff, NULL);
  320. mm->total_vm += pages;
  321. vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
  322. if (vma->vm_flags & VM_LOCKED) {
  323. mm->locked_vm += pages;
  324. make_pages_present(addr + old_len,
  325. addr + new_len);
  326. }
  327. ret = addr;
  328. goto out;
  329. }
  330. }
  331. /*
  332. * We weren't able to just expand or shrink the area,
  333. * we need to create a new one and move it..
  334. */
  335. ret = -ENOMEM;
  336. if (flags & MREMAP_MAYMOVE) {
  337. if (!(flags & MREMAP_FIXED)) {
  338. unsigned long map_flags = 0;
  339. if (vma->vm_flags & VM_MAYSHARE)
  340. map_flags |= MAP_SHARED;
  341. new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
  342. vma->vm_pgoff, map_flags);
  343. ret = new_addr;
  344. if (new_addr & ~PAGE_MASK)
  345. goto out;
  346. }
  347. ret = move_vma(vma, addr, old_len, new_len, new_addr);
  348. }
  349. out:
  350. if (ret & ~PAGE_MASK)
  351. vm_unacct_memory(charged);
  352. out_nc:
  353. return ret;
  354. }
  355. asmlinkage unsigned long sys_mremap(unsigned long addr,
  356. unsigned long old_len, unsigned long new_len,
  357. unsigned long flags, unsigned long new_addr)
  358. {
  359. unsigned long ret;
  360. down_write(&current->mm->mmap_sem);
  361. ret = do_mremap(addr, old_len, new_len, flags, new_addr);
  362. up_write(&current->mm->mmap_sem);
  363. return ret;
  364. }