mmap.c 56 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177
  1. /*
  2. * mm/mmap.c
  3. *
  4. * Written by obz.
  5. *
  6. * Address space accounting code <alan@redhat.com>
  7. */
  8. #include <linux/slab.h>
  9. #include <linux/mm.h>
  10. #include <linux/shm.h>
  11. #include <linux/mman.h>
  12. #include <linux/pagemap.h>
  13. #include <linux/swap.h>
  14. #include <linux/syscalls.h>
  15. #include <linux/capability.h>
  16. #include <linux/init.h>
  17. #include <linux/file.h>
  18. #include <linux/fs.h>
  19. #include <linux/personality.h>
  20. #include <linux/security.h>
  21. #include <linux/hugetlb.h>
  22. #include <linux/profile.h>
  23. #include <linux/module.h>
  24. #include <linux/mount.h>
  25. #include <linux/mempolicy.h>
  26. #include <linux/rmap.h>
  27. #include <asm/uaccess.h>
  28. #include <asm/cacheflush.h>
  29. #include <asm/tlb.h>
  30. #ifndef arch_mmap_check
  31. #define arch_mmap_check(addr, len, flags) (0)
  32. #endif
  33. static void unmap_region(struct mm_struct *mm,
  34. struct vm_area_struct *vma, struct vm_area_struct *prev,
  35. unsigned long start, unsigned long end);
  36. /*
  37. * WARNING: the debugging will use recursive algorithms so never enable this
  38. * unless you know what you are doing.
  39. */
  40. #undef DEBUG_MM_RB
  41. /* description of effects of mapping type and prot in current implementation.
  42. * this is due to the limited x86 page protection hardware. The expected
  43. * behavior is in parens:
  44. *
  45. * map_type prot
  46. * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
  47. * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
  48. * w: (no) no w: (no) no w: (yes) yes w: (no) no
  49. * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
  50. *
  51. * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
  52. * w: (no) no w: (no) no w: (copy) copy w: (no) no
  53. * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
  54. *
  55. */
  56. pgprot_t protection_map[16] = {
  57. __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
  58. __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
  59. };
  60. pgprot_t vm_get_page_prot(unsigned long vm_flags)
  61. {
  62. return protection_map[vm_flags &
  63. (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)];
  64. }
  65. EXPORT_SYMBOL(vm_get_page_prot);
  66. int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
  67. int sysctl_overcommit_ratio = 50; /* default is 50% */
  68. int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
  69. atomic_t vm_committed_space = ATOMIC_INIT(0);
  70. /*
  71. * Check that a process has enough memory to allocate a new virtual
  72. * mapping. 0 means there is enough memory for the allocation to
  73. * succeed and -ENOMEM implies there is not.
  74. *
  75. * We currently support three overcommit policies, which are set via the
  76. * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
  77. *
  78. * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
  79. * Additional code 2002 Jul 20 by Robert Love.
  80. *
  81. * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
  82. *
  83. * Note this is a helper function intended to be used by LSMs which
  84. * wish to use this logic.
  85. */
  86. int __vm_enough_memory(long pages, int cap_sys_admin)
  87. {
  88. unsigned long free, allowed;
  89. vm_acct_memory(pages);
  90. /*
  91. * Sometimes we want to use more memory than we have
  92. */
  93. if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
  94. return 0;
  95. if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
  96. unsigned long n;
  97. free = global_page_state(NR_FILE_PAGES);
  98. free += nr_swap_pages;
  99. /*
  100. * Any slabs which are created with the
  101. * SLAB_RECLAIM_ACCOUNT flag claim to have contents
  102. * which are reclaimable, under pressure. The dentry
  103. * cache and most inode caches should fall into this
  104. */
  105. free += global_page_state(NR_SLAB_RECLAIMABLE);
  106. /*
  107. * Leave the last 3% for root
  108. */
  109. if (!cap_sys_admin)
  110. free -= free / 32;
  111. if (free > pages)
  112. return 0;
  113. /*
  114. * nr_free_pages() is very expensive on large systems,
  115. * only call if we're about to fail.
  116. */
  117. n = nr_free_pages();
  118. /*
  119. * Leave reserved pages. The pages are not for anonymous pages.
  120. */
  121. if (n <= totalreserve_pages)
  122. goto error;
  123. else
  124. n -= totalreserve_pages;
  125. /*
  126. * Leave the last 3% for root
  127. */
  128. if (!cap_sys_admin)
  129. n -= n / 32;
  130. free += n;
  131. if (free > pages)
  132. return 0;
  133. goto error;
  134. }
  135. allowed = (totalram_pages - hugetlb_total_pages())
  136. * sysctl_overcommit_ratio / 100;
  137. /*
  138. * Leave the last 3% for root
  139. */
  140. if (!cap_sys_admin)
  141. allowed -= allowed / 32;
  142. allowed += total_swap_pages;
  143. /* Don't let a single process grow too big:
  144. leave 3% of the size of this process for other processes */
  145. allowed -= current->mm->total_vm / 32;
  146. /*
  147. * cast `allowed' as a signed long because vm_committed_space
  148. * sometimes has a negative value
  149. */
  150. if (atomic_read(&vm_committed_space) < (long)allowed)
  151. return 0;
  152. error:
  153. vm_unacct_memory(pages);
  154. return -ENOMEM;
  155. }
  156. EXPORT_SYMBOL(__vm_enough_memory);
  157. /*
  158. * Requires inode->i_mapping->i_mmap_lock
  159. */
  160. static void __remove_shared_vm_struct(struct vm_area_struct *vma,
  161. struct file *file, struct address_space *mapping)
  162. {
  163. if (vma->vm_flags & VM_DENYWRITE)
  164. atomic_inc(&file->f_path.dentry->d_inode->i_writecount);
  165. if (vma->vm_flags & VM_SHARED)
  166. mapping->i_mmap_writable--;
  167. flush_dcache_mmap_lock(mapping);
  168. if (unlikely(vma->vm_flags & VM_NONLINEAR))
  169. list_del_init(&vma->shared.vm_set.list);
  170. else
  171. vma_prio_tree_remove(vma, &mapping->i_mmap);
  172. flush_dcache_mmap_unlock(mapping);
  173. }
  174. /*
  175. * Unlink a file-based vm structure from its prio_tree, to hide
  176. * vma from rmap and vmtruncate before freeing its page tables.
  177. */
  178. void unlink_file_vma(struct vm_area_struct *vma)
  179. {
  180. struct file *file = vma->vm_file;
  181. if (file) {
  182. struct address_space *mapping = file->f_mapping;
  183. spin_lock(&mapping->i_mmap_lock);
  184. __remove_shared_vm_struct(vma, file, mapping);
  185. spin_unlock(&mapping->i_mmap_lock);
  186. }
  187. }
  188. /*
  189. * Close a vm structure and free it, returning the next.
  190. */
  191. static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
  192. {
  193. struct vm_area_struct *next = vma->vm_next;
  194. might_sleep();
  195. if (vma->vm_ops && vma->vm_ops->close)
  196. vma->vm_ops->close(vma);
  197. if (vma->vm_file)
  198. fput(vma->vm_file);
  199. mpol_free(vma_policy(vma));
  200. kmem_cache_free(vm_area_cachep, vma);
  201. return next;
  202. }
  203. asmlinkage unsigned long sys_brk(unsigned long brk)
  204. {
  205. unsigned long rlim, retval;
  206. unsigned long newbrk, oldbrk;
  207. struct mm_struct *mm = current->mm;
  208. down_write(&mm->mmap_sem);
  209. if (brk < mm->end_code)
  210. goto out;
  211. /*
  212. * Check against rlimit here. If this check is done later after the test
  213. * of oldbrk with newbrk then it can escape the test and let the data
  214. * segment grow beyond its set limit the in case where the limit is
  215. * not page aligned -Ram Gupta
  216. */
  217. rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
  218. if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim)
  219. goto out;
  220. newbrk = PAGE_ALIGN(brk);
  221. oldbrk = PAGE_ALIGN(mm->brk);
  222. if (oldbrk == newbrk)
  223. goto set_brk;
  224. /* Always allow shrinking brk. */
  225. if (brk <= mm->brk) {
  226. if (!do_munmap(mm, newbrk, oldbrk-newbrk))
  227. goto set_brk;
  228. goto out;
  229. }
  230. /* Check against existing mmap mappings. */
  231. if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
  232. goto out;
  233. /* Ok, looks good - let it rip. */
  234. if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
  235. goto out;
  236. set_brk:
  237. mm->brk = brk;
  238. out:
  239. retval = mm->brk;
  240. up_write(&mm->mmap_sem);
  241. return retval;
  242. }
  243. #ifdef DEBUG_MM_RB
  244. static int browse_rb(struct rb_root *root)
  245. {
  246. int i = 0, j;
  247. struct rb_node *nd, *pn = NULL;
  248. unsigned long prev = 0, pend = 0;
  249. for (nd = rb_first(root); nd; nd = rb_next(nd)) {
  250. struct vm_area_struct *vma;
  251. vma = rb_entry(nd, struct vm_area_struct, vm_rb);
  252. if (vma->vm_start < prev)
  253. printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
  254. if (vma->vm_start < pend)
  255. printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
  256. if (vma->vm_start > vma->vm_end)
  257. printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
  258. i++;
  259. pn = nd;
  260. prev = vma->vm_start;
  261. pend = vma->vm_end;
  262. }
  263. j = 0;
  264. for (nd = pn; nd; nd = rb_prev(nd)) {
  265. j++;
  266. }
  267. if (i != j)
  268. printk("backwards %d, forwards %d\n", j, i), i = 0;
  269. return i;
  270. }
  271. void validate_mm(struct mm_struct *mm)
  272. {
  273. int bug = 0;
  274. int i = 0;
  275. struct vm_area_struct *tmp = mm->mmap;
  276. while (tmp) {
  277. tmp = tmp->vm_next;
  278. i++;
  279. }
  280. if (i != mm->map_count)
  281. printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
  282. i = browse_rb(&mm->mm_rb);
  283. if (i != mm->map_count)
  284. printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
  285. BUG_ON(bug);
  286. }
  287. #else
  288. #define validate_mm(mm) do { } while (0)
  289. #endif
  290. static struct vm_area_struct *
  291. find_vma_prepare(struct mm_struct *mm, unsigned long addr,
  292. struct vm_area_struct **pprev, struct rb_node ***rb_link,
  293. struct rb_node ** rb_parent)
  294. {
  295. struct vm_area_struct * vma;
  296. struct rb_node ** __rb_link, * __rb_parent, * rb_prev;
  297. __rb_link = &mm->mm_rb.rb_node;
  298. rb_prev = __rb_parent = NULL;
  299. vma = NULL;
  300. while (*__rb_link) {
  301. struct vm_area_struct *vma_tmp;
  302. __rb_parent = *__rb_link;
  303. vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
  304. if (vma_tmp->vm_end > addr) {
  305. vma = vma_tmp;
  306. if (vma_tmp->vm_start <= addr)
  307. return vma;
  308. __rb_link = &__rb_parent->rb_left;
  309. } else {
  310. rb_prev = __rb_parent;
  311. __rb_link = &__rb_parent->rb_right;
  312. }
  313. }
  314. *pprev = NULL;
  315. if (rb_prev)
  316. *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
  317. *rb_link = __rb_link;
  318. *rb_parent = __rb_parent;
  319. return vma;
  320. }
  321. static inline void
  322. __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
  323. struct vm_area_struct *prev, struct rb_node *rb_parent)
  324. {
  325. if (prev) {
  326. vma->vm_next = prev->vm_next;
  327. prev->vm_next = vma;
  328. } else {
  329. mm->mmap = vma;
  330. if (rb_parent)
  331. vma->vm_next = rb_entry(rb_parent,
  332. struct vm_area_struct, vm_rb);
  333. else
  334. vma->vm_next = NULL;
  335. }
  336. }
  337. void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
  338. struct rb_node **rb_link, struct rb_node *rb_parent)
  339. {
  340. rb_link_node(&vma->vm_rb, rb_parent, rb_link);
  341. rb_insert_color(&vma->vm_rb, &mm->mm_rb);
  342. }
  343. static inline void __vma_link_file(struct vm_area_struct *vma)
  344. {
  345. struct file * file;
  346. file = vma->vm_file;
  347. if (file) {
  348. struct address_space *mapping = file->f_mapping;
  349. if (vma->vm_flags & VM_DENYWRITE)
  350. atomic_dec(&file->f_path.dentry->d_inode->i_writecount);
  351. if (vma->vm_flags & VM_SHARED)
  352. mapping->i_mmap_writable++;
  353. flush_dcache_mmap_lock(mapping);
  354. if (unlikely(vma->vm_flags & VM_NONLINEAR))
  355. vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
  356. else
  357. vma_prio_tree_insert(vma, &mapping->i_mmap);
  358. flush_dcache_mmap_unlock(mapping);
  359. }
  360. }
  361. static void
  362. __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
  363. struct vm_area_struct *prev, struct rb_node **rb_link,
  364. struct rb_node *rb_parent)
  365. {
  366. __vma_link_list(mm, vma, prev, rb_parent);
  367. __vma_link_rb(mm, vma, rb_link, rb_parent);
  368. __anon_vma_link(vma);
  369. }
  370. static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
  371. struct vm_area_struct *prev, struct rb_node **rb_link,
  372. struct rb_node *rb_parent)
  373. {
  374. struct address_space *mapping = NULL;
  375. if (vma->vm_file)
  376. mapping = vma->vm_file->f_mapping;
  377. if (mapping) {
  378. spin_lock(&mapping->i_mmap_lock);
  379. vma->vm_truncate_count = mapping->truncate_count;
  380. }
  381. anon_vma_lock(vma);
  382. __vma_link(mm, vma, prev, rb_link, rb_parent);
  383. __vma_link_file(vma);
  384. anon_vma_unlock(vma);
  385. if (mapping)
  386. spin_unlock(&mapping->i_mmap_lock);
  387. mm->map_count++;
  388. validate_mm(mm);
  389. }
  390. /*
  391. * Helper for vma_adjust in the split_vma insert case:
  392. * insert vm structure into list and rbtree and anon_vma,
  393. * but it has already been inserted into prio_tree earlier.
  394. */
  395. static void
  396. __insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
  397. {
  398. struct vm_area_struct * __vma, * prev;
  399. struct rb_node ** rb_link, * rb_parent;
  400. __vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
  401. BUG_ON(__vma && __vma->vm_start < vma->vm_end);
  402. __vma_link(mm, vma, prev, rb_link, rb_parent);
  403. mm->map_count++;
  404. }
  405. static inline void
  406. __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
  407. struct vm_area_struct *prev)
  408. {
  409. prev->vm_next = vma->vm_next;
  410. rb_erase(&vma->vm_rb, &mm->mm_rb);
  411. if (mm->mmap_cache == vma)
  412. mm->mmap_cache = prev;
  413. }
  414. /*
  415. * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
  416. * is already present in an i_mmap tree without adjusting the tree.
  417. * The following helper function should be used when such adjustments
  418. * are necessary. The "insert" vma (if any) is to be inserted
  419. * before we drop the necessary locks.
  420. */
  421. void vma_adjust(struct vm_area_struct *vma, unsigned long start,
  422. unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
  423. {
  424. struct mm_struct *mm = vma->vm_mm;
  425. struct vm_area_struct *next = vma->vm_next;
  426. struct vm_area_struct *importer = NULL;
  427. struct address_space *mapping = NULL;
  428. struct prio_tree_root *root = NULL;
  429. struct file *file = vma->vm_file;
  430. struct anon_vma *anon_vma = NULL;
  431. long adjust_next = 0;
  432. int remove_next = 0;
  433. if (next && !insert) {
  434. if (end >= next->vm_end) {
  435. /*
  436. * vma expands, overlapping all the next, and
  437. * perhaps the one after too (mprotect case 6).
  438. */
  439. again: remove_next = 1 + (end > next->vm_end);
  440. end = next->vm_end;
  441. anon_vma = next->anon_vma;
  442. importer = vma;
  443. } else if (end > next->vm_start) {
  444. /*
  445. * vma expands, overlapping part of the next:
  446. * mprotect case 5 shifting the boundary up.
  447. */
  448. adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
  449. anon_vma = next->anon_vma;
  450. importer = vma;
  451. } else if (end < vma->vm_end) {
  452. /*
  453. * vma shrinks, and !insert tells it's not
  454. * split_vma inserting another: so it must be
  455. * mprotect case 4 shifting the boundary down.
  456. */
  457. adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
  458. anon_vma = next->anon_vma;
  459. importer = next;
  460. }
  461. }
  462. if (file) {
  463. mapping = file->f_mapping;
  464. if (!(vma->vm_flags & VM_NONLINEAR))
  465. root = &mapping->i_mmap;
  466. spin_lock(&mapping->i_mmap_lock);
  467. if (importer &&
  468. vma->vm_truncate_count != next->vm_truncate_count) {
  469. /*
  470. * unmap_mapping_range might be in progress:
  471. * ensure that the expanding vma is rescanned.
  472. */
  473. importer->vm_truncate_count = 0;
  474. }
  475. if (insert) {
  476. insert->vm_truncate_count = vma->vm_truncate_count;
  477. /*
  478. * Put into prio_tree now, so instantiated pages
  479. * are visible to arm/parisc __flush_dcache_page
  480. * throughout; but we cannot insert into address
  481. * space until vma start or end is updated.
  482. */
  483. __vma_link_file(insert);
  484. }
  485. }
  486. /*
  487. * When changing only vma->vm_end, we don't really need
  488. * anon_vma lock: but is that case worth optimizing out?
  489. */
  490. if (vma->anon_vma)
  491. anon_vma = vma->anon_vma;
  492. if (anon_vma) {
  493. spin_lock(&anon_vma->lock);
  494. /*
  495. * Easily overlooked: when mprotect shifts the boundary,
  496. * make sure the expanding vma has anon_vma set if the
  497. * shrinking vma had, to cover any anon pages imported.
  498. */
  499. if (importer && !importer->anon_vma) {
  500. importer->anon_vma = anon_vma;
  501. __anon_vma_link(importer);
  502. }
  503. }
  504. if (root) {
  505. flush_dcache_mmap_lock(mapping);
  506. vma_prio_tree_remove(vma, root);
  507. if (adjust_next)
  508. vma_prio_tree_remove(next, root);
  509. }
  510. vma->vm_start = start;
  511. vma->vm_end = end;
  512. vma->vm_pgoff = pgoff;
  513. if (adjust_next) {
  514. next->vm_start += adjust_next << PAGE_SHIFT;
  515. next->vm_pgoff += adjust_next;
  516. }
  517. if (root) {
  518. if (adjust_next)
  519. vma_prio_tree_insert(next, root);
  520. vma_prio_tree_insert(vma, root);
  521. flush_dcache_mmap_unlock(mapping);
  522. }
  523. if (remove_next) {
  524. /*
  525. * vma_merge has merged next into vma, and needs
  526. * us to remove next before dropping the locks.
  527. */
  528. __vma_unlink(mm, next, vma);
  529. if (file)
  530. __remove_shared_vm_struct(next, file, mapping);
  531. if (next->anon_vma)
  532. __anon_vma_merge(vma, next);
  533. } else if (insert) {
  534. /*
  535. * split_vma has split insert from vma, and needs
  536. * us to insert it before dropping the locks
  537. * (it may either follow vma or precede it).
  538. */
  539. __insert_vm_struct(mm, insert);
  540. }
  541. if (anon_vma)
  542. spin_unlock(&anon_vma->lock);
  543. if (mapping)
  544. spin_unlock(&mapping->i_mmap_lock);
  545. if (remove_next) {
  546. if (file)
  547. fput(file);
  548. mm->map_count--;
  549. mpol_free(vma_policy(next));
  550. kmem_cache_free(vm_area_cachep, next);
  551. /*
  552. * In mprotect's case 6 (see comments on vma_merge),
  553. * we must remove another next too. It would clutter
  554. * up the code too much to do both in one go.
  555. */
  556. if (remove_next == 2) {
  557. next = vma->vm_next;
  558. goto again;
  559. }
  560. }
  561. validate_mm(mm);
  562. }
  563. /*
  564. * If the vma has a ->close operation then the driver probably needs to release
  565. * per-vma resources, so we don't attempt to merge those.
  566. */
  567. #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_RESERVED | VM_PFNMAP)
  568. static inline int is_mergeable_vma(struct vm_area_struct *vma,
  569. struct file *file, unsigned long vm_flags)
  570. {
  571. if (vma->vm_flags != vm_flags)
  572. return 0;
  573. if (vma->vm_file != file)
  574. return 0;
  575. if (vma->vm_ops && vma->vm_ops->close)
  576. return 0;
  577. return 1;
  578. }
  579. static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
  580. struct anon_vma *anon_vma2)
  581. {
  582. return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
  583. }
  584. /*
  585. * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
  586. * in front of (at a lower virtual address and file offset than) the vma.
  587. *
  588. * We cannot merge two vmas if they have differently assigned (non-NULL)
  589. * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
  590. *
  591. * We don't check here for the merged mmap wrapping around the end of pagecache
  592. * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
  593. * wrap, nor mmaps which cover the final page at index -1UL.
  594. */
  595. static int
  596. can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
  597. struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
  598. {
  599. if (is_mergeable_vma(vma, file, vm_flags) &&
  600. is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
  601. if (vma->vm_pgoff == vm_pgoff)
  602. return 1;
  603. }
  604. return 0;
  605. }
  606. /*
  607. * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
  608. * beyond (at a higher virtual address and file offset than) the vma.
  609. *
  610. * We cannot merge two vmas if they have differently assigned (non-NULL)
  611. * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
  612. */
  613. static int
  614. can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
  615. struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
  616. {
  617. if (is_mergeable_vma(vma, file, vm_flags) &&
  618. is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
  619. pgoff_t vm_pglen;
  620. vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  621. if (vma->vm_pgoff + vm_pglen == vm_pgoff)
  622. return 1;
  623. }
  624. return 0;
  625. }
  626. /*
  627. * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
  628. * whether that can be merged with its predecessor or its successor.
  629. * Or both (it neatly fills a hole).
  630. *
  631. * In most cases - when called for mmap, brk or mremap - [addr,end) is
  632. * certain not to be mapped by the time vma_merge is called; but when
  633. * called for mprotect, it is certain to be already mapped (either at
  634. * an offset within prev, or at the start of next), and the flags of
  635. * this area are about to be changed to vm_flags - and the no-change
  636. * case has already been eliminated.
  637. *
  638. * The following mprotect cases have to be considered, where AAAA is
  639. * the area passed down from mprotect_fixup, never extending beyond one
  640. * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
  641. *
  642. * AAAA AAAA AAAA AAAA
  643. * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
  644. * cannot merge might become might become might become
  645. * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
  646. * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
  647. * mremap move: PPPPNNNNNNNN 8
  648. * AAAA
  649. * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
  650. * might become case 1 below case 2 below case 3 below
  651. *
  652. * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
  653. * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
  654. */
  655. struct vm_area_struct *vma_merge(struct mm_struct *mm,
  656. struct vm_area_struct *prev, unsigned long addr,
  657. unsigned long end, unsigned long vm_flags,
  658. struct anon_vma *anon_vma, struct file *file,
  659. pgoff_t pgoff, struct mempolicy *policy)
  660. {
  661. pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
  662. struct vm_area_struct *area, *next;
  663. /*
  664. * We later require that vma->vm_flags == vm_flags,
  665. * so this tests vma->vm_flags & VM_SPECIAL, too.
  666. */
  667. if (vm_flags & VM_SPECIAL)
  668. return NULL;
  669. if (prev)
  670. next = prev->vm_next;
  671. else
  672. next = mm->mmap;
  673. area = next;
  674. if (next && next->vm_end == end) /* cases 6, 7, 8 */
  675. next = next->vm_next;
  676. /*
  677. * Can it merge with the predecessor?
  678. */
  679. if (prev && prev->vm_end == addr &&
  680. mpol_equal(vma_policy(prev), policy) &&
  681. can_vma_merge_after(prev, vm_flags,
  682. anon_vma, file, pgoff)) {
  683. /*
  684. * OK, it can. Can we now merge in the successor as well?
  685. */
  686. if (next && end == next->vm_start &&
  687. mpol_equal(policy, vma_policy(next)) &&
  688. can_vma_merge_before(next, vm_flags,
  689. anon_vma, file, pgoff+pglen) &&
  690. is_mergeable_anon_vma(prev->anon_vma,
  691. next->anon_vma)) {
  692. /* cases 1, 6 */
  693. vma_adjust(prev, prev->vm_start,
  694. next->vm_end, prev->vm_pgoff, NULL);
  695. } else /* cases 2, 5, 7 */
  696. vma_adjust(prev, prev->vm_start,
  697. end, prev->vm_pgoff, NULL);
  698. return prev;
  699. }
  700. /*
  701. * Can this new request be merged in front of next?
  702. */
  703. if (next && end == next->vm_start &&
  704. mpol_equal(policy, vma_policy(next)) &&
  705. can_vma_merge_before(next, vm_flags,
  706. anon_vma, file, pgoff+pglen)) {
  707. if (prev && addr < prev->vm_end) /* case 4 */
  708. vma_adjust(prev, prev->vm_start,
  709. addr, prev->vm_pgoff, NULL);
  710. else /* cases 3, 8 */
  711. vma_adjust(area, addr, next->vm_end,
  712. next->vm_pgoff - pglen, NULL);
  713. return area;
  714. }
  715. return NULL;
  716. }
  717. /*
  718. * find_mergeable_anon_vma is used by anon_vma_prepare, to check
  719. * neighbouring vmas for a suitable anon_vma, before it goes off
  720. * to allocate a new anon_vma. It checks because a repetitive
  721. * sequence of mprotects and faults may otherwise lead to distinct
  722. * anon_vmas being allocated, preventing vma merge in subsequent
  723. * mprotect.
  724. */
  725. struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
  726. {
  727. struct vm_area_struct *near;
  728. unsigned long vm_flags;
  729. near = vma->vm_next;
  730. if (!near)
  731. goto try_prev;
  732. /*
  733. * Since only mprotect tries to remerge vmas, match flags
  734. * which might be mprotected into each other later on.
  735. * Neither mlock nor madvise tries to remerge at present,
  736. * so leave their flags as obstructing a merge.
  737. */
  738. vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
  739. vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
  740. if (near->anon_vma && vma->vm_end == near->vm_start &&
  741. mpol_equal(vma_policy(vma), vma_policy(near)) &&
  742. can_vma_merge_before(near, vm_flags,
  743. NULL, vma->vm_file, vma->vm_pgoff +
  744. ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
  745. return near->anon_vma;
  746. try_prev:
  747. /*
  748. * It is potentially slow to have to call find_vma_prev here.
  749. * But it's only on the first write fault on the vma, not
  750. * every time, and we could devise a way to avoid it later
  751. * (e.g. stash info in next's anon_vma_node when assigning
  752. * an anon_vma, or when trying vma_merge). Another time.
  753. */
  754. BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);
  755. if (!near)
  756. goto none;
  757. vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
  758. vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
  759. if (near->anon_vma && near->vm_end == vma->vm_start &&
  760. mpol_equal(vma_policy(near), vma_policy(vma)) &&
  761. can_vma_merge_after(near, vm_flags,
  762. NULL, vma->vm_file, vma->vm_pgoff))
  763. return near->anon_vma;
  764. none:
  765. /*
  766. * There's no absolute need to look only at touching neighbours:
  767. * we could search further afield for "compatible" anon_vmas.
  768. * But it would probably just be a waste of time searching,
  769. * or lead to too many vmas hanging off the same anon_vma.
  770. * We're trying to allow mprotect remerging later on,
  771. * not trying to minimize memory used for anon_vmas.
  772. */
  773. return NULL;
  774. }
  775. #ifdef CONFIG_PROC_FS
  776. void vm_stat_account(struct mm_struct *mm, unsigned long flags,
  777. struct file *file, long pages)
  778. {
  779. const unsigned long stack_flags
  780. = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
  781. if (file) {
  782. mm->shared_vm += pages;
  783. if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
  784. mm->exec_vm += pages;
  785. } else if (flags & stack_flags)
  786. mm->stack_vm += pages;
  787. if (flags & (VM_RESERVED|VM_IO))
  788. mm->reserved_vm += pages;
  789. }
  790. #endif /* CONFIG_PROC_FS */
  791. /*
  792. * The caller must hold down_write(current->mm->mmap_sem).
  793. */
  794. unsigned long do_mmap_pgoff(struct file * file, unsigned long addr,
  795. unsigned long len, unsigned long prot,
  796. unsigned long flags, unsigned long pgoff)
  797. {
  798. struct mm_struct * mm = current->mm;
  799. struct vm_area_struct * vma, * prev;
  800. struct inode *inode;
  801. unsigned int vm_flags;
  802. int correct_wcount = 0;
  803. int error;
  804. struct rb_node ** rb_link, * rb_parent;
  805. int accountable = 1;
  806. unsigned long charged = 0, reqprot = prot;
  807. /*
  808. * Does the application expect PROT_READ to imply PROT_EXEC?
  809. *
  810. * (the exception is when the underlying filesystem is noexec
  811. * mounted, in which case we dont add PROT_EXEC.)
  812. */
  813. if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
  814. if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
  815. prot |= PROT_EXEC;
  816. if (!len)
  817. return -EINVAL;
  818. error = arch_mmap_check(addr, len, flags);
  819. if (error)
  820. return error;
  821. /* Careful about overflows.. */
  822. len = PAGE_ALIGN(len);
  823. if (!len || len > TASK_SIZE)
  824. return -ENOMEM;
  825. /* offset overflow? */
  826. if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
  827. return -EOVERFLOW;
  828. /* Too many mappings? */
  829. if (mm->map_count > sysctl_max_map_count)
  830. return -ENOMEM;
  831. /* Obtain the address to map to. we verify (or select) it and ensure
  832. * that it represents a valid section of the address space.
  833. */
  834. addr = get_unmapped_area(file, addr, len, pgoff, flags);
  835. if (addr & ~PAGE_MASK)
  836. return addr;
  837. /* Do simple checking here so the lower-level routines won't have
  838. * to. we assume access permissions have been handled by the open
  839. * of the memory object, so we don't do any here.
  840. */
  841. vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
  842. mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
  843. if (flags & MAP_LOCKED) {
  844. if (!can_do_mlock())
  845. return -EPERM;
  846. vm_flags |= VM_LOCKED;
  847. }
  848. /* mlock MCL_FUTURE? */
  849. if (vm_flags & VM_LOCKED) {
  850. unsigned long locked, lock_limit;
  851. locked = len >> PAGE_SHIFT;
  852. locked += mm->locked_vm;
  853. lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
  854. lock_limit >>= PAGE_SHIFT;
  855. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  856. return -EAGAIN;
  857. }
  858. inode = file ? file->f_path.dentry->d_inode : NULL;
  859. if (file) {
  860. switch (flags & MAP_TYPE) {
  861. case MAP_SHARED:
  862. if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
  863. return -EACCES;
  864. /*
  865. * Make sure we don't allow writing to an append-only
  866. * file..
  867. */
  868. if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
  869. return -EACCES;
  870. /*
  871. * Make sure there are no mandatory locks on the file.
  872. */
  873. if (locks_verify_locked(inode))
  874. return -EAGAIN;
  875. vm_flags |= VM_SHARED | VM_MAYSHARE;
  876. if (!(file->f_mode & FMODE_WRITE))
  877. vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
  878. /* fall through */
  879. case MAP_PRIVATE:
  880. if (!(file->f_mode & FMODE_READ))
  881. return -EACCES;
  882. if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
  883. if (vm_flags & VM_EXEC)
  884. return -EPERM;
  885. vm_flags &= ~VM_MAYEXEC;
  886. }
  887. if (is_file_hugepages(file))
  888. accountable = 0;
  889. if (!file->f_op || !file->f_op->mmap)
  890. return -ENODEV;
  891. break;
  892. default:
  893. return -EINVAL;
  894. }
  895. } else {
  896. switch (flags & MAP_TYPE) {
  897. case MAP_SHARED:
  898. vm_flags |= VM_SHARED | VM_MAYSHARE;
  899. break;
  900. case MAP_PRIVATE:
  901. /*
  902. * Set pgoff according to addr for anon_vma.
  903. */
  904. pgoff = addr >> PAGE_SHIFT;
  905. break;
  906. default:
  907. return -EINVAL;
  908. }
  909. }
  910. error = security_file_mmap(file, reqprot, prot, flags);
  911. if (error)
  912. return error;
  913. /* Clear old maps */
  914. error = -ENOMEM;
  915. munmap_back:
  916. vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  917. if (vma && vma->vm_start < addr + len) {
  918. if (do_munmap(mm, addr, len))
  919. return -ENOMEM;
  920. goto munmap_back;
  921. }
  922. /* Check against address space limit. */
  923. if (!may_expand_vm(mm, len >> PAGE_SHIFT))
  924. return -ENOMEM;
  925. if (accountable && (!(flags & MAP_NORESERVE) ||
  926. sysctl_overcommit_memory == OVERCOMMIT_NEVER)) {
  927. if (vm_flags & VM_SHARED) {
  928. /* Check memory availability in shmem_file_setup? */
  929. vm_flags |= VM_ACCOUNT;
  930. } else if (vm_flags & VM_WRITE) {
  931. /*
  932. * Private writable mapping: check memory availability
  933. */
  934. charged = len >> PAGE_SHIFT;
  935. if (security_vm_enough_memory(charged))
  936. return -ENOMEM;
  937. vm_flags |= VM_ACCOUNT;
  938. }
  939. }
  940. /*
  941. * Can we just expand an old private anonymous mapping?
  942. * The VM_SHARED test is necessary because shmem_zero_setup
  943. * will create the file object for a shared anonymous map below.
  944. */
  945. if (!file && !(vm_flags & VM_SHARED) &&
  946. vma_merge(mm, prev, addr, addr + len, vm_flags,
  947. NULL, NULL, pgoff, NULL))
  948. goto out;
  949. /*
  950. * Determine the object being mapped and call the appropriate
  951. * specific mapper. the address has already been validated, but
  952. * not unmapped, but the maps are removed from the list.
  953. */
  954. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  955. if (!vma) {
  956. error = -ENOMEM;
  957. goto unacct_error;
  958. }
  959. vma->vm_mm = mm;
  960. vma->vm_start = addr;
  961. vma->vm_end = addr + len;
  962. vma->vm_flags = vm_flags;
  963. vma->vm_page_prot = protection_map[vm_flags &
  964. (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)];
  965. vma->vm_pgoff = pgoff;
  966. if (file) {
  967. error = -EINVAL;
  968. if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
  969. goto free_vma;
  970. if (vm_flags & VM_DENYWRITE) {
  971. error = deny_write_access(file);
  972. if (error)
  973. goto free_vma;
  974. correct_wcount = 1;
  975. }
  976. vma->vm_file = file;
  977. get_file(file);
  978. error = file->f_op->mmap(file, vma);
  979. if (error)
  980. goto unmap_and_free_vma;
  981. } else if (vm_flags & VM_SHARED) {
  982. error = shmem_zero_setup(vma);
  983. if (error)
  984. goto free_vma;
  985. }
  986. /* We set VM_ACCOUNT in a shared mapping's vm_flags, to inform
  987. * shmem_zero_setup (perhaps called through /dev/zero's ->mmap)
  988. * that memory reservation must be checked; but that reservation
  989. * belongs to shared memory object, not to vma: so now clear it.
  990. */
  991. if ((vm_flags & (VM_SHARED|VM_ACCOUNT)) == (VM_SHARED|VM_ACCOUNT))
  992. vma->vm_flags &= ~VM_ACCOUNT;
  993. /* Can addr have changed??
  994. *
  995. * Answer: Yes, several device drivers can do it in their
  996. * f_op->mmap method. -DaveM
  997. */
  998. addr = vma->vm_start;
  999. pgoff = vma->vm_pgoff;
  1000. vm_flags = vma->vm_flags;
  1001. if (vma_wants_writenotify(vma))
  1002. vma->vm_page_prot =
  1003. protection_map[vm_flags & (VM_READ|VM_WRITE|VM_EXEC)];
  1004. if (!file || !vma_merge(mm, prev, addr, vma->vm_end,
  1005. vma->vm_flags, NULL, file, pgoff, vma_policy(vma))) {
  1006. file = vma->vm_file;
  1007. vma_link(mm, vma, prev, rb_link, rb_parent);
  1008. if (correct_wcount)
  1009. atomic_inc(&inode->i_writecount);
  1010. } else {
  1011. if (file) {
  1012. if (correct_wcount)
  1013. atomic_inc(&inode->i_writecount);
  1014. fput(file);
  1015. }
  1016. mpol_free(vma_policy(vma));
  1017. kmem_cache_free(vm_area_cachep, vma);
  1018. }
  1019. out:
  1020. mm->total_vm += len >> PAGE_SHIFT;
  1021. vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
  1022. if (vm_flags & VM_LOCKED) {
  1023. mm->locked_vm += len >> PAGE_SHIFT;
  1024. make_pages_present(addr, addr + len);
  1025. }
  1026. if (flags & MAP_POPULATE) {
  1027. up_write(&mm->mmap_sem);
  1028. sys_remap_file_pages(addr, len, 0,
  1029. pgoff, flags & MAP_NONBLOCK);
  1030. down_write(&mm->mmap_sem);
  1031. }
  1032. return addr;
  1033. unmap_and_free_vma:
  1034. if (correct_wcount)
  1035. atomic_inc(&inode->i_writecount);
  1036. vma->vm_file = NULL;
  1037. fput(file);
  1038. /* Undo any partial mapping done by a device driver. */
  1039. unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
  1040. charged = 0;
  1041. free_vma:
  1042. kmem_cache_free(vm_area_cachep, vma);
  1043. unacct_error:
  1044. if (charged)
  1045. vm_unacct_memory(charged);
  1046. return error;
  1047. }
  1048. EXPORT_SYMBOL(do_mmap_pgoff);
  1049. /* Get an address range which is currently unmapped.
  1050. * For shmat() with addr=0.
  1051. *
  1052. * Ugly calling convention alert:
  1053. * Return value with the low bits set means error value,
  1054. * ie
  1055. * if (ret & ~PAGE_MASK)
  1056. * error = ret;
  1057. *
  1058. * This function "knows" that -ENOMEM has the bits set.
  1059. */
  1060. #ifndef HAVE_ARCH_UNMAPPED_AREA
  1061. unsigned long
  1062. arch_get_unmapped_area(struct file *filp, unsigned long addr,
  1063. unsigned long len, unsigned long pgoff, unsigned long flags)
  1064. {
  1065. struct mm_struct *mm = current->mm;
  1066. struct vm_area_struct *vma;
  1067. unsigned long start_addr;
  1068. if (len > TASK_SIZE)
  1069. return -ENOMEM;
  1070. if (addr) {
  1071. addr = PAGE_ALIGN(addr);
  1072. vma = find_vma(mm, addr);
  1073. if (TASK_SIZE - len >= addr &&
  1074. (!vma || addr + len <= vma->vm_start))
  1075. return addr;
  1076. }
  1077. if (len > mm->cached_hole_size) {
  1078. start_addr = addr = mm->free_area_cache;
  1079. } else {
  1080. start_addr = addr = TASK_UNMAPPED_BASE;
  1081. mm->cached_hole_size = 0;
  1082. }
  1083. full_search:
  1084. for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
  1085. /* At this point: (!vma || addr < vma->vm_end). */
  1086. if (TASK_SIZE - len < addr) {
  1087. /*
  1088. * Start a new search - just in case we missed
  1089. * some holes.
  1090. */
  1091. if (start_addr != TASK_UNMAPPED_BASE) {
  1092. addr = TASK_UNMAPPED_BASE;
  1093. start_addr = addr;
  1094. mm->cached_hole_size = 0;
  1095. goto full_search;
  1096. }
  1097. return -ENOMEM;
  1098. }
  1099. if (!vma || addr + len <= vma->vm_start) {
  1100. /*
  1101. * Remember the place where we stopped the search:
  1102. */
  1103. mm->free_area_cache = addr + len;
  1104. return addr;
  1105. }
  1106. if (addr + mm->cached_hole_size < vma->vm_start)
  1107. mm->cached_hole_size = vma->vm_start - addr;
  1108. addr = vma->vm_end;
  1109. }
  1110. }
  1111. #endif
  1112. void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
  1113. {
  1114. /*
  1115. * Is this a new hole at the lowest possible address?
  1116. */
  1117. if (addr >= TASK_UNMAPPED_BASE && addr < mm->free_area_cache) {
  1118. mm->free_area_cache = addr;
  1119. mm->cached_hole_size = ~0UL;
  1120. }
  1121. }
  1122. /*
  1123. * This mmap-allocator allocates new areas top-down from below the
  1124. * stack's low limit (the base):
  1125. */
  1126. #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
  1127. unsigned long
  1128. arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
  1129. const unsigned long len, const unsigned long pgoff,
  1130. const unsigned long flags)
  1131. {
  1132. struct vm_area_struct *vma;
  1133. struct mm_struct *mm = current->mm;
  1134. unsigned long addr = addr0;
  1135. /* requested length too big for entire address space */
  1136. if (len > TASK_SIZE)
  1137. return -ENOMEM;
  1138. /* requesting a specific address */
  1139. if (addr) {
  1140. addr = PAGE_ALIGN(addr);
  1141. vma = find_vma(mm, addr);
  1142. if (TASK_SIZE - len >= addr &&
  1143. (!vma || addr + len <= vma->vm_start))
  1144. return addr;
  1145. }
  1146. /* check if free_area_cache is useful for us */
  1147. if (len <= mm->cached_hole_size) {
  1148. mm->cached_hole_size = 0;
  1149. mm->free_area_cache = mm->mmap_base;
  1150. }
  1151. /* either no address requested or can't fit in requested address hole */
  1152. addr = mm->free_area_cache;
  1153. /* make sure it can fit in the remaining address space */
  1154. if (addr > len) {
  1155. vma = find_vma(mm, addr-len);
  1156. if (!vma || addr <= vma->vm_start)
  1157. /* remember the address as a hint for next time */
  1158. return (mm->free_area_cache = addr-len);
  1159. }
  1160. if (mm->mmap_base < len)
  1161. goto bottomup;
  1162. addr = mm->mmap_base-len;
  1163. do {
  1164. /*
  1165. * Lookup failure means no vma is above this address,
  1166. * else if new region fits below vma->vm_start,
  1167. * return with success:
  1168. */
  1169. vma = find_vma(mm, addr);
  1170. if (!vma || addr+len <= vma->vm_start)
  1171. /* remember the address as a hint for next time */
  1172. return (mm->free_area_cache = addr);
  1173. /* remember the largest hole we saw so far */
  1174. if (addr + mm->cached_hole_size < vma->vm_start)
  1175. mm->cached_hole_size = vma->vm_start - addr;
  1176. /* try just below the current vma->vm_start */
  1177. addr = vma->vm_start-len;
  1178. } while (len < vma->vm_start);
  1179. bottomup:
  1180. /*
  1181. * A failed mmap() very likely causes application failure,
  1182. * so fall back to the bottom-up function here. This scenario
  1183. * can happen with large stack limits and large mmap()
  1184. * allocations.
  1185. */
  1186. mm->cached_hole_size = ~0UL;
  1187. mm->free_area_cache = TASK_UNMAPPED_BASE;
  1188. addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
  1189. /*
  1190. * Restore the topdown base:
  1191. */
  1192. mm->free_area_cache = mm->mmap_base;
  1193. mm->cached_hole_size = ~0UL;
  1194. return addr;
  1195. }
  1196. #endif
  1197. void arch_unmap_area_topdown(struct mm_struct *mm, unsigned long addr)
  1198. {
  1199. /*
  1200. * Is this a new hole at the highest possible address?
  1201. */
  1202. if (addr > mm->free_area_cache)
  1203. mm->free_area_cache = addr;
  1204. /* dont allow allocations above current base */
  1205. if (mm->free_area_cache > mm->mmap_base)
  1206. mm->free_area_cache = mm->mmap_base;
  1207. }
  1208. unsigned long
  1209. get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
  1210. unsigned long pgoff, unsigned long flags)
  1211. {
  1212. unsigned long ret;
  1213. if (!(flags & MAP_FIXED)) {
  1214. unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
  1215. get_area = current->mm->get_unmapped_area;
  1216. if (file && file->f_op && file->f_op->get_unmapped_area)
  1217. get_area = file->f_op->get_unmapped_area;
  1218. addr = get_area(file, addr, len, pgoff, flags);
  1219. if (IS_ERR_VALUE(addr))
  1220. return addr;
  1221. }
  1222. if (addr > TASK_SIZE - len)
  1223. return -ENOMEM;
  1224. if (addr & ~PAGE_MASK)
  1225. return -EINVAL;
  1226. if (file && is_file_hugepages(file)) {
  1227. /*
  1228. * Check if the given range is hugepage aligned, and
  1229. * can be made suitable for hugepages.
  1230. */
  1231. ret = prepare_hugepage_range(addr, len, pgoff);
  1232. } else {
  1233. /*
  1234. * Ensure that a normal request is not falling in a
  1235. * reserved hugepage range. For some archs like IA-64,
  1236. * there is a separate region for hugepages.
  1237. */
  1238. ret = is_hugepage_only_range(current->mm, addr, len);
  1239. }
  1240. if (ret)
  1241. return -EINVAL;
  1242. return addr;
  1243. }
  1244. EXPORT_SYMBOL(get_unmapped_area);
  1245. /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
  1246. struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr)
  1247. {
  1248. struct vm_area_struct *vma = NULL;
  1249. if (mm) {
  1250. /* Check the cache first. */
  1251. /* (Cache hit rate is typically around 35%.) */
  1252. vma = mm->mmap_cache;
  1253. if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
  1254. struct rb_node * rb_node;
  1255. rb_node = mm->mm_rb.rb_node;
  1256. vma = NULL;
  1257. while (rb_node) {
  1258. struct vm_area_struct * vma_tmp;
  1259. vma_tmp = rb_entry(rb_node,
  1260. struct vm_area_struct, vm_rb);
  1261. if (vma_tmp->vm_end > addr) {
  1262. vma = vma_tmp;
  1263. if (vma_tmp->vm_start <= addr)
  1264. break;
  1265. rb_node = rb_node->rb_left;
  1266. } else
  1267. rb_node = rb_node->rb_right;
  1268. }
  1269. if (vma)
  1270. mm->mmap_cache = vma;
  1271. }
  1272. }
  1273. return vma;
  1274. }
  1275. EXPORT_SYMBOL(find_vma);
  1276. /* Same as find_vma, but also return a pointer to the previous VMA in *pprev. */
  1277. struct vm_area_struct *
  1278. find_vma_prev(struct mm_struct *mm, unsigned long addr,
  1279. struct vm_area_struct **pprev)
  1280. {
  1281. struct vm_area_struct *vma = NULL, *prev = NULL;
  1282. struct rb_node * rb_node;
  1283. if (!mm)
  1284. goto out;
  1285. /* Guard against addr being lower than the first VMA */
  1286. vma = mm->mmap;
  1287. /* Go through the RB tree quickly. */
  1288. rb_node = mm->mm_rb.rb_node;
  1289. while (rb_node) {
  1290. struct vm_area_struct *vma_tmp;
  1291. vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
  1292. if (addr < vma_tmp->vm_end) {
  1293. rb_node = rb_node->rb_left;
  1294. } else {
  1295. prev = vma_tmp;
  1296. if (!prev->vm_next || (addr < prev->vm_next->vm_end))
  1297. break;
  1298. rb_node = rb_node->rb_right;
  1299. }
  1300. }
  1301. out:
  1302. *pprev = prev;
  1303. return prev ? prev->vm_next : vma;
  1304. }
  1305. /*
  1306. * Verify that the stack growth is acceptable and
  1307. * update accounting. This is shared with both the
  1308. * grow-up and grow-down cases.
  1309. */
  1310. static int acct_stack_growth(struct vm_area_struct * vma, unsigned long size, unsigned long grow)
  1311. {
  1312. struct mm_struct *mm = vma->vm_mm;
  1313. struct rlimit *rlim = current->signal->rlim;
  1314. unsigned long new_start;
  1315. /* address space limit tests */
  1316. if (!may_expand_vm(mm, grow))
  1317. return -ENOMEM;
  1318. /* Stack limit test */
  1319. if (size > rlim[RLIMIT_STACK].rlim_cur)
  1320. return -ENOMEM;
  1321. /* mlock limit tests */
  1322. if (vma->vm_flags & VM_LOCKED) {
  1323. unsigned long locked;
  1324. unsigned long limit;
  1325. locked = mm->locked_vm + grow;
  1326. limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
  1327. if (locked > limit && !capable(CAP_IPC_LOCK))
  1328. return -ENOMEM;
  1329. }
  1330. /* Check to ensure the stack will not grow into a hugetlb-only region */
  1331. new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
  1332. vma->vm_end - size;
  1333. if (is_hugepage_only_range(vma->vm_mm, new_start, size))
  1334. return -EFAULT;
  1335. /*
  1336. * Overcommit.. This must be the final test, as it will
  1337. * update security statistics.
  1338. */
  1339. if (security_vm_enough_memory(grow))
  1340. return -ENOMEM;
  1341. /* Ok, everything looks good - let it rip */
  1342. mm->total_vm += grow;
  1343. if (vma->vm_flags & VM_LOCKED)
  1344. mm->locked_vm += grow;
  1345. vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
  1346. return 0;
  1347. }
  1348. #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
  1349. /*
  1350. * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
  1351. * vma is the last one with address > vma->vm_end. Have to extend vma.
  1352. */
  1353. #ifndef CONFIG_IA64
  1354. static inline
  1355. #endif
  1356. int expand_upwards(struct vm_area_struct *vma, unsigned long address)
  1357. {
  1358. int error;
  1359. if (!(vma->vm_flags & VM_GROWSUP))
  1360. return -EFAULT;
  1361. /*
  1362. * We must make sure the anon_vma is allocated
  1363. * so that the anon_vma locking is not a noop.
  1364. */
  1365. if (unlikely(anon_vma_prepare(vma)))
  1366. return -ENOMEM;
  1367. anon_vma_lock(vma);
  1368. /*
  1369. * vma->vm_start/vm_end cannot change under us because the caller
  1370. * is required to hold the mmap_sem in read mode. We need the
  1371. * anon_vma lock to serialize against concurrent expand_stacks.
  1372. */
  1373. address += 4 + PAGE_SIZE - 1;
  1374. address &= PAGE_MASK;
  1375. error = 0;
  1376. /* Somebody else might have raced and expanded it already */
  1377. if (address > vma->vm_end) {
  1378. unsigned long size, grow;
  1379. size = address - vma->vm_start;
  1380. grow = (address - vma->vm_end) >> PAGE_SHIFT;
  1381. error = acct_stack_growth(vma, size, grow);
  1382. if (!error)
  1383. vma->vm_end = address;
  1384. }
  1385. anon_vma_unlock(vma);
  1386. return error;
  1387. }
  1388. #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
  1389. #ifdef CONFIG_STACK_GROWSUP
  1390. int expand_stack(struct vm_area_struct *vma, unsigned long address)
  1391. {
  1392. return expand_upwards(vma, address);
  1393. }
  1394. struct vm_area_struct *
  1395. find_extend_vma(struct mm_struct *mm, unsigned long addr)
  1396. {
  1397. struct vm_area_struct *vma, *prev;
  1398. addr &= PAGE_MASK;
  1399. vma = find_vma_prev(mm, addr, &prev);
  1400. if (vma && (vma->vm_start <= addr))
  1401. return vma;
  1402. if (!prev || expand_stack(prev, addr))
  1403. return NULL;
  1404. if (prev->vm_flags & VM_LOCKED) {
  1405. make_pages_present(addr, prev->vm_end);
  1406. }
  1407. return prev;
  1408. }
  1409. #else
  1410. /*
  1411. * vma is the first one with address < vma->vm_start. Have to extend vma.
  1412. */
  1413. int expand_stack(struct vm_area_struct *vma, unsigned long address)
  1414. {
  1415. int error;
  1416. /*
  1417. * We must make sure the anon_vma is allocated
  1418. * so that the anon_vma locking is not a noop.
  1419. */
  1420. if (unlikely(anon_vma_prepare(vma)))
  1421. return -ENOMEM;
  1422. anon_vma_lock(vma);
  1423. /*
  1424. * vma->vm_start/vm_end cannot change under us because the caller
  1425. * is required to hold the mmap_sem in read mode. We need the
  1426. * anon_vma lock to serialize against concurrent expand_stacks.
  1427. */
  1428. address &= PAGE_MASK;
  1429. error = 0;
  1430. /* Somebody else might have raced and expanded it already */
  1431. if (address < vma->vm_start) {
  1432. unsigned long size, grow;
  1433. size = vma->vm_end - address;
  1434. grow = (vma->vm_start - address) >> PAGE_SHIFT;
  1435. error = acct_stack_growth(vma, size, grow);
  1436. if (!error) {
  1437. vma->vm_start = address;
  1438. vma->vm_pgoff -= grow;
  1439. }
  1440. }
  1441. anon_vma_unlock(vma);
  1442. return error;
  1443. }
  1444. struct vm_area_struct *
  1445. find_extend_vma(struct mm_struct * mm, unsigned long addr)
  1446. {
  1447. struct vm_area_struct * vma;
  1448. unsigned long start;
  1449. addr &= PAGE_MASK;
  1450. vma = find_vma(mm,addr);
  1451. if (!vma)
  1452. return NULL;
  1453. if (vma->vm_start <= addr)
  1454. return vma;
  1455. if (!(vma->vm_flags & VM_GROWSDOWN))
  1456. return NULL;
  1457. start = vma->vm_start;
  1458. if (expand_stack(vma, addr))
  1459. return NULL;
  1460. if (vma->vm_flags & VM_LOCKED) {
  1461. make_pages_present(addr, start);
  1462. }
  1463. return vma;
  1464. }
  1465. #endif
  1466. /*
  1467. * Ok - we have the memory areas we should free on the vma list,
  1468. * so release them, and do the vma updates.
  1469. *
  1470. * Called with the mm semaphore held.
  1471. */
  1472. static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
  1473. {
  1474. /* Update high watermark before we lower total_vm */
  1475. update_hiwater_vm(mm);
  1476. do {
  1477. long nrpages = vma_pages(vma);
  1478. mm->total_vm -= nrpages;
  1479. if (vma->vm_flags & VM_LOCKED)
  1480. mm->locked_vm -= nrpages;
  1481. vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
  1482. vma = remove_vma(vma);
  1483. } while (vma);
  1484. validate_mm(mm);
  1485. }
  1486. /*
  1487. * Get rid of page table information in the indicated region.
  1488. *
  1489. * Called with the mm semaphore held.
  1490. */
  1491. static void unmap_region(struct mm_struct *mm,
  1492. struct vm_area_struct *vma, struct vm_area_struct *prev,
  1493. unsigned long start, unsigned long end)
  1494. {
  1495. struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
  1496. struct mmu_gather *tlb;
  1497. unsigned long nr_accounted = 0;
  1498. lru_add_drain();
  1499. tlb = tlb_gather_mmu(mm, 0);
  1500. update_hiwater_rss(mm);
  1501. unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
  1502. vm_unacct_memory(nr_accounted);
  1503. free_pgtables(&tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
  1504. next? next->vm_start: 0);
  1505. tlb_finish_mmu(tlb, start, end);
  1506. }
  1507. /*
  1508. * Create a list of vma's touched by the unmap, removing them from the mm's
  1509. * vma list as we go..
  1510. */
  1511. static void
  1512. detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
  1513. struct vm_area_struct *prev, unsigned long end)
  1514. {
  1515. struct vm_area_struct **insertion_point;
  1516. struct vm_area_struct *tail_vma = NULL;
  1517. unsigned long addr;
  1518. insertion_point = (prev ? &prev->vm_next : &mm->mmap);
  1519. do {
  1520. rb_erase(&vma->vm_rb, &mm->mm_rb);
  1521. mm->map_count--;
  1522. tail_vma = vma;
  1523. vma = vma->vm_next;
  1524. } while (vma && vma->vm_start < end);
  1525. *insertion_point = vma;
  1526. tail_vma->vm_next = NULL;
  1527. if (mm->unmap_area == arch_unmap_area)
  1528. addr = prev ? prev->vm_end : mm->mmap_base;
  1529. else
  1530. addr = vma ? vma->vm_start : mm->mmap_base;
  1531. mm->unmap_area(mm, addr);
  1532. mm->mmap_cache = NULL; /* Kill the cache. */
  1533. }
  1534. /*
  1535. * Split a vma into two pieces at address 'addr', a new vma is allocated
  1536. * either for the first part or the the tail.
  1537. */
  1538. int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
  1539. unsigned long addr, int new_below)
  1540. {
  1541. struct mempolicy *pol;
  1542. struct vm_area_struct *new;
  1543. if (is_vm_hugetlb_page(vma) && (addr & ~HPAGE_MASK))
  1544. return -EINVAL;
  1545. if (mm->map_count >= sysctl_max_map_count)
  1546. return -ENOMEM;
  1547. new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  1548. if (!new)
  1549. return -ENOMEM;
  1550. /* most fields are the same, copy all, and then fixup */
  1551. *new = *vma;
  1552. if (new_below)
  1553. new->vm_end = addr;
  1554. else {
  1555. new->vm_start = addr;
  1556. new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
  1557. }
  1558. pol = mpol_copy(vma_policy(vma));
  1559. if (IS_ERR(pol)) {
  1560. kmem_cache_free(vm_area_cachep, new);
  1561. return PTR_ERR(pol);
  1562. }
  1563. vma_set_policy(new, pol);
  1564. if (new->vm_file)
  1565. get_file(new->vm_file);
  1566. if (new->vm_ops && new->vm_ops->open)
  1567. new->vm_ops->open(new);
  1568. if (new_below)
  1569. vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
  1570. ((addr - new->vm_start) >> PAGE_SHIFT), new);
  1571. else
  1572. vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
  1573. return 0;
  1574. }
  1575. /* Munmap is split into 2 main parts -- this part which finds
  1576. * what needs doing, and the areas themselves, which do the
  1577. * work. This now handles partial unmappings.
  1578. * Jeremy Fitzhardinge <jeremy@goop.org>
  1579. */
  1580. int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
  1581. {
  1582. unsigned long end;
  1583. struct vm_area_struct *vma, *prev, *last;
  1584. if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
  1585. return -EINVAL;
  1586. if ((len = PAGE_ALIGN(len)) == 0)
  1587. return -EINVAL;
  1588. /* Find the first overlapping VMA */
  1589. vma = find_vma_prev(mm, start, &prev);
  1590. if (!vma)
  1591. return 0;
  1592. /* we have start < vma->vm_end */
  1593. /* if it doesn't overlap, we have nothing.. */
  1594. end = start + len;
  1595. if (vma->vm_start >= end)
  1596. return 0;
  1597. /*
  1598. * If we need to split any vma, do it now to save pain later.
  1599. *
  1600. * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
  1601. * unmapped vm_area_struct will remain in use: so lower split_vma
  1602. * places tmp vma above, and higher split_vma places tmp vma below.
  1603. */
  1604. if (start > vma->vm_start) {
  1605. int error = split_vma(mm, vma, start, 0);
  1606. if (error)
  1607. return error;
  1608. prev = vma;
  1609. }
  1610. /* Does it split the last one? */
  1611. last = find_vma(mm, end);
  1612. if (last && end > last->vm_start) {
  1613. int error = split_vma(mm, last, end, 1);
  1614. if (error)
  1615. return error;
  1616. }
  1617. vma = prev? prev->vm_next: mm->mmap;
  1618. /*
  1619. * Remove the vma's, and unmap the actual pages
  1620. */
  1621. detach_vmas_to_be_unmapped(mm, vma, prev, end);
  1622. unmap_region(mm, vma, prev, start, end);
  1623. /* Fix up all other VM information */
  1624. remove_vma_list(mm, vma);
  1625. return 0;
  1626. }
  1627. EXPORT_SYMBOL(do_munmap);
  1628. asmlinkage long sys_munmap(unsigned long addr, size_t len)
  1629. {
  1630. int ret;
  1631. struct mm_struct *mm = current->mm;
  1632. profile_munmap(addr);
  1633. down_write(&mm->mmap_sem);
  1634. ret = do_munmap(mm, addr, len);
  1635. up_write(&mm->mmap_sem);
  1636. return ret;
  1637. }
  1638. static inline void verify_mm_writelocked(struct mm_struct *mm)
  1639. {
  1640. #ifdef CONFIG_DEBUG_VM
  1641. if (unlikely(down_read_trylock(&mm->mmap_sem))) {
  1642. WARN_ON(1);
  1643. up_read(&mm->mmap_sem);
  1644. }
  1645. #endif
  1646. }
  1647. /*
  1648. * this is really a simplified "do_mmap". it only handles
  1649. * anonymous maps. eventually we may be able to do some
  1650. * brk-specific accounting here.
  1651. */
  1652. unsigned long do_brk(unsigned long addr, unsigned long len)
  1653. {
  1654. struct mm_struct * mm = current->mm;
  1655. struct vm_area_struct * vma, * prev;
  1656. unsigned long flags;
  1657. struct rb_node ** rb_link, * rb_parent;
  1658. pgoff_t pgoff = addr >> PAGE_SHIFT;
  1659. int error;
  1660. len = PAGE_ALIGN(len);
  1661. if (!len)
  1662. return addr;
  1663. if ((addr + len) > TASK_SIZE || (addr + len) < addr)
  1664. return -EINVAL;
  1665. if (is_hugepage_only_range(mm, addr, len))
  1666. return -EINVAL;
  1667. flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
  1668. error = arch_mmap_check(addr, len, flags);
  1669. if (error)
  1670. return error;
  1671. /*
  1672. * mlock MCL_FUTURE?
  1673. */
  1674. if (mm->def_flags & VM_LOCKED) {
  1675. unsigned long locked, lock_limit;
  1676. locked = len >> PAGE_SHIFT;
  1677. locked += mm->locked_vm;
  1678. lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
  1679. lock_limit >>= PAGE_SHIFT;
  1680. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  1681. return -EAGAIN;
  1682. }
  1683. /*
  1684. * mm->mmap_sem is required to protect against another thread
  1685. * changing the mappings in case we sleep.
  1686. */
  1687. verify_mm_writelocked(mm);
  1688. /*
  1689. * Clear old maps. this also does some error checking for us
  1690. */
  1691. munmap_back:
  1692. vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  1693. if (vma && vma->vm_start < addr + len) {
  1694. if (do_munmap(mm, addr, len))
  1695. return -ENOMEM;
  1696. goto munmap_back;
  1697. }
  1698. /* Check against address space limits *after* clearing old maps... */
  1699. if (!may_expand_vm(mm, len >> PAGE_SHIFT))
  1700. return -ENOMEM;
  1701. if (mm->map_count > sysctl_max_map_count)
  1702. return -ENOMEM;
  1703. if (security_vm_enough_memory(len >> PAGE_SHIFT))
  1704. return -ENOMEM;
  1705. /* Can we just expand an old private anonymous mapping? */
  1706. if (vma_merge(mm, prev, addr, addr + len, flags,
  1707. NULL, NULL, pgoff, NULL))
  1708. goto out;
  1709. /*
  1710. * create a vma struct for an anonymous mapping
  1711. */
  1712. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  1713. if (!vma) {
  1714. vm_unacct_memory(len >> PAGE_SHIFT);
  1715. return -ENOMEM;
  1716. }
  1717. vma->vm_mm = mm;
  1718. vma->vm_start = addr;
  1719. vma->vm_end = addr + len;
  1720. vma->vm_pgoff = pgoff;
  1721. vma->vm_flags = flags;
  1722. vma->vm_page_prot = protection_map[flags &
  1723. (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)];
  1724. vma_link(mm, vma, prev, rb_link, rb_parent);
  1725. out:
  1726. mm->total_vm += len >> PAGE_SHIFT;
  1727. if (flags & VM_LOCKED) {
  1728. mm->locked_vm += len >> PAGE_SHIFT;
  1729. make_pages_present(addr, addr + len);
  1730. }
  1731. return addr;
  1732. }
  1733. EXPORT_SYMBOL(do_brk);
  1734. /* Release all mmaps. */
  1735. void exit_mmap(struct mm_struct *mm)
  1736. {
  1737. struct mmu_gather *tlb;
  1738. struct vm_area_struct *vma = mm->mmap;
  1739. unsigned long nr_accounted = 0;
  1740. unsigned long end;
  1741. lru_add_drain();
  1742. flush_cache_mm(mm);
  1743. tlb = tlb_gather_mmu(mm, 1);
  1744. /* Don't update_hiwater_rss(mm) here, do_exit already did */
  1745. /* Use -1 here to ensure all VMAs in the mm are unmapped */
  1746. end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
  1747. vm_unacct_memory(nr_accounted);
  1748. free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, 0);
  1749. tlb_finish_mmu(tlb, 0, end);
  1750. /*
  1751. * Walk the list again, actually closing and freeing it,
  1752. * with preemption enabled, without holding any MM locks.
  1753. */
  1754. while (vma)
  1755. vma = remove_vma(vma);
  1756. BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
  1757. }
  1758. /* Insert vm structure into process list sorted by address
  1759. * and into the inode's i_mmap tree. If vm_file is non-NULL
  1760. * then i_mmap_lock is taken here.
  1761. */
  1762. int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
  1763. {
  1764. struct vm_area_struct * __vma, * prev;
  1765. struct rb_node ** rb_link, * rb_parent;
  1766. /*
  1767. * The vm_pgoff of a purely anonymous vma should be irrelevant
  1768. * until its first write fault, when page's anon_vma and index
  1769. * are set. But now set the vm_pgoff it will almost certainly
  1770. * end up with (unless mremap moves it elsewhere before that
  1771. * first wfault), so /proc/pid/maps tells a consistent story.
  1772. *
  1773. * By setting it to reflect the virtual start address of the
  1774. * vma, merges and splits can happen in a seamless way, just
  1775. * using the existing file pgoff checks and manipulations.
  1776. * Similarly in do_mmap_pgoff and in do_brk.
  1777. */
  1778. if (!vma->vm_file) {
  1779. BUG_ON(vma->anon_vma);
  1780. vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
  1781. }
  1782. __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent);
  1783. if (__vma && __vma->vm_start < vma->vm_end)
  1784. return -ENOMEM;
  1785. if ((vma->vm_flags & VM_ACCOUNT) &&
  1786. security_vm_enough_memory(vma_pages(vma)))
  1787. return -ENOMEM;
  1788. vma_link(mm, vma, prev, rb_link, rb_parent);
  1789. return 0;
  1790. }
  1791. /*
  1792. * Copy the vma structure to a new location in the same mm,
  1793. * prior to moving page table entries, to effect an mremap move.
  1794. */
  1795. struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
  1796. unsigned long addr, unsigned long len, pgoff_t pgoff)
  1797. {
  1798. struct vm_area_struct *vma = *vmap;
  1799. unsigned long vma_start = vma->vm_start;
  1800. struct mm_struct *mm = vma->vm_mm;
  1801. struct vm_area_struct *new_vma, *prev;
  1802. struct rb_node **rb_link, *rb_parent;
  1803. struct mempolicy *pol;
  1804. /*
  1805. * If anonymous vma has not yet been faulted, update new pgoff
  1806. * to match new location, to increase its chance of merging.
  1807. */
  1808. if (!vma->vm_file && !vma->anon_vma)
  1809. pgoff = addr >> PAGE_SHIFT;
  1810. find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  1811. new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
  1812. vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
  1813. if (new_vma) {
  1814. /*
  1815. * Source vma may have been merged into new_vma
  1816. */
  1817. if (vma_start >= new_vma->vm_start &&
  1818. vma_start < new_vma->vm_end)
  1819. *vmap = new_vma;
  1820. } else {
  1821. new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  1822. if (new_vma) {
  1823. *new_vma = *vma;
  1824. pol = mpol_copy(vma_policy(vma));
  1825. if (IS_ERR(pol)) {
  1826. kmem_cache_free(vm_area_cachep, new_vma);
  1827. return NULL;
  1828. }
  1829. vma_set_policy(new_vma, pol);
  1830. new_vma->vm_start = addr;
  1831. new_vma->vm_end = addr + len;
  1832. new_vma->vm_pgoff = pgoff;
  1833. if (new_vma->vm_file)
  1834. get_file(new_vma->vm_file);
  1835. if (new_vma->vm_ops && new_vma->vm_ops->open)
  1836. new_vma->vm_ops->open(new_vma);
  1837. vma_link(mm, new_vma, prev, rb_link, rb_parent);
  1838. }
  1839. }
  1840. return new_vma;
  1841. }
  1842. /*
  1843. * Return true if the calling process may expand its vm space by the passed
  1844. * number of pages
  1845. */
  1846. int may_expand_vm(struct mm_struct *mm, unsigned long npages)
  1847. {
  1848. unsigned long cur = mm->total_vm; /* pages */
  1849. unsigned long lim;
  1850. lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
  1851. if (cur + npages > lim)
  1852. return 0;
  1853. return 1;
  1854. }
  1855. static struct page *special_mapping_nopage(struct vm_area_struct *vma,
  1856. unsigned long address, int *type)
  1857. {
  1858. struct page **pages;
  1859. BUG_ON(address < vma->vm_start || address >= vma->vm_end);
  1860. address -= vma->vm_start;
  1861. for (pages = vma->vm_private_data; address > 0 && *pages; ++pages)
  1862. address -= PAGE_SIZE;
  1863. if (*pages) {
  1864. struct page *page = *pages;
  1865. get_page(page);
  1866. return page;
  1867. }
  1868. return NOPAGE_SIGBUS;
  1869. }
  1870. /*
  1871. * Having a close hook prevents vma merging regardless of flags.
  1872. */
  1873. static void special_mapping_close(struct vm_area_struct *vma)
  1874. {
  1875. }
  1876. static struct vm_operations_struct special_mapping_vmops = {
  1877. .close = special_mapping_close,
  1878. .nopage = special_mapping_nopage,
  1879. };
  1880. /*
  1881. * Called with mm->mmap_sem held for writing.
  1882. * Insert a new vma covering the given region, with the given flags.
  1883. * Its pages are supplied by the given array of struct page *.
  1884. * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
  1885. * The region past the last page supplied will always produce SIGBUS.
  1886. * The array pointer and the pages it points to are assumed to stay alive
  1887. * for as long as this mapping might exist.
  1888. */
  1889. int install_special_mapping(struct mm_struct *mm,
  1890. unsigned long addr, unsigned long len,
  1891. unsigned long vm_flags, struct page **pages)
  1892. {
  1893. struct vm_area_struct *vma;
  1894. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  1895. if (unlikely(vma == NULL))
  1896. return -ENOMEM;
  1897. vma->vm_mm = mm;
  1898. vma->vm_start = addr;
  1899. vma->vm_end = addr + len;
  1900. vma->vm_flags = vm_flags | mm->def_flags;
  1901. vma->vm_page_prot = protection_map[vma->vm_flags & 7];
  1902. vma->vm_ops = &special_mapping_vmops;
  1903. vma->vm_private_data = pages;
  1904. if (unlikely(insert_vm_struct(mm, vma))) {
  1905. kmem_cache_free(vm_area_cachep, vma);
  1906. return -ENOMEM;
  1907. }
  1908. mm->total_vm += len >> PAGE_SHIFT;
  1909. return 0;
  1910. }