nommu.c 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/nommu.c
  4. *
  5. * Replacement code for mm functions to support CPU's that don't
  6. * have any form of memory management unit (thus no virtual memory).
  7. *
  8. * See Documentation/admin-guide/mm/nommu-mmap.rst
  9. *
  10. * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
  11. * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
  12. * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
  13. * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
  14. * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
  15. */
  16. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  17. #include <linux/export.h>
  18. #include <linux/mm.h>
  19. #include <linux/sched/mm.h>
  20. #include <linux/vmacache.h>
  21. #include <linux/mman.h>
  22. #include <linux/swap.h>
  23. #include <linux/file.h>
  24. #include <linux/highmem.h>
  25. #include <linux/pagemap.h>
  26. #include <linux/slab.h>
  27. #include <linux/vmalloc.h>
  28. #include <linux/blkdev.h>
  29. #include <linux/backing-dev.h>
  30. #include <linux/compiler.h>
  31. #include <linux/mount.h>
  32. #include <linux/personality.h>
  33. #include <linux/security.h>
  34. #include <linux/syscalls.h>
  35. #include <linux/audit.h>
  36. #include <linux/printk.h>
  37. #include <linux/uaccess.h>
  38. #include <asm/tlb.h>
  39. #include <asm/tlbflush.h>
  40. #include <asm/mmu_context.h>
  41. #include "internal.h"
  42. void *high_memory;
  43. EXPORT_SYMBOL(high_memory);
  44. struct page *mem_map;
  45. unsigned long max_mapnr;
  46. EXPORT_SYMBOL(max_mapnr);
  47. unsigned long highest_memmap_pfn;
  48. int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
  49. int heap_stack_gap = 0;
  50. atomic_long_t mmap_pages_allocated;
  51. EXPORT_SYMBOL(mem_map);
  52. /* list of mapped, potentially shareable regions */
  53. static struct kmem_cache *vm_region_jar;
  54. struct rb_root nommu_region_tree = RB_ROOT;
  55. DECLARE_RWSEM(nommu_region_sem);
  56. const struct vm_operations_struct generic_file_vm_ops = {
  57. };
  58. /*
  59. * Return the total memory allocated for this pointer, not
  60. * just what the caller asked for.
  61. *
  62. * Doesn't have to be accurate, i.e. may have races.
  63. */
  64. unsigned int kobjsize(const void *objp)
  65. {
  66. struct page *page;
  67. /*
  68. * If the object we have should not have ksize performed on it,
  69. * return size of 0
  70. */
  71. if (!objp || !virt_addr_valid(objp))
  72. return 0;
  73. page = virt_to_head_page(objp);
  74. /*
  75. * If the allocator sets PageSlab, we know the pointer came from
  76. * kmalloc().
  77. */
  78. if (PageSlab(page))
  79. return ksize(objp);
  80. /*
  81. * If it's not a compound page, see if we have a matching VMA
  82. * region. This test is intentionally done in reverse order,
  83. * so if there's no VMA, we still fall through and hand back
  84. * PAGE_SIZE for 0-order pages.
  85. */
  86. if (!PageCompound(page)) {
  87. struct vm_area_struct *vma;
  88. vma = find_vma(current->mm, (unsigned long)objp);
  89. if (vma)
  90. return vma->vm_end - vma->vm_start;
  91. }
  92. /*
  93. * The ksize() function is only guaranteed to work for pointers
  94. * returned by kmalloc(). So handle arbitrary pointers here.
  95. */
  96. return page_size(page);
  97. }
  98. /**
  99. * follow_pfn - look up PFN at a user virtual address
  100. * @vma: memory mapping
  101. * @address: user virtual address
  102. * @pfn: location to store found PFN
  103. *
  104. * Only IO mappings and raw PFN mappings are allowed.
  105. *
  106. * Returns zero and the pfn at @pfn on success, -ve otherwise.
  107. */
  108. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  109. unsigned long *pfn)
  110. {
  111. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  112. return -EINVAL;
  113. *pfn = address >> PAGE_SHIFT;
  114. return 0;
  115. }
  116. EXPORT_SYMBOL(follow_pfn);
  117. LIST_HEAD(vmap_area_list);
  118. void vfree(const void *addr)
  119. {
  120. kfree(addr);
  121. }
  122. EXPORT_SYMBOL(vfree);
  123. void *__vmalloc(unsigned long size, gfp_t gfp_mask)
  124. {
  125. /*
  126. * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
  127. * returns only a logical address.
  128. */
  129. return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
  130. }
  131. EXPORT_SYMBOL(__vmalloc);
  132. void *__vmalloc_node_range(unsigned long size, unsigned long align,
  133. unsigned long start, unsigned long end, gfp_t gfp_mask,
  134. pgprot_t prot, unsigned long vm_flags, int node,
  135. const void *caller)
  136. {
  137. return __vmalloc(size, gfp_mask);
  138. }
  139. void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask,
  140. int node, const void *caller)
  141. {
  142. return __vmalloc(size, gfp_mask);
  143. }
  144. static void *__vmalloc_user_flags(unsigned long size, gfp_t flags)
  145. {
  146. void *ret;
  147. ret = __vmalloc(size, flags);
  148. if (ret) {
  149. struct vm_area_struct *vma;
  150. mmap_write_lock(current->mm);
  151. vma = find_vma(current->mm, (unsigned long)ret);
  152. if (vma)
  153. vma->vm_flags |= VM_USERMAP;
  154. mmap_write_unlock(current->mm);
  155. }
  156. return ret;
  157. }
  158. void *vmalloc_user(unsigned long size)
  159. {
  160. return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO);
  161. }
  162. EXPORT_SYMBOL(vmalloc_user);
  163. struct page *vmalloc_to_page(const void *addr)
  164. {
  165. return virt_to_page(addr);
  166. }
  167. EXPORT_SYMBOL(vmalloc_to_page);
  168. unsigned long vmalloc_to_pfn(const void *addr)
  169. {
  170. return page_to_pfn(virt_to_page(addr));
  171. }
  172. EXPORT_SYMBOL(vmalloc_to_pfn);
  173. long vread(char *buf, char *addr, unsigned long count)
  174. {
  175. /* Don't allow overflow */
  176. if ((unsigned long) buf + count < count)
  177. count = -(unsigned long) buf;
  178. memcpy(buf, addr, count);
  179. return count;
  180. }
  181. long vwrite(char *buf, char *addr, unsigned long count)
  182. {
  183. /* Don't allow overflow */
  184. if ((unsigned long) addr + count < count)
  185. count = -(unsigned long) addr;
  186. memcpy(addr, buf, count);
  187. return count;
  188. }
  189. /*
  190. * vmalloc - allocate virtually contiguous memory
  191. *
  192. * @size: allocation size
  193. *
  194. * Allocate enough pages to cover @size from the page level
  195. * allocator and map them into contiguous kernel virtual space.
  196. *
  197. * For tight control over page level allocator and protection flags
  198. * use __vmalloc() instead.
  199. */
  200. void *vmalloc(unsigned long size)
  201. {
  202. return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM);
  203. }
  204. EXPORT_SYMBOL(vmalloc);
  205. /*
  206. * vzalloc - allocate virtually contiguous memory with zero fill
  207. *
  208. * @size: allocation size
  209. *
  210. * Allocate enough pages to cover @size from the page level
  211. * allocator and map them into contiguous kernel virtual space.
  212. * The memory allocated is set to zero.
  213. *
  214. * For tight control over page level allocator and protection flags
  215. * use __vmalloc() instead.
  216. */
  217. void *vzalloc(unsigned long size)
  218. {
  219. return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  220. }
  221. EXPORT_SYMBOL(vzalloc);
  222. /**
  223. * vmalloc_node - allocate memory on a specific node
  224. * @size: allocation size
  225. * @node: numa node
  226. *
  227. * Allocate enough pages to cover @size from the page level
  228. * allocator and map them into contiguous kernel virtual space.
  229. *
  230. * For tight control over page level allocator and protection flags
  231. * use __vmalloc() instead.
  232. */
  233. void *vmalloc_node(unsigned long size, int node)
  234. {
  235. return vmalloc(size);
  236. }
  237. EXPORT_SYMBOL(vmalloc_node);
  238. /**
  239. * vzalloc_node - allocate memory on a specific node with zero fill
  240. * @size: allocation size
  241. * @node: numa node
  242. *
  243. * Allocate enough pages to cover @size from the page level
  244. * allocator and map them into contiguous kernel virtual space.
  245. * The memory allocated is set to zero.
  246. *
  247. * For tight control over page level allocator and protection flags
  248. * use __vmalloc() instead.
  249. */
  250. void *vzalloc_node(unsigned long size, int node)
  251. {
  252. return vzalloc(size);
  253. }
  254. EXPORT_SYMBOL(vzalloc_node);
  255. /**
  256. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  257. * @size: allocation size
  258. *
  259. * Allocate enough 32bit PA addressable pages to cover @size from the
  260. * page level allocator and map them into contiguous kernel virtual space.
  261. */
  262. void *vmalloc_32(unsigned long size)
  263. {
  264. return __vmalloc(size, GFP_KERNEL);
  265. }
  266. EXPORT_SYMBOL(vmalloc_32);
  267. /**
  268. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  269. * @size: allocation size
  270. *
  271. * The resulting memory area is 32bit addressable and zeroed so it can be
  272. * mapped to userspace without leaking data.
  273. *
  274. * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
  275. * remap_vmalloc_range() are permissible.
  276. */
  277. void *vmalloc_32_user(unsigned long size)
  278. {
  279. /*
  280. * We'll have to sort out the ZONE_DMA bits for 64-bit,
  281. * but for now this can simply use vmalloc_user() directly.
  282. */
  283. return vmalloc_user(size);
  284. }
  285. EXPORT_SYMBOL(vmalloc_32_user);
  286. void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
  287. {
  288. BUG();
  289. return NULL;
  290. }
  291. EXPORT_SYMBOL(vmap);
  292. void vunmap(const void *addr)
  293. {
  294. BUG();
  295. }
  296. EXPORT_SYMBOL(vunmap);
  297. void *vm_map_ram(struct page **pages, unsigned int count, int node)
  298. {
  299. BUG();
  300. return NULL;
  301. }
  302. EXPORT_SYMBOL(vm_map_ram);
  303. void vm_unmap_ram(const void *mem, unsigned int count)
  304. {
  305. BUG();
  306. }
  307. EXPORT_SYMBOL(vm_unmap_ram);
  308. void vm_unmap_aliases(void)
  309. {
  310. }
  311. EXPORT_SYMBOL_GPL(vm_unmap_aliases);
  312. void free_vm_area(struct vm_struct *area)
  313. {
  314. BUG();
  315. }
  316. EXPORT_SYMBOL_GPL(free_vm_area);
  317. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  318. struct page *page)
  319. {
  320. return -EINVAL;
  321. }
  322. EXPORT_SYMBOL(vm_insert_page);
  323. int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
  324. unsigned long num)
  325. {
  326. return -EINVAL;
  327. }
  328. EXPORT_SYMBOL(vm_map_pages);
  329. int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
  330. unsigned long num)
  331. {
  332. return -EINVAL;
  333. }
  334. EXPORT_SYMBOL(vm_map_pages_zero);
  335. /*
  336. * sys_brk() for the most part doesn't need the global kernel
  337. * lock, except when an application is doing something nasty
  338. * like trying to un-brk an area that has already been mapped
  339. * to a regular file. in this case, the unmapping will need
  340. * to invoke file system routines that need the global lock.
  341. */
  342. SYSCALL_DEFINE1(brk, unsigned long, brk)
  343. {
  344. struct mm_struct *mm = current->mm;
  345. if (brk < mm->start_brk || brk > mm->context.end_brk)
  346. return mm->brk;
  347. if (mm->brk == brk)
  348. return mm->brk;
  349. /*
  350. * Always allow shrinking brk
  351. */
  352. if (brk <= mm->brk) {
  353. mm->brk = brk;
  354. return brk;
  355. }
  356. /*
  357. * Ok, looks good - let it rip.
  358. */
  359. flush_icache_user_range(mm->brk, brk);
  360. return mm->brk = brk;
  361. }
  362. /*
  363. * initialise the percpu counter for VM and region record slabs
  364. */
  365. void __init mmap_init(void)
  366. {
  367. int ret;
  368. ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
  369. VM_BUG_ON(ret);
  370. vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
  371. }
  372. /*
  373. * validate the region tree
  374. * - the caller must hold the region lock
  375. */
  376. #ifdef CONFIG_DEBUG_NOMMU_REGIONS
  377. static noinline void validate_nommu_regions(void)
  378. {
  379. struct vm_region *region, *last;
  380. struct rb_node *p, *lastp;
  381. lastp = rb_first(&nommu_region_tree);
  382. if (!lastp)
  383. return;
  384. last = rb_entry(lastp, struct vm_region, vm_rb);
  385. BUG_ON(last->vm_end <= last->vm_start);
  386. BUG_ON(last->vm_top < last->vm_end);
  387. while ((p = rb_next(lastp))) {
  388. region = rb_entry(p, struct vm_region, vm_rb);
  389. last = rb_entry(lastp, struct vm_region, vm_rb);
  390. BUG_ON(region->vm_end <= region->vm_start);
  391. BUG_ON(region->vm_top < region->vm_end);
  392. BUG_ON(region->vm_start < last->vm_top);
  393. lastp = p;
  394. }
  395. }
  396. #else
  397. static void validate_nommu_regions(void)
  398. {
  399. }
  400. #endif
  401. /*
  402. * add a region into the global tree
  403. */
  404. static void add_nommu_region(struct vm_region *region)
  405. {
  406. struct vm_region *pregion;
  407. struct rb_node **p, *parent;
  408. validate_nommu_regions();
  409. parent = NULL;
  410. p = &nommu_region_tree.rb_node;
  411. while (*p) {
  412. parent = *p;
  413. pregion = rb_entry(parent, struct vm_region, vm_rb);
  414. if (region->vm_start < pregion->vm_start)
  415. p = &(*p)->rb_left;
  416. else if (region->vm_start > pregion->vm_start)
  417. p = &(*p)->rb_right;
  418. else if (pregion == region)
  419. return;
  420. else
  421. BUG();
  422. }
  423. rb_link_node(&region->vm_rb, parent, p);
  424. rb_insert_color(&region->vm_rb, &nommu_region_tree);
  425. validate_nommu_regions();
  426. }
  427. /*
  428. * delete a region from the global tree
  429. */
  430. static void delete_nommu_region(struct vm_region *region)
  431. {
  432. BUG_ON(!nommu_region_tree.rb_node);
  433. validate_nommu_regions();
  434. rb_erase(&region->vm_rb, &nommu_region_tree);
  435. validate_nommu_regions();
  436. }
  437. /*
  438. * free a contiguous series of pages
  439. */
  440. static void free_page_series(unsigned long from, unsigned long to)
  441. {
  442. for (; from < to; from += PAGE_SIZE) {
  443. struct page *page = virt_to_page(from);
  444. atomic_long_dec(&mmap_pages_allocated);
  445. put_page(page);
  446. }
  447. }
  448. /*
  449. * release a reference to a region
  450. * - the caller must hold the region semaphore for writing, which this releases
  451. * - the region may not have been added to the tree yet, in which case vm_top
  452. * will equal vm_start
  453. */
  454. static void __put_nommu_region(struct vm_region *region)
  455. __releases(nommu_region_sem)
  456. {
  457. BUG_ON(!nommu_region_tree.rb_node);
  458. if (--region->vm_usage == 0) {
  459. if (region->vm_top > region->vm_start)
  460. delete_nommu_region(region);
  461. up_write(&nommu_region_sem);
  462. if (region->vm_file)
  463. fput(region->vm_file);
  464. /* IO memory and memory shared directly out of the pagecache
  465. * from ramfs/tmpfs mustn't be released here */
  466. if (region->vm_flags & VM_MAPPED_COPY)
  467. free_page_series(region->vm_start, region->vm_top);
  468. kmem_cache_free(vm_region_jar, region);
  469. } else {
  470. up_write(&nommu_region_sem);
  471. }
  472. }
  473. /*
  474. * release a reference to a region
  475. */
  476. static void put_nommu_region(struct vm_region *region)
  477. {
  478. down_write(&nommu_region_sem);
  479. __put_nommu_region(region);
  480. }
  481. /*
  482. * add a VMA into a process's mm_struct in the appropriate place in the list
  483. * and tree and add to the address space's page tree also if not an anonymous
  484. * page
  485. * - should be called with mm->mmap_lock held writelocked
  486. */
  487. static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
  488. {
  489. struct vm_area_struct *pvma, *prev;
  490. struct address_space *mapping;
  491. struct rb_node **p, *parent, *rb_prev;
  492. BUG_ON(!vma->vm_region);
  493. mm->map_count++;
  494. vma->vm_mm = mm;
  495. /* add the VMA to the mapping */
  496. if (vma->vm_file) {
  497. mapping = vma->vm_file->f_mapping;
  498. i_mmap_lock_write(mapping);
  499. flush_dcache_mmap_lock(mapping);
  500. vma_interval_tree_insert(vma, &mapping->i_mmap);
  501. flush_dcache_mmap_unlock(mapping);
  502. i_mmap_unlock_write(mapping);
  503. }
  504. /* add the VMA to the tree */
  505. parent = rb_prev = NULL;
  506. p = &mm->mm_rb.rb_node;
  507. while (*p) {
  508. parent = *p;
  509. pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
  510. /* sort by: start addr, end addr, VMA struct addr in that order
  511. * (the latter is necessary as we may get identical VMAs) */
  512. if (vma->vm_start < pvma->vm_start)
  513. p = &(*p)->rb_left;
  514. else if (vma->vm_start > pvma->vm_start) {
  515. rb_prev = parent;
  516. p = &(*p)->rb_right;
  517. } else if (vma->vm_end < pvma->vm_end)
  518. p = &(*p)->rb_left;
  519. else if (vma->vm_end > pvma->vm_end) {
  520. rb_prev = parent;
  521. p = &(*p)->rb_right;
  522. } else if (vma < pvma)
  523. p = &(*p)->rb_left;
  524. else if (vma > pvma) {
  525. rb_prev = parent;
  526. p = &(*p)->rb_right;
  527. } else
  528. BUG();
  529. }
  530. rb_link_node(&vma->vm_rb, parent, p);
  531. rb_insert_color(&vma->vm_rb, &mm->mm_rb);
  532. /* add VMA to the VMA list also */
  533. prev = NULL;
  534. if (rb_prev)
  535. prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
  536. __vma_link_list(mm, vma, prev);
  537. }
  538. /*
  539. * delete a VMA from its owning mm_struct and address space
  540. */
  541. static void delete_vma_from_mm(struct vm_area_struct *vma)
  542. {
  543. int i;
  544. struct address_space *mapping;
  545. struct mm_struct *mm = vma->vm_mm;
  546. struct task_struct *curr = current;
  547. mm->map_count--;
  548. for (i = 0; i < VMACACHE_SIZE; i++) {
  549. /* if the vma is cached, invalidate the entire cache */
  550. if (curr->vmacache.vmas[i] == vma) {
  551. vmacache_invalidate(mm);
  552. break;
  553. }
  554. }
  555. /* remove the VMA from the mapping */
  556. if (vma->vm_file) {
  557. mapping = vma->vm_file->f_mapping;
  558. i_mmap_lock_write(mapping);
  559. flush_dcache_mmap_lock(mapping);
  560. vma_interval_tree_remove(vma, &mapping->i_mmap);
  561. flush_dcache_mmap_unlock(mapping);
  562. i_mmap_unlock_write(mapping);
  563. }
  564. /* remove from the MM's tree and list */
  565. rb_erase(&vma->vm_rb, &mm->mm_rb);
  566. __vma_unlink_list(mm, vma);
  567. }
  568. /*
  569. * destroy a VMA record
  570. */
  571. static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
  572. {
  573. if (vma->vm_ops && vma->vm_ops->close)
  574. vma->vm_ops->close(vma);
  575. if (vma->vm_file)
  576. fput(vma->vm_file);
  577. put_nommu_region(vma->vm_region);
  578. vm_area_free(vma);
  579. }
  580. /*
  581. * look up the first VMA in which addr resides, NULL if none
  582. * - should be called with mm->mmap_lock at least held readlocked
  583. */
  584. struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
  585. {
  586. struct vm_area_struct *vma;
  587. /* check the cache first */
  588. vma = vmacache_find(mm, addr);
  589. if (likely(vma))
  590. return vma;
  591. /* trawl the list (there may be multiple mappings in which addr
  592. * resides) */
  593. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  594. if (vma->vm_start > addr)
  595. return NULL;
  596. if (vma->vm_end > addr) {
  597. vmacache_update(addr, vma);
  598. return vma;
  599. }
  600. }
  601. return NULL;
  602. }
  603. EXPORT_SYMBOL(find_vma);
  604. /*
  605. * find a VMA
  606. * - we don't extend stack VMAs under NOMMU conditions
  607. */
  608. struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
  609. {
  610. return find_vma(mm, addr);
  611. }
  612. /*
  613. * expand a stack to a given address
  614. * - not supported under NOMMU conditions
  615. */
  616. int expand_stack(struct vm_area_struct *vma, unsigned long address)
  617. {
  618. return -ENOMEM;
  619. }
  620. /*
  621. * look up the first VMA exactly that exactly matches addr
  622. * - should be called with mm->mmap_lock at least held readlocked
  623. */
  624. static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
  625. unsigned long addr,
  626. unsigned long len)
  627. {
  628. struct vm_area_struct *vma;
  629. unsigned long end = addr + len;
  630. /* check the cache first */
  631. vma = vmacache_find_exact(mm, addr, end);
  632. if (vma)
  633. return vma;
  634. /* trawl the list (there may be multiple mappings in which addr
  635. * resides) */
  636. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  637. if (vma->vm_start < addr)
  638. continue;
  639. if (vma->vm_start > addr)
  640. return NULL;
  641. if (vma->vm_end == end) {
  642. vmacache_update(addr, vma);
  643. return vma;
  644. }
  645. }
  646. return NULL;
  647. }
  648. /*
  649. * determine whether a mapping should be permitted and, if so, what sort of
  650. * mapping we're capable of supporting
  651. */
  652. static int validate_mmap_request(struct file *file,
  653. unsigned long addr,
  654. unsigned long len,
  655. unsigned long prot,
  656. unsigned long flags,
  657. unsigned long pgoff,
  658. unsigned long *_capabilities)
  659. {
  660. unsigned long capabilities, rlen;
  661. int ret;
  662. /* do the simple checks first */
  663. if (flags & MAP_FIXED)
  664. return -EINVAL;
  665. if ((flags & MAP_TYPE) != MAP_PRIVATE &&
  666. (flags & MAP_TYPE) != MAP_SHARED)
  667. return -EINVAL;
  668. if (!len)
  669. return -EINVAL;
  670. /* Careful about overflows.. */
  671. rlen = PAGE_ALIGN(len);
  672. if (!rlen || rlen > TASK_SIZE)
  673. return -ENOMEM;
  674. /* offset overflow? */
  675. if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
  676. return -EOVERFLOW;
  677. if (file) {
  678. /* files must support mmap */
  679. if (!file->f_op->mmap)
  680. return -ENODEV;
  681. /* work out if what we've got could possibly be shared
  682. * - we support chardevs that provide their own "memory"
  683. * - we support files/blockdevs that are memory backed
  684. */
  685. if (file->f_op->mmap_capabilities) {
  686. capabilities = file->f_op->mmap_capabilities(file);
  687. } else {
  688. /* no explicit capabilities set, so assume some
  689. * defaults */
  690. switch (file_inode(file)->i_mode & S_IFMT) {
  691. case S_IFREG:
  692. case S_IFBLK:
  693. capabilities = NOMMU_MAP_COPY;
  694. break;
  695. case S_IFCHR:
  696. capabilities =
  697. NOMMU_MAP_DIRECT |
  698. NOMMU_MAP_READ |
  699. NOMMU_MAP_WRITE;
  700. break;
  701. default:
  702. return -EINVAL;
  703. }
  704. }
  705. /* eliminate any capabilities that we can't support on this
  706. * device */
  707. if (!file->f_op->get_unmapped_area)
  708. capabilities &= ~NOMMU_MAP_DIRECT;
  709. if (!(file->f_mode & FMODE_CAN_READ))
  710. capabilities &= ~NOMMU_MAP_COPY;
  711. /* The file shall have been opened with read permission. */
  712. if (!(file->f_mode & FMODE_READ))
  713. return -EACCES;
  714. if (flags & MAP_SHARED) {
  715. /* do checks for writing, appending and locking */
  716. if ((prot & PROT_WRITE) &&
  717. !(file->f_mode & FMODE_WRITE))
  718. return -EACCES;
  719. if (IS_APPEND(file_inode(file)) &&
  720. (file->f_mode & FMODE_WRITE))
  721. return -EACCES;
  722. if (locks_verify_locked(file))
  723. return -EAGAIN;
  724. if (!(capabilities & NOMMU_MAP_DIRECT))
  725. return -ENODEV;
  726. /* we mustn't privatise shared mappings */
  727. capabilities &= ~NOMMU_MAP_COPY;
  728. } else {
  729. /* we're going to read the file into private memory we
  730. * allocate */
  731. if (!(capabilities & NOMMU_MAP_COPY))
  732. return -ENODEV;
  733. /* we don't permit a private writable mapping to be
  734. * shared with the backing device */
  735. if (prot & PROT_WRITE)
  736. capabilities &= ~NOMMU_MAP_DIRECT;
  737. }
  738. if (capabilities & NOMMU_MAP_DIRECT) {
  739. if (((prot & PROT_READ) && !(capabilities & NOMMU_MAP_READ)) ||
  740. ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
  741. ((prot & PROT_EXEC) && !(capabilities & NOMMU_MAP_EXEC))
  742. ) {
  743. capabilities &= ~NOMMU_MAP_DIRECT;
  744. if (flags & MAP_SHARED) {
  745. pr_warn("MAP_SHARED not completely supported on !MMU\n");
  746. return -EINVAL;
  747. }
  748. }
  749. }
  750. /* handle executable mappings and implied executable
  751. * mappings */
  752. if (path_noexec(&file->f_path)) {
  753. if (prot & PROT_EXEC)
  754. return -EPERM;
  755. } else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
  756. /* handle implication of PROT_EXEC by PROT_READ */
  757. if (current->personality & READ_IMPLIES_EXEC) {
  758. if (capabilities & NOMMU_MAP_EXEC)
  759. prot |= PROT_EXEC;
  760. }
  761. } else if ((prot & PROT_READ) &&
  762. (prot & PROT_EXEC) &&
  763. !(capabilities & NOMMU_MAP_EXEC)
  764. ) {
  765. /* backing file is not executable, try to copy */
  766. capabilities &= ~NOMMU_MAP_DIRECT;
  767. }
  768. } else {
  769. /* anonymous mappings are always memory backed and can be
  770. * privately mapped
  771. */
  772. capabilities = NOMMU_MAP_COPY;
  773. /* handle PROT_EXEC implication by PROT_READ */
  774. if ((prot & PROT_READ) &&
  775. (current->personality & READ_IMPLIES_EXEC))
  776. prot |= PROT_EXEC;
  777. }
  778. /* allow the security API to have its say */
  779. ret = security_mmap_addr(addr);
  780. if (ret < 0)
  781. return ret;
  782. /* looks okay */
  783. *_capabilities = capabilities;
  784. return 0;
  785. }
  786. /*
  787. * we've determined that we can make the mapping, now translate what we
  788. * now know into VMA flags
  789. */
  790. static unsigned long determine_vm_flags(struct file *file,
  791. unsigned long prot,
  792. unsigned long flags,
  793. unsigned long capabilities)
  794. {
  795. unsigned long vm_flags;
  796. vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
  797. /* vm_flags |= mm->def_flags; */
  798. if (!(capabilities & NOMMU_MAP_DIRECT)) {
  799. /* attempt to share read-only copies of mapped file chunks */
  800. vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
  801. if (file && !(prot & PROT_WRITE))
  802. vm_flags |= VM_MAYSHARE;
  803. } else {
  804. /* overlay a shareable mapping on the backing device or inode
  805. * if possible - used for chardevs, ramfs/tmpfs/shmfs and
  806. * romfs/cramfs */
  807. vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
  808. if (flags & MAP_SHARED)
  809. vm_flags |= VM_SHARED;
  810. }
  811. /* refuse to let anyone share private mappings with this process if
  812. * it's being traced - otherwise breakpoints set in it may interfere
  813. * with another untraced process
  814. */
  815. if ((flags & MAP_PRIVATE) && current->ptrace)
  816. vm_flags &= ~VM_MAYSHARE;
  817. return vm_flags;
  818. }
  819. /*
  820. * set up a shared mapping on a file (the driver or filesystem provides and
  821. * pins the storage)
  822. */
  823. static int do_mmap_shared_file(struct vm_area_struct *vma)
  824. {
  825. int ret;
  826. ret = call_mmap(vma->vm_file, vma);
  827. if (ret == 0) {
  828. vma->vm_region->vm_top = vma->vm_region->vm_end;
  829. return 0;
  830. }
  831. if (ret != -ENOSYS)
  832. return ret;
  833. /* getting -ENOSYS indicates that direct mmap isn't possible (as
  834. * opposed to tried but failed) so we can only give a suitable error as
  835. * it's not possible to make a private copy if MAP_SHARED was given */
  836. return -ENODEV;
  837. }
  838. /*
  839. * set up a private mapping or an anonymous shared mapping
  840. */
  841. static int do_mmap_private(struct vm_area_struct *vma,
  842. struct vm_region *region,
  843. unsigned long len,
  844. unsigned long capabilities)
  845. {
  846. unsigned long total, point;
  847. void *base;
  848. int ret, order;
  849. /* invoke the file's mapping function so that it can keep track of
  850. * shared mappings on devices or memory
  851. * - VM_MAYSHARE will be set if it may attempt to share
  852. */
  853. if (capabilities & NOMMU_MAP_DIRECT) {
  854. ret = call_mmap(vma->vm_file, vma);
  855. if (ret == 0) {
  856. /* shouldn't return success if we're not sharing */
  857. BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
  858. vma->vm_region->vm_top = vma->vm_region->vm_end;
  859. return 0;
  860. }
  861. if (ret != -ENOSYS)
  862. return ret;
  863. /* getting an ENOSYS error indicates that direct mmap isn't
  864. * possible (as opposed to tried but failed) so we'll try to
  865. * make a private copy of the data and map that instead */
  866. }
  867. /* allocate some memory to hold the mapping
  868. * - note that this may not return a page-aligned address if the object
  869. * we're allocating is smaller than a page
  870. */
  871. order = get_order(len);
  872. total = 1 << order;
  873. point = len >> PAGE_SHIFT;
  874. /* we don't want to allocate a power-of-2 sized page set */
  875. if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
  876. total = point;
  877. base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
  878. if (!base)
  879. goto enomem;
  880. atomic_long_add(total, &mmap_pages_allocated);
  881. region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
  882. region->vm_start = (unsigned long) base;
  883. region->vm_end = region->vm_start + len;
  884. region->vm_top = region->vm_start + (total << PAGE_SHIFT);
  885. vma->vm_start = region->vm_start;
  886. vma->vm_end = region->vm_start + len;
  887. if (vma->vm_file) {
  888. /* read the contents of a file into the copy */
  889. loff_t fpos;
  890. fpos = vma->vm_pgoff;
  891. fpos <<= PAGE_SHIFT;
  892. ret = kernel_read(vma->vm_file, base, len, &fpos);
  893. if (ret < 0)
  894. goto error_free;
  895. /* clear the last little bit */
  896. if (ret < len)
  897. memset(base + ret, 0, len - ret);
  898. } else {
  899. vma_set_anonymous(vma);
  900. }
  901. return 0;
  902. error_free:
  903. free_page_series(region->vm_start, region->vm_top);
  904. region->vm_start = vma->vm_start = 0;
  905. region->vm_end = vma->vm_end = 0;
  906. region->vm_top = 0;
  907. return ret;
  908. enomem:
  909. pr_err("Allocation of length %lu from process %d (%s) failed\n",
  910. len, current->pid, current->comm);
  911. show_free_areas(0, NULL);
  912. return -ENOMEM;
  913. }
  914. /*
  915. * handle mapping creation for uClinux
  916. */
  917. unsigned long do_mmap(struct file *file,
  918. unsigned long addr,
  919. unsigned long len,
  920. unsigned long prot,
  921. unsigned long flags,
  922. unsigned long pgoff,
  923. unsigned long *populate,
  924. struct list_head *uf)
  925. {
  926. struct vm_area_struct *vma;
  927. struct vm_region *region;
  928. struct rb_node *rb;
  929. vm_flags_t vm_flags;
  930. unsigned long capabilities, result;
  931. int ret;
  932. *populate = 0;
  933. /* decide whether we should attempt the mapping, and if so what sort of
  934. * mapping */
  935. ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
  936. &capabilities);
  937. if (ret < 0)
  938. return ret;
  939. /* we ignore the address hint */
  940. addr = 0;
  941. len = PAGE_ALIGN(len);
  942. /* we've determined that we can make the mapping, now translate what we
  943. * now know into VMA flags */
  944. vm_flags = determine_vm_flags(file, prot, flags, capabilities);
  945. /* we're going to need to record the mapping */
  946. region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
  947. if (!region)
  948. goto error_getting_region;
  949. vma = vm_area_alloc(current->mm);
  950. if (!vma)
  951. goto error_getting_vma;
  952. region->vm_usage = 1;
  953. region->vm_flags = vm_flags;
  954. region->vm_pgoff = pgoff;
  955. vma->vm_flags = vm_flags;
  956. vma->vm_pgoff = pgoff;
  957. if (file) {
  958. region->vm_file = get_file(file);
  959. vma->vm_file = get_file(file);
  960. }
  961. down_write(&nommu_region_sem);
  962. /* if we want to share, we need to check for regions created by other
  963. * mmap() calls that overlap with our proposed mapping
  964. * - we can only share with a superset match on most regular files
  965. * - shared mappings on character devices and memory backed files are
  966. * permitted to overlap inexactly as far as we are concerned for in
  967. * these cases, sharing is handled in the driver or filesystem rather
  968. * than here
  969. */
  970. if (vm_flags & VM_MAYSHARE) {
  971. struct vm_region *pregion;
  972. unsigned long pglen, rpglen, pgend, rpgend, start;
  973. pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  974. pgend = pgoff + pglen;
  975. for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
  976. pregion = rb_entry(rb, struct vm_region, vm_rb);
  977. if (!(pregion->vm_flags & VM_MAYSHARE))
  978. continue;
  979. /* search for overlapping mappings on the same file */
  980. if (file_inode(pregion->vm_file) !=
  981. file_inode(file))
  982. continue;
  983. if (pregion->vm_pgoff >= pgend)
  984. continue;
  985. rpglen = pregion->vm_end - pregion->vm_start;
  986. rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  987. rpgend = pregion->vm_pgoff + rpglen;
  988. if (pgoff >= rpgend)
  989. continue;
  990. /* handle inexactly overlapping matches between
  991. * mappings */
  992. if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
  993. !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
  994. /* new mapping is not a subset of the region */
  995. if (!(capabilities & NOMMU_MAP_DIRECT))
  996. goto sharing_violation;
  997. continue;
  998. }
  999. /* we've found a region we can share */
  1000. pregion->vm_usage++;
  1001. vma->vm_region = pregion;
  1002. start = pregion->vm_start;
  1003. start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
  1004. vma->vm_start = start;
  1005. vma->vm_end = start + len;
  1006. if (pregion->vm_flags & VM_MAPPED_COPY)
  1007. vma->vm_flags |= VM_MAPPED_COPY;
  1008. else {
  1009. ret = do_mmap_shared_file(vma);
  1010. if (ret < 0) {
  1011. vma->vm_region = NULL;
  1012. vma->vm_start = 0;
  1013. vma->vm_end = 0;
  1014. pregion->vm_usage--;
  1015. pregion = NULL;
  1016. goto error_just_free;
  1017. }
  1018. }
  1019. fput(region->vm_file);
  1020. kmem_cache_free(vm_region_jar, region);
  1021. region = pregion;
  1022. result = start;
  1023. goto share;
  1024. }
  1025. /* obtain the address at which to make a shared mapping
  1026. * - this is the hook for quasi-memory character devices to
  1027. * tell us the location of a shared mapping
  1028. */
  1029. if (capabilities & NOMMU_MAP_DIRECT) {
  1030. addr = file->f_op->get_unmapped_area(file, addr, len,
  1031. pgoff, flags);
  1032. if (IS_ERR_VALUE(addr)) {
  1033. ret = addr;
  1034. if (ret != -ENOSYS)
  1035. goto error_just_free;
  1036. /* the driver refused to tell us where to site
  1037. * the mapping so we'll have to attempt to copy
  1038. * it */
  1039. ret = -ENODEV;
  1040. if (!(capabilities & NOMMU_MAP_COPY))
  1041. goto error_just_free;
  1042. capabilities &= ~NOMMU_MAP_DIRECT;
  1043. } else {
  1044. vma->vm_start = region->vm_start = addr;
  1045. vma->vm_end = region->vm_end = addr + len;
  1046. }
  1047. }
  1048. }
  1049. vma->vm_region = region;
  1050. /* set up the mapping
  1051. * - the region is filled in if NOMMU_MAP_DIRECT is still set
  1052. */
  1053. if (file && vma->vm_flags & VM_SHARED)
  1054. ret = do_mmap_shared_file(vma);
  1055. else
  1056. ret = do_mmap_private(vma, region, len, capabilities);
  1057. if (ret < 0)
  1058. goto error_just_free;
  1059. add_nommu_region(region);
  1060. /* clear anonymous mappings that don't ask for uninitialized data */
  1061. if (!vma->vm_file &&
  1062. (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) ||
  1063. !(flags & MAP_UNINITIALIZED)))
  1064. memset((void *)region->vm_start, 0,
  1065. region->vm_end - region->vm_start);
  1066. /* okay... we have a mapping; now we have to register it */
  1067. result = vma->vm_start;
  1068. current->mm->total_vm += len >> PAGE_SHIFT;
  1069. share:
  1070. add_vma_to_mm(current->mm, vma);
  1071. /* we flush the region from the icache only when the first executable
  1072. * mapping of it is made */
  1073. if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
  1074. flush_icache_user_range(region->vm_start, region->vm_end);
  1075. region->vm_icache_flushed = true;
  1076. }
  1077. up_write(&nommu_region_sem);
  1078. return result;
  1079. error_just_free:
  1080. up_write(&nommu_region_sem);
  1081. error:
  1082. if (region->vm_file)
  1083. fput(region->vm_file);
  1084. kmem_cache_free(vm_region_jar, region);
  1085. if (vma->vm_file)
  1086. fput(vma->vm_file);
  1087. vm_area_free(vma);
  1088. return ret;
  1089. sharing_violation:
  1090. up_write(&nommu_region_sem);
  1091. pr_warn("Attempt to share mismatched mappings\n");
  1092. ret = -EINVAL;
  1093. goto error;
  1094. error_getting_vma:
  1095. kmem_cache_free(vm_region_jar, region);
  1096. pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
  1097. len, current->pid);
  1098. show_free_areas(0, NULL);
  1099. return -ENOMEM;
  1100. error_getting_region:
  1101. pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
  1102. len, current->pid);
  1103. show_free_areas(0, NULL);
  1104. return -ENOMEM;
  1105. }
  1106. unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
  1107. unsigned long prot, unsigned long flags,
  1108. unsigned long fd, unsigned long pgoff)
  1109. {
  1110. struct file *file = NULL;
  1111. unsigned long retval = -EBADF;
  1112. audit_mmap_fd(fd, flags);
  1113. if (!(flags & MAP_ANONYMOUS)) {
  1114. file = fget(fd);
  1115. if (!file)
  1116. goto out;
  1117. }
  1118. flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
  1119. retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
  1120. if (file)
  1121. fput(file);
  1122. out:
  1123. return retval;
  1124. }
  1125. SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
  1126. unsigned long, prot, unsigned long, flags,
  1127. unsigned long, fd, unsigned long, pgoff)
  1128. {
  1129. return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
  1130. }
  1131. #ifdef __ARCH_WANT_SYS_OLD_MMAP
  1132. struct mmap_arg_struct {
  1133. unsigned long addr;
  1134. unsigned long len;
  1135. unsigned long prot;
  1136. unsigned long flags;
  1137. unsigned long fd;
  1138. unsigned long offset;
  1139. };
  1140. SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
  1141. {
  1142. struct mmap_arg_struct a;
  1143. if (copy_from_user(&a, arg, sizeof(a)))
  1144. return -EFAULT;
  1145. if (offset_in_page(a.offset))
  1146. return -EINVAL;
  1147. return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
  1148. a.offset >> PAGE_SHIFT);
  1149. }
  1150. #endif /* __ARCH_WANT_SYS_OLD_MMAP */
  1151. /*
  1152. * split a vma into two pieces at address 'addr', a new vma is allocated either
  1153. * for the first part or the tail.
  1154. */
  1155. int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
  1156. unsigned long addr, int new_below)
  1157. {
  1158. struct vm_area_struct *new;
  1159. struct vm_region *region;
  1160. unsigned long npages;
  1161. /* we're only permitted to split anonymous regions (these should have
  1162. * only a single usage on the region) */
  1163. if (vma->vm_file)
  1164. return -ENOMEM;
  1165. if (mm->map_count >= sysctl_max_map_count)
  1166. return -ENOMEM;
  1167. region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
  1168. if (!region)
  1169. return -ENOMEM;
  1170. new = vm_area_dup(vma);
  1171. if (!new) {
  1172. kmem_cache_free(vm_region_jar, region);
  1173. return -ENOMEM;
  1174. }
  1175. /* most fields are the same, copy all, and then fixup */
  1176. *region = *vma->vm_region;
  1177. new->vm_region = region;
  1178. npages = (addr - vma->vm_start) >> PAGE_SHIFT;
  1179. if (new_below) {
  1180. region->vm_top = region->vm_end = new->vm_end = addr;
  1181. } else {
  1182. region->vm_start = new->vm_start = addr;
  1183. region->vm_pgoff = new->vm_pgoff += npages;
  1184. }
  1185. if (new->vm_ops && new->vm_ops->open)
  1186. new->vm_ops->open(new);
  1187. delete_vma_from_mm(vma);
  1188. down_write(&nommu_region_sem);
  1189. delete_nommu_region(vma->vm_region);
  1190. if (new_below) {
  1191. vma->vm_region->vm_start = vma->vm_start = addr;
  1192. vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
  1193. } else {
  1194. vma->vm_region->vm_end = vma->vm_end = addr;
  1195. vma->vm_region->vm_top = addr;
  1196. }
  1197. add_nommu_region(vma->vm_region);
  1198. add_nommu_region(new->vm_region);
  1199. up_write(&nommu_region_sem);
  1200. add_vma_to_mm(mm, vma);
  1201. add_vma_to_mm(mm, new);
  1202. return 0;
  1203. }
  1204. /*
  1205. * shrink a VMA by removing the specified chunk from either the beginning or
  1206. * the end
  1207. */
  1208. static int shrink_vma(struct mm_struct *mm,
  1209. struct vm_area_struct *vma,
  1210. unsigned long from, unsigned long to)
  1211. {
  1212. struct vm_region *region;
  1213. /* adjust the VMA's pointers, which may reposition it in the MM's tree
  1214. * and list */
  1215. delete_vma_from_mm(vma);
  1216. if (from > vma->vm_start)
  1217. vma->vm_end = from;
  1218. else
  1219. vma->vm_start = to;
  1220. add_vma_to_mm(mm, vma);
  1221. /* cut the backing region down to size */
  1222. region = vma->vm_region;
  1223. BUG_ON(region->vm_usage != 1);
  1224. down_write(&nommu_region_sem);
  1225. delete_nommu_region(region);
  1226. if (from > region->vm_start) {
  1227. to = region->vm_top;
  1228. region->vm_top = region->vm_end = from;
  1229. } else {
  1230. region->vm_start = to;
  1231. }
  1232. add_nommu_region(region);
  1233. up_write(&nommu_region_sem);
  1234. free_page_series(from, to);
  1235. return 0;
  1236. }
  1237. /*
  1238. * release a mapping
  1239. * - under NOMMU conditions the chunk to be unmapped must be backed by a single
  1240. * VMA, though it need not cover the whole VMA
  1241. */
  1242. int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
  1243. {
  1244. struct vm_area_struct *vma;
  1245. unsigned long end;
  1246. int ret;
  1247. len = PAGE_ALIGN(len);
  1248. if (len == 0)
  1249. return -EINVAL;
  1250. end = start + len;
  1251. /* find the first potentially overlapping VMA */
  1252. vma = find_vma(mm, start);
  1253. if (!vma) {
  1254. static int limit;
  1255. if (limit < 5) {
  1256. pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
  1257. current->pid, current->comm,
  1258. start, start + len - 1);
  1259. limit++;
  1260. }
  1261. return -EINVAL;
  1262. }
  1263. /* we're allowed to split an anonymous VMA but not a file-backed one */
  1264. if (vma->vm_file) {
  1265. do {
  1266. if (start > vma->vm_start)
  1267. return -EINVAL;
  1268. if (end == vma->vm_end)
  1269. goto erase_whole_vma;
  1270. vma = vma->vm_next;
  1271. } while (vma);
  1272. return -EINVAL;
  1273. } else {
  1274. /* the chunk must be a subset of the VMA found */
  1275. if (start == vma->vm_start && end == vma->vm_end)
  1276. goto erase_whole_vma;
  1277. if (start < vma->vm_start || end > vma->vm_end)
  1278. return -EINVAL;
  1279. if (offset_in_page(start))
  1280. return -EINVAL;
  1281. if (end != vma->vm_end && offset_in_page(end))
  1282. return -EINVAL;
  1283. if (start != vma->vm_start && end != vma->vm_end) {
  1284. ret = split_vma(mm, vma, start, 1);
  1285. if (ret < 0)
  1286. return ret;
  1287. }
  1288. return shrink_vma(mm, vma, start, end);
  1289. }
  1290. erase_whole_vma:
  1291. delete_vma_from_mm(vma);
  1292. delete_vma(mm, vma);
  1293. return 0;
  1294. }
  1295. EXPORT_SYMBOL(do_munmap);
  1296. int vm_munmap(unsigned long addr, size_t len)
  1297. {
  1298. struct mm_struct *mm = current->mm;
  1299. int ret;
  1300. mmap_write_lock(mm);
  1301. ret = do_munmap(mm, addr, len, NULL);
  1302. mmap_write_unlock(mm);
  1303. return ret;
  1304. }
  1305. EXPORT_SYMBOL(vm_munmap);
  1306. SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
  1307. {
  1308. return vm_munmap(addr, len);
  1309. }
  1310. /*
  1311. * release all the mappings made in a process's VM space
  1312. */
  1313. void exit_mmap(struct mm_struct *mm)
  1314. {
  1315. struct vm_area_struct *vma;
  1316. if (!mm)
  1317. return;
  1318. mm->total_vm = 0;
  1319. while ((vma = mm->mmap)) {
  1320. mm->mmap = vma->vm_next;
  1321. delete_vma_from_mm(vma);
  1322. delete_vma(mm, vma);
  1323. cond_resched();
  1324. }
  1325. }
  1326. int vm_brk(unsigned long addr, unsigned long len)
  1327. {
  1328. return -ENOMEM;
  1329. }
  1330. /*
  1331. * expand (or shrink) an existing mapping, potentially moving it at the same
  1332. * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
  1333. *
  1334. * under NOMMU conditions, we only permit changing a mapping's size, and only
  1335. * as long as it stays within the region allocated by do_mmap_private() and the
  1336. * block is not shareable
  1337. *
  1338. * MREMAP_FIXED is not supported under NOMMU conditions
  1339. */
  1340. static unsigned long do_mremap(unsigned long addr,
  1341. unsigned long old_len, unsigned long new_len,
  1342. unsigned long flags, unsigned long new_addr)
  1343. {
  1344. struct vm_area_struct *vma;
  1345. /* insanity checks first */
  1346. old_len = PAGE_ALIGN(old_len);
  1347. new_len = PAGE_ALIGN(new_len);
  1348. if (old_len == 0 || new_len == 0)
  1349. return (unsigned long) -EINVAL;
  1350. if (offset_in_page(addr))
  1351. return -EINVAL;
  1352. if (flags & MREMAP_FIXED && new_addr != addr)
  1353. return (unsigned long) -EINVAL;
  1354. vma = find_vma_exact(current->mm, addr, old_len);
  1355. if (!vma)
  1356. return (unsigned long) -EINVAL;
  1357. if (vma->vm_end != vma->vm_start + old_len)
  1358. return (unsigned long) -EFAULT;
  1359. if (vma->vm_flags & VM_MAYSHARE)
  1360. return (unsigned long) -EPERM;
  1361. if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
  1362. return (unsigned long) -ENOMEM;
  1363. /* all checks complete - do it */
  1364. vma->vm_end = vma->vm_start + new_len;
  1365. return vma->vm_start;
  1366. }
  1367. SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
  1368. unsigned long, new_len, unsigned long, flags,
  1369. unsigned long, new_addr)
  1370. {
  1371. unsigned long ret;
  1372. mmap_write_lock(current->mm);
  1373. ret = do_mremap(addr, old_len, new_len, flags, new_addr);
  1374. mmap_write_unlock(current->mm);
  1375. return ret;
  1376. }
  1377. struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
  1378. unsigned int foll_flags)
  1379. {
  1380. return NULL;
  1381. }
  1382. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  1383. unsigned long pfn, unsigned long size, pgprot_t prot)
  1384. {
  1385. if (addr != (pfn << PAGE_SHIFT))
  1386. return -EINVAL;
  1387. vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
  1388. return 0;
  1389. }
  1390. EXPORT_SYMBOL(remap_pfn_range);
  1391. int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
  1392. {
  1393. unsigned long pfn = start >> PAGE_SHIFT;
  1394. unsigned long vm_len = vma->vm_end - vma->vm_start;
  1395. pfn += vma->vm_pgoff;
  1396. return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
  1397. }
  1398. EXPORT_SYMBOL(vm_iomap_memory);
  1399. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  1400. unsigned long pgoff)
  1401. {
  1402. unsigned int size = vma->vm_end - vma->vm_start;
  1403. if (!(vma->vm_flags & VM_USERMAP))
  1404. return -EINVAL;
  1405. vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
  1406. vma->vm_end = vma->vm_start + size;
  1407. return 0;
  1408. }
  1409. EXPORT_SYMBOL(remap_vmalloc_range);
  1410. unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
  1411. unsigned long len, unsigned long pgoff, unsigned long flags)
  1412. {
  1413. return -ENOMEM;
  1414. }
  1415. vm_fault_t filemap_fault(struct vm_fault *vmf)
  1416. {
  1417. BUG();
  1418. return 0;
  1419. }
  1420. EXPORT_SYMBOL(filemap_fault);
  1421. vm_fault_t filemap_map_pages(struct vm_fault *vmf,
  1422. pgoff_t start_pgoff, pgoff_t end_pgoff)
  1423. {
  1424. BUG();
  1425. return 0;
  1426. }
  1427. EXPORT_SYMBOL(filemap_map_pages);
  1428. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  1429. bool filemap_allow_speculation(void)
  1430. {
  1431. BUG();
  1432. return false;
  1433. }
  1434. #endif
  1435. int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
  1436. unsigned long addr, void *buf, int len, unsigned int gup_flags)
  1437. {
  1438. struct vm_area_struct *vma;
  1439. int write = gup_flags & FOLL_WRITE;
  1440. if (mmap_read_lock_killable(mm))
  1441. return 0;
  1442. /* the access must start within one of the target process's mappings */
  1443. vma = find_vma(mm, addr);
  1444. if (vma) {
  1445. /* don't overrun this mapping */
  1446. if (addr + len >= vma->vm_end)
  1447. len = vma->vm_end - addr;
  1448. /* only read or write mappings where it is permitted */
  1449. if (write && vma->vm_flags & VM_MAYWRITE)
  1450. copy_to_user_page(vma, NULL, addr,
  1451. (void *) addr, buf, len);
  1452. else if (!write && vma->vm_flags & VM_MAYREAD)
  1453. copy_from_user_page(vma, NULL, addr,
  1454. buf, (void *) addr, len);
  1455. else
  1456. len = 0;
  1457. } else {
  1458. len = 0;
  1459. }
  1460. mmap_read_unlock(mm);
  1461. return len;
  1462. }
  1463. /**
  1464. * access_remote_vm - access another process' address space
  1465. * @mm: the mm_struct of the target address space
  1466. * @addr: start address to access
  1467. * @buf: source or destination buffer
  1468. * @len: number of bytes to transfer
  1469. * @gup_flags: flags modifying lookup behaviour
  1470. *
  1471. * The caller must hold a reference on @mm.
  1472. */
  1473. int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  1474. void *buf, int len, unsigned int gup_flags)
  1475. {
  1476. return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
  1477. }
  1478. /*
  1479. * Access another process' address space.
  1480. * - source/target buffer must be kernel space
  1481. */
  1482. int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
  1483. unsigned int gup_flags)
  1484. {
  1485. struct mm_struct *mm;
  1486. if (addr + len < addr)
  1487. return 0;
  1488. mm = get_task_mm(tsk);
  1489. if (!mm)
  1490. return 0;
  1491. len = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
  1492. mmput(mm);
  1493. return len;
  1494. }
  1495. EXPORT_SYMBOL_GPL(access_process_vm);
  1496. /**
  1497. * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
  1498. * @inode: The inode to check
  1499. * @size: The current filesize of the inode
  1500. * @newsize: The proposed filesize of the inode
  1501. *
  1502. * Check the shared mappings on an inode on behalf of a shrinking truncate to
  1503. * make sure that any outstanding VMAs aren't broken and then shrink the
  1504. * vm_regions that extend beyond so that do_mmap() doesn't
  1505. * automatically grant mappings that are too large.
  1506. */
  1507. int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
  1508. size_t newsize)
  1509. {
  1510. struct vm_area_struct *vma;
  1511. struct vm_region *region;
  1512. pgoff_t low, high;
  1513. size_t r_size, r_top;
  1514. low = newsize >> PAGE_SHIFT;
  1515. high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1516. down_write(&nommu_region_sem);
  1517. i_mmap_lock_read(inode->i_mapping);
  1518. /* search for VMAs that fall within the dead zone */
  1519. vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
  1520. /* found one - only interested if it's shared out of the page
  1521. * cache */
  1522. if (vma->vm_flags & VM_SHARED) {
  1523. i_mmap_unlock_read(inode->i_mapping);
  1524. up_write(&nommu_region_sem);
  1525. return -ETXTBSY; /* not quite true, but near enough */
  1526. }
  1527. }
  1528. /* reduce any regions that overlap the dead zone - if in existence,
  1529. * these will be pointed to by VMAs that don't overlap the dead zone
  1530. *
  1531. * we don't check for any regions that start beyond the EOF as there
  1532. * shouldn't be any
  1533. */
  1534. vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
  1535. if (!(vma->vm_flags & VM_SHARED))
  1536. continue;
  1537. region = vma->vm_region;
  1538. r_size = region->vm_top - region->vm_start;
  1539. r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
  1540. if (r_top > newsize) {
  1541. region->vm_top -= r_top - newsize;
  1542. if (region->vm_end > region->vm_top)
  1543. region->vm_end = region->vm_top;
  1544. }
  1545. }
  1546. i_mmap_unlock_read(inode->i_mapping);
  1547. up_write(&nommu_region_sem);
  1548. return 0;
  1549. }
  1550. /*
  1551. * Initialise sysctl_user_reserve_kbytes.
  1552. *
  1553. * This is intended to prevent a user from starting a single memory hogging
  1554. * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
  1555. * mode.
  1556. *
  1557. * The default value is min(3% of free memory, 128MB)
  1558. * 128MB is enough to recover with sshd/login, bash, and top/kill.
  1559. */
  1560. static int __meminit init_user_reserve(void)
  1561. {
  1562. unsigned long free_kbytes;
  1563. free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
  1564. sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
  1565. return 0;
  1566. }
  1567. subsys_initcall(init_user_reserve);
  1568. /*
  1569. * Initialise sysctl_admin_reserve_kbytes.
  1570. *
  1571. * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
  1572. * to log in and kill a memory hogging process.
  1573. *
  1574. * Systems with more than 256MB will reserve 8MB, enough to recover
  1575. * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
  1576. * only reserve 3% of free pages by default.
  1577. */
  1578. static int __meminit init_admin_reserve(void)
  1579. {
  1580. unsigned long free_kbytes;
  1581. free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
  1582. sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
  1583. return 0;
  1584. }
  1585. subsys_initcall(init_admin_reserve);