memory.c 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738
  1. /*
  2. * linux/mm/memory.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. */
  6. /*
  7. * demand-loading started 01.12.91 - seems it is high on the list of
  8. * things wanted, and it should be easy to implement. - Linus
  9. */
  10. /*
  11. * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  12. * pages started 02.12.91, seems to work. - Linus.
  13. *
  14. * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  15. * would have taken more than the 6M I have free, but it worked well as
  16. * far as I could see.
  17. *
  18. * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  19. */
  20. /*
  21. * Real VM (paging to/from disk) started 18.12.91. Much more work and
  22. * thought has to go into this. Oh, well..
  23. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
  24. * Found it. Everything seems to work now.
  25. * 20.12.91 - Ok, making the swap-device changeable like the root.
  26. */
  27. /*
  28. * 05.04.94 - Multi-page memory management added for v1.1.
  29. * Idea by Alex Bligh (alex@cconcepts.co.uk)
  30. *
  31. * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
  32. * (Gerhard.Wichert@pdb.siemens.de)
  33. *
  34. * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  35. */
  36. #include <linux/kernel_stat.h>
  37. #include <linux/mm.h>
  38. #include <linux/hugetlb.h>
  39. #include <linux/mman.h>
  40. #include <linux/swap.h>
  41. #include <linux/highmem.h>
  42. #include <linux/pagemap.h>
  43. #include <linux/rmap.h>
  44. #include <linux/module.h>
  45. #include <linux/delayacct.h>
  46. #include <linux/init.h>
  47. #include <linux/writeback.h>
  48. #include <asm/pgalloc.h>
  49. #include <asm/uaccess.h>
  50. #include <asm/tlb.h>
  51. #include <asm/tlbflush.h>
  52. #include <asm/pgtable.h>
  53. #include <linux/swapops.h>
  54. #include <linux/elf.h>
  55. #ifndef CONFIG_NEED_MULTIPLE_NODES
  56. /* use the per-pgdat data instead for discontigmem - mbligh */
  57. unsigned long max_mapnr;
  58. struct page *mem_map;
  59. EXPORT_SYMBOL(max_mapnr);
  60. EXPORT_SYMBOL(mem_map);
  61. #endif
  62. unsigned long num_physpages;
  63. /*
  64. * A number of key systems in x86 including ioremap() rely on the assumption
  65. * that high_memory defines the upper bound on direct map memory, then end
  66. * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
  67. * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
  68. * and ZONE_HIGHMEM.
  69. */
  70. void * high_memory;
  71. unsigned long vmalloc_earlyreserve;
  72. EXPORT_SYMBOL(num_physpages);
  73. EXPORT_SYMBOL(high_memory);
  74. EXPORT_SYMBOL(vmalloc_earlyreserve);
  75. int randomize_va_space __read_mostly = 1;
  76. static int __init disable_randmaps(char *s)
  77. {
  78. randomize_va_space = 0;
  79. return 1;
  80. }
  81. __setup("norandmaps", disable_randmaps);
  82. /*
  83. * If a p?d_bad entry is found while walking page tables, report
  84. * the error, before resetting entry to p?d_none. Usually (but
  85. * very seldom) called out from the p?d_none_or_clear_bad macros.
  86. */
  87. void pgd_clear_bad(pgd_t *pgd)
  88. {
  89. pgd_ERROR(*pgd);
  90. pgd_clear(pgd);
  91. }
  92. void pud_clear_bad(pud_t *pud)
  93. {
  94. pud_ERROR(*pud);
  95. pud_clear(pud);
  96. }
  97. void pmd_clear_bad(pmd_t *pmd)
  98. {
  99. pmd_ERROR(*pmd);
  100. pmd_clear(pmd);
  101. }
  102. /*
  103. * Note: this doesn't free the actual pages themselves. That
  104. * has been handled earlier when unmapping all the memory regions.
  105. */
  106. static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
  107. {
  108. struct page *page = pmd_page(*pmd);
  109. pmd_clear(pmd);
  110. pte_lock_deinit(page);
  111. pte_free_tlb(tlb, page);
  112. dec_zone_page_state(page, NR_PAGETABLE);
  113. tlb->mm->nr_ptes--;
  114. }
  115. static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  116. unsigned long addr, unsigned long end,
  117. unsigned long floor, unsigned long ceiling)
  118. {
  119. pmd_t *pmd;
  120. unsigned long next;
  121. unsigned long start;
  122. start = addr;
  123. pmd = pmd_offset(pud, addr);
  124. do {
  125. next = pmd_addr_end(addr, end);
  126. if (pmd_none_or_clear_bad(pmd))
  127. continue;
  128. free_pte_range(tlb, pmd);
  129. } while (pmd++, addr = next, addr != end);
  130. start &= PUD_MASK;
  131. if (start < floor)
  132. return;
  133. if (ceiling) {
  134. ceiling &= PUD_MASK;
  135. if (!ceiling)
  136. return;
  137. }
  138. if (end - 1 > ceiling - 1)
  139. return;
  140. pmd = pmd_offset(pud, start);
  141. pud_clear(pud);
  142. pmd_free_tlb(tlb, pmd);
  143. }
  144. static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  145. unsigned long addr, unsigned long end,
  146. unsigned long floor, unsigned long ceiling)
  147. {
  148. pud_t *pud;
  149. unsigned long next;
  150. unsigned long start;
  151. start = addr;
  152. pud = pud_offset(pgd, addr);
  153. do {
  154. next = pud_addr_end(addr, end);
  155. if (pud_none_or_clear_bad(pud))
  156. continue;
  157. free_pmd_range(tlb, pud, addr, next, floor, ceiling);
  158. } while (pud++, addr = next, addr != end);
  159. start &= PGDIR_MASK;
  160. if (start < floor)
  161. return;
  162. if (ceiling) {
  163. ceiling &= PGDIR_MASK;
  164. if (!ceiling)
  165. return;
  166. }
  167. if (end - 1 > ceiling - 1)
  168. return;
  169. pud = pud_offset(pgd, start);
  170. pgd_clear(pgd);
  171. pud_free_tlb(tlb, pud);
  172. }
  173. /*
  174. * This function frees user-level page tables of a process.
  175. *
  176. * Must be called with pagetable lock held.
  177. */
  178. void free_pgd_range(struct mmu_gather **tlb,
  179. unsigned long addr, unsigned long end,
  180. unsigned long floor, unsigned long ceiling)
  181. {
  182. pgd_t *pgd;
  183. unsigned long next;
  184. unsigned long start;
  185. /*
  186. * The next few lines have given us lots of grief...
  187. *
  188. * Why are we testing PMD* at this top level? Because often
  189. * there will be no work to do at all, and we'd prefer not to
  190. * go all the way down to the bottom just to discover that.
  191. *
  192. * Why all these "- 1"s? Because 0 represents both the bottom
  193. * of the address space and the top of it (using -1 for the
  194. * top wouldn't help much: the masks would do the wrong thing).
  195. * The rule is that addr 0 and floor 0 refer to the bottom of
  196. * the address space, but end 0 and ceiling 0 refer to the top
  197. * Comparisons need to use "end - 1" and "ceiling - 1" (though
  198. * that end 0 case should be mythical).
  199. *
  200. * Wherever addr is brought up or ceiling brought down, we must
  201. * be careful to reject "the opposite 0" before it confuses the
  202. * subsequent tests. But what about where end is brought down
  203. * by PMD_SIZE below? no, end can't go down to 0 there.
  204. *
  205. * Whereas we round start (addr) and ceiling down, by different
  206. * masks at different levels, in order to test whether a table
  207. * now has no other vmas using it, so can be freed, we don't
  208. * bother to round floor or end up - the tests don't need that.
  209. */
  210. addr &= PMD_MASK;
  211. if (addr < floor) {
  212. addr += PMD_SIZE;
  213. if (!addr)
  214. return;
  215. }
  216. if (ceiling) {
  217. ceiling &= PMD_MASK;
  218. if (!ceiling)
  219. return;
  220. }
  221. if (end - 1 > ceiling - 1)
  222. end -= PMD_SIZE;
  223. if (addr > end - 1)
  224. return;
  225. start = addr;
  226. pgd = pgd_offset((*tlb)->mm, addr);
  227. do {
  228. next = pgd_addr_end(addr, end);
  229. if (pgd_none_or_clear_bad(pgd))
  230. continue;
  231. free_pud_range(*tlb, pgd, addr, next, floor, ceiling);
  232. } while (pgd++, addr = next, addr != end);
  233. if (!(*tlb)->fullmm)
  234. flush_tlb_pgtables((*tlb)->mm, start, end);
  235. }
  236. void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
  237. unsigned long floor, unsigned long ceiling)
  238. {
  239. while (vma) {
  240. struct vm_area_struct *next = vma->vm_next;
  241. unsigned long addr = vma->vm_start;
  242. /*
  243. * Hide vma from rmap and vmtruncate before freeing pgtables
  244. */
  245. anon_vma_unlink(vma);
  246. unlink_file_vma(vma);
  247. if (is_vm_hugetlb_page(vma)) {
  248. hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
  249. floor, next? next->vm_start: ceiling);
  250. } else {
  251. /*
  252. * Optimization: gather nearby vmas into one call down
  253. */
  254. while (next && next->vm_start <= vma->vm_end + PMD_SIZE
  255. && !is_vm_hugetlb_page(next)) {
  256. vma = next;
  257. next = vma->vm_next;
  258. anon_vma_unlink(vma);
  259. unlink_file_vma(vma);
  260. }
  261. free_pgd_range(tlb, addr, vma->vm_end,
  262. floor, next? next->vm_start: ceiling);
  263. }
  264. vma = next;
  265. }
  266. }
  267. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
  268. {
  269. struct page *new = pte_alloc_one(mm, address);
  270. if (!new)
  271. return -ENOMEM;
  272. pte_lock_init(new);
  273. spin_lock(&mm->page_table_lock);
  274. if (pmd_present(*pmd)) { /* Another has populated it */
  275. pte_lock_deinit(new);
  276. pte_free(new);
  277. } else {
  278. mm->nr_ptes++;
  279. inc_zone_page_state(new, NR_PAGETABLE);
  280. pmd_populate(mm, pmd, new);
  281. }
  282. spin_unlock(&mm->page_table_lock);
  283. return 0;
  284. }
  285. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
  286. {
  287. pte_t *new = pte_alloc_one_kernel(&init_mm, address);
  288. if (!new)
  289. return -ENOMEM;
  290. spin_lock(&init_mm.page_table_lock);
  291. if (pmd_present(*pmd)) /* Another has populated it */
  292. pte_free_kernel(new);
  293. else
  294. pmd_populate_kernel(&init_mm, pmd, new);
  295. spin_unlock(&init_mm.page_table_lock);
  296. return 0;
  297. }
  298. static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
  299. {
  300. if (file_rss)
  301. add_mm_counter(mm, file_rss, file_rss);
  302. if (anon_rss)
  303. add_mm_counter(mm, anon_rss, anon_rss);
  304. }
  305. /*
  306. * This function is called to print an error when a bad pte
  307. * is found. For example, we might have a PFN-mapped pte in
  308. * a region that doesn't allow it.
  309. *
  310. * The calling function must still handle the error.
  311. */
  312. void print_bad_pte(struct vm_area_struct *vma, pte_t pte, unsigned long vaddr)
  313. {
  314. printk(KERN_ERR "Bad pte = %08llx, process = %s, "
  315. "vm_flags = %lx, vaddr = %lx\n",
  316. (long long)pte_val(pte),
  317. (vma->vm_mm == current->mm ? current->comm : "???"),
  318. vma->vm_flags, vaddr);
  319. dump_stack();
  320. }
  321. static inline int is_cow_mapping(unsigned int flags)
  322. {
  323. return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  324. }
  325. /*
  326. * This function gets the "struct page" associated with a pte.
  327. *
  328. * NOTE! Some mappings do not have "struct pages". A raw PFN mapping
  329. * will have each page table entry just pointing to a raw page frame
  330. * number, and as far as the VM layer is concerned, those do not have
  331. * pages associated with them - even if the PFN might point to memory
  332. * that otherwise is perfectly fine and has a "struct page".
  333. *
  334. * The way we recognize those mappings is through the rules set up
  335. * by "remap_pfn_range()": the vma will have the VM_PFNMAP bit set,
  336. * and the vm_pgoff will point to the first PFN mapped: thus every
  337. * page that is a raw mapping will always honor the rule
  338. *
  339. * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  340. *
  341. * and if that isn't true, the page has been COW'ed (in which case it
  342. * _does_ have a "struct page" associated with it even if it is in a
  343. * VM_PFNMAP range).
  344. */
  345. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
  346. {
  347. unsigned long pfn = pte_pfn(pte);
  348. if (unlikely(vma->vm_flags & VM_PFNMAP)) {
  349. unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT;
  350. if (pfn == vma->vm_pgoff + off)
  351. return NULL;
  352. if (!is_cow_mapping(vma->vm_flags))
  353. return NULL;
  354. }
  355. /*
  356. * Add some anal sanity checks for now. Eventually,
  357. * we should just do "return pfn_to_page(pfn)", but
  358. * in the meantime we check that we get a valid pfn,
  359. * and that the resulting page looks ok.
  360. */
  361. if (unlikely(!pfn_valid(pfn))) {
  362. print_bad_pte(vma, pte, addr);
  363. return NULL;
  364. }
  365. /*
  366. * NOTE! We still have PageReserved() pages in the page
  367. * tables.
  368. *
  369. * The PAGE_ZERO() pages and various VDSO mappings can
  370. * cause them to exist.
  371. */
  372. return pfn_to_page(pfn);
  373. }
  374. /*
  375. * copy one vm_area from one task to the other. Assumes the page tables
  376. * already present in the new task to be cleared in the whole range
  377. * covered by this vma.
  378. */
  379. static inline void
  380. copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  381. pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
  382. unsigned long addr, int *rss)
  383. {
  384. unsigned long vm_flags = vma->vm_flags;
  385. pte_t pte = *src_pte;
  386. struct page *page;
  387. /* pte contains position in swap or file, so copy. */
  388. if (unlikely(!pte_present(pte))) {
  389. if (!pte_file(pte)) {
  390. swp_entry_t entry = pte_to_swp_entry(pte);
  391. swap_duplicate(entry);
  392. /* make sure dst_mm is on swapoff's mmlist. */
  393. if (unlikely(list_empty(&dst_mm->mmlist))) {
  394. spin_lock(&mmlist_lock);
  395. if (list_empty(&dst_mm->mmlist))
  396. list_add(&dst_mm->mmlist,
  397. &src_mm->mmlist);
  398. spin_unlock(&mmlist_lock);
  399. }
  400. if (is_write_migration_entry(entry) &&
  401. is_cow_mapping(vm_flags)) {
  402. /*
  403. * COW mappings require pages in both parent
  404. * and child to be set to read.
  405. */
  406. make_migration_entry_read(&entry);
  407. pte = swp_entry_to_pte(entry);
  408. set_pte_at(src_mm, addr, src_pte, pte);
  409. }
  410. }
  411. goto out_set_pte;
  412. }
  413. /*
  414. * If it's a COW mapping, write protect it both
  415. * in the parent and the child
  416. */
  417. if (is_cow_mapping(vm_flags)) {
  418. ptep_set_wrprotect(src_mm, addr, src_pte);
  419. pte = pte_wrprotect(pte);
  420. }
  421. /*
  422. * If it's a shared mapping, mark it clean in
  423. * the child
  424. */
  425. if (vm_flags & VM_SHARED)
  426. pte = pte_mkclean(pte);
  427. pte = pte_mkold(pte);
  428. page = vm_normal_page(vma, addr, pte);
  429. if (page) {
  430. get_page(page);
  431. page_dup_rmap(page);
  432. rss[!!PageAnon(page)]++;
  433. }
  434. out_set_pte:
  435. set_pte_at(dst_mm, addr, dst_pte, pte);
  436. }
  437. static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  438. pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
  439. unsigned long addr, unsigned long end)
  440. {
  441. pte_t *src_pte, *dst_pte;
  442. spinlock_t *src_ptl, *dst_ptl;
  443. int progress = 0;
  444. int rss[2];
  445. again:
  446. rss[1] = rss[0] = 0;
  447. dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
  448. if (!dst_pte)
  449. return -ENOMEM;
  450. src_pte = pte_offset_map_nested(src_pmd, addr);
  451. src_ptl = pte_lockptr(src_mm, src_pmd);
  452. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  453. arch_enter_lazy_mmu_mode();
  454. do {
  455. /*
  456. * We are holding two locks at this point - either of them
  457. * could generate latencies in another task on another CPU.
  458. */
  459. if (progress >= 32) {
  460. progress = 0;
  461. if (need_resched() ||
  462. need_lockbreak(src_ptl) ||
  463. need_lockbreak(dst_ptl))
  464. break;
  465. }
  466. if (pte_none(*src_pte)) {
  467. progress++;
  468. continue;
  469. }
  470. copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
  471. progress += 8;
  472. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  473. arch_leave_lazy_mmu_mode();
  474. spin_unlock(src_ptl);
  475. pte_unmap_nested(src_pte - 1);
  476. add_mm_rss(dst_mm, rss[0], rss[1]);
  477. pte_unmap_unlock(dst_pte - 1, dst_ptl);
  478. cond_resched();
  479. if (addr != end)
  480. goto again;
  481. return 0;
  482. }
  483. static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  484. pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
  485. unsigned long addr, unsigned long end)
  486. {
  487. pmd_t *src_pmd, *dst_pmd;
  488. unsigned long next;
  489. dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
  490. if (!dst_pmd)
  491. return -ENOMEM;
  492. src_pmd = pmd_offset(src_pud, addr);
  493. do {
  494. next = pmd_addr_end(addr, end);
  495. if (pmd_none_or_clear_bad(src_pmd))
  496. continue;
  497. if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
  498. vma, addr, next))
  499. return -ENOMEM;
  500. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  501. return 0;
  502. }
  503. static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  504. pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
  505. unsigned long addr, unsigned long end)
  506. {
  507. pud_t *src_pud, *dst_pud;
  508. unsigned long next;
  509. dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
  510. if (!dst_pud)
  511. return -ENOMEM;
  512. src_pud = pud_offset(src_pgd, addr);
  513. do {
  514. next = pud_addr_end(addr, end);
  515. if (pud_none_or_clear_bad(src_pud))
  516. continue;
  517. if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
  518. vma, addr, next))
  519. return -ENOMEM;
  520. } while (dst_pud++, src_pud++, addr = next, addr != end);
  521. return 0;
  522. }
  523. int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  524. struct vm_area_struct *vma)
  525. {
  526. pgd_t *src_pgd, *dst_pgd;
  527. unsigned long next;
  528. unsigned long addr = vma->vm_start;
  529. unsigned long end = vma->vm_end;
  530. /*
  531. * Don't copy ptes where a page fault will fill them correctly.
  532. * Fork becomes much lighter when there are big shared or private
  533. * readonly mappings. The tradeoff is that copy_page_range is more
  534. * efficient than faulting.
  535. */
  536. if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) {
  537. if (!vma->anon_vma)
  538. return 0;
  539. }
  540. if (is_vm_hugetlb_page(vma))
  541. return copy_hugetlb_page_range(dst_mm, src_mm, vma);
  542. dst_pgd = pgd_offset(dst_mm, addr);
  543. src_pgd = pgd_offset(src_mm, addr);
  544. do {
  545. next = pgd_addr_end(addr, end);
  546. if (pgd_none_or_clear_bad(src_pgd))
  547. continue;
  548. if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
  549. vma, addr, next))
  550. return -ENOMEM;
  551. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  552. return 0;
  553. }
  554. static unsigned long zap_pte_range(struct mmu_gather *tlb,
  555. struct vm_area_struct *vma, pmd_t *pmd,
  556. unsigned long addr, unsigned long end,
  557. long *zap_work, struct zap_details *details)
  558. {
  559. struct mm_struct *mm = tlb->mm;
  560. pte_t *pte;
  561. spinlock_t *ptl;
  562. int file_rss = 0;
  563. int anon_rss = 0;
  564. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  565. arch_enter_lazy_mmu_mode();
  566. do {
  567. pte_t ptent = *pte;
  568. if (pte_none(ptent)) {
  569. (*zap_work)--;
  570. continue;
  571. }
  572. (*zap_work) -= PAGE_SIZE;
  573. if (pte_present(ptent)) {
  574. struct page *page;
  575. page = vm_normal_page(vma, addr, ptent);
  576. if (unlikely(details) && page) {
  577. /*
  578. * unmap_shared_mapping_pages() wants to
  579. * invalidate cache without truncating:
  580. * unmap shared but keep private pages.
  581. */
  582. if (details->check_mapping &&
  583. details->check_mapping != page->mapping)
  584. continue;
  585. /*
  586. * Each page->index must be checked when
  587. * invalidating or truncating nonlinear.
  588. */
  589. if (details->nonlinear_vma &&
  590. (page->index < details->first_index ||
  591. page->index > details->last_index))
  592. continue;
  593. }
  594. ptent = ptep_get_and_clear_full(mm, addr, pte,
  595. tlb->fullmm);
  596. tlb_remove_tlb_entry(tlb, pte, addr);
  597. if (unlikely(!page))
  598. continue;
  599. if (unlikely(details) && details->nonlinear_vma
  600. && linear_page_index(details->nonlinear_vma,
  601. addr) != page->index)
  602. set_pte_at(mm, addr, pte,
  603. pgoff_to_pte(page->index));
  604. if (PageAnon(page))
  605. anon_rss--;
  606. else {
  607. if (pte_dirty(ptent))
  608. set_page_dirty(page);
  609. if (pte_young(ptent))
  610. SetPageReferenced(page);
  611. file_rss--;
  612. }
  613. page_remove_rmap(page, vma);
  614. tlb_remove_page(tlb, page);
  615. continue;
  616. }
  617. /*
  618. * If details->check_mapping, we leave swap entries;
  619. * if details->nonlinear_vma, we leave file entries.
  620. */
  621. if (unlikely(details))
  622. continue;
  623. if (!pte_file(ptent))
  624. free_swap_and_cache(pte_to_swp_entry(ptent));
  625. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  626. } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
  627. add_mm_rss(mm, file_rss, anon_rss);
  628. arch_leave_lazy_mmu_mode();
  629. pte_unmap_unlock(pte - 1, ptl);
  630. return addr;
  631. }
  632. static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
  633. struct vm_area_struct *vma, pud_t *pud,
  634. unsigned long addr, unsigned long end,
  635. long *zap_work, struct zap_details *details)
  636. {
  637. pmd_t *pmd;
  638. unsigned long next;
  639. pmd = pmd_offset(pud, addr);
  640. do {
  641. next = pmd_addr_end(addr, end);
  642. if (pmd_none_or_clear_bad(pmd)) {
  643. (*zap_work)--;
  644. continue;
  645. }
  646. next = zap_pte_range(tlb, vma, pmd, addr, next,
  647. zap_work, details);
  648. } while (pmd++, addr = next, (addr != end && *zap_work > 0));
  649. return addr;
  650. }
  651. static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
  652. struct vm_area_struct *vma, pgd_t *pgd,
  653. unsigned long addr, unsigned long end,
  654. long *zap_work, struct zap_details *details)
  655. {
  656. pud_t *pud;
  657. unsigned long next;
  658. pud = pud_offset(pgd, addr);
  659. do {
  660. next = pud_addr_end(addr, end);
  661. if (pud_none_or_clear_bad(pud)) {
  662. (*zap_work)--;
  663. continue;
  664. }
  665. next = zap_pmd_range(tlb, vma, pud, addr, next,
  666. zap_work, details);
  667. } while (pud++, addr = next, (addr != end && *zap_work > 0));
  668. return addr;
  669. }
  670. static unsigned long unmap_page_range(struct mmu_gather *tlb,
  671. struct vm_area_struct *vma,
  672. unsigned long addr, unsigned long end,
  673. long *zap_work, struct zap_details *details)
  674. {
  675. pgd_t *pgd;
  676. unsigned long next;
  677. if (details && !details->check_mapping && !details->nonlinear_vma)
  678. details = NULL;
  679. BUG_ON(addr >= end);
  680. tlb_start_vma(tlb, vma);
  681. pgd = pgd_offset(vma->vm_mm, addr);
  682. do {
  683. next = pgd_addr_end(addr, end);
  684. if (pgd_none_or_clear_bad(pgd)) {
  685. (*zap_work)--;
  686. continue;
  687. }
  688. next = zap_pud_range(tlb, vma, pgd, addr, next,
  689. zap_work, details);
  690. } while (pgd++, addr = next, (addr != end && *zap_work > 0));
  691. tlb_end_vma(tlb, vma);
  692. return addr;
  693. }
  694. #ifdef CONFIG_PREEMPT
  695. # define ZAP_BLOCK_SIZE (8 * PAGE_SIZE)
  696. #else
  697. /* No preempt: go for improved straight-line efficiency */
  698. # define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE)
  699. #endif
  700. /**
  701. * unmap_vmas - unmap a range of memory covered by a list of vma's
  702. * @tlbp: address of the caller's struct mmu_gather
  703. * @vma: the starting vma
  704. * @start_addr: virtual address at which to start unmapping
  705. * @end_addr: virtual address at which to end unmapping
  706. * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
  707. * @details: details of nonlinear truncation or shared cache invalidation
  708. *
  709. * Returns the end address of the unmapping (restart addr if interrupted).
  710. *
  711. * Unmap all pages in the vma list.
  712. *
  713. * We aim to not hold locks for too long (for scheduling latency reasons).
  714. * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to
  715. * return the ending mmu_gather to the caller.
  716. *
  717. * Only addresses between `start' and `end' will be unmapped.
  718. *
  719. * The VMA list must be sorted in ascending virtual address order.
  720. *
  721. * unmap_vmas() assumes that the caller will flush the whole unmapped address
  722. * range after unmap_vmas() returns. So the only responsibility here is to
  723. * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  724. * drops the lock and schedules.
  725. */
  726. unsigned long unmap_vmas(struct mmu_gather **tlbp,
  727. struct vm_area_struct *vma, unsigned long start_addr,
  728. unsigned long end_addr, unsigned long *nr_accounted,
  729. struct zap_details *details)
  730. {
  731. long zap_work = ZAP_BLOCK_SIZE;
  732. unsigned long tlb_start = 0; /* For tlb_finish_mmu */
  733. int tlb_start_valid = 0;
  734. unsigned long start = start_addr;
  735. spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
  736. int fullmm = (*tlbp)->fullmm;
  737. for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
  738. unsigned long end;
  739. start = max(vma->vm_start, start_addr);
  740. if (start >= vma->vm_end)
  741. continue;
  742. end = min(vma->vm_end, end_addr);
  743. if (end <= vma->vm_start)
  744. continue;
  745. if (vma->vm_flags & VM_ACCOUNT)
  746. *nr_accounted += (end - start) >> PAGE_SHIFT;
  747. while (start != end) {
  748. if (!tlb_start_valid) {
  749. tlb_start = start;
  750. tlb_start_valid = 1;
  751. }
  752. if (unlikely(is_vm_hugetlb_page(vma))) {
  753. unmap_hugepage_range(vma, start, end);
  754. zap_work -= (end - start) /
  755. (HPAGE_SIZE / PAGE_SIZE);
  756. start = end;
  757. } else
  758. start = unmap_page_range(*tlbp, vma,
  759. start, end, &zap_work, details);
  760. if (zap_work > 0) {
  761. BUG_ON(start != end);
  762. break;
  763. }
  764. tlb_finish_mmu(*tlbp, tlb_start, start);
  765. if (need_resched() ||
  766. (i_mmap_lock && need_lockbreak(i_mmap_lock))) {
  767. if (i_mmap_lock) {
  768. *tlbp = NULL;
  769. goto out;
  770. }
  771. cond_resched();
  772. }
  773. *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm);
  774. tlb_start_valid = 0;
  775. zap_work = ZAP_BLOCK_SIZE;
  776. }
  777. }
  778. out:
  779. return start; /* which is now the end (or restart) address */
  780. }
  781. /**
  782. * zap_page_range - remove user pages in a given range
  783. * @vma: vm_area_struct holding the applicable pages
  784. * @address: starting address of pages to zap
  785. * @size: number of bytes to zap
  786. * @details: details of nonlinear truncation or shared cache invalidation
  787. */
  788. unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
  789. unsigned long size, struct zap_details *details)
  790. {
  791. struct mm_struct *mm = vma->vm_mm;
  792. struct mmu_gather *tlb;
  793. unsigned long end = address + size;
  794. unsigned long nr_accounted = 0;
  795. lru_add_drain();
  796. tlb = tlb_gather_mmu(mm, 0);
  797. update_hiwater_rss(mm);
  798. end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
  799. if (tlb)
  800. tlb_finish_mmu(tlb, address, end);
  801. return end;
  802. }
  803. /*
  804. * Do a quick page-table lookup for a single page.
  805. */
  806. struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
  807. unsigned int flags)
  808. {
  809. pgd_t *pgd;
  810. pud_t *pud;
  811. pmd_t *pmd;
  812. pte_t *ptep, pte;
  813. spinlock_t *ptl;
  814. struct page *page;
  815. struct mm_struct *mm = vma->vm_mm;
  816. page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
  817. if (!IS_ERR(page)) {
  818. BUG_ON(flags & FOLL_GET);
  819. goto out;
  820. }
  821. page = NULL;
  822. pgd = pgd_offset(mm, address);
  823. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  824. goto no_page_table;
  825. pud = pud_offset(pgd, address);
  826. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  827. goto no_page_table;
  828. pmd = pmd_offset(pud, address);
  829. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  830. goto no_page_table;
  831. if (pmd_huge(*pmd)) {
  832. BUG_ON(flags & FOLL_GET);
  833. page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
  834. goto out;
  835. }
  836. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  837. if (!ptep)
  838. goto out;
  839. pte = *ptep;
  840. if (!pte_present(pte))
  841. goto unlock;
  842. if ((flags & FOLL_WRITE) && !pte_write(pte))
  843. goto unlock;
  844. page = vm_normal_page(vma, address, pte);
  845. if (unlikely(!page))
  846. goto unlock;
  847. if (flags & FOLL_GET)
  848. get_page(page);
  849. if (flags & FOLL_TOUCH) {
  850. if ((flags & FOLL_WRITE) &&
  851. !pte_dirty(pte) && !PageDirty(page))
  852. set_page_dirty(page);
  853. mark_page_accessed(page);
  854. }
  855. unlock:
  856. pte_unmap_unlock(ptep, ptl);
  857. out:
  858. return page;
  859. no_page_table:
  860. /*
  861. * When core dumping an enormous anonymous area that nobody
  862. * has touched so far, we don't want to allocate page tables.
  863. */
  864. if (flags & FOLL_ANON) {
  865. page = ZERO_PAGE(address);
  866. if (flags & FOLL_GET)
  867. get_page(page);
  868. BUG_ON(flags & FOLL_WRITE);
  869. }
  870. return page;
  871. }
  872. int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  873. unsigned long start, int len, int write, int force,
  874. struct page **pages, struct vm_area_struct **vmas)
  875. {
  876. int i;
  877. unsigned int vm_flags;
  878. /*
  879. * Require read or write permissions.
  880. * If 'force' is set, we only require the "MAY" flags.
  881. */
  882. vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
  883. vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
  884. i = 0;
  885. do {
  886. struct vm_area_struct *vma;
  887. unsigned int foll_flags;
  888. vma = find_extend_vma(mm, start);
  889. if (!vma && in_gate_area(tsk, start)) {
  890. unsigned long pg = start & PAGE_MASK;
  891. struct vm_area_struct *gate_vma = get_gate_vma(tsk);
  892. pgd_t *pgd;
  893. pud_t *pud;
  894. pmd_t *pmd;
  895. pte_t *pte;
  896. if (write) /* user gate pages are read-only */
  897. return i ? : -EFAULT;
  898. if (pg > TASK_SIZE)
  899. pgd = pgd_offset_k(pg);
  900. else
  901. pgd = pgd_offset_gate(mm, pg);
  902. BUG_ON(pgd_none(*pgd));
  903. pud = pud_offset(pgd, pg);
  904. BUG_ON(pud_none(*pud));
  905. pmd = pmd_offset(pud, pg);
  906. if (pmd_none(*pmd))
  907. return i ? : -EFAULT;
  908. pte = pte_offset_map(pmd, pg);
  909. if (pte_none(*pte)) {
  910. pte_unmap(pte);
  911. return i ? : -EFAULT;
  912. }
  913. if (pages) {
  914. struct page *page = vm_normal_page(gate_vma, start, *pte);
  915. pages[i] = page;
  916. if (page)
  917. get_page(page);
  918. }
  919. pte_unmap(pte);
  920. if (vmas)
  921. vmas[i] = gate_vma;
  922. i++;
  923. start += PAGE_SIZE;
  924. len--;
  925. continue;
  926. }
  927. if (!vma || (vma->vm_flags & (VM_IO | VM_PFNMAP))
  928. || !(vm_flags & vma->vm_flags))
  929. return i ? : -EFAULT;
  930. if (is_vm_hugetlb_page(vma)) {
  931. i = follow_hugetlb_page(mm, vma, pages, vmas,
  932. &start, &len, i);
  933. continue;
  934. }
  935. foll_flags = FOLL_TOUCH;
  936. if (pages)
  937. foll_flags |= FOLL_GET;
  938. if (!write && !(vma->vm_flags & VM_LOCKED) &&
  939. (!vma->vm_ops || !vma->vm_ops->nopage))
  940. foll_flags |= FOLL_ANON;
  941. do {
  942. struct page *page;
  943. if (write)
  944. foll_flags |= FOLL_WRITE;
  945. cond_resched();
  946. while (!(page = follow_page(vma, start, foll_flags))) {
  947. int ret;
  948. ret = __handle_mm_fault(mm, vma, start,
  949. foll_flags & FOLL_WRITE);
  950. /*
  951. * The VM_FAULT_WRITE bit tells us that do_wp_page has
  952. * broken COW when necessary, even if maybe_mkwrite
  953. * decided not to set pte_write. We can thus safely do
  954. * subsequent page lookups as if they were reads.
  955. */
  956. if (ret & VM_FAULT_WRITE)
  957. foll_flags &= ~FOLL_WRITE;
  958. switch (ret & ~VM_FAULT_WRITE) {
  959. case VM_FAULT_MINOR:
  960. tsk->min_flt++;
  961. break;
  962. case VM_FAULT_MAJOR:
  963. tsk->maj_flt++;
  964. break;
  965. case VM_FAULT_SIGBUS:
  966. return i ? i : -EFAULT;
  967. case VM_FAULT_OOM:
  968. return i ? i : -ENOMEM;
  969. default:
  970. BUG();
  971. }
  972. cond_resched();
  973. }
  974. if (pages) {
  975. pages[i] = page;
  976. flush_anon_page(vma, page, start);
  977. flush_dcache_page(page);
  978. }
  979. if (vmas)
  980. vmas[i] = vma;
  981. i++;
  982. start += PAGE_SIZE;
  983. len--;
  984. } while (len && start < vma->vm_end);
  985. } while (len);
  986. return i;
  987. }
  988. EXPORT_SYMBOL(get_user_pages);
  989. static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  990. unsigned long addr, unsigned long end, pgprot_t prot)
  991. {
  992. pte_t *pte;
  993. spinlock_t *ptl;
  994. int err = 0;
  995. pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  996. if (!pte)
  997. return -EAGAIN;
  998. arch_enter_lazy_mmu_mode();
  999. do {
  1000. struct page *page = ZERO_PAGE(addr);
  1001. pte_t zero_pte = pte_wrprotect(mk_pte(page, prot));
  1002. if (unlikely(!pte_none(*pte))) {
  1003. err = -EEXIST;
  1004. pte++;
  1005. break;
  1006. }
  1007. page_cache_get(page);
  1008. page_add_file_rmap(page);
  1009. inc_mm_counter(mm, file_rss);
  1010. set_pte_at(mm, addr, pte, zero_pte);
  1011. } while (pte++, addr += PAGE_SIZE, addr != end);
  1012. arch_leave_lazy_mmu_mode();
  1013. pte_unmap_unlock(pte - 1, ptl);
  1014. return err;
  1015. }
  1016. static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud,
  1017. unsigned long addr, unsigned long end, pgprot_t prot)
  1018. {
  1019. pmd_t *pmd;
  1020. unsigned long next;
  1021. int err;
  1022. pmd = pmd_alloc(mm, pud, addr);
  1023. if (!pmd)
  1024. return -EAGAIN;
  1025. do {
  1026. next = pmd_addr_end(addr, end);
  1027. err = zeromap_pte_range(mm, pmd, addr, next, prot);
  1028. if (err)
  1029. break;
  1030. } while (pmd++, addr = next, addr != end);
  1031. return err;
  1032. }
  1033. static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1034. unsigned long addr, unsigned long end, pgprot_t prot)
  1035. {
  1036. pud_t *pud;
  1037. unsigned long next;
  1038. int err;
  1039. pud = pud_alloc(mm, pgd, addr);
  1040. if (!pud)
  1041. return -EAGAIN;
  1042. do {
  1043. next = pud_addr_end(addr, end);
  1044. err = zeromap_pmd_range(mm, pud, addr, next, prot);
  1045. if (err)
  1046. break;
  1047. } while (pud++, addr = next, addr != end);
  1048. return err;
  1049. }
  1050. int zeromap_page_range(struct vm_area_struct *vma,
  1051. unsigned long addr, unsigned long size, pgprot_t prot)
  1052. {
  1053. pgd_t *pgd;
  1054. unsigned long next;
  1055. unsigned long end = addr + size;
  1056. struct mm_struct *mm = vma->vm_mm;
  1057. int err;
  1058. BUG_ON(addr >= end);
  1059. pgd = pgd_offset(mm, addr);
  1060. flush_cache_range(vma, addr, end);
  1061. do {
  1062. next = pgd_addr_end(addr, end);
  1063. err = zeromap_pud_range(mm, pgd, addr, next, prot);
  1064. if (err)
  1065. break;
  1066. } while (pgd++, addr = next, addr != end);
  1067. return err;
  1068. }
  1069. pte_t * fastcall get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl)
  1070. {
  1071. pgd_t * pgd = pgd_offset(mm, addr);
  1072. pud_t * pud = pud_alloc(mm, pgd, addr);
  1073. if (pud) {
  1074. pmd_t * pmd = pmd_alloc(mm, pud, addr);
  1075. if (pmd)
  1076. return pte_alloc_map_lock(mm, pmd, addr, ptl);
  1077. }
  1078. return NULL;
  1079. }
  1080. /*
  1081. * This is the old fallback for page remapping.
  1082. *
  1083. * For historical reasons, it only allows reserved pages. Only
  1084. * old drivers should use this, and they needed to mark their
  1085. * pages reserved for the old functions anyway.
  1086. */
  1087. static int insert_page(struct mm_struct *mm, unsigned long addr, struct page *page, pgprot_t prot)
  1088. {
  1089. int retval;
  1090. pte_t *pte;
  1091. spinlock_t *ptl;
  1092. retval = -EINVAL;
  1093. if (PageAnon(page))
  1094. goto out;
  1095. retval = -ENOMEM;
  1096. flush_dcache_page(page);
  1097. pte = get_locked_pte(mm, addr, &ptl);
  1098. if (!pte)
  1099. goto out;
  1100. retval = -EBUSY;
  1101. if (!pte_none(*pte))
  1102. goto out_unlock;
  1103. /* Ok, finally just insert the thing.. */
  1104. get_page(page);
  1105. inc_mm_counter(mm, file_rss);
  1106. page_add_file_rmap(page);
  1107. set_pte_at(mm, addr, pte, mk_pte(page, prot));
  1108. retval = 0;
  1109. out_unlock:
  1110. pte_unmap_unlock(pte, ptl);
  1111. out:
  1112. return retval;
  1113. }
  1114. /**
  1115. * vm_insert_page - insert single page into user vma
  1116. * @vma: user vma to map to
  1117. * @addr: target user address of this page
  1118. * @page: source kernel page
  1119. *
  1120. * This allows drivers to insert individual pages they've allocated
  1121. * into a user vma.
  1122. *
  1123. * The page has to be a nice clean _individual_ kernel allocation.
  1124. * If you allocate a compound page, you need to have marked it as
  1125. * such (__GFP_COMP), or manually just split the page up yourself
  1126. * (see split_page()).
  1127. *
  1128. * NOTE! Traditionally this was done with "remap_pfn_range()" which
  1129. * took an arbitrary page protection parameter. This doesn't allow
  1130. * that. Your vma protection will have to be set up correctly, which
  1131. * means that if you want a shared writable mapping, you'd better
  1132. * ask for a shared writable mapping!
  1133. *
  1134. * The page does not need to be reserved.
  1135. */
  1136. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page)
  1137. {
  1138. if (addr < vma->vm_start || addr >= vma->vm_end)
  1139. return -EFAULT;
  1140. if (!page_count(page))
  1141. return -EINVAL;
  1142. vma->vm_flags |= VM_INSERTPAGE;
  1143. return insert_page(vma->vm_mm, addr, page, vma->vm_page_prot);
  1144. }
  1145. EXPORT_SYMBOL(vm_insert_page);
  1146. /**
  1147. * vm_insert_pfn - insert single pfn into user vma
  1148. * @vma: user vma to map to
  1149. * @addr: target user address of this page
  1150. * @pfn: source kernel pfn
  1151. *
  1152. * Similar to vm_inert_page, this allows drivers to insert individual pages
  1153. * they've allocated into a user vma. Same comments apply.
  1154. *
  1155. * This function should only be called from a vm_ops->fault handler, and
  1156. * in that case the handler should return NULL.
  1157. */
  1158. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1159. unsigned long pfn)
  1160. {
  1161. struct mm_struct *mm = vma->vm_mm;
  1162. int retval;
  1163. pte_t *pte, entry;
  1164. spinlock_t *ptl;
  1165. BUG_ON(!(vma->vm_flags & VM_PFNMAP));
  1166. BUG_ON(is_cow_mapping(vma->vm_flags));
  1167. retval = -ENOMEM;
  1168. pte = get_locked_pte(mm, addr, &ptl);
  1169. if (!pte)
  1170. goto out;
  1171. retval = -EBUSY;
  1172. if (!pte_none(*pte))
  1173. goto out_unlock;
  1174. /* Ok, finally just insert the thing.. */
  1175. entry = pfn_pte(pfn, vma->vm_page_prot);
  1176. set_pte_at(mm, addr, pte, entry);
  1177. update_mmu_cache(vma, addr, entry);
  1178. retval = 0;
  1179. out_unlock:
  1180. pte_unmap_unlock(pte, ptl);
  1181. out:
  1182. return retval;
  1183. }
  1184. EXPORT_SYMBOL(vm_insert_pfn);
  1185. /*
  1186. * maps a range of physical memory into the requested pages. the old
  1187. * mappings are removed. any references to nonexistent pages results
  1188. * in null mappings (currently treated as "copy-on-access")
  1189. */
  1190. static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1191. unsigned long addr, unsigned long end,
  1192. unsigned long pfn, pgprot_t prot)
  1193. {
  1194. pte_t *pte;
  1195. spinlock_t *ptl;
  1196. pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1197. if (!pte)
  1198. return -ENOMEM;
  1199. arch_enter_lazy_mmu_mode();
  1200. do {
  1201. BUG_ON(!pte_none(*pte));
  1202. set_pte_at(mm, addr, pte, pfn_pte(pfn, prot));
  1203. pfn++;
  1204. } while (pte++, addr += PAGE_SIZE, addr != end);
  1205. arch_leave_lazy_mmu_mode();
  1206. pte_unmap_unlock(pte - 1, ptl);
  1207. return 0;
  1208. }
  1209. static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
  1210. unsigned long addr, unsigned long end,
  1211. unsigned long pfn, pgprot_t prot)
  1212. {
  1213. pmd_t *pmd;
  1214. unsigned long next;
  1215. pfn -= addr >> PAGE_SHIFT;
  1216. pmd = pmd_alloc(mm, pud, addr);
  1217. if (!pmd)
  1218. return -ENOMEM;
  1219. do {
  1220. next = pmd_addr_end(addr, end);
  1221. if (remap_pte_range(mm, pmd, addr, next,
  1222. pfn + (addr >> PAGE_SHIFT), prot))
  1223. return -ENOMEM;
  1224. } while (pmd++, addr = next, addr != end);
  1225. return 0;
  1226. }
  1227. static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1228. unsigned long addr, unsigned long end,
  1229. unsigned long pfn, pgprot_t prot)
  1230. {
  1231. pud_t *pud;
  1232. unsigned long next;
  1233. pfn -= addr >> PAGE_SHIFT;
  1234. pud = pud_alloc(mm, pgd, addr);
  1235. if (!pud)
  1236. return -ENOMEM;
  1237. do {
  1238. next = pud_addr_end(addr, end);
  1239. if (remap_pmd_range(mm, pud, addr, next,
  1240. pfn + (addr >> PAGE_SHIFT), prot))
  1241. return -ENOMEM;
  1242. } while (pud++, addr = next, addr != end);
  1243. return 0;
  1244. }
  1245. /**
  1246. * remap_pfn_range - remap kernel memory to userspace
  1247. * @vma: user vma to map to
  1248. * @addr: target user address to start at
  1249. * @pfn: physical address of kernel memory
  1250. * @size: size of map area
  1251. * @prot: page protection flags for this mapping
  1252. *
  1253. * Note: this is only safe if the mm semaphore is held when called.
  1254. */
  1255. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  1256. unsigned long pfn, unsigned long size, pgprot_t prot)
  1257. {
  1258. pgd_t *pgd;
  1259. unsigned long next;
  1260. unsigned long end = addr + PAGE_ALIGN(size);
  1261. struct mm_struct *mm = vma->vm_mm;
  1262. int err;
  1263. /*
  1264. * Physically remapped pages are special. Tell the
  1265. * rest of the world about it:
  1266. * VM_IO tells people not to look at these pages
  1267. * (accesses can have side effects).
  1268. * VM_RESERVED is specified all over the place, because
  1269. * in 2.4 it kept swapout's vma scan off this vma; but
  1270. * in 2.6 the LRU scan won't even find its pages, so this
  1271. * flag means no more than count its pages in reserved_vm,
  1272. * and omit it from core dump, even when VM_IO turned off.
  1273. * VM_PFNMAP tells the core MM that the base pages are just
  1274. * raw PFN mappings, and do not have a "struct page" associated
  1275. * with them.
  1276. *
  1277. * There's a horrible special case to handle copy-on-write
  1278. * behaviour that some programs depend on. We mark the "original"
  1279. * un-COW'ed pages by matching them up with "vma->vm_pgoff".
  1280. */
  1281. if (is_cow_mapping(vma->vm_flags)) {
  1282. if (addr != vma->vm_start || end != vma->vm_end)
  1283. return -EINVAL;
  1284. vma->vm_pgoff = pfn;
  1285. }
  1286. vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
  1287. BUG_ON(addr >= end);
  1288. pfn -= addr >> PAGE_SHIFT;
  1289. pgd = pgd_offset(mm, addr);
  1290. flush_cache_range(vma, addr, end);
  1291. do {
  1292. next = pgd_addr_end(addr, end);
  1293. err = remap_pud_range(mm, pgd, addr, next,
  1294. pfn + (addr >> PAGE_SHIFT), prot);
  1295. if (err)
  1296. break;
  1297. } while (pgd++, addr = next, addr != end);
  1298. return err;
  1299. }
  1300. EXPORT_SYMBOL(remap_pfn_range);
  1301. /*
  1302. * handle_pte_fault chooses page fault handler according to an entry
  1303. * which was read non-atomically. Before making any commitment, on
  1304. * those architectures or configurations (e.g. i386 with PAE) which
  1305. * might give a mix of unmatched parts, do_swap_page and do_file_page
  1306. * must check under lock before unmapping the pte and proceeding
  1307. * (but do_wp_page is only called after already making such a check;
  1308. * and do_anonymous_page and do_no_page can safely check later on).
  1309. */
  1310. static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
  1311. pte_t *page_table, pte_t orig_pte)
  1312. {
  1313. int same = 1;
  1314. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
  1315. if (sizeof(pte_t) > sizeof(unsigned long)) {
  1316. spinlock_t *ptl = pte_lockptr(mm, pmd);
  1317. spin_lock(ptl);
  1318. same = pte_same(*page_table, orig_pte);
  1319. spin_unlock(ptl);
  1320. }
  1321. #endif
  1322. pte_unmap(page_table);
  1323. return same;
  1324. }
  1325. /*
  1326. * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
  1327. * servicing faults for write access. In the normal case, do always want
  1328. * pte_mkwrite. But get_user_pages can cause write faults for mappings
  1329. * that do not have writing enabled, when used by access_process_vm.
  1330. */
  1331. static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  1332. {
  1333. if (likely(vma->vm_flags & VM_WRITE))
  1334. pte = pte_mkwrite(pte);
  1335. return pte;
  1336. }
  1337. static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
  1338. {
  1339. /*
  1340. * If the source page was a PFN mapping, we don't have
  1341. * a "struct page" for it. We do a best-effort copy by
  1342. * just copying from the original user address. If that
  1343. * fails, we just zero-fill it. Live with it.
  1344. */
  1345. if (unlikely(!src)) {
  1346. void *kaddr = kmap_atomic(dst, KM_USER0);
  1347. void __user *uaddr = (void __user *)(va & PAGE_MASK);
  1348. /*
  1349. * This really shouldn't fail, because the page is there
  1350. * in the page tables. But it might just be unreadable,
  1351. * in which case we just give up and fill the result with
  1352. * zeroes.
  1353. */
  1354. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
  1355. memset(kaddr, 0, PAGE_SIZE);
  1356. kunmap_atomic(kaddr, KM_USER0);
  1357. flush_dcache_page(dst);
  1358. return;
  1359. }
  1360. copy_user_highpage(dst, src, va, vma);
  1361. }
  1362. /*
  1363. * This routine handles present pages, when users try to write
  1364. * to a shared page. It is done by copying the page to a new address
  1365. * and decrementing the shared-page counter for the old page.
  1366. *
  1367. * Note that this routine assumes that the protection checks have been
  1368. * done by the caller (the low-level page fault routine in most cases).
  1369. * Thus we can safely just mark it writable once we've done any necessary
  1370. * COW.
  1371. *
  1372. * We also mark the page dirty at this point even though the page will
  1373. * change only once the write actually happens. This avoids a few races,
  1374. * and potentially makes it more efficient.
  1375. *
  1376. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1377. * but allow concurrent faults), with pte both mapped and locked.
  1378. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1379. */
  1380. static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1381. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1382. spinlock_t *ptl, pte_t orig_pte)
  1383. {
  1384. struct page *old_page, *new_page;
  1385. pte_t entry;
  1386. int reuse = 0, ret = VM_FAULT_MINOR;
  1387. struct page *dirty_page = NULL;
  1388. old_page = vm_normal_page(vma, address, orig_pte);
  1389. if (!old_page)
  1390. goto gotten;
  1391. /*
  1392. * Take out anonymous pages first, anonymous shared vmas are
  1393. * not dirty accountable.
  1394. */
  1395. if (PageAnon(old_page)) {
  1396. if (!TestSetPageLocked(old_page)) {
  1397. reuse = can_share_swap_page(old_page);
  1398. unlock_page(old_page);
  1399. }
  1400. } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  1401. (VM_WRITE|VM_SHARED))) {
  1402. /*
  1403. * Only catch write-faults on shared writable pages,
  1404. * read-only shared pages can get COWed by
  1405. * get_user_pages(.write=1, .force=1).
  1406. */
  1407. if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
  1408. /*
  1409. * Notify the address space that the page is about to
  1410. * become writable so that it can prohibit this or wait
  1411. * for the page to get into an appropriate state.
  1412. *
  1413. * We do this without the lock held, so that it can
  1414. * sleep if it needs to.
  1415. */
  1416. page_cache_get(old_page);
  1417. pte_unmap_unlock(page_table, ptl);
  1418. if (vma->vm_ops->page_mkwrite(vma, old_page) < 0)
  1419. goto unwritable_page;
  1420. /*
  1421. * Since we dropped the lock we need to revalidate
  1422. * the PTE as someone else may have changed it. If
  1423. * they did, we just return, as we can count on the
  1424. * MMU to tell us if they didn't also make it writable.
  1425. */
  1426. page_table = pte_offset_map_lock(mm, pmd, address,
  1427. &ptl);
  1428. page_cache_release(old_page);
  1429. if (!pte_same(*page_table, orig_pte))
  1430. goto unlock;
  1431. }
  1432. dirty_page = old_page;
  1433. get_page(dirty_page);
  1434. reuse = 1;
  1435. }
  1436. if (reuse) {
  1437. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1438. entry = pte_mkyoung(orig_pte);
  1439. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1440. ptep_set_access_flags(vma, address, page_table, entry, 1);
  1441. update_mmu_cache(vma, address, entry);
  1442. lazy_mmu_prot_update(entry);
  1443. ret |= VM_FAULT_WRITE;
  1444. goto unlock;
  1445. }
  1446. /*
  1447. * Ok, we need to copy. Oh, well..
  1448. */
  1449. page_cache_get(old_page);
  1450. gotten:
  1451. pte_unmap_unlock(page_table, ptl);
  1452. if (unlikely(anon_vma_prepare(vma)))
  1453. goto oom;
  1454. if (old_page == ZERO_PAGE(address)) {
  1455. new_page = alloc_zeroed_user_highpage(vma, address);
  1456. if (!new_page)
  1457. goto oom;
  1458. } else {
  1459. new_page = alloc_page_vma(GFP_HIGHUSER, vma, address);
  1460. if (!new_page)
  1461. goto oom;
  1462. cow_user_page(new_page, old_page, address, vma);
  1463. }
  1464. /*
  1465. * Re-check the pte - we dropped the lock
  1466. */
  1467. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1468. if (likely(pte_same(*page_table, orig_pte))) {
  1469. if (old_page) {
  1470. page_remove_rmap(old_page, vma);
  1471. if (!PageAnon(old_page)) {
  1472. dec_mm_counter(mm, file_rss);
  1473. inc_mm_counter(mm, anon_rss);
  1474. }
  1475. } else
  1476. inc_mm_counter(mm, anon_rss);
  1477. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1478. entry = mk_pte(new_page, vma->vm_page_prot);
  1479. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1480. lazy_mmu_prot_update(entry);
  1481. /*
  1482. * Clear the pte entry and flush it first, before updating the
  1483. * pte with the new entry. This will avoid a race condition
  1484. * seen in the presence of one thread doing SMC and another
  1485. * thread doing COW.
  1486. */
  1487. ptep_clear_flush(vma, address, page_table);
  1488. set_pte_at(mm, address, page_table, entry);
  1489. update_mmu_cache(vma, address, entry);
  1490. lru_cache_add_active(new_page);
  1491. page_add_new_anon_rmap(new_page, vma, address);
  1492. /* Free the old page.. */
  1493. new_page = old_page;
  1494. ret |= VM_FAULT_WRITE;
  1495. }
  1496. if (new_page)
  1497. page_cache_release(new_page);
  1498. if (old_page)
  1499. page_cache_release(old_page);
  1500. unlock:
  1501. pte_unmap_unlock(page_table, ptl);
  1502. if (dirty_page) {
  1503. set_page_dirty_balance(dirty_page);
  1504. put_page(dirty_page);
  1505. }
  1506. return ret;
  1507. oom:
  1508. if (old_page)
  1509. page_cache_release(old_page);
  1510. return VM_FAULT_OOM;
  1511. unwritable_page:
  1512. page_cache_release(old_page);
  1513. return VM_FAULT_SIGBUS;
  1514. }
  1515. /*
  1516. * Helper functions for unmap_mapping_range().
  1517. *
  1518. * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __
  1519. *
  1520. * We have to restart searching the prio_tree whenever we drop the lock,
  1521. * since the iterator is only valid while the lock is held, and anyway
  1522. * a later vma might be split and reinserted earlier while lock dropped.
  1523. *
  1524. * The list of nonlinear vmas could be handled more efficiently, using
  1525. * a placeholder, but handle it in the same way until a need is shown.
  1526. * It is important to search the prio_tree before nonlinear list: a vma
  1527. * may become nonlinear and be shifted from prio_tree to nonlinear list
  1528. * while the lock is dropped; but never shifted from list to prio_tree.
  1529. *
  1530. * In order to make forward progress despite restarting the search,
  1531. * vm_truncate_count is used to mark a vma as now dealt with, so we can
  1532. * quickly skip it next time around. Since the prio_tree search only
  1533. * shows us those vmas affected by unmapping the range in question, we
  1534. * can't efficiently keep all vmas in step with mapping->truncate_count:
  1535. * so instead reset them all whenever it wraps back to 0 (then go to 1).
  1536. * mapping->truncate_count and vma->vm_truncate_count are protected by
  1537. * i_mmap_lock.
  1538. *
  1539. * In order to make forward progress despite repeatedly restarting some
  1540. * large vma, note the restart_addr from unmap_vmas when it breaks out:
  1541. * and restart from that address when we reach that vma again. It might
  1542. * have been split or merged, shrunk or extended, but never shifted: so
  1543. * restart_addr remains valid so long as it remains in the vma's range.
  1544. * unmap_mapping_range forces truncate_count to leap over page-aligned
  1545. * values so we can save vma's restart_addr in its truncate_count field.
  1546. */
  1547. #define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK))
  1548. static void reset_vma_truncate_counts(struct address_space *mapping)
  1549. {
  1550. struct vm_area_struct *vma;
  1551. struct prio_tree_iter iter;
  1552. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
  1553. vma->vm_truncate_count = 0;
  1554. list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
  1555. vma->vm_truncate_count = 0;
  1556. }
  1557. static int unmap_mapping_range_vma(struct vm_area_struct *vma,
  1558. unsigned long start_addr, unsigned long end_addr,
  1559. struct zap_details *details)
  1560. {
  1561. unsigned long restart_addr;
  1562. int need_break;
  1563. again:
  1564. restart_addr = vma->vm_truncate_count;
  1565. if (is_restart_addr(restart_addr) && start_addr < restart_addr) {
  1566. start_addr = restart_addr;
  1567. if (start_addr >= end_addr) {
  1568. /* Top of vma has been split off since last time */
  1569. vma->vm_truncate_count = details->truncate_count;
  1570. return 0;
  1571. }
  1572. }
  1573. restart_addr = zap_page_range(vma, start_addr,
  1574. end_addr - start_addr, details);
  1575. need_break = need_resched() ||
  1576. need_lockbreak(details->i_mmap_lock);
  1577. if (restart_addr >= end_addr) {
  1578. /* We have now completed this vma: mark it so */
  1579. vma->vm_truncate_count = details->truncate_count;
  1580. if (!need_break)
  1581. return 0;
  1582. } else {
  1583. /* Note restart_addr in vma's truncate_count field */
  1584. vma->vm_truncate_count = restart_addr;
  1585. if (!need_break)
  1586. goto again;
  1587. }
  1588. spin_unlock(details->i_mmap_lock);
  1589. cond_resched();
  1590. spin_lock(details->i_mmap_lock);
  1591. return -EINTR;
  1592. }
  1593. static inline void unmap_mapping_range_tree(struct prio_tree_root *root,
  1594. struct zap_details *details)
  1595. {
  1596. struct vm_area_struct *vma;
  1597. struct prio_tree_iter iter;
  1598. pgoff_t vba, vea, zba, zea;
  1599. restart:
  1600. vma_prio_tree_foreach(vma, &iter, root,
  1601. details->first_index, details->last_index) {
  1602. /* Skip quickly over those we have already dealt with */
  1603. if (vma->vm_truncate_count == details->truncate_count)
  1604. continue;
  1605. vba = vma->vm_pgoff;
  1606. vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1;
  1607. /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */
  1608. zba = details->first_index;
  1609. if (zba < vba)
  1610. zba = vba;
  1611. zea = details->last_index;
  1612. if (zea > vea)
  1613. zea = vea;
  1614. if (unmap_mapping_range_vma(vma,
  1615. ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
  1616. ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
  1617. details) < 0)
  1618. goto restart;
  1619. }
  1620. }
  1621. static inline void unmap_mapping_range_list(struct list_head *head,
  1622. struct zap_details *details)
  1623. {
  1624. struct vm_area_struct *vma;
  1625. /*
  1626. * In nonlinear VMAs there is no correspondence between virtual address
  1627. * offset and file offset. So we must perform an exhaustive search
  1628. * across *all* the pages in each nonlinear VMA, not just the pages
  1629. * whose virtual address lies outside the file truncation point.
  1630. */
  1631. restart:
  1632. list_for_each_entry(vma, head, shared.vm_set.list) {
  1633. /* Skip quickly over those we have already dealt with */
  1634. if (vma->vm_truncate_count == details->truncate_count)
  1635. continue;
  1636. details->nonlinear_vma = vma;
  1637. if (unmap_mapping_range_vma(vma, vma->vm_start,
  1638. vma->vm_end, details) < 0)
  1639. goto restart;
  1640. }
  1641. }
  1642. /**
  1643. * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file.
  1644. * @mapping: the address space containing mmaps to be unmapped.
  1645. * @holebegin: byte in first page to unmap, relative to the start of
  1646. * the underlying file. This will be rounded down to a PAGE_SIZE
  1647. * boundary. Note that this is different from vmtruncate(), which
  1648. * must keep the partial page. In contrast, we must get rid of
  1649. * partial pages.
  1650. * @holelen: size of prospective hole in bytes. This will be rounded
  1651. * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
  1652. * end of the file.
  1653. * @even_cows: 1 when truncating a file, unmap even private COWed pages;
  1654. * but 0 when invalidating pagecache, don't throw away private data.
  1655. */
  1656. void unmap_mapping_range(struct address_space *mapping,
  1657. loff_t const holebegin, loff_t const holelen, int even_cows)
  1658. {
  1659. struct zap_details details;
  1660. pgoff_t hba = holebegin >> PAGE_SHIFT;
  1661. pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1662. /* Check for overflow. */
  1663. if (sizeof(holelen) > sizeof(hlen)) {
  1664. long long holeend =
  1665. (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1666. if (holeend & ~(long long)ULONG_MAX)
  1667. hlen = ULONG_MAX - hba + 1;
  1668. }
  1669. details.check_mapping = even_cows? NULL: mapping;
  1670. details.nonlinear_vma = NULL;
  1671. details.first_index = hba;
  1672. details.last_index = hba + hlen - 1;
  1673. if (details.last_index < details.first_index)
  1674. details.last_index = ULONG_MAX;
  1675. details.i_mmap_lock = &mapping->i_mmap_lock;
  1676. spin_lock(&mapping->i_mmap_lock);
  1677. /* serialize i_size write against truncate_count write */
  1678. smp_wmb();
  1679. /* Protect against page faults, and endless unmapping loops */
  1680. mapping->truncate_count++;
  1681. /*
  1682. * For archs where spin_lock has inclusive semantics like ia64
  1683. * this smp_mb() will prevent to read pagetable contents
  1684. * before the truncate_count increment is visible to
  1685. * other cpus.
  1686. */
  1687. smp_mb();
  1688. if (unlikely(is_restart_addr(mapping->truncate_count))) {
  1689. if (mapping->truncate_count == 0)
  1690. reset_vma_truncate_counts(mapping);
  1691. mapping->truncate_count++;
  1692. }
  1693. details.truncate_count = mapping->truncate_count;
  1694. if (unlikely(!prio_tree_empty(&mapping->i_mmap)))
  1695. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  1696. if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
  1697. unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
  1698. spin_unlock(&mapping->i_mmap_lock);
  1699. }
  1700. EXPORT_SYMBOL(unmap_mapping_range);
  1701. /**
  1702. * vmtruncate - unmap mappings "freed" by truncate() syscall
  1703. * @inode: inode of the file used
  1704. * @offset: file offset to start truncating
  1705. *
  1706. * NOTE! We have to be ready to update the memory sharing
  1707. * between the file and the memory map for a potential last
  1708. * incomplete page. Ugly, but necessary.
  1709. */
  1710. int vmtruncate(struct inode * inode, loff_t offset)
  1711. {
  1712. struct address_space *mapping = inode->i_mapping;
  1713. unsigned long limit;
  1714. if (inode->i_size < offset)
  1715. goto do_expand;
  1716. /*
  1717. * truncation of in-use swapfiles is disallowed - it would cause
  1718. * subsequent swapout to scribble on the now-freed blocks.
  1719. */
  1720. if (IS_SWAPFILE(inode))
  1721. goto out_busy;
  1722. i_size_write(inode, offset);
  1723. unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
  1724. truncate_inode_pages(mapping, offset);
  1725. goto out_truncate;
  1726. do_expand:
  1727. limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
  1728. if (limit != RLIM_INFINITY && offset > limit)
  1729. goto out_sig;
  1730. if (offset > inode->i_sb->s_maxbytes)
  1731. goto out_big;
  1732. i_size_write(inode, offset);
  1733. out_truncate:
  1734. if (inode->i_op && inode->i_op->truncate)
  1735. inode->i_op->truncate(inode);
  1736. return 0;
  1737. out_sig:
  1738. send_sig(SIGXFSZ, current, 0);
  1739. out_big:
  1740. return -EFBIG;
  1741. out_busy:
  1742. return -ETXTBSY;
  1743. }
  1744. EXPORT_SYMBOL(vmtruncate);
  1745. int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
  1746. {
  1747. struct address_space *mapping = inode->i_mapping;
  1748. /*
  1749. * If the underlying filesystem is not going to provide
  1750. * a way to truncate a range of blocks (punch a hole) -
  1751. * we should return failure right now.
  1752. */
  1753. if (!inode->i_op || !inode->i_op->truncate_range)
  1754. return -ENOSYS;
  1755. mutex_lock(&inode->i_mutex);
  1756. down_write(&inode->i_alloc_sem);
  1757. unmap_mapping_range(mapping, offset, (end - offset), 1);
  1758. truncate_inode_pages_range(mapping, offset, end);
  1759. inode->i_op->truncate_range(inode, offset, end);
  1760. up_write(&inode->i_alloc_sem);
  1761. mutex_unlock(&inode->i_mutex);
  1762. return 0;
  1763. }
  1764. /**
  1765. * swapin_readahead - swap in pages in hope we need them soon
  1766. * @entry: swap entry of this memory
  1767. * @addr: address to start
  1768. * @vma: user vma this addresses belong to
  1769. *
  1770. * Primitive swap readahead code. We simply read an aligned block of
  1771. * (1 << page_cluster) entries in the swap area. This method is chosen
  1772. * because it doesn't cost us any seek time. We also make sure to queue
  1773. * the 'original' request together with the readahead ones...
  1774. *
  1775. * This has been extended to use the NUMA policies from the mm triggering
  1776. * the readahead.
  1777. *
  1778. * Caller must hold down_read on the vma->vm_mm if vma is not NULL.
  1779. */
  1780. void swapin_readahead(swp_entry_t entry, unsigned long addr,struct vm_area_struct *vma)
  1781. {
  1782. #ifdef CONFIG_NUMA
  1783. struct vm_area_struct *next_vma = vma ? vma->vm_next : NULL;
  1784. #endif
  1785. int i, num;
  1786. struct page *new_page;
  1787. unsigned long offset;
  1788. /*
  1789. * Get the number of handles we should do readahead io to.
  1790. */
  1791. num = valid_swaphandles(entry, &offset);
  1792. for (i = 0; i < num; offset++, i++) {
  1793. /* Ok, do the async read-ahead now */
  1794. new_page = read_swap_cache_async(swp_entry(swp_type(entry),
  1795. offset), vma, addr);
  1796. if (!new_page)
  1797. break;
  1798. page_cache_release(new_page);
  1799. #ifdef CONFIG_NUMA
  1800. /*
  1801. * Find the next applicable VMA for the NUMA policy.
  1802. */
  1803. addr += PAGE_SIZE;
  1804. if (addr == 0)
  1805. vma = NULL;
  1806. if (vma) {
  1807. if (addr >= vma->vm_end) {
  1808. vma = next_vma;
  1809. next_vma = vma ? vma->vm_next : NULL;
  1810. }
  1811. if (vma && addr < vma->vm_start)
  1812. vma = NULL;
  1813. } else {
  1814. if (next_vma && addr >= next_vma->vm_start) {
  1815. vma = next_vma;
  1816. next_vma = vma->vm_next;
  1817. }
  1818. }
  1819. #endif
  1820. }
  1821. lru_add_drain(); /* Push any new pages onto the LRU now */
  1822. }
  1823. /*
  1824. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1825. * but allow concurrent faults), and pte mapped but not yet locked.
  1826. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1827. */
  1828. static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1829. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1830. int write_access, pte_t orig_pte)
  1831. {
  1832. spinlock_t *ptl;
  1833. struct page *page;
  1834. swp_entry_t entry;
  1835. pte_t pte;
  1836. int ret = VM_FAULT_MINOR;
  1837. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  1838. goto out;
  1839. entry = pte_to_swp_entry(orig_pte);
  1840. if (is_migration_entry(entry)) {
  1841. migration_entry_wait(mm, pmd, address);
  1842. goto out;
  1843. }
  1844. delayacct_set_flag(DELAYACCT_PF_SWAPIN);
  1845. page = lookup_swap_cache(entry);
  1846. if (!page) {
  1847. grab_swap_token(); /* Contend for token _before_ read-in */
  1848. swapin_readahead(entry, address, vma);
  1849. page = read_swap_cache_async(entry, vma, address);
  1850. if (!page) {
  1851. /*
  1852. * Back out if somebody else faulted in this pte
  1853. * while we released the pte lock.
  1854. */
  1855. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1856. if (likely(pte_same(*page_table, orig_pte)))
  1857. ret = VM_FAULT_OOM;
  1858. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  1859. goto unlock;
  1860. }
  1861. /* Had to read the page from swap area: Major fault */
  1862. ret = VM_FAULT_MAJOR;
  1863. count_vm_event(PGMAJFAULT);
  1864. }
  1865. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  1866. mark_page_accessed(page);
  1867. lock_page(page);
  1868. /*
  1869. * Back out if somebody else already faulted in this pte.
  1870. */
  1871. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1872. if (unlikely(!pte_same(*page_table, orig_pte)))
  1873. goto out_nomap;
  1874. if (unlikely(!PageUptodate(page))) {
  1875. ret = VM_FAULT_SIGBUS;
  1876. goto out_nomap;
  1877. }
  1878. /* The page isn't present yet, go ahead with the fault. */
  1879. inc_mm_counter(mm, anon_rss);
  1880. pte = mk_pte(page, vma->vm_page_prot);
  1881. if (write_access && can_share_swap_page(page)) {
  1882. pte = maybe_mkwrite(pte_mkdirty(pte), vma);
  1883. write_access = 0;
  1884. }
  1885. flush_icache_page(vma, page);
  1886. set_pte_at(mm, address, page_table, pte);
  1887. page_add_anon_rmap(page, vma, address);
  1888. swap_free(entry);
  1889. if (vm_swap_full())
  1890. remove_exclusive_swap_page(page);
  1891. unlock_page(page);
  1892. if (write_access) {
  1893. if (do_wp_page(mm, vma, address,
  1894. page_table, pmd, ptl, pte) == VM_FAULT_OOM)
  1895. ret = VM_FAULT_OOM;
  1896. goto out;
  1897. }
  1898. /* No need to invalidate - it was non-present before */
  1899. update_mmu_cache(vma, address, pte);
  1900. lazy_mmu_prot_update(pte);
  1901. unlock:
  1902. pte_unmap_unlock(page_table, ptl);
  1903. out:
  1904. return ret;
  1905. out_nomap:
  1906. pte_unmap_unlock(page_table, ptl);
  1907. unlock_page(page);
  1908. page_cache_release(page);
  1909. return ret;
  1910. }
  1911. /*
  1912. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1913. * but allow concurrent faults), and pte mapped but not yet locked.
  1914. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1915. */
  1916. static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1917. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1918. int write_access)
  1919. {
  1920. struct page *page;
  1921. spinlock_t *ptl;
  1922. pte_t entry;
  1923. if (write_access) {
  1924. /* Allocate our own private page. */
  1925. pte_unmap(page_table);
  1926. if (unlikely(anon_vma_prepare(vma)))
  1927. goto oom;
  1928. page = alloc_zeroed_user_highpage(vma, address);
  1929. if (!page)
  1930. goto oom;
  1931. entry = mk_pte(page, vma->vm_page_prot);
  1932. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1933. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1934. if (!pte_none(*page_table))
  1935. goto release;
  1936. inc_mm_counter(mm, anon_rss);
  1937. lru_cache_add_active(page);
  1938. page_add_new_anon_rmap(page, vma, address);
  1939. } else {
  1940. /* Map the ZERO_PAGE - vm_page_prot is readonly */
  1941. page = ZERO_PAGE(address);
  1942. page_cache_get(page);
  1943. entry = mk_pte(page, vma->vm_page_prot);
  1944. ptl = pte_lockptr(mm, pmd);
  1945. spin_lock(ptl);
  1946. if (!pte_none(*page_table))
  1947. goto release;
  1948. inc_mm_counter(mm, file_rss);
  1949. page_add_file_rmap(page);
  1950. }
  1951. set_pte_at(mm, address, page_table, entry);
  1952. /* No need to invalidate - it was non-present before */
  1953. update_mmu_cache(vma, address, entry);
  1954. lazy_mmu_prot_update(entry);
  1955. unlock:
  1956. pte_unmap_unlock(page_table, ptl);
  1957. return VM_FAULT_MINOR;
  1958. release:
  1959. page_cache_release(page);
  1960. goto unlock;
  1961. oom:
  1962. return VM_FAULT_OOM;
  1963. }
  1964. /*
  1965. * do_no_page() tries to create a new page mapping. It aggressively
  1966. * tries to share with existing pages, but makes a separate copy if
  1967. * the "write_access" parameter is true in order to avoid the next
  1968. * page fault.
  1969. *
  1970. * As this is called only for pages that do not currently exist, we
  1971. * do not need to flush old virtual caches or the TLB.
  1972. *
  1973. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1974. * but allow concurrent faults), and pte mapped but not yet locked.
  1975. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1976. */
  1977. static int do_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1978. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1979. int write_access)
  1980. {
  1981. spinlock_t *ptl;
  1982. struct page *new_page;
  1983. struct address_space *mapping = NULL;
  1984. pte_t entry;
  1985. unsigned int sequence = 0;
  1986. int ret = VM_FAULT_MINOR;
  1987. int anon = 0;
  1988. struct page *dirty_page = NULL;
  1989. pte_unmap(page_table);
  1990. BUG_ON(vma->vm_flags & VM_PFNMAP);
  1991. if (vma->vm_file) {
  1992. mapping = vma->vm_file->f_mapping;
  1993. sequence = mapping->truncate_count;
  1994. smp_rmb(); /* serializes i_size against truncate_count */
  1995. }
  1996. retry:
  1997. new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, &ret);
  1998. /*
  1999. * No smp_rmb is needed here as long as there's a full
  2000. * spin_lock/unlock sequence inside the ->nopage callback
  2001. * (for the pagecache lookup) that acts as an implicit
  2002. * smp_mb() and prevents the i_size read to happen
  2003. * after the next truncate_count read.
  2004. */
  2005. /* no page was available -- either SIGBUS, OOM or REFAULT */
  2006. if (unlikely(new_page == NOPAGE_SIGBUS))
  2007. return VM_FAULT_SIGBUS;
  2008. else if (unlikely(new_page == NOPAGE_OOM))
  2009. return VM_FAULT_OOM;
  2010. else if (unlikely(new_page == NOPAGE_REFAULT))
  2011. return VM_FAULT_MINOR;
  2012. /*
  2013. * Should we do an early C-O-W break?
  2014. */
  2015. if (write_access) {
  2016. if (!(vma->vm_flags & VM_SHARED)) {
  2017. struct page *page;
  2018. if (unlikely(anon_vma_prepare(vma)))
  2019. goto oom;
  2020. page = alloc_page_vma(GFP_HIGHUSER, vma, address);
  2021. if (!page)
  2022. goto oom;
  2023. copy_user_highpage(page, new_page, address, vma);
  2024. page_cache_release(new_page);
  2025. new_page = page;
  2026. anon = 1;
  2027. } else {
  2028. /* if the page will be shareable, see if the backing
  2029. * address space wants to know that the page is about
  2030. * to become writable */
  2031. if (vma->vm_ops->page_mkwrite &&
  2032. vma->vm_ops->page_mkwrite(vma, new_page) < 0
  2033. ) {
  2034. page_cache_release(new_page);
  2035. return VM_FAULT_SIGBUS;
  2036. }
  2037. }
  2038. }
  2039. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2040. /*
  2041. * For a file-backed vma, someone could have truncated or otherwise
  2042. * invalidated this page. If unmap_mapping_range got called,
  2043. * retry getting the page.
  2044. */
  2045. if (mapping && unlikely(sequence != mapping->truncate_count)) {
  2046. pte_unmap_unlock(page_table, ptl);
  2047. page_cache_release(new_page);
  2048. cond_resched();
  2049. sequence = mapping->truncate_count;
  2050. smp_rmb();
  2051. goto retry;
  2052. }
  2053. /*
  2054. * This silly early PAGE_DIRTY setting removes a race
  2055. * due to the bad i386 page protection. But it's valid
  2056. * for other architectures too.
  2057. *
  2058. * Note that if write_access is true, we either now have
  2059. * an exclusive copy of the page, or this is a shared mapping,
  2060. * so we can make it writable and dirty to avoid having to
  2061. * handle that later.
  2062. */
  2063. /* Only go through if we didn't race with anybody else... */
  2064. if (pte_none(*page_table)) {
  2065. flush_icache_page(vma, new_page);
  2066. entry = mk_pte(new_page, vma->vm_page_prot);
  2067. if (write_access)
  2068. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2069. set_pte_at(mm, address, page_table, entry);
  2070. if (anon) {
  2071. inc_mm_counter(mm, anon_rss);
  2072. lru_cache_add_active(new_page);
  2073. page_add_new_anon_rmap(new_page, vma, address);
  2074. } else {
  2075. inc_mm_counter(mm, file_rss);
  2076. page_add_file_rmap(new_page);
  2077. if (write_access) {
  2078. dirty_page = new_page;
  2079. get_page(dirty_page);
  2080. }
  2081. }
  2082. } else {
  2083. /* One of our sibling threads was faster, back out. */
  2084. page_cache_release(new_page);
  2085. goto unlock;
  2086. }
  2087. /* no need to invalidate: a not-present page shouldn't be cached */
  2088. update_mmu_cache(vma, address, entry);
  2089. lazy_mmu_prot_update(entry);
  2090. unlock:
  2091. pte_unmap_unlock(page_table, ptl);
  2092. if (dirty_page) {
  2093. set_page_dirty_balance(dirty_page);
  2094. put_page(dirty_page);
  2095. }
  2096. return ret;
  2097. oom:
  2098. page_cache_release(new_page);
  2099. return VM_FAULT_OOM;
  2100. }
  2101. /*
  2102. * do_no_pfn() tries to create a new page mapping for a page without
  2103. * a struct_page backing it
  2104. *
  2105. * As this is called only for pages that do not currently exist, we
  2106. * do not need to flush old virtual caches or the TLB.
  2107. *
  2108. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2109. * but allow concurrent faults), and pte mapped but not yet locked.
  2110. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2111. *
  2112. * It is expected that the ->nopfn handler always returns the same pfn
  2113. * for a given virtual mapping.
  2114. *
  2115. * Mark this `noinline' to prevent it from bloating the main pagefault code.
  2116. */
  2117. static noinline int do_no_pfn(struct mm_struct *mm, struct vm_area_struct *vma,
  2118. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2119. int write_access)
  2120. {
  2121. spinlock_t *ptl;
  2122. pte_t entry;
  2123. unsigned long pfn;
  2124. int ret = VM_FAULT_MINOR;
  2125. pte_unmap(page_table);
  2126. BUG_ON(!(vma->vm_flags & VM_PFNMAP));
  2127. BUG_ON(is_cow_mapping(vma->vm_flags));
  2128. pfn = vma->vm_ops->nopfn(vma, address & PAGE_MASK);
  2129. if (unlikely(pfn == NOPFN_OOM))
  2130. return VM_FAULT_OOM;
  2131. else if (unlikely(pfn == NOPFN_SIGBUS))
  2132. return VM_FAULT_SIGBUS;
  2133. else if (unlikely(pfn == NOPFN_REFAULT))
  2134. return VM_FAULT_MINOR;
  2135. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2136. /* Only go through if we didn't race with anybody else... */
  2137. if (pte_none(*page_table)) {
  2138. entry = pfn_pte(pfn, vma->vm_page_prot);
  2139. if (write_access)
  2140. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2141. set_pte_at(mm, address, page_table, entry);
  2142. }
  2143. pte_unmap_unlock(page_table, ptl);
  2144. return ret;
  2145. }
  2146. /*
  2147. * Fault of a previously existing named mapping. Repopulate the pte
  2148. * from the encoded file_pte if possible. This enables swappable
  2149. * nonlinear vmas.
  2150. *
  2151. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2152. * but allow concurrent faults), and pte mapped but not yet locked.
  2153. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2154. */
  2155. static int do_file_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2156. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2157. int write_access, pte_t orig_pte)
  2158. {
  2159. pgoff_t pgoff;
  2160. int err;
  2161. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  2162. return VM_FAULT_MINOR;
  2163. if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) {
  2164. /*
  2165. * Page table corrupted: show pte and kill process.
  2166. */
  2167. print_bad_pte(vma, orig_pte, address);
  2168. return VM_FAULT_OOM;
  2169. }
  2170. /* We can then assume vm->vm_ops && vma->vm_ops->populate */
  2171. pgoff = pte_to_pgoff(orig_pte);
  2172. err = vma->vm_ops->populate(vma, address & PAGE_MASK, PAGE_SIZE,
  2173. vma->vm_page_prot, pgoff, 0);
  2174. if (err == -ENOMEM)
  2175. return VM_FAULT_OOM;
  2176. if (err)
  2177. return VM_FAULT_SIGBUS;
  2178. return VM_FAULT_MAJOR;
  2179. }
  2180. /*
  2181. * These routines also need to handle stuff like marking pages dirty
  2182. * and/or accessed for architectures that don't do it in hardware (most
  2183. * RISC architectures). The early dirtying is also good on the i386.
  2184. *
  2185. * There is also a hook called "update_mmu_cache()" that architectures
  2186. * with external mmu caches can use to update those (ie the Sparc or
  2187. * PowerPC hashed page tables that act as extended TLBs).
  2188. *
  2189. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2190. * but allow concurrent faults), and pte mapped but not yet locked.
  2191. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2192. */
  2193. static inline int handle_pte_fault(struct mm_struct *mm,
  2194. struct vm_area_struct *vma, unsigned long address,
  2195. pte_t *pte, pmd_t *pmd, int write_access)
  2196. {
  2197. pte_t entry;
  2198. pte_t old_entry;
  2199. spinlock_t *ptl;
  2200. old_entry = entry = *pte;
  2201. if (!pte_present(entry)) {
  2202. if (pte_none(entry)) {
  2203. if (vma->vm_ops) {
  2204. if (vma->vm_ops->nopage)
  2205. return do_no_page(mm, vma, address,
  2206. pte, pmd,
  2207. write_access);
  2208. if (unlikely(vma->vm_ops->nopfn))
  2209. return do_no_pfn(mm, vma, address, pte,
  2210. pmd, write_access);
  2211. }
  2212. return do_anonymous_page(mm, vma, address,
  2213. pte, pmd, write_access);
  2214. }
  2215. if (pte_file(entry))
  2216. return do_file_page(mm, vma, address,
  2217. pte, pmd, write_access, entry);
  2218. return do_swap_page(mm, vma, address,
  2219. pte, pmd, write_access, entry);
  2220. }
  2221. ptl = pte_lockptr(mm, pmd);
  2222. spin_lock(ptl);
  2223. if (unlikely(!pte_same(*pte, entry)))
  2224. goto unlock;
  2225. if (write_access) {
  2226. if (!pte_write(entry))
  2227. return do_wp_page(mm, vma, address,
  2228. pte, pmd, ptl, entry);
  2229. entry = pte_mkdirty(entry);
  2230. }
  2231. entry = pte_mkyoung(entry);
  2232. if (!pte_same(old_entry, entry)) {
  2233. ptep_set_access_flags(vma, address, pte, entry, write_access);
  2234. update_mmu_cache(vma, address, entry);
  2235. lazy_mmu_prot_update(entry);
  2236. } else {
  2237. /*
  2238. * This is needed only for protection faults but the arch code
  2239. * is not yet telling us if this is a protection fault or not.
  2240. * This still avoids useless tlb flushes for .text page faults
  2241. * with threads.
  2242. */
  2243. if (write_access)
  2244. flush_tlb_page(vma, address);
  2245. }
  2246. unlock:
  2247. pte_unmap_unlock(pte, ptl);
  2248. return VM_FAULT_MINOR;
  2249. }
  2250. /*
  2251. * By the time we get here, we already hold the mm semaphore
  2252. */
  2253. int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2254. unsigned long address, int write_access)
  2255. {
  2256. pgd_t *pgd;
  2257. pud_t *pud;
  2258. pmd_t *pmd;
  2259. pte_t *pte;
  2260. __set_current_state(TASK_RUNNING);
  2261. count_vm_event(PGFAULT);
  2262. if (unlikely(is_vm_hugetlb_page(vma)))
  2263. return hugetlb_fault(mm, vma, address, write_access);
  2264. pgd = pgd_offset(mm, address);
  2265. pud = pud_alloc(mm, pgd, address);
  2266. if (!pud)
  2267. return VM_FAULT_OOM;
  2268. pmd = pmd_alloc(mm, pud, address);
  2269. if (!pmd)
  2270. return VM_FAULT_OOM;
  2271. pte = pte_alloc_map(mm, pmd, address);
  2272. if (!pte)
  2273. return VM_FAULT_OOM;
  2274. return handle_pte_fault(mm, vma, address, pte, pmd, write_access);
  2275. }
  2276. EXPORT_SYMBOL_GPL(__handle_mm_fault);
  2277. #ifndef __PAGETABLE_PUD_FOLDED
  2278. /*
  2279. * Allocate page upper directory.
  2280. * We've already handled the fast-path in-line.
  2281. */
  2282. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  2283. {
  2284. pud_t *new = pud_alloc_one(mm, address);
  2285. if (!new)
  2286. return -ENOMEM;
  2287. spin_lock(&mm->page_table_lock);
  2288. if (pgd_present(*pgd)) /* Another has populated it */
  2289. pud_free(new);
  2290. else
  2291. pgd_populate(mm, pgd, new);
  2292. spin_unlock(&mm->page_table_lock);
  2293. return 0;
  2294. }
  2295. #else
  2296. /* Workaround for gcc 2.96 */
  2297. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  2298. {
  2299. return 0;
  2300. }
  2301. #endif /* __PAGETABLE_PUD_FOLDED */
  2302. #ifndef __PAGETABLE_PMD_FOLDED
  2303. /*
  2304. * Allocate page middle directory.
  2305. * We've already handled the fast-path in-line.
  2306. */
  2307. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  2308. {
  2309. pmd_t *new = pmd_alloc_one(mm, address);
  2310. if (!new)
  2311. return -ENOMEM;
  2312. spin_lock(&mm->page_table_lock);
  2313. #ifndef __ARCH_HAS_4LEVEL_HACK
  2314. if (pud_present(*pud)) /* Another has populated it */
  2315. pmd_free(new);
  2316. else
  2317. pud_populate(mm, pud, new);
  2318. #else
  2319. if (pgd_present(*pud)) /* Another has populated it */
  2320. pmd_free(new);
  2321. else
  2322. pgd_populate(mm, pud, new);
  2323. #endif /* __ARCH_HAS_4LEVEL_HACK */
  2324. spin_unlock(&mm->page_table_lock);
  2325. return 0;
  2326. }
  2327. #else
  2328. /* Workaround for gcc 2.96 */
  2329. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  2330. {
  2331. return 0;
  2332. }
  2333. #endif /* __PAGETABLE_PMD_FOLDED */
  2334. int make_pages_present(unsigned long addr, unsigned long end)
  2335. {
  2336. int ret, len, write;
  2337. struct vm_area_struct * vma;
  2338. vma = find_vma(current->mm, addr);
  2339. if (!vma)
  2340. return -1;
  2341. write = (vma->vm_flags & VM_WRITE) != 0;
  2342. BUG_ON(addr >= end);
  2343. BUG_ON(end > vma->vm_end);
  2344. len = (end+PAGE_SIZE-1)/PAGE_SIZE-addr/PAGE_SIZE;
  2345. ret = get_user_pages(current, current->mm, addr,
  2346. len, write, 0, NULL, NULL);
  2347. if (ret < 0)
  2348. return ret;
  2349. return ret == len ? 0 : -1;
  2350. }
  2351. /*
  2352. * Map a vmalloc()-space virtual address to the physical page.
  2353. */
  2354. struct page * vmalloc_to_page(void * vmalloc_addr)
  2355. {
  2356. unsigned long addr = (unsigned long) vmalloc_addr;
  2357. struct page *page = NULL;
  2358. pgd_t *pgd = pgd_offset_k(addr);
  2359. pud_t *pud;
  2360. pmd_t *pmd;
  2361. pte_t *ptep, pte;
  2362. if (!pgd_none(*pgd)) {
  2363. pud = pud_offset(pgd, addr);
  2364. if (!pud_none(*pud)) {
  2365. pmd = pmd_offset(pud, addr);
  2366. if (!pmd_none(*pmd)) {
  2367. ptep = pte_offset_map(pmd, addr);
  2368. pte = *ptep;
  2369. if (pte_present(pte))
  2370. page = pte_page(pte);
  2371. pte_unmap(ptep);
  2372. }
  2373. }
  2374. }
  2375. return page;
  2376. }
  2377. EXPORT_SYMBOL(vmalloc_to_page);
  2378. /*
  2379. * Map a vmalloc()-space virtual address to the physical page frame number.
  2380. */
  2381. unsigned long vmalloc_to_pfn(void * vmalloc_addr)
  2382. {
  2383. return page_to_pfn(vmalloc_to_page(vmalloc_addr));
  2384. }
  2385. EXPORT_SYMBOL(vmalloc_to_pfn);
  2386. #if !defined(__HAVE_ARCH_GATE_AREA)
  2387. #if defined(AT_SYSINFO_EHDR)
  2388. static struct vm_area_struct gate_vma;
  2389. static int __init gate_vma_init(void)
  2390. {
  2391. gate_vma.vm_mm = NULL;
  2392. gate_vma.vm_start = FIXADDR_USER_START;
  2393. gate_vma.vm_end = FIXADDR_USER_END;
  2394. gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
  2395. gate_vma.vm_page_prot = __P101;
  2396. /*
  2397. * Make sure the vDSO gets into every core dump.
  2398. * Dumping its contents makes post-mortem fully interpretable later
  2399. * without matching up the same kernel and hardware config to see
  2400. * what PC values meant.
  2401. */
  2402. gate_vma.vm_flags |= VM_ALWAYSDUMP;
  2403. return 0;
  2404. }
  2405. __initcall(gate_vma_init);
  2406. #endif
  2407. struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
  2408. {
  2409. #ifdef AT_SYSINFO_EHDR
  2410. return &gate_vma;
  2411. #else
  2412. return NULL;
  2413. #endif
  2414. }
  2415. int in_gate_area_no_task(unsigned long addr)
  2416. {
  2417. #ifdef AT_SYSINFO_EHDR
  2418. if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END))
  2419. return 1;
  2420. #endif
  2421. return 0;
  2422. }
  2423. #endif /* __HAVE_ARCH_GATE_AREA */
  2424. /*
  2425. * Access another process' address space.
  2426. * Source/target buffer must be kernel space,
  2427. * Do not walk the page table directly, use get_user_pages
  2428. */
  2429. int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
  2430. {
  2431. struct mm_struct *mm;
  2432. struct vm_area_struct *vma;
  2433. struct page *page;
  2434. void *old_buf = buf;
  2435. mm = get_task_mm(tsk);
  2436. if (!mm)
  2437. return 0;
  2438. down_read(&mm->mmap_sem);
  2439. /* ignore errors, just check how much was sucessfully transfered */
  2440. while (len) {
  2441. int bytes, ret, offset;
  2442. void *maddr;
  2443. ret = get_user_pages(tsk, mm, addr, 1,
  2444. write, 1, &page, &vma);
  2445. if (ret <= 0)
  2446. break;
  2447. bytes = len;
  2448. offset = addr & (PAGE_SIZE-1);
  2449. if (bytes > PAGE_SIZE-offset)
  2450. bytes = PAGE_SIZE-offset;
  2451. maddr = kmap(page);
  2452. if (write) {
  2453. copy_to_user_page(vma, page, addr,
  2454. maddr + offset, buf, bytes);
  2455. set_page_dirty_lock(page);
  2456. } else {
  2457. copy_from_user_page(vma, page, addr,
  2458. buf, maddr + offset, bytes);
  2459. }
  2460. kunmap(page);
  2461. page_cache_release(page);
  2462. len -= bytes;
  2463. buf += bytes;
  2464. addr += bytes;
  2465. }
  2466. up_read(&mm->mmap_sem);
  2467. mmput(mm);
  2468. return buf - old_buf;
  2469. }