filemap.c 101 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/filemap.c
  4. *
  5. * Copyright (C) 1994-1999 Linus Torvalds
  6. */
  7. /*
  8. * This file handles the generic file mmap semantics used by
  9. * most "normal" filesystems (but you don't /have/ to use this:
  10. * the NFS filesystem used to do this differently, for example)
  11. */
  12. #include <linux/export.h>
  13. #include <linux/compiler.h>
  14. #include <linux/dax.h>
  15. #include <linux/fs.h>
  16. #include <linux/sched/signal.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/capability.h>
  19. #include <linux/kernel_stat.h>
  20. #include <linux/gfp.h>
  21. #include <linux/mm.h>
  22. #include <linux/swap.h>
  23. #include <linux/mman.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/file.h>
  26. #include <linux/uio.h>
  27. #include <linux/error-injection.h>
  28. #include <linux/hash.h>
  29. #include <linux/writeback.h>
  30. #include <linux/backing-dev.h>
  31. #include <linux/pagevec.h>
  32. #include <linux/blkdev.h>
  33. #include <linux/security.h>
  34. #include <linux/cpuset.h>
  35. #include <linux/hugetlb.h>
  36. #include <linux/memcontrol.h>
  37. #include <linux/cleancache.h>
  38. #include <linux/shmem_fs.h>
  39. #include <linux/rmap.h>
  40. #include <linux/delayacct.h>
  41. #include <linux/psi.h>
  42. #include <linux/ramfs.h>
  43. #include <linux/page_idle.h>
  44. #include <asm/pgalloc.h>
  45. #include <asm/tlbflush.h>
  46. #include "internal.h"
  47. #define CREATE_TRACE_POINTS
  48. #include <trace/events/filemap.h>
  49. #undef CREATE_TRACE_POINTS
  50. #include <trace/hooks/mm.h>
  51. /*
  52. * FIXME: remove all knowledge of the buffer layer from the core VM
  53. */
  54. #include <linux/buffer_head.h> /* for try_to_free_buffers */
  55. #include <asm/mman.h>
  56. /*
  57. * Shared mappings implemented 30.11.1994. It's not fully working yet,
  58. * though.
  59. *
  60. * Shared mappings now work. 15.8.1995 Bruno.
  61. *
  62. * finished 'unifying' the page and buffer cache and SMP-threaded the
  63. * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  64. *
  65. * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  66. */
  67. /*
  68. * Lock ordering:
  69. *
  70. * ->i_mmap_rwsem (truncate_pagecache)
  71. * ->private_lock (__free_pte->__set_page_dirty_buffers)
  72. * ->swap_lock (exclusive_swap_page, others)
  73. * ->i_pages lock
  74. *
  75. * ->i_mutex
  76. * ->i_mmap_rwsem (truncate->unmap_mapping_range)
  77. *
  78. * ->mmap_lock
  79. * ->i_mmap_rwsem
  80. * ->page_table_lock or pte_lock (various, mainly in memory.c)
  81. * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
  82. *
  83. * ->mmap_lock
  84. * ->lock_page (access_process_vm)
  85. *
  86. * ->i_mutex (generic_perform_write)
  87. * ->mmap_lock (fault_in_pages_readable->do_page_fault)
  88. *
  89. * bdi->wb.list_lock
  90. * sb_lock (fs/fs-writeback.c)
  91. * ->i_pages lock (__sync_single_inode)
  92. *
  93. * ->i_mmap_rwsem
  94. * ->anon_vma.lock (vma_adjust)
  95. *
  96. * ->anon_vma.lock
  97. * ->page_table_lock or pte_lock (anon_vma_prepare and various)
  98. *
  99. * ->page_table_lock or pte_lock
  100. * ->swap_lock (try_to_unmap_one)
  101. * ->private_lock (try_to_unmap_one)
  102. * ->i_pages lock (try_to_unmap_one)
  103. * ->pgdat->lru_lock (follow_page->mark_page_accessed)
  104. * ->pgdat->lru_lock (check_pte_range->isolate_lru_page)
  105. * ->private_lock (page_remove_rmap->set_page_dirty)
  106. * ->i_pages lock (page_remove_rmap->set_page_dirty)
  107. * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
  108. * ->inode->i_lock (page_remove_rmap->set_page_dirty)
  109. * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
  110. * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
  111. * ->inode->i_lock (zap_pte_range->set_page_dirty)
  112. * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
  113. *
  114. * ->i_mmap_rwsem
  115. * ->tasklist_lock (memory_failure, collect_procs_ao)
  116. */
  117. static void page_cache_delete(struct address_space *mapping,
  118. struct page *page, void *shadow)
  119. {
  120. XA_STATE(xas, &mapping->i_pages, page->index);
  121. unsigned int nr = 1;
  122. mapping_set_update(&xas, mapping);
  123. /* hugetlb pages are represented by a single entry in the xarray */
  124. if (!PageHuge(page)) {
  125. xas_set_order(&xas, page->index, compound_order(page));
  126. nr = compound_nr(page);
  127. }
  128. VM_BUG_ON_PAGE(!PageLocked(page), page);
  129. VM_BUG_ON_PAGE(PageTail(page), page);
  130. VM_BUG_ON_PAGE(nr != 1 && shadow, page);
  131. xas_store(&xas, shadow);
  132. xas_init_marks(&xas);
  133. page->mapping = NULL;
  134. /* Leave page->index set: truncation lookup relies upon it */
  135. if (shadow) {
  136. mapping->nrexceptional += nr;
  137. /*
  138. * Make sure the nrexceptional update is committed before
  139. * the nrpages update so that final truncate racing
  140. * with reclaim does not see both counters 0 at the
  141. * same time and miss a shadow entry.
  142. */
  143. smp_wmb();
  144. }
  145. mapping->nrpages -= nr;
  146. }
  147. static void unaccount_page_cache_page(struct address_space *mapping,
  148. struct page *page)
  149. {
  150. int nr;
  151. /*
  152. * if we're uptodate, flush out into the cleancache, otherwise
  153. * invalidate any existing cleancache entries. We can't leave
  154. * stale data around in the cleancache once our page is gone
  155. */
  156. if (PageUptodate(page) && PageMappedToDisk(page))
  157. cleancache_put_page(page);
  158. else
  159. cleancache_invalidate_page(mapping, page);
  160. VM_BUG_ON_PAGE(PageTail(page), page);
  161. VM_BUG_ON_PAGE(page_mapped(page), page);
  162. if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
  163. int mapcount;
  164. pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
  165. current->comm, page_to_pfn(page));
  166. dump_page(page, "still mapped when deleted");
  167. dump_stack();
  168. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  169. mapcount = page_mapcount(page);
  170. if (mapping_exiting(mapping) &&
  171. page_count(page) >= mapcount + 2) {
  172. /*
  173. * All vmas have already been torn down, so it's
  174. * a good bet that actually the page is unmapped,
  175. * and we'd prefer not to leak it: if we're wrong,
  176. * some other bad page check should catch it later.
  177. */
  178. page_mapcount_reset(page);
  179. page_ref_sub(page, mapcount);
  180. }
  181. }
  182. /* hugetlb pages do not participate in page cache accounting. */
  183. if (PageHuge(page))
  184. return;
  185. nr = thp_nr_pages(page);
  186. __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr);
  187. if (PageSwapBacked(page)) {
  188. __mod_lruvec_page_state(page, NR_SHMEM, -nr);
  189. if (PageTransHuge(page))
  190. __dec_node_page_state(page, NR_SHMEM_THPS);
  191. } else if (PageTransHuge(page)) {
  192. __dec_node_page_state(page, NR_FILE_THPS);
  193. filemap_nr_thps_dec(mapping);
  194. }
  195. /*
  196. * At this point page must be either written or cleaned by
  197. * truncate. Dirty page here signals a bug and loss of
  198. * unwritten data.
  199. *
  200. * This fixes dirty accounting after removing the page entirely
  201. * but leaves PageDirty set: it has no effect for truncated
  202. * page and anyway will be cleared before returning page into
  203. * buddy allocator.
  204. */
  205. if (WARN_ON_ONCE(PageDirty(page)))
  206. account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
  207. }
  208. /*
  209. * Delete a page from the page cache and free it. Caller has to make
  210. * sure the page is locked and that nobody else uses it - or that usage
  211. * is safe. The caller must hold the i_pages lock.
  212. */
  213. void __delete_from_page_cache(struct page *page, void *shadow)
  214. {
  215. struct address_space *mapping = page->mapping;
  216. trace_mm_filemap_delete_from_page_cache(page);
  217. unaccount_page_cache_page(mapping, page);
  218. page_cache_delete(mapping, page, shadow);
  219. }
  220. static void page_cache_free_page(struct address_space *mapping,
  221. struct page *page)
  222. {
  223. void (*freepage)(struct page *);
  224. freepage = mapping->a_ops->freepage;
  225. if (freepage)
  226. freepage(page);
  227. if (PageTransHuge(page) && !PageHuge(page)) {
  228. page_ref_sub(page, thp_nr_pages(page));
  229. VM_BUG_ON_PAGE(page_count(page) <= 0, page);
  230. } else {
  231. put_page(page);
  232. }
  233. }
  234. /**
  235. * delete_from_page_cache - delete page from page cache
  236. * @page: the page which the kernel is trying to remove from page cache
  237. *
  238. * This must be called only on pages that have been verified to be in the page
  239. * cache and locked. It will never put the page into the free list, the caller
  240. * has a reference on the page.
  241. */
  242. void delete_from_page_cache(struct page *page)
  243. {
  244. struct address_space *mapping = page_mapping(page);
  245. unsigned long flags;
  246. BUG_ON(!PageLocked(page));
  247. xa_lock_irqsave(&mapping->i_pages, flags);
  248. __delete_from_page_cache(page, NULL);
  249. xa_unlock_irqrestore(&mapping->i_pages, flags);
  250. page_cache_free_page(mapping, page);
  251. }
  252. EXPORT_SYMBOL(delete_from_page_cache);
  253. /*
  254. * page_cache_delete_batch - delete several pages from page cache
  255. * @mapping: the mapping to which pages belong
  256. * @pvec: pagevec with pages to delete
  257. *
  258. * The function walks over mapping->i_pages and removes pages passed in @pvec
  259. * from the mapping. The function expects @pvec to be sorted by page index
  260. * and is optimised for it to be dense.
  261. * It tolerates holes in @pvec (mapping entries at those indices are not
  262. * modified). The function expects only THP head pages to be present in the
  263. * @pvec.
  264. *
  265. * The function expects the i_pages lock to be held.
  266. */
  267. static void page_cache_delete_batch(struct address_space *mapping,
  268. struct pagevec *pvec)
  269. {
  270. XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
  271. int total_pages = 0;
  272. int i = 0;
  273. struct page *page;
  274. mapping_set_update(&xas, mapping);
  275. xas_for_each(&xas, page, ULONG_MAX) {
  276. if (i >= pagevec_count(pvec))
  277. break;
  278. /* A swap/dax/shadow entry got inserted? Skip it. */
  279. if (xa_is_value(page))
  280. continue;
  281. /*
  282. * A page got inserted in our range? Skip it. We have our
  283. * pages locked so they are protected from being removed.
  284. * If we see a page whose index is higher than ours, it
  285. * means our page has been removed, which shouldn't be
  286. * possible because we're holding the PageLock.
  287. */
  288. if (page != pvec->pages[i]) {
  289. VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index,
  290. page);
  291. continue;
  292. }
  293. WARN_ON_ONCE(!PageLocked(page));
  294. if (page->index == xas.xa_index)
  295. page->mapping = NULL;
  296. /* Leave page->index set: truncation lookup relies on it */
  297. /*
  298. * Move to the next page in the vector if this is a regular
  299. * page or the index is of the last sub-page of this compound
  300. * page.
  301. */
  302. if (page->index + compound_nr(page) - 1 == xas.xa_index)
  303. i++;
  304. xas_store(&xas, NULL);
  305. total_pages++;
  306. }
  307. mapping->nrpages -= total_pages;
  308. }
  309. void delete_from_page_cache_batch(struct address_space *mapping,
  310. struct pagevec *pvec)
  311. {
  312. int i;
  313. unsigned long flags;
  314. if (!pagevec_count(pvec))
  315. return;
  316. xa_lock_irqsave(&mapping->i_pages, flags);
  317. for (i = 0; i < pagevec_count(pvec); i++) {
  318. trace_mm_filemap_delete_from_page_cache(pvec->pages[i]);
  319. unaccount_page_cache_page(mapping, pvec->pages[i]);
  320. }
  321. page_cache_delete_batch(mapping, pvec);
  322. xa_unlock_irqrestore(&mapping->i_pages, flags);
  323. for (i = 0; i < pagevec_count(pvec); i++)
  324. page_cache_free_page(mapping, pvec->pages[i]);
  325. }
  326. int filemap_check_errors(struct address_space *mapping)
  327. {
  328. int ret = 0;
  329. /* Check for outstanding write errors */
  330. if (test_bit(AS_ENOSPC, &mapping->flags) &&
  331. test_and_clear_bit(AS_ENOSPC, &mapping->flags))
  332. ret = -ENOSPC;
  333. if (test_bit(AS_EIO, &mapping->flags) &&
  334. test_and_clear_bit(AS_EIO, &mapping->flags))
  335. ret = -EIO;
  336. return ret;
  337. }
  338. EXPORT_SYMBOL(filemap_check_errors);
  339. static int filemap_check_and_keep_errors(struct address_space *mapping)
  340. {
  341. /* Check for outstanding write errors */
  342. if (test_bit(AS_EIO, &mapping->flags))
  343. return -EIO;
  344. if (test_bit(AS_ENOSPC, &mapping->flags))
  345. return -ENOSPC;
  346. return 0;
  347. }
  348. /**
  349. * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  350. * @mapping: address space structure to write
  351. * @start: offset in bytes where the range starts
  352. * @end: offset in bytes where the range ends (inclusive)
  353. * @sync_mode: enable synchronous operation
  354. *
  355. * Start writeback against all of a mapping's dirty pages that lie
  356. * within the byte offsets <start, end> inclusive.
  357. *
  358. * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
  359. * opposed to a regular memory cleansing writeback. The difference between
  360. * these two operations is that if a dirty page/buffer is encountered, it must
  361. * be waited upon, and not just skipped over.
  362. *
  363. * Return: %0 on success, negative error code otherwise.
  364. */
  365. int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  366. loff_t end, int sync_mode)
  367. {
  368. int ret;
  369. struct writeback_control wbc = {
  370. .sync_mode = sync_mode,
  371. .nr_to_write = LONG_MAX,
  372. .range_start = start,
  373. .range_end = end,
  374. };
  375. if (!mapping_can_writeback(mapping) ||
  376. !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
  377. return 0;
  378. wbc_attach_fdatawrite_inode(&wbc, mapping->host);
  379. ret = do_writepages(mapping, &wbc);
  380. wbc_detach_inode(&wbc);
  381. return ret;
  382. }
  383. static inline int __filemap_fdatawrite(struct address_space *mapping,
  384. int sync_mode)
  385. {
  386. return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
  387. }
  388. int filemap_fdatawrite(struct address_space *mapping)
  389. {
  390. return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
  391. }
  392. EXPORT_SYMBOL(filemap_fdatawrite);
  393. int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  394. loff_t end)
  395. {
  396. return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
  397. }
  398. EXPORT_SYMBOL(filemap_fdatawrite_range);
  399. /**
  400. * filemap_flush - mostly a non-blocking flush
  401. * @mapping: target address_space
  402. *
  403. * This is a mostly non-blocking flush. Not suitable for data-integrity
  404. * purposes - I/O may not be started against all dirty pages.
  405. *
  406. * Return: %0 on success, negative error code otherwise.
  407. */
  408. int filemap_flush(struct address_space *mapping)
  409. {
  410. return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
  411. }
  412. EXPORT_SYMBOL(filemap_flush);
  413. /**
  414. * filemap_range_has_page - check if a page exists in range.
  415. * @mapping: address space within which to check
  416. * @start_byte: offset in bytes where the range starts
  417. * @end_byte: offset in bytes where the range ends (inclusive)
  418. *
  419. * Find at least one page in the range supplied, usually used to check if
  420. * direct writing in this range will trigger a writeback.
  421. *
  422. * Return: %true if at least one page exists in the specified range,
  423. * %false otherwise.
  424. */
  425. bool filemap_range_has_page(struct address_space *mapping,
  426. loff_t start_byte, loff_t end_byte)
  427. {
  428. struct page *page;
  429. XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
  430. pgoff_t max = end_byte >> PAGE_SHIFT;
  431. if (end_byte < start_byte)
  432. return false;
  433. rcu_read_lock();
  434. for (;;) {
  435. page = xas_find(&xas, max);
  436. if (xas_retry(&xas, page))
  437. continue;
  438. /* Shadow entries don't count */
  439. if (xa_is_value(page))
  440. continue;
  441. /*
  442. * We don't need to try to pin this page; we're about to
  443. * release the RCU lock anyway. It is enough to know that
  444. * there was a page here recently.
  445. */
  446. break;
  447. }
  448. rcu_read_unlock();
  449. return page != NULL;
  450. }
  451. EXPORT_SYMBOL(filemap_range_has_page);
  452. static void __filemap_fdatawait_range(struct address_space *mapping,
  453. loff_t start_byte, loff_t end_byte)
  454. {
  455. pgoff_t index = start_byte >> PAGE_SHIFT;
  456. pgoff_t end = end_byte >> PAGE_SHIFT;
  457. struct pagevec pvec;
  458. int nr_pages;
  459. if (end_byte < start_byte)
  460. return;
  461. pagevec_init(&pvec);
  462. while (index <= end) {
  463. unsigned i;
  464. nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
  465. end, PAGECACHE_TAG_WRITEBACK);
  466. if (!nr_pages)
  467. break;
  468. for (i = 0; i < nr_pages; i++) {
  469. struct page *page = pvec.pages[i];
  470. wait_on_page_writeback(page);
  471. ClearPageError(page);
  472. }
  473. pagevec_release(&pvec);
  474. cond_resched();
  475. }
  476. }
  477. /**
  478. * filemap_fdatawait_range - wait for writeback to complete
  479. * @mapping: address space structure to wait for
  480. * @start_byte: offset in bytes where the range starts
  481. * @end_byte: offset in bytes where the range ends (inclusive)
  482. *
  483. * Walk the list of under-writeback pages of the given address space
  484. * in the given range and wait for all of them. Check error status of
  485. * the address space and return it.
  486. *
  487. * Since the error status of the address space is cleared by this function,
  488. * callers are responsible for checking the return value and handling and/or
  489. * reporting the error.
  490. *
  491. * Return: error status of the address space.
  492. */
  493. int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
  494. loff_t end_byte)
  495. {
  496. __filemap_fdatawait_range(mapping, start_byte, end_byte);
  497. return filemap_check_errors(mapping);
  498. }
  499. EXPORT_SYMBOL(filemap_fdatawait_range);
  500. /**
  501. * filemap_fdatawait_range_keep_errors - wait for writeback to complete
  502. * @mapping: address space structure to wait for
  503. * @start_byte: offset in bytes where the range starts
  504. * @end_byte: offset in bytes where the range ends (inclusive)
  505. *
  506. * Walk the list of under-writeback pages of the given address space in the
  507. * given range and wait for all of them. Unlike filemap_fdatawait_range(),
  508. * this function does not clear error status of the address space.
  509. *
  510. * Use this function if callers don't handle errors themselves. Expected
  511. * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
  512. * fsfreeze(8)
  513. */
  514. int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
  515. loff_t start_byte, loff_t end_byte)
  516. {
  517. __filemap_fdatawait_range(mapping, start_byte, end_byte);
  518. return filemap_check_and_keep_errors(mapping);
  519. }
  520. EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors);
  521. /**
  522. * file_fdatawait_range - wait for writeback to complete
  523. * @file: file pointing to address space structure to wait for
  524. * @start_byte: offset in bytes where the range starts
  525. * @end_byte: offset in bytes where the range ends (inclusive)
  526. *
  527. * Walk the list of under-writeback pages of the address space that file
  528. * refers to, in the given range and wait for all of them. Check error
  529. * status of the address space vs. the file->f_wb_err cursor and return it.
  530. *
  531. * Since the error status of the file is advanced by this function,
  532. * callers are responsible for checking the return value and handling and/or
  533. * reporting the error.
  534. *
  535. * Return: error status of the address space vs. the file->f_wb_err cursor.
  536. */
  537. int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
  538. {
  539. struct address_space *mapping = file->f_mapping;
  540. __filemap_fdatawait_range(mapping, start_byte, end_byte);
  541. return file_check_and_advance_wb_err(file);
  542. }
  543. EXPORT_SYMBOL(file_fdatawait_range);
  544. /**
  545. * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
  546. * @mapping: address space structure to wait for
  547. *
  548. * Walk the list of under-writeback pages of the given address space
  549. * and wait for all of them. Unlike filemap_fdatawait(), this function
  550. * does not clear error status of the address space.
  551. *
  552. * Use this function if callers don't handle errors themselves. Expected
  553. * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
  554. * fsfreeze(8)
  555. *
  556. * Return: error status of the address space.
  557. */
  558. int filemap_fdatawait_keep_errors(struct address_space *mapping)
  559. {
  560. __filemap_fdatawait_range(mapping, 0, LLONG_MAX);
  561. return filemap_check_and_keep_errors(mapping);
  562. }
  563. EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
  564. /* Returns true if writeback might be needed or already in progress. */
  565. static bool mapping_needs_writeback(struct address_space *mapping)
  566. {
  567. if (dax_mapping(mapping))
  568. return mapping->nrexceptional;
  569. return mapping->nrpages;
  570. }
  571. /**
  572. * filemap_write_and_wait_range - write out & wait on a file range
  573. * @mapping: the address_space for the pages
  574. * @lstart: offset in bytes where the range starts
  575. * @lend: offset in bytes where the range ends (inclusive)
  576. *
  577. * Write out and wait upon file offsets lstart->lend, inclusive.
  578. *
  579. * Note that @lend is inclusive (describes the last byte to be written) so
  580. * that this function can be used to write to the very end-of-file (end = -1).
  581. *
  582. * Return: error status of the address space.
  583. */
  584. int filemap_write_and_wait_range(struct address_space *mapping,
  585. loff_t lstart, loff_t lend)
  586. {
  587. int err = 0;
  588. if (mapping_needs_writeback(mapping)) {
  589. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  590. WB_SYNC_ALL);
  591. /*
  592. * Even if the above returned error, the pages may be
  593. * written partially (e.g. -ENOSPC), so we wait for it.
  594. * But the -EIO is special case, it may indicate the worst
  595. * thing (e.g. bug) happened, so we avoid waiting for it.
  596. */
  597. if (err != -EIO) {
  598. int err2 = filemap_fdatawait_range(mapping,
  599. lstart, lend);
  600. if (!err)
  601. err = err2;
  602. } else {
  603. /* Clear any previously stored errors */
  604. filemap_check_errors(mapping);
  605. }
  606. } else {
  607. err = filemap_check_errors(mapping);
  608. }
  609. return err;
  610. }
  611. EXPORT_SYMBOL(filemap_write_and_wait_range);
  612. void __filemap_set_wb_err(struct address_space *mapping, int err)
  613. {
  614. errseq_t eseq = errseq_set(&mapping->wb_err, err);
  615. trace_filemap_set_wb_err(mapping, eseq);
  616. }
  617. EXPORT_SYMBOL(__filemap_set_wb_err);
  618. /**
  619. * file_check_and_advance_wb_err - report wb error (if any) that was previously
  620. * and advance wb_err to current one
  621. * @file: struct file on which the error is being reported
  622. *
  623. * When userland calls fsync (or something like nfsd does the equivalent), we
  624. * want to report any writeback errors that occurred since the last fsync (or
  625. * since the file was opened if there haven't been any).
  626. *
  627. * Grab the wb_err from the mapping. If it matches what we have in the file,
  628. * then just quickly return 0. The file is all caught up.
  629. *
  630. * If it doesn't match, then take the mapping value, set the "seen" flag in
  631. * it and try to swap it into place. If it works, or another task beat us
  632. * to it with the new value, then update the f_wb_err and return the error
  633. * portion. The error at this point must be reported via proper channels
  634. * (a'la fsync, or NFS COMMIT operation, etc.).
  635. *
  636. * While we handle mapping->wb_err with atomic operations, the f_wb_err
  637. * value is protected by the f_lock since we must ensure that it reflects
  638. * the latest value swapped in for this file descriptor.
  639. *
  640. * Return: %0 on success, negative error code otherwise.
  641. */
  642. int file_check_and_advance_wb_err(struct file *file)
  643. {
  644. int err = 0;
  645. errseq_t old = READ_ONCE(file->f_wb_err);
  646. struct address_space *mapping = file->f_mapping;
  647. /* Locklessly handle the common case where nothing has changed */
  648. if (errseq_check(&mapping->wb_err, old)) {
  649. /* Something changed, must use slow path */
  650. spin_lock(&file->f_lock);
  651. old = file->f_wb_err;
  652. err = errseq_check_and_advance(&mapping->wb_err,
  653. &file->f_wb_err);
  654. trace_file_check_and_advance_wb_err(file, old);
  655. spin_unlock(&file->f_lock);
  656. }
  657. /*
  658. * We're mostly using this function as a drop in replacement for
  659. * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
  660. * that the legacy code would have had on these flags.
  661. */
  662. clear_bit(AS_EIO, &mapping->flags);
  663. clear_bit(AS_ENOSPC, &mapping->flags);
  664. return err;
  665. }
  666. EXPORT_SYMBOL(file_check_and_advance_wb_err);
  667. /**
  668. * file_write_and_wait_range - write out & wait on a file range
  669. * @file: file pointing to address_space with pages
  670. * @lstart: offset in bytes where the range starts
  671. * @lend: offset in bytes where the range ends (inclusive)
  672. *
  673. * Write out and wait upon file offsets lstart->lend, inclusive.
  674. *
  675. * Note that @lend is inclusive (describes the last byte to be written) so
  676. * that this function can be used to write to the very end-of-file (end = -1).
  677. *
  678. * After writing out and waiting on the data, we check and advance the
  679. * f_wb_err cursor to the latest value, and return any errors detected there.
  680. *
  681. * Return: %0 on success, negative error code otherwise.
  682. */
  683. int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
  684. {
  685. int err = 0, err2;
  686. struct address_space *mapping = file->f_mapping;
  687. if (mapping_needs_writeback(mapping)) {
  688. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  689. WB_SYNC_ALL);
  690. /* See comment of filemap_write_and_wait() */
  691. if (err != -EIO)
  692. __filemap_fdatawait_range(mapping, lstart, lend);
  693. }
  694. err2 = file_check_and_advance_wb_err(file);
  695. if (!err)
  696. err = err2;
  697. return err;
  698. }
  699. EXPORT_SYMBOL(file_write_and_wait_range);
  700. /**
  701. * replace_page_cache_page - replace a pagecache page with a new one
  702. * @old: page to be replaced
  703. * @new: page to replace with
  704. * @gfp_mask: allocation mode
  705. *
  706. * This function replaces a page in the pagecache with a new one. On
  707. * success it acquires the pagecache reference for the new page and
  708. * drops it for the old page. Both the old and new pages must be
  709. * locked. This function does not add the new page to the LRU, the
  710. * caller must do that.
  711. *
  712. * The remove + add is atomic. This function cannot fail.
  713. *
  714. * Return: %0
  715. */
  716. int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
  717. {
  718. struct address_space *mapping = old->mapping;
  719. void (*freepage)(struct page *) = mapping->a_ops->freepage;
  720. pgoff_t offset = old->index;
  721. XA_STATE(xas, &mapping->i_pages, offset);
  722. unsigned long flags;
  723. VM_BUG_ON_PAGE(!PageLocked(old), old);
  724. VM_BUG_ON_PAGE(!PageLocked(new), new);
  725. VM_BUG_ON_PAGE(new->mapping, new);
  726. get_page(new);
  727. new->mapping = mapping;
  728. new->index = offset;
  729. mem_cgroup_migrate(old, new);
  730. xas_lock_irqsave(&xas, flags);
  731. xas_store(&xas, new);
  732. old->mapping = NULL;
  733. /* hugetlb pages do not participate in page cache accounting. */
  734. if (!PageHuge(old))
  735. __dec_lruvec_page_state(old, NR_FILE_PAGES);
  736. if (!PageHuge(new))
  737. __inc_lruvec_page_state(new, NR_FILE_PAGES);
  738. if (PageSwapBacked(old))
  739. __dec_lruvec_page_state(old, NR_SHMEM);
  740. if (PageSwapBacked(new))
  741. __inc_lruvec_page_state(new, NR_SHMEM);
  742. xas_unlock_irqrestore(&xas, flags);
  743. if (freepage)
  744. freepage(old);
  745. put_page(old);
  746. return 0;
  747. }
  748. EXPORT_SYMBOL_GPL(replace_page_cache_page);
  749. noinline int __add_to_page_cache_locked(struct page *page,
  750. struct address_space *mapping,
  751. pgoff_t offset, gfp_t gfp,
  752. void **shadowp)
  753. {
  754. XA_STATE(xas, &mapping->i_pages, offset);
  755. int huge = PageHuge(page);
  756. int error;
  757. bool charged = false;
  758. VM_BUG_ON_PAGE(!PageLocked(page), page);
  759. VM_BUG_ON_PAGE(PageSwapBacked(page), page);
  760. mapping_set_update(&xas, mapping);
  761. get_page(page);
  762. page->mapping = mapping;
  763. page->index = offset;
  764. if (!huge) {
  765. error = mem_cgroup_charge(page, current->mm, gfp);
  766. if (error)
  767. goto error;
  768. charged = true;
  769. }
  770. gfp &= GFP_RECLAIM_MASK;
  771. do {
  772. unsigned int order = xa_get_order(xas.xa, xas.xa_index);
  773. void *entry, *old = NULL;
  774. if (order > thp_order(page))
  775. xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),
  776. order, gfp);
  777. xas_lock_irq(&xas);
  778. xas_for_each_conflict(&xas, entry) {
  779. old = entry;
  780. if (!xa_is_value(entry)) {
  781. xas_set_err(&xas, -EEXIST);
  782. goto unlock;
  783. }
  784. }
  785. if (old) {
  786. if (shadowp)
  787. *shadowp = old;
  788. /* entry may have been split before we acquired lock */
  789. order = xa_get_order(xas.xa, xas.xa_index);
  790. if (order > thp_order(page)) {
  791. xas_split(&xas, old, order);
  792. xas_reset(&xas);
  793. }
  794. }
  795. xas_store(&xas, page);
  796. if (xas_error(&xas))
  797. goto unlock;
  798. if (old)
  799. mapping->nrexceptional--;
  800. mapping->nrpages++;
  801. /* hugetlb pages do not participate in page cache accounting */
  802. if (!huge)
  803. __inc_lruvec_page_state(page, NR_FILE_PAGES);
  804. unlock:
  805. xas_unlock_irq(&xas);
  806. } while (xas_nomem(&xas, gfp));
  807. if (xas_error(&xas)) {
  808. error = xas_error(&xas);
  809. if (charged)
  810. mem_cgroup_uncharge(page);
  811. goto error;
  812. }
  813. trace_mm_filemap_add_to_page_cache(page);
  814. return 0;
  815. error:
  816. page->mapping = NULL;
  817. /* Leave page->index set: truncation relies upon it */
  818. put_page(page);
  819. return error;
  820. }
  821. ALLOW_ERROR_INJECTION(__add_to_page_cache_locked, ERRNO);
  822. /**
  823. * add_to_page_cache_locked - add a locked page to the pagecache
  824. * @page: page to add
  825. * @mapping: the page's address_space
  826. * @offset: page index
  827. * @gfp_mask: page allocation mode
  828. *
  829. * This function is used to add a page to the pagecache. It must be locked.
  830. * This function does not add the page to the LRU. The caller must do that.
  831. *
  832. * Return: %0 on success, negative error code otherwise.
  833. */
  834. int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
  835. pgoff_t offset, gfp_t gfp_mask)
  836. {
  837. return __add_to_page_cache_locked(page, mapping, offset,
  838. gfp_mask, NULL);
  839. }
  840. EXPORT_SYMBOL(add_to_page_cache_locked);
  841. int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
  842. pgoff_t offset, gfp_t gfp_mask)
  843. {
  844. void *shadow = NULL;
  845. int ret;
  846. __SetPageLocked(page);
  847. ret = __add_to_page_cache_locked(page, mapping, offset,
  848. gfp_mask, &shadow);
  849. if (unlikely(ret))
  850. __ClearPageLocked(page);
  851. else {
  852. /*
  853. * The page might have been evicted from cache only
  854. * recently, in which case it should be activated like
  855. * any other repeatedly accessed page.
  856. * The exception is pages getting rewritten; evicting other
  857. * data from the working set, only to cache data that will
  858. * get overwritten with something else, is a waste of memory.
  859. */
  860. WARN_ON_ONCE(PageActive(page));
  861. if (!(gfp_mask & __GFP_WRITE) && shadow)
  862. workingset_refault(page, shadow);
  863. lru_cache_add(page);
  864. }
  865. return ret;
  866. }
  867. EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
  868. #ifdef CONFIG_NUMA
  869. struct page *__page_cache_alloc(gfp_t gfp)
  870. {
  871. int n;
  872. struct page *page;
  873. if (cpuset_do_page_mem_spread()) {
  874. unsigned int cpuset_mems_cookie;
  875. do {
  876. cpuset_mems_cookie = read_mems_allowed_begin();
  877. n = cpuset_mem_spread_node();
  878. page = __alloc_pages_node(n, gfp, 0);
  879. } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
  880. return page;
  881. }
  882. return alloc_pages(gfp, 0);
  883. }
  884. EXPORT_SYMBOL(__page_cache_alloc);
  885. #endif
  886. /*
  887. * In order to wait for pages to become available there must be
  888. * waitqueues associated with pages. By using a hash table of
  889. * waitqueues where the bucket discipline is to maintain all
  890. * waiters on the same queue and wake all when any of the pages
  891. * become available, and for the woken contexts to check to be
  892. * sure the appropriate page became available, this saves space
  893. * at a cost of "thundering herd" phenomena during rare hash
  894. * collisions.
  895. */
  896. #define PAGE_WAIT_TABLE_BITS 8
  897. #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
  898. static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
  899. static wait_queue_head_t *page_waitqueue(struct page *page)
  900. {
  901. return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
  902. }
  903. void __init pagecache_init(void)
  904. {
  905. int i;
  906. for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
  907. init_waitqueue_head(&page_wait_table[i]);
  908. page_writeback_init();
  909. }
  910. /*
  911. * The page wait code treats the "wait->flags" somewhat unusually, because
  912. * we have multiple different kinds of waits, not just the usual "exclusive"
  913. * one.
  914. *
  915. * We have:
  916. *
  917. * (a) no special bits set:
  918. *
  919. * We're just waiting for the bit to be released, and when a waker
  920. * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up,
  921. * and remove it from the wait queue.
  922. *
  923. * Simple and straightforward.
  924. *
  925. * (b) WQ_FLAG_EXCLUSIVE:
  926. *
  927. * The waiter is waiting to get the lock, and only one waiter should
  928. * be woken up to avoid any thundering herd behavior. We'll set the
  929. * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue.
  930. *
  931. * This is the traditional exclusive wait.
  932. *
  933. * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM:
  934. *
  935. * The waiter is waiting to get the bit, and additionally wants the
  936. * lock to be transferred to it for fair lock behavior. If the lock
  937. * cannot be taken, we stop walking the wait queue without waking
  938. * the waiter.
  939. *
  940. * This is the "fair lock handoff" case, and in addition to setting
  941. * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see
  942. * that it now has the lock.
  943. */
  944. static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
  945. {
  946. unsigned int flags;
  947. struct wait_page_key *key = arg;
  948. struct wait_page_queue *wait_page
  949. = container_of(wait, struct wait_page_queue, wait);
  950. if (!wake_page_match(wait_page, key))
  951. return 0;
  952. /*
  953. * If it's a lock handoff wait, we get the bit for it, and
  954. * stop walking (and do not wake it up) if we can't.
  955. */
  956. flags = wait->flags;
  957. if (flags & WQ_FLAG_EXCLUSIVE) {
  958. if (test_bit(key->bit_nr, &key->page->flags))
  959. return -1;
  960. if (flags & WQ_FLAG_CUSTOM) {
  961. if (test_and_set_bit(key->bit_nr, &key->page->flags))
  962. return -1;
  963. flags |= WQ_FLAG_DONE;
  964. }
  965. }
  966. /*
  967. * We are holding the wait-queue lock, but the waiter that
  968. * is waiting for this will be checking the flags without
  969. * any locking.
  970. *
  971. * So update the flags atomically, and wake up the waiter
  972. * afterwards to avoid any races. This store-release pairs
  973. * with the load-acquire in wait_on_page_bit_common().
  974. */
  975. smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
  976. wake_up_state(wait->private, mode);
  977. /*
  978. * Ok, we have successfully done what we're waiting for,
  979. * and we can unconditionally remove the wait entry.
  980. *
  981. * Note that this pairs with the "finish_wait()" in the
  982. * waiter, and has to be the absolute last thing we do.
  983. * After this list_del_init(&wait->entry) the wait entry
  984. * might be de-allocated and the process might even have
  985. * exited.
  986. */
  987. list_del_init_careful(&wait->entry);
  988. return (flags & WQ_FLAG_EXCLUSIVE) != 0;
  989. }
  990. static void wake_up_page_bit(struct page *page, int bit_nr)
  991. {
  992. wait_queue_head_t *q = page_waitqueue(page);
  993. struct wait_page_key key;
  994. unsigned long flags;
  995. wait_queue_entry_t bookmark;
  996. key.page = page;
  997. key.bit_nr = bit_nr;
  998. key.page_match = 0;
  999. bookmark.flags = 0;
  1000. bookmark.private = NULL;
  1001. bookmark.func = NULL;
  1002. INIT_LIST_HEAD(&bookmark.entry);
  1003. spin_lock_irqsave(&q->lock, flags);
  1004. __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
  1005. while (bookmark.flags & WQ_FLAG_BOOKMARK) {
  1006. /*
  1007. * Take a breather from holding the lock,
  1008. * allow pages that finish wake up asynchronously
  1009. * to acquire the lock and remove themselves
  1010. * from wait queue
  1011. */
  1012. spin_unlock_irqrestore(&q->lock, flags);
  1013. cpu_relax();
  1014. spin_lock_irqsave(&q->lock, flags);
  1015. __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
  1016. }
  1017. /*
  1018. * It is possible for other pages to have collided on the waitqueue
  1019. * hash, so in that case check for a page match. That prevents a long-
  1020. * term waiter
  1021. *
  1022. * It is still possible to miss a case here, when we woke page waiters
  1023. * and removed them from the waitqueue, but there are still other
  1024. * page waiters.
  1025. */
  1026. if (!waitqueue_active(q) || !key.page_match) {
  1027. ClearPageWaiters(page);
  1028. /*
  1029. * It's possible to miss clearing Waiters here, when we woke
  1030. * our page waiters, but the hashed waitqueue has waiters for
  1031. * other pages on it.
  1032. *
  1033. * That's okay, it's a rare case. The next waker will clear it.
  1034. */
  1035. }
  1036. spin_unlock_irqrestore(&q->lock, flags);
  1037. }
  1038. static void wake_up_page(struct page *page, int bit)
  1039. {
  1040. if (!PageWaiters(page))
  1041. return;
  1042. wake_up_page_bit(page, bit);
  1043. }
  1044. /*
  1045. * A choice of three behaviors for wait_on_page_bit_common():
  1046. */
  1047. enum behavior {
  1048. EXCLUSIVE, /* Hold ref to page and take the bit when woken, like
  1049. * __lock_page() waiting on then setting PG_locked.
  1050. */
  1051. SHARED, /* Hold ref to page and check the bit when woken, like
  1052. * wait_on_page_writeback() waiting on PG_writeback.
  1053. */
  1054. DROP, /* Drop ref to page before wait, no check when woken,
  1055. * like put_and_wait_on_page_locked() on PG_locked.
  1056. */
  1057. };
  1058. /*
  1059. * Attempt to check (or get) the page bit, and mark us done
  1060. * if successful.
  1061. */
  1062. static inline bool trylock_page_bit_common(struct page *page, int bit_nr,
  1063. struct wait_queue_entry *wait)
  1064. {
  1065. if (wait->flags & WQ_FLAG_EXCLUSIVE) {
  1066. if (test_and_set_bit(bit_nr, &page->flags))
  1067. return false;
  1068. } else if (test_bit(bit_nr, &page->flags))
  1069. return false;
  1070. wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
  1071. return true;
  1072. }
  1073. /* How many times do we accept lock stealing from under a waiter? */
  1074. int sysctl_page_lock_unfairness = 5;
  1075. static inline __sched int wait_on_page_bit_common(wait_queue_head_t *q,
  1076. struct page *page, int bit_nr, int state, enum behavior behavior)
  1077. {
  1078. int unfairness = sysctl_page_lock_unfairness;
  1079. struct wait_page_queue wait_page;
  1080. wait_queue_entry_t *wait = &wait_page.wait;
  1081. bool thrashing = false;
  1082. bool delayacct = false;
  1083. unsigned long pflags;
  1084. if (bit_nr == PG_locked &&
  1085. !PageUptodate(page) && PageWorkingset(page)) {
  1086. if (!PageSwapBacked(page)) {
  1087. delayacct_thrashing_start();
  1088. delayacct = true;
  1089. }
  1090. psi_memstall_enter(&pflags);
  1091. thrashing = true;
  1092. }
  1093. init_wait(wait);
  1094. wait->func = wake_page_function;
  1095. wait_page.page = page;
  1096. wait_page.bit_nr = bit_nr;
  1097. repeat:
  1098. wait->flags = 0;
  1099. if (behavior == EXCLUSIVE) {
  1100. wait->flags = WQ_FLAG_EXCLUSIVE;
  1101. if (--unfairness < 0)
  1102. wait->flags |= WQ_FLAG_CUSTOM;
  1103. }
  1104. /*
  1105. * Do one last check whether we can get the
  1106. * page bit synchronously.
  1107. *
  1108. * Do the SetPageWaiters() marking before that
  1109. * to let any waker we _just_ missed know they
  1110. * need to wake us up (otherwise they'll never
  1111. * even go to the slow case that looks at the
  1112. * page queue), and add ourselves to the wait
  1113. * queue if we need to sleep.
  1114. *
  1115. * This part needs to be done under the queue
  1116. * lock to avoid races.
  1117. */
  1118. spin_lock_irq(&q->lock);
  1119. SetPageWaiters(page);
  1120. if (!trylock_page_bit_common(page, bit_nr, wait))
  1121. __add_wait_queue_entry_tail(q, wait);
  1122. spin_unlock_irq(&q->lock);
  1123. /*
  1124. * From now on, all the logic will be based on
  1125. * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to
  1126. * see whether the page bit testing has already
  1127. * been done by the wake function.
  1128. *
  1129. * We can drop our reference to the page.
  1130. */
  1131. if (behavior == DROP)
  1132. put_page(page);
  1133. /*
  1134. * Note that until the "finish_wait()", or until
  1135. * we see the WQ_FLAG_WOKEN flag, we need to
  1136. * be very careful with the 'wait->flags', because
  1137. * we may race with a waker that sets them.
  1138. */
  1139. for (;;) {
  1140. unsigned int flags;
  1141. set_current_state(state);
  1142. /* Loop until we've been woken or interrupted */
  1143. flags = smp_load_acquire(&wait->flags);
  1144. if (!(flags & WQ_FLAG_WOKEN)) {
  1145. if (signal_pending_state(state, current))
  1146. break;
  1147. io_schedule();
  1148. continue;
  1149. }
  1150. /* If we were non-exclusive, we're done */
  1151. if (behavior != EXCLUSIVE)
  1152. break;
  1153. /* If the waker got the lock for us, we're done */
  1154. if (flags & WQ_FLAG_DONE)
  1155. break;
  1156. /*
  1157. * Otherwise, if we're getting the lock, we need to
  1158. * try to get it ourselves.
  1159. *
  1160. * And if that fails, we'll have to retry this all.
  1161. */
  1162. if (unlikely(test_and_set_bit(bit_nr, &page->flags)))
  1163. goto repeat;
  1164. wait->flags |= WQ_FLAG_DONE;
  1165. break;
  1166. }
  1167. /*
  1168. * If a signal happened, this 'finish_wait()' may remove the last
  1169. * waiter from the wait-queues, but the PageWaiters bit will remain
  1170. * set. That's ok. The next wakeup will take care of it, and trying
  1171. * to do it here would be difficult and prone to races.
  1172. */
  1173. finish_wait(q, wait);
  1174. if (thrashing) {
  1175. if (delayacct)
  1176. delayacct_thrashing_end();
  1177. psi_memstall_leave(&pflags);
  1178. }
  1179. /*
  1180. * NOTE! The wait->flags weren't stable until we've done the
  1181. * 'finish_wait()', and we could have exited the loop above due
  1182. * to a signal, and had a wakeup event happen after the signal
  1183. * test but before the 'finish_wait()'.
  1184. *
  1185. * So only after the finish_wait() can we reliably determine
  1186. * if we got woken up or not, so we can now figure out the final
  1187. * return value based on that state without races.
  1188. *
  1189. * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
  1190. * waiter, but an exclusive one requires WQ_FLAG_DONE.
  1191. */
  1192. if (behavior == EXCLUSIVE)
  1193. return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
  1194. return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
  1195. }
  1196. __sched void wait_on_page_bit(struct page *page, int bit_nr)
  1197. {
  1198. wait_queue_head_t *q = page_waitqueue(page);
  1199. wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
  1200. }
  1201. EXPORT_SYMBOL(wait_on_page_bit);
  1202. __sched int wait_on_page_bit_killable(struct page *page, int bit_nr)
  1203. {
  1204. wait_queue_head_t *q = page_waitqueue(page);
  1205. return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, SHARED);
  1206. }
  1207. EXPORT_SYMBOL(wait_on_page_bit_killable);
  1208. static int __wait_on_page_locked_async(struct page *page,
  1209. struct wait_page_queue *wait, bool set)
  1210. {
  1211. struct wait_queue_head *q = page_waitqueue(page);
  1212. int ret = 0;
  1213. wait->page = page;
  1214. wait->bit_nr = PG_locked;
  1215. spin_lock_irq(&q->lock);
  1216. __add_wait_queue_entry_tail(q, &wait->wait);
  1217. SetPageWaiters(page);
  1218. if (set)
  1219. ret = !trylock_page(page);
  1220. else
  1221. ret = PageLocked(page);
  1222. /*
  1223. * If we were succesful now, we know we're still on the
  1224. * waitqueue as we're still under the lock. This means it's
  1225. * safe to remove and return success, we know the callback
  1226. * isn't going to trigger.
  1227. */
  1228. if (!ret)
  1229. __remove_wait_queue(q, &wait->wait);
  1230. else
  1231. ret = -EIOCBQUEUED;
  1232. spin_unlock_irq(&q->lock);
  1233. return ret;
  1234. }
  1235. static int wait_on_page_locked_async(struct page *page,
  1236. struct wait_page_queue *wait)
  1237. {
  1238. if (!PageLocked(page))
  1239. return 0;
  1240. return __wait_on_page_locked_async(compound_head(page), wait, false);
  1241. }
  1242. /**
  1243. * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
  1244. * @page: The page to wait for.
  1245. *
  1246. * The caller should hold a reference on @page. They expect the page to
  1247. * become unlocked relatively soon, but do not wish to hold up migration
  1248. * (for example) by holding the reference while waiting for the page to
  1249. * come unlocked. After this function returns, the caller should not
  1250. * dereference @page.
  1251. */
  1252. void put_and_wait_on_page_locked(struct page *page)
  1253. {
  1254. wait_queue_head_t *q;
  1255. page = compound_head(page);
  1256. q = page_waitqueue(page);
  1257. wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, DROP);
  1258. }
  1259. /**
  1260. * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
  1261. * @page: Page defining the wait queue of interest
  1262. * @waiter: Waiter to add to the queue
  1263. *
  1264. * Add an arbitrary @waiter to the wait queue for the nominated @page.
  1265. */
  1266. void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
  1267. {
  1268. wait_queue_head_t *q = page_waitqueue(page);
  1269. unsigned long flags;
  1270. spin_lock_irqsave(&q->lock, flags);
  1271. __add_wait_queue_entry_tail(q, waiter);
  1272. SetPageWaiters(page);
  1273. spin_unlock_irqrestore(&q->lock, flags);
  1274. }
  1275. EXPORT_SYMBOL_GPL(add_page_wait_queue);
  1276. #ifndef clear_bit_unlock_is_negative_byte
  1277. /*
  1278. * PG_waiters is the high bit in the same byte as PG_lock.
  1279. *
  1280. * On x86 (and on many other architectures), we can clear PG_lock and
  1281. * test the sign bit at the same time. But if the architecture does
  1282. * not support that special operation, we just do this all by hand
  1283. * instead.
  1284. *
  1285. * The read of PG_waiters has to be after (or concurrently with) PG_locked
  1286. * being cleared, but a memory barrier should be unnecessary since it is
  1287. * in the same byte as PG_locked.
  1288. */
  1289. static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
  1290. {
  1291. clear_bit_unlock(nr, mem);
  1292. /* smp_mb__after_atomic(); */
  1293. return test_bit(PG_waiters, mem);
  1294. }
  1295. #endif
  1296. /**
  1297. * unlock_page - unlock a locked page
  1298. * @page: the page
  1299. *
  1300. * Unlocks the page and wakes up sleepers in wait_on_page_locked().
  1301. * Also wakes sleepers in wait_on_page_writeback() because the wakeup
  1302. * mechanism between PageLocked pages and PageWriteback pages is shared.
  1303. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  1304. *
  1305. * Note that this depends on PG_waiters being the sign bit in the byte
  1306. * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
  1307. * clear the PG_locked bit and test PG_waiters at the same time fairly
  1308. * portably (architectures that do LL/SC can test any bit, while x86 can
  1309. * test the sign bit).
  1310. */
  1311. void unlock_page(struct page *page)
  1312. {
  1313. BUILD_BUG_ON(PG_waiters != 7);
  1314. page = compound_head(page);
  1315. VM_BUG_ON_PAGE(!PageLocked(page), page);
  1316. if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags))
  1317. wake_up_page_bit(page, PG_locked);
  1318. }
  1319. EXPORT_SYMBOL(unlock_page);
  1320. /**
  1321. * end_page_writeback - end writeback against a page
  1322. * @page: the page
  1323. */
  1324. void end_page_writeback(struct page *page)
  1325. {
  1326. /*
  1327. * TestClearPageReclaim could be used here but it is an atomic
  1328. * operation and overkill in this particular case. Failing to
  1329. * shuffle a page marked for immediate reclaim is too mild to
  1330. * justify taking an atomic operation penalty at the end of
  1331. * ever page writeback.
  1332. */
  1333. if (PageReclaim(page)) {
  1334. ClearPageReclaim(page);
  1335. rotate_reclaimable_page(page);
  1336. }
  1337. /*
  1338. * Writeback does not hold a page reference of its own, relying
  1339. * on truncation to wait for the clearing of PG_writeback.
  1340. * But here we must make sure that the page is not freed and
  1341. * reused before the wake_up_page().
  1342. */
  1343. get_page(page);
  1344. if (!test_clear_page_writeback(page))
  1345. BUG();
  1346. smp_mb__after_atomic();
  1347. wake_up_page(page, PG_writeback);
  1348. put_page(page);
  1349. }
  1350. EXPORT_SYMBOL(end_page_writeback);
  1351. /*
  1352. * After completing I/O on a page, call this routine to update the page
  1353. * flags appropriately
  1354. */
  1355. void page_endio(struct page *page, bool is_write, int err)
  1356. {
  1357. if (!is_write) {
  1358. if (!err) {
  1359. SetPageUptodate(page);
  1360. } else {
  1361. ClearPageUptodate(page);
  1362. SetPageError(page);
  1363. }
  1364. unlock_page(page);
  1365. } else {
  1366. if (err) {
  1367. struct address_space *mapping;
  1368. SetPageError(page);
  1369. mapping = page_mapping(page);
  1370. if (mapping)
  1371. mapping_set_error(mapping, err);
  1372. }
  1373. end_page_writeback(page);
  1374. }
  1375. }
  1376. EXPORT_SYMBOL_GPL(page_endio);
  1377. /**
  1378. * __lock_page - get a lock on the page, assuming we need to sleep to get it
  1379. * @__page: the page to lock
  1380. */
  1381. __sched void __lock_page(struct page *__page)
  1382. {
  1383. struct page *page = compound_head(__page);
  1384. wait_queue_head_t *q = page_waitqueue(page);
  1385. wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE,
  1386. EXCLUSIVE);
  1387. }
  1388. EXPORT_SYMBOL(__lock_page);
  1389. __sched int __lock_page_killable(struct page *__page)
  1390. {
  1391. struct page *page = compound_head(__page);
  1392. wait_queue_head_t *q = page_waitqueue(page);
  1393. return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE,
  1394. EXCLUSIVE);
  1395. }
  1396. EXPORT_SYMBOL_GPL(__lock_page_killable);
  1397. __sched int __lock_page_async(struct page *page, struct wait_page_queue *wait)
  1398. {
  1399. return __wait_on_page_locked_async(page, wait, true);
  1400. }
  1401. /*
  1402. * Return values:
  1403. * 1 - page is locked; mmap_lock is still held.
  1404. * 0 - page is not locked.
  1405. * mmap_lock has been released (mmap_read_unlock(), unless flags had both
  1406. * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
  1407. * which case mmap_lock is still held.
  1408. *
  1409. * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
  1410. * with the page locked and the mmap_lock unperturbed.
  1411. */
  1412. __sched int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
  1413. unsigned int flags)
  1414. {
  1415. if (fault_flag_allow_retry_first(flags)) {
  1416. /*
  1417. * CAUTION! In this case, mmap_lock is not released
  1418. * even though return 0.
  1419. */
  1420. if (flags & FAULT_FLAG_RETRY_NOWAIT)
  1421. return 0;
  1422. mmap_read_unlock(mm);
  1423. if (flags & FAULT_FLAG_KILLABLE)
  1424. wait_on_page_locked_killable(page);
  1425. else
  1426. wait_on_page_locked(page);
  1427. return 0;
  1428. } else {
  1429. if (flags & FAULT_FLAG_KILLABLE) {
  1430. int ret;
  1431. ret = __lock_page_killable(page);
  1432. if (ret) {
  1433. mmap_read_unlock(mm);
  1434. return 0;
  1435. }
  1436. } else
  1437. __lock_page(page);
  1438. return 1;
  1439. }
  1440. }
  1441. /**
  1442. * page_cache_next_miss() - Find the next gap in the page cache.
  1443. * @mapping: Mapping.
  1444. * @index: Index.
  1445. * @max_scan: Maximum range to search.
  1446. *
  1447. * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
  1448. * gap with the lowest index.
  1449. *
  1450. * This function may be called under the rcu_read_lock. However, this will
  1451. * not atomically search a snapshot of the cache at a single point in time.
  1452. * For example, if a gap is created at index 5, then subsequently a gap is
  1453. * created at index 10, page_cache_next_miss covering both indices may
  1454. * return 10 if called under the rcu_read_lock.
  1455. *
  1456. * Return: The index of the gap if found, otherwise an index outside the
  1457. * range specified (in which case 'return - index >= max_scan' will be true).
  1458. * In the rare case of index wrap-around, 0 will be returned.
  1459. */
  1460. pgoff_t page_cache_next_miss(struct address_space *mapping,
  1461. pgoff_t index, unsigned long max_scan)
  1462. {
  1463. XA_STATE(xas, &mapping->i_pages, index);
  1464. while (max_scan--) {
  1465. void *entry = xas_next(&xas);
  1466. if (!entry || xa_is_value(entry))
  1467. break;
  1468. if (xas.xa_index == 0)
  1469. break;
  1470. }
  1471. return xas.xa_index;
  1472. }
  1473. EXPORT_SYMBOL(page_cache_next_miss);
  1474. /**
  1475. * page_cache_prev_miss() - Find the previous gap in the page cache.
  1476. * @mapping: Mapping.
  1477. * @index: Index.
  1478. * @max_scan: Maximum range to search.
  1479. *
  1480. * Search the range [max(index - max_scan + 1, 0), index] for the
  1481. * gap with the highest index.
  1482. *
  1483. * This function may be called under the rcu_read_lock. However, this will
  1484. * not atomically search a snapshot of the cache at a single point in time.
  1485. * For example, if a gap is created at index 10, then subsequently a gap is
  1486. * created at index 5, page_cache_prev_miss() covering both indices may
  1487. * return 5 if called under the rcu_read_lock.
  1488. *
  1489. * Return: The index of the gap if found, otherwise an index outside the
  1490. * range specified (in which case 'index - return >= max_scan' will be true).
  1491. * In the rare case of wrap-around, ULONG_MAX will be returned.
  1492. */
  1493. pgoff_t page_cache_prev_miss(struct address_space *mapping,
  1494. pgoff_t index, unsigned long max_scan)
  1495. {
  1496. XA_STATE(xas, &mapping->i_pages, index);
  1497. while (max_scan--) {
  1498. void *entry = xas_prev(&xas);
  1499. if (!entry || xa_is_value(entry))
  1500. break;
  1501. if (xas.xa_index == ULONG_MAX)
  1502. break;
  1503. }
  1504. return xas.xa_index;
  1505. }
  1506. EXPORT_SYMBOL(page_cache_prev_miss);
  1507. /**
  1508. * find_get_entry - find and get a page cache entry
  1509. * @mapping: the address_space to search
  1510. * @index: The page cache index.
  1511. *
  1512. * Looks up the page cache slot at @mapping & @offset. If there is a
  1513. * page cache page, the head page is returned with an increased refcount.
  1514. *
  1515. * If the slot holds a shadow entry of a previously evicted page, or a
  1516. * swap entry from shmem/tmpfs, it is returned.
  1517. *
  1518. * Return: The head page or shadow entry, %NULL if nothing is found.
  1519. */
  1520. struct page *find_get_entry(struct address_space *mapping, pgoff_t index)
  1521. {
  1522. XA_STATE(xas, &mapping->i_pages, index);
  1523. struct page *page;
  1524. rcu_read_lock();
  1525. repeat:
  1526. xas_reset(&xas);
  1527. page = xas_load(&xas);
  1528. if (xas_retry(&xas, page))
  1529. goto repeat;
  1530. /*
  1531. * A shadow entry of a recently evicted page, or a swap entry from
  1532. * shmem/tmpfs. Return it without attempting to raise page count.
  1533. */
  1534. if (!page || xa_is_value(page))
  1535. goto out;
  1536. if (!page_cache_get_speculative(page))
  1537. goto repeat;
  1538. /*
  1539. * Has the page moved or been split?
  1540. * This is part of the lockless pagecache protocol. See
  1541. * include/linux/pagemap.h for details.
  1542. */
  1543. if (unlikely(page != xas_reload(&xas))) {
  1544. put_page(page);
  1545. goto repeat;
  1546. }
  1547. out:
  1548. rcu_read_unlock();
  1549. return page;
  1550. }
  1551. /**
  1552. * find_lock_entry - Locate and lock a page cache entry.
  1553. * @mapping: The address_space to search.
  1554. * @index: The page cache index.
  1555. *
  1556. * Looks up the page at @mapping & @index. If there is a page in the
  1557. * cache, the head page is returned locked and with an increased refcount.
  1558. *
  1559. * If the slot holds a shadow entry of a previously evicted page, or a
  1560. * swap entry from shmem/tmpfs, it is returned.
  1561. *
  1562. * Context: May sleep.
  1563. * Return: The head page or shadow entry, %NULL if nothing is found.
  1564. */
  1565. struct page *find_lock_entry(struct address_space *mapping, pgoff_t index)
  1566. {
  1567. struct page *page;
  1568. repeat:
  1569. page = find_get_entry(mapping, index);
  1570. if (page && !xa_is_value(page)) {
  1571. lock_page(page);
  1572. /* Has the page been truncated? */
  1573. if (unlikely(page->mapping != mapping)) {
  1574. unlock_page(page);
  1575. put_page(page);
  1576. goto repeat;
  1577. }
  1578. VM_BUG_ON_PAGE(!thp_contains(page, index), page);
  1579. }
  1580. return page;
  1581. }
  1582. /**
  1583. * pagecache_get_page - Find and get a reference to a page.
  1584. * @mapping: The address_space to search.
  1585. * @index: The page index.
  1586. * @fgp_flags: %FGP flags modify how the page is returned.
  1587. * @gfp_mask: Memory allocation flags to use if %FGP_CREAT is specified.
  1588. *
  1589. * Looks up the page cache entry at @mapping & @index.
  1590. *
  1591. * @fgp_flags can be zero or more of these flags:
  1592. *
  1593. * * %FGP_ACCESSED - The page will be marked accessed.
  1594. * * %FGP_LOCK - The page is returned locked.
  1595. * * %FGP_HEAD - If the page is present and a THP, return the head page
  1596. * rather than the exact page specified by the index.
  1597. * * %FGP_CREAT - If no page is present then a new page is allocated using
  1598. * @gfp_mask and added to the page cache and the VM's LRU list.
  1599. * The page is returned locked and with an increased refcount.
  1600. * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
  1601. * page is already in cache. If the page was allocated, unlock it before
  1602. * returning so the caller can do the same dance.
  1603. * * %FGP_WRITE - The page will be written
  1604. * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
  1605. * * %FGP_NOWAIT - Don't get blocked by page lock
  1606. *
  1607. * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even
  1608. * if the %GFP flags specified for %FGP_CREAT are atomic.
  1609. *
  1610. * If there is a page cache page, it is returned with an increased refcount.
  1611. *
  1612. * Return: The found page or %NULL otherwise.
  1613. */
  1614. struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
  1615. int fgp_flags, gfp_t gfp_mask)
  1616. {
  1617. struct page *page;
  1618. repeat:
  1619. page = find_get_entry(mapping, index);
  1620. if (xa_is_value(page))
  1621. page = NULL;
  1622. trace_android_vh_pagecache_get_page(mapping, index, fgp_flags,
  1623. gfp_mask, page);
  1624. if (!page)
  1625. goto no_page;
  1626. if (fgp_flags & FGP_LOCK) {
  1627. if (fgp_flags & FGP_NOWAIT) {
  1628. if (!trylock_page(page)) {
  1629. put_page(page);
  1630. return NULL;
  1631. }
  1632. } else {
  1633. lock_page(page);
  1634. }
  1635. /* Has the page been truncated? */
  1636. if (unlikely(page->mapping != mapping)) {
  1637. unlock_page(page);
  1638. put_page(page);
  1639. goto repeat;
  1640. }
  1641. VM_BUG_ON_PAGE(!thp_contains(page, index), page);
  1642. }
  1643. if (fgp_flags & FGP_ACCESSED)
  1644. mark_page_accessed(page);
  1645. else if (fgp_flags & FGP_WRITE) {
  1646. /* Clear idle flag for buffer write */
  1647. if (page_is_idle(page))
  1648. clear_page_idle(page);
  1649. }
  1650. if (!(fgp_flags & FGP_HEAD))
  1651. page = find_subpage(page, index);
  1652. no_page:
  1653. if (!page && (fgp_flags & FGP_CREAT)) {
  1654. int err;
  1655. if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
  1656. gfp_mask |= __GFP_WRITE;
  1657. if (fgp_flags & FGP_NOFS)
  1658. gfp_mask &= ~__GFP_FS;
  1659. page = __page_cache_alloc(gfp_mask);
  1660. if (!page)
  1661. return NULL;
  1662. if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
  1663. fgp_flags |= FGP_LOCK;
  1664. /* Init accessed so avoid atomic mark_page_accessed later */
  1665. if (fgp_flags & FGP_ACCESSED)
  1666. __SetPageReferenced(page);
  1667. err = add_to_page_cache_lru(page, mapping, index, gfp_mask);
  1668. if (unlikely(err)) {
  1669. put_page(page);
  1670. page = NULL;
  1671. if (err == -EEXIST)
  1672. goto repeat;
  1673. }
  1674. /*
  1675. * add_to_page_cache_lru locks the page, and for mmap we expect
  1676. * an unlocked page.
  1677. */
  1678. if (page && (fgp_flags & FGP_FOR_MMAP))
  1679. unlock_page(page);
  1680. }
  1681. return page;
  1682. }
  1683. EXPORT_SYMBOL(pagecache_get_page);
  1684. /**
  1685. * find_get_entries - gang pagecache lookup
  1686. * @mapping: The address_space to search
  1687. * @start: The starting page cache index
  1688. * @nr_entries: The maximum number of entries
  1689. * @entries: Where the resulting entries are placed
  1690. * @indices: The cache indices corresponding to the entries in @entries
  1691. *
  1692. * find_get_entries() will search for and return a group of up to
  1693. * @nr_entries entries in the mapping. The entries are placed at
  1694. * @entries. find_get_entries() takes a reference against any actual
  1695. * pages it returns.
  1696. *
  1697. * The search returns a group of mapping-contiguous page cache entries
  1698. * with ascending indexes. There may be holes in the indices due to
  1699. * not-present pages.
  1700. *
  1701. * Any shadow entries of evicted pages, or swap entries from
  1702. * shmem/tmpfs, are included in the returned array.
  1703. *
  1704. * If it finds a Transparent Huge Page, head or tail, find_get_entries()
  1705. * stops at that page: the caller is likely to have a better way to handle
  1706. * the compound page as a whole, and then skip its extent, than repeatedly
  1707. * calling find_get_entries() to return all its tails.
  1708. *
  1709. * Return: the number of pages and shadow entries which were found.
  1710. */
  1711. unsigned find_get_entries(struct address_space *mapping,
  1712. pgoff_t start, unsigned int nr_entries,
  1713. struct page **entries, pgoff_t *indices)
  1714. {
  1715. XA_STATE(xas, &mapping->i_pages, start);
  1716. struct page *page;
  1717. unsigned int ret = 0;
  1718. if (!nr_entries)
  1719. return 0;
  1720. rcu_read_lock();
  1721. xas_for_each(&xas, page, ULONG_MAX) {
  1722. if (xas_retry(&xas, page))
  1723. continue;
  1724. /*
  1725. * A shadow entry of a recently evicted page, a swap
  1726. * entry from shmem/tmpfs or a DAX entry. Return it
  1727. * without attempting to raise page count.
  1728. */
  1729. if (xa_is_value(page))
  1730. goto export;
  1731. if (!page_cache_get_speculative(page))
  1732. goto retry;
  1733. /* Has the page moved or been split? */
  1734. if (unlikely(page != xas_reload(&xas)))
  1735. goto put_page;
  1736. /*
  1737. * Terminate early on finding a THP, to allow the caller to
  1738. * handle it all at once; but continue if this is hugetlbfs.
  1739. */
  1740. if (PageTransHuge(page) && !PageHuge(page)) {
  1741. page = find_subpage(page, xas.xa_index);
  1742. nr_entries = ret + 1;
  1743. }
  1744. export:
  1745. indices[ret] = xas.xa_index;
  1746. entries[ret] = page;
  1747. if (++ret == nr_entries)
  1748. break;
  1749. continue;
  1750. put_page:
  1751. put_page(page);
  1752. retry:
  1753. xas_reset(&xas);
  1754. }
  1755. rcu_read_unlock();
  1756. return ret;
  1757. }
  1758. /**
  1759. * find_get_pages_range - gang pagecache lookup
  1760. * @mapping: The address_space to search
  1761. * @start: The starting page index
  1762. * @end: The final page index (inclusive)
  1763. * @nr_pages: The maximum number of pages
  1764. * @pages: Where the resulting pages are placed
  1765. *
  1766. * find_get_pages_range() will search for and return a group of up to @nr_pages
  1767. * pages in the mapping starting at index @start and up to index @end
  1768. * (inclusive). The pages are placed at @pages. find_get_pages_range() takes
  1769. * a reference against the returned pages.
  1770. *
  1771. * The search returns a group of mapping-contiguous pages with ascending
  1772. * indexes. There may be holes in the indices due to not-present pages.
  1773. * We also update @start to index the next page for the traversal.
  1774. *
  1775. * Return: the number of pages which were found. If this number is
  1776. * smaller than @nr_pages, the end of specified range has been
  1777. * reached.
  1778. */
  1779. unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
  1780. pgoff_t end, unsigned int nr_pages,
  1781. struct page **pages)
  1782. {
  1783. XA_STATE(xas, &mapping->i_pages, *start);
  1784. struct page *page;
  1785. unsigned ret = 0;
  1786. if (unlikely(!nr_pages))
  1787. return 0;
  1788. rcu_read_lock();
  1789. xas_for_each(&xas, page, end) {
  1790. if (xas_retry(&xas, page))
  1791. continue;
  1792. /* Skip over shadow, swap and DAX entries */
  1793. if (xa_is_value(page))
  1794. continue;
  1795. if (!page_cache_get_speculative(page))
  1796. goto retry;
  1797. /* Has the page moved or been split? */
  1798. if (unlikely(page != xas_reload(&xas)))
  1799. goto put_page;
  1800. pages[ret] = find_subpage(page, xas.xa_index);
  1801. if (++ret == nr_pages) {
  1802. *start = xas.xa_index + 1;
  1803. goto out;
  1804. }
  1805. continue;
  1806. put_page:
  1807. put_page(page);
  1808. retry:
  1809. xas_reset(&xas);
  1810. }
  1811. /*
  1812. * We come here when there is no page beyond @end. We take care to not
  1813. * overflow the index @start as it confuses some of the callers. This
  1814. * breaks the iteration when there is a page at index -1 but that is
  1815. * already broken anyway.
  1816. */
  1817. if (end == (pgoff_t)-1)
  1818. *start = (pgoff_t)-1;
  1819. else
  1820. *start = end + 1;
  1821. out:
  1822. rcu_read_unlock();
  1823. return ret;
  1824. }
  1825. /**
  1826. * find_get_pages_contig - gang contiguous pagecache lookup
  1827. * @mapping: The address_space to search
  1828. * @index: The starting page index
  1829. * @nr_pages: The maximum number of pages
  1830. * @pages: Where the resulting pages are placed
  1831. *
  1832. * find_get_pages_contig() works exactly like find_get_pages(), except
  1833. * that the returned number of pages are guaranteed to be contiguous.
  1834. *
  1835. * Return: the number of pages which were found.
  1836. */
  1837. unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
  1838. unsigned int nr_pages, struct page **pages)
  1839. {
  1840. XA_STATE(xas, &mapping->i_pages, index);
  1841. struct page *page;
  1842. unsigned int ret = 0;
  1843. if (unlikely(!nr_pages))
  1844. return 0;
  1845. rcu_read_lock();
  1846. for (page = xas_load(&xas); page; page = xas_next(&xas)) {
  1847. if (xas_retry(&xas, page))
  1848. continue;
  1849. /*
  1850. * If the entry has been swapped out, we can stop looking.
  1851. * No current caller is looking for DAX entries.
  1852. */
  1853. if (xa_is_value(page))
  1854. break;
  1855. if (!page_cache_get_speculative(page))
  1856. goto retry;
  1857. /* Has the page moved or been split? */
  1858. if (unlikely(page != xas_reload(&xas)))
  1859. goto put_page;
  1860. pages[ret] = find_subpage(page, xas.xa_index);
  1861. if (++ret == nr_pages)
  1862. break;
  1863. continue;
  1864. put_page:
  1865. put_page(page);
  1866. retry:
  1867. xas_reset(&xas);
  1868. }
  1869. rcu_read_unlock();
  1870. return ret;
  1871. }
  1872. EXPORT_SYMBOL(find_get_pages_contig);
  1873. /**
  1874. * find_get_pages_range_tag - find and return pages in given range matching @tag
  1875. * @mapping: the address_space to search
  1876. * @index: the starting page index
  1877. * @end: The final page index (inclusive)
  1878. * @tag: the tag index
  1879. * @nr_pages: the maximum number of pages
  1880. * @pages: where the resulting pages are placed
  1881. *
  1882. * Like find_get_pages, except we only return pages which are tagged with
  1883. * @tag. We update @index to index the next page for the traversal.
  1884. *
  1885. * Return: the number of pages which were found.
  1886. */
  1887. unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
  1888. pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
  1889. struct page **pages)
  1890. {
  1891. XA_STATE(xas, &mapping->i_pages, *index);
  1892. struct page *page;
  1893. unsigned ret = 0;
  1894. if (unlikely(!nr_pages))
  1895. return 0;
  1896. rcu_read_lock();
  1897. xas_for_each_marked(&xas, page, end, tag) {
  1898. if (xas_retry(&xas, page))
  1899. continue;
  1900. /*
  1901. * Shadow entries should never be tagged, but this iteration
  1902. * is lockless so there is a window for page reclaim to evict
  1903. * a page we saw tagged. Skip over it.
  1904. */
  1905. if (xa_is_value(page))
  1906. continue;
  1907. if (!page_cache_get_speculative(page))
  1908. goto retry;
  1909. /* Has the page moved or been split? */
  1910. if (unlikely(page != xas_reload(&xas)))
  1911. goto put_page;
  1912. pages[ret] = find_subpage(page, xas.xa_index);
  1913. if (++ret == nr_pages) {
  1914. *index = xas.xa_index + 1;
  1915. goto out;
  1916. }
  1917. continue;
  1918. put_page:
  1919. put_page(page);
  1920. retry:
  1921. xas_reset(&xas);
  1922. }
  1923. /*
  1924. * We come here when we got to @end. We take care to not overflow the
  1925. * index @index as it confuses some of the callers. This breaks the
  1926. * iteration when there is a page at index -1 but that is already
  1927. * broken anyway.
  1928. */
  1929. if (end == (pgoff_t)-1)
  1930. *index = (pgoff_t)-1;
  1931. else
  1932. *index = end + 1;
  1933. out:
  1934. rcu_read_unlock();
  1935. return ret;
  1936. }
  1937. EXPORT_SYMBOL(find_get_pages_range_tag);
  1938. /*
  1939. * CD/DVDs are error prone. When a medium error occurs, the driver may fail
  1940. * a _large_ part of the i/o request. Imagine the worst scenario:
  1941. *
  1942. * ---R__________________________________________B__________
  1943. * ^ reading here ^ bad block(assume 4k)
  1944. *
  1945. * read(R) => miss => readahead(R...B) => media error => frustrating retries
  1946. * => failing the whole request => read(R) => read(R+1) =>
  1947. * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
  1948. * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
  1949. * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
  1950. *
  1951. * It is going insane. Fix it by quickly scaling down the readahead size.
  1952. */
  1953. static void shrink_readahead_size_eio(struct file_ra_state *ra)
  1954. {
  1955. ra->ra_pages /= 4;
  1956. }
  1957. /**
  1958. * generic_file_buffered_read - generic file read routine
  1959. * @iocb: the iocb to read
  1960. * @iter: data destination
  1961. * @written: already copied
  1962. *
  1963. * This is a generic file read routine, and uses the
  1964. * mapping->a_ops->readpage() function for the actual low-level stuff.
  1965. *
  1966. * This is really ugly. But the goto's actually try to clarify some
  1967. * of the logic when it comes to error handling etc.
  1968. *
  1969. * Return:
  1970. * * total number of bytes copied, including those the were already @written
  1971. * * negative error code if nothing was copied
  1972. */
  1973. ssize_t generic_file_buffered_read(struct kiocb *iocb,
  1974. struct iov_iter *iter, ssize_t written)
  1975. {
  1976. struct file *filp = iocb->ki_filp;
  1977. struct address_space *mapping = filp->f_mapping;
  1978. struct inode *inode = mapping->host;
  1979. struct file_ra_state *ra = &filp->f_ra;
  1980. loff_t *ppos = &iocb->ki_pos;
  1981. pgoff_t index;
  1982. pgoff_t last_index;
  1983. pgoff_t prev_index;
  1984. unsigned long offset; /* offset into pagecache page */
  1985. unsigned int prev_offset;
  1986. int error = 0;
  1987. if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
  1988. return 0;
  1989. iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
  1990. index = *ppos >> PAGE_SHIFT;
  1991. prev_index = ra->prev_pos >> PAGE_SHIFT;
  1992. prev_offset = ra->prev_pos & (PAGE_SIZE-1);
  1993. last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
  1994. offset = *ppos & ~PAGE_MASK;
  1995. /*
  1996. * If we've already successfully copied some data, then we
  1997. * can no longer safely return -EIOCBQUEUED. Hence mark
  1998. * an async read NOWAIT at that point.
  1999. */
  2000. if (written && (iocb->ki_flags & IOCB_WAITQ))
  2001. iocb->ki_flags |= IOCB_NOWAIT;
  2002. for (;;) {
  2003. struct page *page;
  2004. pgoff_t end_index;
  2005. loff_t isize;
  2006. unsigned long nr, ret;
  2007. cond_resched();
  2008. find_page:
  2009. if (fatal_signal_pending(current)) {
  2010. error = -EINTR;
  2011. goto out;
  2012. }
  2013. page = find_get_page(mapping, index);
  2014. if (!page) {
  2015. if (iocb->ki_flags & IOCB_NOIO)
  2016. goto would_block;
  2017. page_cache_sync_readahead(mapping,
  2018. ra, filp,
  2019. index, last_index - index);
  2020. page = find_get_page(mapping, index);
  2021. if (unlikely(page == NULL))
  2022. goto no_cached_page;
  2023. }
  2024. if (PageReadahead(page)) {
  2025. if (iocb->ki_flags & IOCB_NOIO) {
  2026. put_page(page);
  2027. goto out;
  2028. }
  2029. page_cache_async_readahead(mapping,
  2030. ra, filp, page,
  2031. index, last_index - index);
  2032. }
  2033. if (!PageUptodate(page)) {
  2034. /*
  2035. * See comment in do_read_cache_page on why
  2036. * wait_on_page_locked is used to avoid unnecessarily
  2037. * serialisations and why it's safe.
  2038. */
  2039. if (iocb->ki_flags & IOCB_WAITQ) {
  2040. if (written) {
  2041. put_page(page);
  2042. goto out;
  2043. }
  2044. error = wait_on_page_locked_async(page,
  2045. iocb->ki_waitq);
  2046. } else {
  2047. if (iocb->ki_flags & IOCB_NOWAIT) {
  2048. put_page(page);
  2049. goto would_block;
  2050. }
  2051. error = wait_on_page_locked_killable(page);
  2052. }
  2053. if (unlikely(error))
  2054. goto readpage_error;
  2055. if (PageUptodate(page))
  2056. goto page_ok;
  2057. if (inode->i_blkbits == PAGE_SHIFT ||
  2058. !mapping->a_ops->is_partially_uptodate)
  2059. goto page_not_up_to_date;
  2060. /* pipes can't handle partially uptodate pages */
  2061. if (unlikely(iov_iter_is_pipe(iter)))
  2062. goto page_not_up_to_date;
  2063. if (!trylock_page(page))
  2064. goto page_not_up_to_date;
  2065. /* Did it get truncated before we got the lock? */
  2066. if (!page->mapping)
  2067. goto page_not_up_to_date_locked;
  2068. if (!mapping->a_ops->is_partially_uptodate(page,
  2069. offset, iter->count))
  2070. goto page_not_up_to_date_locked;
  2071. unlock_page(page);
  2072. }
  2073. page_ok:
  2074. /*
  2075. * i_size must be checked after we know the page is Uptodate.
  2076. *
  2077. * Checking i_size after the check allows us to calculate
  2078. * the correct value for "nr", which means the zero-filled
  2079. * part of the page is not copied back to userspace (unless
  2080. * another truncate extends the file - this is desired though).
  2081. */
  2082. isize = i_size_read(inode);
  2083. end_index = (isize - 1) >> PAGE_SHIFT;
  2084. if (unlikely(!isize || index > end_index)) {
  2085. put_page(page);
  2086. goto out;
  2087. }
  2088. /* nr is the maximum number of bytes to copy from this page */
  2089. nr = PAGE_SIZE;
  2090. if (index == end_index) {
  2091. nr = ((isize - 1) & ~PAGE_MASK) + 1;
  2092. if (nr <= offset) {
  2093. put_page(page);
  2094. goto out;
  2095. }
  2096. }
  2097. nr = nr - offset;
  2098. /* If users can be writing to this page using arbitrary
  2099. * virtual addresses, take care about potential aliasing
  2100. * before reading the page on the kernel side.
  2101. */
  2102. if (mapping_writably_mapped(mapping))
  2103. flush_dcache_page(page);
  2104. /*
  2105. * When a sequential read accesses a page several times,
  2106. * only mark it as accessed the first time.
  2107. */
  2108. if (prev_index != index || offset != prev_offset)
  2109. mark_page_accessed(page);
  2110. prev_index = index;
  2111. /*
  2112. * Ok, we have the page, and it's up-to-date, so
  2113. * now we can copy it to user space...
  2114. */
  2115. ret = copy_page_to_iter(page, offset, nr, iter);
  2116. offset += ret;
  2117. index += offset >> PAGE_SHIFT;
  2118. offset &= ~PAGE_MASK;
  2119. prev_offset = offset;
  2120. put_page(page);
  2121. written += ret;
  2122. if (!iov_iter_count(iter))
  2123. goto out;
  2124. if (ret < nr) {
  2125. error = -EFAULT;
  2126. goto out;
  2127. }
  2128. continue;
  2129. page_not_up_to_date:
  2130. /* Get exclusive access to the page ... */
  2131. if (iocb->ki_flags & IOCB_WAITQ) {
  2132. if (written) {
  2133. put_page(page);
  2134. goto out;
  2135. }
  2136. error = lock_page_async(page, iocb->ki_waitq);
  2137. } else {
  2138. error = lock_page_killable(page);
  2139. }
  2140. if (unlikely(error))
  2141. goto readpage_error;
  2142. page_not_up_to_date_locked:
  2143. /* Did it get truncated before we got the lock? */
  2144. if (!page->mapping) {
  2145. unlock_page(page);
  2146. put_page(page);
  2147. continue;
  2148. }
  2149. /* Did somebody else fill it already? */
  2150. if (PageUptodate(page)) {
  2151. unlock_page(page);
  2152. goto page_ok;
  2153. }
  2154. readpage:
  2155. if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) {
  2156. unlock_page(page);
  2157. put_page(page);
  2158. goto would_block;
  2159. }
  2160. /*
  2161. * A previous I/O error may have been due to temporary
  2162. * failures, eg. multipath errors.
  2163. * PG_error will be set again if readpage fails.
  2164. */
  2165. ClearPageError(page);
  2166. /* Start the actual read. The read will unlock the page. */
  2167. error = mapping->a_ops->readpage(filp, page);
  2168. if (unlikely(error)) {
  2169. if (error == AOP_TRUNCATED_PAGE) {
  2170. put_page(page);
  2171. error = 0;
  2172. goto find_page;
  2173. }
  2174. goto readpage_error;
  2175. }
  2176. if (!PageUptodate(page)) {
  2177. if (iocb->ki_flags & IOCB_WAITQ) {
  2178. if (written) {
  2179. put_page(page);
  2180. goto out;
  2181. }
  2182. error = lock_page_async(page, iocb->ki_waitq);
  2183. } else {
  2184. error = lock_page_killable(page);
  2185. }
  2186. if (unlikely(error))
  2187. goto readpage_error;
  2188. if (!PageUptodate(page)) {
  2189. if (page->mapping == NULL) {
  2190. /*
  2191. * invalidate_mapping_pages got it
  2192. */
  2193. unlock_page(page);
  2194. put_page(page);
  2195. goto find_page;
  2196. }
  2197. unlock_page(page);
  2198. shrink_readahead_size_eio(ra);
  2199. error = -EIO;
  2200. goto readpage_error;
  2201. }
  2202. unlock_page(page);
  2203. }
  2204. goto page_ok;
  2205. readpage_error:
  2206. /* UHHUH! A synchronous read error occurred. Report it */
  2207. put_page(page);
  2208. goto out;
  2209. no_cached_page:
  2210. /*
  2211. * Ok, it wasn't cached, so we need to create a new
  2212. * page..
  2213. */
  2214. page = page_cache_alloc(mapping);
  2215. if (!page) {
  2216. error = -ENOMEM;
  2217. goto out;
  2218. }
  2219. error = add_to_page_cache_lru(page, mapping, index,
  2220. mapping_gfp_constraint(mapping, GFP_KERNEL));
  2221. if (error) {
  2222. put_page(page);
  2223. if (error == -EEXIST) {
  2224. error = 0;
  2225. goto find_page;
  2226. }
  2227. goto out;
  2228. }
  2229. goto readpage;
  2230. }
  2231. would_block:
  2232. error = -EAGAIN;
  2233. out:
  2234. ra->prev_pos = prev_index;
  2235. ra->prev_pos <<= PAGE_SHIFT;
  2236. ra->prev_pos |= prev_offset;
  2237. *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
  2238. file_accessed(filp);
  2239. return written ? written : error;
  2240. }
  2241. EXPORT_SYMBOL_GPL(generic_file_buffered_read);
  2242. /**
  2243. * generic_file_read_iter - generic filesystem read routine
  2244. * @iocb: kernel I/O control block
  2245. * @iter: destination for the data read
  2246. *
  2247. * This is the "read_iter()" routine for all filesystems
  2248. * that can use the page cache directly.
  2249. *
  2250. * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
  2251. * be returned when no data can be read without waiting for I/O requests
  2252. * to complete; it doesn't prevent readahead.
  2253. *
  2254. * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
  2255. * requests shall be made for the read or for readahead. When no data
  2256. * can be read, -EAGAIN shall be returned. When readahead would be
  2257. * triggered, a partial, possibly empty read shall be returned.
  2258. *
  2259. * Return:
  2260. * * number of bytes copied, even for partial reads
  2261. * * negative error code (or 0 if IOCB_NOIO) if nothing was read
  2262. */
  2263. ssize_t
  2264. generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
  2265. {
  2266. size_t count = iov_iter_count(iter);
  2267. ssize_t retval = 0;
  2268. if (!count)
  2269. goto out; /* skip atime */
  2270. if (iocb->ki_flags & IOCB_DIRECT) {
  2271. struct file *file = iocb->ki_filp;
  2272. struct address_space *mapping = file->f_mapping;
  2273. struct inode *inode = mapping->host;
  2274. loff_t size;
  2275. size = i_size_read(inode);
  2276. if (iocb->ki_flags & IOCB_NOWAIT) {
  2277. if (filemap_range_has_page(mapping, iocb->ki_pos,
  2278. iocb->ki_pos + count - 1))
  2279. return -EAGAIN;
  2280. } else {
  2281. retval = filemap_write_and_wait_range(mapping,
  2282. iocb->ki_pos,
  2283. iocb->ki_pos + count - 1);
  2284. if (retval < 0)
  2285. goto out;
  2286. }
  2287. file_accessed(file);
  2288. retval = mapping->a_ops->direct_IO(iocb, iter);
  2289. if (retval >= 0) {
  2290. iocb->ki_pos += retval;
  2291. count -= retval;
  2292. }
  2293. iov_iter_revert(iter, count - iov_iter_count(iter));
  2294. /*
  2295. * Btrfs can have a short DIO read if we encounter
  2296. * compressed extents, so if there was an error, or if
  2297. * we've already read everything we wanted to, or if
  2298. * there was a short read because we hit EOF, go ahead
  2299. * and return. Otherwise fallthrough to buffered io for
  2300. * the rest of the read. Buffered reads will not work for
  2301. * DAX files, so don't bother trying.
  2302. */
  2303. if (retval < 0 || !count || iocb->ki_pos >= size ||
  2304. IS_DAX(inode))
  2305. goto out;
  2306. }
  2307. retval = generic_file_buffered_read(iocb, iter, retval);
  2308. out:
  2309. return retval;
  2310. }
  2311. EXPORT_SYMBOL(generic_file_read_iter);
  2312. #ifdef CONFIG_MMU
  2313. #define MMAP_LOTSAMISS (100)
  2314. /*
  2315. * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
  2316. * @vmf - the vm_fault for this fault.
  2317. * @page - the page to lock.
  2318. * @fpin - the pointer to the file we may pin (or is already pinned).
  2319. *
  2320. * This works similar to lock_page_or_retry in that it can drop the mmap_lock.
  2321. * It differs in that it actually returns the page locked if it returns 1 and 0
  2322. * if it couldn't lock the page. If we did have to drop the mmap_lock then fpin
  2323. * will point to the pinned file and needs to be fput()'ed at a later point.
  2324. */
  2325. static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page,
  2326. struct file **fpin)
  2327. {
  2328. if (trylock_page(page))
  2329. return 1;
  2330. /*
  2331. * NOTE! This will make us return with VM_FAULT_RETRY, but with
  2332. * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT
  2333. * is supposed to work. We have way too many special cases..
  2334. */
  2335. if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
  2336. return 0;
  2337. *fpin = maybe_unlock_mmap_for_io(vmf, *fpin);
  2338. if (vmf->flags & FAULT_FLAG_KILLABLE) {
  2339. if (__lock_page_killable(page)) {
  2340. /*
  2341. * We didn't have the right flags to drop the mmap_lock,
  2342. * but all fault_handlers only check for fatal signals
  2343. * if we return VM_FAULT_RETRY, so we need to drop the
  2344. * mmap_lock here and return 0 if we don't have a fpin.
  2345. */
  2346. if (*fpin == NULL)
  2347. mmap_read_unlock(vmf->vma->vm_mm);
  2348. return 0;
  2349. }
  2350. } else
  2351. __lock_page(page);
  2352. return 1;
  2353. }
  2354. /*
  2355. * Synchronous readahead happens when we don't even find a page in the page
  2356. * cache at all. We don't want to perform IO under the mmap sem, so if we have
  2357. * to drop the mmap sem we return the file that was pinned in order for us to do
  2358. * that. If we didn't pin a file then we return NULL. The file that is
  2359. * returned needs to be fput()'ed when we're done with it.
  2360. */
  2361. static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
  2362. {
  2363. struct file *file = vmf->vma->vm_file;
  2364. struct file_ra_state *ra = &file->f_ra;
  2365. struct address_space *mapping = file->f_mapping;
  2366. DEFINE_READAHEAD(ractl, file, mapping, vmf->pgoff);
  2367. struct file *fpin = NULL;
  2368. unsigned int mmap_miss;
  2369. /* If we don't want any read-ahead, don't bother */
  2370. if (vmf->vma->vm_flags & VM_RAND_READ)
  2371. return fpin;
  2372. if (!ra->ra_pages)
  2373. return fpin;
  2374. if (vmf->vma->vm_flags & VM_SEQ_READ) {
  2375. fpin = maybe_unlock_mmap_for_io(vmf, fpin);
  2376. page_cache_sync_ra(&ractl, ra, ra->ra_pages);
  2377. return fpin;
  2378. }
  2379. /* Avoid banging the cache line if not needed */
  2380. mmap_miss = READ_ONCE(ra->mmap_miss);
  2381. if (mmap_miss < MMAP_LOTSAMISS * 10)
  2382. WRITE_ONCE(ra->mmap_miss, ++mmap_miss);
  2383. /*
  2384. * Do we miss much more than hit in this file? If so,
  2385. * stop bothering with read-ahead. It will only hurt.
  2386. */
  2387. if (mmap_miss > MMAP_LOTSAMISS)
  2388. return fpin;
  2389. /*
  2390. * mmap read-around
  2391. */
  2392. fpin = maybe_unlock_mmap_for_io(vmf, fpin);
  2393. ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
  2394. ra->size = ra->ra_pages;
  2395. ra->async_size = ra->ra_pages / 4;
  2396. ractl._index = ra->start;
  2397. do_page_cache_ra(&ractl, ra->size, ra->async_size);
  2398. return fpin;
  2399. }
  2400. /*
  2401. * Asynchronous readahead happens when we find the page and PG_readahead,
  2402. * so we want to possibly extend the readahead further. We return the file that
  2403. * was pinned if we have to drop the mmap_lock in order to do IO.
  2404. */
  2405. static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
  2406. struct page *page)
  2407. {
  2408. struct file *file = vmf->vma->vm_file;
  2409. struct file_ra_state *ra = &file->f_ra;
  2410. struct address_space *mapping = file->f_mapping;
  2411. struct file *fpin = NULL;
  2412. unsigned int mmap_miss;
  2413. pgoff_t offset = vmf->pgoff;
  2414. /* If we don't want any read-ahead, don't bother */
  2415. if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
  2416. return fpin;
  2417. mmap_miss = READ_ONCE(ra->mmap_miss);
  2418. if (mmap_miss)
  2419. WRITE_ONCE(ra->mmap_miss, --mmap_miss);
  2420. if (PageReadahead(page)) {
  2421. fpin = maybe_unlock_mmap_for_io(vmf, fpin);
  2422. page_cache_async_readahead(mapping, ra, file,
  2423. page, offset, ra->ra_pages);
  2424. }
  2425. return fpin;
  2426. }
  2427. /**
  2428. * filemap_fault - read in file data for page fault handling
  2429. * @vmf: struct vm_fault containing details of the fault
  2430. *
  2431. * filemap_fault() is invoked via the vma operations vector for a
  2432. * mapped memory region to read in file data during a page fault.
  2433. *
  2434. * The goto's are kind of ugly, but this streamlines the normal case of having
  2435. * it in the page cache, and handles the special cases reasonably without
  2436. * having a lot of duplicated code.
  2437. *
  2438. * vma->vm_mm->mmap_lock must be held on entry.
  2439. *
  2440. * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
  2441. * may be dropped before doing I/O or by lock_page_maybe_drop_mmap().
  2442. *
  2443. * If our return value does not have VM_FAULT_RETRY set, the mmap_lock
  2444. * has not been released.
  2445. *
  2446. * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
  2447. *
  2448. * Return: bitwise-OR of %VM_FAULT_ codes.
  2449. */
  2450. vm_fault_t filemap_fault(struct vm_fault *vmf)
  2451. {
  2452. int error;
  2453. struct file *file = vmf->vma->vm_file;
  2454. struct file *fpin = NULL;
  2455. struct address_space *mapping = file->f_mapping;
  2456. struct file_ra_state *ra = &file->f_ra;
  2457. struct inode *inode = mapping->host;
  2458. pgoff_t offset = vmf->pgoff;
  2459. pgoff_t max_off;
  2460. struct page *page = NULL;
  2461. vm_fault_t ret = 0;
  2462. bool retry = false;
  2463. max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
  2464. if (unlikely(offset >= max_off))
  2465. return VM_FAULT_SIGBUS;
  2466. trace_android_vh_filemap_fault_get_page(vmf, &page, &retry);
  2467. if (unlikely(retry))
  2468. goto out_retry;
  2469. if (unlikely(page))
  2470. goto page_ok;
  2471. /*
  2472. * Do we have something in the page cache already?
  2473. */
  2474. page = find_get_page(mapping, offset);
  2475. if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
  2476. /*
  2477. * We found the page, so try async readahead before
  2478. * waiting for the lock.
  2479. */
  2480. fpin = do_async_mmap_readahead(vmf, page);
  2481. } else if (!page) {
  2482. /* No page in the page cache at all */
  2483. count_vm_event(PGMAJFAULT);
  2484. count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
  2485. ret = VM_FAULT_MAJOR;
  2486. fpin = do_sync_mmap_readahead(vmf);
  2487. retry_find:
  2488. page = pagecache_get_page(mapping, offset,
  2489. FGP_CREAT|FGP_FOR_MMAP,
  2490. vmf->gfp_mask);
  2491. if (!page) {
  2492. if (fpin)
  2493. goto out_retry;
  2494. return VM_FAULT_OOM;
  2495. }
  2496. }
  2497. if (!lock_page_maybe_drop_mmap(vmf, page, &fpin))
  2498. goto out_retry;
  2499. /* Did it get truncated? */
  2500. if (unlikely(compound_head(page)->mapping != mapping)) {
  2501. unlock_page(page);
  2502. put_page(page);
  2503. goto retry_find;
  2504. }
  2505. VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
  2506. /*
  2507. * We have a locked page in the page cache, now we need to check
  2508. * that it's up-to-date. If not, it is going to be due to an error.
  2509. */
  2510. if (unlikely(!PageUptodate(page)))
  2511. goto page_not_uptodate;
  2512. /*
  2513. * We've made it this far and we had to drop our mmap_lock, now is the
  2514. * time to return to the upper layer and have it re-find the vma and
  2515. * redo the fault.
  2516. */
  2517. if (fpin) {
  2518. unlock_page(page);
  2519. goto out_retry;
  2520. }
  2521. page_ok:
  2522. /*
  2523. * Found the page and have a reference on it.
  2524. * We must recheck i_size under page lock.
  2525. */
  2526. max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
  2527. if (unlikely(offset >= max_off)) {
  2528. unlock_page(page);
  2529. put_page(page);
  2530. return VM_FAULT_SIGBUS;
  2531. }
  2532. vmf->page = page;
  2533. return ret | VM_FAULT_LOCKED;
  2534. page_not_uptodate:
  2535. /*
  2536. * Umm, take care of errors if the page isn't up-to-date.
  2537. * Try to re-read it _once_. We do this synchronously,
  2538. * because there really aren't any performance issues here
  2539. * and we need to check for errors.
  2540. */
  2541. ClearPageError(page);
  2542. fpin = maybe_unlock_mmap_for_io(vmf, fpin);
  2543. error = mapping->a_ops->readpage(file, page);
  2544. if (!error) {
  2545. wait_on_page_locked(page);
  2546. if (!PageUptodate(page))
  2547. error = -EIO;
  2548. }
  2549. if (fpin)
  2550. goto out_retry;
  2551. put_page(page);
  2552. if (!error || error == AOP_TRUNCATED_PAGE)
  2553. goto retry_find;
  2554. shrink_readahead_size_eio(ra);
  2555. return VM_FAULT_SIGBUS;
  2556. out_retry:
  2557. /*
  2558. * We dropped the mmap_lock, we need to return to the fault handler to
  2559. * re-find the vma and come back and find our hopefully still populated
  2560. * page.
  2561. */
  2562. if (page) {
  2563. trace_android_vh_filemap_fault_cache_page(vmf, page);
  2564. put_page(page);
  2565. }
  2566. if (fpin)
  2567. fput(fpin);
  2568. return ret | VM_FAULT_RETRY;
  2569. }
  2570. EXPORT_SYMBOL(filemap_fault);
  2571. static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page)
  2572. {
  2573. struct mm_struct *mm = vmf->vma->vm_mm;
  2574. /* Huge page is mapped? No need to proceed. */
  2575. if (pmd_trans_huge(*vmf->pmd)) {
  2576. unlock_page(page);
  2577. put_page(page);
  2578. return true;
  2579. }
  2580. if (pmd_none(*vmf->pmd) && PageTransHuge(page)) {
  2581. vm_fault_t ret = do_set_pmd(vmf, page);
  2582. if (!ret) {
  2583. /* The page is mapped successfully, reference consumed. */
  2584. unlock_page(page);
  2585. return true;
  2586. }
  2587. }
  2588. if (pmd_none(*vmf->pmd)) {
  2589. if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
  2590. unlock_page(page);
  2591. put_page(page);
  2592. return true;
  2593. }
  2594. vmf->ptl = pmd_lock(mm, vmf->pmd);
  2595. if (likely(pmd_none(*vmf->pmd))) {
  2596. mm_inc_nr_ptes(mm);
  2597. pmd_populate(mm, vmf->pmd, vmf->prealloc_pte);
  2598. vmf->prealloc_pte = NULL;
  2599. }
  2600. spin_unlock(vmf->ptl);
  2601. }
  2602. /* See comment in handle_pte_fault() */
  2603. if (pmd_devmap_trans_unstable(vmf->pmd)) {
  2604. unlock_page(page);
  2605. put_page(page);
  2606. return true;
  2607. }
  2608. return false;
  2609. }
  2610. static struct page *next_uptodate_page(struct page *page,
  2611. struct address_space *mapping,
  2612. struct xa_state *xas, pgoff_t end_pgoff)
  2613. {
  2614. unsigned long max_idx;
  2615. do {
  2616. if (!page)
  2617. return NULL;
  2618. if (xas_retry(xas, page))
  2619. continue;
  2620. if (xa_is_value(page))
  2621. continue;
  2622. if (PageLocked(page))
  2623. continue;
  2624. if (!page_cache_get_speculative(page))
  2625. continue;
  2626. /* Has the page moved or been split? */
  2627. if (unlikely(page != xas_reload(xas)))
  2628. goto skip;
  2629. if (!PageUptodate(page) || PageReadahead(page))
  2630. goto skip;
  2631. if (PageHWPoison(page))
  2632. goto skip;
  2633. if (!trylock_page(page))
  2634. goto skip;
  2635. if (page->mapping != mapping)
  2636. goto unlock;
  2637. if (!PageUptodate(page))
  2638. goto unlock;
  2639. max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
  2640. if (xas->xa_index >= max_idx)
  2641. goto unlock;
  2642. return page;
  2643. unlock:
  2644. unlock_page(page);
  2645. skip:
  2646. put_page(page);
  2647. } while ((page = xas_next_entry(xas, end_pgoff)) != NULL);
  2648. return NULL;
  2649. }
  2650. static inline struct page *first_map_page(struct address_space *mapping,
  2651. struct xa_state *xas,
  2652. pgoff_t end_pgoff)
  2653. {
  2654. return next_uptodate_page(xas_find(xas, end_pgoff),
  2655. mapping, xas, end_pgoff);
  2656. }
  2657. static inline struct page *next_map_page(struct address_space *mapping,
  2658. struct xa_state *xas,
  2659. pgoff_t end_pgoff)
  2660. {
  2661. return next_uptodate_page(xas_next_entry(xas, end_pgoff),
  2662. mapping, xas, end_pgoff);
  2663. }
  2664. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  2665. bool filemap_allow_speculation(void)
  2666. {
  2667. return true;
  2668. }
  2669. EXPORT_SYMBOL_GPL(filemap_allow_speculation);
  2670. #endif
  2671. vm_fault_t filemap_map_pages(struct vm_fault *vmf,
  2672. pgoff_t start_pgoff, pgoff_t end_pgoff)
  2673. {
  2674. struct vm_area_struct *vma = vmf->vma;
  2675. struct file *file = vma->vm_file;
  2676. struct address_space *mapping = file->f_mapping;
  2677. pgoff_t last_pgoff = start_pgoff;
  2678. unsigned long addr;
  2679. XA_STATE(xas, &mapping->i_pages, start_pgoff);
  2680. struct page *head, *page;
  2681. unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss);
  2682. vm_fault_t ret = 0;
  2683. rcu_read_lock();
  2684. head = first_map_page(mapping, &xas, end_pgoff);
  2685. if (!head)
  2686. goto out;
  2687. if (filemap_map_pmd(vmf, head)) {
  2688. if (pmd_none(*vmf->pmd) &&
  2689. vmf->flags & FAULT_FLAG_SPECULATIVE) {
  2690. ret = VM_FAULT_RETRY;
  2691. goto out;
  2692. }
  2693. ret = VM_FAULT_NOPAGE;
  2694. goto out;
  2695. }
  2696. addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
  2697. if (!pte_map_lock_addr(vmf, addr)) {
  2698. unlock_page(head);
  2699. put_page(head);
  2700. ret = VM_FAULT_RETRY;
  2701. goto out;
  2702. }
  2703. do {
  2704. page = find_subpage(head, xas.xa_index);
  2705. if (PageHWPoison(page))
  2706. goto unlock;
  2707. if (mmap_miss > 0)
  2708. mmap_miss--;
  2709. addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
  2710. vmf->pte += xas.xa_index - last_pgoff;
  2711. last_pgoff = xas.xa_index;
  2712. if (!pte_none(*vmf->pte))
  2713. goto unlock;
  2714. /* We're about to handle the fault */
  2715. if (vmf->address == addr)
  2716. ret = VM_FAULT_NOPAGE;
  2717. do_set_pte(vmf, page, addr);
  2718. /* no need to invalidate: a not-present page won't be cached */
  2719. update_mmu_cache(vma, addr, vmf->pte);
  2720. unlock_page(head);
  2721. continue;
  2722. unlock:
  2723. unlock_page(head);
  2724. put_page(head);
  2725. } while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL);
  2726. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2727. out:
  2728. rcu_read_unlock();
  2729. WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss);
  2730. return ret;
  2731. }
  2732. EXPORT_SYMBOL(filemap_map_pages);
  2733. vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
  2734. {
  2735. struct page *page = vmf->page;
  2736. struct inode *inode = file_inode(vmf->vma->vm_file);
  2737. vm_fault_t ret = VM_FAULT_LOCKED;
  2738. sb_start_pagefault(inode->i_sb);
  2739. file_update_time(vmf->vma->vm_file);
  2740. lock_page(page);
  2741. if (page->mapping != inode->i_mapping) {
  2742. unlock_page(page);
  2743. ret = VM_FAULT_NOPAGE;
  2744. goto out;
  2745. }
  2746. /*
  2747. * We mark the page dirty already here so that when freeze is in
  2748. * progress, we are guaranteed that writeback during freezing will
  2749. * see the dirty page and writeprotect it again.
  2750. */
  2751. set_page_dirty(page);
  2752. wait_for_stable_page(page);
  2753. out:
  2754. sb_end_pagefault(inode->i_sb);
  2755. return ret;
  2756. }
  2757. const struct vm_operations_struct generic_file_vm_ops = {
  2758. .fault = filemap_fault,
  2759. .map_pages = filemap_map_pages,
  2760. .page_mkwrite = filemap_page_mkwrite,
  2761. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  2762. .allow_speculation = filemap_allow_speculation,
  2763. #endif
  2764. };
  2765. /* This is used for a general mmap of a disk file */
  2766. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  2767. {
  2768. struct address_space *mapping = file->f_mapping;
  2769. if (!mapping->a_ops->readpage)
  2770. return -ENOEXEC;
  2771. file_accessed(file);
  2772. vma->vm_ops = &generic_file_vm_ops;
  2773. return 0;
  2774. }
  2775. /*
  2776. * This is for filesystems which do not implement ->writepage.
  2777. */
  2778. int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
  2779. {
  2780. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  2781. return -EINVAL;
  2782. return generic_file_mmap(file, vma);
  2783. }
  2784. #else
  2785. vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
  2786. {
  2787. return VM_FAULT_SIGBUS;
  2788. }
  2789. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  2790. {
  2791. return -ENOSYS;
  2792. }
  2793. int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
  2794. {
  2795. return -ENOSYS;
  2796. }
  2797. #endif /* CONFIG_MMU */
  2798. EXPORT_SYMBOL(filemap_page_mkwrite);
  2799. EXPORT_SYMBOL(generic_file_mmap);
  2800. EXPORT_SYMBOL(generic_file_readonly_mmap);
  2801. static struct page *wait_on_page_read(struct page *page)
  2802. {
  2803. if (!IS_ERR(page)) {
  2804. wait_on_page_locked(page);
  2805. if (!PageUptodate(page)) {
  2806. put_page(page);
  2807. page = ERR_PTR(-EIO);
  2808. }
  2809. }
  2810. return page;
  2811. }
  2812. static struct page *do_read_cache_page(struct address_space *mapping,
  2813. pgoff_t index,
  2814. int (*filler)(void *, struct page *),
  2815. void *data,
  2816. gfp_t gfp)
  2817. {
  2818. struct page *page;
  2819. int err;
  2820. repeat:
  2821. page = find_get_page(mapping, index);
  2822. if (!page) {
  2823. page = __page_cache_alloc(gfp);
  2824. if (!page)
  2825. return ERR_PTR(-ENOMEM);
  2826. err = add_to_page_cache_lru(page, mapping, index, gfp);
  2827. if (unlikely(err)) {
  2828. put_page(page);
  2829. if (err == -EEXIST)
  2830. goto repeat;
  2831. /* Presumably ENOMEM for xarray node */
  2832. return ERR_PTR(err);
  2833. }
  2834. filler:
  2835. if (filler)
  2836. err = filler(data, page);
  2837. else
  2838. err = mapping->a_ops->readpage(data, page);
  2839. if (err < 0) {
  2840. put_page(page);
  2841. return ERR_PTR(err);
  2842. }
  2843. page = wait_on_page_read(page);
  2844. if (IS_ERR(page))
  2845. return page;
  2846. goto out;
  2847. }
  2848. if (PageUptodate(page))
  2849. goto out;
  2850. /*
  2851. * Page is not up to date and may be locked due to one of the following
  2852. * case a: Page is being filled and the page lock is held
  2853. * case b: Read/write error clearing the page uptodate status
  2854. * case c: Truncation in progress (page locked)
  2855. * case d: Reclaim in progress
  2856. *
  2857. * Case a, the page will be up to date when the page is unlocked.
  2858. * There is no need to serialise on the page lock here as the page
  2859. * is pinned so the lock gives no additional protection. Even if the
  2860. * page is truncated, the data is still valid if PageUptodate as
  2861. * it's a race vs truncate race.
  2862. * Case b, the page will not be up to date
  2863. * Case c, the page may be truncated but in itself, the data may still
  2864. * be valid after IO completes as it's a read vs truncate race. The
  2865. * operation must restart if the page is not uptodate on unlock but
  2866. * otherwise serialising on page lock to stabilise the mapping gives
  2867. * no additional guarantees to the caller as the page lock is
  2868. * released before return.
  2869. * Case d, similar to truncation. If reclaim holds the page lock, it
  2870. * will be a race with remove_mapping that determines if the mapping
  2871. * is valid on unlock but otherwise the data is valid and there is
  2872. * no need to serialise with page lock.
  2873. *
  2874. * As the page lock gives no additional guarantee, we optimistically
  2875. * wait on the page to be unlocked and check if it's up to date and
  2876. * use the page if it is. Otherwise, the page lock is required to
  2877. * distinguish between the different cases. The motivation is that we
  2878. * avoid spurious serialisations and wakeups when multiple processes
  2879. * wait on the same page for IO to complete.
  2880. */
  2881. wait_on_page_locked(page);
  2882. if (PageUptodate(page))
  2883. goto out;
  2884. /* Distinguish between all the cases under the safety of the lock */
  2885. lock_page(page);
  2886. /* Case c or d, restart the operation */
  2887. if (!page->mapping) {
  2888. unlock_page(page);
  2889. put_page(page);
  2890. goto repeat;
  2891. }
  2892. /* Someone else locked and filled the page in a very small window */
  2893. if (PageUptodate(page)) {
  2894. unlock_page(page);
  2895. goto out;
  2896. }
  2897. /*
  2898. * A previous I/O error may have been due to temporary
  2899. * failures.
  2900. * Clear page error before actual read, PG_error will be
  2901. * set again if read page fails.
  2902. */
  2903. ClearPageError(page);
  2904. goto filler;
  2905. out:
  2906. mark_page_accessed(page);
  2907. return page;
  2908. }
  2909. /**
  2910. * read_cache_page - read into page cache, fill it if needed
  2911. * @mapping: the page's address_space
  2912. * @index: the page index
  2913. * @filler: function to perform the read
  2914. * @data: first arg to filler(data, page) function, often left as NULL
  2915. *
  2916. * Read into the page cache. If a page already exists, and PageUptodate() is
  2917. * not set, try to fill the page and wait for it to become unlocked.
  2918. *
  2919. * If the page does not get brought uptodate, return -EIO.
  2920. *
  2921. * Return: up to date page on success, ERR_PTR() on failure.
  2922. */
  2923. struct page *read_cache_page(struct address_space *mapping,
  2924. pgoff_t index,
  2925. int (*filler)(void *, struct page *),
  2926. void *data)
  2927. {
  2928. return do_read_cache_page(mapping, index, filler, data,
  2929. mapping_gfp_mask(mapping));
  2930. }
  2931. EXPORT_SYMBOL(read_cache_page);
  2932. /**
  2933. * read_cache_page_gfp - read into page cache, using specified page allocation flags.
  2934. * @mapping: the page's address_space
  2935. * @index: the page index
  2936. * @gfp: the page allocator flags to use if allocating
  2937. *
  2938. * This is the same as "read_mapping_page(mapping, index, NULL)", but with
  2939. * any new page allocations done using the specified allocation flags.
  2940. *
  2941. * If the page does not get brought uptodate, return -EIO.
  2942. *
  2943. * Return: up to date page on success, ERR_PTR() on failure.
  2944. */
  2945. struct page *read_cache_page_gfp(struct address_space *mapping,
  2946. pgoff_t index,
  2947. gfp_t gfp)
  2948. {
  2949. return do_read_cache_page(mapping, index, NULL, NULL, gfp);
  2950. }
  2951. EXPORT_SYMBOL(read_cache_page_gfp);
  2952. int pagecache_write_begin(struct file *file, struct address_space *mapping,
  2953. loff_t pos, unsigned len, unsigned flags,
  2954. struct page **pagep, void **fsdata)
  2955. {
  2956. const struct address_space_operations *aops = mapping->a_ops;
  2957. return aops->write_begin(file, mapping, pos, len, flags,
  2958. pagep, fsdata);
  2959. }
  2960. EXPORT_SYMBOL(pagecache_write_begin);
  2961. int pagecache_write_end(struct file *file, struct address_space *mapping,
  2962. loff_t pos, unsigned len, unsigned copied,
  2963. struct page *page, void *fsdata)
  2964. {
  2965. const struct address_space_operations *aops = mapping->a_ops;
  2966. return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
  2967. }
  2968. EXPORT_SYMBOL(pagecache_write_end);
  2969. /*
  2970. * Warn about a page cache invalidation failure during a direct I/O write.
  2971. */
  2972. void dio_warn_stale_pagecache(struct file *filp)
  2973. {
  2974. static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
  2975. char pathname[128];
  2976. struct inode *inode = file_inode(filp);
  2977. char *path;
  2978. errseq_set(&inode->i_mapping->wb_err, -EIO);
  2979. if (__ratelimit(&_rs)) {
  2980. path = file_path(filp, pathname, sizeof(pathname));
  2981. if (IS_ERR(path))
  2982. path = "(unknown)";
  2983. pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
  2984. pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
  2985. current->comm);
  2986. }
  2987. }
  2988. ssize_t
  2989. generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
  2990. {
  2991. struct file *file = iocb->ki_filp;
  2992. struct address_space *mapping = file->f_mapping;
  2993. struct inode *inode = mapping->host;
  2994. loff_t pos = iocb->ki_pos;
  2995. ssize_t written;
  2996. size_t write_len;
  2997. pgoff_t end;
  2998. write_len = iov_iter_count(from);
  2999. end = (pos + write_len - 1) >> PAGE_SHIFT;
  3000. if (iocb->ki_flags & IOCB_NOWAIT) {
  3001. /* If there are pages to writeback, return */
  3002. if (filemap_range_has_page(inode->i_mapping, pos,
  3003. pos + write_len - 1))
  3004. return -EAGAIN;
  3005. } else {
  3006. written = filemap_write_and_wait_range(mapping, pos,
  3007. pos + write_len - 1);
  3008. if (written)
  3009. goto out;
  3010. }
  3011. /*
  3012. * After a write we want buffered reads to be sure to go to disk to get
  3013. * the new data. We invalidate clean cached page from the region we're
  3014. * about to write. We do this *before* the write so that we can return
  3015. * without clobbering -EIOCBQUEUED from ->direct_IO().
  3016. */
  3017. written = invalidate_inode_pages2_range(mapping,
  3018. pos >> PAGE_SHIFT, end);
  3019. /*
  3020. * If a page can not be invalidated, return 0 to fall back
  3021. * to buffered write.
  3022. */
  3023. if (written) {
  3024. if (written == -EBUSY)
  3025. return 0;
  3026. goto out;
  3027. }
  3028. written = mapping->a_ops->direct_IO(iocb, from);
  3029. /*
  3030. * Finally, try again to invalidate clean pages which might have been
  3031. * cached by non-direct readahead, or faulted in by get_user_pages()
  3032. * if the source of the write was an mmap'ed region of the file
  3033. * we're writing. Either one is a pretty crazy thing to do,
  3034. * so we don't support it 100%. If this invalidation
  3035. * fails, tough, the write still worked...
  3036. *
  3037. * Most of the time we do not need this since dio_complete() will do
  3038. * the invalidation for us. However there are some file systems that
  3039. * do not end up with dio_complete() being called, so let's not break
  3040. * them by removing it completely.
  3041. *
  3042. * Noticeable example is a blkdev_direct_IO().
  3043. *
  3044. * Skip invalidation for async writes or if mapping has no pages.
  3045. */
  3046. if (written > 0 && mapping->nrpages &&
  3047. invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))
  3048. dio_warn_stale_pagecache(file);
  3049. if (written > 0) {
  3050. pos += written;
  3051. write_len -= written;
  3052. if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
  3053. i_size_write(inode, pos);
  3054. mark_inode_dirty(inode);
  3055. }
  3056. iocb->ki_pos = pos;
  3057. }
  3058. iov_iter_revert(from, write_len - iov_iter_count(from));
  3059. out:
  3060. return written;
  3061. }
  3062. EXPORT_SYMBOL(generic_file_direct_write);
  3063. /*
  3064. * Find or create a page at the given pagecache position. Return the locked
  3065. * page. This function is specifically for buffered writes.
  3066. */
  3067. struct page *grab_cache_page_write_begin(struct address_space *mapping,
  3068. pgoff_t index, unsigned flags)
  3069. {
  3070. struct page *page;
  3071. int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
  3072. if (flags & AOP_FLAG_NOFS)
  3073. fgp_flags |= FGP_NOFS;
  3074. page = pagecache_get_page(mapping, index, fgp_flags,
  3075. mapping_gfp_mask(mapping));
  3076. if (page)
  3077. wait_for_stable_page(page);
  3078. return page;
  3079. }
  3080. EXPORT_SYMBOL(grab_cache_page_write_begin);
  3081. ssize_t generic_perform_write(struct file *file,
  3082. struct iov_iter *i, loff_t pos)
  3083. {
  3084. struct address_space *mapping = file->f_mapping;
  3085. const struct address_space_operations *a_ops = mapping->a_ops;
  3086. long status = 0;
  3087. ssize_t written = 0;
  3088. unsigned int flags = 0;
  3089. do {
  3090. struct page *page;
  3091. unsigned long offset; /* Offset into pagecache page */
  3092. unsigned long bytes; /* Bytes to write to page */
  3093. size_t copied; /* Bytes copied from user */
  3094. void *fsdata;
  3095. offset = (pos & (PAGE_SIZE - 1));
  3096. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  3097. iov_iter_count(i));
  3098. again:
  3099. /*
  3100. * Bring in the user page that we will copy from _first_.
  3101. * Otherwise there's a nasty deadlock on copying from the
  3102. * same page as we're writing to, without it being marked
  3103. * up-to-date.
  3104. *
  3105. * Not only is this an optimisation, but it is also required
  3106. * to check that the address is actually valid, when atomic
  3107. * usercopies are used, below.
  3108. */
  3109. if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
  3110. status = -EFAULT;
  3111. break;
  3112. }
  3113. if (fatal_signal_pending(current)) {
  3114. status = -EINTR;
  3115. break;
  3116. }
  3117. status = a_ops->write_begin(file, mapping, pos, bytes, flags,
  3118. &page, &fsdata);
  3119. if (unlikely(status < 0))
  3120. break;
  3121. if (mapping_writably_mapped(mapping))
  3122. flush_dcache_page(page);
  3123. copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
  3124. flush_dcache_page(page);
  3125. status = a_ops->write_end(file, mapping, pos, bytes, copied,
  3126. page, fsdata);
  3127. if (unlikely(status < 0))
  3128. break;
  3129. copied = status;
  3130. cond_resched();
  3131. iov_iter_advance(i, copied);
  3132. if (unlikely(copied == 0)) {
  3133. /*
  3134. * If we were unable to copy any data at all, we must
  3135. * fall back to a single segment length write.
  3136. *
  3137. * If we didn't fallback here, we could livelock
  3138. * because not all segments in the iov can be copied at
  3139. * once without a pagefault.
  3140. */
  3141. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  3142. iov_iter_single_seg_count(i));
  3143. goto again;
  3144. }
  3145. pos += copied;
  3146. written += copied;
  3147. balance_dirty_pages_ratelimited(mapping);
  3148. } while (iov_iter_count(i));
  3149. return written ? written : status;
  3150. }
  3151. EXPORT_SYMBOL(generic_perform_write);
  3152. /**
  3153. * __generic_file_write_iter - write data to a file
  3154. * @iocb: IO state structure (file, offset, etc.)
  3155. * @from: iov_iter with data to write
  3156. *
  3157. * This function does all the work needed for actually writing data to a
  3158. * file. It does all basic checks, removes SUID from the file, updates
  3159. * modification times and calls proper subroutines depending on whether we
  3160. * do direct IO or a standard buffered write.
  3161. *
  3162. * It expects i_mutex to be grabbed unless we work on a block device or similar
  3163. * object which does not need locking at all.
  3164. *
  3165. * This function does *not* take care of syncing data in case of O_SYNC write.
  3166. * A caller has to handle it. This is mainly due to the fact that we want to
  3167. * avoid syncing under i_mutex.
  3168. *
  3169. * Return:
  3170. * * number of bytes written, even for truncated writes
  3171. * * negative error code if no data has been written at all
  3172. */
  3173. ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
  3174. {
  3175. struct file *file = iocb->ki_filp;
  3176. struct address_space * mapping = file->f_mapping;
  3177. struct inode *inode = mapping->host;
  3178. ssize_t written = 0;
  3179. ssize_t err;
  3180. ssize_t status;
  3181. /* We can write back this queue in page reclaim */
  3182. current->backing_dev_info = inode_to_bdi(inode);
  3183. err = file_remove_privs(file);
  3184. if (err)
  3185. goto out;
  3186. err = file_update_time(file);
  3187. if (err)
  3188. goto out;
  3189. if (iocb->ki_flags & IOCB_DIRECT) {
  3190. loff_t pos, endbyte;
  3191. written = generic_file_direct_write(iocb, from);
  3192. /*
  3193. * If the write stopped short of completing, fall back to
  3194. * buffered writes. Some filesystems do this for writes to
  3195. * holes, for example. For DAX files, a buffered write will
  3196. * not succeed (even if it did, DAX does not handle dirty
  3197. * page-cache pages correctly).
  3198. */
  3199. if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
  3200. goto out;
  3201. status = generic_perform_write(file, from, pos = iocb->ki_pos);
  3202. /*
  3203. * If generic_perform_write() returned a synchronous error
  3204. * then we want to return the number of bytes which were
  3205. * direct-written, or the error code if that was zero. Note
  3206. * that this differs from normal direct-io semantics, which
  3207. * will return -EFOO even if some bytes were written.
  3208. */
  3209. if (unlikely(status < 0)) {
  3210. err = status;
  3211. goto out;
  3212. }
  3213. /*
  3214. * We need to ensure that the page cache pages are written to
  3215. * disk and invalidated to preserve the expected O_DIRECT
  3216. * semantics.
  3217. */
  3218. endbyte = pos + status - 1;
  3219. err = filemap_write_and_wait_range(mapping, pos, endbyte);
  3220. if (err == 0) {
  3221. iocb->ki_pos = endbyte + 1;
  3222. written += status;
  3223. invalidate_mapping_pages(mapping,
  3224. pos >> PAGE_SHIFT,
  3225. endbyte >> PAGE_SHIFT);
  3226. } else {
  3227. /*
  3228. * We don't know how much we wrote, so just return
  3229. * the number of bytes which were direct-written
  3230. */
  3231. }
  3232. } else {
  3233. written = generic_perform_write(file, from, iocb->ki_pos);
  3234. if (likely(written > 0))
  3235. iocb->ki_pos += written;
  3236. }
  3237. out:
  3238. current->backing_dev_info = NULL;
  3239. return written ? written : err;
  3240. }
  3241. EXPORT_SYMBOL(__generic_file_write_iter);
  3242. /**
  3243. * generic_file_write_iter - write data to a file
  3244. * @iocb: IO state structure
  3245. * @from: iov_iter with data to write
  3246. *
  3247. * This is a wrapper around __generic_file_write_iter() to be used by most
  3248. * filesystems. It takes care of syncing the file in case of O_SYNC file
  3249. * and acquires i_mutex as needed.
  3250. * Return:
  3251. * * negative error code if no data has been written at all of
  3252. * vfs_fsync_range() failed for a synchronous write
  3253. * * number of bytes written, even for truncated writes
  3254. */
  3255. ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
  3256. {
  3257. struct file *file = iocb->ki_filp;
  3258. struct inode *inode = file->f_mapping->host;
  3259. ssize_t ret;
  3260. inode_lock(inode);
  3261. ret = generic_write_checks(iocb, from);
  3262. if (ret > 0)
  3263. ret = __generic_file_write_iter(iocb, from);
  3264. inode_unlock(inode);
  3265. if (ret > 0)
  3266. ret = generic_write_sync(iocb, ret);
  3267. return ret;
  3268. }
  3269. EXPORT_SYMBOL(generic_file_write_iter);
  3270. /**
  3271. * try_to_release_page() - release old fs-specific metadata on a page
  3272. *
  3273. * @page: the page which the kernel is trying to free
  3274. * @gfp_mask: memory allocation flags (and I/O mode)
  3275. *
  3276. * The address_space is to try to release any data against the page
  3277. * (presumably at page->private).
  3278. *
  3279. * This may also be called if PG_fscache is set on a page, indicating that the
  3280. * page is known to the local caching routines.
  3281. *
  3282. * The @gfp_mask argument specifies whether I/O may be performed to release
  3283. * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
  3284. *
  3285. * Return: %1 if the release was successful, otherwise return zero.
  3286. */
  3287. int try_to_release_page(struct page *page, gfp_t gfp_mask)
  3288. {
  3289. struct address_space * const mapping = page->mapping;
  3290. BUG_ON(!PageLocked(page));
  3291. if (PageWriteback(page))
  3292. return 0;
  3293. if (mapping && mapping->a_ops->releasepage)
  3294. return mapping->a_ops->releasepage(page, gfp_mask);
  3295. return try_to_free_buffers(page);
  3296. }
  3297. EXPORT_SYMBOL(try_to_release_page);