buffer.c 90 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/buffer.c
  4. *
  5. * Copyright (C) 1991, 1992, 2002 Linus Torvalds
  6. */
  7. /*
  8. * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
  9. *
  10. * Removed a lot of unnecessary code and simplified things now that
  11. * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
  12. *
  13. * Speed up hash, lru, and free list operations. Use gfp() for allocating
  14. * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
  15. *
  16. * Added 32k buffer block sizes - these are required older ARM systems. - RMK
  17. *
  18. * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
  19. */
  20. #include <linux/kernel.h>
  21. #include <linux/sched/signal.h>
  22. #include <linux/syscalls.h>
  23. #include <linux/fs.h>
  24. #include <linux/iomap.h>
  25. #include <linux/mm.h>
  26. #include <linux/percpu.h>
  27. #include <linux/slab.h>
  28. #include <linux/capability.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/file.h>
  31. #include <linux/quotaops.h>
  32. #include <linux/highmem.h>
  33. #include <linux/export.h>
  34. #include <linux/backing-dev.h>
  35. #include <linux/writeback.h>
  36. #include <linux/hash.h>
  37. #include <linux/suspend.h>
  38. #include <linux/buffer_head.h>
  39. #include <linux/task_io_accounting_ops.h>
  40. #include <linux/bio.h>
  41. #include <linux/cpu.h>
  42. #include <linux/bitops.h>
  43. #include <linux/mpage.h>
  44. #include <linux/bit_spinlock.h>
  45. #include <linux/pagevec.h>
  46. #include <linux/sched/mm.h>
  47. #include <trace/events/block.h>
  48. #include <linux/fscrypt.h>
  49. #include "internal.h"
  50. static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
  51. static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
  52. enum rw_hint hint, struct writeback_control *wbc);
  53. #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
  54. inline void touch_buffer(struct buffer_head *bh)
  55. {
  56. trace_block_touch_buffer(bh);
  57. mark_page_accessed(bh->b_page);
  58. }
  59. EXPORT_SYMBOL(touch_buffer);
  60. void __lock_buffer(struct buffer_head *bh)
  61. {
  62. wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
  63. }
  64. EXPORT_SYMBOL(__lock_buffer);
  65. void unlock_buffer(struct buffer_head *bh)
  66. {
  67. clear_bit_unlock(BH_Lock, &bh->b_state);
  68. smp_mb__after_atomic();
  69. wake_up_bit(&bh->b_state, BH_Lock);
  70. }
  71. EXPORT_SYMBOL(unlock_buffer);
  72. /*
  73. * Returns if the page has dirty or writeback buffers. If all the buffers
  74. * are unlocked and clean then the PageDirty information is stale. If
  75. * any of the pages are locked, it is assumed they are locked for IO.
  76. */
  77. void buffer_check_dirty_writeback(struct page *page,
  78. bool *dirty, bool *writeback)
  79. {
  80. struct buffer_head *head, *bh;
  81. *dirty = false;
  82. *writeback = false;
  83. BUG_ON(!PageLocked(page));
  84. if (!page_has_buffers(page))
  85. return;
  86. if (PageWriteback(page))
  87. *writeback = true;
  88. head = page_buffers(page);
  89. bh = head;
  90. do {
  91. if (buffer_locked(bh))
  92. *writeback = true;
  93. if (buffer_dirty(bh))
  94. *dirty = true;
  95. bh = bh->b_this_page;
  96. } while (bh != head);
  97. }
  98. EXPORT_SYMBOL(buffer_check_dirty_writeback);
  99. /*
  100. * Block until a buffer comes unlocked. This doesn't stop it
  101. * from becoming locked again - you have to lock it yourself
  102. * if you want to preserve its state.
  103. */
  104. void __wait_on_buffer(struct buffer_head * bh)
  105. {
  106. wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
  107. }
  108. EXPORT_SYMBOL(__wait_on_buffer);
  109. static void buffer_io_error(struct buffer_head *bh, char *msg)
  110. {
  111. if (!test_bit(BH_Quiet, &bh->b_state))
  112. printk_ratelimited(KERN_ERR
  113. "Buffer I/O error on dev %pg, logical block %llu%s\n",
  114. bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
  115. }
  116. /*
  117. * End-of-IO handler helper function which does not touch the bh after
  118. * unlocking it.
  119. * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
  120. * a race there is benign: unlock_buffer() only use the bh's address for
  121. * hashing after unlocking the buffer, so it doesn't actually touch the bh
  122. * itself.
  123. */
  124. static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
  125. {
  126. if (uptodate) {
  127. set_buffer_uptodate(bh);
  128. } else {
  129. /* This happens, due to failed read-ahead attempts. */
  130. clear_buffer_uptodate(bh);
  131. }
  132. unlock_buffer(bh);
  133. }
  134. /*
  135. * Default synchronous end-of-IO handler.. Just mark it up-to-date and
  136. * unlock the buffer. This is what ll_rw_block uses too.
  137. */
  138. void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
  139. {
  140. __end_buffer_read_notouch(bh, uptodate);
  141. put_bh(bh);
  142. }
  143. EXPORT_SYMBOL(end_buffer_read_sync);
  144. void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
  145. {
  146. if (uptodate) {
  147. set_buffer_uptodate(bh);
  148. } else {
  149. buffer_io_error(bh, ", lost sync page write");
  150. mark_buffer_write_io_error(bh);
  151. clear_buffer_uptodate(bh);
  152. }
  153. unlock_buffer(bh);
  154. put_bh(bh);
  155. }
  156. EXPORT_SYMBOL_NS(end_buffer_write_sync, ANDROID_GKI_VFS_EXPORT_ONLY);
  157. /*
  158. * Various filesystems appear to want __find_get_block to be non-blocking.
  159. * But it's the page lock which protects the buffers. To get around this,
  160. * we get exclusion from try_to_free_buffers with the blockdev mapping's
  161. * private_lock.
  162. *
  163. * Hack idea: for the blockdev mapping, private_lock contention
  164. * may be quite high. This code could TryLock the page, and if that
  165. * succeeds, there is no need to take private_lock.
  166. */
  167. static struct buffer_head *
  168. __find_get_block_slow(struct block_device *bdev, sector_t block)
  169. {
  170. struct inode *bd_inode = bdev->bd_inode;
  171. struct address_space *bd_mapping = bd_inode->i_mapping;
  172. struct buffer_head *ret = NULL;
  173. pgoff_t index;
  174. struct buffer_head *bh;
  175. struct buffer_head *head;
  176. struct page *page;
  177. int all_mapped = 1;
  178. static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
  179. index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
  180. page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
  181. if (!page)
  182. goto out;
  183. spin_lock(&bd_mapping->private_lock);
  184. if (!page_has_buffers(page))
  185. goto out_unlock;
  186. head = page_buffers(page);
  187. bh = head;
  188. do {
  189. if (!buffer_mapped(bh))
  190. all_mapped = 0;
  191. else if (bh->b_blocknr == block) {
  192. ret = bh;
  193. get_bh(bh);
  194. goto out_unlock;
  195. }
  196. bh = bh->b_this_page;
  197. } while (bh != head);
  198. /* we might be here because some of the buffers on this page are
  199. * not mapped. This is due to various races between
  200. * file io on the block device and getblk. It gets dealt with
  201. * elsewhere, don't buffer_error if we had some unmapped buffers
  202. */
  203. ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
  204. if (all_mapped && __ratelimit(&last_warned)) {
  205. printk("__find_get_block_slow() failed. block=%llu, "
  206. "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
  207. "device %pg blocksize: %d\n",
  208. (unsigned long long)block,
  209. (unsigned long long)bh->b_blocknr,
  210. bh->b_state, bh->b_size, bdev,
  211. 1 << bd_inode->i_blkbits);
  212. }
  213. out_unlock:
  214. spin_unlock(&bd_mapping->private_lock);
  215. put_page(page);
  216. out:
  217. return ret;
  218. }
  219. static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
  220. {
  221. unsigned long flags;
  222. struct buffer_head *first;
  223. struct buffer_head *tmp;
  224. struct page *page;
  225. int page_uptodate = 1;
  226. BUG_ON(!buffer_async_read(bh));
  227. page = bh->b_page;
  228. if (uptodate) {
  229. set_buffer_uptodate(bh);
  230. } else {
  231. clear_buffer_uptodate(bh);
  232. buffer_io_error(bh, ", async page read");
  233. SetPageError(page);
  234. }
  235. /*
  236. * Be _very_ careful from here on. Bad things can happen if
  237. * two buffer heads end IO at almost the same time and both
  238. * decide that the page is now completely done.
  239. */
  240. first = page_buffers(page);
  241. spin_lock_irqsave(&first->b_uptodate_lock, flags);
  242. clear_buffer_async_read(bh);
  243. unlock_buffer(bh);
  244. tmp = bh;
  245. do {
  246. if (!buffer_uptodate(tmp))
  247. page_uptodate = 0;
  248. if (buffer_async_read(tmp)) {
  249. BUG_ON(!buffer_locked(tmp));
  250. goto still_busy;
  251. }
  252. tmp = tmp->b_this_page;
  253. } while (tmp != bh);
  254. spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
  255. /*
  256. * If none of the buffers had errors and they are all
  257. * uptodate then we can set the page uptodate.
  258. */
  259. if (page_uptodate && !PageError(page))
  260. SetPageUptodate(page);
  261. unlock_page(page);
  262. return;
  263. still_busy:
  264. spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
  265. return;
  266. }
  267. struct decrypt_bh_ctx {
  268. struct work_struct work;
  269. struct buffer_head *bh;
  270. };
  271. static void decrypt_bh(struct work_struct *work)
  272. {
  273. struct decrypt_bh_ctx *ctx =
  274. container_of(work, struct decrypt_bh_ctx, work);
  275. struct buffer_head *bh = ctx->bh;
  276. int err;
  277. err = fscrypt_decrypt_pagecache_blocks(bh->b_page, bh->b_size,
  278. bh_offset(bh));
  279. end_buffer_async_read(bh, err == 0);
  280. kfree(ctx);
  281. }
  282. /*
  283. * I/O completion handler for block_read_full_page() - pages
  284. * which come unlocked at the end of I/O.
  285. */
  286. static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
  287. {
  288. /* Decrypt if needed */
  289. if (uptodate &&
  290. fscrypt_inode_uses_fs_layer_crypto(bh->b_page->mapping->host)) {
  291. struct decrypt_bh_ctx *ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC);
  292. if (ctx) {
  293. INIT_WORK(&ctx->work, decrypt_bh);
  294. ctx->bh = bh;
  295. fscrypt_enqueue_decrypt_work(&ctx->work);
  296. return;
  297. }
  298. uptodate = 0;
  299. }
  300. end_buffer_async_read(bh, uptodate);
  301. }
  302. /*
  303. * Completion handler for block_write_full_page() - pages which are unlocked
  304. * during I/O, and which have PageWriteback cleared upon I/O completion.
  305. */
  306. void end_buffer_async_write(struct buffer_head *bh, int uptodate)
  307. {
  308. unsigned long flags;
  309. struct buffer_head *first;
  310. struct buffer_head *tmp;
  311. struct page *page;
  312. BUG_ON(!buffer_async_write(bh));
  313. page = bh->b_page;
  314. if (uptodate) {
  315. set_buffer_uptodate(bh);
  316. } else {
  317. buffer_io_error(bh, ", lost async page write");
  318. mark_buffer_write_io_error(bh);
  319. clear_buffer_uptodate(bh);
  320. SetPageError(page);
  321. }
  322. first = page_buffers(page);
  323. spin_lock_irqsave(&first->b_uptodate_lock, flags);
  324. clear_buffer_async_write(bh);
  325. unlock_buffer(bh);
  326. tmp = bh->b_this_page;
  327. while (tmp != bh) {
  328. if (buffer_async_write(tmp)) {
  329. BUG_ON(!buffer_locked(tmp));
  330. goto still_busy;
  331. }
  332. tmp = tmp->b_this_page;
  333. }
  334. spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
  335. end_page_writeback(page);
  336. return;
  337. still_busy:
  338. spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
  339. return;
  340. }
  341. EXPORT_SYMBOL(end_buffer_async_write);
  342. /*
  343. * If a page's buffers are under async readin (end_buffer_async_read
  344. * completion) then there is a possibility that another thread of
  345. * control could lock one of the buffers after it has completed
  346. * but while some of the other buffers have not completed. This
  347. * locked buffer would confuse end_buffer_async_read() into not unlocking
  348. * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
  349. * that this buffer is not under async I/O.
  350. *
  351. * The page comes unlocked when it has no locked buffer_async buffers
  352. * left.
  353. *
  354. * PageLocked prevents anyone starting new async I/O reads any of
  355. * the buffers.
  356. *
  357. * PageWriteback is used to prevent simultaneous writeout of the same
  358. * page.
  359. *
  360. * PageLocked prevents anyone from starting writeback of a page which is
  361. * under read I/O (PageWriteback is only ever set against a locked page).
  362. */
  363. static void mark_buffer_async_read(struct buffer_head *bh)
  364. {
  365. bh->b_end_io = end_buffer_async_read_io;
  366. set_buffer_async_read(bh);
  367. }
  368. static void mark_buffer_async_write_endio(struct buffer_head *bh,
  369. bh_end_io_t *handler)
  370. {
  371. bh->b_end_io = handler;
  372. set_buffer_async_write(bh);
  373. }
  374. void mark_buffer_async_write(struct buffer_head *bh)
  375. {
  376. mark_buffer_async_write_endio(bh, end_buffer_async_write);
  377. }
  378. EXPORT_SYMBOL_NS(mark_buffer_async_write, ANDROID_GKI_VFS_EXPORT_ONLY);
  379. /*
  380. * fs/buffer.c contains helper functions for buffer-backed address space's
  381. * fsync functions. A common requirement for buffer-based filesystems is
  382. * that certain data from the backing blockdev needs to be written out for
  383. * a successful fsync(). For example, ext2 indirect blocks need to be
  384. * written back and waited upon before fsync() returns.
  385. *
  386. * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
  387. * inode_has_buffers() and invalidate_inode_buffers() are provided for the
  388. * management of a list of dependent buffers at ->i_mapping->private_list.
  389. *
  390. * Locking is a little subtle: try_to_free_buffers() will remove buffers
  391. * from their controlling inode's queue when they are being freed. But
  392. * try_to_free_buffers() will be operating against the *blockdev* mapping
  393. * at the time, not against the S_ISREG file which depends on those buffers.
  394. * So the locking for private_list is via the private_lock in the address_space
  395. * which backs the buffers. Which is different from the address_space
  396. * against which the buffers are listed. So for a particular address_space,
  397. * mapping->private_lock does *not* protect mapping->private_list! In fact,
  398. * mapping->private_list will always be protected by the backing blockdev's
  399. * ->private_lock.
  400. *
  401. * Which introduces a requirement: all buffers on an address_space's
  402. * ->private_list must be from the same address_space: the blockdev's.
  403. *
  404. * address_spaces which do not place buffers at ->private_list via these
  405. * utility functions are free to use private_lock and private_list for
  406. * whatever they want. The only requirement is that list_empty(private_list)
  407. * be true at clear_inode() time.
  408. *
  409. * FIXME: clear_inode should not call invalidate_inode_buffers(). The
  410. * filesystems should do that. invalidate_inode_buffers() should just go
  411. * BUG_ON(!list_empty).
  412. *
  413. * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
  414. * take an address_space, not an inode. And it should be called
  415. * mark_buffer_dirty_fsync() to clearly define why those buffers are being
  416. * queued up.
  417. *
  418. * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
  419. * list if it is already on a list. Because if the buffer is on a list,
  420. * it *must* already be on the right one. If not, the filesystem is being
  421. * silly. This will save a ton of locking. But first we have to ensure
  422. * that buffers are taken *off* the old inode's list when they are freed
  423. * (presumably in truncate). That requires careful auditing of all
  424. * filesystems (do it inside bforget()). It could also be done by bringing
  425. * b_inode back.
  426. */
  427. /*
  428. * The buffer's backing address_space's private_lock must be held
  429. */
  430. static void __remove_assoc_queue(struct buffer_head *bh)
  431. {
  432. list_del_init(&bh->b_assoc_buffers);
  433. WARN_ON(!bh->b_assoc_map);
  434. bh->b_assoc_map = NULL;
  435. }
  436. int inode_has_buffers(struct inode *inode)
  437. {
  438. return !list_empty(&inode->i_data.private_list);
  439. }
  440. /*
  441. * osync is designed to support O_SYNC io. It waits synchronously for
  442. * all already-submitted IO to complete, but does not queue any new
  443. * writes to the disk.
  444. *
  445. * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
  446. * you dirty the buffers, and then use osync_inode_buffers to wait for
  447. * completion. Any other dirty buffers which are not yet queued for
  448. * write will not be flushed to disk by the osync.
  449. */
  450. static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
  451. {
  452. struct buffer_head *bh;
  453. struct list_head *p;
  454. int err = 0;
  455. spin_lock(lock);
  456. repeat:
  457. list_for_each_prev(p, list) {
  458. bh = BH_ENTRY(p);
  459. if (buffer_locked(bh)) {
  460. get_bh(bh);
  461. spin_unlock(lock);
  462. wait_on_buffer(bh);
  463. if (!buffer_uptodate(bh))
  464. err = -EIO;
  465. brelse(bh);
  466. spin_lock(lock);
  467. goto repeat;
  468. }
  469. }
  470. spin_unlock(lock);
  471. return err;
  472. }
  473. void emergency_thaw_bdev(struct super_block *sb)
  474. {
  475. while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
  476. printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
  477. }
  478. /**
  479. * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
  480. * @mapping: the mapping which wants those buffers written
  481. *
  482. * Starts I/O against the buffers at mapping->private_list, and waits upon
  483. * that I/O.
  484. *
  485. * Basically, this is a convenience function for fsync().
  486. * @mapping is a file or directory which needs those buffers to be written for
  487. * a successful fsync().
  488. */
  489. int sync_mapping_buffers(struct address_space *mapping)
  490. {
  491. struct address_space *buffer_mapping = mapping->private_data;
  492. if (buffer_mapping == NULL || list_empty(&mapping->private_list))
  493. return 0;
  494. return fsync_buffers_list(&buffer_mapping->private_lock,
  495. &mapping->private_list);
  496. }
  497. EXPORT_SYMBOL(sync_mapping_buffers);
  498. /*
  499. * Called when we've recently written block `bblock', and it is known that
  500. * `bblock' was for a buffer_boundary() buffer. This means that the block at
  501. * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
  502. * dirty, schedule it for IO. So that indirects merge nicely with their data.
  503. */
  504. void write_boundary_block(struct block_device *bdev,
  505. sector_t bblock, unsigned blocksize)
  506. {
  507. struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
  508. if (bh) {
  509. if (buffer_dirty(bh))
  510. ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
  511. put_bh(bh);
  512. }
  513. }
  514. void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
  515. {
  516. struct address_space *mapping = inode->i_mapping;
  517. struct address_space *buffer_mapping = bh->b_page->mapping;
  518. mark_buffer_dirty(bh);
  519. if (!mapping->private_data) {
  520. mapping->private_data = buffer_mapping;
  521. } else {
  522. BUG_ON(mapping->private_data != buffer_mapping);
  523. }
  524. if (!bh->b_assoc_map) {
  525. spin_lock(&buffer_mapping->private_lock);
  526. list_move_tail(&bh->b_assoc_buffers,
  527. &mapping->private_list);
  528. bh->b_assoc_map = mapping;
  529. spin_unlock(&buffer_mapping->private_lock);
  530. }
  531. }
  532. EXPORT_SYMBOL(mark_buffer_dirty_inode);
  533. /*
  534. * Mark the page dirty, and set it dirty in the page cache, and mark the inode
  535. * dirty.
  536. *
  537. * If warn is true, then emit a warning if the page is not uptodate and has
  538. * not been truncated.
  539. *
  540. * The caller must hold lock_page_memcg().
  541. */
  542. void __set_page_dirty(struct page *page, struct address_space *mapping,
  543. int warn)
  544. {
  545. unsigned long flags;
  546. xa_lock_irqsave(&mapping->i_pages, flags);
  547. if (page->mapping) { /* Race with truncate? */
  548. WARN_ON_ONCE(warn && !PageUptodate(page));
  549. account_page_dirtied(page, mapping);
  550. __xa_set_mark(&mapping->i_pages, page_index(page),
  551. PAGECACHE_TAG_DIRTY);
  552. }
  553. xa_unlock_irqrestore(&mapping->i_pages, flags);
  554. }
  555. EXPORT_SYMBOL_GPL(__set_page_dirty);
  556. /*
  557. * Add a page to the dirty page list.
  558. *
  559. * It is a sad fact of life that this function is called from several places
  560. * deeply under spinlocking. It may not sleep.
  561. *
  562. * If the page has buffers, the uptodate buffers are set dirty, to preserve
  563. * dirty-state coherency between the page and the buffers. It the page does
  564. * not have buffers then when they are later attached they will all be set
  565. * dirty.
  566. *
  567. * The buffers are dirtied before the page is dirtied. There's a small race
  568. * window in which a writepage caller may see the page cleanness but not the
  569. * buffer dirtiness. That's fine. If this code were to set the page dirty
  570. * before the buffers, a concurrent writepage caller could clear the page dirty
  571. * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
  572. * page on the dirty page list.
  573. *
  574. * We use private_lock to lock against try_to_free_buffers while using the
  575. * page's buffer list. Also use this to protect against clean buffers being
  576. * added to the page after it was set dirty.
  577. *
  578. * FIXME: may need to call ->reservepage here as well. That's rather up to the
  579. * address_space though.
  580. */
  581. int __set_page_dirty_buffers(struct page *page)
  582. {
  583. int newly_dirty;
  584. struct address_space *mapping = page_mapping(page);
  585. if (unlikely(!mapping))
  586. return !TestSetPageDirty(page);
  587. spin_lock(&mapping->private_lock);
  588. if (page_has_buffers(page)) {
  589. struct buffer_head *head = page_buffers(page);
  590. struct buffer_head *bh = head;
  591. do {
  592. set_buffer_dirty(bh);
  593. bh = bh->b_this_page;
  594. } while (bh != head);
  595. }
  596. /*
  597. * Lock out page->mem_cgroup migration to keep PageDirty
  598. * synchronized with per-memcg dirty page counters.
  599. */
  600. lock_page_memcg(page);
  601. newly_dirty = !TestSetPageDirty(page);
  602. spin_unlock(&mapping->private_lock);
  603. if (newly_dirty)
  604. __set_page_dirty(page, mapping, 1);
  605. unlock_page_memcg(page);
  606. if (newly_dirty)
  607. __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
  608. return newly_dirty;
  609. }
  610. EXPORT_SYMBOL_NS(__set_page_dirty_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
  611. /*
  612. * Write out and wait upon a list of buffers.
  613. *
  614. * We have conflicting pressures: we want to make sure that all
  615. * initially dirty buffers get waited on, but that any subsequently
  616. * dirtied buffers don't. After all, we don't want fsync to last
  617. * forever if somebody is actively writing to the file.
  618. *
  619. * Do this in two main stages: first we copy dirty buffers to a
  620. * temporary inode list, queueing the writes as we go. Then we clean
  621. * up, waiting for those writes to complete.
  622. *
  623. * During this second stage, any subsequent updates to the file may end
  624. * up refiling the buffer on the original inode's dirty list again, so
  625. * there is a chance we will end up with a buffer queued for write but
  626. * not yet completed on that list. So, as a final cleanup we go through
  627. * the osync code to catch these locked, dirty buffers without requeuing
  628. * any newly dirty buffers for write.
  629. */
  630. static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
  631. {
  632. struct buffer_head *bh;
  633. struct list_head tmp;
  634. struct address_space *mapping;
  635. int err = 0, err2;
  636. struct blk_plug plug;
  637. INIT_LIST_HEAD(&tmp);
  638. blk_start_plug(&plug);
  639. spin_lock(lock);
  640. while (!list_empty(list)) {
  641. bh = BH_ENTRY(list->next);
  642. mapping = bh->b_assoc_map;
  643. __remove_assoc_queue(bh);
  644. /* Avoid race with mark_buffer_dirty_inode() which does
  645. * a lockless check and we rely on seeing the dirty bit */
  646. smp_mb();
  647. if (buffer_dirty(bh) || buffer_locked(bh)) {
  648. list_add(&bh->b_assoc_buffers, &tmp);
  649. bh->b_assoc_map = mapping;
  650. if (buffer_dirty(bh)) {
  651. get_bh(bh);
  652. spin_unlock(lock);
  653. /*
  654. * Ensure any pending I/O completes so that
  655. * write_dirty_buffer() actually writes the
  656. * current contents - it is a noop if I/O is
  657. * still in flight on potentially older
  658. * contents.
  659. */
  660. write_dirty_buffer(bh, REQ_SYNC);
  661. /*
  662. * Kick off IO for the previous mapping. Note
  663. * that we will not run the very last mapping,
  664. * wait_on_buffer() will do that for us
  665. * through sync_buffer().
  666. */
  667. brelse(bh);
  668. spin_lock(lock);
  669. }
  670. }
  671. }
  672. spin_unlock(lock);
  673. blk_finish_plug(&plug);
  674. spin_lock(lock);
  675. while (!list_empty(&tmp)) {
  676. bh = BH_ENTRY(tmp.prev);
  677. get_bh(bh);
  678. mapping = bh->b_assoc_map;
  679. __remove_assoc_queue(bh);
  680. /* Avoid race with mark_buffer_dirty_inode() which does
  681. * a lockless check and we rely on seeing the dirty bit */
  682. smp_mb();
  683. if (buffer_dirty(bh)) {
  684. list_add(&bh->b_assoc_buffers,
  685. &mapping->private_list);
  686. bh->b_assoc_map = mapping;
  687. }
  688. spin_unlock(lock);
  689. wait_on_buffer(bh);
  690. if (!buffer_uptodate(bh))
  691. err = -EIO;
  692. brelse(bh);
  693. spin_lock(lock);
  694. }
  695. spin_unlock(lock);
  696. err2 = osync_buffers_list(lock, list);
  697. if (err)
  698. return err;
  699. else
  700. return err2;
  701. }
  702. /*
  703. * Invalidate any and all dirty buffers on a given inode. We are
  704. * probably unmounting the fs, but that doesn't mean we have already
  705. * done a sync(). Just drop the buffers from the inode list.
  706. *
  707. * NOTE: we take the inode's blockdev's mapping's private_lock. Which
  708. * assumes that all the buffers are against the blockdev. Not true
  709. * for reiserfs.
  710. */
  711. void invalidate_inode_buffers(struct inode *inode)
  712. {
  713. if (inode_has_buffers(inode)) {
  714. struct address_space *mapping = &inode->i_data;
  715. struct list_head *list = &mapping->private_list;
  716. struct address_space *buffer_mapping = mapping->private_data;
  717. spin_lock(&buffer_mapping->private_lock);
  718. while (!list_empty(list))
  719. __remove_assoc_queue(BH_ENTRY(list->next));
  720. spin_unlock(&buffer_mapping->private_lock);
  721. }
  722. }
  723. EXPORT_SYMBOL(invalidate_inode_buffers);
  724. /*
  725. * Remove any clean buffers from the inode's buffer list. This is called
  726. * when we're trying to free the inode itself. Those buffers can pin it.
  727. *
  728. * Returns true if all buffers were removed.
  729. */
  730. int remove_inode_buffers(struct inode *inode)
  731. {
  732. int ret = 1;
  733. if (inode_has_buffers(inode)) {
  734. struct address_space *mapping = &inode->i_data;
  735. struct list_head *list = &mapping->private_list;
  736. struct address_space *buffer_mapping = mapping->private_data;
  737. spin_lock(&buffer_mapping->private_lock);
  738. while (!list_empty(list)) {
  739. struct buffer_head *bh = BH_ENTRY(list->next);
  740. if (buffer_dirty(bh)) {
  741. ret = 0;
  742. break;
  743. }
  744. __remove_assoc_queue(bh);
  745. }
  746. spin_unlock(&buffer_mapping->private_lock);
  747. }
  748. return ret;
  749. }
  750. /*
  751. * Create the appropriate buffers when given a page for data area and
  752. * the size of each buffer.. Use the bh->b_this_page linked list to
  753. * follow the buffers created. Return NULL if unable to create more
  754. * buffers.
  755. *
  756. * The retry flag is used to differentiate async IO (paging, swapping)
  757. * which may not fail from ordinary buffer allocations.
  758. */
  759. struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
  760. bool retry)
  761. {
  762. struct buffer_head *bh, *head;
  763. gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
  764. long offset;
  765. struct mem_cgroup *memcg, *old_memcg;
  766. if (retry)
  767. gfp |= __GFP_NOFAIL;
  768. memcg = get_mem_cgroup_from_page(page);
  769. old_memcg = set_active_memcg(memcg);
  770. head = NULL;
  771. offset = PAGE_SIZE;
  772. while ((offset -= size) >= 0) {
  773. bh = alloc_buffer_head(gfp);
  774. if (!bh)
  775. goto no_grow;
  776. bh->b_this_page = head;
  777. bh->b_blocknr = -1;
  778. head = bh;
  779. bh->b_size = size;
  780. /* Link the buffer to its page */
  781. set_bh_page(bh, page, offset);
  782. }
  783. out:
  784. set_active_memcg(old_memcg);
  785. mem_cgroup_put(memcg);
  786. return head;
  787. /*
  788. * In case anything failed, we just free everything we got.
  789. */
  790. no_grow:
  791. if (head) {
  792. do {
  793. bh = head;
  794. head = head->b_this_page;
  795. free_buffer_head(bh);
  796. } while (head);
  797. }
  798. goto out;
  799. }
  800. EXPORT_SYMBOL_GPL(alloc_page_buffers);
  801. static inline void
  802. link_dev_buffers(struct page *page, struct buffer_head *head)
  803. {
  804. struct buffer_head *bh, *tail;
  805. bh = head;
  806. do {
  807. tail = bh;
  808. bh = bh->b_this_page;
  809. } while (bh);
  810. tail->b_this_page = head;
  811. attach_page_private(page, head);
  812. }
  813. static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
  814. {
  815. sector_t retval = ~((sector_t)0);
  816. loff_t sz = i_size_read(bdev->bd_inode);
  817. if (sz) {
  818. unsigned int sizebits = blksize_bits(size);
  819. retval = (sz >> sizebits);
  820. }
  821. return retval;
  822. }
  823. /*
  824. * Initialise the state of a blockdev page's buffers.
  825. */
  826. static sector_t
  827. init_page_buffers(struct page *page, struct block_device *bdev,
  828. sector_t block, int size)
  829. {
  830. struct buffer_head *head = page_buffers(page);
  831. struct buffer_head *bh = head;
  832. int uptodate = PageUptodate(page);
  833. sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
  834. do {
  835. if (!buffer_mapped(bh)) {
  836. bh->b_end_io = NULL;
  837. bh->b_private = NULL;
  838. bh->b_bdev = bdev;
  839. bh->b_blocknr = block;
  840. if (uptodate)
  841. set_buffer_uptodate(bh);
  842. if (block < end_block)
  843. set_buffer_mapped(bh);
  844. }
  845. block++;
  846. bh = bh->b_this_page;
  847. } while (bh != head);
  848. /*
  849. * Caller needs to validate requested block against end of device.
  850. */
  851. return end_block;
  852. }
  853. /*
  854. * Create the page-cache page that contains the requested block.
  855. *
  856. * This is used purely for blockdev mappings.
  857. */
  858. static int
  859. grow_dev_page(struct block_device *bdev, sector_t block,
  860. pgoff_t index, int size, int sizebits, gfp_t gfp)
  861. {
  862. struct inode *inode = bdev->bd_inode;
  863. struct page *page;
  864. struct buffer_head *bh;
  865. sector_t end_block;
  866. int ret = 0;
  867. gfp_t gfp_mask;
  868. gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
  869. /*
  870. * XXX: __getblk_slow() can not really deal with failure and
  871. * will endlessly loop on improvised global reclaim. Prefer
  872. * looping in the allocator rather than here, at least that
  873. * code knows what it's doing.
  874. */
  875. gfp_mask |= __GFP_NOFAIL;
  876. page = find_or_create_page(inode->i_mapping, index, gfp_mask);
  877. BUG_ON(!PageLocked(page));
  878. if (page_has_buffers(page)) {
  879. bh = page_buffers(page);
  880. if (bh->b_size == size) {
  881. end_block = init_page_buffers(page, bdev,
  882. (sector_t)index << sizebits,
  883. size);
  884. goto done;
  885. }
  886. if (!try_to_free_buffers(page))
  887. goto failed;
  888. }
  889. /*
  890. * Allocate some buffers for this page
  891. */
  892. bh = alloc_page_buffers(page, size, true);
  893. /*
  894. * Link the page to the buffers and initialise them. Take the
  895. * lock to be atomic wrt __find_get_block(), which does not
  896. * run under the page lock.
  897. */
  898. spin_lock(&inode->i_mapping->private_lock);
  899. link_dev_buffers(page, bh);
  900. end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
  901. size);
  902. spin_unlock(&inode->i_mapping->private_lock);
  903. done:
  904. ret = (block < end_block) ? 1 : -ENXIO;
  905. failed:
  906. unlock_page(page);
  907. put_page(page);
  908. return ret;
  909. }
  910. /*
  911. * Create buffers for the specified block device block's page. If
  912. * that page was dirty, the buffers are set dirty also.
  913. */
  914. static int
  915. grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
  916. {
  917. pgoff_t index;
  918. int sizebits;
  919. sizebits = -1;
  920. do {
  921. sizebits++;
  922. } while ((size << sizebits) < PAGE_SIZE);
  923. index = block >> sizebits;
  924. /*
  925. * Check for a block which wants to lie outside our maximum possible
  926. * pagecache index. (this comparison is done using sector_t types).
  927. */
  928. if (unlikely(index != block >> sizebits)) {
  929. printk(KERN_ERR "%s: requested out-of-range block %llu for "
  930. "device %pg\n",
  931. __func__, (unsigned long long)block,
  932. bdev);
  933. return -EIO;
  934. }
  935. /* Create a page with the proper size buffers.. */
  936. return grow_dev_page(bdev, block, index, size, sizebits, gfp);
  937. }
  938. static struct buffer_head *
  939. __getblk_slow(struct block_device *bdev, sector_t block,
  940. unsigned size, gfp_t gfp)
  941. {
  942. /* Size must be multiple of hard sectorsize */
  943. if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
  944. (size < 512 || size > PAGE_SIZE))) {
  945. printk(KERN_ERR "getblk(): invalid block size %d requested\n",
  946. size);
  947. printk(KERN_ERR "logical block size: %d\n",
  948. bdev_logical_block_size(bdev));
  949. dump_stack();
  950. return NULL;
  951. }
  952. for (;;) {
  953. struct buffer_head *bh;
  954. int ret;
  955. bh = __find_get_block(bdev, block, size);
  956. if (bh)
  957. return bh;
  958. ret = grow_buffers(bdev, block, size, gfp);
  959. if (ret < 0)
  960. return NULL;
  961. }
  962. }
  963. /*
  964. * The relationship between dirty buffers and dirty pages:
  965. *
  966. * Whenever a page has any dirty buffers, the page's dirty bit is set, and
  967. * the page is tagged dirty in the page cache.
  968. *
  969. * At all times, the dirtiness of the buffers represents the dirtiness of
  970. * subsections of the page. If the page has buffers, the page dirty bit is
  971. * merely a hint about the true dirty state.
  972. *
  973. * When a page is set dirty in its entirety, all its buffers are marked dirty
  974. * (if the page has buffers).
  975. *
  976. * When a buffer is marked dirty, its page is dirtied, but the page's other
  977. * buffers are not.
  978. *
  979. * Also. When blockdev buffers are explicitly read with bread(), they
  980. * individually become uptodate. But their backing page remains not
  981. * uptodate - even if all of its buffers are uptodate. A subsequent
  982. * block_read_full_page() against that page will discover all the uptodate
  983. * buffers, will set the page uptodate and will perform no I/O.
  984. */
  985. /**
  986. * mark_buffer_dirty - mark a buffer_head as needing writeout
  987. * @bh: the buffer_head to mark dirty
  988. *
  989. * mark_buffer_dirty() will set the dirty bit against the buffer, then set
  990. * its backing page dirty, then tag the page as dirty in the page cache
  991. * and then attach the address_space's inode to its superblock's dirty
  992. * inode list.
  993. *
  994. * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
  995. * i_pages lock and mapping->host->i_lock.
  996. */
  997. void mark_buffer_dirty(struct buffer_head *bh)
  998. {
  999. WARN_ON_ONCE(!buffer_uptodate(bh));
  1000. trace_block_dirty_buffer(bh);
  1001. /*
  1002. * Very *carefully* optimize the it-is-already-dirty case.
  1003. *
  1004. * Don't let the final "is it dirty" escape to before we
  1005. * perhaps modified the buffer.
  1006. */
  1007. if (buffer_dirty(bh)) {
  1008. smp_mb();
  1009. if (buffer_dirty(bh))
  1010. return;
  1011. }
  1012. if (!test_set_buffer_dirty(bh)) {
  1013. struct page *page = bh->b_page;
  1014. struct address_space *mapping = NULL;
  1015. lock_page_memcg(page);
  1016. if (!TestSetPageDirty(page)) {
  1017. mapping = page_mapping(page);
  1018. if (mapping)
  1019. __set_page_dirty(page, mapping, 0);
  1020. }
  1021. unlock_page_memcg(page);
  1022. if (mapping)
  1023. __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
  1024. }
  1025. }
  1026. EXPORT_SYMBOL_NS(mark_buffer_dirty, ANDROID_GKI_VFS_EXPORT_ONLY);
  1027. void mark_buffer_write_io_error(struct buffer_head *bh)
  1028. {
  1029. struct super_block *sb;
  1030. set_buffer_write_io_error(bh);
  1031. /* FIXME: do we need to set this in both places? */
  1032. if (bh->b_page && bh->b_page->mapping)
  1033. mapping_set_error(bh->b_page->mapping, -EIO);
  1034. if (bh->b_assoc_map)
  1035. mapping_set_error(bh->b_assoc_map, -EIO);
  1036. rcu_read_lock();
  1037. sb = READ_ONCE(bh->b_bdev->bd_super);
  1038. if (sb)
  1039. errseq_set(&sb->s_wb_err, -EIO);
  1040. rcu_read_unlock();
  1041. }
  1042. EXPORT_SYMBOL_NS(mark_buffer_write_io_error, ANDROID_GKI_VFS_EXPORT_ONLY);
  1043. /*
  1044. * Decrement a buffer_head's reference count. If all buffers against a page
  1045. * have zero reference count, are clean and unlocked, and if the page is clean
  1046. * and unlocked then try_to_free_buffers() may strip the buffers from the page
  1047. * in preparation for freeing it (sometimes, rarely, buffers are removed from
  1048. * a page but it ends up not being freed, and buffers may later be reattached).
  1049. */
  1050. void __brelse(struct buffer_head * buf)
  1051. {
  1052. if (atomic_read(&buf->b_count)) {
  1053. put_bh(buf);
  1054. return;
  1055. }
  1056. WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
  1057. }
  1058. EXPORT_SYMBOL_NS(__brelse, ANDROID_GKI_VFS_EXPORT_ONLY);
  1059. /*
  1060. * bforget() is like brelse(), except it discards any
  1061. * potentially dirty data.
  1062. */
  1063. void __bforget(struct buffer_head *bh)
  1064. {
  1065. clear_buffer_dirty(bh);
  1066. if (bh->b_assoc_map) {
  1067. struct address_space *buffer_mapping = bh->b_page->mapping;
  1068. spin_lock(&buffer_mapping->private_lock);
  1069. list_del_init(&bh->b_assoc_buffers);
  1070. bh->b_assoc_map = NULL;
  1071. spin_unlock(&buffer_mapping->private_lock);
  1072. }
  1073. __brelse(bh);
  1074. }
  1075. EXPORT_SYMBOL_NS(__bforget, ANDROID_GKI_VFS_EXPORT_ONLY);
  1076. static struct buffer_head *__bread_slow(struct buffer_head *bh)
  1077. {
  1078. lock_buffer(bh);
  1079. if (buffer_uptodate(bh)) {
  1080. unlock_buffer(bh);
  1081. return bh;
  1082. } else {
  1083. get_bh(bh);
  1084. bh->b_end_io = end_buffer_read_sync;
  1085. submit_bh(REQ_OP_READ, 0, bh);
  1086. wait_on_buffer(bh);
  1087. if (buffer_uptodate(bh))
  1088. return bh;
  1089. }
  1090. brelse(bh);
  1091. return NULL;
  1092. }
  1093. /*
  1094. * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
  1095. * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
  1096. * refcount elevated by one when they're in an LRU. A buffer can only appear
  1097. * once in a particular CPU's LRU. A single buffer can be present in multiple
  1098. * CPU's LRUs at the same time.
  1099. *
  1100. * This is a transparent caching front-end to sb_bread(), sb_getblk() and
  1101. * sb_find_get_block().
  1102. *
  1103. * The LRUs themselves only need locking against invalidate_bh_lrus. We use
  1104. * a local interrupt disable for that.
  1105. */
  1106. #define BH_LRU_SIZE 16
  1107. struct bh_lru {
  1108. struct buffer_head *bhs[BH_LRU_SIZE];
  1109. };
  1110. static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
  1111. #ifdef CONFIG_SMP
  1112. #define bh_lru_lock() local_irq_disable()
  1113. #define bh_lru_unlock() local_irq_enable()
  1114. #else
  1115. #define bh_lru_lock() preempt_disable()
  1116. #define bh_lru_unlock() preempt_enable()
  1117. #endif
  1118. static inline void check_irqs_on(void)
  1119. {
  1120. #ifdef irqs_disabled
  1121. BUG_ON(irqs_disabled());
  1122. #endif
  1123. }
  1124. /*
  1125. * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
  1126. * inserted at the front, and the buffer_head at the back if any is evicted.
  1127. * Or, if already in the LRU it is moved to the front.
  1128. */
  1129. static void bh_lru_install(struct buffer_head *bh)
  1130. {
  1131. struct buffer_head *evictee = bh;
  1132. struct bh_lru *b;
  1133. int i;
  1134. check_irqs_on();
  1135. /*
  1136. * the refcount of buffer_head in bh_lru prevents dropping the
  1137. * attached page(i.e., try_to_free_buffers) so it could cause
  1138. * failing page migration.
  1139. * Skip putting upcoming bh into bh_lru until migration is done.
  1140. */
  1141. if (lru_cache_disabled())
  1142. return;
  1143. bh_lru_lock();
  1144. b = this_cpu_ptr(&bh_lrus);
  1145. for (i = 0; i < BH_LRU_SIZE; i++) {
  1146. swap(evictee, b->bhs[i]);
  1147. if (evictee == bh) {
  1148. bh_lru_unlock();
  1149. return;
  1150. }
  1151. }
  1152. get_bh(bh);
  1153. bh_lru_unlock();
  1154. brelse(evictee);
  1155. }
  1156. /*
  1157. * Look up the bh in this cpu's LRU. If it's there, move it to the head.
  1158. */
  1159. static struct buffer_head *
  1160. lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
  1161. {
  1162. struct buffer_head *ret = NULL;
  1163. unsigned int i;
  1164. check_irqs_on();
  1165. bh_lru_lock();
  1166. for (i = 0; i < BH_LRU_SIZE; i++) {
  1167. struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
  1168. if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
  1169. bh->b_size == size) {
  1170. if (i) {
  1171. while (i) {
  1172. __this_cpu_write(bh_lrus.bhs[i],
  1173. __this_cpu_read(bh_lrus.bhs[i - 1]));
  1174. i--;
  1175. }
  1176. __this_cpu_write(bh_lrus.bhs[0], bh);
  1177. }
  1178. get_bh(bh);
  1179. ret = bh;
  1180. break;
  1181. }
  1182. }
  1183. bh_lru_unlock();
  1184. return ret;
  1185. }
  1186. /*
  1187. * Perform a pagecache lookup for the matching buffer. If it's there, refresh
  1188. * it in the LRU and mark it as accessed. If it is not present then return
  1189. * NULL
  1190. */
  1191. struct buffer_head *
  1192. __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
  1193. {
  1194. struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
  1195. if (bh == NULL) {
  1196. /* __find_get_block_slow will mark the page accessed */
  1197. bh = __find_get_block_slow(bdev, block);
  1198. if (bh)
  1199. bh_lru_install(bh);
  1200. } else
  1201. touch_buffer(bh);
  1202. return bh;
  1203. }
  1204. EXPORT_SYMBOL(__find_get_block);
  1205. /*
  1206. * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
  1207. * which corresponds to the passed block_device, block and size. The
  1208. * returned buffer has its reference count incremented.
  1209. *
  1210. * __getblk_gfp() will lock up the machine if grow_dev_page's
  1211. * try_to_free_buffers() attempt is failing. FIXME, perhaps?
  1212. */
  1213. struct buffer_head *
  1214. __getblk_gfp(struct block_device *bdev, sector_t block,
  1215. unsigned size, gfp_t gfp)
  1216. {
  1217. struct buffer_head *bh = __find_get_block(bdev, block, size);
  1218. might_sleep();
  1219. if (bh == NULL)
  1220. bh = __getblk_slow(bdev, block, size, gfp);
  1221. return bh;
  1222. }
  1223. EXPORT_SYMBOL(__getblk_gfp);
  1224. /*
  1225. * Do async read-ahead on a buffer..
  1226. */
  1227. void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
  1228. {
  1229. struct buffer_head *bh = __getblk(bdev, block, size);
  1230. if (likely(bh)) {
  1231. ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
  1232. brelse(bh);
  1233. }
  1234. }
  1235. EXPORT_SYMBOL_NS(__breadahead, ANDROID_GKI_VFS_EXPORT_ONLY);
  1236. void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size,
  1237. gfp_t gfp)
  1238. {
  1239. struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
  1240. if (likely(bh)) {
  1241. ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
  1242. brelse(bh);
  1243. }
  1244. }
  1245. EXPORT_SYMBOL(__breadahead_gfp);
  1246. /**
  1247. * __bread_gfp() - reads a specified block and returns the bh
  1248. * @bdev: the block_device to read from
  1249. * @block: number of block
  1250. * @size: size (in bytes) to read
  1251. * @gfp: page allocation flag
  1252. *
  1253. * Reads a specified block, and returns buffer head that contains it.
  1254. * The page cache can be allocated from non-movable area
  1255. * not to prevent page migration if you set gfp to zero.
  1256. * It returns NULL if the block was unreadable.
  1257. */
  1258. struct buffer_head *
  1259. __bread_gfp(struct block_device *bdev, sector_t block,
  1260. unsigned size, gfp_t gfp)
  1261. {
  1262. struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
  1263. if (likely(bh) && !buffer_uptodate(bh))
  1264. bh = __bread_slow(bh);
  1265. return bh;
  1266. }
  1267. EXPORT_SYMBOL_NS(__bread_gfp, ANDROID_GKI_VFS_EXPORT_ONLY);
  1268. static void __invalidate_bh_lrus(struct bh_lru *b)
  1269. {
  1270. int i;
  1271. for (i = 0; i < BH_LRU_SIZE; i++) {
  1272. brelse(b->bhs[i]);
  1273. b->bhs[i] = NULL;
  1274. }
  1275. }
  1276. /*
  1277. * invalidate_bh_lrus() is called rarely - but not only at unmount.
  1278. * This doesn't race because it runs in each cpu either in irq
  1279. * or with preempt disabled.
  1280. */
  1281. static void invalidate_bh_lru(void *arg)
  1282. {
  1283. struct bh_lru *b = &get_cpu_var(bh_lrus);
  1284. __invalidate_bh_lrus(b);
  1285. put_cpu_var(bh_lrus);
  1286. }
  1287. bool has_bh_in_lru(int cpu, void *dummy)
  1288. {
  1289. struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
  1290. int i;
  1291. for (i = 0; i < BH_LRU_SIZE; i++) {
  1292. if (b->bhs[i])
  1293. return true;
  1294. }
  1295. return false;
  1296. }
  1297. void invalidate_bh_lrus(void)
  1298. {
  1299. on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
  1300. }
  1301. EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
  1302. /*
  1303. * It's called from workqueue context so we need a bh_lru_lock to close
  1304. * the race with preemption/irq.
  1305. */
  1306. void invalidate_bh_lrus_cpu(void)
  1307. {
  1308. struct bh_lru *b;
  1309. bh_lru_lock();
  1310. b = this_cpu_ptr(&bh_lrus);
  1311. __invalidate_bh_lrus(b);
  1312. bh_lru_unlock();
  1313. }
  1314. void set_bh_page(struct buffer_head *bh,
  1315. struct page *page, unsigned long offset)
  1316. {
  1317. bh->b_page = page;
  1318. BUG_ON(offset >= PAGE_SIZE);
  1319. if (PageHighMem(page))
  1320. /*
  1321. * This catches illegal uses and preserves the offset:
  1322. */
  1323. bh->b_data = (char *)(0 + offset);
  1324. else
  1325. bh->b_data = page_address(page) + offset;
  1326. }
  1327. EXPORT_SYMBOL(set_bh_page);
  1328. /*
  1329. * Called when truncating a buffer on a page completely.
  1330. */
  1331. /* Bits that are cleared during an invalidate */
  1332. #define BUFFER_FLAGS_DISCARD \
  1333. (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
  1334. 1 << BH_Delay | 1 << BH_Unwritten)
  1335. static void discard_buffer(struct buffer_head * bh)
  1336. {
  1337. unsigned long b_state, b_state_old;
  1338. lock_buffer(bh);
  1339. clear_buffer_dirty(bh);
  1340. bh->b_bdev = NULL;
  1341. b_state = bh->b_state;
  1342. for (;;) {
  1343. b_state_old = cmpxchg(&bh->b_state, b_state,
  1344. (b_state & ~BUFFER_FLAGS_DISCARD));
  1345. if (b_state_old == b_state)
  1346. break;
  1347. b_state = b_state_old;
  1348. }
  1349. unlock_buffer(bh);
  1350. }
  1351. /**
  1352. * block_invalidatepage - invalidate part or all of a buffer-backed page
  1353. *
  1354. * @page: the page which is affected
  1355. * @offset: start of the range to invalidate
  1356. * @length: length of the range to invalidate
  1357. *
  1358. * block_invalidatepage() is called when all or part of the page has become
  1359. * invalidated by a truncate operation.
  1360. *
  1361. * block_invalidatepage() does not have to release all buffers, but it must
  1362. * ensure that no dirty buffer is left outside @offset and that no I/O
  1363. * is underway against any of the blocks which are outside the truncation
  1364. * point. Because the caller is about to free (and possibly reuse) those
  1365. * blocks on-disk.
  1366. */
  1367. void block_invalidatepage(struct page *page, unsigned int offset,
  1368. unsigned int length)
  1369. {
  1370. struct buffer_head *head, *bh, *next;
  1371. unsigned int curr_off = 0;
  1372. unsigned int stop = length + offset;
  1373. BUG_ON(!PageLocked(page));
  1374. if (!page_has_buffers(page))
  1375. goto out;
  1376. /*
  1377. * Check for overflow
  1378. */
  1379. BUG_ON(stop > PAGE_SIZE || stop < length);
  1380. head = page_buffers(page);
  1381. bh = head;
  1382. do {
  1383. unsigned int next_off = curr_off + bh->b_size;
  1384. next = bh->b_this_page;
  1385. /*
  1386. * Are we still fully in range ?
  1387. */
  1388. if (next_off > stop)
  1389. goto out;
  1390. /*
  1391. * is this block fully invalidated?
  1392. */
  1393. if (offset <= curr_off)
  1394. discard_buffer(bh);
  1395. curr_off = next_off;
  1396. bh = next;
  1397. } while (bh != head);
  1398. /*
  1399. * We release buffers only if the entire page is being invalidated.
  1400. * The get_block cached value has been unconditionally invalidated,
  1401. * so real IO is not possible anymore.
  1402. */
  1403. if (length == PAGE_SIZE)
  1404. try_to_release_page(page, 0);
  1405. out:
  1406. return;
  1407. }
  1408. EXPORT_SYMBOL_NS(block_invalidatepage, ANDROID_GKI_VFS_EXPORT_ONLY);
  1409. /*
  1410. * We attach and possibly dirty the buffers atomically wrt
  1411. * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
  1412. * is already excluded via the page lock.
  1413. */
  1414. void create_empty_buffers(struct page *page,
  1415. unsigned long blocksize, unsigned long b_state)
  1416. {
  1417. struct buffer_head *bh, *head, *tail;
  1418. head = alloc_page_buffers(page, blocksize, true);
  1419. bh = head;
  1420. do {
  1421. bh->b_state |= b_state;
  1422. tail = bh;
  1423. bh = bh->b_this_page;
  1424. } while (bh);
  1425. tail->b_this_page = head;
  1426. spin_lock(&page->mapping->private_lock);
  1427. if (PageUptodate(page) || PageDirty(page)) {
  1428. bh = head;
  1429. do {
  1430. if (PageDirty(page))
  1431. set_buffer_dirty(bh);
  1432. if (PageUptodate(page))
  1433. set_buffer_uptodate(bh);
  1434. bh = bh->b_this_page;
  1435. } while (bh != head);
  1436. }
  1437. attach_page_private(page, head);
  1438. spin_unlock(&page->mapping->private_lock);
  1439. }
  1440. EXPORT_SYMBOL_NS(create_empty_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
  1441. /**
  1442. * clean_bdev_aliases: clean a range of buffers in block device
  1443. * @bdev: Block device to clean buffers in
  1444. * @block: Start of a range of blocks to clean
  1445. * @len: Number of blocks to clean
  1446. *
  1447. * We are taking a range of blocks for data and we don't want writeback of any
  1448. * buffer-cache aliases starting from return from this function and until the
  1449. * moment when something will explicitly mark the buffer dirty (hopefully that
  1450. * will not happen until we will free that block ;-) We don't even need to mark
  1451. * it not-uptodate - nobody can expect anything from a newly allocated buffer
  1452. * anyway. We used to use unmap_buffer() for such invalidation, but that was
  1453. * wrong. We definitely don't want to mark the alias unmapped, for example - it
  1454. * would confuse anyone who might pick it with bread() afterwards...
  1455. *
  1456. * Also.. Note that bforget() doesn't lock the buffer. So there can be
  1457. * writeout I/O going on against recently-freed buffers. We don't wait on that
  1458. * I/O in bforget() - it's more efficient to wait on the I/O only if we really
  1459. * need to. That happens here.
  1460. */
  1461. void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
  1462. {
  1463. struct inode *bd_inode = bdev->bd_inode;
  1464. struct address_space *bd_mapping = bd_inode->i_mapping;
  1465. struct pagevec pvec;
  1466. pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
  1467. pgoff_t end;
  1468. int i, count;
  1469. struct buffer_head *bh;
  1470. struct buffer_head *head;
  1471. end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
  1472. pagevec_init(&pvec);
  1473. while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
  1474. count = pagevec_count(&pvec);
  1475. for (i = 0; i < count; i++) {
  1476. struct page *page = pvec.pages[i];
  1477. if (!page_has_buffers(page))
  1478. continue;
  1479. /*
  1480. * We use page lock instead of bd_mapping->private_lock
  1481. * to pin buffers here since we can afford to sleep and
  1482. * it scales better than a global spinlock lock.
  1483. */
  1484. lock_page(page);
  1485. /* Recheck when the page is locked which pins bhs */
  1486. if (!page_has_buffers(page))
  1487. goto unlock_page;
  1488. head = page_buffers(page);
  1489. bh = head;
  1490. do {
  1491. if (!buffer_mapped(bh) || (bh->b_blocknr < block))
  1492. goto next;
  1493. if (bh->b_blocknr >= block + len)
  1494. break;
  1495. clear_buffer_dirty(bh);
  1496. wait_on_buffer(bh);
  1497. clear_buffer_req(bh);
  1498. next:
  1499. bh = bh->b_this_page;
  1500. } while (bh != head);
  1501. unlock_page:
  1502. unlock_page(page);
  1503. }
  1504. pagevec_release(&pvec);
  1505. cond_resched();
  1506. /* End of range already reached? */
  1507. if (index > end || !index)
  1508. break;
  1509. }
  1510. }
  1511. EXPORT_SYMBOL_NS(clean_bdev_aliases, ANDROID_GKI_VFS_EXPORT_ONLY);
  1512. /*
  1513. * Size is a power-of-two in the range 512..PAGE_SIZE,
  1514. * and the case we care about most is PAGE_SIZE.
  1515. *
  1516. * So this *could* possibly be written with those
  1517. * constraints in mind (relevant mostly if some
  1518. * architecture has a slow bit-scan instruction)
  1519. */
  1520. static inline int block_size_bits(unsigned int blocksize)
  1521. {
  1522. return ilog2(blocksize);
  1523. }
  1524. static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
  1525. {
  1526. BUG_ON(!PageLocked(page));
  1527. if (!page_has_buffers(page))
  1528. create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
  1529. b_state);
  1530. return page_buffers(page);
  1531. }
  1532. /*
  1533. * NOTE! All mapped/uptodate combinations are valid:
  1534. *
  1535. * Mapped Uptodate Meaning
  1536. *
  1537. * No No "unknown" - must do get_block()
  1538. * No Yes "hole" - zero-filled
  1539. * Yes No "allocated" - allocated on disk, not read in
  1540. * Yes Yes "valid" - allocated and up-to-date in memory.
  1541. *
  1542. * "Dirty" is valid only with the last case (mapped+uptodate).
  1543. */
  1544. /*
  1545. * While block_write_full_page is writing back the dirty buffers under
  1546. * the page lock, whoever dirtied the buffers may decide to clean them
  1547. * again at any time. We handle that by only looking at the buffer
  1548. * state inside lock_buffer().
  1549. *
  1550. * If block_write_full_page() is called for regular writeback
  1551. * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
  1552. * locked buffer. This only can happen if someone has written the buffer
  1553. * directly, with submit_bh(). At the address_space level PageWriteback
  1554. * prevents this contention from occurring.
  1555. *
  1556. * If block_write_full_page() is called with wbc->sync_mode ==
  1557. * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
  1558. * causes the writes to be flagged as synchronous writes.
  1559. */
  1560. int __block_write_full_page(struct inode *inode, struct page *page,
  1561. get_block_t *get_block, struct writeback_control *wbc,
  1562. bh_end_io_t *handler)
  1563. {
  1564. int err;
  1565. sector_t block;
  1566. sector_t last_block;
  1567. struct buffer_head *bh, *head;
  1568. unsigned int blocksize, bbits;
  1569. int nr_underway = 0;
  1570. int write_flags = wbc_to_write_flags(wbc);
  1571. head = create_page_buffers(page, inode,
  1572. (1 << BH_Dirty)|(1 << BH_Uptodate));
  1573. /*
  1574. * Be very careful. We have no exclusion from __set_page_dirty_buffers
  1575. * here, and the (potentially unmapped) buffers may become dirty at
  1576. * any time. If a buffer becomes dirty here after we've inspected it
  1577. * then we just miss that fact, and the page stays dirty.
  1578. *
  1579. * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
  1580. * handle that here by just cleaning them.
  1581. */
  1582. bh = head;
  1583. blocksize = bh->b_size;
  1584. bbits = block_size_bits(blocksize);
  1585. block = (sector_t)page->index << (PAGE_SHIFT - bbits);
  1586. last_block = (i_size_read(inode) - 1) >> bbits;
  1587. /*
  1588. * Get all the dirty buffers mapped to disk addresses and
  1589. * handle any aliases from the underlying blockdev's mapping.
  1590. */
  1591. do {
  1592. if (block > last_block) {
  1593. /*
  1594. * mapped buffers outside i_size will occur, because
  1595. * this page can be outside i_size when there is a
  1596. * truncate in progress.
  1597. */
  1598. /*
  1599. * The buffer was zeroed by block_write_full_page()
  1600. */
  1601. clear_buffer_dirty(bh);
  1602. set_buffer_uptodate(bh);
  1603. } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
  1604. buffer_dirty(bh)) {
  1605. WARN_ON(bh->b_size != blocksize);
  1606. err = get_block(inode, block, bh, 1);
  1607. if (err)
  1608. goto recover;
  1609. clear_buffer_delay(bh);
  1610. if (buffer_new(bh)) {
  1611. /* blockdev mappings never come here */
  1612. clear_buffer_new(bh);
  1613. clean_bdev_bh_alias(bh);
  1614. }
  1615. }
  1616. bh = bh->b_this_page;
  1617. block++;
  1618. } while (bh != head);
  1619. do {
  1620. if (!buffer_mapped(bh))
  1621. continue;
  1622. /*
  1623. * If it's a fully non-blocking write attempt and we cannot
  1624. * lock the buffer then redirty the page. Note that this can
  1625. * potentially cause a busy-wait loop from writeback threads
  1626. * and kswapd activity, but those code paths have their own
  1627. * higher-level throttling.
  1628. */
  1629. if (wbc->sync_mode != WB_SYNC_NONE) {
  1630. lock_buffer(bh);
  1631. } else if (!trylock_buffer(bh)) {
  1632. redirty_page_for_writepage(wbc, page);
  1633. continue;
  1634. }
  1635. if (test_clear_buffer_dirty(bh)) {
  1636. mark_buffer_async_write_endio(bh, handler);
  1637. } else {
  1638. unlock_buffer(bh);
  1639. }
  1640. } while ((bh = bh->b_this_page) != head);
  1641. /*
  1642. * The page and its buffers are protected by PageWriteback(), so we can
  1643. * drop the bh refcounts early.
  1644. */
  1645. BUG_ON(PageWriteback(page));
  1646. set_page_writeback(page);
  1647. do {
  1648. struct buffer_head *next = bh->b_this_page;
  1649. if (buffer_async_write(bh)) {
  1650. submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
  1651. inode->i_write_hint, wbc);
  1652. nr_underway++;
  1653. }
  1654. bh = next;
  1655. } while (bh != head);
  1656. unlock_page(page);
  1657. err = 0;
  1658. done:
  1659. if (nr_underway == 0) {
  1660. /*
  1661. * The page was marked dirty, but the buffers were
  1662. * clean. Someone wrote them back by hand with
  1663. * ll_rw_block/submit_bh. A rare case.
  1664. */
  1665. end_page_writeback(page);
  1666. /*
  1667. * The page and buffer_heads can be released at any time from
  1668. * here on.
  1669. */
  1670. }
  1671. return err;
  1672. recover:
  1673. /*
  1674. * ENOSPC, or some other error. We may already have added some
  1675. * blocks to the file, so we need to write these out to avoid
  1676. * exposing stale data.
  1677. * The page is currently locked and not marked for writeback
  1678. */
  1679. bh = head;
  1680. /* Recovery: lock and submit the mapped buffers */
  1681. do {
  1682. if (buffer_mapped(bh) && buffer_dirty(bh) &&
  1683. !buffer_delay(bh)) {
  1684. lock_buffer(bh);
  1685. mark_buffer_async_write_endio(bh, handler);
  1686. } else {
  1687. /*
  1688. * The buffer may have been set dirty during
  1689. * attachment to a dirty page.
  1690. */
  1691. clear_buffer_dirty(bh);
  1692. }
  1693. } while ((bh = bh->b_this_page) != head);
  1694. SetPageError(page);
  1695. BUG_ON(PageWriteback(page));
  1696. mapping_set_error(page->mapping, err);
  1697. set_page_writeback(page);
  1698. do {
  1699. struct buffer_head *next = bh->b_this_page;
  1700. if (buffer_async_write(bh)) {
  1701. clear_buffer_dirty(bh);
  1702. submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
  1703. inode->i_write_hint, wbc);
  1704. nr_underway++;
  1705. }
  1706. bh = next;
  1707. } while (bh != head);
  1708. unlock_page(page);
  1709. goto done;
  1710. }
  1711. EXPORT_SYMBOL(__block_write_full_page);
  1712. /*
  1713. * If a page has any new buffers, zero them out here, and mark them uptodate
  1714. * and dirty so they'll be written out (in order to prevent uninitialised
  1715. * block data from leaking). And clear the new bit.
  1716. */
  1717. void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
  1718. {
  1719. unsigned int block_start, block_end;
  1720. struct buffer_head *head, *bh;
  1721. BUG_ON(!PageLocked(page));
  1722. if (!page_has_buffers(page))
  1723. return;
  1724. bh = head = page_buffers(page);
  1725. block_start = 0;
  1726. do {
  1727. block_end = block_start + bh->b_size;
  1728. if (buffer_new(bh)) {
  1729. if (block_end > from && block_start < to) {
  1730. if (!PageUptodate(page)) {
  1731. unsigned start, size;
  1732. start = max(from, block_start);
  1733. size = min(to, block_end) - start;
  1734. zero_user(page, start, size);
  1735. set_buffer_uptodate(bh);
  1736. }
  1737. clear_buffer_new(bh);
  1738. mark_buffer_dirty(bh);
  1739. }
  1740. }
  1741. block_start = block_end;
  1742. bh = bh->b_this_page;
  1743. } while (bh != head);
  1744. }
  1745. EXPORT_SYMBOL_NS(page_zero_new_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
  1746. static void
  1747. iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
  1748. struct iomap *iomap)
  1749. {
  1750. loff_t offset = block << inode->i_blkbits;
  1751. bh->b_bdev = iomap->bdev;
  1752. /*
  1753. * Block points to offset in file we need to map, iomap contains
  1754. * the offset at which the map starts. If the map ends before the
  1755. * current block, then do not map the buffer and let the caller
  1756. * handle it.
  1757. */
  1758. BUG_ON(offset >= iomap->offset + iomap->length);
  1759. switch (iomap->type) {
  1760. case IOMAP_HOLE:
  1761. /*
  1762. * If the buffer is not up to date or beyond the current EOF,
  1763. * we need to mark it as new to ensure sub-block zeroing is
  1764. * executed if necessary.
  1765. */
  1766. if (!buffer_uptodate(bh) ||
  1767. (offset >= i_size_read(inode)))
  1768. set_buffer_new(bh);
  1769. break;
  1770. case IOMAP_DELALLOC:
  1771. if (!buffer_uptodate(bh) ||
  1772. (offset >= i_size_read(inode)))
  1773. set_buffer_new(bh);
  1774. set_buffer_uptodate(bh);
  1775. set_buffer_mapped(bh);
  1776. set_buffer_delay(bh);
  1777. break;
  1778. case IOMAP_UNWRITTEN:
  1779. /*
  1780. * For unwritten regions, we always need to ensure that regions
  1781. * in the block we are not writing to are zeroed. Mark the
  1782. * buffer as new to ensure this.
  1783. */
  1784. set_buffer_new(bh);
  1785. set_buffer_unwritten(bh);
  1786. fallthrough;
  1787. case IOMAP_MAPPED:
  1788. if ((iomap->flags & IOMAP_F_NEW) ||
  1789. offset >= i_size_read(inode))
  1790. set_buffer_new(bh);
  1791. bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
  1792. inode->i_blkbits;
  1793. set_buffer_mapped(bh);
  1794. break;
  1795. }
  1796. }
  1797. int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
  1798. get_block_t *get_block, struct iomap *iomap)
  1799. {
  1800. unsigned from = pos & (PAGE_SIZE - 1);
  1801. unsigned to = from + len;
  1802. struct inode *inode = page->mapping->host;
  1803. unsigned block_start, block_end;
  1804. sector_t block;
  1805. int err = 0;
  1806. unsigned blocksize, bbits;
  1807. struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
  1808. BUG_ON(!PageLocked(page));
  1809. BUG_ON(from > PAGE_SIZE);
  1810. BUG_ON(to > PAGE_SIZE);
  1811. BUG_ON(from > to);
  1812. head = create_page_buffers(page, inode, 0);
  1813. blocksize = head->b_size;
  1814. bbits = block_size_bits(blocksize);
  1815. block = (sector_t)page->index << (PAGE_SHIFT - bbits);
  1816. for(bh = head, block_start = 0; bh != head || !block_start;
  1817. block++, block_start=block_end, bh = bh->b_this_page) {
  1818. block_end = block_start + blocksize;
  1819. if (block_end <= from || block_start >= to) {
  1820. if (PageUptodate(page)) {
  1821. if (!buffer_uptodate(bh))
  1822. set_buffer_uptodate(bh);
  1823. }
  1824. continue;
  1825. }
  1826. if (buffer_new(bh))
  1827. clear_buffer_new(bh);
  1828. if (!buffer_mapped(bh)) {
  1829. WARN_ON(bh->b_size != blocksize);
  1830. if (get_block) {
  1831. err = get_block(inode, block, bh, 1);
  1832. if (err)
  1833. break;
  1834. } else {
  1835. iomap_to_bh(inode, block, bh, iomap);
  1836. }
  1837. if (buffer_new(bh)) {
  1838. clean_bdev_bh_alias(bh);
  1839. if (PageUptodate(page)) {
  1840. clear_buffer_new(bh);
  1841. set_buffer_uptodate(bh);
  1842. mark_buffer_dirty(bh);
  1843. continue;
  1844. }
  1845. if (block_end > to || block_start < from)
  1846. zero_user_segments(page,
  1847. to, block_end,
  1848. block_start, from);
  1849. continue;
  1850. }
  1851. }
  1852. if (PageUptodate(page)) {
  1853. if (!buffer_uptodate(bh))
  1854. set_buffer_uptodate(bh);
  1855. continue;
  1856. }
  1857. if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
  1858. !buffer_unwritten(bh) &&
  1859. (block_start < from || block_end > to)) {
  1860. ll_rw_block(REQ_OP_READ, 0, 1, &bh);
  1861. *wait_bh++=bh;
  1862. }
  1863. }
  1864. /*
  1865. * If we issued read requests - let them complete.
  1866. */
  1867. while(wait_bh > wait) {
  1868. wait_on_buffer(*--wait_bh);
  1869. if (!buffer_uptodate(*wait_bh))
  1870. err = -EIO;
  1871. }
  1872. if (unlikely(err))
  1873. page_zero_new_buffers(page, from, to);
  1874. return err;
  1875. }
  1876. int __block_write_begin(struct page *page, loff_t pos, unsigned len,
  1877. get_block_t *get_block)
  1878. {
  1879. return __block_write_begin_int(page, pos, len, get_block, NULL);
  1880. }
  1881. EXPORT_SYMBOL(__block_write_begin);
  1882. static int __block_commit_write(struct inode *inode, struct page *page,
  1883. unsigned from, unsigned to)
  1884. {
  1885. unsigned block_start, block_end;
  1886. int partial = 0;
  1887. unsigned blocksize;
  1888. struct buffer_head *bh, *head;
  1889. bh = head = page_buffers(page);
  1890. blocksize = bh->b_size;
  1891. block_start = 0;
  1892. do {
  1893. block_end = block_start + blocksize;
  1894. if (block_end <= from || block_start >= to) {
  1895. if (!buffer_uptodate(bh))
  1896. partial = 1;
  1897. } else {
  1898. set_buffer_uptodate(bh);
  1899. mark_buffer_dirty(bh);
  1900. }
  1901. clear_buffer_new(bh);
  1902. block_start = block_end;
  1903. bh = bh->b_this_page;
  1904. } while (bh != head);
  1905. /*
  1906. * If this is a partial write which happened to make all buffers
  1907. * uptodate then we can optimize away a bogus readpage() for
  1908. * the next read(). Here we 'discover' whether the page went
  1909. * uptodate as a result of this (potentially partial) write.
  1910. */
  1911. if (!partial)
  1912. SetPageUptodate(page);
  1913. return 0;
  1914. }
  1915. /*
  1916. * block_write_begin takes care of the basic task of block allocation and
  1917. * bringing partial write blocks uptodate first.
  1918. *
  1919. * The filesystem needs to handle block truncation upon failure.
  1920. */
  1921. int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
  1922. unsigned flags, struct page **pagep, get_block_t *get_block)
  1923. {
  1924. pgoff_t index = pos >> PAGE_SHIFT;
  1925. struct page *page;
  1926. int status;
  1927. page = grab_cache_page_write_begin(mapping, index, flags);
  1928. if (!page)
  1929. return -ENOMEM;
  1930. status = __block_write_begin(page, pos, len, get_block);
  1931. if (unlikely(status)) {
  1932. unlock_page(page);
  1933. put_page(page);
  1934. page = NULL;
  1935. }
  1936. *pagep = page;
  1937. return status;
  1938. }
  1939. EXPORT_SYMBOL(block_write_begin);
  1940. int block_write_end(struct file *file, struct address_space *mapping,
  1941. loff_t pos, unsigned len, unsigned copied,
  1942. struct page *page, void *fsdata)
  1943. {
  1944. struct inode *inode = mapping->host;
  1945. unsigned start;
  1946. start = pos & (PAGE_SIZE - 1);
  1947. if (unlikely(copied < len)) {
  1948. /*
  1949. * The buffers that were written will now be uptodate, so we
  1950. * don't have to worry about a readpage reading them and
  1951. * overwriting a partial write. However if we have encountered
  1952. * a short write and only partially written into a buffer, it
  1953. * will not be marked uptodate, so a readpage might come in and
  1954. * destroy our partial write.
  1955. *
  1956. * Do the simplest thing, and just treat any short write to a
  1957. * non uptodate page as a zero-length write, and force the
  1958. * caller to redo the whole thing.
  1959. */
  1960. if (!PageUptodate(page))
  1961. copied = 0;
  1962. page_zero_new_buffers(page, start+copied, start+len);
  1963. }
  1964. flush_dcache_page(page);
  1965. /* This could be a short (even 0-length) commit */
  1966. __block_commit_write(inode, page, start, start+copied);
  1967. return copied;
  1968. }
  1969. EXPORT_SYMBOL(block_write_end);
  1970. int generic_write_end(struct file *file, struct address_space *mapping,
  1971. loff_t pos, unsigned len, unsigned copied,
  1972. struct page *page, void *fsdata)
  1973. {
  1974. struct inode *inode = mapping->host;
  1975. loff_t old_size = inode->i_size;
  1976. bool i_size_changed = false;
  1977. copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
  1978. /*
  1979. * No need to use i_size_read() here, the i_size cannot change under us
  1980. * because we hold i_rwsem.
  1981. *
  1982. * But it's important to update i_size while still holding page lock:
  1983. * page writeout could otherwise come in and zero beyond i_size.
  1984. */
  1985. if (pos + copied > inode->i_size) {
  1986. i_size_write(inode, pos + copied);
  1987. i_size_changed = true;
  1988. }
  1989. unlock_page(page);
  1990. put_page(page);
  1991. if (old_size < pos)
  1992. pagecache_isize_extended(inode, old_size, pos);
  1993. /*
  1994. * Don't mark the inode dirty under page lock. First, it unnecessarily
  1995. * makes the holding time of page lock longer. Second, it forces lock
  1996. * ordering of page lock and transaction start for journaling
  1997. * filesystems.
  1998. */
  1999. if (i_size_changed)
  2000. mark_inode_dirty(inode);
  2001. return copied;
  2002. }
  2003. EXPORT_SYMBOL(generic_write_end);
  2004. /*
  2005. * block_is_partially_uptodate checks whether buffers within a page are
  2006. * uptodate or not.
  2007. *
  2008. * Returns true if all buffers which correspond to a file portion
  2009. * we want to read are uptodate.
  2010. */
  2011. int block_is_partially_uptodate(struct page *page, unsigned long from,
  2012. unsigned long count)
  2013. {
  2014. unsigned block_start, block_end, blocksize;
  2015. unsigned to;
  2016. struct buffer_head *bh, *head;
  2017. int ret = 1;
  2018. if (!page_has_buffers(page))
  2019. return 0;
  2020. head = page_buffers(page);
  2021. blocksize = head->b_size;
  2022. to = min_t(unsigned, PAGE_SIZE - from, count);
  2023. to = from + to;
  2024. if (from < blocksize && to > PAGE_SIZE - blocksize)
  2025. return 0;
  2026. bh = head;
  2027. block_start = 0;
  2028. do {
  2029. block_end = block_start + blocksize;
  2030. if (block_end > from && block_start < to) {
  2031. if (!buffer_uptodate(bh)) {
  2032. ret = 0;
  2033. break;
  2034. }
  2035. if (block_end >= to)
  2036. break;
  2037. }
  2038. block_start = block_end;
  2039. bh = bh->b_this_page;
  2040. } while (bh != head);
  2041. return ret;
  2042. }
  2043. EXPORT_SYMBOL_NS(block_is_partially_uptodate, ANDROID_GKI_VFS_EXPORT_ONLY);
  2044. /*
  2045. * Generic "read page" function for block devices that have the normal
  2046. * get_block functionality. This is most of the block device filesystems.
  2047. * Reads the page asynchronously --- the unlock_buffer() and
  2048. * set/clear_buffer_uptodate() functions propagate buffer state into the
  2049. * page struct once IO has completed.
  2050. */
  2051. int block_read_full_page(struct page *page, get_block_t *get_block)
  2052. {
  2053. struct inode *inode = page->mapping->host;
  2054. sector_t iblock, lblock;
  2055. struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
  2056. unsigned int blocksize, bbits;
  2057. int nr, i;
  2058. int fully_mapped = 1;
  2059. head = create_page_buffers(page, inode, 0);
  2060. blocksize = head->b_size;
  2061. bbits = block_size_bits(blocksize);
  2062. iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
  2063. lblock = (i_size_read(inode)+blocksize-1) >> bbits;
  2064. bh = head;
  2065. nr = 0;
  2066. i = 0;
  2067. do {
  2068. if (buffer_uptodate(bh))
  2069. continue;
  2070. if (!buffer_mapped(bh)) {
  2071. int err = 0;
  2072. fully_mapped = 0;
  2073. if (iblock < lblock) {
  2074. WARN_ON(bh->b_size != blocksize);
  2075. err = get_block(inode, iblock, bh, 0);
  2076. if (err)
  2077. SetPageError(page);
  2078. }
  2079. if (!buffer_mapped(bh)) {
  2080. zero_user(page, i * blocksize, blocksize);
  2081. if (!err)
  2082. set_buffer_uptodate(bh);
  2083. continue;
  2084. }
  2085. /*
  2086. * get_block() might have updated the buffer
  2087. * synchronously
  2088. */
  2089. if (buffer_uptodate(bh))
  2090. continue;
  2091. }
  2092. arr[nr++] = bh;
  2093. } while (i++, iblock++, (bh = bh->b_this_page) != head);
  2094. if (fully_mapped)
  2095. SetPageMappedToDisk(page);
  2096. if (!nr) {
  2097. /*
  2098. * All buffers are uptodate - we can set the page uptodate
  2099. * as well. But not if get_block() returned an error.
  2100. */
  2101. if (!PageError(page))
  2102. SetPageUptodate(page);
  2103. unlock_page(page);
  2104. return 0;
  2105. }
  2106. /* Stage two: lock the buffers */
  2107. for (i = 0; i < nr; i++) {
  2108. bh = arr[i];
  2109. lock_buffer(bh);
  2110. mark_buffer_async_read(bh);
  2111. }
  2112. /*
  2113. * Stage 3: start the IO. Check for uptodateness
  2114. * inside the buffer lock in case another process reading
  2115. * the underlying blockdev brought it uptodate (the sct fix).
  2116. */
  2117. for (i = 0; i < nr; i++) {
  2118. bh = arr[i];
  2119. if (buffer_uptodate(bh))
  2120. end_buffer_async_read(bh, 1);
  2121. else
  2122. submit_bh(REQ_OP_READ, 0, bh);
  2123. }
  2124. return 0;
  2125. }
  2126. EXPORT_SYMBOL(block_read_full_page);
  2127. /* utility function for filesystems that need to do work on expanding
  2128. * truncates. Uses filesystem pagecache writes to allow the filesystem to
  2129. * deal with the hole.
  2130. */
  2131. int generic_cont_expand_simple(struct inode *inode, loff_t size)
  2132. {
  2133. struct address_space *mapping = inode->i_mapping;
  2134. struct page *page;
  2135. void *fsdata;
  2136. int err;
  2137. err = inode_newsize_ok(inode, size);
  2138. if (err)
  2139. goto out;
  2140. err = pagecache_write_begin(NULL, mapping, size, 0,
  2141. AOP_FLAG_CONT_EXPAND, &page, &fsdata);
  2142. if (err)
  2143. goto out;
  2144. err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
  2145. BUG_ON(err > 0);
  2146. out:
  2147. return err;
  2148. }
  2149. EXPORT_SYMBOL(generic_cont_expand_simple);
  2150. static int cont_expand_zero(struct file *file, struct address_space *mapping,
  2151. loff_t pos, loff_t *bytes)
  2152. {
  2153. struct inode *inode = mapping->host;
  2154. unsigned int blocksize = i_blocksize(inode);
  2155. struct page *page;
  2156. void *fsdata;
  2157. pgoff_t index, curidx;
  2158. loff_t curpos;
  2159. unsigned zerofrom, offset, len;
  2160. int err = 0;
  2161. index = pos >> PAGE_SHIFT;
  2162. offset = pos & ~PAGE_MASK;
  2163. while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
  2164. zerofrom = curpos & ~PAGE_MASK;
  2165. if (zerofrom & (blocksize-1)) {
  2166. *bytes |= (blocksize-1);
  2167. (*bytes)++;
  2168. }
  2169. len = PAGE_SIZE - zerofrom;
  2170. err = pagecache_write_begin(file, mapping, curpos, len, 0,
  2171. &page, &fsdata);
  2172. if (err)
  2173. goto out;
  2174. zero_user(page, zerofrom, len);
  2175. err = pagecache_write_end(file, mapping, curpos, len, len,
  2176. page, fsdata);
  2177. if (err < 0)
  2178. goto out;
  2179. BUG_ON(err != len);
  2180. err = 0;
  2181. balance_dirty_pages_ratelimited(mapping);
  2182. if (fatal_signal_pending(current)) {
  2183. err = -EINTR;
  2184. goto out;
  2185. }
  2186. }
  2187. /* page covers the boundary, find the boundary offset */
  2188. if (index == curidx) {
  2189. zerofrom = curpos & ~PAGE_MASK;
  2190. /* if we will expand the thing last block will be filled */
  2191. if (offset <= zerofrom) {
  2192. goto out;
  2193. }
  2194. if (zerofrom & (blocksize-1)) {
  2195. *bytes |= (blocksize-1);
  2196. (*bytes)++;
  2197. }
  2198. len = offset - zerofrom;
  2199. err = pagecache_write_begin(file, mapping, curpos, len, 0,
  2200. &page, &fsdata);
  2201. if (err)
  2202. goto out;
  2203. zero_user(page, zerofrom, len);
  2204. err = pagecache_write_end(file, mapping, curpos, len, len,
  2205. page, fsdata);
  2206. if (err < 0)
  2207. goto out;
  2208. BUG_ON(err != len);
  2209. err = 0;
  2210. }
  2211. out:
  2212. return err;
  2213. }
  2214. /*
  2215. * For moronic filesystems that do not allow holes in file.
  2216. * We may have to extend the file.
  2217. */
  2218. int cont_write_begin(struct file *file, struct address_space *mapping,
  2219. loff_t pos, unsigned len, unsigned flags,
  2220. struct page **pagep, void **fsdata,
  2221. get_block_t *get_block, loff_t *bytes)
  2222. {
  2223. struct inode *inode = mapping->host;
  2224. unsigned int blocksize = i_blocksize(inode);
  2225. unsigned int zerofrom;
  2226. int err;
  2227. err = cont_expand_zero(file, mapping, pos, bytes);
  2228. if (err)
  2229. return err;
  2230. zerofrom = *bytes & ~PAGE_MASK;
  2231. if (pos+len > *bytes && zerofrom & (blocksize-1)) {
  2232. *bytes |= (blocksize-1);
  2233. (*bytes)++;
  2234. }
  2235. return block_write_begin(mapping, pos, len, flags, pagep, get_block);
  2236. }
  2237. EXPORT_SYMBOL(cont_write_begin);
  2238. int block_commit_write(struct page *page, unsigned from, unsigned to)
  2239. {
  2240. struct inode *inode = page->mapping->host;
  2241. __block_commit_write(inode,page,from,to);
  2242. return 0;
  2243. }
  2244. EXPORT_SYMBOL(block_commit_write);
  2245. /*
  2246. * block_page_mkwrite() is not allowed to change the file size as it gets
  2247. * called from a page fault handler when a page is first dirtied. Hence we must
  2248. * be careful to check for EOF conditions here. We set the page up correctly
  2249. * for a written page which means we get ENOSPC checking when writing into
  2250. * holes and correct delalloc and unwritten extent mapping on filesystems that
  2251. * support these features.
  2252. *
  2253. * We are not allowed to take the i_mutex here so we have to play games to
  2254. * protect against truncate races as the page could now be beyond EOF. Because
  2255. * truncate writes the inode size before removing pages, once we have the
  2256. * page lock we can determine safely if the page is beyond EOF. If it is not
  2257. * beyond EOF, then the page is guaranteed safe against truncation until we
  2258. * unlock the page.
  2259. *
  2260. * Direct callers of this function should protect against filesystem freezing
  2261. * using sb_start_pagefault() - sb_end_pagefault() functions.
  2262. */
  2263. int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
  2264. get_block_t get_block)
  2265. {
  2266. struct page *page = vmf->page;
  2267. struct inode *inode = file_inode(vma->vm_file);
  2268. unsigned long end;
  2269. loff_t size;
  2270. int ret;
  2271. lock_page(page);
  2272. size = i_size_read(inode);
  2273. if ((page->mapping != inode->i_mapping) ||
  2274. (page_offset(page) > size)) {
  2275. /* We overload EFAULT to mean page got truncated */
  2276. ret = -EFAULT;
  2277. goto out_unlock;
  2278. }
  2279. /* page is wholly or partially inside EOF */
  2280. if (((page->index + 1) << PAGE_SHIFT) > size)
  2281. end = size & ~PAGE_MASK;
  2282. else
  2283. end = PAGE_SIZE;
  2284. ret = __block_write_begin(page, 0, end, get_block);
  2285. if (!ret)
  2286. ret = block_commit_write(page, 0, end);
  2287. if (unlikely(ret < 0))
  2288. goto out_unlock;
  2289. set_page_dirty(page);
  2290. wait_for_stable_page(page);
  2291. return 0;
  2292. out_unlock:
  2293. unlock_page(page);
  2294. return ret;
  2295. }
  2296. EXPORT_SYMBOL(block_page_mkwrite);
  2297. /*
  2298. * nobh_write_begin()'s prereads are special: the buffer_heads are freed
  2299. * immediately, while under the page lock. So it needs a special end_io
  2300. * handler which does not touch the bh after unlocking it.
  2301. */
  2302. static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
  2303. {
  2304. __end_buffer_read_notouch(bh, uptodate);
  2305. }
  2306. /*
  2307. * Attach the singly-linked list of buffers created by nobh_write_begin, to
  2308. * the page (converting it to circular linked list and taking care of page
  2309. * dirty races).
  2310. */
  2311. static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
  2312. {
  2313. struct buffer_head *bh;
  2314. BUG_ON(!PageLocked(page));
  2315. spin_lock(&page->mapping->private_lock);
  2316. bh = head;
  2317. do {
  2318. if (PageDirty(page))
  2319. set_buffer_dirty(bh);
  2320. if (!bh->b_this_page)
  2321. bh->b_this_page = head;
  2322. bh = bh->b_this_page;
  2323. } while (bh != head);
  2324. attach_page_private(page, head);
  2325. spin_unlock(&page->mapping->private_lock);
  2326. }
  2327. /*
  2328. * On entry, the page is fully not uptodate.
  2329. * On exit the page is fully uptodate in the areas outside (from,to)
  2330. * The filesystem needs to handle block truncation upon failure.
  2331. */
  2332. int nobh_write_begin(struct address_space *mapping,
  2333. loff_t pos, unsigned len, unsigned flags,
  2334. struct page **pagep, void **fsdata,
  2335. get_block_t *get_block)
  2336. {
  2337. struct inode *inode = mapping->host;
  2338. const unsigned blkbits = inode->i_blkbits;
  2339. const unsigned blocksize = 1 << blkbits;
  2340. struct buffer_head *head, *bh;
  2341. struct page *page;
  2342. pgoff_t index;
  2343. unsigned from, to;
  2344. unsigned block_in_page;
  2345. unsigned block_start, block_end;
  2346. sector_t block_in_file;
  2347. int nr_reads = 0;
  2348. int ret = 0;
  2349. int is_mapped_to_disk = 1;
  2350. index = pos >> PAGE_SHIFT;
  2351. from = pos & (PAGE_SIZE - 1);
  2352. to = from + len;
  2353. page = grab_cache_page_write_begin(mapping, index, flags);
  2354. if (!page)
  2355. return -ENOMEM;
  2356. *pagep = page;
  2357. *fsdata = NULL;
  2358. if (page_has_buffers(page)) {
  2359. ret = __block_write_begin(page, pos, len, get_block);
  2360. if (unlikely(ret))
  2361. goto out_release;
  2362. return ret;
  2363. }
  2364. if (PageMappedToDisk(page))
  2365. return 0;
  2366. /*
  2367. * Allocate buffers so that we can keep track of state, and potentially
  2368. * attach them to the page if an error occurs. In the common case of
  2369. * no error, they will just be freed again without ever being attached
  2370. * to the page (which is all OK, because we're under the page lock).
  2371. *
  2372. * Be careful: the buffer linked list is a NULL terminated one, rather
  2373. * than the circular one we're used to.
  2374. */
  2375. head = alloc_page_buffers(page, blocksize, false);
  2376. if (!head) {
  2377. ret = -ENOMEM;
  2378. goto out_release;
  2379. }
  2380. block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
  2381. /*
  2382. * We loop across all blocks in the page, whether or not they are
  2383. * part of the affected region. This is so we can discover if the
  2384. * page is fully mapped-to-disk.
  2385. */
  2386. for (block_start = 0, block_in_page = 0, bh = head;
  2387. block_start < PAGE_SIZE;
  2388. block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
  2389. int create;
  2390. block_end = block_start + blocksize;
  2391. bh->b_state = 0;
  2392. create = 1;
  2393. if (block_start >= to)
  2394. create = 0;
  2395. ret = get_block(inode, block_in_file + block_in_page,
  2396. bh, create);
  2397. if (ret)
  2398. goto failed;
  2399. if (!buffer_mapped(bh))
  2400. is_mapped_to_disk = 0;
  2401. if (buffer_new(bh))
  2402. clean_bdev_bh_alias(bh);
  2403. if (PageUptodate(page)) {
  2404. set_buffer_uptodate(bh);
  2405. continue;
  2406. }
  2407. if (buffer_new(bh) || !buffer_mapped(bh)) {
  2408. zero_user_segments(page, block_start, from,
  2409. to, block_end);
  2410. continue;
  2411. }
  2412. if (buffer_uptodate(bh))
  2413. continue; /* reiserfs does this */
  2414. if (block_start < from || block_end > to) {
  2415. lock_buffer(bh);
  2416. bh->b_end_io = end_buffer_read_nobh;
  2417. submit_bh(REQ_OP_READ, 0, bh);
  2418. nr_reads++;
  2419. }
  2420. }
  2421. if (nr_reads) {
  2422. /*
  2423. * The page is locked, so these buffers are protected from
  2424. * any VM or truncate activity. Hence we don't need to care
  2425. * for the buffer_head refcounts.
  2426. */
  2427. for (bh = head; bh; bh = bh->b_this_page) {
  2428. wait_on_buffer(bh);
  2429. if (!buffer_uptodate(bh))
  2430. ret = -EIO;
  2431. }
  2432. if (ret)
  2433. goto failed;
  2434. }
  2435. if (is_mapped_to_disk)
  2436. SetPageMappedToDisk(page);
  2437. *fsdata = head; /* to be released by nobh_write_end */
  2438. return 0;
  2439. failed:
  2440. BUG_ON(!ret);
  2441. /*
  2442. * Error recovery is a bit difficult. We need to zero out blocks that
  2443. * were newly allocated, and dirty them to ensure they get written out.
  2444. * Buffers need to be attached to the page at this point, otherwise
  2445. * the handling of potential IO errors during writeout would be hard
  2446. * (could try doing synchronous writeout, but what if that fails too?)
  2447. */
  2448. attach_nobh_buffers(page, head);
  2449. page_zero_new_buffers(page, from, to);
  2450. out_release:
  2451. unlock_page(page);
  2452. put_page(page);
  2453. *pagep = NULL;
  2454. return ret;
  2455. }
  2456. EXPORT_SYMBOL(nobh_write_begin);
  2457. int nobh_write_end(struct file *file, struct address_space *mapping,
  2458. loff_t pos, unsigned len, unsigned copied,
  2459. struct page *page, void *fsdata)
  2460. {
  2461. struct inode *inode = page->mapping->host;
  2462. struct buffer_head *head = fsdata;
  2463. struct buffer_head *bh;
  2464. BUG_ON(fsdata != NULL && page_has_buffers(page));
  2465. if (unlikely(copied < len) && head)
  2466. attach_nobh_buffers(page, head);
  2467. if (page_has_buffers(page))
  2468. return generic_write_end(file, mapping, pos, len,
  2469. copied, page, fsdata);
  2470. SetPageUptodate(page);
  2471. set_page_dirty(page);
  2472. if (pos+copied > inode->i_size) {
  2473. i_size_write(inode, pos+copied);
  2474. mark_inode_dirty(inode);
  2475. }
  2476. unlock_page(page);
  2477. put_page(page);
  2478. while (head) {
  2479. bh = head;
  2480. head = head->b_this_page;
  2481. free_buffer_head(bh);
  2482. }
  2483. return copied;
  2484. }
  2485. EXPORT_SYMBOL(nobh_write_end);
  2486. /*
  2487. * nobh_writepage() - based on block_full_write_page() except
  2488. * that it tries to operate without attaching bufferheads to
  2489. * the page.
  2490. */
  2491. int nobh_writepage(struct page *page, get_block_t *get_block,
  2492. struct writeback_control *wbc)
  2493. {
  2494. struct inode * const inode = page->mapping->host;
  2495. loff_t i_size = i_size_read(inode);
  2496. const pgoff_t end_index = i_size >> PAGE_SHIFT;
  2497. unsigned offset;
  2498. int ret;
  2499. /* Is the page fully inside i_size? */
  2500. if (page->index < end_index)
  2501. goto out;
  2502. /* Is the page fully outside i_size? (truncate in progress) */
  2503. offset = i_size & (PAGE_SIZE-1);
  2504. if (page->index >= end_index+1 || !offset) {
  2505. unlock_page(page);
  2506. return 0; /* don't care */
  2507. }
  2508. /*
  2509. * The page straddles i_size. It must be zeroed out on each and every
  2510. * writepage invocation because it may be mmapped. "A file is mapped
  2511. * in multiples of the page size. For a file that is not a multiple of
  2512. * the page size, the remaining memory is zeroed when mapped, and
  2513. * writes to that region are not written out to the file."
  2514. */
  2515. zero_user_segment(page, offset, PAGE_SIZE);
  2516. out:
  2517. ret = mpage_writepage(page, get_block, wbc);
  2518. if (ret == -EAGAIN)
  2519. ret = __block_write_full_page(inode, page, get_block, wbc,
  2520. end_buffer_async_write);
  2521. return ret;
  2522. }
  2523. EXPORT_SYMBOL(nobh_writepage);
  2524. int nobh_truncate_page(struct address_space *mapping,
  2525. loff_t from, get_block_t *get_block)
  2526. {
  2527. pgoff_t index = from >> PAGE_SHIFT;
  2528. unsigned offset = from & (PAGE_SIZE-1);
  2529. unsigned blocksize;
  2530. sector_t iblock;
  2531. unsigned length, pos;
  2532. struct inode *inode = mapping->host;
  2533. struct page *page;
  2534. struct buffer_head map_bh;
  2535. int err;
  2536. blocksize = i_blocksize(inode);
  2537. length = offset & (blocksize - 1);
  2538. /* Block boundary? Nothing to do */
  2539. if (!length)
  2540. return 0;
  2541. length = blocksize - length;
  2542. iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
  2543. page = grab_cache_page(mapping, index);
  2544. err = -ENOMEM;
  2545. if (!page)
  2546. goto out;
  2547. if (page_has_buffers(page)) {
  2548. has_buffers:
  2549. unlock_page(page);
  2550. put_page(page);
  2551. return block_truncate_page(mapping, from, get_block);
  2552. }
  2553. /* Find the buffer that contains "offset" */
  2554. pos = blocksize;
  2555. while (offset >= pos) {
  2556. iblock++;
  2557. pos += blocksize;
  2558. }
  2559. map_bh.b_size = blocksize;
  2560. map_bh.b_state = 0;
  2561. err = get_block(inode, iblock, &map_bh, 0);
  2562. if (err)
  2563. goto unlock;
  2564. /* unmapped? It's a hole - nothing to do */
  2565. if (!buffer_mapped(&map_bh))
  2566. goto unlock;
  2567. /* Ok, it's mapped. Make sure it's up-to-date */
  2568. if (!PageUptodate(page)) {
  2569. err = mapping->a_ops->readpage(NULL, page);
  2570. if (err) {
  2571. put_page(page);
  2572. goto out;
  2573. }
  2574. lock_page(page);
  2575. if (!PageUptodate(page)) {
  2576. err = -EIO;
  2577. goto unlock;
  2578. }
  2579. if (page_has_buffers(page))
  2580. goto has_buffers;
  2581. }
  2582. zero_user(page, offset, length);
  2583. set_page_dirty(page);
  2584. err = 0;
  2585. unlock:
  2586. unlock_page(page);
  2587. put_page(page);
  2588. out:
  2589. return err;
  2590. }
  2591. EXPORT_SYMBOL(nobh_truncate_page);
  2592. int block_truncate_page(struct address_space *mapping,
  2593. loff_t from, get_block_t *get_block)
  2594. {
  2595. pgoff_t index = from >> PAGE_SHIFT;
  2596. unsigned offset = from & (PAGE_SIZE-1);
  2597. unsigned blocksize;
  2598. sector_t iblock;
  2599. unsigned length, pos;
  2600. struct inode *inode = mapping->host;
  2601. struct page *page;
  2602. struct buffer_head *bh;
  2603. int err;
  2604. blocksize = i_blocksize(inode);
  2605. length = offset & (blocksize - 1);
  2606. /* Block boundary? Nothing to do */
  2607. if (!length)
  2608. return 0;
  2609. length = blocksize - length;
  2610. iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
  2611. page = grab_cache_page(mapping, index);
  2612. err = -ENOMEM;
  2613. if (!page)
  2614. goto out;
  2615. if (!page_has_buffers(page))
  2616. create_empty_buffers(page, blocksize, 0);
  2617. /* Find the buffer that contains "offset" */
  2618. bh = page_buffers(page);
  2619. pos = blocksize;
  2620. while (offset >= pos) {
  2621. bh = bh->b_this_page;
  2622. iblock++;
  2623. pos += blocksize;
  2624. }
  2625. err = 0;
  2626. if (!buffer_mapped(bh)) {
  2627. WARN_ON(bh->b_size != blocksize);
  2628. err = get_block(inode, iblock, bh, 0);
  2629. if (err)
  2630. goto unlock;
  2631. /* unmapped? It's a hole - nothing to do */
  2632. if (!buffer_mapped(bh))
  2633. goto unlock;
  2634. }
  2635. /* Ok, it's mapped. Make sure it's up-to-date */
  2636. if (PageUptodate(page))
  2637. set_buffer_uptodate(bh);
  2638. if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
  2639. err = -EIO;
  2640. ll_rw_block(REQ_OP_READ, 0, 1, &bh);
  2641. wait_on_buffer(bh);
  2642. /* Uhhuh. Read error. Complain and punt. */
  2643. if (!buffer_uptodate(bh))
  2644. goto unlock;
  2645. }
  2646. zero_user(page, offset, length);
  2647. mark_buffer_dirty(bh);
  2648. err = 0;
  2649. unlock:
  2650. unlock_page(page);
  2651. put_page(page);
  2652. out:
  2653. return err;
  2654. }
  2655. EXPORT_SYMBOL(block_truncate_page);
  2656. /*
  2657. * The generic ->writepage function for buffer-backed address_spaces
  2658. */
  2659. int block_write_full_page(struct page *page, get_block_t *get_block,
  2660. struct writeback_control *wbc)
  2661. {
  2662. struct inode * const inode = page->mapping->host;
  2663. loff_t i_size = i_size_read(inode);
  2664. const pgoff_t end_index = i_size >> PAGE_SHIFT;
  2665. unsigned offset;
  2666. /* Is the page fully inside i_size? */
  2667. if (page->index < end_index)
  2668. return __block_write_full_page(inode, page, get_block, wbc,
  2669. end_buffer_async_write);
  2670. /* Is the page fully outside i_size? (truncate in progress) */
  2671. offset = i_size & (PAGE_SIZE-1);
  2672. if (page->index >= end_index+1 || !offset) {
  2673. unlock_page(page);
  2674. return 0; /* don't care */
  2675. }
  2676. /*
  2677. * The page straddles i_size. It must be zeroed out on each and every
  2678. * writepage invocation because it may be mmapped. "A file is mapped
  2679. * in multiples of the page size. For a file that is not a multiple of
  2680. * the page size, the remaining memory is zeroed when mapped, and
  2681. * writes to that region are not written out to the file."
  2682. */
  2683. zero_user_segment(page, offset, PAGE_SIZE);
  2684. return __block_write_full_page(inode, page, get_block, wbc,
  2685. end_buffer_async_write);
  2686. }
  2687. EXPORT_SYMBOL(block_write_full_page);
  2688. sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
  2689. get_block_t *get_block)
  2690. {
  2691. struct inode *inode = mapping->host;
  2692. struct buffer_head tmp = {
  2693. .b_size = i_blocksize(inode),
  2694. };
  2695. get_block(inode, block, &tmp, 0);
  2696. return tmp.b_blocknr;
  2697. }
  2698. EXPORT_SYMBOL(generic_block_bmap);
  2699. static void end_bio_bh_io_sync(struct bio *bio)
  2700. {
  2701. struct buffer_head *bh = bio->bi_private;
  2702. if (unlikely(bio_flagged(bio, BIO_QUIET)))
  2703. set_bit(BH_Quiet, &bh->b_state);
  2704. bh->b_end_io(bh, !bio->bi_status);
  2705. bio_put(bio);
  2706. }
  2707. static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
  2708. enum rw_hint write_hint, struct writeback_control *wbc)
  2709. {
  2710. struct bio *bio;
  2711. BUG_ON(!buffer_locked(bh));
  2712. BUG_ON(!buffer_mapped(bh));
  2713. BUG_ON(!bh->b_end_io);
  2714. BUG_ON(buffer_delay(bh));
  2715. BUG_ON(buffer_unwritten(bh));
  2716. /*
  2717. * Only clear out a write error when rewriting
  2718. */
  2719. if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
  2720. clear_buffer_write_io_error(bh);
  2721. bio = bio_alloc(GFP_NOIO, 1);
  2722. fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
  2723. bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
  2724. bio_set_dev(bio, bh->b_bdev);
  2725. bio->bi_write_hint = write_hint;
  2726. bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
  2727. BUG_ON(bio->bi_iter.bi_size != bh->b_size);
  2728. bio->bi_end_io = end_bio_bh_io_sync;
  2729. bio->bi_private = bh;
  2730. if (buffer_meta(bh))
  2731. op_flags |= REQ_META;
  2732. if (buffer_prio(bh))
  2733. op_flags |= REQ_PRIO;
  2734. bio_set_op_attrs(bio, op, op_flags);
  2735. /* Take care of bh's that straddle the end of the device */
  2736. guard_bio_eod(bio);
  2737. if (wbc) {
  2738. wbc_init_bio(wbc, bio);
  2739. wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
  2740. }
  2741. submit_bio(bio);
  2742. return 0;
  2743. }
  2744. int submit_bh(int op, int op_flags, struct buffer_head *bh)
  2745. {
  2746. return submit_bh_wbc(op, op_flags, bh, 0, NULL);
  2747. }
  2748. EXPORT_SYMBOL(submit_bh);
  2749. /**
  2750. * ll_rw_block: low-level access to block devices (DEPRECATED)
  2751. * @op: whether to %READ or %WRITE
  2752. * @op_flags: req_flag_bits
  2753. * @nr: number of &struct buffer_heads in the array
  2754. * @bhs: array of pointers to &struct buffer_head
  2755. *
  2756. * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
  2757. * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
  2758. * @op_flags contains flags modifying the detailed I/O behavior, most notably
  2759. * %REQ_RAHEAD.
  2760. *
  2761. * This function drops any buffer that it cannot get a lock on (with the
  2762. * BH_Lock state bit), any buffer that appears to be clean when doing a write
  2763. * request, and any buffer that appears to be up-to-date when doing read
  2764. * request. Further it marks as clean buffers that are processed for
  2765. * writing (the buffer cache won't assume that they are actually clean
  2766. * until the buffer gets unlocked).
  2767. *
  2768. * ll_rw_block sets b_end_io to simple completion handler that marks
  2769. * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
  2770. * any waiters.
  2771. *
  2772. * All of the buffers must be for the same device, and must also be a
  2773. * multiple of the current approved size for the device.
  2774. */
  2775. void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
  2776. {
  2777. int i;
  2778. for (i = 0; i < nr; i++) {
  2779. struct buffer_head *bh = bhs[i];
  2780. if (!trylock_buffer(bh))
  2781. continue;
  2782. if (op == WRITE) {
  2783. if (test_clear_buffer_dirty(bh)) {
  2784. bh->b_end_io = end_buffer_write_sync;
  2785. get_bh(bh);
  2786. submit_bh(op, op_flags, bh);
  2787. continue;
  2788. }
  2789. } else {
  2790. if (!buffer_uptodate(bh)) {
  2791. bh->b_end_io = end_buffer_read_sync;
  2792. get_bh(bh);
  2793. submit_bh(op, op_flags, bh);
  2794. continue;
  2795. }
  2796. }
  2797. unlock_buffer(bh);
  2798. }
  2799. }
  2800. EXPORT_SYMBOL_NS(ll_rw_block, ANDROID_GKI_VFS_EXPORT_ONLY);
  2801. void write_dirty_buffer(struct buffer_head *bh, int op_flags)
  2802. {
  2803. lock_buffer(bh);
  2804. if (!test_clear_buffer_dirty(bh)) {
  2805. unlock_buffer(bh);
  2806. return;
  2807. }
  2808. bh->b_end_io = end_buffer_write_sync;
  2809. get_bh(bh);
  2810. submit_bh(REQ_OP_WRITE, op_flags, bh);
  2811. }
  2812. EXPORT_SYMBOL(write_dirty_buffer);
  2813. /*
  2814. * For a data-integrity writeout, we need to wait upon any in-progress I/O
  2815. * and then start new I/O and then wait upon it. The caller must have a ref on
  2816. * the buffer_head.
  2817. */
  2818. int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
  2819. {
  2820. int ret = 0;
  2821. WARN_ON(atomic_read(&bh->b_count) < 1);
  2822. lock_buffer(bh);
  2823. if (test_clear_buffer_dirty(bh)) {
  2824. /*
  2825. * The bh should be mapped, but it might not be if the
  2826. * device was hot-removed. Not much we can do but fail the I/O.
  2827. */
  2828. if (!buffer_mapped(bh)) {
  2829. unlock_buffer(bh);
  2830. return -EIO;
  2831. }
  2832. get_bh(bh);
  2833. bh->b_end_io = end_buffer_write_sync;
  2834. ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
  2835. wait_on_buffer(bh);
  2836. if (!ret && !buffer_uptodate(bh))
  2837. ret = -EIO;
  2838. } else {
  2839. unlock_buffer(bh);
  2840. }
  2841. return ret;
  2842. }
  2843. EXPORT_SYMBOL(__sync_dirty_buffer);
  2844. int sync_dirty_buffer(struct buffer_head *bh)
  2845. {
  2846. return __sync_dirty_buffer(bh, REQ_SYNC);
  2847. }
  2848. EXPORT_SYMBOL_NS(sync_dirty_buffer, ANDROID_GKI_VFS_EXPORT_ONLY);
  2849. /*
  2850. * try_to_free_buffers() checks if all the buffers on this particular page
  2851. * are unused, and releases them if so.
  2852. *
  2853. * Exclusion against try_to_free_buffers may be obtained by either
  2854. * locking the page or by holding its mapping's private_lock.
  2855. *
  2856. * If the page is dirty but all the buffers are clean then we need to
  2857. * be sure to mark the page clean as well. This is because the page
  2858. * may be against a block device, and a later reattachment of buffers
  2859. * to a dirty page will set *all* buffers dirty. Which would corrupt
  2860. * filesystem data on the same device.
  2861. *
  2862. * The same applies to regular filesystem pages: if all the buffers are
  2863. * clean then we set the page clean and proceed. To do that, we require
  2864. * total exclusion from __set_page_dirty_buffers(). That is obtained with
  2865. * private_lock.
  2866. *
  2867. * try_to_free_buffers() is non-blocking.
  2868. */
  2869. static inline int buffer_busy(struct buffer_head *bh)
  2870. {
  2871. return atomic_read(&bh->b_count) |
  2872. (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
  2873. }
  2874. static int
  2875. drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
  2876. {
  2877. struct buffer_head *head = page_buffers(page);
  2878. struct buffer_head *bh;
  2879. bh = head;
  2880. do {
  2881. if (buffer_busy(bh))
  2882. goto failed;
  2883. bh = bh->b_this_page;
  2884. } while (bh != head);
  2885. do {
  2886. struct buffer_head *next = bh->b_this_page;
  2887. if (bh->b_assoc_map)
  2888. __remove_assoc_queue(bh);
  2889. bh = next;
  2890. } while (bh != head);
  2891. *buffers_to_free = head;
  2892. detach_page_private(page);
  2893. return 1;
  2894. failed:
  2895. return 0;
  2896. }
  2897. int try_to_free_buffers(struct page *page)
  2898. {
  2899. struct address_space * const mapping = page->mapping;
  2900. struct buffer_head *buffers_to_free = NULL;
  2901. int ret = 0;
  2902. BUG_ON(!PageLocked(page));
  2903. if (PageWriteback(page))
  2904. return 0;
  2905. if (mapping == NULL) { /* can this still happen? */
  2906. ret = drop_buffers(page, &buffers_to_free);
  2907. goto out;
  2908. }
  2909. spin_lock(&mapping->private_lock);
  2910. ret = drop_buffers(page, &buffers_to_free);
  2911. /*
  2912. * If the filesystem writes its buffers by hand (eg ext3)
  2913. * then we can have clean buffers against a dirty page. We
  2914. * clean the page here; otherwise the VM will never notice
  2915. * that the filesystem did any IO at all.
  2916. *
  2917. * Also, during truncate, discard_buffer will have marked all
  2918. * the page's buffers clean. We discover that here and clean
  2919. * the page also.
  2920. *
  2921. * private_lock must be held over this entire operation in order
  2922. * to synchronise against __set_page_dirty_buffers and prevent the
  2923. * dirty bit from being lost.
  2924. */
  2925. if (ret)
  2926. cancel_dirty_page(page);
  2927. spin_unlock(&mapping->private_lock);
  2928. out:
  2929. if (buffers_to_free) {
  2930. struct buffer_head *bh = buffers_to_free;
  2931. do {
  2932. struct buffer_head *next = bh->b_this_page;
  2933. free_buffer_head(bh);
  2934. bh = next;
  2935. } while (bh != buffers_to_free);
  2936. }
  2937. return ret;
  2938. }
  2939. EXPORT_SYMBOL(try_to_free_buffers);
  2940. /*
  2941. * There are no bdflush tunables left. But distributions are
  2942. * still running obsolete flush daemons, so we terminate them here.
  2943. *
  2944. * Use of bdflush() is deprecated and will be removed in a future kernel.
  2945. * The `flush-X' kernel threads fully replace bdflush daemons and this call.
  2946. */
  2947. SYSCALL_DEFINE2(bdflush, int, func, long, data)
  2948. {
  2949. static int msg_count;
  2950. if (!capable(CAP_SYS_ADMIN))
  2951. return -EPERM;
  2952. if (msg_count < 5) {
  2953. msg_count++;
  2954. printk(KERN_INFO
  2955. "warning: process `%s' used the obsolete bdflush"
  2956. " system call\n", current->comm);
  2957. printk(KERN_INFO "Fix your initscripts?\n");
  2958. }
  2959. if (func == 1)
  2960. do_exit(0);
  2961. return 0;
  2962. }
  2963. /*
  2964. * Buffer-head allocation
  2965. */
  2966. static struct kmem_cache *bh_cachep __read_mostly;
  2967. /*
  2968. * Once the number of bh's in the machine exceeds this level, we start
  2969. * stripping them in writeback.
  2970. */
  2971. static unsigned long max_buffer_heads;
  2972. int buffer_heads_over_limit;
  2973. struct bh_accounting {
  2974. int nr; /* Number of live bh's */
  2975. int ratelimit; /* Limit cacheline bouncing */
  2976. };
  2977. static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
  2978. static void recalc_bh_state(void)
  2979. {
  2980. int i;
  2981. int tot = 0;
  2982. if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
  2983. return;
  2984. __this_cpu_write(bh_accounting.ratelimit, 0);
  2985. for_each_online_cpu(i)
  2986. tot += per_cpu(bh_accounting, i).nr;
  2987. buffer_heads_over_limit = (tot > max_buffer_heads);
  2988. }
  2989. struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
  2990. {
  2991. struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
  2992. if (ret) {
  2993. INIT_LIST_HEAD(&ret->b_assoc_buffers);
  2994. spin_lock_init(&ret->b_uptodate_lock);
  2995. preempt_disable();
  2996. __this_cpu_inc(bh_accounting.nr);
  2997. recalc_bh_state();
  2998. preempt_enable();
  2999. }
  3000. return ret;
  3001. }
  3002. EXPORT_SYMBOL(alloc_buffer_head);
  3003. void free_buffer_head(struct buffer_head *bh)
  3004. {
  3005. BUG_ON(!list_empty(&bh->b_assoc_buffers));
  3006. kmem_cache_free(bh_cachep, bh);
  3007. preempt_disable();
  3008. __this_cpu_dec(bh_accounting.nr);
  3009. recalc_bh_state();
  3010. preempt_enable();
  3011. }
  3012. EXPORT_SYMBOL(free_buffer_head);
  3013. static int buffer_exit_cpu_dead(unsigned int cpu)
  3014. {
  3015. int i;
  3016. struct bh_lru *b = &per_cpu(bh_lrus, cpu);
  3017. for (i = 0; i < BH_LRU_SIZE; i++) {
  3018. brelse(b->bhs[i]);
  3019. b->bhs[i] = NULL;
  3020. }
  3021. this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
  3022. per_cpu(bh_accounting, cpu).nr = 0;
  3023. return 0;
  3024. }
  3025. /**
  3026. * bh_uptodate_or_lock - Test whether the buffer is uptodate
  3027. * @bh: struct buffer_head
  3028. *
  3029. * Return true if the buffer is up-to-date and false,
  3030. * with the buffer locked, if not.
  3031. */
  3032. int bh_uptodate_or_lock(struct buffer_head *bh)
  3033. {
  3034. if (!buffer_uptodate(bh)) {
  3035. lock_buffer(bh);
  3036. if (!buffer_uptodate(bh))
  3037. return 0;
  3038. unlock_buffer(bh);
  3039. }
  3040. return 1;
  3041. }
  3042. EXPORT_SYMBOL(bh_uptodate_or_lock);
  3043. /**
  3044. * bh_submit_read - Submit a locked buffer for reading
  3045. * @bh: struct buffer_head
  3046. *
  3047. * Returns zero on success and -EIO on error.
  3048. */
  3049. int bh_submit_read(struct buffer_head *bh)
  3050. {
  3051. BUG_ON(!buffer_locked(bh));
  3052. if (buffer_uptodate(bh)) {
  3053. unlock_buffer(bh);
  3054. return 0;
  3055. }
  3056. get_bh(bh);
  3057. bh->b_end_io = end_buffer_read_sync;
  3058. submit_bh(REQ_OP_READ, 0, bh);
  3059. wait_on_buffer(bh);
  3060. if (buffer_uptodate(bh))
  3061. return 0;
  3062. return -EIO;
  3063. }
  3064. EXPORT_SYMBOL(bh_submit_read);
  3065. void __init buffer_init(void)
  3066. {
  3067. unsigned long nrpages;
  3068. int ret;
  3069. bh_cachep = kmem_cache_create("buffer_head",
  3070. sizeof(struct buffer_head), 0,
  3071. (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
  3072. SLAB_MEM_SPREAD),
  3073. NULL);
  3074. /*
  3075. * Limit the bh occupancy to 10% of ZONE_NORMAL
  3076. */
  3077. nrpages = (nr_free_buffer_pages() * 10) / 100;
  3078. max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
  3079. ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
  3080. NULL, buffer_exit_cpu_dead);
  3081. WARN_ON(ret < 0);
  3082. }