write.c 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/nfs/write.c
  4. *
  5. * Write file data over NFS.
  6. *
  7. * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
  8. */
  9. #include <linux/types.h>
  10. #include <linux/slab.h>
  11. #include <linux/mm.h>
  12. #include <linux/pagemap.h>
  13. #include <linux/file.h>
  14. #include <linux/writeback.h>
  15. #include <linux/swap.h>
  16. #include <linux/migrate.h>
  17. #include <linux/sunrpc/clnt.h>
  18. #include <linux/nfs_fs.h>
  19. #include <linux/nfs_mount.h>
  20. #include <linux/nfs_page.h>
  21. #include <linux/backing-dev.h>
  22. #include <linux/export.h>
  23. #include <linux/freezer.h>
  24. #include <linux/wait.h>
  25. #include <linux/iversion.h>
  26. #include <linux/uaccess.h>
  27. #include <linux/sched/mm.h>
  28. #include "delegation.h"
  29. #include "internal.h"
  30. #include "iostat.h"
  31. #include "nfs4_fs.h"
  32. #include "fscache.h"
  33. #include "pnfs.h"
  34. #include "nfstrace.h"
  35. #define NFSDBG_FACILITY NFSDBG_PAGECACHE
  36. #define MIN_POOL_WRITE (32)
  37. #define MIN_POOL_COMMIT (4)
  38. struct nfs_io_completion {
  39. void (*complete)(void *data);
  40. void *data;
  41. struct kref refcount;
  42. };
  43. /*
  44. * Local function declarations
  45. */
  46. static void nfs_redirty_request(struct nfs_page *req);
  47. static const struct rpc_call_ops nfs_commit_ops;
  48. static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
  49. static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
  50. static const struct nfs_rw_ops nfs_rw_write_ops;
  51. static void nfs_inode_remove_request(struct nfs_page *req);
  52. static void nfs_clear_request_commit(struct nfs_page *req);
  53. static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
  54. struct inode *inode);
  55. static struct nfs_page *
  56. nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
  57. struct page *page);
  58. static struct kmem_cache *nfs_wdata_cachep;
  59. static mempool_t *nfs_wdata_mempool;
  60. static struct kmem_cache *nfs_cdata_cachep;
  61. static mempool_t *nfs_commit_mempool;
  62. struct nfs_commit_data *nfs_commitdata_alloc(void)
  63. {
  64. struct nfs_commit_data *p;
  65. p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask());
  66. if (!p) {
  67. p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
  68. if (!p)
  69. return NULL;
  70. memset(p, 0, sizeof(*p));
  71. }
  72. INIT_LIST_HEAD(&p->pages);
  73. return p;
  74. }
  75. EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
  76. void nfs_commit_free(struct nfs_commit_data *p)
  77. {
  78. mempool_free(p, nfs_commit_mempool);
  79. }
  80. EXPORT_SYMBOL_GPL(nfs_commit_free);
  81. static struct nfs_pgio_header *nfs_writehdr_alloc(void)
  82. {
  83. struct nfs_pgio_header *p;
  84. p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask());
  85. if (!p) {
  86. p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT);
  87. if (!p)
  88. return NULL;
  89. memset(p, 0, sizeof(*p));
  90. }
  91. p->rw_mode = FMODE_WRITE;
  92. return p;
  93. }
  94. static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
  95. {
  96. mempool_free(hdr, nfs_wdata_mempool);
  97. }
  98. static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
  99. {
  100. return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
  101. }
  102. static void nfs_io_completion_init(struct nfs_io_completion *ioc,
  103. void (*complete)(void *), void *data)
  104. {
  105. ioc->complete = complete;
  106. ioc->data = data;
  107. kref_init(&ioc->refcount);
  108. }
  109. static void nfs_io_completion_release(struct kref *kref)
  110. {
  111. struct nfs_io_completion *ioc = container_of(kref,
  112. struct nfs_io_completion, refcount);
  113. ioc->complete(ioc->data);
  114. kfree(ioc);
  115. }
  116. static void nfs_io_completion_get(struct nfs_io_completion *ioc)
  117. {
  118. if (ioc != NULL)
  119. kref_get(&ioc->refcount);
  120. }
  121. static void nfs_io_completion_put(struct nfs_io_completion *ioc)
  122. {
  123. if (ioc != NULL)
  124. kref_put(&ioc->refcount, nfs_io_completion_release);
  125. }
  126. static void
  127. nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode)
  128. {
  129. if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) {
  130. kref_get(&req->wb_kref);
  131. atomic_long_inc(&NFS_I(inode)->nrequests);
  132. }
  133. }
  134. static int
  135. nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode)
  136. {
  137. int ret;
  138. if (!test_bit(PG_REMOVE, &req->wb_flags))
  139. return 0;
  140. ret = nfs_page_group_lock(req);
  141. if (ret)
  142. return ret;
  143. if (test_and_clear_bit(PG_REMOVE, &req->wb_flags))
  144. nfs_page_set_inode_ref(req, inode);
  145. nfs_page_group_unlock(req);
  146. return 0;
  147. }
  148. static struct nfs_page *
  149. nfs_page_private_request(struct page *page)
  150. {
  151. if (!PagePrivate(page))
  152. return NULL;
  153. return (struct nfs_page *)page_private(page);
  154. }
  155. /*
  156. * nfs_page_find_head_request_locked - find head request associated with @page
  157. *
  158. * must be called while holding the inode lock.
  159. *
  160. * returns matching head request with reference held, or NULL if not found.
  161. */
  162. static struct nfs_page *
  163. nfs_page_find_private_request(struct page *page)
  164. {
  165. struct address_space *mapping = page_file_mapping(page);
  166. struct nfs_page *req;
  167. if (!PagePrivate(page))
  168. return NULL;
  169. spin_lock(&mapping->private_lock);
  170. req = nfs_page_private_request(page);
  171. if (req) {
  172. WARN_ON_ONCE(req->wb_head != req);
  173. kref_get(&req->wb_kref);
  174. }
  175. spin_unlock(&mapping->private_lock);
  176. return req;
  177. }
  178. static struct nfs_page *
  179. nfs_page_find_swap_request(struct page *page)
  180. {
  181. struct inode *inode = page_file_mapping(page)->host;
  182. struct nfs_inode *nfsi = NFS_I(inode);
  183. struct nfs_page *req = NULL;
  184. if (!PageSwapCache(page))
  185. return NULL;
  186. mutex_lock(&nfsi->commit_mutex);
  187. if (PageSwapCache(page)) {
  188. req = nfs_page_search_commits_for_head_request_locked(nfsi,
  189. page);
  190. if (req) {
  191. WARN_ON_ONCE(req->wb_head != req);
  192. kref_get(&req->wb_kref);
  193. }
  194. }
  195. mutex_unlock(&nfsi->commit_mutex);
  196. return req;
  197. }
  198. /*
  199. * nfs_page_find_head_request - find head request associated with @page
  200. *
  201. * returns matching head request with reference held, or NULL if not found.
  202. */
  203. static struct nfs_page *nfs_page_find_head_request(struct page *page)
  204. {
  205. struct nfs_page *req;
  206. req = nfs_page_find_private_request(page);
  207. if (!req)
  208. req = nfs_page_find_swap_request(page);
  209. return req;
  210. }
  211. static struct nfs_page *nfs_find_and_lock_page_request(struct page *page)
  212. {
  213. struct inode *inode = page_file_mapping(page)->host;
  214. struct nfs_page *req, *head;
  215. int ret;
  216. for (;;) {
  217. req = nfs_page_find_head_request(page);
  218. if (!req)
  219. return req;
  220. head = nfs_page_group_lock_head(req);
  221. if (head != req)
  222. nfs_release_request(req);
  223. if (IS_ERR(head))
  224. return head;
  225. ret = nfs_cancel_remove_inode(head, inode);
  226. if (ret < 0) {
  227. nfs_unlock_and_release_request(head);
  228. return ERR_PTR(ret);
  229. }
  230. /* Ensure that nobody removed the request before we locked it */
  231. if (head == nfs_page_private_request(page))
  232. break;
  233. if (PageSwapCache(page))
  234. break;
  235. nfs_unlock_and_release_request(head);
  236. }
  237. return head;
  238. }
  239. /* Adjust the file length if we're writing beyond the end */
  240. static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
  241. {
  242. struct inode *inode = page_file_mapping(page)->host;
  243. loff_t end, i_size;
  244. pgoff_t end_index;
  245. spin_lock(&inode->i_lock);
  246. i_size = i_size_read(inode);
  247. end_index = (i_size - 1) >> PAGE_SHIFT;
  248. if (i_size > 0 && page_index(page) < end_index)
  249. goto out;
  250. end = page_file_offset(page) + ((loff_t)offset+count);
  251. if (i_size >= end)
  252. goto out;
  253. i_size_write(inode, end);
  254. NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
  255. nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
  256. out:
  257. spin_unlock(&inode->i_lock);
  258. }
  259. /* A writeback failed: mark the page as bad, and invalidate the page cache */
  260. static void nfs_set_pageerror(struct address_space *mapping)
  261. {
  262. struct inode *inode = mapping->host;
  263. nfs_zap_mapping(mapping->host, mapping);
  264. /* Force file size revalidation */
  265. spin_lock(&inode->i_lock);
  266. NFS_I(inode)->cache_validity |= NFS_INO_REVAL_FORCED |
  267. NFS_INO_REVAL_PAGECACHE |
  268. NFS_INO_INVALID_SIZE;
  269. spin_unlock(&inode->i_lock);
  270. }
  271. static void nfs_mapping_set_error(struct page *page, int error)
  272. {
  273. struct address_space *mapping = page_file_mapping(page);
  274. SetPageError(page);
  275. filemap_set_wb_err(mapping, error);
  276. if (mapping->host)
  277. errseq_set(&mapping->host->i_sb->s_wb_err,
  278. error == -ENOSPC ? -ENOSPC : -EIO);
  279. nfs_set_pageerror(mapping);
  280. }
  281. /*
  282. * nfs_page_group_search_locked
  283. * @head - head request of page group
  284. * @page_offset - offset into page
  285. *
  286. * Search page group with head @head to find a request that contains the
  287. * page offset @page_offset.
  288. *
  289. * Returns a pointer to the first matching nfs request, or NULL if no
  290. * match is found.
  291. *
  292. * Must be called with the page group lock held
  293. */
  294. static struct nfs_page *
  295. nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
  296. {
  297. struct nfs_page *req;
  298. req = head;
  299. do {
  300. if (page_offset >= req->wb_pgbase &&
  301. page_offset < (req->wb_pgbase + req->wb_bytes))
  302. return req;
  303. req = req->wb_this_page;
  304. } while (req != head);
  305. return NULL;
  306. }
  307. /*
  308. * nfs_page_group_covers_page
  309. * @head - head request of page group
  310. *
  311. * Return true if the page group with head @head covers the whole page,
  312. * returns false otherwise
  313. */
  314. static bool nfs_page_group_covers_page(struct nfs_page *req)
  315. {
  316. struct nfs_page *tmp;
  317. unsigned int pos = 0;
  318. unsigned int len = nfs_page_length(req->wb_page);
  319. nfs_page_group_lock(req);
  320. for (;;) {
  321. tmp = nfs_page_group_search_locked(req->wb_head, pos);
  322. if (!tmp)
  323. break;
  324. pos = tmp->wb_pgbase + tmp->wb_bytes;
  325. }
  326. nfs_page_group_unlock(req);
  327. return pos >= len;
  328. }
  329. /* We can set the PG_uptodate flag if we see that a write request
  330. * covers the full page.
  331. */
  332. static void nfs_mark_uptodate(struct nfs_page *req)
  333. {
  334. if (PageUptodate(req->wb_page))
  335. return;
  336. if (!nfs_page_group_covers_page(req))
  337. return;
  338. SetPageUptodate(req->wb_page);
  339. }
  340. static int wb_priority(struct writeback_control *wbc)
  341. {
  342. int ret = 0;
  343. if (wbc->sync_mode == WB_SYNC_ALL)
  344. ret = FLUSH_COND_STABLE;
  345. return ret;
  346. }
  347. /*
  348. * NFS congestion control
  349. */
  350. int nfs_congestion_kb;
  351. #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
  352. #define NFS_CONGESTION_OFF_THRESH \
  353. (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
  354. static void nfs_set_page_writeback(struct page *page)
  355. {
  356. struct inode *inode = page_file_mapping(page)->host;
  357. struct nfs_server *nfss = NFS_SERVER(inode);
  358. int ret = test_set_page_writeback(page);
  359. WARN_ON_ONCE(ret != 0);
  360. if (atomic_long_inc_return(&nfss->writeback) >
  361. NFS_CONGESTION_ON_THRESH)
  362. set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
  363. }
  364. static void nfs_end_page_writeback(struct nfs_page *req)
  365. {
  366. struct inode *inode = page_file_mapping(req->wb_page)->host;
  367. struct nfs_server *nfss = NFS_SERVER(inode);
  368. bool is_done;
  369. is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
  370. nfs_unlock_request(req);
  371. if (!is_done)
  372. return;
  373. end_page_writeback(req->wb_page);
  374. if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
  375. clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
  376. }
  377. /*
  378. * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
  379. *
  380. * @destroy_list - request list (using wb_this_page) terminated by @old_head
  381. * @old_head - the old head of the list
  382. *
  383. * All subrequests must be locked and removed from all lists, so at this point
  384. * they are only "active" in this function, and possibly in nfs_wait_on_request
  385. * with a reference held by some other context.
  386. */
  387. static void
  388. nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
  389. struct nfs_page *old_head,
  390. struct inode *inode)
  391. {
  392. while (destroy_list) {
  393. struct nfs_page *subreq = destroy_list;
  394. destroy_list = (subreq->wb_this_page == old_head) ?
  395. NULL : subreq->wb_this_page;
  396. /* Note: lock subreq in order to change subreq->wb_head */
  397. nfs_page_set_headlock(subreq);
  398. WARN_ON_ONCE(old_head != subreq->wb_head);
  399. /* make sure old group is not used */
  400. subreq->wb_this_page = subreq;
  401. subreq->wb_head = subreq;
  402. clear_bit(PG_REMOVE, &subreq->wb_flags);
  403. /* Note: races with nfs_page_group_destroy() */
  404. if (!kref_read(&subreq->wb_kref)) {
  405. /* Check if we raced with nfs_page_group_destroy() */
  406. if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) {
  407. nfs_page_clear_headlock(subreq);
  408. nfs_free_request(subreq);
  409. } else
  410. nfs_page_clear_headlock(subreq);
  411. continue;
  412. }
  413. nfs_page_clear_headlock(subreq);
  414. nfs_release_request(old_head);
  415. if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
  416. nfs_release_request(subreq);
  417. atomic_long_dec(&NFS_I(inode)->nrequests);
  418. }
  419. /* subreq is now totally disconnected from page group or any
  420. * write / commit lists. last chance to wake any waiters */
  421. nfs_unlock_and_release_request(subreq);
  422. }
  423. }
  424. /*
  425. * nfs_join_page_group - destroy subrequests of the head req
  426. * @head: the page used to lookup the "page group" of nfs_page structures
  427. * @inode: Inode to which the request belongs.
  428. *
  429. * This function joins all sub requests to the head request by first
  430. * locking all requests in the group, cancelling any pending operations
  431. * and finally updating the head request to cover the whole range covered by
  432. * the (former) group. All subrequests are removed from any write or commit
  433. * lists, unlinked from the group and destroyed.
  434. */
  435. void
  436. nfs_join_page_group(struct nfs_page *head, struct inode *inode)
  437. {
  438. struct nfs_page *subreq;
  439. struct nfs_page *destroy_list = NULL;
  440. unsigned int pgbase, off, bytes;
  441. pgbase = head->wb_pgbase;
  442. bytes = head->wb_bytes;
  443. off = head->wb_offset;
  444. for (subreq = head->wb_this_page; subreq != head;
  445. subreq = subreq->wb_this_page) {
  446. /* Subrequests should always form a contiguous range */
  447. if (pgbase > subreq->wb_pgbase) {
  448. off -= pgbase - subreq->wb_pgbase;
  449. bytes += pgbase - subreq->wb_pgbase;
  450. pgbase = subreq->wb_pgbase;
  451. }
  452. bytes = max(subreq->wb_pgbase + subreq->wb_bytes
  453. - pgbase, bytes);
  454. }
  455. /* Set the head request's range to cover the former page group */
  456. head->wb_pgbase = pgbase;
  457. head->wb_bytes = bytes;
  458. head->wb_offset = off;
  459. /* Now that all requests are locked, make sure they aren't on any list.
  460. * Commit list removal accounting is done after locks are dropped */
  461. subreq = head;
  462. do {
  463. nfs_clear_request_commit(subreq);
  464. subreq = subreq->wb_this_page;
  465. } while (subreq != head);
  466. /* unlink subrequests from head, destroy them later */
  467. if (head->wb_this_page != head) {
  468. /* destroy list will be terminated by head */
  469. destroy_list = head->wb_this_page;
  470. head->wb_this_page = head;
  471. }
  472. nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
  473. }
  474. /*
  475. * nfs_lock_and_join_requests - join all subreqs to the head req
  476. * @page: the page used to lookup the "page group" of nfs_page structures
  477. *
  478. * This function joins all sub requests to the head request by first
  479. * locking all requests in the group, cancelling any pending operations
  480. * and finally updating the head request to cover the whole range covered by
  481. * the (former) group. All subrequests are removed from any write or commit
  482. * lists, unlinked from the group and destroyed.
  483. *
  484. * Returns a locked, referenced pointer to the head request - which after
  485. * this call is guaranteed to be the only request associated with the page.
  486. * Returns NULL if no requests are found for @page, or a ERR_PTR if an
  487. * error was encountered.
  488. */
  489. static struct nfs_page *
  490. nfs_lock_and_join_requests(struct page *page)
  491. {
  492. struct inode *inode = page_file_mapping(page)->host;
  493. struct nfs_page *head;
  494. int ret;
  495. /*
  496. * A reference is taken only on the head request which acts as a
  497. * reference to the whole page group - the group will not be destroyed
  498. * until the head reference is released.
  499. */
  500. head = nfs_find_and_lock_page_request(page);
  501. if (IS_ERR_OR_NULL(head))
  502. return head;
  503. /* lock each request in the page group */
  504. ret = nfs_page_group_lock_subrequests(head);
  505. if (ret < 0) {
  506. nfs_unlock_and_release_request(head);
  507. return ERR_PTR(ret);
  508. }
  509. nfs_join_page_group(head, inode);
  510. return head;
  511. }
  512. static void nfs_write_error(struct nfs_page *req, int error)
  513. {
  514. trace_nfs_write_error(req, error);
  515. nfs_mapping_set_error(req->wb_page, error);
  516. nfs_inode_remove_request(req);
  517. nfs_end_page_writeback(req);
  518. nfs_release_request(req);
  519. }
  520. /*
  521. * Find an associated nfs write request, and prepare to flush it out
  522. * May return an error if the user signalled nfs_wait_on_request().
  523. */
  524. static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
  525. struct page *page)
  526. {
  527. struct nfs_page *req;
  528. int ret = 0;
  529. req = nfs_lock_and_join_requests(page);
  530. if (!req)
  531. goto out;
  532. ret = PTR_ERR(req);
  533. if (IS_ERR(req))
  534. goto out;
  535. nfs_set_page_writeback(page);
  536. WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
  537. /* If there is a fatal error that covers this write, just exit */
  538. ret = pgio->pg_error;
  539. if (nfs_error_is_fatal_on_server(ret))
  540. goto out_launder;
  541. ret = 0;
  542. if (!nfs_pageio_add_request(pgio, req)) {
  543. ret = pgio->pg_error;
  544. /*
  545. * Remove the problematic req upon fatal errors on the server
  546. */
  547. if (nfs_error_is_fatal(ret)) {
  548. if (nfs_error_is_fatal_on_server(ret))
  549. goto out_launder;
  550. } else
  551. ret = -EAGAIN;
  552. nfs_redirty_request(req);
  553. pgio->pg_error = 0;
  554. } else
  555. nfs_add_stats(page_file_mapping(page)->host,
  556. NFSIOS_WRITEPAGES, 1);
  557. out:
  558. return ret;
  559. out_launder:
  560. nfs_write_error(req, ret);
  561. return 0;
  562. }
  563. static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
  564. struct nfs_pageio_descriptor *pgio)
  565. {
  566. int ret;
  567. nfs_pageio_cond_complete(pgio, page_index(page));
  568. ret = nfs_page_async_flush(pgio, page);
  569. if (ret == -EAGAIN) {
  570. redirty_page_for_writepage(wbc, page);
  571. ret = AOP_WRITEPAGE_ACTIVATE;
  572. }
  573. return ret;
  574. }
  575. /*
  576. * Write an mmapped page to the server.
  577. */
  578. static int nfs_writepage_locked(struct page *page,
  579. struct writeback_control *wbc)
  580. {
  581. struct nfs_pageio_descriptor pgio;
  582. struct inode *inode = page_file_mapping(page)->host;
  583. int err;
  584. nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
  585. nfs_pageio_init_write(&pgio, inode, 0,
  586. false, &nfs_async_write_completion_ops);
  587. err = nfs_do_writepage(page, wbc, &pgio);
  588. pgio.pg_error = 0;
  589. nfs_pageio_complete(&pgio);
  590. if (err < 0)
  591. return err;
  592. if (nfs_error_is_fatal(pgio.pg_error))
  593. return pgio.pg_error;
  594. return 0;
  595. }
  596. int nfs_writepage(struct page *page, struct writeback_control *wbc)
  597. {
  598. int ret;
  599. ret = nfs_writepage_locked(page, wbc);
  600. if (ret != AOP_WRITEPAGE_ACTIVATE)
  601. unlock_page(page);
  602. return ret;
  603. }
  604. static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
  605. {
  606. int ret;
  607. ret = nfs_do_writepage(page, wbc, data);
  608. if (ret != AOP_WRITEPAGE_ACTIVATE)
  609. unlock_page(page);
  610. return ret;
  611. }
  612. static void nfs_io_completion_commit(void *inode)
  613. {
  614. nfs_commit_inode(inode, 0);
  615. }
  616. int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
  617. {
  618. struct inode *inode = mapping->host;
  619. struct nfs_pageio_descriptor pgio;
  620. struct nfs_io_completion *ioc;
  621. int err;
  622. nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
  623. ioc = nfs_io_completion_alloc(GFP_KERNEL);
  624. if (ioc)
  625. nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);
  626. nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
  627. &nfs_async_write_completion_ops);
  628. pgio.pg_io_completion = ioc;
  629. err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
  630. pgio.pg_error = 0;
  631. nfs_pageio_complete(&pgio);
  632. nfs_io_completion_put(ioc);
  633. if (err < 0)
  634. goto out_err;
  635. err = pgio.pg_error;
  636. if (nfs_error_is_fatal(err))
  637. goto out_err;
  638. return 0;
  639. out_err:
  640. return err;
  641. }
  642. /*
  643. * Insert a write request into an inode
  644. */
  645. static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
  646. {
  647. struct address_space *mapping = page_file_mapping(req->wb_page);
  648. struct nfs_inode *nfsi = NFS_I(inode);
  649. WARN_ON_ONCE(req->wb_this_page != req);
  650. /* Lock the request! */
  651. nfs_lock_request(req);
  652. /*
  653. * Swap-space should not get truncated. Hence no need to plug the race
  654. * with invalidate/truncate.
  655. */
  656. spin_lock(&mapping->private_lock);
  657. if (!nfs_have_writebacks(inode) &&
  658. NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
  659. inode_inc_iversion_raw(inode);
  660. if (likely(!PageSwapCache(req->wb_page))) {
  661. set_bit(PG_MAPPED, &req->wb_flags);
  662. SetPagePrivate(req->wb_page);
  663. set_page_private(req->wb_page, (unsigned long)req);
  664. }
  665. spin_unlock(&mapping->private_lock);
  666. atomic_long_inc(&nfsi->nrequests);
  667. /* this a head request for a page group - mark it as having an
  668. * extra reference so sub groups can follow suit.
  669. * This flag also informs pgio layer when to bump nrequests when
  670. * adding subrequests. */
  671. WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
  672. kref_get(&req->wb_kref);
  673. }
  674. /*
  675. * Remove a write request from an inode
  676. */
  677. static void nfs_inode_remove_request(struct nfs_page *req)
  678. {
  679. struct address_space *mapping = page_file_mapping(req->wb_page);
  680. struct inode *inode = mapping->host;
  681. struct nfs_inode *nfsi = NFS_I(inode);
  682. struct nfs_page *head;
  683. if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
  684. head = req->wb_head;
  685. spin_lock(&mapping->private_lock);
  686. if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
  687. set_page_private(head->wb_page, 0);
  688. ClearPagePrivate(head->wb_page);
  689. clear_bit(PG_MAPPED, &head->wb_flags);
  690. }
  691. spin_unlock(&mapping->private_lock);
  692. }
  693. if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) {
  694. nfs_release_request(req);
  695. atomic_long_dec(&nfsi->nrequests);
  696. }
  697. }
  698. static void
  699. nfs_mark_request_dirty(struct nfs_page *req)
  700. {
  701. if (req->wb_page)
  702. __set_page_dirty_nobuffers(req->wb_page);
  703. }
  704. /*
  705. * nfs_page_search_commits_for_head_request_locked
  706. *
  707. * Search through commit lists on @inode for the head request for @page.
  708. * Must be called while holding the inode (which is cinfo) lock.
  709. *
  710. * Returns the head request if found, or NULL if not found.
  711. */
  712. static struct nfs_page *
  713. nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
  714. struct page *page)
  715. {
  716. struct nfs_page *freq, *t;
  717. struct nfs_commit_info cinfo;
  718. struct inode *inode = &nfsi->vfs_inode;
  719. nfs_init_cinfo_from_inode(&cinfo, inode);
  720. /* search through pnfs commit lists */
  721. freq = pnfs_search_commit_reqs(inode, &cinfo, page);
  722. if (freq)
  723. return freq->wb_head;
  724. /* Linearly search the commit list for the correct request */
  725. list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
  726. if (freq->wb_page == page)
  727. return freq->wb_head;
  728. }
  729. return NULL;
  730. }
  731. /**
  732. * nfs_request_add_commit_list_locked - add request to a commit list
  733. * @req: pointer to a struct nfs_page
  734. * @dst: commit list head
  735. * @cinfo: holds list lock and accounting info
  736. *
  737. * This sets the PG_CLEAN bit, updates the cinfo count of
  738. * number of outstanding requests requiring a commit as well as
  739. * the MM page stats.
  740. *
  741. * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
  742. * nfs_page lock.
  743. */
  744. void
  745. nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
  746. struct nfs_commit_info *cinfo)
  747. {
  748. set_bit(PG_CLEAN, &req->wb_flags);
  749. nfs_list_add_request(req, dst);
  750. atomic_long_inc(&cinfo->mds->ncommit);
  751. }
  752. EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);
  753. /**
  754. * nfs_request_add_commit_list - add request to a commit list
  755. * @req: pointer to a struct nfs_page
  756. * @cinfo: holds list lock and accounting info
  757. *
  758. * This sets the PG_CLEAN bit, updates the cinfo count of
  759. * number of outstanding requests requiring a commit as well as
  760. * the MM page stats.
  761. *
  762. * The caller must _not_ hold the cinfo->lock, but must be
  763. * holding the nfs_page lock.
  764. */
  765. void
  766. nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
  767. {
  768. mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
  769. nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
  770. mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
  771. if (req->wb_page)
  772. nfs_mark_page_unstable(req->wb_page, cinfo);
  773. }
  774. EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
  775. /**
  776. * nfs_request_remove_commit_list - Remove request from a commit list
  777. * @req: pointer to a nfs_page
  778. * @cinfo: holds list lock and accounting info
  779. *
  780. * This clears the PG_CLEAN bit, and updates the cinfo's count of
  781. * number of outstanding requests requiring a commit
  782. * It does not update the MM page stats.
  783. *
  784. * The caller _must_ hold the cinfo->lock and the nfs_page lock.
  785. */
  786. void
  787. nfs_request_remove_commit_list(struct nfs_page *req,
  788. struct nfs_commit_info *cinfo)
  789. {
  790. if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
  791. return;
  792. nfs_list_remove_request(req);
  793. atomic_long_dec(&cinfo->mds->ncommit);
  794. }
  795. EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
  796. static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
  797. struct inode *inode)
  798. {
  799. cinfo->inode = inode;
  800. cinfo->mds = &NFS_I(inode)->commit_info;
  801. cinfo->ds = pnfs_get_ds_info(inode);
  802. cinfo->dreq = NULL;
  803. cinfo->completion_ops = &nfs_commit_completion_ops;
  804. }
  805. void nfs_init_cinfo(struct nfs_commit_info *cinfo,
  806. struct inode *inode,
  807. struct nfs_direct_req *dreq)
  808. {
  809. if (dreq)
  810. nfs_init_cinfo_from_dreq(cinfo, dreq);
  811. else
  812. nfs_init_cinfo_from_inode(cinfo, inode);
  813. }
  814. EXPORT_SYMBOL_GPL(nfs_init_cinfo);
  815. /*
  816. * Add a request to the inode's commit list.
  817. */
  818. void
  819. nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
  820. struct nfs_commit_info *cinfo, u32 ds_commit_idx)
  821. {
  822. if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
  823. return;
  824. nfs_request_add_commit_list(req, cinfo);
  825. }
  826. static void
  827. nfs_clear_page_commit(struct page *page)
  828. {
  829. dec_node_page_state(page, NR_WRITEBACK);
  830. dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
  831. WB_WRITEBACK);
  832. }
  833. /* Called holding the request lock on @req */
  834. static void
  835. nfs_clear_request_commit(struct nfs_page *req)
  836. {
  837. if (test_bit(PG_CLEAN, &req->wb_flags)) {
  838. struct nfs_open_context *ctx = nfs_req_openctx(req);
  839. struct inode *inode = d_inode(ctx->dentry);
  840. struct nfs_commit_info cinfo;
  841. nfs_init_cinfo_from_inode(&cinfo, inode);
  842. mutex_lock(&NFS_I(inode)->commit_mutex);
  843. if (!pnfs_clear_request_commit(req, &cinfo)) {
  844. nfs_request_remove_commit_list(req, &cinfo);
  845. }
  846. mutex_unlock(&NFS_I(inode)->commit_mutex);
  847. nfs_clear_page_commit(req->wb_page);
  848. }
  849. }
  850. int nfs_write_need_commit(struct nfs_pgio_header *hdr)
  851. {
  852. if (hdr->verf.committed == NFS_DATA_SYNC)
  853. return hdr->lseg == NULL;
  854. return hdr->verf.committed != NFS_FILE_SYNC;
  855. }
  856. static void nfs_async_write_init(struct nfs_pgio_header *hdr)
  857. {
  858. nfs_io_completion_get(hdr->io_completion);
  859. }
  860. static void nfs_write_completion(struct nfs_pgio_header *hdr)
  861. {
  862. struct nfs_commit_info cinfo;
  863. unsigned long bytes = 0;
  864. if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
  865. goto out;
  866. nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
  867. while (!list_empty(&hdr->pages)) {
  868. struct nfs_page *req = nfs_list_entry(hdr->pages.next);
  869. bytes += req->wb_bytes;
  870. nfs_list_remove_request(req);
  871. if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
  872. (hdr->good_bytes < bytes)) {
  873. trace_nfs_comp_error(req, hdr->error);
  874. nfs_mapping_set_error(req->wb_page, hdr->error);
  875. goto remove_req;
  876. }
  877. if (nfs_write_need_commit(hdr)) {
  878. /* Reset wb_nio, since the write was successful. */
  879. req->wb_nio = 0;
  880. memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
  881. nfs_mark_request_commit(req, hdr->lseg, &cinfo,
  882. hdr->pgio_mirror_idx);
  883. goto next;
  884. }
  885. remove_req:
  886. nfs_inode_remove_request(req);
  887. next:
  888. nfs_end_page_writeback(req);
  889. nfs_release_request(req);
  890. }
  891. out:
  892. nfs_io_completion_put(hdr->io_completion);
  893. hdr->release(hdr);
  894. }
  895. unsigned long
  896. nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
  897. {
  898. return atomic_long_read(&cinfo->mds->ncommit);
  899. }
  900. /* NFS_I(cinfo->inode)->commit_mutex held by caller */
  901. int
  902. nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
  903. struct nfs_commit_info *cinfo, int max)
  904. {
  905. struct nfs_page *req, *tmp;
  906. int ret = 0;
  907. list_for_each_entry_safe(req, tmp, src, wb_list) {
  908. kref_get(&req->wb_kref);
  909. if (!nfs_lock_request(req)) {
  910. nfs_release_request(req);
  911. continue;
  912. }
  913. nfs_request_remove_commit_list(req, cinfo);
  914. clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
  915. nfs_list_add_request(req, dst);
  916. ret++;
  917. if ((ret == max) && !cinfo->dreq)
  918. break;
  919. cond_resched();
  920. }
  921. return ret;
  922. }
  923. EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
  924. /*
  925. * nfs_scan_commit - Scan an inode for commit requests
  926. * @inode: NFS inode to scan
  927. * @dst: mds destination list
  928. * @cinfo: mds and ds lists of reqs ready to commit
  929. *
  930. * Moves requests from the inode's 'commit' request list.
  931. * The requests are *not* checked to ensure that they form a contiguous set.
  932. */
  933. int
  934. nfs_scan_commit(struct inode *inode, struct list_head *dst,
  935. struct nfs_commit_info *cinfo)
  936. {
  937. int ret = 0;
  938. if (!atomic_long_read(&cinfo->mds->ncommit))
  939. return 0;
  940. mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
  941. if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
  942. const int max = INT_MAX;
  943. ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
  944. cinfo, max);
  945. ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
  946. }
  947. mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
  948. return ret;
  949. }
  950. /*
  951. * Search for an existing write request, and attempt to update
  952. * it to reflect a new dirty region on a given page.
  953. *
  954. * If the attempt fails, then the existing request is flushed out
  955. * to disk.
  956. */
  957. static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
  958. struct page *page,
  959. unsigned int offset,
  960. unsigned int bytes)
  961. {
  962. struct nfs_page *req;
  963. unsigned int rqend;
  964. unsigned int end;
  965. int error;
  966. end = offset + bytes;
  967. req = nfs_lock_and_join_requests(page);
  968. if (IS_ERR_OR_NULL(req))
  969. return req;
  970. rqend = req->wb_offset + req->wb_bytes;
  971. /*
  972. * Tell the caller to flush out the request if
  973. * the offsets are non-contiguous.
  974. * Note: nfs_flush_incompatible() will already
  975. * have flushed out requests having wrong owners.
  976. */
  977. if (offset > rqend || end < req->wb_offset)
  978. goto out_flushme;
  979. /* Okay, the request matches. Update the region */
  980. if (offset < req->wb_offset) {
  981. req->wb_offset = offset;
  982. req->wb_pgbase = offset;
  983. }
  984. if (end > rqend)
  985. req->wb_bytes = end - req->wb_offset;
  986. else
  987. req->wb_bytes = rqend - req->wb_offset;
  988. req->wb_nio = 0;
  989. return req;
  990. out_flushme:
  991. /*
  992. * Note: we mark the request dirty here because
  993. * nfs_lock_and_join_requests() cannot preserve
  994. * commit flags, so we have to replay the write.
  995. */
  996. nfs_mark_request_dirty(req);
  997. nfs_unlock_and_release_request(req);
  998. error = nfs_wb_page(inode, page);
  999. return (error < 0) ? ERR_PTR(error) : NULL;
  1000. }
  1001. /*
  1002. * Try to update an existing write request, or create one if there is none.
  1003. *
  1004. * Note: Should always be called with the Page Lock held to prevent races
  1005. * if we have to add a new request. Also assumes that the caller has
  1006. * already called nfs_flush_incompatible() if necessary.
  1007. */
  1008. static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
  1009. struct page *page, unsigned int offset, unsigned int bytes)
  1010. {
  1011. struct inode *inode = page_file_mapping(page)->host;
  1012. struct nfs_page *req;
  1013. req = nfs_try_to_update_request(inode, page, offset, bytes);
  1014. if (req != NULL)
  1015. goto out;
  1016. req = nfs_create_request(ctx, page, offset, bytes);
  1017. if (IS_ERR(req))
  1018. goto out;
  1019. nfs_inode_add_request(inode, req);
  1020. out:
  1021. return req;
  1022. }
  1023. static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
  1024. unsigned int offset, unsigned int count)
  1025. {
  1026. struct nfs_page *req;
  1027. req = nfs_setup_write_request(ctx, page, offset, count);
  1028. if (IS_ERR(req))
  1029. return PTR_ERR(req);
  1030. /* Update file length */
  1031. nfs_grow_file(page, offset, count);
  1032. nfs_mark_uptodate(req);
  1033. nfs_mark_request_dirty(req);
  1034. nfs_unlock_and_release_request(req);
  1035. return 0;
  1036. }
  1037. int nfs_flush_incompatible(struct file *file, struct page *page)
  1038. {
  1039. struct nfs_open_context *ctx = nfs_file_open_context(file);
  1040. struct nfs_lock_context *l_ctx;
  1041. struct file_lock_context *flctx = file_inode(file)->i_flctx;
  1042. struct nfs_page *req;
  1043. int do_flush, status;
  1044. /*
  1045. * Look for a request corresponding to this page. If there
  1046. * is one, and it belongs to another file, we flush it out
  1047. * before we try to copy anything into the page. Do this
  1048. * due to the lack of an ACCESS-type call in NFSv2.
  1049. * Also do the same if we find a request from an existing
  1050. * dropped page.
  1051. */
  1052. do {
  1053. req = nfs_page_find_head_request(page);
  1054. if (req == NULL)
  1055. return 0;
  1056. l_ctx = req->wb_lock_context;
  1057. do_flush = req->wb_page != page ||
  1058. !nfs_match_open_context(nfs_req_openctx(req), ctx);
  1059. if (l_ctx && flctx &&
  1060. !(list_empty_careful(&flctx->flc_posix) &&
  1061. list_empty_careful(&flctx->flc_flock))) {
  1062. do_flush |= l_ctx->lockowner != current->files;
  1063. }
  1064. nfs_release_request(req);
  1065. if (!do_flush)
  1066. return 0;
  1067. status = nfs_wb_page(page_file_mapping(page)->host, page);
  1068. } while (status == 0);
  1069. return status;
  1070. }
  1071. /*
  1072. * Avoid buffered writes when a open context credential's key would
  1073. * expire soon.
  1074. *
  1075. * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
  1076. *
  1077. * Return 0 and set a credential flag which triggers the inode to flush
  1078. * and performs NFS_FILE_SYNC writes if the key will expired within
  1079. * RPC_KEY_EXPIRE_TIMEO.
  1080. */
  1081. int
  1082. nfs_key_timeout_notify(struct file *filp, struct inode *inode)
  1083. {
  1084. struct nfs_open_context *ctx = nfs_file_open_context(filp);
  1085. if (nfs_ctx_key_to_expire(ctx, inode) &&
  1086. !ctx->ll_cred)
  1087. /* Already expired! */
  1088. return -EACCES;
  1089. return 0;
  1090. }
  1091. /*
  1092. * Test if the open context credential key is marked to expire soon.
  1093. */
  1094. bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
  1095. {
  1096. struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
  1097. struct rpc_cred *cred = ctx->ll_cred;
  1098. struct auth_cred acred = {
  1099. .cred = ctx->cred,
  1100. };
  1101. if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) {
  1102. put_rpccred(cred);
  1103. ctx->ll_cred = NULL;
  1104. cred = NULL;
  1105. }
  1106. if (!cred)
  1107. cred = auth->au_ops->lookup_cred(auth, &acred, 0);
  1108. if (!cred || IS_ERR(cred))
  1109. return true;
  1110. ctx->ll_cred = cred;
  1111. return !!(cred->cr_ops->crkey_timeout &&
  1112. cred->cr_ops->crkey_timeout(cred));
  1113. }
  1114. /*
  1115. * If the page cache is marked as unsafe or invalid, then we can't rely on
  1116. * the PageUptodate() flag. In this case, we will need to turn off
  1117. * write optimisations that depend on the page contents being correct.
  1118. */
  1119. static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
  1120. {
  1121. struct nfs_inode *nfsi = NFS_I(inode);
  1122. if (nfs_have_delegated_attributes(inode))
  1123. goto out;
  1124. if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
  1125. return false;
  1126. smp_rmb();
  1127. if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
  1128. return false;
  1129. out:
  1130. if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
  1131. return false;
  1132. return PageUptodate(page) != 0;
  1133. }
  1134. static bool
  1135. is_whole_file_wrlock(struct file_lock *fl)
  1136. {
  1137. return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
  1138. fl->fl_type == F_WRLCK;
  1139. }
  1140. /* If we know the page is up to date, and we're not using byte range locks (or
  1141. * if we have the whole file locked for writing), it may be more efficient to
  1142. * extend the write to cover the entire page in order to avoid fragmentation
  1143. * inefficiencies.
  1144. *
  1145. * If the file is opened for synchronous writes then we can just skip the rest
  1146. * of the checks.
  1147. */
  1148. static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
  1149. {
  1150. int ret;
  1151. struct file_lock_context *flctx = inode->i_flctx;
  1152. struct file_lock *fl;
  1153. if (file->f_flags & O_DSYNC)
  1154. return 0;
  1155. if (!nfs_write_pageuptodate(page, inode))
  1156. return 0;
  1157. if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
  1158. return 1;
  1159. if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
  1160. list_empty_careful(&flctx->flc_posix)))
  1161. return 1;
  1162. /* Check to see if there are whole file write locks */
  1163. ret = 0;
  1164. spin_lock(&flctx->flc_lock);
  1165. if (!list_empty(&flctx->flc_posix)) {
  1166. fl = list_first_entry(&flctx->flc_posix, struct file_lock,
  1167. fl_list);
  1168. if (is_whole_file_wrlock(fl))
  1169. ret = 1;
  1170. } else if (!list_empty(&flctx->flc_flock)) {
  1171. fl = list_first_entry(&flctx->flc_flock, struct file_lock,
  1172. fl_list);
  1173. if (fl->fl_type == F_WRLCK)
  1174. ret = 1;
  1175. }
  1176. spin_unlock(&flctx->flc_lock);
  1177. return ret;
  1178. }
  1179. /*
  1180. * Update and possibly write a cached page of an NFS file.
  1181. *
  1182. * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
  1183. * things with a page scheduled for an RPC call (e.g. invalidate it).
  1184. */
  1185. int nfs_updatepage(struct file *file, struct page *page,
  1186. unsigned int offset, unsigned int count)
  1187. {
  1188. struct nfs_open_context *ctx = nfs_file_open_context(file);
  1189. struct address_space *mapping = page_file_mapping(page);
  1190. struct inode *inode = mapping->host;
  1191. int status = 0;
  1192. nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
  1193. dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n",
  1194. file, count, (long long)(page_file_offset(page) + offset));
  1195. if (!count)
  1196. goto out;
  1197. if (nfs_can_extend_write(file, page, inode)) {
  1198. count = max(count + offset, nfs_page_length(page));
  1199. offset = 0;
  1200. }
  1201. status = nfs_writepage_setup(ctx, page, offset, count);
  1202. if (status < 0)
  1203. nfs_set_pageerror(mapping);
  1204. else
  1205. __set_page_dirty_nobuffers(page);
  1206. out:
  1207. dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
  1208. status, (long long)i_size_read(inode));
  1209. return status;
  1210. }
  1211. static int flush_task_priority(int how)
  1212. {
  1213. switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
  1214. case FLUSH_HIGHPRI:
  1215. return RPC_PRIORITY_HIGH;
  1216. case FLUSH_LOWPRI:
  1217. return RPC_PRIORITY_LOW;
  1218. }
  1219. return RPC_PRIORITY_NORMAL;
  1220. }
  1221. static void nfs_initiate_write(struct nfs_pgio_header *hdr,
  1222. struct rpc_message *msg,
  1223. const struct nfs_rpc_ops *rpc_ops,
  1224. struct rpc_task_setup *task_setup_data, int how)
  1225. {
  1226. int priority = flush_task_priority(how);
  1227. task_setup_data->priority = priority;
  1228. rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
  1229. trace_nfs_initiate_write(hdr);
  1230. }
  1231. /* If a nfs_flush_* function fails, it should remove reqs from @head and
  1232. * call this on each, which will prepare them to be retried on next
  1233. * writeback using standard nfs.
  1234. */
  1235. static void nfs_redirty_request(struct nfs_page *req)
  1236. {
  1237. /* Bump the transmission count */
  1238. req->wb_nio++;
  1239. nfs_mark_request_dirty(req);
  1240. set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
  1241. nfs_end_page_writeback(req);
  1242. nfs_release_request(req);
  1243. }
  1244. static void nfs_async_write_error(struct list_head *head, int error)
  1245. {
  1246. struct nfs_page *req;
  1247. while (!list_empty(head)) {
  1248. req = nfs_list_entry(head->next);
  1249. nfs_list_remove_request(req);
  1250. if (nfs_error_is_fatal(error))
  1251. nfs_write_error(req, error);
  1252. else
  1253. nfs_redirty_request(req);
  1254. }
  1255. }
  1256. static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
  1257. {
  1258. nfs_async_write_error(&hdr->pages, 0);
  1259. filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset,
  1260. hdr->args.offset + hdr->args.count - 1);
  1261. }
  1262. static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
  1263. .init_hdr = nfs_async_write_init,
  1264. .error_cleanup = nfs_async_write_error,
  1265. .completion = nfs_write_completion,
  1266. .reschedule_io = nfs_async_write_reschedule_io,
  1267. };
  1268. void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
  1269. struct inode *inode, int ioflags, bool force_mds,
  1270. const struct nfs_pgio_completion_ops *compl_ops)
  1271. {
  1272. struct nfs_server *server = NFS_SERVER(inode);
  1273. const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
  1274. #ifdef CONFIG_NFS_V4_1
  1275. if (server->pnfs_curr_ld && !force_mds)
  1276. pg_ops = server->pnfs_curr_ld->pg_write_ops;
  1277. #endif
  1278. nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
  1279. server->wsize, ioflags);
  1280. }
  1281. EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
  1282. void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
  1283. {
  1284. struct nfs_pgio_mirror *mirror;
  1285. if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
  1286. pgio->pg_ops->pg_cleanup(pgio);
  1287. pgio->pg_ops = &nfs_pgio_rw_ops;
  1288. nfs_pageio_stop_mirroring(pgio);
  1289. mirror = &pgio->pg_mirrors[0];
  1290. mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
  1291. }
  1292. EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
  1293. void nfs_commit_prepare(struct rpc_task *task, void *calldata)
  1294. {
  1295. struct nfs_commit_data *data = calldata;
  1296. NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
  1297. }
  1298. /*
  1299. * Special version of should_remove_suid() that ignores capabilities.
  1300. */
  1301. static int nfs_should_remove_suid(const struct inode *inode)
  1302. {
  1303. umode_t mode = inode->i_mode;
  1304. int kill = 0;
  1305. /* suid always must be killed */
  1306. if (unlikely(mode & S_ISUID))
  1307. kill = ATTR_KILL_SUID;
  1308. /*
  1309. * sgid without any exec bits is just a mandatory locking mark; leave
  1310. * it alone. If some exec bits are set, it's a real sgid; kill it.
  1311. */
  1312. if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
  1313. kill |= ATTR_KILL_SGID;
  1314. if (unlikely(kill && S_ISREG(mode)))
  1315. return kill;
  1316. return 0;
  1317. }
  1318. static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
  1319. struct nfs_fattr *fattr)
  1320. {
  1321. struct nfs_pgio_args *argp = &hdr->args;
  1322. struct nfs_pgio_res *resp = &hdr->res;
  1323. u64 size = argp->offset + resp->count;
  1324. if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
  1325. fattr->size = size;
  1326. if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
  1327. fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
  1328. return;
  1329. }
  1330. if (size != fattr->size)
  1331. return;
  1332. /* Set attribute barrier */
  1333. nfs_fattr_set_barrier(fattr);
  1334. /* ...and update size */
  1335. fattr->valid |= NFS_ATTR_FATTR_SIZE;
  1336. }
  1337. void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
  1338. {
  1339. struct nfs_fattr *fattr = &hdr->fattr;
  1340. struct inode *inode = hdr->inode;
  1341. spin_lock(&inode->i_lock);
  1342. nfs_writeback_check_extend(hdr, fattr);
  1343. nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
  1344. spin_unlock(&inode->i_lock);
  1345. }
  1346. EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);
  1347. /*
  1348. * This function is called when the WRITE call is complete.
  1349. */
  1350. static int nfs_writeback_done(struct rpc_task *task,
  1351. struct nfs_pgio_header *hdr,
  1352. struct inode *inode)
  1353. {
  1354. int status;
  1355. /*
  1356. * ->write_done will attempt to use post-op attributes to detect
  1357. * conflicting writes by other clients. A strict interpretation
  1358. * of close-to-open would allow us to continue caching even if
  1359. * another writer had changed the file, but some applications
  1360. * depend on tighter cache coherency when writing.
  1361. */
  1362. status = NFS_PROTO(inode)->write_done(task, hdr);
  1363. if (status != 0)
  1364. return status;
  1365. nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
  1366. trace_nfs_writeback_done(task, hdr);
  1367. if (hdr->res.verf->committed < hdr->args.stable &&
  1368. task->tk_status >= 0) {
  1369. /* We tried a write call, but the server did not
  1370. * commit data to stable storage even though we
  1371. * requested it.
  1372. * Note: There is a known bug in Tru64 < 5.0 in which
  1373. * the server reports NFS_DATA_SYNC, but performs
  1374. * NFS_FILE_SYNC. We therefore implement this checking
  1375. * as a dprintk() in order to avoid filling syslog.
  1376. */
  1377. static unsigned long complain;
  1378. /* Note this will print the MDS for a DS write */
  1379. if (time_before(complain, jiffies)) {
  1380. dprintk("NFS: faulty NFS server %s:"
  1381. " (committed = %d) != (stable = %d)\n",
  1382. NFS_SERVER(inode)->nfs_client->cl_hostname,
  1383. hdr->res.verf->committed, hdr->args.stable);
  1384. complain = jiffies + 300 * HZ;
  1385. }
  1386. }
  1387. /* Deal with the suid/sgid bit corner case */
  1388. if (nfs_should_remove_suid(inode)) {
  1389. spin_lock(&inode->i_lock);
  1390. NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
  1391. spin_unlock(&inode->i_lock);
  1392. }
  1393. return 0;
  1394. }
  1395. /*
  1396. * This function is called when the WRITE call is complete.
  1397. */
  1398. static void nfs_writeback_result(struct rpc_task *task,
  1399. struct nfs_pgio_header *hdr)
  1400. {
  1401. struct nfs_pgio_args *argp = &hdr->args;
  1402. struct nfs_pgio_res *resp = &hdr->res;
  1403. if (resp->count < argp->count) {
  1404. static unsigned long complain;
  1405. /* This a short write! */
  1406. nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
  1407. /* Has the server at least made some progress? */
  1408. if (resp->count == 0) {
  1409. if (time_before(complain, jiffies)) {
  1410. printk(KERN_WARNING
  1411. "NFS: Server wrote zero bytes, expected %u.\n",
  1412. argp->count);
  1413. complain = jiffies + 300 * HZ;
  1414. }
  1415. nfs_set_pgio_error(hdr, -EIO, argp->offset);
  1416. task->tk_status = -EIO;
  1417. return;
  1418. }
  1419. /* For non rpc-based layout drivers, retry-through-MDS */
  1420. if (!task->tk_ops) {
  1421. hdr->pnfs_error = -EAGAIN;
  1422. return;
  1423. }
  1424. /* Was this an NFSv2 write or an NFSv3 stable write? */
  1425. if (resp->verf->committed != NFS_UNSTABLE) {
  1426. /* Resend from where the server left off */
  1427. hdr->mds_offset += resp->count;
  1428. argp->offset += resp->count;
  1429. argp->pgbase += resp->count;
  1430. argp->count -= resp->count;
  1431. } else {
  1432. /* Resend as a stable write in order to avoid
  1433. * headaches in the case of a server crash.
  1434. */
  1435. argp->stable = NFS_FILE_SYNC;
  1436. }
  1437. resp->count = 0;
  1438. resp->verf->committed = 0;
  1439. rpc_restart_call_prepare(task);
  1440. }
  1441. }
  1442. static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
  1443. {
  1444. return wait_var_event_killable(&cinfo->rpcs_out,
  1445. !atomic_read(&cinfo->rpcs_out));
  1446. }
  1447. static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
  1448. {
  1449. atomic_inc(&cinfo->rpcs_out);
  1450. }
  1451. bool nfs_commit_end(struct nfs_mds_commit_info *cinfo)
  1452. {
  1453. if (atomic_dec_and_test(&cinfo->rpcs_out)) {
  1454. wake_up_var(&cinfo->rpcs_out);
  1455. return true;
  1456. }
  1457. return false;
  1458. }
  1459. void nfs_commitdata_release(struct nfs_commit_data *data)
  1460. {
  1461. put_nfs_open_context(data->context);
  1462. nfs_commit_free(data);
  1463. }
  1464. EXPORT_SYMBOL_GPL(nfs_commitdata_release);
  1465. int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
  1466. const struct nfs_rpc_ops *nfs_ops,
  1467. const struct rpc_call_ops *call_ops,
  1468. int how, int flags)
  1469. {
  1470. struct rpc_task *task;
  1471. int priority = flush_task_priority(how);
  1472. struct rpc_message msg = {
  1473. .rpc_argp = &data->args,
  1474. .rpc_resp = &data->res,
  1475. .rpc_cred = data->cred,
  1476. };
  1477. struct rpc_task_setup task_setup_data = {
  1478. .task = &data->task,
  1479. .rpc_client = clnt,
  1480. .rpc_message = &msg,
  1481. .callback_ops = call_ops,
  1482. .callback_data = data,
  1483. .workqueue = nfsiod_workqueue,
  1484. .flags = RPC_TASK_ASYNC | flags,
  1485. .priority = priority,
  1486. };
  1487. /* Set up the initial task struct. */
  1488. nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
  1489. trace_nfs_initiate_commit(data);
  1490. dprintk("NFS: initiated commit call\n");
  1491. task = rpc_run_task(&task_setup_data);
  1492. if (IS_ERR(task))
  1493. return PTR_ERR(task);
  1494. if (how & FLUSH_SYNC)
  1495. rpc_wait_for_completion_task(task);
  1496. rpc_put_task(task);
  1497. return 0;
  1498. }
  1499. EXPORT_SYMBOL_GPL(nfs_initiate_commit);
  1500. static loff_t nfs_get_lwb(struct list_head *head)
  1501. {
  1502. loff_t lwb = 0;
  1503. struct nfs_page *req;
  1504. list_for_each_entry(req, head, wb_list)
  1505. if (lwb < (req_offset(req) + req->wb_bytes))
  1506. lwb = req_offset(req) + req->wb_bytes;
  1507. return lwb;
  1508. }
  1509. /*
  1510. * Set up the argument/result storage required for the RPC call.
  1511. */
  1512. void nfs_init_commit(struct nfs_commit_data *data,
  1513. struct list_head *head,
  1514. struct pnfs_layout_segment *lseg,
  1515. struct nfs_commit_info *cinfo)
  1516. {
  1517. struct nfs_page *first;
  1518. struct nfs_open_context *ctx;
  1519. struct inode *inode;
  1520. /* Set up the RPC argument and reply structs
  1521. * NB: take care not to mess about with data->commit et al. */
  1522. if (head)
  1523. list_splice_init(head, &data->pages);
  1524. first = nfs_list_entry(data->pages.next);
  1525. ctx = nfs_req_openctx(first);
  1526. inode = d_inode(ctx->dentry);
  1527. data->inode = inode;
  1528. data->cred = ctx->cred;
  1529. data->lseg = lseg; /* reference transferred */
  1530. /* only set lwb for pnfs commit */
  1531. if (lseg)
  1532. data->lwb = nfs_get_lwb(&data->pages);
  1533. data->mds_ops = &nfs_commit_ops;
  1534. data->completion_ops = cinfo->completion_ops;
  1535. data->dreq = cinfo->dreq;
  1536. data->args.fh = NFS_FH(data->inode);
  1537. /* Note: we always request a commit of the entire inode */
  1538. data->args.offset = 0;
  1539. data->args.count = 0;
  1540. data->context = get_nfs_open_context(ctx);
  1541. data->res.fattr = &data->fattr;
  1542. data->res.verf = &data->verf;
  1543. nfs_fattr_init(&data->fattr);
  1544. nfs_commit_begin(cinfo->mds);
  1545. }
  1546. EXPORT_SYMBOL_GPL(nfs_init_commit);
  1547. void nfs_retry_commit(struct list_head *page_list,
  1548. struct pnfs_layout_segment *lseg,
  1549. struct nfs_commit_info *cinfo,
  1550. u32 ds_commit_idx)
  1551. {
  1552. struct nfs_page *req;
  1553. while (!list_empty(page_list)) {
  1554. req = nfs_list_entry(page_list->next);
  1555. nfs_list_remove_request(req);
  1556. nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
  1557. if (!cinfo->dreq)
  1558. nfs_clear_page_commit(req->wb_page);
  1559. nfs_unlock_and_release_request(req);
  1560. }
  1561. }
  1562. EXPORT_SYMBOL_GPL(nfs_retry_commit);
  1563. static void
  1564. nfs_commit_resched_write(struct nfs_commit_info *cinfo,
  1565. struct nfs_page *req)
  1566. {
  1567. __set_page_dirty_nobuffers(req->wb_page);
  1568. }
  1569. /*
  1570. * Commit dirty pages
  1571. */
  1572. static int
  1573. nfs_commit_list(struct inode *inode, struct list_head *head, int how,
  1574. struct nfs_commit_info *cinfo)
  1575. {
  1576. struct nfs_commit_data *data;
  1577. /* another commit raced with us */
  1578. if (list_empty(head))
  1579. return 0;
  1580. data = nfs_commitdata_alloc();
  1581. if (!data) {
  1582. nfs_retry_commit(head, NULL, cinfo, -1);
  1583. return -ENOMEM;
  1584. }
  1585. /* Set up the argument struct */
  1586. nfs_init_commit(data, head, NULL, cinfo);
  1587. return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
  1588. data->mds_ops, how, RPC_TASK_CRED_NOREF);
  1589. }
  1590. /*
  1591. * COMMIT call returned
  1592. */
  1593. static void nfs_commit_done(struct rpc_task *task, void *calldata)
  1594. {
  1595. struct nfs_commit_data *data = calldata;
  1596. dprintk("NFS: %5u nfs_commit_done (status %d)\n",
  1597. task->tk_pid, task->tk_status);
  1598. /* Call the NFS version-specific code */
  1599. NFS_PROTO(data->inode)->commit_done(task, data);
  1600. trace_nfs_commit_done(task, data);
  1601. }
  1602. static void nfs_commit_release_pages(struct nfs_commit_data *data)
  1603. {
  1604. const struct nfs_writeverf *verf = data->res.verf;
  1605. struct nfs_page *req;
  1606. int status = data->task.tk_status;
  1607. struct nfs_commit_info cinfo;
  1608. struct nfs_server *nfss;
  1609. while (!list_empty(&data->pages)) {
  1610. req = nfs_list_entry(data->pages.next);
  1611. nfs_list_remove_request(req);
  1612. if (req->wb_page)
  1613. nfs_clear_page_commit(req->wb_page);
  1614. dprintk("NFS: commit (%s/%llu %d@%lld)",
  1615. nfs_req_openctx(req)->dentry->d_sb->s_id,
  1616. (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)),
  1617. req->wb_bytes,
  1618. (long long)req_offset(req));
  1619. if (status < 0) {
  1620. if (req->wb_page) {
  1621. trace_nfs_commit_error(req, status);
  1622. nfs_mapping_set_error(req->wb_page, status);
  1623. nfs_inode_remove_request(req);
  1624. }
  1625. dprintk_cont(", error = %d\n", status);
  1626. goto next;
  1627. }
  1628. /* Okay, COMMIT succeeded, apparently. Check the verifier
  1629. * returned by the server against all stored verfs. */
  1630. if (nfs_write_match_verf(verf, req)) {
  1631. /* We have a match */
  1632. if (req->wb_page)
  1633. nfs_inode_remove_request(req);
  1634. dprintk_cont(" OK\n");
  1635. goto next;
  1636. }
  1637. /* We have a mismatch. Write the page again */
  1638. dprintk_cont(" mismatch\n");
  1639. nfs_mark_request_dirty(req);
  1640. set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
  1641. next:
  1642. nfs_unlock_and_release_request(req);
  1643. /* Latency breaker */
  1644. cond_resched();
  1645. }
  1646. nfss = NFS_SERVER(data->inode);
  1647. if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
  1648. clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
  1649. nfs_init_cinfo(&cinfo, data->inode, data->dreq);
  1650. nfs_commit_end(cinfo.mds);
  1651. }
  1652. static void nfs_commit_release(void *calldata)
  1653. {
  1654. struct nfs_commit_data *data = calldata;
  1655. data->completion_ops->completion(data);
  1656. nfs_commitdata_release(calldata);
  1657. }
  1658. static const struct rpc_call_ops nfs_commit_ops = {
  1659. .rpc_call_prepare = nfs_commit_prepare,
  1660. .rpc_call_done = nfs_commit_done,
  1661. .rpc_release = nfs_commit_release,
  1662. };
  1663. static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
  1664. .completion = nfs_commit_release_pages,
  1665. .resched_write = nfs_commit_resched_write,
  1666. };
  1667. int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
  1668. int how, struct nfs_commit_info *cinfo)
  1669. {
  1670. int status;
  1671. status = pnfs_commit_list(inode, head, how, cinfo);
  1672. if (status == PNFS_NOT_ATTEMPTED)
  1673. status = nfs_commit_list(inode, head, how, cinfo);
  1674. return status;
  1675. }
  1676. static int __nfs_commit_inode(struct inode *inode, int how,
  1677. struct writeback_control *wbc)
  1678. {
  1679. LIST_HEAD(head);
  1680. struct nfs_commit_info cinfo;
  1681. int may_wait = how & FLUSH_SYNC;
  1682. int ret, nscan;
  1683. how &= ~FLUSH_SYNC;
  1684. nfs_init_cinfo_from_inode(&cinfo, inode);
  1685. nfs_commit_begin(cinfo.mds);
  1686. for (;;) {
  1687. ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
  1688. if (ret <= 0)
  1689. break;
  1690. ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
  1691. if (ret < 0)
  1692. break;
  1693. ret = 0;
  1694. if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
  1695. if (nscan < wbc->nr_to_write)
  1696. wbc->nr_to_write -= nscan;
  1697. else
  1698. wbc->nr_to_write = 0;
  1699. }
  1700. if (nscan < INT_MAX)
  1701. break;
  1702. cond_resched();
  1703. }
  1704. nfs_commit_end(cinfo.mds);
  1705. if (ret || !may_wait)
  1706. return ret;
  1707. return wait_on_commit(cinfo.mds);
  1708. }
  1709. int nfs_commit_inode(struct inode *inode, int how)
  1710. {
  1711. return __nfs_commit_inode(inode, how, NULL);
  1712. }
  1713. EXPORT_SYMBOL_GPL(nfs_commit_inode);
  1714. int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
  1715. {
  1716. struct nfs_inode *nfsi = NFS_I(inode);
  1717. int flags = FLUSH_SYNC;
  1718. int ret = 0;
  1719. if (wbc->sync_mode == WB_SYNC_NONE) {
  1720. /* no commits means nothing needs to be done */
  1721. if (!atomic_long_read(&nfsi->commit_info.ncommit))
  1722. goto check_requests_outstanding;
  1723. /* Don't commit yet if this is a non-blocking flush and there
  1724. * are a lot of outstanding writes for this mapping.
  1725. */
  1726. if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
  1727. goto out_mark_dirty;
  1728. /* don't wait for the COMMIT response */
  1729. flags = 0;
  1730. }
  1731. ret = __nfs_commit_inode(inode, flags, wbc);
  1732. if (!ret) {
  1733. if (flags & FLUSH_SYNC)
  1734. return 0;
  1735. } else if (atomic_long_read(&nfsi->commit_info.ncommit))
  1736. goto out_mark_dirty;
  1737. check_requests_outstanding:
  1738. if (!atomic_read(&nfsi->commit_info.rpcs_out))
  1739. return ret;
  1740. out_mark_dirty:
  1741. __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
  1742. return ret;
  1743. }
  1744. EXPORT_SYMBOL_GPL(nfs_write_inode);
  1745. /*
  1746. * Wrapper for filemap_write_and_wait_range()
  1747. *
  1748. * Needed for pNFS in order to ensure data becomes visible to the
  1749. * client.
  1750. */
  1751. int nfs_filemap_write_and_wait_range(struct address_space *mapping,
  1752. loff_t lstart, loff_t lend)
  1753. {
  1754. int ret;
  1755. ret = filemap_write_and_wait_range(mapping, lstart, lend);
  1756. if (ret == 0)
  1757. ret = pnfs_sync_inode(mapping->host, true);
  1758. return ret;
  1759. }
  1760. EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);
  1761. /*
  1762. * flush the inode to disk.
  1763. */
  1764. int nfs_wb_all(struct inode *inode)
  1765. {
  1766. int ret;
  1767. trace_nfs_writeback_inode_enter(inode);
  1768. ret = filemap_write_and_wait(inode->i_mapping);
  1769. if (ret)
  1770. goto out;
  1771. ret = nfs_commit_inode(inode, FLUSH_SYNC);
  1772. if (ret < 0)
  1773. goto out;
  1774. pnfs_sync_inode(inode, true);
  1775. ret = 0;
  1776. out:
  1777. trace_nfs_writeback_inode_exit(inode, ret);
  1778. return ret;
  1779. }
  1780. EXPORT_SYMBOL_GPL(nfs_wb_all);
  1781. int nfs_wb_page_cancel(struct inode *inode, struct page *page)
  1782. {
  1783. struct nfs_page *req;
  1784. int ret = 0;
  1785. wait_on_page_writeback(page);
  1786. /* blocking call to cancel all requests and join to a single (head)
  1787. * request */
  1788. req = nfs_lock_and_join_requests(page);
  1789. if (IS_ERR(req)) {
  1790. ret = PTR_ERR(req);
  1791. } else if (req) {
  1792. /* all requests from this page have been cancelled by
  1793. * nfs_lock_and_join_requests, so just remove the head
  1794. * request from the inode / page_private pointer and
  1795. * release it */
  1796. nfs_inode_remove_request(req);
  1797. nfs_unlock_and_release_request(req);
  1798. }
  1799. return ret;
  1800. }
  1801. /*
  1802. * Write back all requests on one page - we do this before reading it.
  1803. */
  1804. int nfs_wb_page(struct inode *inode, struct page *page)
  1805. {
  1806. loff_t range_start = page_file_offset(page);
  1807. loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
  1808. struct writeback_control wbc = {
  1809. .sync_mode = WB_SYNC_ALL,
  1810. .nr_to_write = 0,
  1811. .range_start = range_start,
  1812. .range_end = range_end,
  1813. };
  1814. int ret;
  1815. trace_nfs_writeback_page_enter(inode);
  1816. for (;;) {
  1817. wait_on_page_writeback(page);
  1818. if (clear_page_dirty_for_io(page)) {
  1819. ret = nfs_writepage_locked(page, &wbc);
  1820. if (ret < 0)
  1821. goto out_error;
  1822. continue;
  1823. }
  1824. ret = 0;
  1825. if (!PagePrivate(page))
  1826. break;
  1827. ret = nfs_commit_inode(inode, FLUSH_SYNC);
  1828. if (ret < 0)
  1829. goto out_error;
  1830. }
  1831. out_error:
  1832. trace_nfs_writeback_page_exit(inode, ret);
  1833. return ret;
  1834. }
  1835. #ifdef CONFIG_MIGRATION
  1836. int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
  1837. struct page *page, enum migrate_mode mode)
  1838. {
  1839. /*
  1840. * If PagePrivate is set, then the page is currently associated with
  1841. * an in-progress read or write request. Don't try to migrate it.
  1842. *
  1843. * FIXME: we could do this in principle, but we'll need a way to ensure
  1844. * that we can safely release the inode reference while holding
  1845. * the page lock.
  1846. */
  1847. if (PagePrivate(page))
  1848. return -EBUSY;
  1849. if (!nfs_fscache_release_page(page, GFP_KERNEL))
  1850. return -EBUSY;
  1851. return migrate_page(mapping, newpage, page, mode);
  1852. }
  1853. #endif
  1854. int __init nfs_init_writepagecache(void)
  1855. {
  1856. nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
  1857. sizeof(struct nfs_pgio_header),
  1858. 0, SLAB_HWCACHE_ALIGN,
  1859. NULL);
  1860. if (nfs_wdata_cachep == NULL)
  1861. return -ENOMEM;
  1862. nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
  1863. nfs_wdata_cachep);
  1864. if (nfs_wdata_mempool == NULL)
  1865. goto out_destroy_write_cache;
  1866. nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
  1867. sizeof(struct nfs_commit_data),
  1868. 0, SLAB_HWCACHE_ALIGN,
  1869. NULL);
  1870. if (nfs_cdata_cachep == NULL)
  1871. goto out_destroy_write_mempool;
  1872. nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
  1873. nfs_cdata_cachep);
  1874. if (nfs_commit_mempool == NULL)
  1875. goto out_destroy_commit_cache;
  1876. /*
  1877. * NFS congestion size, scale with available memory.
  1878. *
  1879. * 64MB: 8192k
  1880. * 128MB: 11585k
  1881. * 256MB: 16384k
  1882. * 512MB: 23170k
  1883. * 1GB: 32768k
  1884. * 2GB: 46340k
  1885. * 4GB: 65536k
  1886. * 8GB: 92681k
  1887. * 16GB: 131072k
  1888. *
  1889. * This allows larger machines to have larger/more transfers.
  1890. * Limit the default to 256M
  1891. */
  1892. nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
  1893. if (nfs_congestion_kb > 256*1024)
  1894. nfs_congestion_kb = 256*1024;
  1895. return 0;
  1896. out_destroy_commit_cache:
  1897. kmem_cache_destroy(nfs_cdata_cachep);
  1898. out_destroy_write_mempool:
  1899. mempool_destroy(nfs_wdata_mempool);
  1900. out_destroy_write_cache:
  1901. kmem_cache_destroy(nfs_wdata_cachep);
  1902. return -ENOMEM;
  1903. }
  1904. void nfs_destroy_writepagecache(void)
  1905. {
  1906. mempool_destroy(nfs_commit_mempool);
  1907. kmem_cache_destroy(nfs_cdata_cachep);
  1908. mempool_destroy(nfs_wdata_mempool);
  1909. kmem_cache_destroy(nfs_wdata_cachep);
  1910. }
  1911. static const struct nfs_rw_ops nfs_rw_write_ops = {
  1912. .rw_alloc_header = nfs_writehdr_alloc,
  1913. .rw_free_header = nfs_writehdr_free,
  1914. .rw_done = nfs_writeback_done,
  1915. .rw_result = nfs_writeback_result,
  1916. .rw_initiate = nfs_initiate_write,
  1917. };