pagelist.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/nfs/pagelist.c
  4. *
  5. * A set of helper functions for managing NFS read and write requests.
  6. * The main purpose of these routines is to provide support for the
  7. * coalescing of several requests into a single RPC call.
  8. *
  9. * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
  10. *
  11. */
  12. #include <linux/slab.h>
  13. #include <linux/file.h>
  14. #include <linux/sched.h>
  15. #include <linux/sunrpc/clnt.h>
  16. #include <linux/nfs.h>
  17. #include <linux/nfs3.h>
  18. #include <linux/nfs4.h>
  19. #include <linux/nfs_fs.h>
  20. #include <linux/nfs_page.h>
  21. #include <linux/nfs_mount.h>
  22. #include <linux/export.h>
  23. #include "internal.h"
  24. #include "pnfs.h"
  25. #include "nfstrace.h"
  26. #define NFSDBG_FACILITY NFSDBG_PAGECACHE
  27. static struct kmem_cache *nfs_page_cachep;
  28. static const struct rpc_call_ops nfs_pgio_common_ops;
  29. static struct nfs_pgio_mirror *
  30. nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
  31. {
  32. if (desc->pg_ops->pg_get_mirror)
  33. return desc->pg_ops->pg_get_mirror(desc, idx);
  34. return &desc->pg_mirrors[0];
  35. }
  36. struct nfs_pgio_mirror *
  37. nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)
  38. {
  39. return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);
  40. }
  41. EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror);
  42. static u32
  43. nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx)
  44. {
  45. if (desc->pg_ops->pg_set_mirror)
  46. return desc->pg_ops->pg_set_mirror(desc, idx);
  47. return desc->pg_mirror_idx;
  48. }
  49. void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
  50. struct nfs_pgio_header *hdr,
  51. void (*release)(struct nfs_pgio_header *hdr))
  52. {
  53. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  54. hdr->req = nfs_list_entry(mirror->pg_list.next);
  55. hdr->inode = desc->pg_inode;
  56. hdr->cred = nfs_req_openctx(hdr->req)->cred;
  57. hdr->io_start = req_offset(hdr->req);
  58. hdr->good_bytes = mirror->pg_count;
  59. hdr->io_completion = desc->pg_io_completion;
  60. hdr->dreq = desc->pg_dreq;
  61. hdr->release = release;
  62. hdr->completion_ops = desc->pg_completion_ops;
  63. if (hdr->completion_ops->init_hdr)
  64. hdr->completion_ops->init_hdr(hdr);
  65. hdr->pgio_mirror_idx = desc->pg_mirror_idx;
  66. }
  67. EXPORT_SYMBOL_GPL(nfs_pgheader_init);
  68. void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
  69. {
  70. unsigned int new = pos - hdr->io_start;
  71. trace_nfs_pgio_error(hdr, error, pos);
  72. if (hdr->good_bytes > new) {
  73. hdr->good_bytes = new;
  74. clear_bit(NFS_IOHDR_EOF, &hdr->flags);
  75. if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags))
  76. hdr->error = error;
  77. }
  78. }
  79. static inline struct nfs_page *nfs_page_alloc(void)
  80. {
  81. struct nfs_page *p =
  82. kmem_cache_zalloc(nfs_page_cachep, nfs_io_gfp_mask());
  83. if (p)
  84. INIT_LIST_HEAD(&p->wb_list);
  85. return p;
  86. }
  87. static inline void
  88. nfs_page_free(struct nfs_page *p)
  89. {
  90. kmem_cache_free(nfs_page_cachep, p);
  91. }
  92. /**
  93. * nfs_iocounter_wait - wait for i/o to complete
  94. * @l_ctx: nfs_lock_context with io_counter to use
  95. *
  96. * returns -ERESTARTSYS if interrupted by a fatal signal.
  97. * Otherwise returns 0 once the io_count hits 0.
  98. */
  99. int
  100. nfs_iocounter_wait(struct nfs_lock_context *l_ctx)
  101. {
  102. return wait_var_event_killable(&l_ctx->io_count,
  103. !atomic_read(&l_ctx->io_count));
  104. }
  105. /**
  106. * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
  107. * to complete
  108. * @task: the rpc_task that should wait
  109. * @l_ctx: nfs_lock_context with io_counter to check
  110. *
  111. * Returns true if there is outstanding I/O to wait on and the
  112. * task has been put to sleep.
  113. */
  114. bool
  115. nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx)
  116. {
  117. struct inode *inode = d_inode(l_ctx->open_context->dentry);
  118. bool ret = false;
  119. if (atomic_read(&l_ctx->io_count) > 0) {
  120. rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL);
  121. ret = true;
  122. }
  123. if (atomic_read(&l_ctx->io_count) == 0) {
  124. rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task);
  125. ret = false;
  126. }
  127. return ret;
  128. }
  129. EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait);
  130. /*
  131. * nfs_page_lock_head_request - page lock the head of the page group
  132. * @req: any member of the page group
  133. */
  134. struct nfs_page *
  135. nfs_page_group_lock_head(struct nfs_page *req)
  136. {
  137. struct nfs_page *head = req->wb_head;
  138. while (!nfs_lock_request(head)) {
  139. int ret = nfs_wait_on_request(head);
  140. if (ret < 0)
  141. return ERR_PTR(ret);
  142. }
  143. if (head != req)
  144. kref_get(&head->wb_kref);
  145. return head;
  146. }
  147. /*
  148. * nfs_unroll_locks - unlock all newly locked reqs and wait on @req
  149. * @head: head request of page group, must be holding head lock
  150. * @req: request that couldn't lock and needs to wait on the req bit lock
  151. *
  152. * This is a helper function for nfs_lock_and_join_requests
  153. * returns 0 on success, < 0 on error.
  154. */
  155. static void
  156. nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req)
  157. {
  158. struct nfs_page *tmp;
  159. /* relinquish all the locks successfully grabbed this run */
  160. for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
  161. if (!kref_read(&tmp->wb_kref))
  162. continue;
  163. nfs_unlock_and_release_request(tmp);
  164. }
  165. }
  166. /*
  167. * nfs_page_group_lock_subreq - try to lock a subrequest
  168. * @head: head request of page group
  169. * @subreq: request to lock
  170. *
  171. * This is a helper function for nfs_lock_and_join_requests which
  172. * must be called with the head request and page group both locked.
  173. * On error, it returns with the page group unlocked.
  174. */
  175. static int
  176. nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq)
  177. {
  178. int ret;
  179. if (!kref_get_unless_zero(&subreq->wb_kref))
  180. return 0;
  181. while (!nfs_lock_request(subreq)) {
  182. nfs_page_group_unlock(head);
  183. ret = nfs_wait_on_request(subreq);
  184. if (!ret)
  185. ret = nfs_page_group_lock(head);
  186. if (ret < 0) {
  187. nfs_unroll_locks(head, subreq);
  188. nfs_release_request(subreq);
  189. return ret;
  190. }
  191. }
  192. return 0;
  193. }
  194. /*
  195. * nfs_page_group_lock_subrequests - try to lock the subrequests
  196. * @head: head request of page group
  197. *
  198. * This is a helper function for nfs_lock_and_join_requests which
  199. * must be called with the head request locked.
  200. */
  201. int nfs_page_group_lock_subrequests(struct nfs_page *head)
  202. {
  203. struct nfs_page *subreq;
  204. int ret;
  205. ret = nfs_page_group_lock(head);
  206. if (ret < 0)
  207. return ret;
  208. /* lock each request in the page group */
  209. for (subreq = head->wb_this_page; subreq != head;
  210. subreq = subreq->wb_this_page) {
  211. ret = nfs_page_group_lock_subreq(head, subreq);
  212. if (ret < 0)
  213. return ret;
  214. }
  215. nfs_page_group_unlock(head);
  216. return 0;
  217. }
  218. /*
  219. * nfs_page_set_headlock - set the request PG_HEADLOCK
  220. * @req: request that is to be locked
  221. *
  222. * this lock must be held when modifying req->wb_head
  223. *
  224. * return 0 on success, < 0 on error
  225. */
  226. int
  227. nfs_page_set_headlock(struct nfs_page *req)
  228. {
  229. if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags))
  230. return 0;
  231. set_bit(PG_CONTENDED1, &req->wb_flags);
  232. smp_mb__after_atomic();
  233. return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK,
  234. TASK_UNINTERRUPTIBLE);
  235. }
  236. /*
  237. * nfs_page_clear_headlock - clear the request PG_HEADLOCK
  238. * @req: request that is to be locked
  239. */
  240. void
  241. nfs_page_clear_headlock(struct nfs_page *req)
  242. {
  243. smp_mb__before_atomic();
  244. clear_bit(PG_HEADLOCK, &req->wb_flags);
  245. smp_mb__after_atomic();
  246. if (!test_bit(PG_CONTENDED1, &req->wb_flags))
  247. return;
  248. wake_up_bit(&req->wb_flags, PG_HEADLOCK);
  249. }
  250. /*
  251. * nfs_page_group_lock - lock the head of the page group
  252. * @req: request in group that is to be locked
  253. *
  254. * this lock must be held when traversing or modifying the page
  255. * group list
  256. *
  257. * return 0 on success, < 0 on error
  258. */
  259. int
  260. nfs_page_group_lock(struct nfs_page *req)
  261. {
  262. int ret;
  263. ret = nfs_page_set_headlock(req);
  264. if (ret || req->wb_head == req)
  265. return ret;
  266. return nfs_page_set_headlock(req->wb_head);
  267. }
  268. /*
  269. * nfs_page_group_unlock - unlock the head of the page group
  270. * @req: request in group that is to be unlocked
  271. */
  272. void
  273. nfs_page_group_unlock(struct nfs_page *req)
  274. {
  275. if (req != req->wb_head)
  276. nfs_page_clear_headlock(req->wb_head);
  277. nfs_page_clear_headlock(req);
  278. }
  279. /*
  280. * nfs_page_group_sync_on_bit_locked
  281. *
  282. * must be called with page group lock held
  283. */
  284. static bool
  285. nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)
  286. {
  287. struct nfs_page *head = req->wb_head;
  288. struct nfs_page *tmp;
  289. WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));
  290. WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));
  291. tmp = req->wb_this_page;
  292. while (tmp != req) {
  293. if (!test_bit(bit, &tmp->wb_flags))
  294. return false;
  295. tmp = tmp->wb_this_page;
  296. }
  297. /* true! reset all bits */
  298. tmp = req;
  299. do {
  300. clear_bit(bit, &tmp->wb_flags);
  301. tmp = tmp->wb_this_page;
  302. } while (tmp != req);
  303. return true;
  304. }
  305. /*
  306. * nfs_page_group_sync_on_bit - set bit on current request, but only
  307. * return true if the bit is set for all requests in page group
  308. * @req - request in page group
  309. * @bit - PG_* bit that is used to sync page group
  310. */
  311. bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)
  312. {
  313. bool ret;
  314. nfs_page_group_lock(req);
  315. ret = nfs_page_group_sync_on_bit_locked(req, bit);
  316. nfs_page_group_unlock(req);
  317. return ret;
  318. }
  319. /*
  320. * nfs_page_group_init - Initialize the page group linkage for @req
  321. * @req - a new nfs request
  322. * @prev - the previous request in page group, or NULL if @req is the first
  323. * or only request in the group (the head).
  324. */
  325. static inline void
  326. nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)
  327. {
  328. struct inode *inode;
  329. WARN_ON_ONCE(prev == req);
  330. if (!prev) {
  331. /* a head request */
  332. req->wb_head = req;
  333. req->wb_this_page = req;
  334. } else {
  335. /* a subrequest */
  336. WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);
  337. WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));
  338. req->wb_head = prev->wb_head;
  339. req->wb_this_page = prev->wb_this_page;
  340. prev->wb_this_page = req;
  341. /* All subrequests take a ref on the head request until
  342. * nfs_page_group_destroy is called */
  343. kref_get(&req->wb_head->wb_kref);
  344. /* grab extra ref and bump the request count if head request
  345. * has extra ref from the write/commit path to handle handoff
  346. * between write and commit lists. */
  347. if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {
  348. inode = page_file_mapping(req->wb_page)->host;
  349. set_bit(PG_INODE_REF, &req->wb_flags);
  350. kref_get(&req->wb_kref);
  351. atomic_long_inc(&NFS_I(inode)->nrequests);
  352. }
  353. }
  354. }
  355. /*
  356. * nfs_page_group_destroy - sync the destruction of page groups
  357. * @req - request that no longer needs the page group
  358. *
  359. * releases the page group reference from each member once all
  360. * members have called this function.
  361. */
  362. static void
  363. nfs_page_group_destroy(struct kref *kref)
  364. {
  365. struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
  366. struct nfs_page *head = req->wb_head;
  367. struct nfs_page *tmp, *next;
  368. if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN))
  369. goto out;
  370. tmp = req;
  371. do {
  372. next = tmp->wb_this_page;
  373. /* unlink and free */
  374. tmp->wb_this_page = tmp;
  375. tmp->wb_head = tmp;
  376. nfs_free_request(tmp);
  377. tmp = next;
  378. } while (tmp != req);
  379. out:
  380. /* subrequests must release the ref on the head request */
  381. if (head != req)
  382. nfs_release_request(head);
  383. }
  384. static struct nfs_page *
  385. __nfs_create_request(struct nfs_lock_context *l_ctx, struct page *page,
  386. unsigned int pgbase, unsigned int offset,
  387. unsigned int count)
  388. {
  389. struct nfs_page *req;
  390. struct nfs_open_context *ctx = l_ctx->open_context;
  391. if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
  392. return ERR_PTR(-EBADF);
  393. /* try to allocate the request struct */
  394. req = nfs_page_alloc();
  395. if (req == NULL)
  396. return ERR_PTR(-ENOMEM);
  397. req->wb_lock_context = l_ctx;
  398. refcount_inc(&l_ctx->count);
  399. atomic_inc(&l_ctx->io_count);
  400. /* Initialize the request struct. Initially, we assume a
  401. * long write-back delay. This will be adjusted in
  402. * update_nfs_request below if the region is not locked. */
  403. req->wb_page = page;
  404. if (page) {
  405. req->wb_index = page_index(page);
  406. get_page(page);
  407. }
  408. req->wb_offset = offset;
  409. req->wb_pgbase = pgbase;
  410. req->wb_bytes = count;
  411. kref_init(&req->wb_kref);
  412. req->wb_nio = 0;
  413. return req;
  414. }
  415. /**
  416. * nfs_create_request - Create an NFS read/write request.
  417. * @ctx: open context to use
  418. * @page: page to write
  419. * @offset: starting offset within the page for the write
  420. * @count: number of bytes to read/write
  421. *
  422. * The page must be locked by the caller. This makes sure we never
  423. * create two different requests for the same page.
  424. * User should ensure it is safe to sleep in this function.
  425. */
  426. struct nfs_page *
  427. nfs_create_request(struct nfs_open_context *ctx, struct page *page,
  428. unsigned int offset, unsigned int count)
  429. {
  430. struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx);
  431. struct nfs_page *ret;
  432. if (IS_ERR(l_ctx))
  433. return ERR_CAST(l_ctx);
  434. ret = __nfs_create_request(l_ctx, page, offset, offset, count);
  435. if (!IS_ERR(ret))
  436. nfs_page_group_init(ret, NULL);
  437. nfs_put_lock_context(l_ctx);
  438. return ret;
  439. }
  440. static struct nfs_page *
  441. nfs_create_subreq(struct nfs_page *req,
  442. unsigned int pgbase,
  443. unsigned int offset,
  444. unsigned int count)
  445. {
  446. struct nfs_page *last;
  447. struct nfs_page *ret;
  448. ret = __nfs_create_request(req->wb_lock_context, req->wb_page,
  449. pgbase, offset, count);
  450. if (!IS_ERR(ret)) {
  451. /* find the last request */
  452. for (last = req->wb_head;
  453. last->wb_this_page != req->wb_head;
  454. last = last->wb_this_page)
  455. ;
  456. nfs_lock_request(ret);
  457. ret->wb_index = req->wb_index;
  458. nfs_page_group_init(ret, last);
  459. ret->wb_nio = req->wb_nio;
  460. }
  461. return ret;
  462. }
  463. /**
  464. * nfs_unlock_request - Unlock request and wake up sleepers.
  465. * @req: pointer to request
  466. */
  467. void nfs_unlock_request(struct nfs_page *req)
  468. {
  469. if (!NFS_WBACK_BUSY(req)) {
  470. printk(KERN_ERR "NFS: Invalid unlock attempted\n");
  471. BUG();
  472. }
  473. smp_mb__before_atomic();
  474. clear_bit(PG_BUSY, &req->wb_flags);
  475. smp_mb__after_atomic();
  476. if (!test_bit(PG_CONTENDED2, &req->wb_flags))
  477. return;
  478. wake_up_bit(&req->wb_flags, PG_BUSY);
  479. }
  480. /**
  481. * nfs_unlock_and_release_request - Unlock request and release the nfs_page
  482. * @req: pointer to request
  483. */
  484. void nfs_unlock_and_release_request(struct nfs_page *req)
  485. {
  486. nfs_unlock_request(req);
  487. nfs_release_request(req);
  488. }
  489. /*
  490. * nfs_clear_request - Free up all resources allocated to the request
  491. * @req:
  492. *
  493. * Release page and open context resources associated with a read/write
  494. * request after it has completed.
  495. */
  496. static void nfs_clear_request(struct nfs_page *req)
  497. {
  498. struct page *page = req->wb_page;
  499. struct nfs_lock_context *l_ctx = req->wb_lock_context;
  500. struct nfs_open_context *ctx;
  501. if (page != NULL) {
  502. put_page(page);
  503. req->wb_page = NULL;
  504. }
  505. if (l_ctx != NULL) {
  506. if (atomic_dec_and_test(&l_ctx->io_count)) {
  507. wake_up_var(&l_ctx->io_count);
  508. ctx = l_ctx->open_context;
  509. if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))
  510. rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);
  511. }
  512. nfs_put_lock_context(l_ctx);
  513. req->wb_lock_context = NULL;
  514. }
  515. }
  516. /**
  517. * nfs_release_request - Release the count on an NFS read/write request
  518. * @req: request to release
  519. *
  520. * Note: Should never be called with the spinlock held!
  521. */
  522. void nfs_free_request(struct nfs_page *req)
  523. {
  524. WARN_ON_ONCE(req->wb_this_page != req);
  525. /* extra debug: make sure no sync bits are still set */
  526. WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
  527. WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));
  528. WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));
  529. WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));
  530. WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));
  531. /* Release struct file and open context */
  532. nfs_clear_request(req);
  533. nfs_page_free(req);
  534. }
  535. void nfs_release_request(struct nfs_page *req)
  536. {
  537. kref_put(&req->wb_kref, nfs_page_group_destroy);
  538. }
  539. EXPORT_SYMBOL_GPL(nfs_release_request);
  540. /**
  541. * nfs_wait_on_request - Wait for a request to complete.
  542. * @req: request to wait upon.
  543. *
  544. * Interruptible by fatal signals only.
  545. * The user is responsible for holding a count on the request.
  546. */
  547. int
  548. nfs_wait_on_request(struct nfs_page *req)
  549. {
  550. if (!test_bit(PG_BUSY, &req->wb_flags))
  551. return 0;
  552. set_bit(PG_CONTENDED2, &req->wb_flags);
  553. smp_mb__after_atomic();
  554. return wait_on_bit_io(&req->wb_flags, PG_BUSY,
  555. TASK_UNINTERRUPTIBLE);
  556. }
  557. EXPORT_SYMBOL_GPL(nfs_wait_on_request);
  558. /*
  559. * nfs_generic_pg_test - determine if requests can be coalesced
  560. * @desc: pointer to descriptor
  561. * @prev: previous request in desc, or NULL
  562. * @req: this request
  563. *
  564. * Returns zero if @req cannot be coalesced into @desc, otherwise it returns
  565. * the size of the request.
  566. */
  567. size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
  568. struct nfs_page *prev, struct nfs_page *req)
  569. {
  570. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  571. if (mirror->pg_count > mirror->pg_bsize) {
  572. /* should never happen */
  573. WARN_ON_ONCE(1);
  574. return 0;
  575. }
  576. /*
  577. * Limit the request size so that we can still allocate a page array
  578. * for it without upsetting the slab allocator.
  579. */
  580. if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
  581. sizeof(struct page *) > PAGE_SIZE)
  582. return 0;
  583. return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
  584. }
  585. EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
  586. struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops)
  587. {
  588. struct nfs_pgio_header *hdr = ops->rw_alloc_header();
  589. if (hdr) {
  590. INIT_LIST_HEAD(&hdr->pages);
  591. hdr->rw_ops = ops;
  592. }
  593. return hdr;
  594. }
  595. EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc);
  596. /**
  597. * nfs_pgio_data_destroy - make @hdr suitable for reuse
  598. *
  599. * Frees memory and releases refs from nfs_generic_pgio, so that it may
  600. * be called again.
  601. *
  602. * @hdr: A header that has had nfs_generic_pgio called
  603. */
  604. static void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr)
  605. {
  606. if (hdr->args.context)
  607. put_nfs_open_context(hdr->args.context);
  608. if (hdr->page_array.pagevec != hdr->page_array.page_array)
  609. kfree(hdr->page_array.pagevec);
  610. }
  611. /*
  612. * nfs_pgio_header_free - Free a read or write header
  613. * @hdr: The header to free
  614. */
  615. void nfs_pgio_header_free(struct nfs_pgio_header *hdr)
  616. {
  617. nfs_pgio_data_destroy(hdr);
  618. hdr->rw_ops->rw_free_header(hdr);
  619. }
  620. EXPORT_SYMBOL_GPL(nfs_pgio_header_free);
  621. /**
  622. * nfs_pgio_rpcsetup - Set up arguments for a pageio call
  623. * @hdr: The pageio hdr
  624. * @count: Number of bytes to read
  625. * @how: How to commit data (writes only)
  626. * @cinfo: Commit information for the call (writes only)
  627. */
  628. static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr,
  629. unsigned int count,
  630. int how, struct nfs_commit_info *cinfo)
  631. {
  632. struct nfs_page *req = hdr->req;
  633. /* Set up the RPC argument and reply structs
  634. * NB: take care not to mess about with hdr->commit et al. */
  635. hdr->args.fh = NFS_FH(hdr->inode);
  636. hdr->args.offset = req_offset(req);
  637. /* pnfs_set_layoutcommit needs this */
  638. hdr->mds_offset = hdr->args.offset;
  639. hdr->args.pgbase = req->wb_pgbase;
  640. hdr->args.pages = hdr->page_array.pagevec;
  641. hdr->args.count = count;
  642. hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));
  643. hdr->args.lock_context = req->wb_lock_context;
  644. hdr->args.stable = NFS_UNSTABLE;
  645. switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
  646. case 0:
  647. break;
  648. case FLUSH_COND_STABLE:
  649. if (nfs_reqs_to_commit(cinfo))
  650. break;
  651. fallthrough;
  652. default:
  653. hdr->args.stable = NFS_FILE_SYNC;
  654. }
  655. hdr->res.fattr = &hdr->fattr;
  656. hdr->res.count = 0;
  657. hdr->res.eof = 0;
  658. hdr->res.verf = &hdr->verf;
  659. nfs_fattr_init(&hdr->fattr);
  660. }
  661. /**
  662. * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
  663. * @task: The current task
  664. * @calldata: pageio header to prepare
  665. */
  666. static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
  667. {
  668. struct nfs_pgio_header *hdr = calldata;
  669. int err;
  670. err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);
  671. if (err)
  672. rpc_exit(task, err);
  673. }
  674. int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
  675. const struct cred *cred, const struct nfs_rpc_ops *rpc_ops,
  676. const struct rpc_call_ops *call_ops, int how, int flags)
  677. {
  678. struct rpc_task *task;
  679. struct rpc_message msg = {
  680. .rpc_argp = &hdr->args,
  681. .rpc_resp = &hdr->res,
  682. .rpc_cred = cred,
  683. };
  684. struct rpc_task_setup task_setup_data = {
  685. .rpc_client = clnt,
  686. .task = &hdr->task,
  687. .rpc_message = &msg,
  688. .callback_ops = call_ops,
  689. .callback_data = hdr,
  690. .workqueue = nfsiod_workqueue,
  691. .flags = RPC_TASK_ASYNC | flags,
  692. };
  693. hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
  694. dprintk("NFS: initiated pgio call "
  695. "(req %s/%llu, %u bytes @ offset %llu)\n",
  696. hdr->inode->i_sb->s_id,
  697. (unsigned long long)NFS_FILEID(hdr->inode),
  698. hdr->args.count,
  699. (unsigned long long)hdr->args.offset);
  700. task = rpc_run_task(&task_setup_data);
  701. if (IS_ERR(task))
  702. return PTR_ERR(task);
  703. rpc_put_task(task);
  704. return 0;
  705. }
  706. EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
  707. /**
  708. * nfs_pgio_error - Clean up from a pageio error
  709. * @hdr: pageio header
  710. */
  711. static void nfs_pgio_error(struct nfs_pgio_header *hdr)
  712. {
  713. set_bit(NFS_IOHDR_REDO, &hdr->flags);
  714. hdr->completion_ops->completion(hdr);
  715. }
  716. /**
  717. * nfs_pgio_release - Release pageio data
  718. * @calldata: The pageio header to release
  719. */
  720. static void nfs_pgio_release(void *calldata)
  721. {
  722. struct nfs_pgio_header *hdr = calldata;
  723. hdr->completion_ops->completion(hdr);
  724. }
  725. static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
  726. unsigned int bsize)
  727. {
  728. INIT_LIST_HEAD(&mirror->pg_list);
  729. mirror->pg_bytes_written = 0;
  730. mirror->pg_count = 0;
  731. mirror->pg_bsize = bsize;
  732. mirror->pg_base = 0;
  733. mirror->pg_recoalesce = 0;
  734. }
  735. /**
  736. * nfs_pageio_init - initialise a page io descriptor
  737. * @desc: pointer to descriptor
  738. * @inode: pointer to inode
  739. * @pg_ops: pointer to pageio operations
  740. * @compl_ops: pointer to pageio completion operations
  741. * @rw_ops: pointer to nfs read/write operations
  742. * @bsize: io block size
  743. * @io_flags: extra parameters for the io function
  744. */
  745. void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
  746. struct inode *inode,
  747. const struct nfs_pageio_ops *pg_ops,
  748. const struct nfs_pgio_completion_ops *compl_ops,
  749. const struct nfs_rw_ops *rw_ops,
  750. size_t bsize,
  751. int io_flags)
  752. {
  753. desc->pg_moreio = 0;
  754. desc->pg_inode = inode;
  755. desc->pg_ops = pg_ops;
  756. desc->pg_completion_ops = compl_ops;
  757. desc->pg_rw_ops = rw_ops;
  758. desc->pg_ioflags = io_flags;
  759. desc->pg_error = 0;
  760. desc->pg_lseg = NULL;
  761. desc->pg_io_completion = NULL;
  762. desc->pg_dreq = NULL;
  763. desc->pg_bsize = bsize;
  764. desc->pg_mirror_count = 1;
  765. desc->pg_mirror_idx = 0;
  766. desc->pg_mirrors_dynamic = NULL;
  767. desc->pg_mirrors = desc->pg_mirrors_static;
  768. nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
  769. desc->pg_maxretrans = 0;
  770. }
  771. /**
  772. * nfs_pgio_result - Basic pageio error handling
  773. * @task: The task that ran
  774. * @calldata: Pageio header to check
  775. */
  776. static void nfs_pgio_result(struct rpc_task *task, void *calldata)
  777. {
  778. struct nfs_pgio_header *hdr = calldata;
  779. struct inode *inode = hdr->inode;
  780. dprintk("NFS: %s: %5u, (status %d)\n", __func__,
  781. task->tk_pid, task->tk_status);
  782. if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)
  783. return;
  784. if (task->tk_status < 0)
  785. nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);
  786. else
  787. hdr->rw_ops->rw_result(task, hdr);
  788. }
  789. /*
  790. * Create an RPC task for the given read or write request and kick it.
  791. * The page must have been locked by the caller.
  792. *
  793. * It may happen that the page we're passed is not marked dirty.
  794. * This is the case if nfs_updatepage detects a conflicting request
  795. * that has been written but not committed.
  796. */
  797. int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
  798. struct nfs_pgio_header *hdr)
  799. {
  800. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  801. struct nfs_page *req;
  802. struct page **pages,
  803. *last_page;
  804. struct list_head *head = &mirror->pg_list;
  805. struct nfs_commit_info cinfo;
  806. struct nfs_page_array *pg_array = &hdr->page_array;
  807. unsigned int pagecount, pageused;
  808. gfp_t gfp_flags = nfs_io_gfp_mask();
  809. pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
  810. pg_array->npages = pagecount;
  811. if (pagecount <= ARRAY_SIZE(pg_array->page_array))
  812. pg_array->pagevec = pg_array->page_array;
  813. else {
  814. pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags);
  815. if (!pg_array->pagevec) {
  816. pg_array->npages = 0;
  817. nfs_pgio_error(hdr);
  818. desc->pg_error = -ENOMEM;
  819. return desc->pg_error;
  820. }
  821. }
  822. nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
  823. pages = hdr->page_array.pagevec;
  824. last_page = NULL;
  825. pageused = 0;
  826. while (!list_empty(head)) {
  827. req = nfs_list_entry(head->next);
  828. nfs_list_move_request(req, &hdr->pages);
  829. if (!last_page || last_page != req->wb_page) {
  830. pageused++;
  831. if (pageused > pagecount)
  832. break;
  833. *pages++ = last_page = req->wb_page;
  834. }
  835. }
  836. if (WARN_ON_ONCE(pageused != pagecount)) {
  837. nfs_pgio_error(hdr);
  838. desc->pg_error = -EINVAL;
  839. return desc->pg_error;
  840. }
  841. if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
  842. (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
  843. desc->pg_ioflags &= ~FLUSH_COND_STABLE;
  844. /* Set up the argument struct */
  845. nfs_pgio_rpcsetup(hdr, mirror->pg_count, desc->pg_ioflags, &cinfo);
  846. desc->pg_rpc_callops = &nfs_pgio_common_ops;
  847. return 0;
  848. }
  849. EXPORT_SYMBOL_GPL(nfs_generic_pgio);
  850. static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
  851. {
  852. struct nfs_pgio_header *hdr;
  853. int ret;
  854. hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
  855. if (!hdr) {
  856. desc->pg_error = -ENOMEM;
  857. return desc->pg_error;
  858. }
  859. nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
  860. ret = nfs_generic_pgio(desc, hdr);
  861. if (ret == 0)
  862. ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),
  863. hdr,
  864. hdr->cred,
  865. NFS_PROTO(hdr->inode),
  866. desc->pg_rpc_callops,
  867. desc->pg_ioflags,
  868. RPC_TASK_CRED_NOREF);
  869. return ret;
  870. }
  871. static struct nfs_pgio_mirror *
  872. nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc,
  873. unsigned int mirror_count)
  874. {
  875. struct nfs_pgio_mirror *ret;
  876. unsigned int i;
  877. kfree(desc->pg_mirrors_dynamic);
  878. desc->pg_mirrors_dynamic = NULL;
  879. if (mirror_count == 1)
  880. return desc->pg_mirrors_static;
  881. ret = kmalloc_array(mirror_count, sizeof(*ret), nfs_io_gfp_mask());
  882. if (ret != NULL) {
  883. for (i = 0; i < mirror_count; i++)
  884. nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);
  885. desc->pg_mirrors_dynamic = ret;
  886. }
  887. return ret;
  888. }
  889. /*
  890. * nfs_pageio_setup_mirroring - determine if mirroring is to be used
  891. * by calling the pg_get_mirror_count op
  892. */
  893. static void nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
  894. struct nfs_page *req)
  895. {
  896. unsigned int mirror_count = 1;
  897. if (pgio->pg_ops->pg_get_mirror_count)
  898. mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
  899. if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0)
  900. return;
  901. if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX) {
  902. pgio->pg_error = -EINVAL;
  903. return;
  904. }
  905. pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count);
  906. if (pgio->pg_mirrors == NULL) {
  907. pgio->pg_error = -ENOMEM;
  908. pgio->pg_mirrors = pgio->pg_mirrors_static;
  909. mirror_count = 1;
  910. }
  911. pgio->pg_mirror_count = mirror_count;
  912. }
  913. static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
  914. {
  915. pgio->pg_mirror_count = 1;
  916. pgio->pg_mirror_idx = 0;
  917. pgio->pg_mirrors = pgio->pg_mirrors_static;
  918. kfree(pgio->pg_mirrors_dynamic);
  919. pgio->pg_mirrors_dynamic = NULL;
  920. }
  921. static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
  922. const struct nfs_lock_context *l2)
  923. {
  924. return l1->lockowner == l2->lockowner;
  925. }
  926. /**
  927. * nfs_coalesce_size - test two requests for compatibility
  928. * @prev: pointer to nfs_page
  929. * @req: pointer to nfs_page
  930. * @pgio: pointer to nfs_pagio_descriptor
  931. *
  932. * The nfs_page structures 'prev' and 'req' are compared to ensure that the
  933. * page data area they describe is contiguous, and that their RPC
  934. * credentials, NFSv4 open state, and lockowners are the same.
  935. *
  936. * Returns size of the request that can be coalesced
  937. */
  938. static unsigned int nfs_coalesce_size(struct nfs_page *prev,
  939. struct nfs_page *req,
  940. struct nfs_pageio_descriptor *pgio)
  941. {
  942. struct file_lock_context *flctx;
  943. if (prev) {
  944. if (!nfs_match_open_context(nfs_req_openctx(req), nfs_req_openctx(prev)))
  945. return 0;
  946. flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx;
  947. if (flctx != NULL &&
  948. !(list_empty_careful(&flctx->flc_posix) &&
  949. list_empty_careful(&flctx->flc_flock)) &&
  950. !nfs_match_lock_context(req->wb_lock_context,
  951. prev->wb_lock_context))
  952. return 0;
  953. if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
  954. return 0;
  955. if (req->wb_page == prev->wb_page) {
  956. if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)
  957. return 0;
  958. } else {
  959. if (req->wb_pgbase != 0 ||
  960. prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE)
  961. return 0;
  962. }
  963. }
  964. return pgio->pg_ops->pg_test(pgio, prev, req);
  965. }
  966. /**
  967. * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
  968. * @desc: destination io descriptor
  969. * @req: request
  970. *
  971. * If the request 'req' was successfully coalesced into the existing list
  972. * of pages 'desc', it returns the size of req.
  973. */
  974. static unsigned int
  975. nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
  976. struct nfs_page *req)
  977. {
  978. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  979. struct nfs_page *prev = NULL;
  980. unsigned int size;
  981. if (list_empty(&mirror->pg_list)) {
  982. if (desc->pg_ops->pg_init)
  983. desc->pg_ops->pg_init(desc, req);
  984. if (desc->pg_error < 0)
  985. return 0;
  986. mirror->pg_base = req->wb_pgbase;
  987. mirror->pg_count = 0;
  988. mirror->pg_recoalesce = 0;
  989. } else
  990. prev = nfs_list_entry(mirror->pg_list.prev);
  991. if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {
  992. if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)
  993. desc->pg_error = -ETIMEDOUT;
  994. else
  995. desc->pg_error = -EIO;
  996. return 0;
  997. }
  998. size = nfs_coalesce_size(prev, req, desc);
  999. if (size < req->wb_bytes)
  1000. return size;
  1001. nfs_list_move_request(req, &mirror->pg_list);
  1002. mirror->pg_count += req->wb_bytes;
  1003. return req->wb_bytes;
  1004. }
  1005. /*
  1006. * Helper for nfs_pageio_add_request and nfs_pageio_complete
  1007. */
  1008. static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
  1009. {
  1010. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  1011. if (!list_empty(&mirror->pg_list)) {
  1012. int error = desc->pg_ops->pg_doio(desc);
  1013. if (error < 0)
  1014. desc->pg_error = error;
  1015. if (list_empty(&mirror->pg_list)) {
  1016. mirror->pg_bytes_written += mirror->pg_count;
  1017. mirror->pg_count = 0;
  1018. mirror->pg_base = 0;
  1019. mirror->pg_recoalesce = 0;
  1020. }
  1021. }
  1022. }
  1023. static void
  1024. nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc,
  1025. struct nfs_page *req)
  1026. {
  1027. LIST_HEAD(head);
  1028. nfs_list_move_request(req, &head);
  1029. desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);
  1030. }
  1031. /**
  1032. * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
  1033. * @desc: destination io descriptor
  1034. * @req: request
  1035. *
  1036. * This may split a request into subrequests which are all part of the
  1037. * same page group. If so, it will submit @req as the last one, to ensure
  1038. * the pointer to @req is still valid in case of failure.
  1039. *
  1040. * Returns true if the request 'req' was successfully coalesced into the
  1041. * existing list of pages 'desc'.
  1042. */
  1043. static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
  1044. struct nfs_page *req)
  1045. {
  1046. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  1047. struct nfs_page *subreq;
  1048. unsigned int size, subreq_size;
  1049. nfs_page_group_lock(req);
  1050. subreq = req;
  1051. subreq_size = subreq->wb_bytes;
  1052. for(;;) {
  1053. size = nfs_pageio_do_add_request(desc, subreq);
  1054. if (size == subreq_size) {
  1055. /* We successfully submitted a request */
  1056. if (subreq == req)
  1057. break;
  1058. req->wb_pgbase += size;
  1059. req->wb_bytes -= size;
  1060. req->wb_offset += size;
  1061. subreq_size = req->wb_bytes;
  1062. subreq = req;
  1063. continue;
  1064. }
  1065. if (WARN_ON_ONCE(subreq != req)) {
  1066. nfs_page_group_unlock(req);
  1067. nfs_pageio_cleanup_request(desc, subreq);
  1068. subreq = req;
  1069. subreq_size = req->wb_bytes;
  1070. nfs_page_group_lock(req);
  1071. }
  1072. if (!size) {
  1073. /* Can't coalesce any more, so do I/O */
  1074. nfs_page_group_unlock(req);
  1075. desc->pg_moreio = 1;
  1076. nfs_pageio_doio(desc);
  1077. if (desc->pg_error < 0 || mirror->pg_recoalesce)
  1078. return 0;
  1079. /* retry add_request for this subreq */
  1080. nfs_page_group_lock(req);
  1081. continue;
  1082. }
  1083. subreq = nfs_create_subreq(req, req->wb_pgbase,
  1084. req->wb_offset, size);
  1085. if (IS_ERR(subreq))
  1086. goto err_ptr;
  1087. subreq_size = size;
  1088. }
  1089. nfs_page_group_unlock(req);
  1090. return 1;
  1091. err_ptr:
  1092. desc->pg_error = PTR_ERR(subreq);
  1093. nfs_page_group_unlock(req);
  1094. return 0;
  1095. }
  1096. static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
  1097. {
  1098. struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
  1099. LIST_HEAD(head);
  1100. do {
  1101. list_splice_init(&mirror->pg_list, &head);
  1102. mirror->pg_count = 0;
  1103. mirror->pg_base = 0;
  1104. mirror->pg_recoalesce = 0;
  1105. while (!list_empty(&head)) {
  1106. struct nfs_page *req;
  1107. req = list_first_entry(&head, struct nfs_page, wb_list);
  1108. if (__nfs_pageio_add_request(desc, req))
  1109. continue;
  1110. if (desc->pg_error < 0) {
  1111. list_splice_tail(&head, &mirror->pg_list);
  1112. mirror->pg_recoalesce = 1;
  1113. return 0;
  1114. }
  1115. break;
  1116. }
  1117. } while (mirror->pg_recoalesce);
  1118. return 1;
  1119. }
  1120. static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
  1121. struct nfs_page *req)
  1122. {
  1123. int ret;
  1124. do {
  1125. ret = __nfs_pageio_add_request(desc, req);
  1126. if (ret)
  1127. break;
  1128. if (desc->pg_error < 0)
  1129. break;
  1130. ret = nfs_do_recoalesce(desc);
  1131. } while (ret);
  1132. return ret;
  1133. }
  1134. static void nfs_pageio_error_cleanup(struct nfs_pageio_descriptor *desc)
  1135. {
  1136. u32 midx;
  1137. struct nfs_pgio_mirror *mirror;
  1138. if (!desc->pg_error)
  1139. return;
  1140. for (midx = 0; midx < desc->pg_mirror_count; midx++) {
  1141. mirror = nfs_pgio_get_mirror(desc, midx);
  1142. desc->pg_completion_ops->error_cleanup(&mirror->pg_list,
  1143. desc->pg_error);
  1144. }
  1145. }
  1146. int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
  1147. struct nfs_page *req)
  1148. {
  1149. u32 midx;
  1150. unsigned int pgbase, offset, bytes;
  1151. struct nfs_page *dupreq;
  1152. pgbase = req->wb_pgbase;
  1153. offset = req->wb_offset;
  1154. bytes = req->wb_bytes;
  1155. nfs_pageio_setup_mirroring(desc, req);
  1156. if (desc->pg_error < 0)
  1157. goto out_failed;
  1158. /* Create the mirror instances first, and fire them off */
  1159. for (midx = 1; midx < desc->pg_mirror_count; midx++) {
  1160. nfs_page_group_lock(req);
  1161. dupreq = nfs_create_subreq(req,
  1162. pgbase, offset, bytes);
  1163. nfs_page_group_unlock(req);
  1164. if (IS_ERR(dupreq)) {
  1165. desc->pg_error = PTR_ERR(dupreq);
  1166. goto out_failed;
  1167. }
  1168. nfs_pgio_set_current_mirror(desc, midx);
  1169. if (!nfs_pageio_add_request_mirror(desc, dupreq))
  1170. goto out_cleanup_subreq;
  1171. }
  1172. nfs_pgio_set_current_mirror(desc, 0);
  1173. if (!nfs_pageio_add_request_mirror(desc, req))
  1174. goto out_failed;
  1175. return 1;
  1176. out_cleanup_subreq:
  1177. nfs_pageio_cleanup_request(desc, dupreq);
  1178. out_failed:
  1179. nfs_pageio_error_cleanup(desc);
  1180. return 0;
  1181. }
  1182. /*
  1183. * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
  1184. * nfs_pageio_descriptor
  1185. * @desc: pointer to io descriptor
  1186. * @mirror_idx: pointer to mirror index
  1187. */
  1188. static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
  1189. u32 mirror_idx)
  1190. {
  1191. struct nfs_pgio_mirror *mirror;
  1192. u32 restore_idx;
  1193. restore_idx = nfs_pgio_set_current_mirror(desc, mirror_idx);
  1194. mirror = nfs_pgio_current_mirror(desc);
  1195. for (;;) {
  1196. nfs_pageio_doio(desc);
  1197. if (desc->pg_error < 0 || !mirror->pg_recoalesce)
  1198. break;
  1199. if (!nfs_do_recoalesce(desc))
  1200. break;
  1201. }
  1202. nfs_pgio_set_current_mirror(desc, restore_idx);
  1203. }
  1204. /*
  1205. * nfs_pageio_resend - Transfer requests to new descriptor and resend
  1206. * @hdr - the pgio header to move request from
  1207. * @desc - the pageio descriptor to add requests to
  1208. *
  1209. * Try to move each request (nfs_page) from @hdr to @desc then attempt
  1210. * to send them.
  1211. *
  1212. * Returns 0 on success and < 0 on error.
  1213. */
  1214. int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
  1215. struct nfs_pgio_header *hdr)
  1216. {
  1217. LIST_HEAD(pages);
  1218. desc->pg_io_completion = hdr->io_completion;
  1219. desc->pg_dreq = hdr->dreq;
  1220. list_splice_init(&hdr->pages, &pages);
  1221. while (!list_empty(&pages)) {
  1222. struct nfs_page *req = nfs_list_entry(pages.next);
  1223. if (!nfs_pageio_add_request(desc, req))
  1224. break;
  1225. }
  1226. nfs_pageio_complete(desc);
  1227. if (!list_empty(&pages)) {
  1228. int err = desc->pg_error < 0 ? desc->pg_error : -EIO;
  1229. hdr->completion_ops->error_cleanup(&pages, err);
  1230. nfs_set_pgio_error(hdr, err, hdr->io_start);
  1231. return err;
  1232. }
  1233. return 0;
  1234. }
  1235. EXPORT_SYMBOL_GPL(nfs_pageio_resend);
  1236. /**
  1237. * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
  1238. * @desc: pointer to io descriptor
  1239. */
  1240. void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
  1241. {
  1242. u32 midx;
  1243. for (midx = 0; midx < desc->pg_mirror_count; midx++)
  1244. nfs_pageio_complete_mirror(desc, midx);
  1245. if (desc->pg_error < 0)
  1246. nfs_pageio_error_cleanup(desc);
  1247. if (desc->pg_ops->pg_cleanup)
  1248. desc->pg_ops->pg_cleanup(desc);
  1249. nfs_pageio_cleanup_mirroring(desc);
  1250. }
  1251. /**
  1252. * nfs_pageio_cond_complete - Conditional I/O completion
  1253. * @desc: pointer to io descriptor
  1254. * @index: page index
  1255. *
  1256. * It is important to ensure that processes don't try to take locks
  1257. * on non-contiguous ranges of pages as that might deadlock. This
  1258. * function should be called before attempting to wait on a locked
  1259. * nfs_page. It will complete the I/O if the page index 'index'
  1260. * is not contiguous with the existing list of pages in 'desc'.
  1261. */
  1262. void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
  1263. {
  1264. struct nfs_pgio_mirror *mirror;
  1265. struct nfs_page *prev;
  1266. u32 midx;
  1267. for (midx = 0; midx < desc->pg_mirror_count; midx++) {
  1268. mirror = nfs_pgio_get_mirror(desc, midx);
  1269. if (!list_empty(&mirror->pg_list)) {
  1270. prev = nfs_list_entry(mirror->pg_list.prev);
  1271. if (index != prev->wb_index + 1) {
  1272. nfs_pageio_complete(desc);
  1273. break;
  1274. }
  1275. }
  1276. }
  1277. }
  1278. /*
  1279. * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
  1280. */
  1281. void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
  1282. {
  1283. nfs_pageio_complete(pgio);
  1284. }
  1285. int __init nfs_init_nfspagecache(void)
  1286. {
  1287. nfs_page_cachep = kmem_cache_create("nfs_page",
  1288. sizeof(struct nfs_page),
  1289. 0, SLAB_HWCACHE_ALIGN,
  1290. NULL);
  1291. if (nfs_page_cachep == NULL)
  1292. return -ENOMEM;
  1293. return 0;
  1294. }
  1295. void nfs_destroy_nfspagecache(void)
  1296. {
  1297. kmem_cache_destroy(nfs_page_cachep);
  1298. }
  1299. static const struct rpc_call_ops nfs_pgio_common_ops = {
  1300. .rpc_call_prepare = nfs_pgio_prepare,
  1301. .rpc_call_done = nfs_pgio_result,
  1302. .rpc_release = nfs_pgio_release,
  1303. };
  1304. const struct nfs_pageio_ops nfs_pgio_rw_ops = {
  1305. .pg_test = nfs_generic_pg_test,
  1306. .pg_doio = nfs_generic_pg_pgios,
  1307. };