direct.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/nfs/direct.c
  4. *
  5. * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
  6. *
  7. * High-performance uncached I/O for the Linux NFS client
  8. *
  9. * There are important applications whose performance or correctness
  10. * depends on uncached access to file data. Database clusters
  11. * (multiple copies of the same instance running on separate hosts)
  12. * implement their own cache coherency protocol that subsumes file
  13. * system cache protocols. Applications that process datasets
  14. * considerably larger than the client's memory do not always benefit
  15. * from a local cache. A streaming video server, for instance, has no
  16. * need to cache the contents of a file.
  17. *
  18. * When an application requests uncached I/O, all read and write requests
  19. * are made directly to the server; data stored or fetched via these
  20. * requests is not cached in the Linux page cache. The client does not
  21. * correct unaligned requests from applications. All requested bytes are
  22. * held on permanent storage before a direct write system call returns to
  23. * an application.
  24. *
  25. * Solaris implements an uncached I/O facility called directio() that
  26. * is used for backups and sequential I/O to very large files. Solaris
  27. * also supports uncaching whole NFS partitions with "-o forcedirectio,"
  28. * an undocumented mount option.
  29. *
  30. * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
  31. * help from Andrew Morton.
  32. *
  33. * 18 Dec 2001 Initial implementation for 2.4 --cel
  34. * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
  35. * 08 Jun 2003 Port to 2.5 APIs --cel
  36. * 31 Mar 2004 Handle direct I/O without VFS support --cel
  37. * 15 Sep 2004 Parallel async reads --cel
  38. * 04 May 2005 support O_DIRECT with aio --cel
  39. *
  40. */
  41. #include <linux/errno.h>
  42. #include <linux/sched.h>
  43. #include <linux/kernel.h>
  44. #include <linux/file.h>
  45. #include <linux/pagemap.h>
  46. #include <linux/kref.h>
  47. #include <linux/slab.h>
  48. #include <linux/task_io_accounting_ops.h>
  49. #include <linux/module.h>
  50. #include <linux/nfs_fs.h>
  51. #include <linux/nfs_page.h>
  52. #include <linux/sunrpc/clnt.h>
  53. #include <linux/uaccess.h>
  54. #include <linux/atomic.h>
  55. #include "internal.h"
  56. #include "iostat.h"
  57. #include "pnfs.h"
  58. #define NFSDBG_FACILITY NFSDBG_VFS
  59. static struct kmem_cache *nfs_direct_cachep;
  60. struct nfs_direct_req {
  61. struct kref kref; /* release manager */
  62. /* I/O parameters */
  63. struct nfs_open_context *ctx; /* file open context info */
  64. struct nfs_lock_context *l_ctx; /* Lock context info */
  65. struct kiocb * iocb; /* controlling i/o request */
  66. struct inode * inode; /* target file of i/o */
  67. /* completion state */
  68. atomic_t io_count; /* i/os we're waiting for */
  69. spinlock_t lock; /* protect completion state */
  70. loff_t io_start; /* Start offset for I/O */
  71. ssize_t count, /* bytes actually processed */
  72. max_count, /* max expected count */
  73. bytes_left, /* bytes left to be sent */
  74. error; /* any reported error */
  75. struct completion completion; /* wait for i/o completion */
  76. /* commit state */
  77. struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
  78. struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
  79. struct work_struct work;
  80. int flags;
  81. /* for write */
  82. #define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
  83. #define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
  84. /* for read */
  85. #define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */
  86. #define NFS_ODIRECT_DONE INT_MAX /* write verification failed */
  87. };
  88. static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
  89. static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
  90. static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
  91. static void nfs_direct_write_schedule_work(struct work_struct *work);
  92. static inline void get_dreq(struct nfs_direct_req *dreq)
  93. {
  94. atomic_inc(&dreq->io_count);
  95. }
  96. static inline int put_dreq(struct nfs_direct_req *dreq)
  97. {
  98. return atomic_dec_and_test(&dreq->io_count);
  99. }
  100. static void
  101. nfs_direct_handle_truncated(struct nfs_direct_req *dreq,
  102. const struct nfs_pgio_header *hdr,
  103. ssize_t dreq_len)
  104. {
  105. if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) ||
  106. test_bit(NFS_IOHDR_EOF, &hdr->flags)))
  107. return;
  108. if (dreq->max_count >= dreq_len) {
  109. dreq->max_count = dreq_len;
  110. if (dreq->count > dreq_len)
  111. dreq->count = dreq_len;
  112. if (test_bit(NFS_IOHDR_ERROR, &hdr->flags))
  113. dreq->error = hdr->error;
  114. else /* Clear outstanding error if this is EOF */
  115. dreq->error = 0;
  116. }
  117. }
  118. static void
  119. nfs_direct_count_bytes(struct nfs_direct_req *dreq,
  120. const struct nfs_pgio_header *hdr)
  121. {
  122. loff_t hdr_end = hdr->io_start + hdr->good_bytes;
  123. ssize_t dreq_len = 0;
  124. if (hdr_end > dreq->io_start)
  125. dreq_len = hdr_end - dreq->io_start;
  126. nfs_direct_handle_truncated(dreq, hdr, dreq_len);
  127. if (dreq_len > dreq->max_count)
  128. dreq_len = dreq->max_count;
  129. if (dreq->count < dreq_len)
  130. dreq->count = dreq_len;
  131. }
  132. /**
  133. * nfs_direct_IO - NFS address space operation for direct I/O
  134. * @iocb: target I/O control block
  135. * @iter: I/O buffer
  136. *
  137. * The presence of this routine in the address space ops vector means
  138. * the NFS client supports direct I/O. However, for most direct IO, we
  139. * shunt off direct read and write requests before the VFS gets them,
  140. * so this method is only ever called for swap.
  141. */
  142. ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  143. {
  144. struct inode *inode = iocb->ki_filp->f_mapping->host;
  145. /* we only support swap file calling nfs_direct_IO */
  146. if (!IS_SWAPFILE(inode))
  147. return 0;
  148. VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
  149. if (iov_iter_rw(iter) == READ)
  150. return nfs_file_direct_read(iocb, iter, true);
  151. return nfs_file_direct_write(iocb, iter, true);
  152. }
  153. static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
  154. {
  155. unsigned int i;
  156. for (i = 0; i < npages; i++)
  157. put_page(pages[i]);
  158. }
  159. void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
  160. struct nfs_direct_req *dreq)
  161. {
  162. cinfo->inode = dreq->inode;
  163. cinfo->mds = &dreq->mds_cinfo;
  164. cinfo->ds = &dreq->ds_cinfo;
  165. cinfo->dreq = dreq;
  166. cinfo->completion_ops = &nfs_direct_commit_completion_ops;
  167. }
  168. static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
  169. {
  170. struct nfs_direct_req *dreq;
  171. dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
  172. if (!dreq)
  173. return NULL;
  174. kref_init(&dreq->kref);
  175. kref_get(&dreq->kref);
  176. init_completion(&dreq->completion);
  177. INIT_LIST_HEAD(&dreq->mds_cinfo.list);
  178. pnfs_init_ds_commit_info(&dreq->ds_cinfo);
  179. INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
  180. spin_lock_init(&dreq->lock);
  181. return dreq;
  182. }
  183. static void nfs_direct_req_free(struct kref *kref)
  184. {
  185. struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
  186. pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);
  187. if (dreq->l_ctx != NULL)
  188. nfs_put_lock_context(dreq->l_ctx);
  189. if (dreq->ctx != NULL)
  190. put_nfs_open_context(dreq->ctx);
  191. kmem_cache_free(nfs_direct_cachep, dreq);
  192. }
  193. static void nfs_direct_req_release(struct nfs_direct_req *dreq)
  194. {
  195. kref_put(&dreq->kref, nfs_direct_req_free);
  196. }
  197. ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
  198. {
  199. return dreq->bytes_left;
  200. }
  201. EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
  202. /*
  203. * Collects and returns the final error value/byte-count.
  204. */
  205. static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
  206. {
  207. ssize_t result = -EIOCBQUEUED;
  208. /* Async requests don't wait here */
  209. if (dreq->iocb)
  210. goto out;
  211. result = wait_for_completion_killable(&dreq->completion);
  212. if (!result) {
  213. result = dreq->count;
  214. WARN_ON_ONCE(dreq->count < 0);
  215. }
  216. if (!result)
  217. result = dreq->error;
  218. out:
  219. return (ssize_t) result;
  220. }
  221. /*
  222. * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
  223. * the iocb is still valid here if this is a synchronous request.
  224. */
  225. static void nfs_direct_complete(struct nfs_direct_req *dreq)
  226. {
  227. struct inode *inode = dreq->inode;
  228. inode_dio_end(inode);
  229. if (dreq->iocb) {
  230. long res = (long) dreq->error;
  231. if (dreq->count != 0) {
  232. res = (long) dreq->count;
  233. WARN_ON_ONCE(dreq->count < 0);
  234. }
  235. dreq->iocb->ki_complete(dreq->iocb, res, 0);
  236. }
  237. complete(&dreq->completion);
  238. nfs_direct_req_release(dreq);
  239. }
  240. static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
  241. {
  242. unsigned long bytes = 0;
  243. struct nfs_direct_req *dreq = hdr->dreq;
  244. spin_lock(&dreq->lock);
  245. if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
  246. spin_unlock(&dreq->lock);
  247. goto out_put;
  248. }
  249. nfs_direct_count_bytes(dreq, hdr);
  250. spin_unlock(&dreq->lock);
  251. while (!list_empty(&hdr->pages)) {
  252. struct nfs_page *req = nfs_list_entry(hdr->pages.next);
  253. struct page *page = req->wb_page;
  254. if (!PageCompound(page) && bytes < hdr->good_bytes &&
  255. (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))
  256. set_page_dirty(page);
  257. bytes += req->wb_bytes;
  258. nfs_list_remove_request(req);
  259. nfs_release_request(req);
  260. }
  261. out_put:
  262. if (put_dreq(dreq))
  263. nfs_direct_complete(dreq);
  264. hdr->release(hdr);
  265. }
  266. static void nfs_read_sync_pgio_error(struct list_head *head, int error)
  267. {
  268. struct nfs_page *req;
  269. while (!list_empty(head)) {
  270. req = nfs_list_entry(head->next);
  271. nfs_list_remove_request(req);
  272. nfs_release_request(req);
  273. }
  274. }
  275. static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
  276. {
  277. get_dreq(hdr->dreq);
  278. }
  279. static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
  280. .error_cleanup = nfs_read_sync_pgio_error,
  281. .init_hdr = nfs_direct_pgio_init,
  282. .completion = nfs_direct_read_completion,
  283. };
  284. /*
  285. * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
  286. * operation. If nfs_readdata_alloc() or get_user_pages() fails,
  287. * bail and stop sending more reads. Read length accounting is
  288. * handled automatically by nfs_direct_read_result(). Otherwise, if
  289. * no requests have been sent, just return an error.
  290. */
  291. static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
  292. struct iov_iter *iter,
  293. loff_t pos)
  294. {
  295. struct nfs_pageio_descriptor desc;
  296. struct inode *inode = dreq->inode;
  297. ssize_t result = -EINVAL;
  298. size_t requested_bytes = 0;
  299. size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
  300. nfs_pageio_init_read(&desc, dreq->inode, false,
  301. &nfs_direct_read_completion_ops);
  302. get_dreq(dreq);
  303. desc.pg_dreq = dreq;
  304. inode_dio_begin(inode);
  305. while (iov_iter_count(iter)) {
  306. struct page **pagevec;
  307. size_t bytes;
  308. size_t pgbase;
  309. unsigned npages, i;
  310. result = iov_iter_get_pages_alloc(iter, &pagevec,
  311. rsize, &pgbase);
  312. if (result < 0)
  313. break;
  314. bytes = result;
  315. iov_iter_advance(iter, bytes);
  316. npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
  317. for (i = 0; i < npages; i++) {
  318. struct nfs_page *req;
  319. unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
  320. /* XXX do we need to do the eof zeroing found in async_filler? */
  321. req = nfs_create_request(dreq->ctx, pagevec[i],
  322. pgbase, req_len);
  323. if (IS_ERR(req)) {
  324. result = PTR_ERR(req);
  325. break;
  326. }
  327. req->wb_index = pos >> PAGE_SHIFT;
  328. req->wb_offset = pos & ~PAGE_MASK;
  329. if (!nfs_pageio_add_request(&desc, req)) {
  330. result = desc.pg_error;
  331. nfs_release_request(req);
  332. break;
  333. }
  334. pgbase = 0;
  335. bytes -= req_len;
  336. requested_bytes += req_len;
  337. pos += req_len;
  338. dreq->bytes_left -= req_len;
  339. }
  340. nfs_direct_release_pages(pagevec, npages);
  341. kvfree(pagevec);
  342. if (result < 0)
  343. break;
  344. }
  345. nfs_pageio_complete(&desc);
  346. /*
  347. * If no bytes were started, return the error, and let the
  348. * generic layer handle the completion.
  349. */
  350. if (requested_bytes == 0) {
  351. inode_dio_end(inode);
  352. nfs_direct_req_release(dreq);
  353. return result < 0 ? result : -EIO;
  354. }
  355. if (put_dreq(dreq))
  356. nfs_direct_complete(dreq);
  357. return requested_bytes;
  358. }
  359. /**
  360. * nfs_file_direct_read - file direct read operation for NFS files
  361. * @iocb: target I/O control block
  362. * @iter: vector of user buffers into which to read data
  363. * @swap: flag indicating this is swap IO, not O_DIRECT IO
  364. *
  365. * We use this function for direct reads instead of calling
  366. * generic_file_aio_read() in order to avoid gfar's check to see if
  367. * the request starts before the end of the file. For that check
  368. * to work, we must generate a GETATTR before each direct read, and
  369. * even then there is a window between the GETATTR and the subsequent
  370. * READ where the file size could change. Our preference is simply
  371. * to do all reads the application wants, and the server will take
  372. * care of managing the end of file boundary.
  373. *
  374. * This function also eliminates unnecessarily updating the file's
  375. * atime locally, as the NFS server sets the file's atime, and this
  376. * client must read the updated atime from the server back into its
  377. * cache.
  378. */
  379. ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
  380. bool swap)
  381. {
  382. struct file *file = iocb->ki_filp;
  383. struct address_space *mapping = file->f_mapping;
  384. struct inode *inode = mapping->host;
  385. struct nfs_direct_req *dreq;
  386. struct nfs_lock_context *l_ctx;
  387. ssize_t result, requested;
  388. size_t count = iov_iter_count(iter);
  389. nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
  390. dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
  391. file, count, (long long) iocb->ki_pos);
  392. result = 0;
  393. if (!count)
  394. goto out;
  395. task_io_account_read(count);
  396. result = -ENOMEM;
  397. dreq = nfs_direct_req_alloc();
  398. if (dreq == NULL)
  399. goto out;
  400. dreq->inode = inode;
  401. dreq->bytes_left = dreq->max_count = count;
  402. dreq->io_start = iocb->ki_pos;
  403. dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
  404. l_ctx = nfs_get_lock_context(dreq->ctx);
  405. if (IS_ERR(l_ctx)) {
  406. result = PTR_ERR(l_ctx);
  407. nfs_direct_req_release(dreq);
  408. goto out_release;
  409. }
  410. dreq->l_ctx = l_ctx;
  411. if (!is_sync_kiocb(iocb))
  412. dreq->iocb = iocb;
  413. if (iter_is_iovec(iter))
  414. dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
  415. if (!swap)
  416. nfs_start_io_direct(inode);
  417. NFS_I(inode)->read_io += count;
  418. requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
  419. if (!swap)
  420. nfs_end_io_direct(inode);
  421. if (requested > 0) {
  422. result = nfs_direct_wait(dreq);
  423. if (result > 0) {
  424. requested -= result;
  425. iocb->ki_pos += result;
  426. }
  427. iov_iter_revert(iter, requested);
  428. } else {
  429. result = requested;
  430. }
  431. out_release:
  432. nfs_direct_req_release(dreq);
  433. out:
  434. return result;
  435. }
  436. static void
  437. nfs_direct_join_group(struct list_head *list, struct inode *inode)
  438. {
  439. struct nfs_page *req, *next;
  440. list_for_each_entry(req, list, wb_list) {
  441. if (req->wb_head != req || req->wb_this_page == req)
  442. continue;
  443. for (next = req->wb_this_page;
  444. next != req->wb_head;
  445. next = next->wb_this_page) {
  446. nfs_list_remove_request(next);
  447. nfs_release_request(next);
  448. }
  449. nfs_join_page_group(req, inode);
  450. }
  451. }
  452. static void
  453. nfs_direct_write_scan_commit_list(struct inode *inode,
  454. struct list_head *list,
  455. struct nfs_commit_info *cinfo)
  456. {
  457. mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
  458. pnfs_recover_commit_reqs(list, cinfo);
  459. nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
  460. mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
  461. }
  462. static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
  463. {
  464. struct nfs_pageio_descriptor desc;
  465. struct nfs_page *req, *tmp;
  466. LIST_HEAD(reqs);
  467. struct nfs_commit_info cinfo;
  468. LIST_HEAD(failed);
  469. nfs_init_cinfo_from_dreq(&cinfo, dreq);
  470. nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
  471. nfs_direct_join_group(&reqs, dreq->inode);
  472. dreq->count = 0;
  473. dreq->max_count = 0;
  474. list_for_each_entry(req, &reqs, wb_list)
  475. dreq->max_count += req->wb_bytes;
  476. nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
  477. get_dreq(dreq);
  478. nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
  479. &nfs_direct_write_completion_ops);
  480. desc.pg_dreq = dreq;
  481. list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
  482. /* Bump the transmission count */
  483. req->wb_nio++;
  484. if (!nfs_pageio_add_request(&desc, req)) {
  485. nfs_list_move_request(req, &failed);
  486. spin_lock(&cinfo.inode->i_lock);
  487. dreq->flags = 0;
  488. if (desc.pg_error < 0)
  489. dreq->error = desc.pg_error;
  490. else
  491. dreq->error = -EIO;
  492. spin_unlock(&cinfo.inode->i_lock);
  493. }
  494. nfs_release_request(req);
  495. }
  496. nfs_pageio_complete(&desc);
  497. while (!list_empty(&failed)) {
  498. req = nfs_list_entry(failed.next);
  499. nfs_list_remove_request(req);
  500. nfs_unlock_and_release_request(req);
  501. }
  502. if (put_dreq(dreq))
  503. nfs_direct_write_complete(dreq);
  504. }
  505. static void nfs_direct_commit_complete(struct nfs_commit_data *data)
  506. {
  507. const struct nfs_writeverf *verf = data->res.verf;
  508. struct nfs_direct_req *dreq = data->dreq;
  509. struct nfs_commit_info cinfo;
  510. struct nfs_page *req;
  511. int status = data->task.tk_status;
  512. if (status < 0) {
  513. /* Errors in commit are fatal */
  514. dreq->error = status;
  515. dreq->max_count = 0;
  516. dreq->count = 0;
  517. dreq->flags = NFS_ODIRECT_DONE;
  518. } else if (dreq->flags == NFS_ODIRECT_DONE)
  519. status = dreq->error;
  520. nfs_init_cinfo_from_dreq(&cinfo, dreq);
  521. while (!list_empty(&data->pages)) {
  522. req = nfs_list_entry(data->pages.next);
  523. nfs_list_remove_request(req);
  524. if (status >= 0 && !nfs_write_match_verf(verf, req)) {
  525. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  526. /*
  527. * Despite the reboot, the write was successful,
  528. * so reset wb_nio.
  529. */
  530. req->wb_nio = 0;
  531. nfs_mark_request_commit(req, NULL, &cinfo, 0);
  532. } else /* Error or match */
  533. nfs_release_request(req);
  534. nfs_unlock_and_release_request(req);
  535. }
  536. if (nfs_commit_end(cinfo.mds))
  537. nfs_direct_write_complete(dreq);
  538. }
  539. static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
  540. struct nfs_page *req)
  541. {
  542. struct nfs_direct_req *dreq = cinfo->dreq;
  543. spin_lock(&dreq->lock);
  544. if (dreq->flags != NFS_ODIRECT_DONE)
  545. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  546. spin_unlock(&dreq->lock);
  547. nfs_mark_request_commit(req, NULL, cinfo, 0);
  548. }
  549. static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
  550. .completion = nfs_direct_commit_complete,
  551. .resched_write = nfs_direct_resched_write,
  552. };
  553. static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
  554. {
  555. int res;
  556. struct nfs_commit_info cinfo;
  557. LIST_HEAD(mds_list);
  558. nfs_init_cinfo_from_dreq(&cinfo, dreq);
  559. nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
  560. res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
  561. if (res < 0) /* res == -ENOMEM */
  562. nfs_direct_write_reschedule(dreq);
  563. }
  564. static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)
  565. {
  566. struct nfs_commit_info cinfo;
  567. struct nfs_page *req;
  568. LIST_HEAD(reqs);
  569. nfs_init_cinfo_from_dreq(&cinfo, dreq);
  570. nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
  571. while (!list_empty(&reqs)) {
  572. req = nfs_list_entry(reqs.next);
  573. nfs_list_remove_request(req);
  574. nfs_release_request(req);
  575. nfs_unlock_and_release_request(req);
  576. }
  577. }
  578. static void nfs_direct_write_schedule_work(struct work_struct *work)
  579. {
  580. struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
  581. int flags = dreq->flags;
  582. dreq->flags = 0;
  583. switch (flags) {
  584. case NFS_ODIRECT_DO_COMMIT:
  585. nfs_direct_commit_schedule(dreq);
  586. break;
  587. case NFS_ODIRECT_RESCHED_WRITES:
  588. nfs_direct_write_reschedule(dreq);
  589. break;
  590. default:
  591. nfs_direct_write_clear_reqs(dreq);
  592. nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
  593. nfs_direct_complete(dreq);
  594. }
  595. }
  596. static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
  597. {
  598. queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
  599. }
  600. static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
  601. {
  602. struct nfs_direct_req *dreq = hdr->dreq;
  603. struct nfs_commit_info cinfo;
  604. struct nfs_page *req = nfs_list_entry(hdr->pages.next);
  605. int flags = NFS_ODIRECT_DONE;
  606. nfs_init_cinfo_from_dreq(&cinfo, dreq);
  607. spin_lock(&dreq->lock);
  608. if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
  609. spin_unlock(&dreq->lock);
  610. goto out_put;
  611. }
  612. nfs_direct_count_bytes(dreq, hdr);
  613. if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) {
  614. if (!dreq->flags)
  615. dreq->flags = NFS_ODIRECT_DO_COMMIT;
  616. flags = dreq->flags;
  617. }
  618. spin_unlock(&dreq->lock);
  619. while (!list_empty(&hdr->pages)) {
  620. req = nfs_list_entry(hdr->pages.next);
  621. nfs_list_remove_request(req);
  622. if (flags == NFS_ODIRECT_DO_COMMIT) {
  623. kref_get(&req->wb_kref);
  624. memcpy(&req->wb_verf, &hdr->verf.verifier,
  625. sizeof(req->wb_verf));
  626. nfs_mark_request_commit(req, hdr->lseg, &cinfo,
  627. hdr->ds_commit_idx);
  628. } else if (flags == NFS_ODIRECT_RESCHED_WRITES) {
  629. kref_get(&req->wb_kref);
  630. nfs_mark_request_commit(req, NULL, &cinfo, 0);
  631. }
  632. nfs_unlock_and_release_request(req);
  633. }
  634. out_put:
  635. if (put_dreq(dreq))
  636. nfs_direct_write_complete(dreq);
  637. hdr->release(hdr);
  638. }
  639. static void nfs_write_sync_pgio_error(struct list_head *head, int error)
  640. {
  641. struct nfs_page *req;
  642. while (!list_empty(head)) {
  643. req = nfs_list_entry(head->next);
  644. nfs_list_remove_request(req);
  645. nfs_unlock_and_release_request(req);
  646. }
  647. }
  648. static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
  649. {
  650. struct nfs_direct_req *dreq = hdr->dreq;
  651. spin_lock(&dreq->lock);
  652. if (dreq->error == 0) {
  653. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  654. /* fake unstable write to let common nfs resend pages */
  655. hdr->verf.committed = NFS_UNSTABLE;
  656. hdr->good_bytes = hdr->args.offset + hdr->args.count -
  657. hdr->io_start;
  658. }
  659. spin_unlock(&dreq->lock);
  660. }
  661. static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
  662. .error_cleanup = nfs_write_sync_pgio_error,
  663. .init_hdr = nfs_direct_pgio_init,
  664. .completion = nfs_direct_write_completion,
  665. .reschedule_io = nfs_direct_write_reschedule_io,
  666. };
  667. /*
  668. * NB: Return the value of the first error return code. Subsequent
  669. * errors after the first one are ignored.
  670. */
  671. /*
  672. * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
  673. * operation. If nfs_writedata_alloc() or get_user_pages() fails,
  674. * bail and stop sending more writes. Write length accounting is
  675. * handled automatically by nfs_direct_write_result(). Otherwise, if
  676. * no requests have been sent, just return an error.
  677. */
  678. static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
  679. struct iov_iter *iter,
  680. loff_t pos, int ioflags)
  681. {
  682. struct nfs_pageio_descriptor desc;
  683. struct inode *inode = dreq->inode;
  684. ssize_t result = 0;
  685. size_t requested_bytes = 0;
  686. size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
  687. nfs_pageio_init_write(&desc, inode, ioflags, false,
  688. &nfs_direct_write_completion_ops);
  689. desc.pg_dreq = dreq;
  690. get_dreq(dreq);
  691. inode_dio_begin(inode);
  692. NFS_I(inode)->write_io += iov_iter_count(iter);
  693. while (iov_iter_count(iter)) {
  694. struct page **pagevec;
  695. size_t bytes;
  696. size_t pgbase;
  697. unsigned npages, i;
  698. result = iov_iter_get_pages_alloc(iter, &pagevec,
  699. wsize, &pgbase);
  700. if (result < 0)
  701. break;
  702. bytes = result;
  703. iov_iter_advance(iter, bytes);
  704. npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
  705. for (i = 0; i < npages; i++) {
  706. struct nfs_page *req;
  707. unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
  708. req = nfs_create_request(dreq->ctx, pagevec[i],
  709. pgbase, req_len);
  710. if (IS_ERR(req)) {
  711. result = PTR_ERR(req);
  712. break;
  713. }
  714. if (desc.pg_error < 0) {
  715. nfs_free_request(req);
  716. result = desc.pg_error;
  717. break;
  718. }
  719. nfs_lock_request(req);
  720. req->wb_index = pos >> PAGE_SHIFT;
  721. req->wb_offset = pos & ~PAGE_MASK;
  722. if (!nfs_pageio_add_request(&desc, req)) {
  723. result = desc.pg_error;
  724. nfs_unlock_and_release_request(req);
  725. break;
  726. }
  727. pgbase = 0;
  728. bytes -= req_len;
  729. requested_bytes += req_len;
  730. pos += req_len;
  731. dreq->bytes_left -= req_len;
  732. }
  733. nfs_direct_release_pages(pagevec, npages);
  734. kvfree(pagevec);
  735. if (result < 0)
  736. break;
  737. }
  738. nfs_pageio_complete(&desc);
  739. /*
  740. * If no bytes were started, return the error, and let the
  741. * generic layer handle the completion.
  742. */
  743. if (requested_bytes == 0) {
  744. inode_dio_end(inode);
  745. nfs_direct_req_release(dreq);
  746. return result < 0 ? result : -EIO;
  747. }
  748. if (put_dreq(dreq))
  749. nfs_direct_write_complete(dreq);
  750. return requested_bytes;
  751. }
  752. /**
  753. * nfs_file_direct_write - file direct write operation for NFS files
  754. * @iocb: target I/O control block
  755. * @iter: vector of user buffers from which to write data
  756. * @swap: flag indicating this is swap IO, not O_DIRECT IO
  757. *
  758. * We use this function for direct writes instead of calling
  759. * generic_file_aio_write() in order to avoid taking the inode
  760. * semaphore and updating the i_size. The NFS server will set
  761. * the new i_size and this client must read the updated size
  762. * back into its cache. We let the server do generic write
  763. * parameter checking and report problems.
  764. *
  765. * We eliminate local atime updates, see direct read above.
  766. *
  767. * We avoid unnecessary page cache invalidations for normal cached
  768. * readers of this file.
  769. *
  770. * Note that O_APPEND is not supported for NFS direct writes, as there
  771. * is no atomic O_APPEND write facility in the NFS protocol.
  772. */
  773. ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
  774. bool swap)
  775. {
  776. ssize_t result, requested;
  777. size_t count;
  778. struct file *file = iocb->ki_filp;
  779. struct address_space *mapping = file->f_mapping;
  780. struct inode *inode = mapping->host;
  781. struct nfs_direct_req *dreq;
  782. struct nfs_lock_context *l_ctx;
  783. loff_t pos, end;
  784. dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
  785. file, iov_iter_count(iter), (long long) iocb->ki_pos);
  786. if (swap)
  787. /* bypass generic checks */
  788. result = iov_iter_count(iter);
  789. else
  790. result = generic_write_checks(iocb, iter);
  791. if (result <= 0)
  792. return result;
  793. count = result;
  794. nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
  795. pos = iocb->ki_pos;
  796. end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
  797. task_io_account_write(count);
  798. result = -ENOMEM;
  799. dreq = nfs_direct_req_alloc();
  800. if (!dreq)
  801. goto out;
  802. dreq->inode = inode;
  803. dreq->bytes_left = dreq->max_count = count;
  804. dreq->io_start = pos;
  805. dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
  806. l_ctx = nfs_get_lock_context(dreq->ctx);
  807. if (IS_ERR(l_ctx)) {
  808. result = PTR_ERR(l_ctx);
  809. nfs_direct_req_release(dreq);
  810. goto out_release;
  811. }
  812. dreq->l_ctx = l_ctx;
  813. if (!is_sync_kiocb(iocb))
  814. dreq->iocb = iocb;
  815. pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
  816. if (swap) {
  817. requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
  818. FLUSH_STABLE);
  819. } else {
  820. nfs_start_io_direct(inode);
  821. requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
  822. FLUSH_COND_STABLE);
  823. if (mapping->nrpages) {
  824. invalidate_inode_pages2_range(mapping,
  825. pos >> PAGE_SHIFT, end);
  826. }
  827. nfs_end_io_direct(inode);
  828. }
  829. if (requested > 0) {
  830. result = nfs_direct_wait(dreq);
  831. if (result > 0) {
  832. requested -= result;
  833. iocb->ki_pos = pos + result;
  834. /* XXX: should check the generic_write_sync retval */
  835. generic_write_sync(iocb, result);
  836. }
  837. iov_iter_revert(iter, requested);
  838. } else {
  839. result = requested;
  840. }
  841. out_release:
  842. nfs_direct_req_release(dreq);
  843. out:
  844. return result;
  845. }
  846. /**
  847. * nfs_init_directcache - create a slab cache for nfs_direct_req structures
  848. *
  849. */
  850. int __init nfs_init_directcache(void)
  851. {
  852. nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
  853. sizeof(struct nfs_direct_req),
  854. 0, (SLAB_RECLAIM_ACCOUNT|
  855. SLAB_MEM_SPREAD),
  856. NULL);
  857. if (nfs_direct_cachep == NULL)
  858. return -ENOMEM;
  859. return 0;
  860. }
  861. /**
  862. * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
  863. *
  864. */
  865. void nfs_destroy_directcache(void)
  866. {
  867. kmem_cache_destroy(nfs_direct_cachep);
  868. }