direct.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865
  1. /*
  2. * linux/fs/nfs/direct.c
  3. *
  4. * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
  5. *
  6. * High-performance uncached I/O for the Linux NFS client
  7. *
  8. * There are important applications whose performance or correctness
  9. * depends on uncached access to file data. Database clusters
  10. * (multiple copies of the same instance running on separate hosts)
  11. * implement their own cache coherency protocol that subsumes file
  12. * system cache protocols. Applications that process datasets
  13. * considerably larger than the client's memory do not always benefit
  14. * from a local cache. A streaming video server, for instance, has no
  15. * need to cache the contents of a file.
  16. *
  17. * When an application requests uncached I/O, all read and write requests
  18. * are made directly to the server; data stored or fetched via these
  19. * requests is not cached in the Linux page cache. The client does not
  20. * correct unaligned requests from applications. All requested bytes are
  21. * held on permanent storage before a direct write system call returns to
  22. * an application.
  23. *
  24. * Solaris implements an uncached I/O facility called directio() that
  25. * is used for backups and sequential I/O to very large files. Solaris
  26. * also supports uncaching whole NFS partitions with "-o forcedirectio,"
  27. * an undocumented mount option.
  28. *
  29. * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
  30. * help from Andrew Morton.
  31. *
  32. * 18 Dec 2001 Initial implementation for 2.4 --cel
  33. * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
  34. * 08 Jun 2003 Port to 2.5 APIs --cel
  35. * 31 Mar 2004 Handle direct I/O without VFS support --cel
  36. * 15 Sep 2004 Parallel async reads --cel
  37. * 04 May 2005 support O_DIRECT with aio --cel
  38. *
  39. */
  40. #include <linux/errno.h>
  41. #include <linux/sched.h>
  42. #include <linux/kernel.h>
  43. #include <linux/smp_lock.h>
  44. #include <linux/file.h>
  45. #include <linux/pagemap.h>
  46. #include <linux/kref.h>
  47. #include <linux/nfs_fs.h>
  48. #include <linux/nfs_page.h>
  49. #include <linux/sunrpc/clnt.h>
  50. #include <asm/system.h>
  51. #include <asm/uaccess.h>
  52. #include <asm/atomic.h>
  53. #include "iostat.h"
  54. #define NFSDBG_FACILITY NFSDBG_VFS
  55. static struct kmem_cache *nfs_direct_cachep;
  56. /*
  57. * This represents a set of asynchronous requests that we're waiting on
  58. */
  59. struct nfs_direct_req {
  60. struct kref kref; /* release manager */
  61. /* I/O parameters */
  62. struct nfs_open_context *ctx; /* file open context info */
  63. struct kiocb * iocb; /* controlling i/o request */
  64. struct inode * inode; /* target file of i/o */
  65. /* completion state */
  66. atomic_t io_count; /* i/os we're waiting for */
  67. spinlock_t lock; /* protect completion state */
  68. ssize_t count, /* bytes actually processed */
  69. error; /* any reported error */
  70. struct completion completion; /* wait for i/o completion */
  71. /* commit state */
  72. struct list_head rewrite_list; /* saved nfs_write_data structs */
  73. struct nfs_write_data * commit_data; /* special write_data for commits */
  74. int flags;
  75. #define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
  76. #define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
  77. struct nfs_writeverf verf; /* unstable write verifier */
  78. };
  79. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
  80. static const struct rpc_call_ops nfs_write_direct_ops;
  81. static inline void get_dreq(struct nfs_direct_req *dreq)
  82. {
  83. atomic_inc(&dreq->io_count);
  84. }
  85. static inline int put_dreq(struct nfs_direct_req *dreq)
  86. {
  87. return atomic_dec_and_test(&dreq->io_count);
  88. }
  89. /**
  90. * nfs_direct_IO - NFS address space operation for direct I/O
  91. * @rw: direction (read or write)
  92. * @iocb: target I/O control block
  93. * @iov: array of vectors that define I/O buffer
  94. * @pos: offset in file to begin the operation
  95. * @nr_segs: size of iovec array
  96. *
  97. * The presence of this routine in the address space ops vector means
  98. * the NFS client supports direct I/O. However, we shunt off direct
  99. * read and write requests before the VFS gets them, so this method
  100. * should never be called.
  101. */
  102. ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
  103. {
  104. dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
  105. iocb->ki_filp->f_path.dentry->d_name.name,
  106. (long long) pos, nr_segs);
  107. return -EINVAL;
  108. }
  109. static void nfs_direct_dirty_pages(struct page **pages, int npages)
  110. {
  111. int i;
  112. for (i = 0; i < npages; i++) {
  113. struct page *page = pages[i];
  114. if (!PageCompound(page))
  115. set_page_dirty_lock(page);
  116. }
  117. }
  118. static void nfs_direct_release_pages(struct page **pages, int npages)
  119. {
  120. int i;
  121. for (i = 0; i < npages; i++)
  122. page_cache_release(pages[i]);
  123. }
  124. static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
  125. {
  126. struct nfs_direct_req *dreq;
  127. dreq = kmem_cache_alloc(nfs_direct_cachep, GFP_KERNEL);
  128. if (!dreq)
  129. return NULL;
  130. kref_init(&dreq->kref);
  131. kref_get(&dreq->kref);
  132. init_completion(&dreq->completion);
  133. INIT_LIST_HEAD(&dreq->rewrite_list);
  134. dreq->iocb = NULL;
  135. dreq->ctx = NULL;
  136. spin_lock_init(&dreq->lock);
  137. atomic_set(&dreq->io_count, 0);
  138. dreq->count = 0;
  139. dreq->error = 0;
  140. dreq->flags = 0;
  141. return dreq;
  142. }
  143. static void nfs_direct_req_release(struct kref *kref)
  144. {
  145. struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
  146. if (dreq->ctx != NULL)
  147. put_nfs_open_context(dreq->ctx);
  148. kmem_cache_free(nfs_direct_cachep, dreq);
  149. }
  150. /*
  151. * Collects and returns the final error value/byte-count.
  152. */
  153. static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
  154. {
  155. ssize_t result = -EIOCBQUEUED;
  156. /* Async requests don't wait here */
  157. if (dreq->iocb)
  158. goto out;
  159. result = wait_for_completion_interruptible(&dreq->completion);
  160. if (!result)
  161. result = dreq->error;
  162. if (!result)
  163. result = dreq->count;
  164. out:
  165. kref_put(&dreq->kref, nfs_direct_req_release);
  166. return (ssize_t) result;
  167. }
  168. /*
  169. * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
  170. * the iocb is still valid here if this is a synchronous request.
  171. */
  172. static void nfs_direct_complete(struct nfs_direct_req *dreq)
  173. {
  174. if (dreq->iocb) {
  175. long res = (long) dreq->error;
  176. if (!res)
  177. res = (long) dreq->count;
  178. aio_complete(dreq->iocb, res, 0);
  179. }
  180. complete_all(&dreq->completion);
  181. kref_put(&dreq->kref, nfs_direct_req_release);
  182. }
  183. /*
  184. * We must hold a reference to all the pages in this direct read request
  185. * until the RPCs complete. This could be long *after* we are woken up in
  186. * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
  187. */
  188. static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
  189. {
  190. struct nfs_read_data *data = calldata;
  191. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  192. if (nfs_readpage_result(task, data) != 0)
  193. return;
  194. nfs_direct_dirty_pages(data->pagevec, data->npages);
  195. nfs_direct_release_pages(data->pagevec, data->npages);
  196. spin_lock(&dreq->lock);
  197. if (likely(task->tk_status >= 0))
  198. dreq->count += data->res.count;
  199. else
  200. dreq->error = task->tk_status;
  201. spin_unlock(&dreq->lock);
  202. if (put_dreq(dreq))
  203. nfs_direct_complete(dreq);
  204. }
  205. static const struct rpc_call_ops nfs_read_direct_ops = {
  206. .rpc_call_done = nfs_direct_read_result,
  207. .rpc_release = nfs_readdata_release,
  208. };
  209. /*
  210. * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
  211. * operation. If nfs_readdata_alloc() or get_user_pages() fails,
  212. * bail and stop sending more reads. Read length accounting is
  213. * handled automatically by nfs_direct_read_result(). Otherwise, if
  214. * no requests have been sent, just return an error.
  215. */
  216. static ssize_t nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos)
  217. {
  218. struct nfs_open_context *ctx = dreq->ctx;
  219. struct inode *inode = ctx->dentry->d_inode;
  220. size_t rsize = NFS_SERVER(inode)->rsize;
  221. unsigned int pgbase;
  222. int result;
  223. ssize_t started = 0;
  224. get_dreq(dreq);
  225. do {
  226. struct nfs_read_data *data;
  227. size_t bytes;
  228. pgbase = user_addr & ~PAGE_MASK;
  229. bytes = min(rsize,count);
  230. result = -ENOMEM;
  231. data = nfs_readdata_alloc(pgbase + bytes);
  232. if (unlikely(!data))
  233. break;
  234. down_read(&current->mm->mmap_sem);
  235. result = get_user_pages(current, current->mm, user_addr,
  236. data->npages, 1, 0, data->pagevec, NULL);
  237. up_read(&current->mm->mmap_sem);
  238. if (unlikely(result < data->npages)) {
  239. if (result > 0)
  240. nfs_direct_release_pages(data->pagevec, result);
  241. nfs_readdata_release(data);
  242. break;
  243. }
  244. get_dreq(dreq);
  245. data->req = (struct nfs_page *) dreq;
  246. data->inode = inode;
  247. data->cred = ctx->cred;
  248. data->args.fh = NFS_FH(inode);
  249. data->args.context = ctx;
  250. data->args.offset = pos;
  251. data->args.pgbase = pgbase;
  252. data->args.pages = data->pagevec;
  253. data->args.count = bytes;
  254. data->res.fattr = &data->fattr;
  255. data->res.eof = 0;
  256. data->res.count = bytes;
  257. rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
  258. &nfs_read_direct_ops, data);
  259. NFS_PROTO(inode)->read_setup(data);
  260. data->task.tk_cookie = (unsigned long) inode;
  261. rpc_execute(&data->task);
  262. dprintk("NFS: %5u initiated direct read call "
  263. "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
  264. data->task.tk_pid,
  265. inode->i_sb->s_id,
  266. (long long)NFS_FILEID(inode),
  267. bytes,
  268. (unsigned long long)data->args.offset);
  269. started += bytes;
  270. user_addr += bytes;
  271. pos += bytes;
  272. /* FIXME: Remove this unnecessary math from final patch */
  273. pgbase += bytes;
  274. pgbase &= ~PAGE_MASK;
  275. BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
  276. count -= bytes;
  277. } while (count != 0);
  278. if (put_dreq(dreq))
  279. nfs_direct_complete(dreq);
  280. if (started)
  281. return 0;
  282. return result < 0 ? (ssize_t) result : -EFAULT;
  283. }
  284. static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
  285. {
  286. ssize_t result = 0;
  287. sigset_t oldset;
  288. struct inode *inode = iocb->ki_filp->f_mapping->host;
  289. struct rpc_clnt *clnt = NFS_CLIENT(inode);
  290. struct nfs_direct_req *dreq;
  291. dreq = nfs_direct_req_alloc();
  292. if (!dreq)
  293. return -ENOMEM;
  294. dreq->inode = inode;
  295. dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
  296. if (!is_sync_kiocb(iocb))
  297. dreq->iocb = iocb;
  298. nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
  299. rpc_clnt_sigmask(clnt, &oldset);
  300. result = nfs_direct_read_schedule(dreq, user_addr, count, pos);
  301. if (!result)
  302. result = nfs_direct_wait(dreq);
  303. rpc_clnt_sigunmask(clnt, &oldset);
  304. return result;
  305. }
  306. static void nfs_direct_free_writedata(struct nfs_direct_req *dreq)
  307. {
  308. while (!list_empty(&dreq->rewrite_list)) {
  309. struct nfs_write_data *data = list_entry(dreq->rewrite_list.next, struct nfs_write_data, pages);
  310. list_del(&data->pages);
  311. nfs_direct_release_pages(data->pagevec, data->npages);
  312. nfs_writedata_release(data);
  313. }
  314. }
  315. #if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
  316. static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
  317. {
  318. struct inode *inode = dreq->inode;
  319. struct list_head *p;
  320. struct nfs_write_data *data;
  321. dreq->count = 0;
  322. get_dreq(dreq);
  323. list_for_each(p, &dreq->rewrite_list) {
  324. data = list_entry(p, struct nfs_write_data, pages);
  325. get_dreq(dreq);
  326. /*
  327. * Reset data->res.
  328. */
  329. nfs_fattr_init(&data->fattr);
  330. data->res.count = data->args.count;
  331. memset(&data->verf, 0, sizeof(data->verf));
  332. /*
  333. * Reuse data->task; data->args should not have changed
  334. * since the original request was sent.
  335. */
  336. rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
  337. &nfs_write_direct_ops, data);
  338. NFS_PROTO(inode)->write_setup(data, FLUSH_STABLE);
  339. data->task.tk_priority = RPC_PRIORITY_NORMAL;
  340. data->task.tk_cookie = (unsigned long) inode;
  341. /*
  342. * We're called via an RPC callback, so BKL is already held.
  343. */
  344. rpc_execute(&data->task);
  345. dprintk("NFS: %5u rescheduled direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
  346. data->task.tk_pid,
  347. inode->i_sb->s_id,
  348. (long long)NFS_FILEID(inode),
  349. data->args.count,
  350. (unsigned long long)data->args.offset);
  351. }
  352. if (put_dreq(dreq))
  353. nfs_direct_write_complete(dreq, inode);
  354. }
  355. static void nfs_direct_commit_result(struct rpc_task *task, void *calldata)
  356. {
  357. struct nfs_write_data *data = calldata;
  358. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  359. /* Call the NFS version-specific code */
  360. if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
  361. return;
  362. if (unlikely(task->tk_status < 0)) {
  363. dprintk("NFS: %5u commit failed with error %d.\n",
  364. task->tk_pid, task->tk_status);
  365. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  366. } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
  367. dprintk("NFS: %5u commit verify failed\n", task->tk_pid);
  368. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  369. }
  370. dprintk("NFS: %5u commit returned %d\n", task->tk_pid, task->tk_status);
  371. nfs_direct_write_complete(dreq, data->inode);
  372. }
  373. static const struct rpc_call_ops nfs_commit_direct_ops = {
  374. .rpc_call_done = nfs_direct_commit_result,
  375. .rpc_release = nfs_commit_release,
  376. };
  377. static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
  378. {
  379. struct nfs_write_data *data = dreq->commit_data;
  380. data->inode = dreq->inode;
  381. data->cred = dreq->ctx->cred;
  382. data->args.fh = NFS_FH(data->inode);
  383. data->args.offset = 0;
  384. data->args.count = 0;
  385. data->res.count = 0;
  386. data->res.fattr = &data->fattr;
  387. data->res.verf = &data->verf;
  388. rpc_init_task(&data->task, NFS_CLIENT(dreq->inode), RPC_TASK_ASYNC,
  389. &nfs_commit_direct_ops, data);
  390. NFS_PROTO(data->inode)->commit_setup(data, 0);
  391. data->task.tk_priority = RPC_PRIORITY_NORMAL;
  392. data->task.tk_cookie = (unsigned long)data->inode;
  393. /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
  394. dreq->commit_data = NULL;
  395. dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
  396. rpc_execute(&data->task);
  397. }
  398. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
  399. {
  400. int flags = dreq->flags;
  401. dreq->flags = 0;
  402. switch (flags) {
  403. case NFS_ODIRECT_DO_COMMIT:
  404. nfs_direct_commit_schedule(dreq);
  405. break;
  406. case NFS_ODIRECT_RESCHED_WRITES:
  407. nfs_direct_write_reschedule(dreq);
  408. break;
  409. default:
  410. nfs_end_data_update(inode);
  411. if (dreq->commit_data != NULL)
  412. nfs_commit_free(dreq->commit_data);
  413. nfs_direct_free_writedata(dreq);
  414. nfs_zap_mapping(inode, inode->i_mapping);
  415. nfs_direct_complete(dreq);
  416. }
  417. }
  418. static void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
  419. {
  420. dreq->commit_data = nfs_commit_alloc();
  421. if (dreq->commit_data != NULL)
  422. dreq->commit_data->req = (struct nfs_page *) dreq;
  423. }
  424. #else
  425. static inline void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
  426. {
  427. dreq->commit_data = NULL;
  428. }
  429. static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
  430. {
  431. nfs_end_data_update(inode);
  432. nfs_direct_free_writedata(dreq);
  433. nfs_zap_mapping(inode, inode->i_mapping);
  434. nfs_direct_complete(dreq);
  435. }
  436. #endif
  437. static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
  438. {
  439. struct nfs_write_data *data = calldata;
  440. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  441. int status = task->tk_status;
  442. if (nfs_writeback_done(task, data) != 0)
  443. return;
  444. spin_lock(&dreq->lock);
  445. if (unlikely(dreq->error != 0))
  446. goto out_unlock;
  447. if (unlikely(status < 0)) {
  448. /* An error has occured, so we should not commit */
  449. dreq->flags = 0;
  450. dreq->error = status;
  451. }
  452. dreq->count += data->res.count;
  453. if (data->res.verf->committed != NFS_FILE_SYNC) {
  454. switch (dreq->flags) {
  455. case 0:
  456. memcpy(&dreq->verf, &data->verf, sizeof(dreq->verf));
  457. dreq->flags = NFS_ODIRECT_DO_COMMIT;
  458. break;
  459. case NFS_ODIRECT_DO_COMMIT:
  460. if (memcmp(&dreq->verf, &data->verf, sizeof(dreq->verf))) {
  461. dprintk("NFS: %5u write verify failed\n", task->tk_pid);
  462. dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
  463. }
  464. }
  465. }
  466. out_unlock:
  467. spin_unlock(&dreq->lock);
  468. }
  469. /*
  470. * NB: Return the value of the first error return code. Subsequent
  471. * errors after the first one are ignored.
  472. */
  473. static void nfs_direct_write_release(void *calldata)
  474. {
  475. struct nfs_write_data *data = calldata;
  476. struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
  477. if (put_dreq(dreq))
  478. nfs_direct_write_complete(dreq, data->inode);
  479. }
  480. static const struct rpc_call_ops nfs_write_direct_ops = {
  481. .rpc_call_done = nfs_direct_write_result,
  482. .rpc_release = nfs_direct_write_release,
  483. };
  484. /*
  485. * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
  486. * operation. If nfs_writedata_alloc() or get_user_pages() fails,
  487. * bail and stop sending more writes. Write length accounting is
  488. * handled automatically by nfs_direct_write_result(). Otherwise, if
  489. * no requests have been sent, just return an error.
  490. */
  491. static ssize_t nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t pos, int sync)
  492. {
  493. struct nfs_open_context *ctx = dreq->ctx;
  494. struct inode *inode = ctx->dentry->d_inode;
  495. size_t wsize = NFS_SERVER(inode)->wsize;
  496. unsigned int pgbase;
  497. int result;
  498. ssize_t started = 0;
  499. get_dreq(dreq);
  500. do {
  501. struct nfs_write_data *data;
  502. size_t bytes;
  503. pgbase = user_addr & ~PAGE_MASK;
  504. bytes = min(wsize,count);
  505. result = -ENOMEM;
  506. data = nfs_writedata_alloc(pgbase + bytes);
  507. if (unlikely(!data))
  508. break;
  509. down_read(&current->mm->mmap_sem);
  510. result = get_user_pages(current, current->mm, user_addr,
  511. data->npages, 0, 0, data->pagevec, NULL);
  512. up_read(&current->mm->mmap_sem);
  513. if (unlikely(result < data->npages)) {
  514. if (result > 0)
  515. nfs_direct_release_pages(data->pagevec, result);
  516. nfs_writedata_release(data);
  517. break;
  518. }
  519. get_dreq(dreq);
  520. list_move_tail(&data->pages, &dreq->rewrite_list);
  521. data->req = (struct nfs_page *) dreq;
  522. data->inode = inode;
  523. data->cred = ctx->cred;
  524. data->args.fh = NFS_FH(inode);
  525. data->args.context = ctx;
  526. data->args.offset = pos;
  527. data->args.pgbase = pgbase;
  528. data->args.pages = data->pagevec;
  529. data->args.count = bytes;
  530. data->res.fattr = &data->fattr;
  531. data->res.count = bytes;
  532. data->res.verf = &data->verf;
  533. rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
  534. &nfs_write_direct_ops, data);
  535. NFS_PROTO(inode)->write_setup(data, sync);
  536. data->task.tk_priority = RPC_PRIORITY_NORMAL;
  537. data->task.tk_cookie = (unsigned long) inode;
  538. rpc_execute(&data->task);
  539. dprintk("NFS: %5u initiated direct write call "
  540. "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
  541. data->task.tk_pid,
  542. inode->i_sb->s_id,
  543. (long long)NFS_FILEID(inode),
  544. bytes,
  545. (unsigned long long)data->args.offset);
  546. started += bytes;
  547. user_addr += bytes;
  548. pos += bytes;
  549. /* FIXME: Remove this useless math from the final patch */
  550. pgbase += bytes;
  551. pgbase &= ~PAGE_MASK;
  552. BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
  553. count -= bytes;
  554. } while (count != 0);
  555. if (put_dreq(dreq))
  556. nfs_direct_write_complete(dreq, inode);
  557. if (started)
  558. return 0;
  559. return result < 0 ? (ssize_t) result : -EFAULT;
  560. }
  561. static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos)
  562. {
  563. ssize_t result = 0;
  564. sigset_t oldset;
  565. struct inode *inode = iocb->ki_filp->f_mapping->host;
  566. struct rpc_clnt *clnt = NFS_CLIENT(inode);
  567. struct nfs_direct_req *dreq;
  568. size_t wsize = NFS_SERVER(inode)->wsize;
  569. int sync = 0;
  570. dreq = nfs_direct_req_alloc();
  571. if (!dreq)
  572. return -ENOMEM;
  573. nfs_alloc_commit_data(dreq);
  574. if (dreq->commit_data == NULL || count < wsize)
  575. sync = FLUSH_STABLE;
  576. dreq->inode = inode;
  577. dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
  578. if (!is_sync_kiocb(iocb))
  579. dreq->iocb = iocb;
  580. nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, count);
  581. nfs_begin_data_update(inode);
  582. rpc_clnt_sigmask(clnt, &oldset);
  583. result = nfs_direct_write_schedule(dreq, user_addr, count, pos, sync);
  584. if (!result)
  585. result = nfs_direct_wait(dreq);
  586. rpc_clnt_sigunmask(clnt, &oldset);
  587. return result;
  588. }
  589. /**
  590. * nfs_file_direct_read - file direct read operation for NFS files
  591. * @iocb: target I/O control block
  592. * @iov: vector of user buffers into which to read data
  593. * @nr_segs: size of iov vector
  594. * @pos: byte offset in file where reading starts
  595. *
  596. * We use this function for direct reads instead of calling
  597. * generic_file_aio_read() in order to avoid gfar's check to see if
  598. * the request starts before the end of the file. For that check
  599. * to work, we must generate a GETATTR before each direct read, and
  600. * even then there is a window between the GETATTR and the subsequent
  601. * READ where the file size could change. Our preference is simply
  602. * to do all reads the application wants, and the server will take
  603. * care of managing the end of file boundary.
  604. *
  605. * This function also eliminates unnecessarily updating the file's
  606. * atime locally, as the NFS server sets the file's atime, and this
  607. * client must read the updated atime from the server back into its
  608. * cache.
  609. */
  610. ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
  611. unsigned long nr_segs, loff_t pos)
  612. {
  613. ssize_t retval = -EINVAL;
  614. struct file *file = iocb->ki_filp;
  615. struct address_space *mapping = file->f_mapping;
  616. /* XXX: temporary */
  617. const char __user *buf = iov[0].iov_base;
  618. size_t count = iov[0].iov_len;
  619. dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
  620. file->f_path.dentry->d_parent->d_name.name,
  621. file->f_path.dentry->d_name.name,
  622. (unsigned long) count, (long long) pos);
  623. if (nr_segs != 1)
  624. return -EINVAL;
  625. if (count < 0)
  626. goto out;
  627. retval = -EFAULT;
  628. if (!access_ok(VERIFY_WRITE, buf, count))
  629. goto out;
  630. retval = 0;
  631. if (!count)
  632. goto out;
  633. retval = nfs_sync_mapping(mapping);
  634. if (retval)
  635. goto out;
  636. retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos);
  637. if (retval > 0)
  638. iocb->ki_pos = pos + retval;
  639. out:
  640. return retval;
  641. }
  642. /**
  643. * nfs_file_direct_write - file direct write operation for NFS files
  644. * @iocb: target I/O control block
  645. * @iov: vector of user buffers from which to write data
  646. * @nr_segs: size of iov vector
  647. * @pos: byte offset in file where writing starts
  648. *
  649. * We use this function for direct writes instead of calling
  650. * generic_file_aio_write() in order to avoid taking the inode
  651. * semaphore and updating the i_size. The NFS server will set
  652. * the new i_size and this client must read the updated size
  653. * back into its cache. We let the server do generic write
  654. * parameter checking and report problems.
  655. *
  656. * We also avoid an unnecessary invocation of generic_osync_inode(),
  657. * as it is fairly meaningless to sync the metadata of an NFS file.
  658. *
  659. * We eliminate local atime updates, see direct read above.
  660. *
  661. * We avoid unnecessary page cache invalidations for normal cached
  662. * readers of this file.
  663. *
  664. * Note that O_APPEND is not supported for NFS direct writes, as there
  665. * is no atomic O_APPEND write facility in the NFS protocol.
  666. */
  667. ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
  668. unsigned long nr_segs, loff_t pos)
  669. {
  670. ssize_t retval;
  671. struct file *file = iocb->ki_filp;
  672. struct address_space *mapping = file->f_mapping;
  673. /* XXX: temporary */
  674. const char __user *buf = iov[0].iov_base;
  675. size_t count = iov[0].iov_len;
  676. dprintk("nfs: direct write(%s/%s, %lu@%Ld)\n",
  677. file->f_path.dentry->d_parent->d_name.name,
  678. file->f_path.dentry->d_name.name,
  679. (unsigned long) count, (long long) pos);
  680. if (nr_segs != 1)
  681. return -EINVAL;
  682. retval = generic_write_checks(file, &pos, &count, 0);
  683. if (retval)
  684. goto out;
  685. retval = -EINVAL;
  686. if ((ssize_t) count < 0)
  687. goto out;
  688. retval = 0;
  689. if (!count)
  690. goto out;
  691. retval = -EFAULT;
  692. if (!access_ok(VERIFY_READ, buf, count))
  693. goto out;
  694. retval = nfs_sync_mapping(mapping);
  695. if (retval)
  696. goto out;
  697. retval = nfs_direct_write(iocb, (unsigned long) buf, count, pos);
  698. if (retval > 0)
  699. iocb->ki_pos = pos + retval;
  700. out:
  701. return retval;
  702. }
  703. /**
  704. * nfs_init_directcache - create a slab cache for nfs_direct_req structures
  705. *
  706. */
  707. int __init nfs_init_directcache(void)
  708. {
  709. nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
  710. sizeof(struct nfs_direct_req),
  711. 0, (SLAB_RECLAIM_ACCOUNT|
  712. SLAB_MEM_SPREAD),
  713. NULL, NULL);
  714. if (nfs_direct_cachep == NULL)
  715. return -ENOMEM;
  716. return 0;
  717. }
  718. /**
  719. * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
  720. *
  721. */
  722. void nfs_destroy_directcache(void)
  723. {
  724. kmem_cache_destroy(nfs_direct_cachep);
  725. }