rdwr.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* Storage object read/write
  3. *
  4. * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. */
  7. #include <linux/mount.h>
  8. #include <linux/slab.h>
  9. #include <linux/file.h>
  10. #include <linux/swap.h>
  11. #include "internal.h"
  12. /*
  13. * detect wake up events generated by the unlocking of pages in which we're
  14. * interested
  15. * - we use this to detect read completion of backing pages
  16. * - the caller holds the waitqueue lock
  17. */
  18. static int cachefiles_read_waiter(wait_queue_entry_t *wait, unsigned mode,
  19. int sync, void *_key)
  20. {
  21. struct cachefiles_one_read *monitor =
  22. container_of(wait, struct cachefiles_one_read, monitor);
  23. struct cachefiles_object *object;
  24. struct fscache_retrieval *op = monitor->op;
  25. struct wait_page_key *key = _key;
  26. struct page *page = wait->private;
  27. ASSERT(key);
  28. _enter("{%lu},%u,%d,{%p,%u}",
  29. monitor->netfs_page->index, mode, sync,
  30. key->page, key->bit_nr);
  31. if (key->page != page || key->bit_nr != PG_locked)
  32. return 0;
  33. _debug("--- monitor %p %lx ---", page, page->flags);
  34. if (!PageUptodate(page) && !PageError(page)) {
  35. /* unlocked, not uptodate and not erronous? */
  36. _debug("page probably truncated");
  37. }
  38. /* remove from the waitqueue */
  39. list_del(&wait->entry);
  40. /* move onto the action list and queue for FS-Cache thread pool */
  41. ASSERT(op);
  42. /* We need to temporarily bump the usage count as we don't own a ref
  43. * here otherwise cachefiles_read_copier() may free the op between the
  44. * monitor being enqueued on the op->to_do list and the op getting
  45. * enqueued on the work queue.
  46. */
  47. fscache_get_retrieval(op);
  48. object = container_of(op->op.object, struct cachefiles_object, fscache);
  49. spin_lock(&object->work_lock);
  50. list_add_tail(&monitor->op_link, &op->to_do);
  51. fscache_enqueue_retrieval(op);
  52. spin_unlock(&object->work_lock);
  53. fscache_put_retrieval(op);
  54. return 0;
  55. }
  56. /*
  57. * handle a probably truncated page
  58. * - check to see if the page is still relevant and reissue the read if
  59. * possible
  60. * - return -EIO on error, -ENODATA if the page is gone, -EINPROGRESS if we
  61. * must wait again and 0 if successful
  62. */
  63. static int cachefiles_read_reissue(struct cachefiles_object *object,
  64. struct cachefiles_one_read *monitor)
  65. {
  66. struct address_space *bmapping = d_backing_inode(object->backer)->i_mapping;
  67. struct page *backpage = monitor->back_page, *backpage2;
  68. int ret;
  69. _enter("{ino=%lx},{%lx,%lx}",
  70. d_backing_inode(object->backer)->i_ino,
  71. backpage->index, backpage->flags);
  72. /* skip if the page was truncated away completely */
  73. if (backpage->mapping != bmapping) {
  74. _leave(" = -ENODATA [mapping]");
  75. return -ENODATA;
  76. }
  77. backpage2 = find_get_page(bmapping, backpage->index);
  78. if (!backpage2) {
  79. _leave(" = -ENODATA [gone]");
  80. return -ENODATA;
  81. }
  82. if (backpage != backpage2) {
  83. put_page(backpage2);
  84. _leave(" = -ENODATA [different]");
  85. return -ENODATA;
  86. }
  87. /* the page is still there and we already have a ref on it, so we don't
  88. * need a second */
  89. put_page(backpage2);
  90. INIT_LIST_HEAD(&monitor->op_link);
  91. add_page_wait_queue(backpage, &monitor->monitor);
  92. if (trylock_page(backpage)) {
  93. ret = -EIO;
  94. if (PageError(backpage))
  95. goto unlock_discard;
  96. ret = 0;
  97. if (PageUptodate(backpage))
  98. goto unlock_discard;
  99. _debug("reissue read");
  100. ret = bmapping->a_ops->readpage(NULL, backpage);
  101. if (ret < 0)
  102. goto discard;
  103. }
  104. /* but the page may have been read before the monitor was installed, so
  105. * the monitor may miss the event - so we have to ensure that we do get
  106. * one in such a case */
  107. if (trylock_page(backpage)) {
  108. _debug("jumpstart %p {%lx}", backpage, backpage->flags);
  109. unlock_page(backpage);
  110. }
  111. /* it'll reappear on the todo list */
  112. _leave(" = -EINPROGRESS");
  113. return -EINPROGRESS;
  114. unlock_discard:
  115. unlock_page(backpage);
  116. discard:
  117. spin_lock_irq(&object->work_lock);
  118. list_del(&monitor->op_link);
  119. spin_unlock_irq(&object->work_lock);
  120. _leave(" = %d", ret);
  121. return ret;
  122. }
  123. /*
  124. * copy data from backing pages to netfs pages to complete a read operation
  125. * - driven by FS-Cache's thread pool
  126. */
  127. static void cachefiles_read_copier(struct fscache_operation *_op)
  128. {
  129. struct cachefiles_one_read *monitor;
  130. struct cachefiles_object *object;
  131. struct fscache_retrieval *op;
  132. int error, max;
  133. op = container_of(_op, struct fscache_retrieval, op);
  134. object = container_of(op->op.object,
  135. struct cachefiles_object, fscache);
  136. _enter("{ino=%lu}", d_backing_inode(object->backer)->i_ino);
  137. max = 8;
  138. spin_lock_irq(&object->work_lock);
  139. while (!list_empty(&op->to_do)) {
  140. monitor = list_entry(op->to_do.next,
  141. struct cachefiles_one_read, op_link);
  142. list_del(&monitor->op_link);
  143. spin_unlock_irq(&object->work_lock);
  144. _debug("- copy {%lu}", monitor->back_page->index);
  145. recheck:
  146. if (test_bit(FSCACHE_COOKIE_INVALIDATING,
  147. &object->fscache.cookie->flags)) {
  148. error = -ESTALE;
  149. } else if (PageUptodate(monitor->back_page)) {
  150. copy_highpage(monitor->netfs_page, monitor->back_page);
  151. fscache_mark_page_cached(monitor->op,
  152. monitor->netfs_page);
  153. error = 0;
  154. } else if (!PageError(monitor->back_page)) {
  155. /* the page has probably been truncated */
  156. error = cachefiles_read_reissue(object, monitor);
  157. if (error == -EINPROGRESS)
  158. goto next;
  159. goto recheck;
  160. } else {
  161. cachefiles_io_error_obj(
  162. object,
  163. "Readpage failed on backing file %lx",
  164. (unsigned long) monitor->back_page->flags);
  165. error = -EIO;
  166. }
  167. put_page(monitor->back_page);
  168. fscache_end_io(op, monitor->netfs_page, error);
  169. put_page(monitor->netfs_page);
  170. fscache_retrieval_complete(op, 1);
  171. fscache_put_retrieval(op);
  172. kfree(monitor);
  173. next:
  174. /* let the thread pool have some air occasionally */
  175. max--;
  176. if (max < 0 || need_resched()) {
  177. if (!list_empty(&op->to_do))
  178. fscache_enqueue_retrieval(op);
  179. _leave(" [maxed out]");
  180. return;
  181. }
  182. spin_lock_irq(&object->work_lock);
  183. }
  184. spin_unlock_irq(&object->work_lock);
  185. _leave("");
  186. }
  187. /*
  188. * read the corresponding page to the given set from the backing file
  189. * - an uncertain page is simply discarded, to be tried again another time
  190. */
  191. static int cachefiles_read_backing_file_one(struct cachefiles_object *object,
  192. struct fscache_retrieval *op,
  193. struct page *netpage)
  194. {
  195. struct cachefiles_one_read *monitor;
  196. struct address_space *bmapping;
  197. struct page *newpage, *backpage;
  198. int ret;
  199. _enter("");
  200. _debug("read back %p{%lu,%d}",
  201. netpage, netpage->index, page_count(netpage));
  202. monitor = kzalloc(sizeof(*monitor), cachefiles_gfp);
  203. if (!monitor)
  204. goto nomem;
  205. monitor->netfs_page = netpage;
  206. monitor->op = fscache_get_retrieval(op);
  207. init_waitqueue_func_entry(&monitor->monitor, cachefiles_read_waiter);
  208. /* attempt to get hold of the backing page */
  209. bmapping = d_backing_inode(object->backer)->i_mapping;
  210. newpage = NULL;
  211. for (;;) {
  212. backpage = find_get_page(bmapping, netpage->index);
  213. if (backpage)
  214. goto backing_page_already_present;
  215. if (!newpage) {
  216. newpage = __page_cache_alloc(cachefiles_gfp);
  217. if (!newpage)
  218. goto nomem_monitor;
  219. }
  220. ret = add_to_page_cache_lru(newpage, bmapping,
  221. netpage->index, cachefiles_gfp);
  222. if (ret == 0)
  223. goto installed_new_backing_page;
  224. if (ret != -EEXIST)
  225. goto nomem_page;
  226. }
  227. /* we've installed a new backing page, so now we need to start
  228. * it reading */
  229. installed_new_backing_page:
  230. _debug("- new %p", newpage);
  231. backpage = newpage;
  232. newpage = NULL;
  233. read_backing_page:
  234. ret = bmapping->a_ops->readpage(NULL, backpage);
  235. if (ret < 0)
  236. goto read_error;
  237. /* set the monitor to transfer the data across */
  238. monitor_backing_page:
  239. _debug("- monitor add");
  240. /* install the monitor */
  241. get_page(monitor->netfs_page);
  242. get_page(backpage);
  243. monitor->back_page = backpage;
  244. monitor->monitor.private = backpage;
  245. add_page_wait_queue(backpage, &monitor->monitor);
  246. monitor = NULL;
  247. /* but the page may have been read before the monitor was installed, so
  248. * the monitor may miss the event - so we have to ensure that we do get
  249. * one in such a case */
  250. if (trylock_page(backpage)) {
  251. _debug("jumpstart %p {%lx}", backpage, backpage->flags);
  252. unlock_page(backpage);
  253. }
  254. goto success;
  255. /* if the backing page is already present, it can be in one of
  256. * three states: read in progress, read failed or read okay */
  257. backing_page_already_present:
  258. _debug("- present");
  259. if (newpage) {
  260. put_page(newpage);
  261. newpage = NULL;
  262. }
  263. if (PageError(backpage))
  264. goto io_error;
  265. if (PageUptodate(backpage))
  266. goto backing_page_already_uptodate;
  267. if (!trylock_page(backpage))
  268. goto monitor_backing_page;
  269. _debug("read %p {%lx}", backpage, backpage->flags);
  270. goto read_backing_page;
  271. /* the backing page is already up to date, attach the netfs
  272. * page to the pagecache and LRU and copy the data across */
  273. backing_page_already_uptodate:
  274. _debug("- uptodate");
  275. fscache_mark_page_cached(op, netpage);
  276. copy_highpage(netpage, backpage);
  277. fscache_end_io(op, netpage, 0);
  278. fscache_retrieval_complete(op, 1);
  279. success:
  280. _debug("success");
  281. ret = 0;
  282. out:
  283. if (backpage)
  284. put_page(backpage);
  285. if (monitor) {
  286. fscache_put_retrieval(monitor->op);
  287. kfree(monitor);
  288. }
  289. _leave(" = %d", ret);
  290. return ret;
  291. read_error:
  292. _debug("read error %d", ret);
  293. if (ret == -ENOMEM) {
  294. fscache_retrieval_complete(op, 1);
  295. goto out;
  296. }
  297. io_error:
  298. cachefiles_io_error_obj(object, "Page read error on backing file");
  299. fscache_retrieval_complete(op, 1);
  300. ret = -ENOBUFS;
  301. goto out;
  302. nomem_page:
  303. put_page(newpage);
  304. nomem_monitor:
  305. fscache_put_retrieval(monitor->op);
  306. kfree(monitor);
  307. nomem:
  308. fscache_retrieval_complete(op, 1);
  309. _leave(" = -ENOMEM");
  310. return -ENOMEM;
  311. }
  312. /*
  313. * read a page from the cache or allocate a block in which to store it
  314. * - cache withdrawal is prevented by the caller
  315. * - returns -EINTR if interrupted
  316. * - returns -ENOMEM if ran out of memory
  317. * - returns -ENOBUFS if no buffers can be made available
  318. * - returns -ENOBUFS if page is beyond EOF
  319. * - if the page is backed by a block in the cache:
  320. * - a read will be started which will call the callback on completion
  321. * - 0 will be returned
  322. * - else if the page is unbacked:
  323. * - the metadata will be retained
  324. * - -ENODATA will be returned
  325. */
  326. int cachefiles_read_or_alloc_page(struct fscache_retrieval *op,
  327. struct page *page,
  328. gfp_t gfp)
  329. {
  330. struct cachefiles_object *object;
  331. struct cachefiles_cache *cache;
  332. struct inode *inode;
  333. sector_t block;
  334. unsigned shift;
  335. int ret, ret2;
  336. object = container_of(op->op.object,
  337. struct cachefiles_object, fscache);
  338. cache = container_of(object->fscache.cache,
  339. struct cachefiles_cache, cache);
  340. _enter("{%p},{%lx},,,", object, page->index);
  341. if (!object->backer)
  342. goto enobufs;
  343. inode = d_backing_inode(object->backer);
  344. ASSERT(S_ISREG(inode->i_mode));
  345. /* calculate the shift required to use bmap */
  346. shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
  347. op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
  348. op->op.flags |= FSCACHE_OP_ASYNC;
  349. op->op.processor = cachefiles_read_copier;
  350. /* we assume the absence or presence of the first block is a good
  351. * enough indication for the page as a whole
  352. * - TODO: don't use bmap() for this as it is _not_ actually good
  353. * enough for this as it doesn't indicate errors, but it's all we've
  354. * got for the moment
  355. */
  356. block = page->index;
  357. block <<= shift;
  358. ret2 = bmap(inode, &block);
  359. ASSERT(ret2 == 0);
  360. _debug("%llx -> %llx",
  361. (unsigned long long) (page->index << shift),
  362. (unsigned long long) block);
  363. if (block) {
  364. /* submit the apparently valid page to the backing fs to be
  365. * read from disk */
  366. ret = cachefiles_read_backing_file_one(object, op, page);
  367. } else if (cachefiles_has_space(cache, 0, 1) == 0) {
  368. /* there's space in the cache we can use */
  369. fscache_mark_page_cached(op, page);
  370. fscache_retrieval_complete(op, 1);
  371. ret = -ENODATA;
  372. } else {
  373. goto enobufs;
  374. }
  375. _leave(" = %d", ret);
  376. return ret;
  377. enobufs:
  378. fscache_retrieval_complete(op, 1);
  379. _leave(" = -ENOBUFS");
  380. return -ENOBUFS;
  381. }
  382. /*
  383. * read the corresponding pages to the given set from the backing file
  384. * - any uncertain pages are simply discarded, to be tried again another time
  385. */
  386. static int cachefiles_read_backing_file(struct cachefiles_object *object,
  387. struct fscache_retrieval *op,
  388. struct list_head *list)
  389. {
  390. struct cachefiles_one_read *monitor = NULL;
  391. struct address_space *bmapping = d_backing_inode(object->backer)->i_mapping;
  392. struct page *newpage = NULL, *netpage, *_n, *backpage = NULL;
  393. int ret = 0;
  394. _enter("");
  395. list_for_each_entry_safe(netpage, _n, list, lru) {
  396. list_del(&netpage->lru);
  397. _debug("read back %p{%lu,%d}",
  398. netpage, netpage->index, page_count(netpage));
  399. if (!monitor) {
  400. monitor = kzalloc(sizeof(*monitor), cachefiles_gfp);
  401. if (!monitor)
  402. goto nomem;
  403. monitor->op = fscache_get_retrieval(op);
  404. init_waitqueue_func_entry(&monitor->monitor,
  405. cachefiles_read_waiter);
  406. }
  407. for (;;) {
  408. backpage = find_get_page(bmapping, netpage->index);
  409. if (backpage)
  410. goto backing_page_already_present;
  411. if (!newpage) {
  412. newpage = __page_cache_alloc(cachefiles_gfp);
  413. if (!newpage)
  414. goto nomem;
  415. }
  416. ret = add_to_page_cache_lru(newpage, bmapping,
  417. netpage->index,
  418. cachefiles_gfp);
  419. if (ret == 0)
  420. goto installed_new_backing_page;
  421. if (ret != -EEXIST)
  422. goto nomem;
  423. }
  424. /* we've installed a new backing page, so now we need
  425. * to start it reading */
  426. installed_new_backing_page:
  427. _debug("- new %p", newpage);
  428. backpage = newpage;
  429. newpage = NULL;
  430. reread_backing_page:
  431. ret = bmapping->a_ops->readpage(NULL, backpage);
  432. if (ret < 0)
  433. goto read_error;
  434. /* add the netfs page to the pagecache and LRU, and set the
  435. * monitor to transfer the data across */
  436. monitor_backing_page:
  437. _debug("- monitor add");
  438. ret = add_to_page_cache_lru(netpage, op->mapping,
  439. netpage->index, cachefiles_gfp);
  440. if (ret < 0) {
  441. if (ret == -EEXIST) {
  442. put_page(backpage);
  443. backpage = NULL;
  444. put_page(netpage);
  445. netpage = NULL;
  446. fscache_retrieval_complete(op, 1);
  447. continue;
  448. }
  449. goto nomem;
  450. }
  451. /* install a monitor */
  452. get_page(netpage);
  453. monitor->netfs_page = netpage;
  454. get_page(backpage);
  455. monitor->back_page = backpage;
  456. monitor->monitor.private = backpage;
  457. add_page_wait_queue(backpage, &monitor->monitor);
  458. monitor = NULL;
  459. /* but the page may have been read before the monitor was
  460. * installed, so the monitor may miss the event - so we have to
  461. * ensure that we do get one in such a case */
  462. if (trylock_page(backpage)) {
  463. _debug("2unlock %p {%lx}", backpage, backpage->flags);
  464. unlock_page(backpage);
  465. }
  466. put_page(backpage);
  467. backpage = NULL;
  468. put_page(netpage);
  469. netpage = NULL;
  470. continue;
  471. /* if the backing page is already present, it can be in one of
  472. * three states: read in progress, read failed or read okay */
  473. backing_page_already_present:
  474. _debug("- present %p", backpage);
  475. if (PageError(backpage))
  476. goto io_error;
  477. if (PageUptodate(backpage))
  478. goto backing_page_already_uptodate;
  479. _debug("- not ready %p{%lx}", backpage, backpage->flags);
  480. if (!trylock_page(backpage))
  481. goto monitor_backing_page;
  482. if (PageError(backpage)) {
  483. _debug("error %lx", backpage->flags);
  484. unlock_page(backpage);
  485. goto io_error;
  486. }
  487. if (PageUptodate(backpage))
  488. goto backing_page_already_uptodate_unlock;
  489. /* we've locked a page that's neither up to date nor erroneous,
  490. * so we need to attempt to read it again */
  491. goto reread_backing_page;
  492. /* the backing page is already up to date, attach the netfs
  493. * page to the pagecache and LRU and copy the data across */
  494. backing_page_already_uptodate_unlock:
  495. _debug("uptodate %lx", backpage->flags);
  496. unlock_page(backpage);
  497. backing_page_already_uptodate:
  498. _debug("- uptodate");
  499. ret = add_to_page_cache_lru(netpage, op->mapping,
  500. netpage->index, cachefiles_gfp);
  501. if (ret < 0) {
  502. if (ret == -EEXIST) {
  503. put_page(backpage);
  504. backpage = NULL;
  505. put_page(netpage);
  506. netpage = NULL;
  507. fscache_retrieval_complete(op, 1);
  508. continue;
  509. }
  510. goto nomem;
  511. }
  512. copy_highpage(netpage, backpage);
  513. put_page(backpage);
  514. backpage = NULL;
  515. fscache_mark_page_cached(op, netpage);
  516. /* the netpage is unlocked and marked up to date here */
  517. fscache_end_io(op, netpage, 0);
  518. put_page(netpage);
  519. netpage = NULL;
  520. fscache_retrieval_complete(op, 1);
  521. continue;
  522. }
  523. netpage = NULL;
  524. _debug("out");
  525. out:
  526. /* tidy up */
  527. if (newpage)
  528. put_page(newpage);
  529. if (netpage)
  530. put_page(netpage);
  531. if (backpage)
  532. put_page(backpage);
  533. if (monitor) {
  534. fscache_put_retrieval(op);
  535. kfree(monitor);
  536. }
  537. list_for_each_entry_safe(netpage, _n, list, lru) {
  538. list_del(&netpage->lru);
  539. put_page(netpage);
  540. fscache_retrieval_complete(op, 1);
  541. }
  542. _leave(" = %d", ret);
  543. return ret;
  544. nomem:
  545. _debug("nomem");
  546. ret = -ENOMEM;
  547. goto record_page_complete;
  548. read_error:
  549. _debug("read error %d", ret);
  550. if (ret == -ENOMEM)
  551. goto record_page_complete;
  552. io_error:
  553. cachefiles_io_error_obj(object, "Page read error on backing file");
  554. ret = -ENOBUFS;
  555. record_page_complete:
  556. fscache_retrieval_complete(op, 1);
  557. goto out;
  558. }
  559. /*
  560. * read a list of pages from the cache or allocate blocks in which to store
  561. * them
  562. */
  563. int cachefiles_read_or_alloc_pages(struct fscache_retrieval *op,
  564. struct list_head *pages,
  565. unsigned *nr_pages,
  566. gfp_t gfp)
  567. {
  568. struct cachefiles_object *object;
  569. struct cachefiles_cache *cache;
  570. struct list_head backpages;
  571. struct pagevec pagevec;
  572. struct inode *inode;
  573. struct page *page, *_n;
  574. unsigned shift, nrbackpages;
  575. int ret, ret2, space;
  576. object = container_of(op->op.object,
  577. struct cachefiles_object, fscache);
  578. cache = container_of(object->fscache.cache,
  579. struct cachefiles_cache, cache);
  580. _enter("{OBJ%x,%d},,%d,,",
  581. object->fscache.debug_id, atomic_read(&op->op.usage),
  582. *nr_pages);
  583. if (!object->backer)
  584. goto all_enobufs;
  585. space = 1;
  586. if (cachefiles_has_space(cache, 0, *nr_pages) < 0)
  587. space = 0;
  588. inode = d_backing_inode(object->backer);
  589. ASSERT(S_ISREG(inode->i_mode));
  590. /* calculate the shift required to use bmap */
  591. shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
  592. pagevec_init(&pagevec);
  593. op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
  594. op->op.flags |= FSCACHE_OP_ASYNC;
  595. op->op.processor = cachefiles_read_copier;
  596. INIT_LIST_HEAD(&backpages);
  597. nrbackpages = 0;
  598. ret = space ? -ENODATA : -ENOBUFS;
  599. list_for_each_entry_safe(page, _n, pages, lru) {
  600. sector_t block;
  601. /* we assume the absence or presence of the first block is a
  602. * good enough indication for the page as a whole
  603. * - TODO: don't use bmap() for this as it is _not_ actually
  604. * good enough for this as it doesn't indicate errors, but
  605. * it's all we've got for the moment
  606. */
  607. block = page->index;
  608. block <<= shift;
  609. ret2 = bmap(inode, &block);
  610. ASSERT(ret2 == 0);
  611. _debug("%llx -> %llx",
  612. (unsigned long long) (page->index << shift),
  613. (unsigned long long) block);
  614. if (block) {
  615. /* we have data - add it to the list to give to the
  616. * backing fs */
  617. list_move(&page->lru, &backpages);
  618. (*nr_pages)--;
  619. nrbackpages++;
  620. } else if (space && pagevec_add(&pagevec, page) == 0) {
  621. fscache_mark_pages_cached(op, &pagevec);
  622. fscache_retrieval_complete(op, 1);
  623. ret = -ENODATA;
  624. } else {
  625. fscache_retrieval_complete(op, 1);
  626. }
  627. }
  628. if (pagevec_count(&pagevec) > 0)
  629. fscache_mark_pages_cached(op, &pagevec);
  630. if (list_empty(pages))
  631. ret = 0;
  632. /* submit the apparently valid pages to the backing fs to be read from
  633. * disk */
  634. if (nrbackpages > 0) {
  635. ret2 = cachefiles_read_backing_file(object, op, &backpages);
  636. if (ret2 == -ENOMEM || ret2 == -EINTR)
  637. ret = ret2;
  638. }
  639. _leave(" = %d [nr=%u%s]",
  640. ret, *nr_pages, list_empty(pages) ? " empty" : "");
  641. return ret;
  642. all_enobufs:
  643. fscache_retrieval_complete(op, *nr_pages);
  644. return -ENOBUFS;
  645. }
  646. /*
  647. * allocate a block in the cache in which to store a page
  648. * - cache withdrawal is prevented by the caller
  649. * - returns -EINTR if interrupted
  650. * - returns -ENOMEM if ran out of memory
  651. * - returns -ENOBUFS if no buffers can be made available
  652. * - returns -ENOBUFS if page is beyond EOF
  653. * - otherwise:
  654. * - the metadata will be retained
  655. * - 0 will be returned
  656. */
  657. int cachefiles_allocate_page(struct fscache_retrieval *op,
  658. struct page *page,
  659. gfp_t gfp)
  660. {
  661. struct cachefiles_object *object;
  662. struct cachefiles_cache *cache;
  663. int ret;
  664. object = container_of(op->op.object,
  665. struct cachefiles_object, fscache);
  666. cache = container_of(object->fscache.cache,
  667. struct cachefiles_cache, cache);
  668. _enter("%p,{%lx},", object, page->index);
  669. ret = cachefiles_has_space(cache, 0, 1);
  670. if (ret == 0)
  671. fscache_mark_page_cached(op, page);
  672. else
  673. ret = -ENOBUFS;
  674. fscache_retrieval_complete(op, 1);
  675. _leave(" = %d", ret);
  676. return ret;
  677. }
  678. /*
  679. * allocate blocks in the cache in which to store a set of pages
  680. * - cache withdrawal is prevented by the caller
  681. * - returns -EINTR if interrupted
  682. * - returns -ENOMEM if ran out of memory
  683. * - returns -ENOBUFS if some buffers couldn't be made available
  684. * - returns -ENOBUFS if some pages are beyond EOF
  685. * - otherwise:
  686. * - -ENODATA will be returned
  687. * - metadata will be retained for any page marked
  688. */
  689. int cachefiles_allocate_pages(struct fscache_retrieval *op,
  690. struct list_head *pages,
  691. unsigned *nr_pages,
  692. gfp_t gfp)
  693. {
  694. struct cachefiles_object *object;
  695. struct cachefiles_cache *cache;
  696. struct pagevec pagevec;
  697. struct page *page;
  698. int ret;
  699. object = container_of(op->op.object,
  700. struct cachefiles_object, fscache);
  701. cache = container_of(object->fscache.cache,
  702. struct cachefiles_cache, cache);
  703. _enter("%p,,,%d,", object, *nr_pages);
  704. ret = cachefiles_has_space(cache, 0, *nr_pages);
  705. if (ret == 0) {
  706. pagevec_init(&pagevec);
  707. list_for_each_entry(page, pages, lru) {
  708. if (pagevec_add(&pagevec, page) == 0)
  709. fscache_mark_pages_cached(op, &pagevec);
  710. }
  711. if (pagevec_count(&pagevec) > 0)
  712. fscache_mark_pages_cached(op, &pagevec);
  713. ret = -ENODATA;
  714. } else {
  715. ret = -ENOBUFS;
  716. }
  717. fscache_retrieval_complete(op, *nr_pages);
  718. _leave(" = %d", ret);
  719. return ret;
  720. }
  721. /*
  722. * request a page be stored in the cache
  723. * - cache withdrawal is prevented by the caller
  724. * - this request may be ignored if there's no cache block available, in which
  725. * case -ENOBUFS will be returned
  726. * - if the op is in progress, 0 will be returned
  727. */
  728. int cachefiles_write_page(struct fscache_storage *op, struct page *page)
  729. {
  730. struct cachefiles_object *object;
  731. struct cachefiles_cache *cache;
  732. struct file *file;
  733. struct path path;
  734. loff_t pos, eof;
  735. size_t len;
  736. void *data;
  737. int ret = -ENOBUFS;
  738. ASSERT(op != NULL);
  739. ASSERT(page != NULL);
  740. object = container_of(op->op.object,
  741. struct cachefiles_object, fscache);
  742. _enter("%p,%p{%lx},,,", object, page, page->index);
  743. if (!object->backer) {
  744. _leave(" = -ENOBUFS");
  745. return -ENOBUFS;
  746. }
  747. ASSERT(d_is_reg(object->backer));
  748. cache = container_of(object->fscache.cache,
  749. struct cachefiles_cache, cache);
  750. pos = (loff_t)page->index << PAGE_SHIFT;
  751. /* We mustn't write more data than we have, so we have to beware of a
  752. * partial page at EOF.
  753. */
  754. eof = object->fscache.store_limit_l;
  755. if (pos >= eof)
  756. goto error;
  757. /* write the page to the backing filesystem and let it store it in its
  758. * own time */
  759. path.mnt = cache->mnt;
  760. path.dentry = object->backer;
  761. file = dentry_open(&path, O_RDWR | O_LARGEFILE, cache->cache_cred);
  762. if (IS_ERR(file)) {
  763. ret = PTR_ERR(file);
  764. goto error_2;
  765. }
  766. len = PAGE_SIZE;
  767. if (eof & ~PAGE_MASK) {
  768. if (eof - pos < PAGE_SIZE) {
  769. _debug("cut short %llx to %llx",
  770. pos, eof);
  771. len = eof - pos;
  772. ASSERTCMP(pos + len, ==, eof);
  773. }
  774. }
  775. data = kmap(page);
  776. ret = kernel_write(file, data, len, &pos);
  777. kunmap(page);
  778. fput(file);
  779. if (ret != len)
  780. goto error_eio;
  781. _leave(" = 0");
  782. return 0;
  783. error_eio:
  784. ret = -EIO;
  785. error_2:
  786. if (ret == -EIO)
  787. cachefiles_io_error_obj(object,
  788. "Write page to backing file failed");
  789. error:
  790. _leave(" = -ENOBUFS [%d]", ret);
  791. return -ENOBUFS;
  792. }
  793. /*
  794. * detach a backing block from a page
  795. * - cache withdrawal is prevented by the caller
  796. */
  797. void cachefiles_uncache_page(struct fscache_object *_object, struct page *page)
  798. __releases(&object->fscache.cookie->lock)
  799. {
  800. struct cachefiles_object *object;
  801. object = container_of(_object, struct cachefiles_object, fscache);
  802. _enter("%p,{%lu}", object, page->index);
  803. spin_unlock(&object->fscache.cookie->lock);
  804. }