cookie.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* netfs cookie management
  3. *
  4. * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. *
  7. * See Documentation/filesystems/caching/netfs-api.rst for more information on
  8. * the netfs API.
  9. */
  10. #define FSCACHE_DEBUG_LEVEL COOKIE
  11. #include <linux/module.h>
  12. #include <linux/slab.h>
  13. #include "internal.h"
  14. struct kmem_cache *fscache_cookie_jar;
  15. static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  16. #define fscache_cookie_hash_shift 15
  17. static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  18. static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  19. loff_t object_size);
  20. static int fscache_alloc_object(struct fscache_cache *cache,
  21. struct fscache_cookie *cookie);
  22. static int fscache_attach_object(struct fscache_cookie *cookie,
  23. struct fscache_object *object);
  24. static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  25. {
  26. struct hlist_node *object;
  27. const u8 *k;
  28. unsigned loop;
  29. pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  30. prefix, cookie, cookie->parent, cookie->flags,
  31. atomic_read(&cookie->n_children),
  32. atomic_read(&cookie->n_active));
  33. pr_err("%c-cookie d=%p n=%p\n",
  34. prefix, cookie->def, cookie->netfs_data);
  35. object = READ_ONCE(cookie->backing_objects.first);
  36. if (object)
  37. pr_err("%c-cookie o=%p\n",
  38. prefix, hlist_entry(object, struct fscache_object, cookie_link));
  39. pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  40. k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  41. cookie->inline_key : cookie->key;
  42. for (loop = 0; loop < cookie->key_len; loop++)
  43. pr_cont("%02x", k[loop]);
  44. pr_cont("'\n");
  45. }
  46. void fscache_free_cookie(struct fscache_cookie *cookie)
  47. {
  48. if (cookie) {
  49. BUG_ON(!hlist_empty(&cookie->backing_objects));
  50. if (cookie->aux_len > sizeof(cookie->inline_aux))
  51. kfree(cookie->aux);
  52. if (cookie->key_len > sizeof(cookie->inline_key))
  53. kfree(cookie->key);
  54. kmem_cache_free(fscache_cookie_jar, cookie);
  55. }
  56. }
  57. /*
  58. * Set the index key in a cookie. The cookie struct has space for a 16-byte
  59. * key plus length and hash, but if that's not big enough, it's instead a
  60. * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  61. * the key data.
  62. */
  63. static int fscache_set_key(struct fscache_cookie *cookie,
  64. const void *index_key, size_t index_key_len)
  65. {
  66. u32 *buf;
  67. int bufs;
  68. bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  69. if (index_key_len > sizeof(cookie->inline_key)) {
  70. buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  71. if (!buf)
  72. return -ENOMEM;
  73. cookie->key = buf;
  74. } else {
  75. buf = (u32 *)cookie->inline_key;
  76. }
  77. memcpy(buf, index_key, index_key_len);
  78. cookie->key_hash = fscache_hash(0, buf, bufs);
  79. return 0;
  80. }
  81. static long fscache_compare_cookie(const struct fscache_cookie *a,
  82. const struct fscache_cookie *b)
  83. {
  84. const void *ka, *kb;
  85. if (a->key_hash != b->key_hash)
  86. return (long)a->key_hash - (long)b->key_hash;
  87. if (a->parent != b->parent)
  88. return (long)a->parent - (long)b->parent;
  89. if (a->key_len != b->key_len)
  90. return (long)a->key_len - (long)b->key_len;
  91. if (a->type != b->type)
  92. return (long)a->type - (long)b->type;
  93. if (a->key_len <= sizeof(a->inline_key)) {
  94. ka = &a->inline_key;
  95. kb = &b->inline_key;
  96. } else {
  97. ka = a->key;
  98. kb = b->key;
  99. }
  100. return memcmp(ka, kb, a->key_len);
  101. }
  102. /*
  103. * Allocate a cookie.
  104. */
  105. struct fscache_cookie *fscache_alloc_cookie(
  106. struct fscache_cookie *parent,
  107. const struct fscache_cookie_def *def,
  108. const void *index_key, size_t index_key_len,
  109. const void *aux_data, size_t aux_data_len,
  110. void *netfs_data,
  111. loff_t object_size)
  112. {
  113. struct fscache_cookie *cookie;
  114. /* allocate and initialise a cookie */
  115. cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
  116. if (!cookie)
  117. return NULL;
  118. cookie->key_len = index_key_len;
  119. cookie->aux_len = aux_data_len;
  120. if (fscache_set_key(cookie, index_key, index_key_len) < 0)
  121. goto nomem;
  122. if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
  123. memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
  124. } else {
  125. cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
  126. if (!cookie->aux)
  127. goto nomem;
  128. }
  129. atomic_set(&cookie->usage, 1);
  130. atomic_set(&cookie->n_children, 0);
  131. /* We keep the active count elevated until relinquishment to prevent an
  132. * attempt to wake up every time the object operations queue quiesces.
  133. */
  134. atomic_set(&cookie->n_active, 1);
  135. cookie->def = def;
  136. cookie->parent = parent;
  137. cookie->netfs_data = netfs_data;
  138. cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
  139. cookie->type = def->type;
  140. spin_lock_init(&cookie->lock);
  141. spin_lock_init(&cookie->stores_lock);
  142. INIT_HLIST_HEAD(&cookie->backing_objects);
  143. /* radix tree insertion won't use the preallocation pool unless it's
  144. * told it may not wait */
  145. INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
  146. return cookie;
  147. nomem:
  148. fscache_free_cookie(cookie);
  149. return NULL;
  150. }
  151. /*
  152. * Attempt to insert the new cookie into the hash. If there's a collision, we
  153. * return the old cookie if it's not in use and an error otherwise.
  154. */
  155. struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
  156. {
  157. struct fscache_cookie *cursor;
  158. struct hlist_bl_head *h;
  159. struct hlist_bl_node *p;
  160. unsigned int bucket;
  161. bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
  162. h = &fscache_cookie_hash[bucket];
  163. hlist_bl_lock(h);
  164. hlist_bl_for_each_entry(cursor, p, h, hash_link) {
  165. if (fscache_compare_cookie(candidate, cursor) == 0)
  166. goto collision;
  167. }
  168. __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
  169. fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
  170. atomic_inc(&candidate->parent->n_children);
  171. hlist_bl_add_head(&candidate->hash_link, h);
  172. hlist_bl_unlock(h);
  173. return candidate;
  174. collision:
  175. if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
  176. trace_fscache_cookie(cursor, fscache_cookie_collision,
  177. atomic_read(&cursor->usage));
  178. pr_err("Duplicate cookie detected\n");
  179. fscache_print_cookie(cursor, 'O');
  180. fscache_print_cookie(candidate, 'N');
  181. hlist_bl_unlock(h);
  182. return NULL;
  183. }
  184. fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
  185. hlist_bl_unlock(h);
  186. return cursor;
  187. }
  188. /*
  189. * request a cookie to represent an object (index, datafile, xattr, etc)
  190. * - parent specifies the parent object
  191. * - the top level index cookie for each netfs is stored in the fscache_netfs
  192. * struct upon registration
  193. * - def points to the definition
  194. * - the netfs_data will be passed to the functions pointed to in *def
  195. * - all attached caches will be searched to see if they contain this object
  196. * - index objects aren't stored on disk until there's a dependent file that
  197. * needs storing
  198. * - other objects are stored in a selected cache immediately, and all the
  199. * indices forming the path to it are instantiated if necessary
  200. * - we never let on to the netfs about errors
  201. * - we may set a negative cookie pointer, but that's okay
  202. */
  203. struct fscache_cookie *__fscache_acquire_cookie(
  204. struct fscache_cookie *parent,
  205. const struct fscache_cookie_def *def,
  206. const void *index_key, size_t index_key_len,
  207. const void *aux_data, size_t aux_data_len,
  208. void *netfs_data,
  209. loff_t object_size,
  210. bool enable)
  211. {
  212. struct fscache_cookie *candidate, *cookie;
  213. BUG_ON(!def);
  214. _enter("{%s},{%s},%p,%u",
  215. parent ? (char *) parent->def->name : "<no-parent>",
  216. def->name, netfs_data, enable);
  217. if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
  218. return NULL;
  219. if (!aux_data || !aux_data_len) {
  220. aux_data = NULL;
  221. aux_data_len = 0;
  222. }
  223. fscache_stat(&fscache_n_acquires);
  224. /* if there's no parent cookie, then we don't create one here either */
  225. if (!parent) {
  226. fscache_stat(&fscache_n_acquires_null);
  227. _leave(" [no parent]");
  228. return NULL;
  229. }
  230. /* validate the definition */
  231. BUG_ON(!def->name[0]);
  232. BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
  233. parent->type != FSCACHE_COOKIE_TYPE_INDEX);
  234. candidate = fscache_alloc_cookie(parent, def,
  235. index_key, index_key_len,
  236. aux_data, aux_data_len,
  237. netfs_data, object_size);
  238. if (!candidate) {
  239. fscache_stat(&fscache_n_acquires_oom);
  240. _leave(" [ENOMEM]");
  241. return NULL;
  242. }
  243. cookie = fscache_hash_cookie(candidate);
  244. if (!cookie) {
  245. trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
  246. goto out;
  247. }
  248. if (cookie == candidate)
  249. candidate = NULL;
  250. switch (cookie->type) {
  251. case FSCACHE_COOKIE_TYPE_INDEX:
  252. fscache_stat(&fscache_n_cookie_index);
  253. break;
  254. case FSCACHE_COOKIE_TYPE_DATAFILE:
  255. fscache_stat(&fscache_n_cookie_data);
  256. break;
  257. default:
  258. fscache_stat(&fscache_n_cookie_special);
  259. break;
  260. }
  261. trace_fscache_acquire(cookie);
  262. if (enable) {
  263. /* if the object is an index then we need do nothing more here
  264. * - we create indices on disk when we need them as an index
  265. * may exist in multiple caches */
  266. if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
  267. if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
  268. set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
  269. } else {
  270. atomic_dec(&parent->n_children);
  271. fscache_cookie_put(cookie,
  272. fscache_cookie_put_acquire_nobufs);
  273. fscache_stat(&fscache_n_acquires_nobufs);
  274. _leave(" = NULL");
  275. return NULL;
  276. }
  277. } else {
  278. set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
  279. }
  280. }
  281. fscache_stat(&fscache_n_acquires_ok);
  282. out:
  283. fscache_free_cookie(candidate);
  284. return cookie;
  285. }
  286. EXPORT_SYMBOL(__fscache_acquire_cookie);
  287. /*
  288. * Enable a cookie to permit it to accept new operations.
  289. */
  290. void __fscache_enable_cookie(struct fscache_cookie *cookie,
  291. const void *aux_data,
  292. loff_t object_size,
  293. bool (*can_enable)(void *data),
  294. void *data)
  295. {
  296. _enter("%p", cookie);
  297. trace_fscache_enable(cookie);
  298. wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
  299. TASK_UNINTERRUPTIBLE);
  300. fscache_update_aux(cookie, aux_data);
  301. if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
  302. goto out_unlock;
  303. if (can_enable && !can_enable(data)) {
  304. /* The netfs decided it didn't want to enable after all */
  305. } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
  306. /* Wait for outstanding disablement to complete */
  307. __fscache_wait_on_invalidate(cookie);
  308. if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
  309. set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
  310. } else {
  311. set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
  312. }
  313. out_unlock:
  314. clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
  315. wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
  316. }
  317. EXPORT_SYMBOL(__fscache_enable_cookie);
  318. /*
  319. * acquire a non-index cookie
  320. * - this must make sure the index chain is instantiated and instantiate the
  321. * object representation too
  322. */
  323. static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  324. loff_t object_size)
  325. {
  326. struct fscache_object *object;
  327. struct fscache_cache *cache;
  328. int ret;
  329. _enter("");
  330. set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
  331. /* now we need to see whether the backing objects for this cookie yet
  332. * exist, if not there'll be nothing to search */
  333. down_read(&fscache_addremove_sem);
  334. if (list_empty(&fscache_cache_list)) {
  335. up_read(&fscache_addremove_sem);
  336. _leave(" = 0 [no caches]");
  337. return 0;
  338. }
  339. /* select a cache in which to store the object */
  340. cache = fscache_select_cache_for_object(cookie->parent);
  341. if (!cache) {
  342. up_read(&fscache_addremove_sem);
  343. fscache_stat(&fscache_n_acquires_no_cache);
  344. _leave(" = -ENOMEDIUM [no cache]");
  345. return -ENOMEDIUM;
  346. }
  347. _debug("cache %s", cache->tag->name);
  348. set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
  349. /* ask the cache to allocate objects for this cookie and its parent
  350. * chain */
  351. ret = fscache_alloc_object(cache, cookie);
  352. if (ret < 0) {
  353. up_read(&fscache_addremove_sem);
  354. _leave(" = %d", ret);
  355. return ret;
  356. }
  357. spin_lock(&cookie->lock);
  358. if (hlist_empty(&cookie->backing_objects)) {
  359. spin_unlock(&cookie->lock);
  360. goto unavailable;
  361. }
  362. object = hlist_entry(cookie->backing_objects.first,
  363. struct fscache_object, cookie_link);
  364. fscache_set_store_limit(object, object_size);
  365. /* initiate the process of looking up all the objects in the chain
  366. * (done by fscache_initialise_object()) */
  367. fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
  368. spin_unlock(&cookie->lock);
  369. /* we may be required to wait for lookup to complete at this point */
  370. if (!fscache_defer_lookup) {
  371. _debug("non-deferred lookup %p", &cookie->flags);
  372. wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
  373. TASK_UNINTERRUPTIBLE);
  374. _debug("complete");
  375. if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
  376. goto unavailable;
  377. }
  378. up_read(&fscache_addremove_sem);
  379. _leave(" = 0 [deferred]");
  380. return 0;
  381. unavailable:
  382. up_read(&fscache_addremove_sem);
  383. _leave(" = -ENOBUFS");
  384. return -ENOBUFS;
  385. }
  386. /*
  387. * recursively allocate cache object records for a cookie/cache combination
  388. * - caller must be holding the addremove sem
  389. */
  390. static int fscache_alloc_object(struct fscache_cache *cache,
  391. struct fscache_cookie *cookie)
  392. {
  393. struct fscache_object *object;
  394. int ret;
  395. _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
  396. spin_lock(&cookie->lock);
  397. hlist_for_each_entry(object, &cookie->backing_objects,
  398. cookie_link) {
  399. if (object->cache == cache)
  400. goto object_already_extant;
  401. }
  402. spin_unlock(&cookie->lock);
  403. /* ask the cache to allocate an object (we may end up with duplicate
  404. * objects at this stage, but we sort that out later) */
  405. fscache_stat(&fscache_n_cop_alloc_object);
  406. object = cache->ops->alloc_object(cache, cookie);
  407. fscache_stat_d(&fscache_n_cop_alloc_object);
  408. if (IS_ERR(object)) {
  409. fscache_stat(&fscache_n_object_no_alloc);
  410. ret = PTR_ERR(object);
  411. goto error;
  412. }
  413. ASSERTCMP(object->cookie, ==, cookie);
  414. fscache_stat(&fscache_n_object_alloc);
  415. object->debug_id = atomic_inc_return(&fscache_object_debug_id);
  416. _debug("ALLOC OBJ%x: %s {%lx}",
  417. object->debug_id, cookie->def->name, object->events);
  418. ret = fscache_alloc_object(cache, cookie->parent);
  419. if (ret < 0)
  420. goto error_put;
  421. /* only attach if we managed to allocate all we needed, otherwise
  422. * discard the object we just allocated and instead use the one
  423. * attached to the cookie */
  424. if (fscache_attach_object(cookie, object) < 0) {
  425. fscache_stat(&fscache_n_cop_put_object);
  426. cache->ops->put_object(object, fscache_obj_put_attach_fail);
  427. fscache_stat_d(&fscache_n_cop_put_object);
  428. }
  429. _leave(" = 0");
  430. return 0;
  431. object_already_extant:
  432. ret = -ENOBUFS;
  433. if (fscache_object_is_dying(object) ||
  434. fscache_cache_is_broken(object)) {
  435. spin_unlock(&cookie->lock);
  436. goto error;
  437. }
  438. spin_unlock(&cookie->lock);
  439. _leave(" = 0 [found]");
  440. return 0;
  441. error_put:
  442. fscache_stat(&fscache_n_cop_put_object);
  443. cache->ops->put_object(object, fscache_obj_put_alloc_fail);
  444. fscache_stat_d(&fscache_n_cop_put_object);
  445. error:
  446. _leave(" = %d", ret);
  447. return ret;
  448. }
  449. /*
  450. * attach a cache object to a cookie
  451. */
  452. static int fscache_attach_object(struct fscache_cookie *cookie,
  453. struct fscache_object *object)
  454. {
  455. struct fscache_object *p;
  456. struct fscache_cache *cache = object->cache;
  457. int ret;
  458. _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
  459. ASSERTCMP(object->cookie, ==, cookie);
  460. spin_lock(&cookie->lock);
  461. /* there may be multiple initial creations of this object, but we only
  462. * want one */
  463. ret = -EEXIST;
  464. hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
  465. if (p->cache == object->cache) {
  466. if (fscache_object_is_dying(p))
  467. ret = -ENOBUFS;
  468. goto cant_attach_object;
  469. }
  470. }
  471. /* pin the parent object */
  472. spin_lock_nested(&cookie->parent->lock, 1);
  473. hlist_for_each_entry(p, &cookie->parent->backing_objects,
  474. cookie_link) {
  475. if (p->cache == object->cache) {
  476. if (fscache_object_is_dying(p)) {
  477. ret = -ENOBUFS;
  478. spin_unlock(&cookie->parent->lock);
  479. goto cant_attach_object;
  480. }
  481. object->parent = p;
  482. spin_lock(&p->lock);
  483. p->n_children++;
  484. spin_unlock(&p->lock);
  485. break;
  486. }
  487. }
  488. spin_unlock(&cookie->parent->lock);
  489. /* attach to the cache's object list */
  490. if (list_empty(&object->cache_link)) {
  491. spin_lock(&cache->object_list_lock);
  492. list_add(&object->cache_link, &cache->object_list);
  493. spin_unlock(&cache->object_list_lock);
  494. }
  495. /* Attach to the cookie. The object already has a ref on it. */
  496. hlist_add_head(&object->cookie_link, &cookie->backing_objects);
  497. fscache_objlist_add(object);
  498. ret = 0;
  499. cant_attach_object:
  500. spin_unlock(&cookie->lock);
  501. _leave(" = %d", ret);
  502. return ret;
  503. }
  504. /*
  505. * Invalidate an object. Callable with spinlocks held.
  506. */
  507. void __fscache_invalidate(struct fscache_cookie *cookie)
  508. {
  509. struct fscache_object *object;
  510. _enter("{%s}", cookie->def->name);
  511. fscache_stat(&fscache_n_invalidates);
  512. /* Only permit invalidation of data files. Invalidating an index will
  513. * require the caller to release all its attachments to the tree rooted
  514. * there, and if it's doing that, it may as well just retire the
  515. * cookie.
  516. */
  517. ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
  518. /* If there's an object, we tell the object state machine to handle the
  519. * invalidation on our behalf, otherwise there's nothing to do.
  520. */
  521. if (!hlist_empty(&cookie->backing_objects)) {
  522. spin_lock(&cookie->lock);
  523. if (fscache_cookie_enabled(cookie) &&
  524. !hlist_empty(&cookie->backing_objects) &&
  525. !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
  526. &cookie->flags)) {
  527. object = hlist_entry(cookie->backing_objects.first,
  528. struct fscache_object,
  529. cookie_link);
  530. if (fscache_object_is_live(object))
  531. fscache_raise_event(
  532. object, FSCACHE_OBJECT_EV_INVALIDATE);
  533. }
  534. spin_unlock(&cookie->lock);
  535. }
  536. _leave("");
  537. }
  538. EXPORT_SYMBOL(__fscache_invalidate);
  539. /*
  540. * Wait for object invalidation to complete.
  541. */
  542. void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
  543. {
  544. _enter("%p", cookie);
  545. wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
  546. TASK_UNINTERRUPTIBLE);
  547. _leave("");
  548. }
  549. EXPORT_SYMBOL(__fscache_wait_on_invalidate);
  550. /*
  551. * update the index entries backing a cookie
  552. */
  553. void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
  554. {
  555. struct fscache_object *object;
  556. fscache_stat(&fscache_n_updates);
  557. if (!cookie) {
  558. fscache_stat(&fscache_n_updates_null);
  559. _leave(" [no cookie]");
  560. return;
  561. }
  562. _enter("{%s}", cookie->def->name);
  563. spin_lock(&cookie->lock);
  564. fscache_update_aux(cookie, aux_data);
  565. if (fscache_cookie_enabled(cookie)) {
  566. /* update the index entry on disk in each cache backing this
  567. * cookie.
  568. */
  569. hlist_for_each_entry(object,
  570. &cookie->backing_objects, cookie_link) {
  571. fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
  572. }
  573. }
  574. spin_unlock(&cookie->lock);
  575. _leave("");
  576. }
  577. EXPORT_SYMBOL(__fscache_update_cookie);
  578. /*
  579. * Disable a cookie to stop it from accepting new requests from the netfs.
  580. */
  581. void __fscache_disable_cookie(struct fscache_cookie *cookie,
  582. const void *aux_data,
  583. bool invalidate)
  584. {
  585. struct fscache_object *object;
  586. bool awaken = false;
  587. _enter("%p,%u", cookie, invalidate);
  588. trace_fscache_disable(cookie);
  589. ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
  590. if (atomic_read(&cookie->n_children) != 0) {
  591. pr_err("Cookie '%s' still has children\n",
  592. cookie->def->name);
  593. BUG();
  594. }
  595. wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
  596. TASK_UNINTERRUPTIBLE);
  597. fscache_update_aux(cookie, aux_data);
  598. if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
  599. goto out_unlock_enable;
  600. /* If the cookie is being invalidated, wait for that to complete first
  601. * so that we can reuse the flag.
  602. */
  603. __fscache_wait_on_invalidate(cookie);
  604. /* Dispose of the backing objects */
  605. set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
  606. spin_lock(&cookie->lock);
  607. if (!hlist_empty(&cookie->backing_objects)) {
  608. hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
  609. if (invalidate)
  610. set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
  611. clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
  612. fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
  613. }
  614. } else {
  615. if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
  616. awaken = true;
  617. }
  618. spin_unlock(&cookie->lock);
  619. if (awaken)
  620. wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
  621. /* Wait for cessation of activity requiring access to the netfs (when
  622. * n_active reaches 0). This makes sure outstanding reads and writes
  623. * have completed.
  624. */
  625. if (!atomic_dec_and_test(&cookie->n_active)) {
  626. wait_var_event(&cookie->n_active,
  627. !atomic_read(&cookie->n_active));
  628. }
  629. /* Make sure any pending writes are cancelled. */
  630. if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
  631. fscache_invalidate_writes(cookie);
  632. /* Reset the cookie state if it wasn't relinquished */
  633. if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
  634. atomic_inc(&cookie->n_active);
  635. set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
  636. }
  637. out_unlock_enable:
  638. clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
  639. wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
  640. _leave("");
  641. }
  642. EXPORT_SYMBOL(__fscache_disable_cookie);
  643. /*
  644. * release a cookie back to the cache
  645. * - the object will be marked as recyclable on disk if retire is true
  646. * - all dependents of this cookie must have already been unregistered
  647. * (indices/files/pages)
  648. */
  649. void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
  650. const void *aux_data,
  651. bool retire)
  652. {
  653. fscache_stat(&fscache_n_relinquishes);
  654. if (retire)
  655. fscache_stat(&fscache_n_relinquishes_retire);
  656. if (!cookie) {
  657. fscache_stat(&fscache_n_relinquishes_null);
  658. _leave(" [no cookie]");
  659. return;
  660. }
  661. _enter("%p{%s,%p,%d},%d",
  662. cookie, cookie->def->name, cookie->netfs_data,
  663. atomic_read(&cookie->n_active), retire);
  664. trace_fscache_relinquish(cookie, retire);
  665. /* No further netfs-accessing operations on this cookie permitted */
  666. if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
  667. BUG();
  668. __fscache_disable_cookie(cookie, aux_data, retire);
  669. /* Clear pointers back to the netfs */
  670. cookie->netfs_data = NULL;
  671. cookie->def = NULL;
  672. BUG_ON(!radix_tree_empty(&cookie->stores));
  673. if (cookie->parent) {
  674. ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
  675. ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
  676. atomic_dec(&cookie->parent->n_children);
  677. }
  678. /* Dispose of the netfs's link to the cookie */
  679. ASSERTCMP(atomic_read(&cookie->usage), >, 0);
  680. fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
  681. _leave("");
  682. }
  683. EXPORT_SYMBOL(__fscache_relinquish_cookie);
  684. /*
  685. * Remove a cookie from the hash table.
  686. */
  687. static void fscache_unhash_cookie(struct fscache_cookie *cookie)
  688. {
  689. struct hlist_bl_head *h;
  690. unsigned int bucket;
  691. bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
  692. h = &fscache_cookie_hash[bucket];
  693. hlist_bl_lock(h);
  694. hlist_bl_del(&cookie->hash_link);
  695. hlist_bl_unlock(h);
  696. }
  697. /*
  698. * Drop a reference to a cookie.
  699. */
  700. void fscache_cookie_put(struct fscache_cookie *cookie,
  701. enum fscache_cookie_trace where)
  702. {
  703. struct fscache_cookie *parent;
  704. int usage;
  705. _enter("%p", cookie);
  706. do {
  707. usage = atomic_dec_return(&cookie->usage);
  708. trace_fscache_cookie(cookie, where, usage);
  709. if (usage > 0)
  710. return;
  711. BUG_ON(usage < 0);
  712. parent = cookie->parent;
  713. fscache_unhash_cookie(cookie);
  714. fscache_free_cookie(cookie);
  715. cookie = parent;
  716. where = fscache_cookie_put_parent;
  717. } while (cookie);
  718. _leave("");
  719. }
  720. /*
  721. * check the consistency between the netfs inode and the backing cache
  722. *
  723. * NOTE: it only serves no-index type
  724. */
  725. int __fscache_check_consistency(struct fscache_cookie *cookie,
  726. const void *aux_data)
  727. {
  728. struct fscache_operation *op;
  729. struct fscache_object *object;
  730. bool wake_cookie = false;
  731. int ret;
  732. _enter("%p,", cookie);
  733. ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
  734. if (fscache_wait_for_deferred_lookup(cookie) < 0)
  735. return -ERESTARTSYS;
  736. if (hlist_empty(&cookie->backing_objects))
  737. return 0;
  738. op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
  739. if (!op)
  740. return -ENOMEM;
  741. fscache_operation_init(cookie, op, NULL, NULL, NULL);
  742. op->flags = FSCACHE_OP_MYTHREAD |
  743. (1 << FSCACHE_OP_WAITING) |
  744. (1 << FSCACHE_OP_UNUSE_COOKIE);
  745. trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
  746. spin_lock(&cookie->lock);
  747. fscache_update_aux(cookie, aux_data);
  748. if (!fscache_cookie_enabled(cookie) ||
  749. hlist_empty(&cookie->backing_objects))
  750. goto inconsistent;
  751. object = hlist_entry(cookie->backing_objects.first,
  752. struct fscache_object, cookie_link);
  753. if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
  754. goto inconsistent;
  755. op->debug_id = atomic_inc_return(&fscache_op_debug_id);
  756. __fscache_use_cookie(cookie);
  757. if (fscache_submit_op(object, op) < 0)
  758. goto submit_failed;
  759. /* the work queue now carries its own ref on the object */
  760. spin_unlock(&cookie->lock);
  761. ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
  762. if (ret == 0) {
  763. /* ask the cache to honour the operation */
  764. ret = object->cache->ops->check_consistency(op);
  765. fscache_op_complete(op, false);
  766. } else if (ret == -ENOBUFS) {
  767. ret = 0;
  768. }
  769. fscache_put_operation(op);
  770. _leave(" = %d", ret);
  771. return ret;
  772. submit_failed:
  773. wake_cookie = __fscache_unuse_cookie(cookie);
  774. inconsistent:
  775. spin_unlock(&cookie->lock);
  776. if (wake_cookie)
  777. __fscache_wake_unused_cookie(cookie);
  778. kfree(op);
  779. _leave(" = -ESTALE");
  780. return -ESTALE;
  781. }
  782. EXPORT_SYMBOL(__fscache_check_consistency);