btree.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _BCACHE_BTREE_H
  3. #define _BCACHE_BTREE_H
  4. /*
  5. * THE BTREE:
  6. *
  7. * At a high level, bcache's btree is relatively standard b+ tree. All keys and
  8. * pointers are in the leaves; interior nodes only have pointers to the child
  9. * nodes.
  10. *
  11. * In the interior nodes, a struct bkey always points to a child btree node, and
  12. * the key is the highest key in the child node - except that the highest key in
  13. * an interior node is always MAX_KEY. The size field refers to the size on disk
  14. * of the child node - this would allow us to have variable sized btree nodes
  15. * (handy for keeping the depth of the btree 1 by expanding just the root).
  16. *
  17. * Btree nodes are themselves log structured, but this is hidden fairly
  18. * thoroughly. Btree nodes on disk will in practice have extents that overlap
  19. * (because they were written at different times), but in memory we never have
  20. * overlapping extents - when we read in a btree node from disk, the first thing
  21. * we do is resort all the sets of keys with a mergesort, and in the same pass
  22. * we check for overlapping extents and adjust them appropriately.
  23. *
  24. * struct btree_op is a central interface to the btree code. It's used for
  25. * specifying read vs. write locking, and the embedded closure is used for
  26. * waiting on IO or reserve memory.
  27. *
  28. * BTREE CACHE:
  29. *
  30. * Btree nodes are cached in memory; traversing the btree might require reading
  31. * in btree nodes which is handled mostly transparently.
  32. *
  33. * bch_btree_node_get() looks up a btree node in the cache and reads it in from
  34. * disk if necessary. This function is almost never called directly though - the
  35. * btree() macro is used to get a btree node, call some function on it, and
  36. * unlock the node after the function returns.
  37. *
  38. * The root is special cased - it's taken out of the cache's lru (thus pinning
  39. * it in memory), so we can find the root of the btree by just dereferencing a
  40. * pointer instead of looking it up in the cache. This makes locking a bit
  41. * tricky, since the root pointer is protected by the lock in the btree node it
  42. * points to - the btree_root() macro handles this.
  43. *
  44. * In various places we must be able to allocate memory for multiple btree nodes
  45. * in order to make forward progress. To do this we use the btree cache itself
  46. * as a reserve; if __get_free_pages() fails, we'll find a node in the btree
  47. * cache we can reuse. We can't allow more than one thread to be doing this at a
  48. * time, so there's a lock, implemented by a pointer to the btree_op closure -
  49. * this allows the btree_root() macro to implicitly release this lock.
  50. *
  51. * BTREE IO:
  52. *
  53. * Btree nodes never have to be explicitly read in; bch_btree_node_get() handles
  54. * this.
  55. *
  56. * For writing, we have two btree_write structs embeddded in struct btree - one
  57. * write in flight, and one being set up, and we toggle between them.
  58. *
  59. * Writing is done with a single function - bch_btree_write() really serves two
  60. * different purposes and should be broken up into two different functions. When
  61. * passing now = false, it merely indicates that the node is now dirty - calling
  62. * it ensures that the dirty keys will be written at some point in the future.
  63. *
  64. * When passing now = true, bch_btree_write() causes a write to happen
  65. * "immediately" (if there was already a write in flight, it'll cause the write
  66. * to happen as soon as the previous write completes). It returns immediately
  67. * though - but it takes a refcount on the closure in struct btree_op you passed
  68. * to it, so a closure_sync() later can be used to wait for the write to
  69. * complete.
  70. *
  71. * This is handy because btree_split() and garbage collection can issue writes
  72. * in parallel, reducing the amount of time they have to hold write locks.
  73. *
  74. * LOCKING:
  75. *
  76. * When traversing the btree, we may need write locks starting at some level -
  77. * inserting a key into the btree will typically only require a write lock on
  78. * the leaf node.
  79. *
  80. * This is specified with the lock field in struct btree_op; lock = 0 means we
  81. * take write locks at level <= 0, i.e. only leaf nodes. bch_btree_node_get()
  82. * checks this field and returns the node with the appropriate lock held.
  83. *
  84. * If, after traversing the btree, the insertion code discovers it has to split
  85. * then it must restart from the root and take new locks - to do this it changes
  86. * the lock field and returns -EINTR, which causes the btree_root() macro to
  87. * loop.
  88. *
  89. * Handling cache misses require a different mechanism for upgrading to a write
  90. * lock. We do cache lookups with only a read lock held, but if we get a cache
  91. * miss and we wish to insert this data into the cache, we have to insert a
  92. * placeholder key to detect races - otherwise, we could race with a write and
  93. * overwrite the data that was just written to the cache with stale data from
  94. * the backing device.
  95. *
  96. * For this we use a sequence number that write locks and unlocks increment - to
  97. * insert the check key it unlocks the btree node and then takes a write lock,
  98. * and fails if the sequence number doesn't match.
  99. */
  100. #include "bset.h"
  101. #include "debug.h"
  102. struct btree_write {
  103. atomic_t *journal;
  104. /* If btree_split() frees a btree node, it writes a new pointer to that
  105. * btree node indicating it was freed; it takes a refcount on
  106. * c->prio_blocked because we can't write the gens until the new
  107. * pointer is on disk. This allows btree_write_endio() to release the
  108. * refcount that btree_split() took.
  109. */
  110. int prio_blocked;
  111. };
  112. struct btree {
  113. /* Hottest entries first */
  114. struct hlist_node hash;
  115. /* Key/pointer for this btree node */
  116. BKEY_PADDED(key);
  117. unsigned long seq;
  118. struct rw_semaphore lock;
  119. struct cache_set *c;
  120. struct btree *parent;
  121. struct mutex write_lock;
  122. unsigned long flags;
  123. uint16_t written; /* would be nice to kill */
  124. uint8_t level;
  125. struct btree_keys keys;
  126. /* For outstanding btree writes, used as a lock - protects write_idx */
  127. struct closure io;
  128. struct semaphore io_mutex;
  129. struct list_head list;
  130. struct delayed_work work;
  131. struct btree_write writes[2];
  132. struct bio *bio;
  133. };
  134. #define BTREE_FLAG(flag) \
  135. static inline bool btree_node_ ## flag(struct btree *b) \
  136. { return test_bit(BTREE_NODE_ ## flag, &b->flags); } \
  137. \
  138. static inline void set_btree_node_ ## flag(struct btree *b) \
  139. { set_bit(BTREE_NODE_ ## flag, &b->flags); }
  140. enum btree_flags {
  141. BTREE_NODE_io_error,
  142. BTREE_NODE_dirty,
  143. BTREE_NODE_write_idx,
  144. BTREE_NODE_journal_flush,
  145. };
  146. BTREE_FLAG(io_error);
  147. BTREE_FLAG(dirty);
  148. BTREE_FLAG(write_idx);
  149. BTREE_FLAG(journal_flush);
  150. static inline struct btree_write *btree_current_write(struct btree *b)
  151. {
  152. return b->writes + btree_node_write_idx(b);
  153. }
  154. static inline struct btree_write *btree_prev_write(struct btree *b)
  155. {
  156. return b->writes + (btree_node_write_idx(b) ^ 1);
  157. }
  158. static inline struct bset *btree_bset_first(struct btree *b)
  159. {
  160. return b->keys.set->data;
  161. }
  162. static inline struct bset *btree_bset_last(struct btree *b)
  163. {
  164. return bset_tree_last(&b->keys)->data;
  165. }
  166. static inline unsigned int bset_block_offset(struct btree *b, struct bset *i)
  167. {
  168. return bset_sector_offset(&b->keys, i) >> b->c->block_bits;
  169. }
  170. static inline void set_gc_sectors(struct cache_set *c)
  171. {
  172. atomic_set(&c->sectors_to_gc, c->cache->sb.bucket_size * c->nbuckets / 16);
  173. }
  174. void bkey_put(struct cache_set *c, struct bkey *k);
  175. /* Looping macros */
  176. #define for_each_cached_btree(b, c, iter) \
  177. for (iter = 0; \
  178. iter < ARRAY_SIZE((c)->bucket_hash); \
  179. iter++) \
  180. hlist_for_each_entry_rcu((b), (c)->bucket_hash + iter, hash)
  181. /* Recursing down the btree */
  182. struct btree_op {
  183. /* for waiting on btree reserve in btree_split() */
  184. wait_queue_entry_t wait;
  185. /* Btree level at which we start taking write locks */
  186. short lock;
  187. unsigned int insert_collision:1;
  188. };
  189. struct btree_check_state;
  190. struct btree_check_info {
  191. struct btree_check_state *state;
  192. struct task_struct *thread;
  193. int result;
  194. };
  195. #define BCH_BTR_CHKTHREAD_MAX 64
  196. struct btree_check_state {
  197. struct cache_set *c;
  198. int total_threads;
  199. int key_idx;
  200. spinlock_t idx_lock;
  201. atomic_t started;
  202. atomic_t enough;
  203. wait_queue_head_t wait;
  204. struct btree_check_info infos[BCH_BTR_CHKTHREAD_MAX];
  205. };
  206. static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level)
  207. {
  208. memset(op, 0, sizeof(struct btree_op));
  209. init_wait(&op->wait);
  210. op->lock = write_lock_level;
  211. }
  212. static inline void rw_lock(bool w, struct btree *b, int level)
  213. {
  214. w ? down_write_nested(&b->lock, level + 1)
  215. : down_read_nested(&b->lock, level + 1);
  216. if (w)
  217. b->seq++;
  218. }
  219. static inline void rw_unlock(bool w, struct btree *b)
  220. {
  221. if (w)
  222. b->seq++;
  223. (w ? up_write : up_read)(&b->lock);
  224. }
  225. void bch_btree_node_read_done(struct btree *b);
  226. void __bch_btree_node_write(struct btree *b, struct closure *parent);
  227. void bch_btree_node_write(struct btree *b, struct closure *parent);
  228. void bch_btree_set_root(struct btree *b);
  229. struct btree *__bch_btree_node_alloc(struct cache_set *c, struct btree_op *op,
  230. int level, bool wait,
  231. struct btree *parent);
  232. struct btree *bch_btree_node_get(struct cache_set *c, struct btree_op *op,
  233. struct bkey *k, int level, bool write,
  234. struct btree *parent);
  235. int bch_btree_insert_check_key(struct btree *b, struct btree_op *op,
  236. struct bkey *check_key);
  237. int bch_btree_insert(struct cache_set *c, struct keylist *keys,
  238. atomic_t *journal_ref, struct bkey *replace_key);
  239. int bch_gc_thread_start(struct cache_set *c);
  240. void bch_initial_gc_finish(struct cache_set *c);
  241. void bch_moving_gc(struct cache_set *c);
  242. int bch_btree_check(struct cache_set *c);
  243. void bch_initial_mark_key(struct cache_set *c, int level, struct bkey *k);
  244. static inline void wake_up_gc(struct cache_set *c)
  245. {
  246. wake_up(&c->gc_wait);
  247. }
  248. static inline void force_wake_up_gc(struct cache_set *c)
  249. {
  250. /*
  251. * Garbage collection thread only works when sectors_to_gc < 0,
  252. * calling wake_up_gc() won't start gc thread if sectors_to_gc is
  253. * not a nagetive value.
  254. * Therefore sectors_to_gc is set to -1 here, before waking up
  255. * gc thread by calling wake_up_gc(). Then gc_should_run() will
  256. * give a chance to permit gc thread to run. "Give a chance" means
  257. * before going into gc_should_run(), there is still possibility
  258. * that c->sectors_to_gc being set to other positive value. So
  259. * this routine won't 100% make sure gc thread will be woken up
  260. * to run.
  261. */
  262. atomic_set(&c->sectors_to_gc, -1);
  263. wake_up_gc(c);
  264. }
  265. /*
  266. * These macros are for recursing down the btree - they handle the details of
  267. * locking and looking up nodes in the cache for you. They're best treated as
  268. * mere syntax when reading code that uses them.
  269. *
  270. * op->lock determines whether we take a read or a write lock at a given depth.
  271. * If you've got a read lock and find that you need a write lock (i.e. you're
  272. * going to have to split), set op->lock and return -EINTR; btree_root() will
  273. * call you again and you'll have the correct lock.
  274. */
  275. /**
  276. * btree - recurse down the btree on a specified key
  277. * @fn: function to call, which will be passed the child node
  278. * @key: key to recurse on
  279. * @b: parent btree node
  280. * @op: pointer to struct btree_op
  281. */
  282. #define bcache_btree(fn, key, b, op, ...) \
  283. ({ \
  284. int _r, l = (b)->level - 1; \
  285. bool _w = l <= (op)->lock; \
  286. struct btree *_child = bch_btree_node_get((b)->c, op, key, l, \
  287. _w, b); \
  288. if (!IS_ERR(_child)) { \
  289. _r = bch_btree_ ## fn(_child, op, ##__VA_ARGS__); \
  290. rw_unlock(_w, _child); \
  291. } else \
  292. _r = PTR_ERR(_child); \
  293. _r; \
  294. })
  295. /**
  296. * btree_root - call a function on the root of the btree
  297. * @fn: function to call, which will be passed the child node
  298. * @c: cache set
  299. * @op: pointer to struct btree_op
  300. */
  301. #define bcache_btree_root(fn, c, op, ...) \
  302. ({ \
  303. int _r = -EINTR; \
  304. do { \
  305. struct btree *_b = (c)->root; \
  306. bool _w = insert_lock(op, _b); \
  307. rw_lock(_w, _b, _b->level); \
  308. if (_b == (c)->root && \
  309. _w == insert_lock(op, _b)) { \
  310. _r = bch_btree_ ## fn(_b, op, ##__VA_ARGS__); \
  311. } \
  312. rw_unlock(_w, _b); \
  313. bch_cannibalize_unlock(c); \
  314. if (_r == -EINTR) \
  315. schedule(); \
  316. } while (_r == -EINTR); \
  317. \
  318. finish_wait(&(c)->btree_cache_wait, &(op)->wait); \
  319. _r; \
  320. })
  321. #define MAP_DONE 0
  322. #define MAP_CONTINUE 1
  323. #define MAP_ALL_NODES 0
  324. #define MAP_LEAF_NODES 1
  325. #define MAP_END_KEY 1
  326. typedef int (btree_map_nodes_fn)(struct btree_op *b_op, struct btree *b);
  327. int __bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
  328. struct bkey *from, btree_map_nodes_fn *fn, int flags);
  329. static inline int bch_btree_map_nodes(struct btree_op *op, struct cache_set *c,
  330. struct bkey *from, btree_map_nodes_fn *fn)
  331. {
  332. return __bch_btree_map_nodes(op, c, from, fn, MAP_ALL_NODES);
  333. }
  334. static inline int bch_btree_map_leaf_nodes(struct btree_op *op,
  335. struct cache_set *c,
  336. struct bkey *from,
  337. btree_map_nodes_fn *fn)
  338. {
  339. return __bch_btree_map_nodes(op, c, from, fn, MAP_LEAF_NODES);
  340. }
  341. typedef int (btree_map_keys_fn)(struct btree_op *op, struct btree *b,
  342. struct bkey *k);
  343. int bch_btree_map_keys(struct btree_op *op, struct cache_set *c,
  344. struct bkey *from, btree_map_keys_fn *fn, int flags);
  345. int bch_btree_map_keys_recurse(struct btree *b, struct btree_op *op,
  346. struct bkey *from, btree_map_keys_fn *fn,
  347. int flags);
  348. typedef bool (keybuf_pred_fn)(struct keybuf *buf, struct bkey *k);
  349. void bch_keybuf_init(struct keybuf *buf);
  350. void bch_refill_keybuf(struct cache_set *c, struct keybuf *buf,
  351. struct bkey *end, keybuf_pred_fn *pred);
  352. bool bch_keybuf_check_overlapping(struct keybuf *buf, struct bkey *start,
  353. struct bkey *end);
  354. void bch_keybuf_del(struct keybuf *buf, struct keybuf_key *w);
  355. struct keybuf_key *bch_keybuf_next(struct keybuf *buf);
  356. struct keybuf_key *bch_keybuf_next_rescan(struct cache_set *c,
  357. struct keybuf *buf,
  358. struct bkey *end,
  359. keybuf_pred_fn *pred);
  360. void bch_update_bucket_in_use(struct cache_set *c, struct gc_stat *stats);
  361. #endif