delayed-ref.c 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2009 Oracle. All rights reserved.
  4. */
  5. #include <linux/sched.h>
  6. #include <linux/slab.h>
  7. #include <linux/sort.h>
  8. #include "ctree.h"
  9. #include "delayed-ref.h"
  10. #include "transaction.h"
  11. #include "qgroup.h"
  12. #include "space-info.h"
  13. struct kmem_cache *btrfs_delayed_ref_head_cachep;
  14. struct kmem_cache *btrfs_delayed_tree_ref_cachep;
  15. struct kmem_cache *btrfs_delayed_data_ref_cachep;
  16. struct kmem_cache *btrfs_delayed_extent_op_cachep;
  17. /*
  18. * delayed back reference update tracking. For subvolume trees
  19. * we queue up extent allocations and backref maintenance for
  20. * delayed processing. This avoids deep call chains where we
  21. * add extents in the middle of btrfs_search_slot, and it allows
  22. * us to buffer up frequently modified backrefs in an rb tree instead
  23. * of hammering updates on the extent allocation tree.
  24. */
  25. bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info)
  26. {
  27. struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
  28. struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
  29. bool ret = false;
  30. u64 reserved;
  31. spin_lock(&global_rsv->lock);
  32. reserved = global_rsv->reserved;
  33. spin_unlock(&global_rsv->lock);
  34. /*
  35. * Since the global reserve is just kind of magic we don't really want
  36. * to rely on it to save our bacon, so if our size is more than the
  37. * delayed_refs_rsv and the global rsv then it's time to think about
  38. * bailing.
  39. */
  40. spin_lock(&delayed_refs_rsv->lock);
  41. reserved += delayed_refs_rsv->reserved;
  42. if (delayed_refs_rsv->size >= reserved)
  43. ret = true;
  44. spin_unlock(&delayed_refs_rsv->lock);
  45. return ret;
  46. }
  47. int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans)
  48. {
  49. u64 num_entries =
  50. atomic_read(&trans->transaction->delayed_refs.num_entries);
  51. u64 avg_runtime;
  52. u64 val;
  53. smp_mb();
  54. avg_runtime = trans->fs_info->avg_delayed_ref_runtime;
  55. val = num_entries * avg_runtime;
  56. if (val >= NSEC_PER_SEC)
  57. return 1;
  58. if (val >= NSEC_PER_SEC / 2)
  59. return 2;
  60. return btrfs_check_space_for_delayed_refs(trans->fs_info);
  61. }
  62. /**
  63. * btrfs_delayed_refs_rsv_release - release a ref head's reservation.
  64. * @fs_info - the fs_info for our fs.
  65. * @nr - the number of items to drop.
  66. *
  67. * This drops the delayed ref head's count from the delayed refs rsv and frees
  68. * any excess reservation we had.
  69. */
  70. void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr)
  71. {
  72. struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
  73. u64 num_bytes = btrfs_calc_insert_metadata_size(fs_info, nr);
  74. u64 released = 0;
  75. released = btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL);
  76. if (released)
  77. trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
  78. 0, released, 0);
  79. }
  80. /*
  81. * btrfs_update_delayed_refs_rsv - adjust the size of the delayed refs rsv
  82. * @trans - the trans that may have generated delayed refs
  83. *
  84. * This is to be called anytime we may have adjusted trans->delayed_ref_updates,
  85. * it'll calculate the additional size and add it to the delayed_refs_rsv.
  86. */
  87. void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans)
  88. {
  89. struct btrfs_fs_info *fs_info = trans->fs_info;
  90. struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv;
  91. u64 num_bytes;
  92. if (!trans->delayed_ref_updates)
  93. return;
  94. num_bytes = btrfs_calc_insert_metadata_size(fs_info,
  95. trans->delayed_ref_updates);
  96. spin_lock(&delayed_rsv->lock);
  97. delayed_rsv->size += num_bytes;
  98. delayed_rsv->full = 0;
  99. spin_unlock(&delayed_rsv->lock);
  100. trans->delayed_ref_updates = 0;
  101. }
  102. /**
  103. * btrfs_migrate_to_delayed_refs_rsv - transfer bytes to our delayed refs rsv.
  104. * @fs_info - the fs info for our fs.
  105. * @src - the source block rsv to transfer from.
  106. * @num_bytes - the number of bytes to transfer.
  107. *
  108. * This transfers up to the num_bytes amount from the src rsv to the
  109. * delayed_refs_rsv. Any extra bytes are returned to the space info.
  110. */
  111. void btrfs_migrate_to_delayed_refs_rsv(struct btrfs_fs_info *fs_info,
  112. struct btrfs_block_rsv *src,
  113. u64 num_bytes)
  114. {
  115. struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
  116. u64 to_free = 0;
  117. spin_lock(&src->lock);
  118. src->reserved -= num_bytes;
  119. src->size -= num_bytes;
  120. spin_unlock(&src->lock);
  121. spin_lock(&delayed_refs_rsv->lock);
  122. if (delayed_refs_rsv->size > delayed_refs_rsv->reserved) {
  123. u64 delta = delayed_refs_rsv->size -
  124. delayed_refs_rsv->reserved;
  125. if (num_bytes > delta) {
  126. to_free = num_bytes - delta;
  127. num_bytes = delta;
  128. }
  129. } else {
  130. to_free = num_bytes;
  131. num_bytes = 0;
  132. }
  133. if (num_bytes)
  134. delayed_refs_rsv->reserved += num_bytes;
  135. if (delayed_refs_rsv->reserved >= delayed_refs_rsv->size)
  136. delayed_refs_rsv->full = 1;
  137. spin_unlock(&delayed_refs_rsv->lock);
  138. if (num_bytes)
  139. trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
  140. 0, num_bytes, 1);
  141. if (to_free)
  142. btrfs_space_info_free_bytes_may_use(fs_info,
  143. delayed_refs_rsv->space_info, to_free);
  144. }
  145. /**
  146. * btrfs_delayed_refs_rsv_refill - refill based on our delayed refs usage.
  147. * @fs_info - the fs_info for our fs.
  148. * @flush - control how we can flush for this reservation.
  149. *
  150. * This will refill the delayed block_rsv up to 1 items size worth of space and
  151. * will return -ENOSPC if we can't make the reservation.
  152. */
  153. int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info,
  154. enum btrfs_reserve_flush_enum flush)
  155. {
  156. struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv;
  157. u64 limit = btrfs_calc_insert_metadata_size(fs_info, 1);
  158. u64 num_bytes = 0;
  159. int ret = -ENOSPC;
  160. spin_lock(&block_rsv->lock);
  161. if (block_rsv->reserved < block_rsv->size) {
  162. num_bytes = block_rsv->size - block_rsv->reserved;
  163. num_bytes = min(num_bytes, limit);
  164. }
  165. spin_unlock(&block_rsv->lock);
  166. if (!num_bytes)
  167. return 0;
  168. ret = btrfs_reserve_metadata_bytes(fs_info->extent_root, block_rsv,
  169. num_bytes, flush);
  170. if (ret)
  171. return ret;
  172. btrfs_block_rsv_add_bytes(block_rsv, num_bytes, 0);
  173. trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv",
  174. 0, num_bytes, 1);
  175. return 0;
  176. }
  177. /*
  178. * compare two delayed tree backrefs with same bytenr and type
  179. */
  180. static int comp_tree_refs(struct btrfs_delayed_tree_ref *ref1,
  181. struct btrfs_delayed_tree_ref *ref2)
  182. {
  183. if (ref1->node.type == BTRFS_TREE_BLOCK_REF_KEY) {
  184. if (ref1->root < ref2->root)
  185. return -1;
  186. if (ref1->root > ref2->root)
  187. return 1;
  188. } else {
  189. if (ref1->parent < ref2->parent)
  190. return -1;
  191. if (ref1->parent > ref2->parent)
  192. return 1;
  193. }
  194. return 0;
  195. }
  196. /*
  197. * compare two delayed data backrefs with same bytenr and type
  198. */
  199. static int comp_data_refs(struct btrfs_delayed_data_ref *ref1,
  200. struct btrfs_delayed_data_ref *ref2)
  201. {
  202. if (ref1->node.type == BTRFS_EXTENT_DATA_REF_KEY) {
  203. if (ref1->root < ref2->root)
  204. return -1;
  205. if (ref1->root > ref2->root)
  206. return 1;
  207. if (ref1->objectid < ref2->objectid)
  208. return -1;
  209. if (ref1->objectid > ref2->objectid)
  210. return 1;
  211. if (ref1->offset < ref2->offset)
  212. return -1;
  213. if (ref1->offset > ref2->offset)
  214. return 1;
  215. } else {
  216. if (ref1->parent < ref2->parent)
  217. return -1;
  218. if (ref1->parent > ref2->parent)
  219. return 1;
  220. }
  221. return 0;
  222. }
  223. static int comp_refs(struct btrfs_delayed_ref_node *ref1,
  224. struct btrfs_delayed_ref_node *ref2,
  225. bool check_seq)
  226. {
  227. int ret = 0;
  228. if (ref1->type < ref2->type)
  229. return -1;
  230. if (ref1->type > ref2->type)
  231. return 1;
  232. if (ref1->type == BTRFS_TREE_BLOCK_REF_KEY ||
  233. ref1->type == BTRFS_SHARED_BLOCK_REF_KEY)
  234. ret = comp_tree_refs(btrfs_delayed_node_to_tree_ref(ref1),
  235. btrfs_delayed_node_to_tree_ref(ref2));
  236. else
  237. ret = comp_data_refs(btrfs_delayed_node_to_data_ref(ref1),
  238. btrfs_delayed_node_to_data_ref(ref2));
  239. if (ret)
  240. return ret;
  241. if (check_seq) {
  242. if (ref1->seq < ref2->seq)
  243. return -1;
  244. if (ref1->seq > ref2->seq)
  245. return 1;
  246. }
  247. return 0;
  248. }
  249. /* insert a new ref to head ref rbtree */
  250. static struct btrfs_delayed_ref_head *htree_insert(struct rb_root_cached *root,
  251. struct rb_node *node)
  252. {
  253. struct rb_node **p = &root->rb_root.rb_node;
  254. struct rb_node *parent_node = NULL;
  255. struct btrfs_delayed_ref_head *entry;
  256. struct btrfs_delayed_ref_head *ins;
  257. u64 bytenr;
  258. bool leftmost = true;
  259. ins = rb_entry(node, struct btrfs_delayed_ref_head, href_node);
  260. bytenr = ins->bytenr;
  261. while (*p) {
  262. parent_node = *p;
  263. entry = rb_entry(parent_node, struct btrfs_delayed_ref_head,
  264. href_node);
  265. if (bytenr < entry->bytenr) {
  266. p = &(*p)->rb_left;
  267. } else if (bytenr > entry->bytenr) {
  268. p = &(*p)->rb_right;
  269. leftmost = false;
  270. } else {
  271. return entry;
  272. }
  273. }
  274. rb_link_node(node, parent_node, p);
  275. rb_insert_color_cached(node, root, leftmost);
  276. return NULL;
  277. }
  278. static struct btrfs_delayed_ref_node* tree_insert(struct rb_root_cached *root,
  279. struct btrfs_delayed_ref_node *ins)
  280. {
  281. struct rb_node **p = &root->rb_root.rb_node;
  282. struct rb_node *node = &ins->ref_node;
  283. struct rb_node *parent_node = NULL;
  284. struct btrfs_delayed_ref_node *entry;
  285. bool leftmost = true;
  286. while (*p) {
  287. int comp;
  288. parent_node = *p;
  289. entry = rb_entry(parent_node, struct btrfs_delayed_ref_node,
  290. ref_node);
  291. comp = comp_refs(ins, entry, true);
  292. if (comp < 0) {
  293. p = &(*p)->rb_left;
  294. } else if (comp > 0) {
  295. p = &(*p)->rb_right;
  296. leftmost = false;
  297. } else {
  298. return entry;
  299. }
  300. }
  301. rb_link_node(node, parent_node, p);
  302. rb_insert_color_cached(node, root, leftmost);
  303. return NULL;
  304. }
  305. static struct btrfs_delayed_ref_head *find_first_ref_head(
  306. struct btrfs_delayed_ref_root *dr)
  307. {
  308. struct rb_node *n;
  309. struct btrfs_delayed_ref_head *entry;
  310. n = rb_first_cached(&dr->href_root);
  311. if (!n)
  312. return NULL;
  313. entry = rb_entry(n, struct btrfs_delayed_ref_head, href_node);
  314. return entry;
  315. }
  316. /*
  317. * Find a head entry based on bytenr. This returns the delayed ref head if it
  318. * was able to find one, or NULL if nothing was in that spot. If return_bigger
  319. * is given, the next bigger entry is returned if no exact match is found.
  320. */
  321. static struct btrfs_delayed_ref_head *find_ref_head(
  322. struct btrfs_delayed_ref_root *dr, u64 bytenr,
  323. bool return_bigger)
  324. {
  325. struct rb_root *root = &dr->href_root.rb_root;
  326. struct rb_node *n;
  327. struct btrfs_delayed_ref_head *entry;
  328. n = root->rb_node;
  329. entry = NULL;
  330. while (n) {
  331. entry = rb_entry(n, struct btrfs_delayed_ref_head, href_node);
  332. if (bytenr < entry->bytenr)
  333. n = n->rb_left;
  334. else if (bytenr > entry->bytenr)
  335. n = n->rb_right;
  336. else
  337. return entry;
  338. }
  339. if (entry && return_bigger) {
  340. if (bytenr > entry->bytenr) {
  341. n = rb_next(&entry->href_node);
  342. if (!n)
  343. return NULL;
  344. entry = rb_entry(n, struct btrfs_delayed_ref_head,
  345. href_node);
  346. }
  347. return entry;
  348. }
  349. return NULL;
  350. }
  351. int btrfs_delayed_ref_lock(struct btrfs_delayed_ref_root *delayed_refs,
  352. struct btrfs_delayed_ref_head *head)
  353. {
  354. lockdep_assert_held(&delayed_refs->lock);
  355. if (mutex_trylock(&head->mutex))
  356. return 0;
  357. refcount_inc(&head->refs);
  358. spin_unlock(&delayed_refs->lock);
  359. mutex_lock(&head->mutex);
  360. spin_lock(&delayed_refs->lock);
  361. if (RB_EMPTY_NODE(&head->href_node)) {
  362. mutex_unlock(&head->mutex);
  363. btrfs_put_delayed_ref_head(head);
  364. return -EAGAIN;
  365. }
  366. btrfs_put_delayed_ref_head(head);
  367. return 0;
  368. }
  369. static inline void drop_delayed_ref(struct btrfs_trans_handle *trans,
  370. struct btrfs_delayed_ref_root *delayed_refs,
  371. struct btrfs_delayed_ref_head *head,
  372. struct btrfs_delayed_ref_node *ref)
  373. {
  374. lockdep_assert_held(&head->lock);
  375. rb_erase_cached(&ref->ref_node, &head->ref_tree);
  376. RB_CLEAR_NODE(&ref->ref_node);
  377. if (!list_empty(&ref->add_list))
  378. list_del(&ref->add_list);
  379. ref->in_tree = 0;
  380. btrfs_put_delayed_ref(ref);
  381. atomic_dec(&delayed_refs->num_entries);
  382. }
  383. static bool merge_ref(struct btrfs_trans_handle *trans,
  384. struct btrfs_delayed_ref_root *delayed_refs,
  385. struct btrfs_delayed_ref_head *head,
  386. struct btrfs_delayed_ref_node *ref,
  387. u64 seq)
  388. {
  389. struct btrfs_delayed_ref_node *next;
  390. struct rb_node *node = rb_next(&ref->ref_node);
  391. bool done = false;
  392. while (!done && node) {
  393. int mod;
  394. next = rb_entry(node, struct btrfs_delayed_ref_node, ref_node);
  395. node = rb_next(node);
  396. if (seq && next->seq >= seq)
  397. break;
  398. if (comp_refs(ref, next, false))
  399. break;
  400. if (ref->action == next->action) {
  401. mod = next->ref_mod;
  402. } else {
  403. if (ref->ref_mod < next->ref_mod) {
  404. swap(ref, next);
  405. done = true;
  406. }
  407. mod = -next->ref_mod;
  408. }
  409. drop_delayed_ref(trans, delayed_refs, head, next);
  410. ref->ref_mod += mod;
  411. if (ref->ref_mod == 0) {
  412. drop_delayed_ref(trans, delayed_refs, head, ref);
  413. done = true;
  414. } else {
  415. /*
  416. * Can't have multiples of the same ref on a tree block.
  417. */
  418. WARN_ON(ref->type == BTRFS_TREE_BLOCK_REF_KEY ||
  419. ref->type == BTRFS_SHARED_BLOCK_REF_KEY);
  420. }
  421. }
  422. return done;
  423. }
  424. void btrfs_merge_delayed_refs(struct btrfs_trans_handle *trans,
  425. struct btrfs_delayed_ref_root *delayed_refs,
  426. struct btrfs_delayed_ref_head *head)
  427. {
  428. struct btrfs_fs_info *fs_info = trans->fs_info;
  429. struct btrfs_delayed_ref_node *ref;
  430. struct rb_node *node;
  431. u64 seq = 0;
  432. lockdep_assert_held(&head->lock);
  433. if (RB_EMPTY_ROOT(&head->ref_tree.rb_root))
  434. return;
  435. /* We don't have too many refs to merge for data. */
  436. if (head->is_data)
  437. return;
  438. read_lock(&fs_info->tree_mod_log_lock);
  439. if (!list_empty(&fs_info->tree_mod_seq_list)) {
  440. struct seq_list *elem;
  441. elem = list_first_entry(&fs_info->tree_mod_seq_list,
  442. struct seq_list, list);
  443. seq = elem->seq;
  444. }
  445. read_unlock(&fs_info->tree_mod_log_lock);
  446. again:
  447. for (node = rb_first_cached(&head->ref_tree); node;
  448. node = rb_next(node)) {
  449. ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node);
  450. if (seq && ref->seq >= seq)
  451. continue;
  452. if (merge_ref(trans, delayed_refs, head, ref, seq))
  453. goto again;
  454. }
  455. }
  456. int btrfs_check_delayed_seq(struct btrfs_fs_info *fs_info, u64 seq)
  457. {
  458. struct seq_list *elem;
  459. int ret = 0;
  460. read_lock(&fs_info->tree_mod_log_lock);
  461. if (!list_empty(&fs_info->tree_mod_seq_list)) {
  462. elem = list_first_entry(&fs_info->tree_mod_seq_list,
  463. struct seq_list, list);
  464. if (seq >= elem->seq) {
  465. btrfs_debug(fs_info,
  466. "holding back delayed_ref %#x.%x, lowest is %#x.%x",
  467. (u32)(seq >> 32), (u32)seq,
  468. (u32)(elem->seq >> 32), (u32)elem->seq);
  469. ret = 1;
  470. }
  471. }
  472. read_unlock(&fs_info->tree_mod_log_lock);
  473. return ret;
  474. }
  475. struct btrfs_delayed_ref_head *btrfs_select_ref_head(
  476. struct btrfs_delayed_ref_root *delayed_refs)
  477. {
  478. struct btrfs_delayed_ref_head *head;
  479. again:
  480. head = find_ref_head(delayed_refs, delayed_refs->run_delayed_start,
  481. true);
  482. if (!head && delayed_refs->run_delayed_start != 0) {
  483. delayed_refs->run_delayed_start = 0;
  484. head = find_first_ref_head(delayed_refs);
  485. }
  486. if (!head)
  487. return NULL;
  488. while (head->processing) {
  489. struct rb_node *node;
  490. node = rb_next(&head->href_node);
  491. if (!node) {
  492. if (delayed_refs->run_delayed_start == 0)
  493. return NULL;
  494. delayed_refs->run_delayed_start = 0;
  495. goto again;
  496. }
  497. head = rb_entry(node, struct btrfs_delayed_ref_head,
  498. href_node);
  499. }
  500. head->processing = 1;
  501. WARN_ON(delayed_refs->num_heads_ready == 0);
  502. delayed_refs->num_heads_ready--;
  503. delayed_refs->run_delayed_start = head->bytenr +
  504. head->num_bytes;
  505. return head;
  506. }
  507. void btrfs_delete_ref_head(struct btrfs_delayed_ref_root *delayed_refs,
  508. struct btrfs_delayed_ref_head *head)
  509. {
  510. lockdep_assert_held(&delayed_refs->lock);
  511. lockdep_assert_held(&head->lock);
  512. rb_erase_cached(&head->href_node, &delayed_refs->href_root);
  513. RB_CLEAR_NODE(&head->href_node);
  514. atomic_dec(&delayed_refs->num_entries);
  515. delayed_refs->num_heads--;
  516. if (head->processing == 0)
  517. delayed_refs->num_heads_ready--;
  518. }
  519. /*
  520. * Helper to insert the ref_node to the tail or merge with tail.
  521. *
  522. * Return 0 for insert.
  523. * Return >0 for merge.
  524. */
  525. static int insert_delayed_ref(struct btrfs_trans_handle *trans,
  526. struct btrfs_delayed_ref_root *root,
  527. struct btrfs_delayed_ref_head *href,
  528. struct btrfs_delayed_ref_node *ref)
  529. {
  530. struct btrfs_delayed_ref_node *exist;
  531. int mod;
  532. int ret = 0;
  533. spin_lock(&href->lock);
  534. exist = tree_insert(&href->ref_tree, ref);
  535. if (!exist)
  536. goto inserted;
  537. /* Now we are sure we can merge */
  538. ret = 1;
  539. if (exist->action == ref->action) {
  540. mod = ref->ref_mod;
  541. } else {
  542. /* Need to change action */
  543. if (exist->ref_mod < ref->ref_mod) {
  544. exist->action = ref->action;
  545. mod = -exist->ref_mod;
  546. exist->ref_mod = ref->ref_mod;
  547. if (ref->action == BTRFS_ADD_DELAYED_REF)
  548. list_add_tail(&exist->add_list,
  549. &href->ref_add_list);
  550. else if (ref->action == BTRFS_DROP_DELAYED_REF) {
  551. ASSERT(!list_empty(&exist->add_list));
  552. list_del(&exist->add_list);
  553. } else {
  554. ASSERT(0);
  555. }
  556. } else
  557. mod = -ref->ref_mod;
  558. }
  559. exist->ref_mod += mod;
  560. /* remove existing tail if its ref_mod is zero */
  561. if (exist->ref_mod == 0)
  562. drop_delayed_ref(trans, root, href, exist);
  563. spin_unlock(&href->lock);
  564. return ret;
  565. inserted:
  566. if (ref->action == BTRFS_ADD_DELAYED_REF)
  567. list_add_tail(&ref->add_list, &href->ref_add_list);
  568. atomic_inc(&root->num_entries);
  569. spin_unlock(&href->lock);
  570. return ret;
  571. }
  572. /*
  573. * helper function to update the accounting in the head ref
  574. * existing and update must have the same bytenr
  575. */
  576. static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans,
  577. struct btrfs_delayed_ref_head *existing,
  578. struct btrfs_delayed_ref_head *update)
  579. {
  580. struct btrfs_delayed_ref_root *delayed_refs =
  581. &trans->transaction->delayed_refs;
  582. struct btrfs_fs_info *fs_info = trans->fs_info;
  583. u64 flags = btrfs_ref_head_to_space_flags(existing);
  584. int old_ref_mod;
  585. BUG_ON(existing->is_data != update->is_data);
  586. spin_lock(&existing->lock);
  587. if (update->must_insert_reserved) {
  588. /* if the extent was freed and then
  589. * reallocated before the delayed ref
  590. * entries were processed, we can end up
  591. * with an existing head ref without
  592. * the must_insert_reserved flag set.
  593. * Set it again here
  594. */
  595. existing->must_insert_reserved = update->must_insert_reserved;
  596. /*
  597. * update the num_bytes so we make sure the accounting
  598. * is done correctly
  599. */
  600. existing->num_bytes = update->num_bytes;
  601. }
  602. if (update->extent_op) {
  603. if (!existing->extent_op) {
  604. existing->extent_op = update->extent_op;
  605. } else {
  606. if (update->extent_op->update_key) {
  607. memcpy(&existing->extent_op->key,
  608. &update->extent_op->key,
  609. sizeof(update->extent_op->key));
  610. existing->extent_op->update_key = true;
  611. }
  612. if (update->extent_op->update_flags) {
  613. existing->extent_op->flags_to_set |=
  614. update->extent_op->flags_to_set;
  615. existing->extent_op->update_flags = true;
  616. }
  617. btrfs_free_delayed_extent_op(update->extent_op);
  618. }
  619. }
  620. /*
  621. * update the reference mod on the head to reflect this new operation,
  622. * only need the lock for this case cause we could be processing it
  623. * currently, for refs we just added we know we're a-ok.
  624. */
  625. old_ref_mod = existing->total_ref_mod;
  626. existing->ref_mod += update->ref_mod;
  627. existing->total_ref_mod += update->ref_mod;
  628. /*
  629. * If we are going to from a positive ref mod to a negative or vice
  630. * versa we need to make sure to adjust pending_csums accordingly.
  631. */
  632. if (existing->is_data) {
  633. u64 csum_leaves =
  634. btrfs_csum_bytes_to_leaves(fs_info,
  635. existing->num_bytes);
  636. if (existing->total_ref_mod >= 0 && old_ref_mod < 0) {
  637. delayed_refs->pending_csums -= existing->num_bytes;
  638. btrfs_delayed_refs_rsv_release(fs_info, csum_leaves);
  639. }
  640. if (existing->total_ref_mod < 0 && old_ref_mod >= 0) {
  641. delayed_refs->pending_csums += existing->num_bytes;
  642. trans->delayed_ref_updates += csum_leaves;
  643. }
  644. }
  645. /*
  646. * This handles the following conditions:
  647. *
  648. * 1. We had a ref mod of 0 or more and went negative, indicating that
  649. * we may be freeing space, so add our space to the
  650. * total_bytes_pinned counter.
  651. * 2. We were negative and went to 0 or positive, so no longer can say
  652. * that the space would be pinned, decrement our counter from the
  653. * total_bytes_pinned counter.
  654. * 3. We are now at 0 and have ->must_insert_reserved set, which means
  655. * this was a new allocation and then we dropped it, and thus must
  656. * add our space to the total_bytes_pinned counter.
  657. */
  658. if (existing->total_ref_mod < 0 && old_ref_mod >= 0)
  659. btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes);
  660. else if (existing->total_ref_mod >= 0 && old_ref_mod < 0)
  661. btrfs_mod_total_bytes_pinned(fs_info, flags, -existing->num_bytes);
  662. else if (existing->total_ref_mod == 0 && existing->must_insert_reserved)
  663. btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes);
  664. spin_unlock(&existing->lock);
  665. }
  666. static void init_delayed_ref_head(struct btrfs_delayed_ref_head *head_ref,
  667. struct btrfs_qgroup_extent_record *qrecord,
  668. u64 bytenr, u64 num_bytes, u64 ref_root,
  669. u64 reserved, int action, bool is_data,
  670. bool is_system)
  671. {
  672. int count_mod = 1;
  673. int must_insert_reserved = 0;
  674. /* If reserved is provided, it must be a data extent. */
  675. BUG_ON(!is_data && reserved);
  676. /*
  677. * The head node stores the sum of all the mods, so dropping a ref
  678. * should drop the sum in the head node by one.
  679. */
  680. if (action == BTRFS_UPDATE_DELAYED_HEAD)
  681. count_mod = 0;
  682. else if (action == BTRFS_DROP_DELAYED_REF)
  683. count_mod = -1;
  684. /*
  685. * BTRFS_ADD_DELAYED_EXTENT means that we need to update the reserved
  686. * accounting when the extent is finally added, or if a later
  687. * modification deletes the delayed ref without ever inserting the
  688. * extent into the extent allocation tree. ref->must_insert_reserved
  689. * is the flag used to record that accounting mods are required.
  690. *
  691. * Once we record must_insert_reserved, switch the action to
  692. * BTRFS_ADD_DELAYED_REF because other special casing is not required.
  693. */
  694. if (action == BTRFS_ADD_DELAYED_EXTENT)
  695. must_insert_reserved = 1;
  696. else
  697. must_insert_reserved = 0;
  698. refcount_set(&head_ref->refs, 1);
  699. head_ref->bytenr = bytenr;
  700. head_ref->num_bytes = num_bytes;
  701. head_ref->ref_mod = count_mod;
  702. head_ref->must_insert_reserved = must_insert_reserved;
  703. head_ref->is_data = is_data;
  704. head_ref->is_system = is_system;
  705. head_ref->ref_tree = RB_ROOT_CACHED;
  706. INIT_LIST_HEAD(&head_ref->ref_add_list);
  707. RB_CLEAR_NODE(&head_ref->href_node);
  708. head_ref->processing = 0;
  709. head_ref->total_ref_mod = count_mod;
  710. spin_lock_init(&head_ref->lock);
  711. mutex_init(&head_ref->mutex);
  712. if (qrecord) {
  713. if (ref_root && reserved) {
  714. qrecord->data_rsv = reserved;
  715. qrecord->data_rsv_refroot = ref_root;
  716. }
  717. qrecord->bytenr = bytenr;
  718. qrecord->num_bytes = num_bytes;
  719. qrecord->old_roots = NULL;
  720. }
  721. }
  722. /*
  723. * helper function to actually insert a head node into the rbtree.
  724. * this does all the dirty work in terms of maintaining the correct
  725. * overall modification count.
  726. */
  727. static noinline struct btrfs_delayed_ref_head *
  728. add_delayed_ref_head(struct btrfs_trans_handle *trans,
  729. struct btrfs_delayed_ref_head *head_ref,
  730. struct btrfs_qgroup_extent_record *qrecord,
  731. int action, int *qrecord_inserted_ret)
  732. {
  733. struct btrfs_delayed_ref_head *existing;
  734. struct btrfs_delayed_ref_root *delayed_refs;
  735. int qrecord_inserted = 0;
  736. delayed_refs = &trans->transaction->delayed_refs;
  737. /* Record qgroup extent info if provided */
  738. if (qrecord) {
  739. if (btrfs_qgroup_trace_extent_nolock(trans->fs_info,
  740. delayed_refs, qrecord))
  741. kfree(qrecord);
  742. else
  743. qrecord_inserted = 1;
  744. }
  745. trace_add_delayed_ref_head(trans->fs_info, head_ref, action);
  746. existing = htree_insert(&delayed_refs->href_root,
  747. &head_ref->href_node);
  748. if (existing) {
  749. update_existing_head_ref(trans, existing, head_ref);
  750. /*
  751. * we've updated the existing ref, free the newly
  752. * allocated ref
  753. */
  754. kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref);
  755. head_ref = existing;
  756. } else {
  757. u64 flags = btrfs_ref_head_to_space_flags(head_ref);
  758. if (head_ref->is_data && head_ref->ref_mod < 0) {
  759. delayed_refs->pending_csums += head_ref->num_bytes;
  760. trans->delayed_ref_updates +=
  761. btrfs_csum_bytes_to_leaves(trans->fs_info,
  762. head_ref->num_bytes);
  763. }
  764. if (head_ref->ref_mod < 0)
  765. btrfs_mod_total_bytes_pinned(trans->fs_info, flags,
  766. head_ref->num_bytes);
  767. delayed_refs->num_heads++;
  768. delayed_refs->num_heads_ready++;
  769. atomic_inc(&delayed_refs->num_entries);
  770. trans->delayed_ref_updates++;
  771. }
  772. if (qrecord_inserted_ret)
  773. *qrecord_inserted_ret = qrecord_inserted;
  774. return head_ref;
  775. }
  776. /*
  777. * init_delayed_ref_common - Initialize the structure which represents a
  778. * modification to a an extent.
  779. *
  780. * @fs_info: Internal to the mounted filesystem mount structure.
  781. *
  782. * @ref: The structure which is going to be initialized.
  783. *
  784. * @bytenr: The logical address of the extent for which a modification is
  785. * going to be recorded.
  786. *
  787. * @num_bytes: Size of the extent whose modification is being recorded.
  788. *
  789. * @ref_root: The id of the root where this modification has originated, this
  790. * can be either one of the well-known metadata trees or the
  791. * subvolume id which references this extent.
  792. *
  793. * @action: Can be one of BTRFS_ADD_DELAYED_REF/BTRFS_DROP_DELAYED_REF or
  794. * BTRFS_ADD_DELAYED_EXTENT
  795. *
  796. * @ref_type: Holds the type of the extent which is being recorded, can be
  797. * one of BTRFS_SHARED_BLOCK_REF_KEY/BTRFS_TREE_BLOCK_REF_KEY
  798. * when recording a metadata extent or BTRFS_SHARED_DATA_REF_KEY/
  799. * BTRFS_EXTENT_DATA_REF_KEY when recording data extent
  800. */
  801. static void init_delayed_ref_common(struct btrfs_fs_info *fs_info,
  802. struct btrfs_delayed_ref_node *ref,
  803. u64 bytenr, u64 num_bytes, u64 ref_root,
  804. int action, u8 ref_type)
  805. {
  806. u64 seq = 0;
  807. if (action == BTRFS_ADD_DELAYED_EXTENT)
  808. action = BTRFS_ADD_DELAYED_REF;
  809. if (is_fstree(ref_root))
  810. seq = atomic64_read(&fs_info->tree_mod_seq);
  811. refcount_set(&ref->refs, 1);
  812. ref->bytenr = bytenr;
  813. ref->num_bytes = num_bytes;
  814. ref->ref_mod = 1;
  815. ref->action = action;
  816. ref->is_head = 0;
  817. ref->in_tree = 1;
  818. ref->seq = seq;
  819. ref->type = ref_type;
  820. RB_CLEAR_NODE(&ref->ref_node);
  821. INIT_LIST_HEAD(&ref->add_list);
  822. }
  823. /*
  824. * add a delayed tree ref. This does all of the accounting required
  825. * to make sure the delayed ref is eventually processed before this
  826. * transaction commits.
  827. */
  828. int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans,
  829. struct btrfs_ref *generic_ref,
  830. struct btrfs_delayed_extent_op *extent_op)
  831. {
  832. struct btrfs_fs_info *fs_info = trans->fs_info;
  833. struct btrfs_delayed_tree_ref *ref;
  834. struct btrfs_delayed_ref_head *head_ref;
  835. struct btrfs_delayed_ref_root *delayed_refs;
  836. struct btrfs_qgroup_extent_record *record = NULL;
  837. int qrecord_inserted;
  838. bool is_system;
  839. int action = generic_ref->action;
  840. int level = generic_ref->tree_ref.level;
  841. int ret;
  842. u64 bytenr = generic_ref->bytenr;
  843. u64 num_bytes = generic_ref->len;
  844. u64 parent = generic_ref->parent;
  845. u8 ref_type;
  846. is_system = (generic_ref->real_root == BTRFS_CHUNK_TREE_OBJECTID);
  847. ASSERT(generic_ref->type == BTRFS_REF_METADATA && generic_ref->action);
  848. BUG_ON(extent_op && extent_op->is_data);
  849. ref = kmem_cache_alloc(btrfs_delayed_tree_ref_cachep, GFP_NOFS);
  850. if (!ref)
  851. return -ENOMEM;
  852. head_ref = kmem_cache_alloc(btrfs_delayed_ref_head_cachep, GFP_NOFS);
  853. if (!head_ref) {
  854. kmem_cache_free(btrfs_delayed_tree_ref_cachep, ref);
  855. return -ENOMEM;
  856. }
  857. if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags) &&
  858. is_fstree(generic_ref->real_root) &&
  859. is_fstree(generic_ref->tree_ref.root) &&
  860. !generic_ref->skip_qgroup) {
  861. record = kzalloc(sizeof(*record), GFP_NOFS);
  862. if (!record) {
  863. kmem_cache_free(btrfs_delayed_tree_ref_cachep, ref);
  864. kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref);
  865. return -ENOMEM;
  866. }
  867. }
  868. if (parent)
  869. ref_type = BTRFS_SHARED_BLOCK_REF_KEY;
  870. else
  871. ref_type = BTRFS_TREE_BLOCK_REF_KEY;
  872. init_delayed_ref_common(fs_info, &ref->node, bytenr, num_bytes,
  873. generic_ref->tree_ref.root, action, ref_type);
  874. ref->root = generic_ref->tree_ref.root;
  875. ref->parent = parent;
  876. ref->level = level;
  877. init_delayed_ref_head(head_ref, record, bytenr, num_bytes,
  878. generic_ref->tree_ref.root, 0, action, false,
  879. is_system);
  880. head_ref->extent_op = extent_op;
  881. delayed_refs = &trans->transaction->delayed_refs;
  882. spin_lock(&delayed_refs->lock);
  883. /*
  884. * insert both the head node and the new ref without dropping
  885. * the spin lock
  886. */
  887. head_ref = add_delayed_ref_head(trans, head_ref, record,
  888. action, &qrecord_inserted);
  889. ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node);
  890. spin_unlock(&delayed_refs->lock);
  891. /*
  892. * Need to update the delayed_refs_rsv with any changes we may have
  893. * made.
  894. */
  895. btrfs_update_delayed_refs_rsv(trans);
  896. trace_add_delayed_tree_ref(fs_info, &ref->node, ref,
  897. action == BTRFS_ADD_DELAYED_EXTENT ?
  898. BTRFS_ADD_DELAYED_REF : action);
  899. if (ret > 0)
  900. kmem_cache_free(btrfs_delayed_tree_ref_cachep, ref);
  901. if (qrecord_inserted)
  902. btrfs_qgroup_trace_extent_post(fs_info, record);
  903. return 0;
  904. }
  905. /*
  906. * add a delayed data ref. it's similar to btrfs_add_delayed_tree_ref.
  907. */
  908. int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans,
  909. struct btrfs_ref *generic_ref,
  910. u64 reserved)
  911. {
  912. struct btrfs_fs_info *fs_info = trans->fs_info;
  913. struct btrfs_delayed_data_ref *ref;
  914. struct btrfs_delayed_ref_head *head_ref;
  915. struct btrfs_delayed_ref_root *delayed_refs;
  916. struct btrfs_qgroup_extent_record *record = NULL;
  917. int qrecord_inserted;
  918. int action = generic_ref->action;
  919. int ret;
  920. u64 bytenr = generic_ref->bytenr;
  921. u64 num_bytes = generic_ref->len;
  922. u64 parent = generic_ref->parent;
  923. u64 ref_root = generic_ref->data_ref.ref_root;
  924. u64 owner = generic_ref->data_ref.ino;
  925. u64 offset = generic_ref->data_ref.offset;
  926. u8 ref_type;
  927. ASSERT(generic_ref->type == BTRFS_REF_DATA && action);
  928. ref = kmem_cache_alloc(btrfs_delayed_data_ref_cachep, GFP_NOFS);
  929. if (!ref)
  930. return -ENOMEM;
  931. if (parent)
  932. ref_type = BTRFS_SHARED_DATA_REF_KEY;
  933. else
  934. ref_type = BTRFS_EXTENT_DATA_REF_KEY;
  935. init_delayed_ref_common(fs_info, &ref->node, bytenr, num_bytes,
  936. ref_root, action, ref_type);
  937. ref->root = ref_root;
  938. ref->parent = parent;
  939. ref->objectid = owner;
  940. ref->offset = offset;
  941. head_ref = kmem_cache_alloc(btrfs_delayed_ref_head_cachep, GFP_NOFS);
  942. if (!head_ref) {
  943. kmem_cache_free(btrfs_delayed_data_ref_cachep, ref);
  944. return -ENOMEM;
  945. }
  946. if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags) &&
  947. is_fstree(ref_root) &&
  948. is_fstree(generic_ref->real_root) &&
  949. !generic_ref->skip_qgroup) {
  950. record = kzalloc(sizeof(*record), GFP_NOFS);
  951. if (!record) {
  952. kmem_cache_free(btrfs_delayed_data_ref_cachep, ref);
  953. kmem_cache_free(btrfs_delayed_ref_head_cachep,
  954. head_ref);
  955. return -ENOMEM;
  956. }
  957. }
  958. init_delayed_ref_head(head_ref, record, bytenr, num_bytes, ref_root,
  959. reserved, action, true, false);
  960. head_ref->extent_op = NULL;
  961. delayed_refs = &trans->transaction->delayed_refs;
  962. spin_lock(&delayed_refs->lock);
  963. /*
  964. * insert both the head node and the new ref without dropping
  965. * the spin lock
  966. */
  967. head_ref = add_delayed_ref_head(trans, head_ref, record,
  968. action, &qrecord_inserted);
  969. ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node);
  970. spin_unlock(&delayed_refs->lock);
  971. /*
  972. * Need to update the delayed_refs_rsv with any changes we may have
  973. * made.
  974. */
  975. btrfs_update_delayed_refs_rsv(trans);
  976. trace_add_delayed_data_ref(trans->fs_info, &ref->node, ref,
  977. action == BTRFS_ADD_DELAYED_EXTENT ?
  978. BTRFS_ADD_DELAYED_REF : action);
  979. if (ret > 0)
  980. kmem_cache_free(btrfs_delayed_data_ref_cachep, ref);
  981. if (qrecord_inserted)
  982. return btrfs_qgroup_trace_extent_post(fs_info, record);
  983. return 0;
  984. }
  985. int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans,
  986. u64 bytenr, u64 num_bytes,
  987. struct btrfs_delayed_extent_op *extent_op)
  988. {
  989. struct btrfs_delayed_ref_head *head_ref;
  990. struct btrfs_delayed_ref_root *delayed_refs;
  991. head_ref = kmem_cache_alloc(btrfs_delayed_ref_head_cachep, GFP_NOFS);
  992. if (!head_ref)
  993. return -ENOMEM;
  994. init_delayed_ref_head(head_ref, NULL, bytenr, num_bytes, 0, 0,
  995. BTRFS_UPDATE_DELAYED_HEAD, extent_op->is_data,
  996. false);
  997. head_ref->extent_op = extent_op;
  998. delayed_refs = &trans->transaction->delayed_refs;
  999. spin_lock(&delayed_refs->lock);
  1000. add_delayed_ref_head(trans, head_ref, NULL, BTRFS_UPDATE_DELAYED_HEAD,
  1001. NULL);
  1002. spin_unlock(&delayed_refs->lock);
  1003. /*
  1004. * Need to update the delayed_refs_rsv with any changes we may have
  1005. * made.
  1006. */
  1007. btrfs_update_delayed_refs_rsv(trans);
  1008. return 0;
  1009. }
  1010. /*
  1011. * This does a simple search for the head node for a given extent. Returns the
  1012. * head node if found, or NULL if not.
  1013. */
  1014. struct btrfs_delayed_ref_head *
  1015. btrfs_find_delayed_ref_head(struct btrfs_delayed_ref_root *delayed_refs, u64 bytenr)
  1016. {
  1017. lockdep_assert_held(&delayed_refs->lock);
  1018. return find_ref_head(delayed_refs, bytenr, false);
  1019. }
  1020. void __cold btrfs_delayed_ref_exit(void)
  1021. {
  1022. kmem_cache_destroy(btrfs_delayed_ref_head_cachep);
  1023. kmem_cache_destroy(btrfs_delayed_tree_ref_cachep);
  1024. kmem_cache_destroy(btrfs_delayed_data_ref_cachep);
  1025. kmem_cache_destroy(btrfs_delayed_extent_op_cachep);
  1026. }
  1027. int __init btrfs_delayed_ref_init(void)
  1028. {
  1029. btrfs_delayed_ref_head_cachep = kmem_cache_create(
  1030. "btrfs_delayed_ref_head",
  1031. sizeof(struct btrfs_delayed_ref_head), 0,
  1032. SLAB_MEM_SPREAD, NULL);
  1033. if (!btrfs_delayed_ref_head_cachep)
  1034. goto fail;
  1035. btrfs_delayed_tree_ref_cachep = kmem_cache_create(
  1036. "btrfs_delayed_tree_ref",
  1037. sizeof(struct btrfs_delayed_tree_ref), 0,
  1038. SLAB_MEM_SPREAD, NULL);
  1039. if (!btrfs_delayed_tree_ref_cachep)
  1040. goto fail;
  1041. btrfs_delayed_data_ref_cachep = kmem_cache_create(
  1042. "btrfs_delayed_data_ref",
  1043. sizeof(struct btrfs_delayed_data_ref), 0,
  1044. SLAB_MEM_SPREAD, NULL);
  1045. if (!btrfs_delayed_data_ref_cachep)
  1046. goto fail;
  1047. btrfs_delayed_extent_op_cachep = kmem_cache_create(
  1048. "btrfs_delayed_extent_op",
  1049. sizeof(struct btrfs_delayed_extent_op), 0,
  1050. SLAB_MEM_SPREAD, NULL);
  1051. if (!btrfs_delayed_extent_op_cachep)
  1052. goto fail;
  1053. return 0;
  1054. fail:
  1055. btrfs_delayed_ref_exit();
  1056. return -ENOMEM;
  1057. }