file-item.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2007 Oracle. All rights reserved.
  4. */
  5. #include <linux/bio.h>
  6. #include <linux/slab.h>
  7. #include <linux/pagemap.h>
  8. #include <linux/highmem.h>
  9. #include <linux/sched/mm.h>
  10. #include <crypto/hash.h>
  11. #include "ctree.h"
  12. #include "disk-io.h"
  13. #include "transaction.h"
  14. #include "volumes.h"
  15. #include "print-tree.h"
  16. #include "compression.h"
  17. #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \
  18. sizeof(struct btrfs_item) * 2) / \
  19. size) - 1))
  20. #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \
  21. PAGE_SIZE))
  22. /**
  23. * @inode - the inode we want to update the disk_i_size for
  24. * @new_i_size - the i_size we want to set to, 0 if we use i_size
  25. *
  26. * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read()
  27. * returns as it is perfectly fine with a file that has holes without hole file
  28. * extent items.
  29. *
  30. * However without NO_HOLES we need to only return the area that is contiguous
  31. * from the 0 offset of the file. Otherwise we could end up adjust i_size up
  32. * to an extent that has a gap in between.
  33. *
  34. * Finally new_i_size should only be set in the case of truncate where we're not
  35. * ready to use i_size_read() as the limiter yet.
  36. */
  37. void btrfs_inode_safe_disk_i_size_write(struct inode *inode, u64 new_i_size)
  38. {
  39. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  40. u64 start, end, i_size;
  41. int ret;
  42. i_size = new_i_size ?: i_size_read(inode);
  43. if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
  44. BTRFS_I(inode)->disk_i_size = i_size;
  45. return;
  46. }
  47. spin_lock(&BTRFS_I(inode)->lock);
  48. ret = find_contiguous_extent_bit(&BTRFS_I(inode)->file_extent_tree, 0,
  49. &start, &end, EXTENT_DIRTY);
  50. if (!ret && start == 0)
  51. i_size = min(i_size, end + 1);
  52. else
  53. i_size = 0;
  54. BTRFS_I(inode)->disk_i_size = i_size;
  55. spin_unlock(&BTRFS_I(inode)->lock);
  56. }
  57. /**
  58. * @inode - the inode we're modifying
  59. * @start - the start file offset of the file extent we've inserted
  60. * @len - the logical length of the file extent item
  61. *
  62. * Call when we are inserting a new file extent where there was none before.
  63. * Does not need to call this in the case where we're replacing an existing file
  64. * extent, however if not sure it's fine to call this multiple times.
  65. *
  66. * The start and len must match the file extent item, so thus must be sectorsize
  67. * aligned.
  68. */
  69. int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start,
  70. u64 len)
  71. {
  72. if (len == 0)
  73. return 0;
  74. ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize));
  75. if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
  76. return 0;
  77. return set_extent_bits(&inode->file_extent_tree, start, start + len - 1,
  78. EXTENT_DIRTY);
  79. }
  80. /**
  81. * @inode - the inode we're modifying
  82. * @start - the start file offset of the file extent we've inserted
  83. * @len - the logical length of the file extent item
  84. *
  85. * Called when we drop a file extent, for example when we truncate. Doesn't
  86. * need to be called for cases where we're replacing a file extent, like when
  87. * we've COWed a file extent.
  88. *
  89. * The start and len must match the file extent item, so thus must be sectorsize
  90. * aligned.
  91. */
  92. int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start,
  93. u64 len)
  94. {
  95. if (len == 0)
  96. return 0;
  97. ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) ||
  98. len == (u64)-1);
  99. if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
  100. return 0;
  101. return clear_extent_bit(&inode->file_extent_tree, start,
  102. start + len - 1, EXTENT_DIRTY, 0, 0, NULL);
  103. }
  104. static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info,
  105. u16 csum_size)
  106. {
  107. u32 ncsums = (PAGE_SIZE - sizeof(struct btrfs_ordered_sum)) / csum_size;
  108. return ncsums * fs_info->sectorsize;
  109. }
  110. int btrfs_insert_file_extent(struct btrfs_trans_handle *trans,
  111. struct btrfs_root *root,
  112. u64 objectid, u64 pos,
  113. u64 disk_offset, u64 disk_num_bytes,
  114. u64 num_bytes, u64 offset, u64 ram_bytes,
  115. u8 compression, u8 encryption, u16 other_encoding)
  116. {
  117. int ret = 0;
  118. struct btrfs_file_extent_item *item;
  119. struct btrfs_key file_key;
  120. struct btrfs_path *path;
  121. struct extent_buffer *leaf;
  122. path = btrfs_alloc_path();
  123. if (!path)
  124. return -ENOMEM;
  125. file_key.objectid = objectid;
  126. file_key.offset = pos;
  127. file_key.type = BTRFS_EXTENT_DATA_KEY;
  128. path->leave_spinning = 1;
  129. ret = btrfs_insert_empty_item(trans, root, path, &file_key,
  130. sizeof(*item));
  131. if (ret < 0)
  132. goto out;
  133. BUG_ON(ret); /* Can't happen */
  134. leaf = path->nodes[0];
  135. item = btrfs_item_ptr(leaf, path->slots[0],
  136. struct btrfs_file_extent_item);
  137. btrfs_set_file_extent_disk_bytenr(leaf, item, disk_offset);
  138. btrfs_set_file_extent_disk_num_bytes(leaf, item, disk_num_bytes);
  139. btrfs_set_file_extent_offset(leaf, item, offset);
  140. btrfs_set_file_extent_num_bytes(leaf, item, num_bytes);
  141. btrfs_set_file_extent_ram_bytes(leaf, item, ram_bytes);
  142. btrfs_set_file_extent_generation(leaf, item, trans->transid);
  143. btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
  144. btrfs_set_file_extent_compression(leaf, item, compression);
  145. btrfs_set_file_extent_encryption(leaf, item, encryption);
  146. btrfs_set_file_extent_other_encoding(leaf, item, other_encoding);
  147. btrfs_mark_buffer_dirty(leaf);
  148. out:
  149. btrfs_free_path(path);
  150. return ret;
  151. }
  152. static struct btrfs_csum_item *
  153. btrfs_lookup_csum(struct btrfs_trans_handle *trans,
  154. struct btrfs_root *root,
  155. struct btrfs_path *path,
  156. u64 bytenr, int cow)
  157. {
  158. struct btrfs_fs_info *fs_info = root->fs_info;
  159. int ret;
  160. struct btrfs_key file_key;
  161. struct btrfs_key found_key;
  162. struct btrfs_csum_item *item;
  163. struct extent_buffer *leaf;
  164. u64 csum_offset = 0;
  165. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  166. int csums_in_item;
  167. file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
  168. file_key.offset = bytenr;
  169. file_key.type = BTRFS_EXTENT_CSUM_KEY;
  170. ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow);
  171. if (ret < 0)
  172. goto fail;
  173. leaf = path->nodes[0];
  174. if (ret > 0) {
  175. ret = 1;
  176. if (path->slots[0] == 0)
  177. goto fail;
  178. path->slots[0]--;
  179. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  180. if (found_key.type != BTRFS_EXTENT_CSUM_KEY)
  181. goto fail;
  182. csum_offset = (bytenr - found_key.offset) >>
  183. fs_info->sb->s_blocksize_bits;
  184. csums_in_item = btrfs_item_size_nr(leaf, path->slots[0]);
  185. csums_in_item /= csum_size;
  186. if (csum_offset == csums_in_item) {
  187. ret = -EFBIG;
  188. goto fail;
  189. } else if (csum_offset > csums_in_item) {
  190. goto fail;
  191. }
  192. }
  193. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
  194. item = (struct btrfs_csum_item *)((unsigned char *)item +
  195. csum_offset * csum_size);
  196. return item;
  197. fail:
  198. if (ret > 0)
  199. ret = -ENOENT;
  200. return ERR_PTR(ret);
  201. }
  202. int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
  203. struct btrfs_root *root,
  204. struct btrfs_path *path, u64 objectid,
  205. u64 offset, int mod)
  206. {
  207. int ret;
  208. struct btrfs_key file_key;
  209. int ins_len = mod < 0 ? -1 : 0;
  210. int cow = mod != 0;
  211. file_key.objectid = objectid;
  212. file_key.offset = offset;
  213. file_key.type = BTRFS_EXTENT_DATA_KEY;
  214. ret = btrfs_search_slot(trans, root, &file_key, path, ins_len, cow);
  215. return ret;
  216. }
  217. /**
  218. * btrfs_lookup_bio_sums - Look up checksums for a bio.
  219. * @inode: inode that the bio is for.
  220. * @bio: bio to look up.
  221. * @offset: Unless (u64)-1, look up checksums for this offset in the file.
  222. * If (u64)-1, use the page offsets from the bio instead.
  223. * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return
  224. * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If
  225. * NULL, the checksum buffer is allocated and returned in
  226. * btrfs_io_bio(bio)->csum instead.
  227. *
  228. * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise.
  229. */
  230. blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio,
  231. u64 offset, u8 *dst)
  232. {
  233. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  234. struct bio_vec bvec;
  235. struct bvec_iter iter;
  236. struct btrfs_csum_item *item = NULL;
  237. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  238. struct btrfs_path *path;
  239. const bool page_offsets = (offset == (u64)-1);
  240. u8 *csum;
  241. u64 item_start_offset = 0;
  242. u64 item_last_offset = 0;
  243. u64 disk_bytenr;
  244. u64 page_bytes_left;
  245. u32 diff;
  246. int nblocks;
  247. int count = 0;
  248. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  249. path = btrfs_alloc_path();
  250. if (!path)
  251. return BLK_STS_RESOURCE;
  252. nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits;
  253. if (!dst) {
  254. struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio);
  255. if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
  256. btrfs_bio->csum = kmalloc_array(nblocks, csum_size,
  257. GFP_NOFS);
  258. if (!btrfs_bio->csum) {
  259. btrfs_free_path(path);
  260. return BLK_STS_RESOURCE;
  261. }
  262. } else {
  263. btrfs_bio->csum = btrfs_bio->csum_inline;
  264. }
  265. csum = btrfs_bio->csum;
  266. } else {
  267. csum = dst;
  268. }
  269. if (bio->bi_iter.bi_size > PAGE_SIZE * 8)
  270. path->reada = READA_FORWARD;
  271. /*
  272. * the free space stuff is only read when it hasn't been
  273. * updated in the current transaction. So, we can safely
  274. * read from the commit root and sidestep a nasty deadlock
  275. * between reading the free space cache and updating the csum tree.
  276. */
  277. if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
  278. path->search_commit_root = 1;
  279. path->skip_locking = 1;
  280. }
  281. disk_bytenr = (u64)bio->bi_iter.bi_sector << 9;
  282. bio_for_each_segment(bvec, bio, iter) {
  283. page_bytes_left = bvec.bv_len;
  284. if (count)
  285. goto next;
  286. if (page_offsets)
  287. offset = page_offset(bvec.bv_page) + bvec.bv_offset;
  288. count = btrfs_find_ordered_sum(BTRFS_I(inode), offset,
  289. disk_bytenr, csum, nblocks);
  290. if (count)
  291. goto found;
  292. if (!item || disk_bytenr < item_start_offset ||
  293. disk_bytenr >= item_last_offset) {
  294. struct btrfs_key found_key;
  295. u32 item_size;
  296. if (item)
  297. btrfs_release_path(path);
  298. item = btrfs_lookup_csum(NULL, fs_info->csum_root,
  299. path, disk_bytenr, 0);
  300. if (IS_ERR(item)) {
  301. count = 1;
  302. memset(csum, 0, csum_size);
  303. if (BTRFS_I(inode)->root->root_key.objectid ==
  304. BTRFS_DATA_RELOC_TREE_OBJECTID) {
  305. set_extent_bits(io_tree, offset,
  306. offset + fs_info->sectorsize - 1,
  307. EXTENT_NODATASUM);
  308. } else {
  309. btrfs_info_rl(fs_info,
  310. "no csum found for inode %llu start %llu",
  311. btrfs_ino(BTRFS_I(inode)), offset);
  312. }
  313. item = NULL;
  314. btrfs_release_path(path);
  315. goto found;
  316. }
  317. btrfs_item_key_to_cpu(path->nodes[0], &found_key,
  318. path->slots[0]);
  319. item_start_offset = found_key.offset;
  320. item_size = btrfs_item_size_nr(path->nodes[0],
  321. path->slots[0]);
  322. item_last_offset = item_start_offset +
  323. (item_size / csum_size) *
  324. fs_info->sectorsize;
  325. item = btrfs_item_ptr(path->nodes[0], path->slots[0],
  326. struct btrfs_csum_item);
  327. }
  328. /*
  329. * this byte range must be able to fit inside
  330. * a single leaf so it will also fit inside a u32
  331. */
  332. diff = disk_bytenr - item_start_offset;
  333. diff = diff / fs_info->sectorsize;
  334. diff = diff * csum_size;
  335. count = min_t(int, nblocks, (item_last_offset - disk_bytenr) >>
  336. inode->i_sb->s_blocksize_bits);
  337. read_extent_buffer(path->nodes[0], csum,
  338. ((unsigned long)item) + diff,
  339. csum_size * count);
  340. found:
  341. csum += count * csum_size;
  342. nblocks -= count;
  343. next:
  344. while (count > 0) {
  345. count--;
  346. disk_bytenr += fs_info->sectorsize;
  347. offset += fs_info->sectorsize;
  348. page_bytes_left -= fs_info->sectorsize;
  349. if (!page_bytes_left)
  350. break; /* move to next bio */
  351. }
  352. }
  353. WARN_ON_ONCE(count);
  354. btrfs_free_path(path);
  355. return BLK_STS_OK;
  356. }
  357. int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end,
  358. struct list_head *list, int search_commit)
  359. {
  360. struct btrfs_fs_info *fs_info = root->fs_info;
  361. struct btrfs_key key;
  362. struct btrfs_path *path;
  363. struct extent_buffer *leaf;
  364. struct btrfs_ordered_sum *sums;
  365. struct btrfs_csum_item *item;
  366. LIST_HEAD(tmplist);
  367. unsigned long offset;
  368. int ret;
  369. size_t size;
  370. u64 csum_end;
  371. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  372. ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
  373. IS_ALIGNED(end + 1, fs_info->sectorsize));
  374. path = btrfs_alloc_path();
  375. if (!path)
  376. return -ENOMEM;
  377. if (search_commit) {
  378. path->skip_locking = 1;
  379. path->reada = READA_FORWARD;
  380. path->search_commit_root = 1;
  381. }
  382. key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
  383. key.offset = start;
  384. key.type = BTRFS_EXTENT_CSUM_KEY;
  385. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  386. if (ret < 0)
  387. goto fail;
  388. if (ret > 0 && path->slots[0] > 0) {
  389. leaf = path->nodes[0];
  390. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  391. if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
  392. key.type == BTRFS_EXTENT_CSUM_KEY) {
  393. offset = (start - key.offset) >>
  394. fs_info->sb->s_blocksize_bits;
  395. if (offset * csum_size <
  396. btrfs_item_size_nr(leaf, path->slots[0] - 1))
  397. path->slots[0]--;
  398. }
  399. }
  400. while (start <= end) {
  401. leaf = path->nodes[0];
  402. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  403. ret = btrfs_next_leaf(root, path);
  404. if (ret < 0)
  405. goto fail;
  406. if (ret > 0)
  407. break;
  408. leaf = path->nodes[0];
  409. }
  410. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  411. if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
  412. key.type != BTRFS_EXTENT_CSUM_KEY ||
  413. key.offset > end)
  414. break;
  415. if (key.offset > start)
  416. start = key.offset;
  417. size = btrfs_item_size_nr(leaf, path->slots[0]);
  418. csum_end = key.offset + (size / csum_size) * fs_info->sectorsize;
  419. if (csum_end <= start) {
  420. path->slots[0]++;
  421. continue;
  422. }
  423. csum_end = min(csum_end, end + 1);
  424. item = btrfs_item_ptr(path->nodes[0], path->slots[0],
  425. struct btrfs_csum_item);
  426. while (start < csum_end) {
  427. size = min_t(size_t, csum_end - start,
  428. max_ordered_sum_bytes(fs_info, csum_size));
  429. sums = kzalloc(btrfs_ordered_sum_size(fs_info, size),
  430. GFP_NOFS);
  431. if (!sums) {
  432. ret = -ENOMEM;
  433. goto fail;
  434. }
  435. sums->bytenr = start;
  436. sums->len = (int)size;
  437. offset = (start - key.offset) >>
  438. fs_info->sb->s_blocksize_bits;
  439. offset *= csum_size;
  440. size >>= fs_info->sb->s_blocksize_bits;
  441. read_extent_buffer(path->nodes[0],
  442. sums->sums,
  443. ((unsigned long)item) + offset,
  444. csum_size * size);
  445. start += fs_info->sectorsize * size;
  446. list_add_tail(&sums->list, &tmplist);
  447. }
  448. path->slots[0]++;
  449. }
  450. ret = 0;
  451. fail:
  452. while (ret < 0 && !list_empty(&tmplist)) {
  453. sums = list_entry(tmplist.next, struct btrfs_ordered_sum, list);
  454. list_del(&sums->list);
  455. kfree(sums);
  456. }
  457. list_splice_tail(&tmplist, list);
  458. btrfs_free_path(path);
  459. return ret;
  460. }
  461. /*
  462. * btrfs_csum_one_bio - Calculates checksums of the data contained inside a bio
  463. * @inode: Owner of the data inside the bio
  464. * @bio: Contains the data to be checksummed
  465. * @file_start: offset in file this bio begins to describe
  466. * @contig: Boolean. If true/1 means all bio vecs in this bio are
  467. * contiguous and they begin at @file_start in the file. False/0
  468. * means this bio can contains potentially discontigous bio vecs
  469. * so the logical offset of each should be calculated separately.
  470. */
  471. blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio,
  472. u64 file_start, int contig)
  473. {
  474. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  475. SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  476. struct btrfs_ordered_sum *sums;
  477. struct btrfs_ordered_extent *ordered = NULL;
  478. char *data;
  479. struct bvec_iter iter;
  480. struct bio_vec bvec;
  481. int index;
  482. int nr_sectors;
  483. unsigned long total_bytes = 0;
  484. unsigned long this_sum_bytes = 0;
  485. int i;
  486. u64 offset;
  487. unsigned nofs_flag;
  488. const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  489. nofs_flag = memalloc_nofs_save();
  490. sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),
  491. GFP_KERNEL);
  492. memalloc_nofs_restore(nofs_flag);
  493. if (!sums)
  494. return BLK_STS_RESOURCE;
  495. sums->len = bio->bi_iter.bi_size;
  496. INIT_LIST_HEAD(&sums->list);
  497. if (contig)
  498. offset = file_start;
  499. else
  500. offset = 0; /* shut up gcc */
  501. sums->bytenr = (u64)bio->bi_iter.bi_sector << 9;
  502. index = 0;
  503. shash->tfm = fs_info->csum_shash;
  504. bio_for_each_segment(bvec, bio, iter) {
  505. if (!contig)
  506. offset = page_offset(bvec.bv_page) + bvec.bv_offset;
  507. if (!ordered) {
  508. ordered = btrfs_lookup_ordered_extent(inode, offset);
  509. /*
  510. * The bio range is not covered by any ordered extent,
  511. * must be a code logic error.
  512. */
  513. if (unlikely(!ordered)) {
  514. WARN(1, KERN_WARNING
  515. "no ordered extent for root %llu ino %llu offset %llu\n",
  516. inode->root->root_key.objectid,
  517. btrfs_ino(inode), offset);
  518. kvfree(sums);
  519. return BLK_STS_IOERR;
  520. }
  521. }
  522. nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info,
  523. bvec.bv_len + fs_info->sectorsize
  524. - 1);
  525. for (i = 0; i < nr_sectors; i++) {
  526. if (offset >= ordered->file_offset + ordered->num_bytes ||
  527. offset < ordered->file_offset) {
  528. unsigned long bytes_left;
  529. sums->len = this_sum_bytes;
  530. this_sum_bytes = 0;
  531. btrfs_add_ordered_sum(ordered, sums);
  532. btrfs_put_ordered_extent(ordered);
  533. bytes_left = bio->bi_iter.bi_size - total_bytes;
  534. nofs_flag = memalloc_nofs_save();
  535. sums = kvzalloc(btrfs_ordered_sum_size(fs_info,
  536. bytes_left), GFP_KERNEL);
  537. memalloc_nofs_restore(nofs_flag);
  538. BUG_ON(!sums); /* -ENOMEM */
  539. sums->len = bytes_left;
  540. ordered = btrfs_lookup_ordered_extent(inode,
  541. offset);
  542. ASSERT(ordered); /* Logic error */
  543. sums->bytenr = ((u64)bio->bi_iter.bi_sector << 9)
  544. + total_bytes;
  545. index = 0;
  546. }
  547. data = kmap_atomic(bvec.bv_page);
  548. crypto_shash_digest(shash, data + bvec.bv_offset
  549. + (i * fs_info->sectorsize),
  550. fs_info->sectorsize,
  551. sums->sums + index);
  552. kunmap_atomic(data);
  553. index += csum_size;
  554. offset += fs_info->sectorsize;
  555. this_sum_bytes += fs_info->sectorsize;
  556. total_bytes += fs_info->sectorsize;
  557. }
  558. }
  559. this_sum_bytes = 0;
  560. btrfs_add_ordered_sum(ordered, sums);
  561. btrfs_put_ordered_extent(ordered);
  562. return 0;
  563. }
  564. /*
  565. * helper function for csum removal, this expects the
  566. * key to describe the csum pointed to by the path, and it expects
  567. * the csum to overlap the range [bytenr, len]
  568. *
  569. * The csum should not be entirely contained in the range and the
  570. * range should not be entirely contained in the csum.
  571. *
  572. * This calls btrfs_truncate_item with the correct args based on the
  573. * overlap, and fixes up the key as required.
  574. */
  575. static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info,
  576. struct btrfs_path *path,
  577. struct btrfs_key *key,
  578. u64 bytenr, u64 len)
  579. {
  580. struct extent_buffer *leaf;
  581. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  582. u64 csum_end;
  583. u64 end_byte = bytenr + len;
  584. u32 blocksize_bits = fs_info->sb->s_blocksize_bits;
  585. leaf = path->nodes[0];
  586. csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size;
  587. csum_end <<= fs_info->sb->s_blocksize_bits;
  588. csum_end += key->offset;
  589. if (key->offset < bytenr && csum_end <= end_byte) {
  590. /*
  591. * [ bytenr - len ]
  592. * [ ]
  593. * [csum ]
  594. * A simple truncate off the end of the item
  595. */
  596. u32 new_size = (bytenr - key->offset) >> blocksize_bits;
  597. new_size *= csum_size;
  598. btrfs_truncate_item(path, new_size, 1);
  599. } else if (key->offset >= bytenr && csum_end > end_byte &&
  600. end_byte > key->offset) {
  601. /*
  602. * [ bytenr - len ]
  603. * [ ]
  604. * [csum ]
  605. * we need to truncate from the beginning of the csum
  606. */
  607. u32 new_size = (csum_end - end_byte) >> blocksize_bits;
  608. new_size *= csum_size;
  609. btrfs_truncate_item(path, new_size, 0);
  610. key->offset = end_byte;
  611. btrfs_set_item_key_safe(fs_info, path, key);
  612. } else {
  613. BUG();
  614. }
  615. }
  616. /*
  617. * deletes the csum items from the csum tree for a given
  618. * range of bytes.
  619. */
  620. int btrfs_del_csums(struct btrfs_trans_handle *trans,
  621. struct btrfs_root *root, u64 bytenr, u64 len)
  622. {
  623. struct btrfs_fs_info *fs_info = trans->fs_info;
  624. struct btrfs_path *path;
  625. struct btrfs_key key;
  626. u64 end_byte = bytenr + len;
  627. u64 csum_end;
  628. struct extent_buffer *leaf;
  629. int ret = 0;
  630. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  631. int blocksize_bits = fs_info->sb->s_blocksize_bits;
  632. ASSERT(root == fs_info->csum_root ||
  633. root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
  634. path = btrfs_alloc_path();
  635. if (!path)
  636. return -ENOMEM;
  637. while (1) {
  638. key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
  639. key.offset = end_byte - 1;
  640. key.type = BTRFS_EXTENT_CSUM_KEY;
  641. path->leave_spinning = 1;
  642. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  643. if (ret > 0) {
  644. ret = 0;
  645. if (path->slots[0] == 0)
  646. break;
  647. path->slots[0]--;
  648. } else if (ret < 0) {
  649. break;
  650. }
  651. leaf = path->nodes[0];
  652. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  653. if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
  654. key.type != BTRFS_EXTENT_CSUM_KEY) {
  655. break;
  656. }
  657. if (key.offset >= end_byte)
  658. break;
  659. csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size;
  660. csum_end <<= blocksize_bits;
  661. csum_end += key.offset;
  662. /* this csum ends before we start, we're done */
  663. if (csum_end <= bytenr)
  664. break;
  665. /* delete the entire item, it is inside our range */
  666. if (key.offset >= bytenr && csum_end <= end_byte) {
  667. int del_nr = 1;
  668. /*
  669. * Check how many csum items preceding this one in this
  670. * leaf correspond to our range and then delete them all
  671. * at once.
  672. */
  673. if (key.offset > bytenr && path->slots[0] > 0) {
  674. int slot = path->slots[0] - 1;
  675. while (slot >= 0) {
  676. struct btrfs_key pk;
  677. btrfs_item_key_to_cpu(leaf, &pk, slot);
  678. if (pk.offset < bytenr ||
  679. pk.type != BTRFS_EXTENT_CSUM_KEY ||
  680. pk.objectid !=
  681. BTRFS_EXTENT_CSUM_OBJECTID)
  682. break;
  683. path->slots[0] = slot;
  684. del_nr++;
  685. key.offset = pk.offset;
  686. slot--;
  687. }
  688. }
  689. ret = btrfs_del_items(trans, root, path,
  690. path->slots[0], del_nr);
  691. if (ret)
  692. break;
  693. if (key.offset == bytenr)
  694. break;
  695. } else if (key.offset < bytenr && csum_end > end_byte) {
  696. unsigned long offset;
  697. unsigned long shift_len;
  698. unsigned long item_offset;
  699. /*
  700. * [ bytenr - len ]
  701. * [csum ]
  702. *
  703. * Our bytes are in the middle of the csum,
  704. * we need to split this item and insert a new one.
  705. *
  706. * But we can't drop the path because the
  707. * csum could change, get removed, extended etc.
  708. *
  709. * The trick here is the max size of a csum item leaves
  710. * enough room in the tree block for a single
  711. * item header. So, we split the item in place,
  712. * adding a new header pointing to the existing
  713. * bytes. Then we loop around again and we have
  714. * a nicely formed csum item that we can neatly
  715. * truncate.
  716. */
  717. offset = (bytenr - key.offset) >> blocksize_bits;
  718. offset *= csum_size;
  719. shift_len = (len >> blocksize_bits) * csum_size;
  720. item_offset = btrfs_item_ptr_offset(leaf,
  721. path->slots[0]);
  722. memzero_extent_buffer(leaf, item_offset + offset,
  723. shift_len);
  724. key.offset = bytenr;
  725. /*
  726. * btrfs_split_item returns -EAGAIN when the
  727. * item changed size or key
  728. */
  729. ret = btrfs_split_item(trans, root, path, &key, offset);
  730. if (ret && ret != -EAGAIN) {
  731. btrfs_abort_transaction(trans, ret);
  732. break;
  733. }
  734. ret = 0;
  735. key.offset = end_byte - 1;
  736. } else {
  737. truncate_one_csum(fs_info, path, &key, bytenr, len);
  738. if (key.offset < bytenr)
  739. break;
  740. }
  741. btrfs_release_path(path);
  742. }
  743. btrfs_free_path(path);
  744. return ret;
  745. }
  746. int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
  747. struct btrfs_root *root,
  748. struct btrfs_ordered_sum *sums)
  749. {
  750. struct btrfs_fs_info *fs_info = root->fs_info;
  751. struct btrfs_key file_key;
  752. struct btrfs_key found_key;
  753. struct btrfs_path *path;
  754. struct btrfs_csum_item *item;
  755. struct btrfs_csum_item *item_end;
  756. struct extent_buffer *leaf = NULL;
  757. u64 next_offset;
  758. u64 total_bytes = 0;
  759. u64 csum_offset;
  760. u64 bytenr;
  761. u32 nritems;
  762. u32 ins_size;
  763. int index = 0;
  764. int found_next;
  765. int ret;
  766. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  767. path = btrfs_alloc_path();
  768. if (!path)
  769. return -ENOMEM;
  770. again:
  771. next_offset = (u64)-1;
  772. found_next = 0;
  773. bytenr = sums->bytenr + total_bytes;
  774. file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
  775. file_key.offset = bytenr;
  776. file_key.type = BTRFS_EXTENT_CSUM_KEY;
  777. item = btrfs_lookup_csum(trans, root, path, bytenr, 1);
  778. if (!IS_ERR(item)) {
  779. ret = 0;
  780. leaf = path->nodes[0];
  781. item_end = btrfs_item_ptr(leaf, path->slots[0],
  782. struct btrfs_csum_item);
  783. item_end = (struct btrfs_csum_item *)((char *)item_end +
  784. btrfs_item_size_nr(leaf, path->slots[0]));
  785. goto found;
  786. }
  787. ret = PTR_ERR(item);
  788. if (ret != -EFBIG && ret != -ENOENT)
  789. goto out;
  790. if (ret == -EFBIG) {
  791. u32 item_size;
  792. /* we found one, but it isn't big enough yet */
  793. leaf = path->nodes[0];
  794. item_size = btrfs_item_size_nr(leaf, path->slots[0]);
  795. if ((item_size / csum_size) >=
  796. MAX_CSUM_ITEMS(fs_info, csum_size)) {
  797. /* already at max size, make a new one */
  798. goto insert;
  799. }
  800. } else {
  801. int slot = path->slots[0] + 1;
  802. /* we didn't find a csum item, insert one */
  803. nritems = btrfs_header_nritems(path->nodes[0]);
  804. if (!nritems || (path->slots[0] >= nritems - 1)) {
  805. ret = btrfs_next_leaf(root, path);
  806. if (ret < 0) {
  807. goto out;
  808. } else if (ret > 0) {
  809. found_next = 1;
  810. goto insert;
  811. }
  812. slot = path->slots[0];
  813. }
  814. btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot);
  815. if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
  816. found_key.type != BTRFS_EXTENT_CSUM_KEY) {
  817. found_next = 1;
  818. goto insert;
  819. }
  820. next_offset = found_key.offset;
  821. found_next = 1;
  822. goto insert;
  823. }
  824. /*
  825. * At this point, we know the tree has a checksum item that ends at an
  826. * offset matching the start of the checksum range we want to insert.
  827. * We try to extend that item as much as possible and then add as many
  828. * checksums to it as they fit.
  829. *
  830. * First check if the leaf has enough free space for at least one
  831. * checksum. If it has go directly to the item extension code, otherwise
  832. * release the path and do a search for insertion before the extension.
  833. */
  834. if (btrfs_leaf_free_space(leaf) >= csum_size) {
  835. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  836. csum_offset = (bytenr - found_key.offset) >>
  837. fs_info->sb->s_blocksize_bits;
  838. goto extend_csum;
  839. }
  840. btrfs_release_path(path);
  841. ret = btrfs_search_slot(trans, root, &file_key, path,
  842. csum_size, 1);
  843. if (ret < 0)
  844. goto out;
  845. if (ret > 0) {
  846. if (path->slots[0] == 0)
  847. goto insert;
  848. path->slots[0]--;
  849. }
  850. leaf = path->nodes[0];
  851. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  852. csum_offset = (bytenr - found_key.offset) >>
  853. fs_info->sb->s_blocksize_bits;
  854. if (found_key.type != BTRFS_EXTENT_CSUM_KEY ||
  855. found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
  856. csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) {
  857. goto insert;
  858. }
  859. extend_csum:
  860. if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) /
  861. csum_size) {
  862. int extend_nr;
  863. u64 tmp;
  864. u32 diff;
  865. tmp = sums->len - total_bytes;
  866. tmp >>= fs_info->sb->s_blocksize_bits;
  867. WARN_ON(tmp < 1);
  868. extend_nr = max_t(int, 1, (int)tmp);
  869. diff = (csum_offset + extend_nr) * csum_size;
  870. diff = min(diff,
  871. MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);
  872. diff = diff - btrfs_item_size_nr(leaf, path->slots[0]);
  873. diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);
  874. diff /= csum_size;
  875. diff *= csum_size;
  876. btrfs_extend_item(path, diff);
  877. ret = 0;
  878. goto csum;
  879. }
  880. insert:
  881. btrfs_release_path(path);
  882. csum_offset = 0;
  883. if (found_next) {
  884. u64 tmp;
  885. tmp = sums->len - total_bytes;
  886. tmp >>= fs_info->sb->s_blocksize_bits;
  887. tmp = min(tmp, (next_offset - file_key.offset) >>
  888. fs_info->sb->s_blocksize_bits);
  889. tmp = max_t(u64, 1, tmp);
  890. tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size));
  891. ins_size = csum_size * tmp;
  892. } else {
  893. ins_size = csum_size;
  894. }
  895. path->leave_spinning = 1;
  896. ret = btrfs_insert_empty_item(trans, root, path, &file_key,
  897. ins_size);
  898. path->leave_spinning = 0;
  899. if (ret < 0)
  900. goto out;
  901. if (WARN_ON(ret != 0))
  902. goto out;
  903. leaf = path->nodes[0];
  904. csum:
  905. item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
  906. item_end = (struct btrfs_csum_item *)((unsigned char *)item +
  907. btrfs_item_size_nr(leaf, path->slots[0]));
  908. item = (struct btrfs_csum_item *)((unsigned char *)item +
  909. csum_offset * csum_size);
  910. found:
  911. ins_size = (u32)(sums->len - total_bytes) >>
  912. fs_info->sb->s_blocksize_bits;
  913. ins_size *= csum_size;
  914. ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item,
  915. ins_size);
  916. write_extent_buffer(leaf, sums->sums + index, (unsigned long)item,
  917. ins_size);
  918. index += ins_size;
  919. ins_size /= csum_size;
  920. total_bytes += ins_size * fs_info->sectorsize;
  921. btrfs_mark_buffer_dirty(path->nodes[0]);
  922. if (total_bytes < sums->len) {
  923. btrfs_release_path(path);
  924. cond_resched();
  925. goto again;
  926. }
  927. out:
  928. btrfs_free_path(path);
  929. return ret;
  930. }
  931. void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
  932. const struct btrfs_path *path,
  933. struct btrfs_file_extent_item *fi,
  934. const bool new_inline,
  935. struct extent_map *em)
  936. {
  937. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  938. struct btrfs_root *root = inode->root;
  939. struct extent_buffer *leaf = path->nodes[0];
  940. const int slot = path->slots[0];
  941. struct btrfs_key key;
  942. u64 extent_start, extent_end;
  943. u64 bytenr;
  944. u8 type = btrfs_file_extent_type(leaf, fi);
  945. int compress_type = btrfs_file_extent_compression(leaf, fi);
  946. btrfs_item_key_to_cpu(leaf, &key, slot);
  947. extent_start = key.offset;
  948. extent_end = btrfs_file_extent_end(path);
  949. em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  950. if (type == BTRFS_FILE_EXTENT_REG ||
  951. type == BTRFS_FILE_EXTENT_PREALLOC) {
  952. em->start = extent_start;
  953. em->len = extent_end - extent_start;
  954. em->orig_start = extent_start -
  955. btrfs_file_extent_offset(leaf, fi);
  956. em->orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
  957. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  958. if (bytenr == 0) {
  959. em->block_start = EXTENT_MAP_HOLE;
  960. return;
  961. }
  962. if (compress_type != BTRFS_COMPRESS_NONE) {
  963. set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  964. em->compress_type = compress_type;
  965. em->block_start = bytenr;
  966. em->block_len = em->orig_block_len;
  967. } else {
  968. bytenr += btrfs_file_extent_offset(leaf, fi);
  969. em->block_start = bytenr;
  970. em->block_len = em->len;
  971. if (type == BTRFS_FILE_EXTENT_PREALLOC)
  972. set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
  973. }
  974. } else if (type == BTRFS_FILE_EXTENT_INLINE) {
  975. em->block_start = EXTENT_MAP_INLINE;
  976. em->start = extent_start;
  977. em->len = extent_end - extent_start;
  978. /*
  979. * Initialize orig_start and block_len with the same values
  980. * as in inode.c:btrfs_get_extent().
  981. */
  982. em->orig_start = EXTENT_MAP_HOLE;
  983. em->block_len = (u64)-1;
  984. if (!new_inline && compress_type != BTRFS_COMPRESS_NONE) {
  985. set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  986. em->compress_type = compress_type;
  987. }
  988. } else {
  989. btrfs_err(fs_info,
  990. "unknown file extent item type %d, inode %llu, offset %llu, "
  991. "root %llu", type, btrfs_ino(inode), extent_start,
  992. root->root_key.objectid);
  993. }
  994. }
  995. /*
  996. * Returns the end offset (non inclusive) of the file extent item the given path
  997. * points to. If it points to an inline extent, the returned offset is rounded
  998. * up to the sector size.
  999. */
  1000. u64 btrfs_file_extent_end(const struct btrfs_path *path)
  1001. {
  1002. const struct extent_buffer *leaf = path->nodes[0];
  1003. const int slot = path->slots[0];
  1004. struct btrfs_file_extent_item *fi;
  1005. struct btrfs_key key;
  1006. u64 end;
  1007. btrfs_item_key_to_cpu(leaf, &key, slot);
  1008. ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);
  1009. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  1010. if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) {
  1011. end = btrfs_file_extent_ram_bytes(leaf, fi);
  1012. end = ALIGN(key.offset + end, leaf->fs_info->sectorsize);
  1013. } else {
  1014. end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  1015. }
  1016. return end;
  1017. }