remap_range.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/slab.h>
  3. #include <linux/stat.h>
  4. #include <linux/sched/xacct.h>
  5. #include <linux/fcntl.h>
  6. #include <linux/file.h>
  7. #include <linux/uio.h>
  8. #include <linux/fsnotify.h>
  9. #include <linux/security.h>
  10. #include <linux/export.h>
  11. #include <linux/syscalls.h>
  12. #include <linux/pagemap.h>
  13. #include <linux/splice.h>
  14. #include <linux/compat.h>
  15. #include <linux/mount.h>
  16. #include <linux/fs.h>
  17. #include "internal.h"
  18. #include <linux/uaccess.h>
  19. #include <asm/unistd.h>
  20. /*
  21. * Performs necessary checks before doing a clone.
  22. *
  23. * Can adjust amount of bytes to clone via @req_count argument.
  24. * Returns appropriate error code that caller should return or
  25. * zero in case the clone should be allowed.
  26. */
  27. static int generic_remap_checks(struct file *file_in, loff_t pos_in,
  28. struct file *file_out, loff_t pos_out,
  29. loff_t *req_count, unsigned int remap_flags)
  30. {
  31. struct inode *inode_in = file_in->f_mapping->host;
  32. struct inode *inode_out = file_out->f_mapping->host;
  33. uint64_t count = *req_count;
  34. uint64_t bcount;
  35. loff_t size_in, size_out;
  36. loff_t bs = inode_out->i_sb->s_blocksize;
  37. int ret;
  38. /* The start of both ranges must be aligned to an fs block. */
  39. if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs))
  40. return -EINVAL;
  41. /* Ensure offsets don't wrap. */
  42. if (pos_in + count < pos_in || pos_out + count < pos_out)
  43. return -EINVAL;
  44. size_in = i_size_read(inode_in);
  45. size_out = i_size_read(inode_out);
  46. /* Dedupe requires both ranges to be within EOF. */
  47. if ((remap_flags & REMAP_FILE_DEDUP) &&
  48. (pos_in >= size_in || pos_in + count > size_in ||
  49. pos_out >= size_out || pos_out + count > size_out))
  50. return -EINVAL;
  51. /* Ensure the infile range is within the infile. */
  52. if (pos_in >= size_in)
  53. return -EINVAL;
  54. count = min(count, size_in - (uint64_t)pos_in);
  55. ret = generic_write_check_limits(file_out, pos_out, &count);
  56. if (ret)
  57. return ret;
  58. /*
  59. * If the user wanted us to link to the infile's EOF, round up to the
  60. * next block boundary for this check.
  61. *
  62. * Otherwise, make sure the count is also block-aligned, having
  63. * already confirmed the starting offsets' block alignment.
  64. */
  65. if (pos_in + count == size_in) {
  66. bcount = ALIGN(size_in, bs) - pos_in;
  67. } else {
  68. if (!IS_ALIGNED(count, bs))
  69. count = ALIGN_DOWN(count, bs);
  70. bcount = count;
  71. }
  72. /* Don't allow overlapped cloning within the same file. */
  73. if (inode_in == inode_out &&
  74. pos_out + bcount > pos_in &&
  75. pos_out < pos_in + bcount)
  76. return -EINVAL;
  77. /*
  78. * We shortened the request but the caller can't deal with that, so
  79. * bounce the request back to userspace.
  80. */
  81. if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN))
  82. return -EINVAL;
  83. *req_count = count;
  84. return 0;
  85. }
  86. static int remap_verify_area(struct file *file, loff_t pos, loff_t len,
  87. bool write)
  88. {
  89. struct inode *inode = file_inode(file);
  90. if (unlikely(pos < 0 || len < 0))
  91. return -EINVAL;
  92. if (unlikely((loff_t) (pos + len) < 0))
  93. return -EINVAL;
  94. if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
  95. loff_t end = len ? pos + len - 1 : OFFSET_MAX;
  96. int retval;
  97. retval = locks_mandatory_area(inode, file, pos, end,
  98. write ? F_WRLCK : F_RDLCK);
  99. if (retval < 0)
  100. return retval;
  101. }
  102. return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
  103. }
  104. /*
  105. * Ensure that we don't remap a partial EOF block in the middle of something
  106. * else. Assume that the offsets have already been checked for block
  107. * alignment.
  108. *
  109. * For clone we only link a partial EOF block above or at the destination file's
  110. * EOF. For deduplication we accept a partial EOF block only if it ends at the
  111. * destination file's EOF (can not link it into the middle of a file).
  112. *
  113. * Shorten the request if possible.
  114. */
  115. static int generic_remap_check_len(struct inode *inode_in,
  116. struct inode *inode_out,
  117. loff_t pos_out,
  118. loff_t *len,
  119. unsigned int remap_flags)
  120. {
  121. u64 blkmask = i_blocksize(inode_in) - 1;
  122. loff_t new_len = *len;
  123. if ((*len & blkmask) == 0)
  124. return 0;
  125. if (pos_out + *len < i_size_read(inode_out))
  126. new_len &= ~blkmask;
  127. if (new_len == *len)
  128. return 0;
  129. if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
  130. *len = new_len;
  131. return 0;
  132. }
  133. return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
  134. }
  135. /* Read a page's worth of file data into the page cache. */
  136. static struct page *vfs_dedupe_get_page(struct inode *inode, loff_t offset)
  137. {
  138. struct page *page;
  139. page = read_mapping_page(inode->i_mapping, offset >> PAGE_SHIFT, NULL);
  140. if (IS_ERR(page))
  141. return page;
  142. if (!PageUptodate(page)) {
  143. put_page(page);
  144. return ERR_PTR(-EIO);
  145. }
  146. return page;
  147. }
  148. /*
  149. * Lock two pages, ensuring that we lock in offset order if the pages are from
  150. * the same file.
  151. */
  152. static void vfs_lock_two_pages(struct page *page1, struct page *page2)
  153. {
  154. /* Always lock in order of increasing index. */
  155. if (page1->index > page2->index)
  156. swap(page1, page2);
  157. lock_page(page1);
  158. if (page1 != page2)
  159. lock_page(page2);
  160. }
  161. /* Unlock two pages, being careful not to unlock the same page twice. */
  162. static void vfs_unlock_two_pages(struct page *page1, struct page *page2)
  163. {
  164. unlock_page(page1);
  165. if (page1 != page2)
  166. unlock_page(page2);
  167. }
  168. /*
  169. * Compare extents of two files to see if they are the same.
  170. * Caller must have locked both inodes to prevent write races.
  171. */
  172. static int vfs_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
  173. struct inode *dest, loff_t destoff,
  174. loff_t len, bool *is_same)
  175. {
  176. loff_t src_poff;
  177. loff_t dest_poff;
  178. void *src_addr;
  179. void *dest_addr;
  180. struct page *src_page;
  181. struct page *dest_page;
  182. loff_t cmp_len;
  183. bool same;
  184. int error;
  185. error = -EINVAL;
  186. same = true;
  187. while (len) {
  188. src_poff = srcoff & (PAGE_SIZE - 1);
  189. dest_poff = destoff & (PAGE_SIZE - 1);
  190. cmp_len = min(PAGE_SIZE - src_poff,
  191. PAGE_SIZE - dest_poff);
  192. cmp_len = min(cmp_len, len);
  193. if (cmp_len <= 0)
  194. goto out_error;
  195. src_page = vfs_dedupe_get_page(src, srcoff);
  196. if (IS_ERR(src_page)) {
  197. error = PTR_ERR(src_page);
  198. goto out_error;
  199. }
  200. dest_page = vfs_dedupe_get_page(dest, destoff);
  201. if (IS_ERR(dest_page)) {
  202. error = PTR_ERR(dest_page);
  203. put_page(src_page);
  204. goto out_error;
  205. }
  206. vfs_lock_two_pages(src_page, dest_page);
  207. /*
  208. * Now that we've locked both pages, make sure they're still
  209. * mapped to the file data we're interested in. If not,
  210. * someone is invalidating pages on us and we lose.
  211. */
  212. if (!PageUptodate(src_page) || !PageUptodate(dest_page) ||
  213. src_page->mapping != src->i_mapping ||
  214. dest_page->mapping != dest->i_mapping) {
  215. same = false;
  216. goto unlock;
  217. }
  218. src_addr = kmap_atomic(src_page);
  219. dest_addr = kmap_atomic(dest_page);
  220. flush_dcache_page(src_page);
  221. flush_dcache_page(dest_page);
  222. if (memcmp(src_addr + src_poff, dest_addr + dest_poff, cmp_len))
  223. same = false;
  224. kunmap_atomic(dest_addr);
  225. kunmap_atomic(src_addr);
  226. unlock:
  227. vfs_unlock_two_pages(src_page, dest_page);
  228. put_page(dest_page);
  229. put_page(src_page);
  230. if (!same)
  231. break;
  232. srcoff += cmp_len;
  233. destoff += cmp_len;
  234. len -= cmp_len;
  235. }
  236. *is_same = same;
  237. return 0;
  238. out_error:
  239. return error;
  240. }
  241. /*
  242. * Check that the two inodes are eligible for cloning, the ranges make
  243. * sense, and then flush all dirty data. Caller must ensure that the
  244. * inodes have been locked against any other modifications.
  245. *
  246. * If there's an error, then the usual negative error code is returned.
  247. * Otherwise returns 0 with *len set to the request length.
  248. */
  249. int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
  250. struct file *file_out, loff_t pos_out,
  251. loff_t *len, unsigned int remap_flags)
  252. {
  253. struct inode *inode_in = file_inode(file_in);
  254. struct inode *inode_out = file_inode(file_out);
  255. bool same_inode = (inode_in == inode_out);
  256. int ret;
  257. /* Don't touch certain kinds of inodes */
  258. if (IS_IMMUTABLE(inode_out))
  259. return -EPERM;
  260. if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
  261. return -ETXTBSY;
  262. /* Don't reflink dirs, pipes, sockets... */
  263. if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
  264. return -EISDIR;
  265. if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
  266. return -EINVAL;
  267. /* Zero length dedupe exits immediately; reflink goes to EOF. */
  268. if (*len == 0) {
  269. loff_t isize = i_size_read(inode_in);
  270. if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
  271. return 0;
  272. if (pos_in > isize)
  273. return -EINVAL;
  274. *len = isize - pos_in;
  275. if (*len == 0)
  276. return 0;
  277. }
  278. /* Check that we don't violate system file offset limits. */
  279. ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
  280. remap_flags);
  281. if (ret)
  282. return ret;
  283. /* Wait for the completion of any pending IOs on both files */
  284. inode_dio_wait(inode_in);
  285. if (!same_inode)
  286. inode_dio_wait(inode_out);
  287. ret = filemap_write_and_wait_range(inode_in->i_mapping,
  288. pos_in, pos_in + *len - 1);
  289. if (ret)
  290. return ret;
  291. ret = filemap_write_and_wait_range(inode_out->i_mapping,
  292. pos_out, pos_out + *len - 1);
  293. if (ret)
  294. return ret;
  295. /*
  296. * Check that the extents are the same.
  297. */
  298. if (remap_flags & REMAP_FILE_DEDUP) {
  299. bool is_same = false;
  300. ret = vfs_dedupe_file_range_compare(inode_in, pos_in,
  301. inode_out, pos_out, *len, &is_same);
  302. if (ret)
  303. return ret;
  304. if (!is_same)
  305. return -EBADE;
  306. }
  307. ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
  308. remap_flags);
  309. if (ret)
  310. return ret;
  311. /* If can't alter the file contents, we're done. */
  312. if (!(remap_flags & REMAP_FILE_DEDUP))
  313. ret = file_modified(file_out);
  314. return ret;
  315. }
  316. EXPORT_SYMBOL(generic_remap_file_range_prep);
  317. loff_t do_clone_file_range(struct file *file_in, loff_t pos_in,
  318. struct file *file_out, loff_t pos_out,
  319. loff_t len, unsigned int remap_flags)
  320. {
  321. loff_t ret;
  322. WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);
  323. /*
  324. * FICLONE/FICLONERANGE ioctls enforce that src and dest files are on
  325. * the same mount. Practically, they only need to be on the same file
  326. * system.
  327. */
  328. if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
  329. return -EXDEV;
  330. ret = generic_file_rw_checks(file_in, file_out);
  331. if (ret < 0)
  332. return ret;
  333. if (!file_in->f_op->remap_file_range)
  334. return -EOPNOTSUPP;
  335. ret = remap_verify_area(file_in, pos_in, len, false);
  336. if (ret)
  337. return ret;
  338. ret = remap_verify_area(file_out, pos_out, len, true);
  339. if (ret)
  340. return ret;
  341. ret = file_in->f_op->remap_file_range(file_in, pos_in,
  342. file_out, pos_out, len, remap_flags);
  343. if (ret < 0)
  344. return ret;
  345. fsnotify_access(file_in);
  346. fsnotify_modify(file_out);
  347. return ret;
  348. }
  349. EXPORT_SYMBOL(do_clone_file_range);
  350. loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
  351. struct file *file_out, loff_t pos_out,
  352. loff_t len, unsigned int remap_flags)
  353. {
  354. loff_t ret;
  355. file_start_write(file_out);
  356. ret = do_clone_file_range(file_in, pos_in, file_out, pos_out, len,
  357. remap_flags);
  358. file_end_write(file_out);
  359. return ret;
  360. }
  361. EXPORT_SYMBOL(vfs_clone_file_range);
  362. /* Check whether we are allowed to dedupe the destination file */
  363. static bool allow_file_dedupe(struct file *file)
  364. {
  365. if (capable(CAP_SYS_ADMIN))
  366. return true;
  367. if (file->f_mode & FMODE_WRITE)
  368. return true;
  369. if (uid_eq(current_fsuid(), file_inode(file)->i_uid))
  370. return true;
  371. if (!inode_permission(file_inode(file), MAY_WRITE))
  372. return true;
  373. return false;
  374. }
  375. loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
  376. struct file *dst_file, loff_t dst_pos,
  377. loff_t len, unsigned int remap_flags)
  378. {
  379. loff_t ret;
  380. WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
  381. REMAP_FILE_CAN_SHORTEN));
  382. ret = mnt_want_write_file(dst_file);
  383. if (ret)
  384. return ret;
  385. ret = remap_verify_area(dst_file, dst_pos, len, true);
  386. if (ret < 0)
  387. goto out_drop_write;
  388. ret = -EPERM;
  389. if (!allow_file_dedupe(dst_file))
  390. goto out_drop_write;
  391. ret = -EXDEV;
  392. if (src_file->f_path.mnt != dst_file->f_path.mnt)
  393. goto out_drop_write;
  394. ret = -EISDIR;
  395. if (S_ISDIR(file_inode(dst_file)->i_mode))
  396. goto out_drop_write;
  397. ret = -EINVAL;
  398. if (!dst_file->f_op->remap_file_range)
  399. goto out_drop_write;
  400. if (len == 0) {
  401. ret = 0;
  402. goto out_drop_write;
  403. }
  404. ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
  405. dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
  406. out_drop_write:
  407. mnt_drop_write_file(dst_file);
  408. return ret;
  409. }
  410. EXPORT_SYMBOL(vfs_dedupe_file_range_one);
  411. int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
  412. {
  413. struct file_dedupe_range_info *info;
  414. struct inode *src = file_inode(file);
  415. u64 off;
  416. u64 len;
  417. int i;
  418. int ret;
  419. u16 count = same->dest_count;
  420. loff_t deduped;
  421. if (!(file->f_mode & FMODE_READ))
  422. return -EINVAL;
  423. if (same->reserved1 || same->reserved2)
  424. return -EINVAL;
  425. off = same->src_offset;
  426. len = same->src_length;
  427. if (S_ISDIR(src->i_mode))
  428. return -EISDIR;
  429. if (!S_ISREG(src->i_mode))
  430. return -EINVAL;
  431. if (!file->f_op->remap_file_range)
  432. return -EOPNOTSUPP;
  433. ret = remap_verify_area(file, off, len, false);
  434. if (ret < 0)
  435. return ret;
  436. ret = 0;
  437. if (off + len > i_size_read(src))
  438. return -EINVAL;
  439. /* Arbitrary 1G limit on a single dedupe request, can be raised. */
  440. len = min_t(u64, len, 1 << 30);
  441. /* pre-format output fields to sane values */
  442. for (i = 0; i < count; i++) {
  443. same->info[i].bytes_deduped = 0ULL;
  444. same->info[i].status = FILE_DEDUPE_RANGE_SAME;
  445. }
  446. for (i = 0, info = same->info; i < count; i++, info++) {
  447. struct fd dst_fd = fdget(info->dest_fd);
  448. struct file *dst_file = dst_fd.file;
  449. if (!dst_file) {
  450. info->status = -EBADF;
  451. goto next_loop;
  452. }
  453. if (info->reserved) {
  454. info->status = -EINVAL;
  455. goto next_fdput;
  456. }
  457. deduped = vfs_dedupe_file_range_one(file, off, dst_file,
  458. info->dest_offset, len,
  459. REMAP_FILE_CAN_SHORTEN);
  460. if (deduped == -EBADE)
  461. info->status = FILE_DEDUPE_RANGE_DIFFERS;
  462. else if (deduped < 0)
  463. info->status = deduped;
  464. else
  465. info->bytes_deduped = len;
  466. next_fdput:
  467. fdput(dst_fd);
  468. next_loop:
  469. if (fatal_signal_pending(current))
  470. break;
  471. }
  472. return ret;
  473. }
  474. EXPORT_SYMBOL(vfs_dedupe_file_range);