splice.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * "splice": joining two ropes together by interweaving their strands.
  4. *
  5. * This is the "extended pipe" functionality, where a pipe is used as
  6. * an arbitrary in-memory buffer. Think of a pipe as a small kernel
  7. * buffer that you can use to transfer data from one end to the other.
  8. *
  9. * The traditional unix read/write is extended with a "splice()" operation
  10. * that transfers data buffers to or from a pipe buffer.
  11. *
  12. * Named by Larry McVoy, original implementation from Linus, extended by
  13. * Jens to support splicing to files, network, direct splicing, etc and
  14. * fixing lots of bugs.
  15. *
  16. * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
  17. * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
  18. * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  19. *
  20. */
  21. #include <linux/bvec.h>
  22. #include <linux/fs.h>
  23. #include <linux/file.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/splice.h>
  26. #include <linux/memcontrol.h>
  27. #include <linux/mm_inline.h>
  28. #include <linux/swap.h>
  29. #include <linux/writeback.h>
  30. #include <linux/export.h>
  31. #include <linux/syscalls.h>
  32. #include <linux/uio.h>
  33. #include <linux/security.h>
  34. #include <linux/gfp.h>
  35. #include <linux/socket.h>
  36. #include <linux/sched/signal.h>
  37. #include "internal.h"
  38. /*
  39. * Attempt to steal a page from a pipe buffer. This should perhaps go into
  40. * a vm helper function, it's already simplified quite a bit by the
  41. * addition of remove_mapping(). If success is returned, the caller may
  42. * attempt to reuse this page for another destination.
  43. */
  44. static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
  45. struct pipe_buffer *buf)
  46. {
  47. struct page *page = buf->page;
  48. struct address_space *mapping;
  49. lock_page(page);
  50. mapping = page_mapping(page);
  51. if (mapping) {
  52. WARN_ON(!PageUptodate(page));
  53. /*
  54. * At least for ext2 with nobh option, we need to wait on
  55. * writeback completing on this page, since we'll remove it
  56. * from the pagecache. Otherwise truncate wont wait on the
  57. * page, allowing the disk blocks to be reused by someone else
  58. * before we actually wrote our data to them. fs corruption
  59. * ensues.
  60. */
  61. wait_on_page_writeback(page);
  62. if (page_has_private(page) &&
  63. !try_to_release_page(page, GFP_KERNEL))
  64. goto out_unlock;
  65. /*
  66. * If we succeeded in removing the mapping, set LRU flag
  67. * and return good.
  68. */
  69. if (remove_mapping(mapping, page)) {
  70. buf->flags |= PIPE_BUF_FLAG_LRU;
  71. return true;
  72. }
  73. }
  74. /*
  75. * Raced with truncate or failed to remove page from current
  76. * address space, unlock and return failure.
  77. */
  78. out_unlock:
  79. unlock_page(page);
  80. return false;
  81. }
  82. static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
  83. struct pipe_buffer *buf)
  84. {
  85. put_page(buf->page);
  86. buf->flags &= ~PIPE_BUF_FLAG_LRU;
  87. }
  88. /*
  89. * Check whether the contents of buf is OK to access. Since the content
  90. * is a page cache page, IO may be in flight.
  91. */
  92. static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
  93. struct pipe_buffer *buf)
  94. {
  95. struct page *page = buf->page;
  96. int err;
  97. if (!PageUptodate(page)) {
  98. lock_page(page);
  99. /*
  100. * Page got truncated/unhashed. This will cause a 0-byte
  101. * splice, if this is the first page.
  102. */
  103. if (!page->mapping) {
  104. err = -ENODATA;
  105. goto error;
  106. }
  107. /*
  108. * Uh oh, read-error from disk.
  109. */
  110. if (!PageUptodate(page)) {
  111. err = -EIO;
  112. goto error;
  113. }
  114. /*
  115. * Page is ok afterall, we are done.
  116. */
  117. unlock_page(page);
  118. }
  119. return 0;
  120. error:
  121. unlock_page(page);
  122. return err;
  123. }
  124. const struct pipe_buf_operations page_cache_pipe_buf_ops = {
  125. .confirm = page_cache_pipe_buf_confirm,
  126. .release = page_cache_pipe_buf_release,
  127. .try_steal = page_cache_pipe_buf_try_steal,
  128. .get = generic_pipe_buf_get,
  129. };
  130. static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
  131. struct pipe_buffer *buf)
  132. {
  133. if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
  134. return false;
  135. buf->flags |= PIPE_BUF_FLAG_LRU;
  136. return generic_pipe_buf_try_steal(pipe, buf);
  137. }
  138. static const struct pipe_buf_operations user_page_pipe_buf_ops = {
  139. .release = page_cache_pipe_buf_release,
  140. .try_steal = user_page_pipe_buf_try_steal,
  141. .get = generic_pipe_buf_get,
  142. };
  143. static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
  144. {
  145. smp_mb();
  146. if (waitqueue_active(&pipe->rd_wait))
  147. wake_up_interruptible(&pipe->rd_wait);
  148. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  149. }
  150. /**
  151. * splice_to_pipe - fill passed data into a pipe
  152. * @pipe: pipe to fill
  153. * @spd: data to fill
  154. *
  155. * Description:
  156. * @spd contains a map of pages and len/offset tuples, along with
  157. * the struct pipe_buf_operations associated with these pages. This
  158. * function will link that data to the pipe.
  159. *
  160. */
  161. ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
  162. struct splice_pipe_desc *spd)
  163. {
  164. unsigned int spd_pages = spd->nr_pages;
  165. unsigned int tail = pipe->tail;
  166. unsigned int head = pipe->head;
  167. unsigned int mask = pipe->ring_size - 1;
  168. int ret = 0, page_nr = 0;
  169. if (!spd_pages)
  170. return 0;
  171. if (unlikely(!pipe->readers)) {
  172. send_sig(SIGPIPE, current, 0);
  173. ret = -EPIPE;
  174. goto out;
  175. }
  176. while (!pipe_full(head, tail, pipe->max_usage)) {
  177. struct pipe_buffer *buf = &pipe->bufs[head & mask];
  178. buf->page = spd->pages[page_nr];
  179. buf->offset = spd->partial[page_nr].offset;
  180. buf->len = spd->partial[page_nr].len;
  181. buf->private = spd->partial[page_nr].private;
  182. buf->ops = spd->ops;
  183. buf->flags = 0;
  184. head++;
  185. pipe->head = head;
  186. page_nr++;
  187. ret += buf->len;
  188. if (!--spd->nr_pages)
  189. break;
  190. }
  191. if (!ret)
  192. ret = -EAGAIN;
  193. out:
  194. while (page_nr < spd_pages)
  195. spd->spd_release(spd, page_nr++);
  196. return ret;
  197. }
  198. EXPORT_SYMBOL_GPL(splice_to_pipe);
  199. ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
  200. {
  201. unsigned int head = pipe->head;
  202. unsigned int tail = pipe->tail;
  203. unsigned int mask = pipe->ring_size - 1;
  204. int ret;
  205. if (unlikely(!pipe->readers)) {
  206. send_sig(SIGPIPE, current, 0);
  207. ret = -EPIPE;
  208. } else if (pipe_full(head, tail, pipe->max_usage)) {
  209. ret = -EAGAIN;
  210. } else {
  211. pipe->bufs[head & mask] = *buf;
  212. pipe->head = head + 1;
  213. return buf->len;
  214. }
  215. pipe_buf_release(pipe, buf);
  216. return ret;
  217. }
  218. EXPORT_SYMBOL(add_to_pipe);
  219. /*
  220. * Check if we need to grow the arrays holding pages and partial page
  221. * descriptions.
  222. */
  223. int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
  224. {
  225. unsigned int max_usage = READ_ONCE(pipe->max_usage);
  226. spd->nr_pages_max = max_usage;
  227. if (max_usage <= PIPE_DEF_BUFFERS)
  228. return 0;
  229. spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
  230. spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
  231. GFP_KERNEL);
  232. if (spd->pages && spd->partial)
  233. return 0;
  234. kfree(spd->pages);
  235. kfree(spd->partial);
  236. return -ENOMEM;
  237. }
  238. void splice_shrink_spd(struct splice_pipe_desc *spd)
  239. {
  240. if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
  241. return;
  242. kfree(spd->pages);
  243. kfree(spd->partial);
  244. }
  245. /**
  246. * generic_file_splice_read - splice data from file to a pipe
  247. * @in: file to splice from
  248. * @ppos: position in @in
  249. * @pipe: pipe to splice to
  250. * @len: number of bytes to splice
  251. * @flags: splice modifier flags
  252. *
  253. * Description:
  254. * Will read pages from given file and fill them into a pipe. Can be
  255. * used as long as it has more or less sane ->read_iter().
  256. *
  257. */
  258. ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
  259. struct pipe_inode_info *pipe, size_t len,
  260. unsigned int flags)
  261. {
  262. struct iov_iter to;
  263. struct kiocb kiocb;
  264. unsigned int i_head;
  265. int ret;
  266. iov_iter_pipe(&to, READ, pipe, len);
  267. i_head = to.head;
  268. init_sync_kiocb(&kiocb, in);
  269. kiocb.ki_pos = *ppos;
  270. ret = call_read_iter(in, &kiocb, &to);
  271. if (ret > 0) {
  272. *ppos = kiocb.ki_pos;
  273. file_accessed(in);
  274. } else if (ret < 0) {
  275. to.head = i_head;
  276. to.iov_offset = 0;
  277. iov_iter_advance(&to, 0); /* to free what was emitted */
  278. /*
  279. * callers of ->splice_read() expect -EAGAIN on
  280. * "can't put anything in there", rather than -EFAULT.
  281. */
  282. if (ret == -EFAULT)
  283. ret = -EAGAIN;
  284. }
  285. return ret;
  286. }
  287. EXPORT_SYMBOL_NS(generic_file_splice_read, ANDROID_GKI_VFS_EXPORT_ONLY);
  288. const struct pipe_buf_operations default_pipe_buf_ops = {
  289. .release = generic_pipe_buf_release,
  290. .try_steal = generic_pipe_buf_try_steal,
  291. .get = generic_pipe_buf_get,
  292. };
  293. /* Pipe buffer operations for a socket and similar. */
  294. const struct pipe_buf_operations nosteal_pipe_buf_ops = {
  295. .release = generic_pipe_buf_release,
  296. .get = generic_pipe_buf_get,
  297. };
  298. EXPORT_SYMBOL(nosteal_pipe_buf_ops);
  299. /*
  300. * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
  301. * using sendpage(). Return the number of bytes sent.
  302. */
  303. static int pipe_to_sendpage(struct pipe_inode_info *pipe,
  304. struct pipe_buffer *buf, struct splice_desc *sd)
  305. {
  306. struct file *file = sd->u.file;
  307. loff_t pos = sd->pos;
  308. int more;
  309. if (!likely(file->f_op->sendpage))
  310. return -EINVAL;
  311. more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
  312. if (sd->len < sd->total_len &&
  313. pipe_occupancy(pipe->head, pipe->tail) > 1)
  314. more |= MSG_SENDPAGE_NOTLAST;
  315. return file->f_op->sendpage(file, buf->page, buf->offset,
  316. sd->len, &pos, more);
  317. }
  318. static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
  319. {
  320. smp_mb();
  321. if (waitqueue_active(&pipe->wr_wait))
  322. wake_up_interruptible(&pipe->wr_wait);
  323. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  324. }
  325. /**
  326. * splice_from_pipe_feed - feed available data from a pipe to a file
  327. * @pipe: pipe to splice from
  328. * @sd: information to @actor
  329. * @actor: handler that splices the data
  330. *
  331. * Description:
  332. * This function loops over the pipe and calls @actor to do the
  333. * actual moving of a single struct pipe_buffer to the desired
  334. * destination. It returns when there's no more buffers left in
  335. * the pipe or if the requested number of bytes (@sd->total_len)
  336. * have been copied. It returns a positive number (one) if the
  337. * pipe needs to be filled with more data, zero if the required
  338. * number of bytes have been copied and -errno on error.
  339. *
  340. * This, together with splice_from_pipe_{begin,end,next}, may be
  341. * used to implement the functionality of __splice_from_pipe() when
  342. * locking is required around copying the pipe buffers to the
  343. * destination.
  344. */
  345. static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
  346. splice_actor *actor)
  347. {
  348. unsigned int head = pipe->head;
  349. unsigned int tail = pipe->tail;
  350. unsigned int mask = pipe->ring_size - 1;
  351. int ret;
  352. while (!pipe_empty(head, tail)) {
  353. struct pipe_buffer *buf = &pipe->bufs[tail & mask];
  354. sd->len = buf->len;
  355. if (sd->len > sd->total_len)
  356. sd->len = sd->total_len;
  357. ret = pipe_buf_confirm(pipe, buf);
  358. if (unlikely(ret)) {
  359. if (ret == -ENODATA)
  360. ret = 0;
  361. return ret;
  362. }
  363. ret = actor(pipe, buf, sd);
  364. if (ret <= 0)
  365. return ret;
  366. buf->offset += ret;
  367. buf->len -= ret;
  368. sd->num_spliced += ret;
  369. sd->len -= ret;
  370. sd->pos += ret;
  371. sd->total_len -= ret;
  372. if (!buf->len) {
  373. pipe_buf_release(pipe, buf);
  374. tail++;
  375. pipe->tail = tail;
  376. if (pipe->files)
  377. sd->need_wakeup = true;
  378. }
  379. if (!sd->total_len)
  380. return 0;
  381. }
  382. return 1;
  383. }
  384. /* We know we have a pipe buffer, but maybe it's empty? */
  385. static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
  386. {
  387. unsigned int tail = pipe->tail;
  388. unsigned int mask = pipe->ring_size - 1;
  389. struct pipe_buffer *buf = &pipe->bufs[tail & mask];
  390. if (unlikely(!buf->len)) {
  391. pipe_buf_release(pipe, buf);
  392. pipe->tail = tail+1;
  393. return true;
  394. }
  395. return false;
  396. }
  397. /**
  398. * splice_from_pipe_next - wait for some data to splice from
  399. * @pipe: pipe to splice from
  400. * @sd: information about the splice operation
  401. *
  402. * Description:
  403. * This function will wait for some data and return a positive
  404. * value (one) if pipe buffers are available. It will return zero
  405. * or -errno if no more data needs to be spliced.
  406. */
  407. static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
  408. {
  409. /*
  410. * Check for signal early to make process killable when there are
  411. * always buffers available
  412. */
  413. if (signal_pending(current))
  414. return -ERESTARTSYS;
  415. repeat:
  416. while (pipe_empty(pipe->head, pipe->tail)) {
  417. if (!pipe->writers)
  418. return 0;
  419. if (sd->num_spliced)
  420. return 0;
  421. if (sd->flags & SPLICE_F_NONBLOCK)
  422. return -EAGAIN;
  423. if (signal_pending(current))
  424. return -ERESTARTSYS;
  425. if (sd->need_wakeup) {
  426. wakeup_pipe_writers(pipe);
  427. sd->need_wakeup = false;
  428. }
  429. pipe_wait_readable(pipe);
  430. }
  431. if (eat_empty_buffer(pipe))
  432. goto repeat;
  433. return 1;
  434. }
  435. /**
  436. * splice_from_pipe_begin - start splicing from pipe
  437. * @sd: information about the splice operation
  438. *
  439. * Description:
  440. * This function should be called before a loop containing
  441. * splice_from_pipe_next() and splice_from_pipe_feed() to
  442. * initialize the necessary fields of @sd.
  443. */
  444. static void splice_from_pipe_begin(struct splice_desc *sd)
  445. {
  446. sd->num_spliced = 0;
  447. sd->need_wakeup = false;
  448. }
  449. /**
  450. * splice_from_pipe_end - finish splicing from pipe
  451. * @pipe: pipe to splice from
  452. * @sd: information about the splice operation
  453. *
  454. * Description:
  455. * This function will wake up pipe writers if necessary. It should
  456. * be called after a loop containing splice_from_pipe_next() and
  457. * splice_from_pipe_feed().
  458. */
  459. static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
  460. {
  461. if (sd->need_wakeup)
  462. wakeup_pipe_writers(pipe);
  463. }
  464. /**
  465. * __splice_from_pipe - splice data from a pipe to given actor
  466. * @pipe: pipe to splice from
  467. * @sd: information to @actor
  468. * @actor: handler that splices the data
  469. *
  470. * Description:
  471. * This function does little more than loop over the pipe and call
  472. * @actor to do the actual moving of a single struct pipe_buffer to
  473. * the desired destination. See pipe_to_file, pipe_to_sendpage, or
  474. * pipe_to_user.
  475. *
  476. */
  477. ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
  478. splice_actor *actor)
  479. {
  480. int ret;
  481. splice_from_pipe_begin(sd);
  482. do {
  483. cond_resched();
  484. ret = splice_from_pipe_next(pipe, sd);
  485. if (ret > 0)
  486. ret = splice_from_pipe_feed(pipe, sd, actor);
  487. } while (ret > 0);
  488. splice_from_pipe_end(pipe, sd);
  489. return sd->num_spliced ? sd->num_spliced : ret;
  490. }
  491. EXPORT_SYMBOL(__splice_from_pipe);
  492. /**
  493. * splice_from_pipe - splice data from a pipe to a file
  494. * @pipe: pipe to splice from
  495. * @out: file to splice to
  496. * @ppos: position in @out
  497. * @len: how many bytes to splice
  498. * @flags: splice modifier flags
  499. * @actor: handler that splices the data
  500. *
  501. * Description:
  502. * See __splice_from_pipe. This function locks the pipe inode,
  503. * otherwise it's identical to __splice_from_pipe().
  504. *
  505. */
  506. ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
  507. loff_t *ppos, size_t len, unsigned int flags,
  508. splice_actor *actor)
  509. {
  510. ssize_t ret;
  511. struct splice_desc sd = {
  512. .total_len = len,
  513. .flags = flags,
  514. .pos = *ppos,
  515. .u.file = out,
  516. };
  517. pipe_lock(pipe);
  518. ret = __splice_from_pipe(pipe, &sd, actor);
  519. pipe_unlock(pipe);
  520. return ret;
  521. }
  522. /**
  523. * iter_file_splice_write - splice data from a pipe to a file
  524. * @pipe: pipe info
  525. * @out: file to write to
  526. * @ppos: position in @out
  527. * @len: number of bytes to splice
  528. * @flags: splice modifier flags
  529. *
  530. * Description:
  531. * Will either move or copy pages (determined by @flags options) from
  532. * the given pipe inode to the given file.
  533. * This one is ->write_iter-based.
  534. *
  535. */
  536. ssize_t
  537. iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
  538. loff_t *ppos, size_t len, unsigned int flags)
  539. {
  540. struct splice_desc sd = {
  541. .total_len = len,
  542. .flags = flags,
  543. .pos = *ppos,
  544. .u.file = out,
  545. };
  546. int nbufs = pipe->max_usage;
  547. struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
  548. GFP_KERNEL);
  549. ssize_t ret;
  550. if (unlikely(!array))
  551. return -ENOMEM;
  552. pipe_lock(pipe);
  553. splice_from_pipe_begin(&sd);
  554. while (sd.total_len) {
  555. struct iov_iter from;
  556. unsigned int head, tail, mask;
  557. size_t left;
  558. int n;
  559. ret = splice_from_pipe_next(pipe, &sd);
  560. if (ret <= 0)
  561. break;
  562. if (unlikely(nbufs < pipe->max_usage)) {
  563. kfree(array);
  564. nbufs = pipe->max_usage;
  565. array = kcalloc(nbufs, sizeof(struct bio_vec),
  566. GFP_KERNEL);
  567. if (!array) {
  568. ret = -ENOMEM;
  569. break;
  570. }
  571. }
  572. head = pipe->head;
  573. tail = pipe->tail;
  574. mask = pipe->ring_size - 1;
  575. /* build the vector */
  576. left = sd.total_len;
  577. for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
  578. struct pipe_buffer *buf = &pipe->bufs[tail & mask];
  579. size_t this_len = buf->len;
  580. if (this_len > left)
  581. this_len = left;
  582. ret = pipe_buf_confirm(pipe, buf);
  583. if (unlikely(ret)) {
  584. if (ret == -ENODATA)
  585. ret = 0;
  586. goto done;
  587. }
  588. array[n].bv_page = buf->page;
  589. array[n].bv_len = this_len;
  590. array[n].bv_offset = buf->offset;
  591. left -= this_len;
  592. }
  593. iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);
  594. ret = vfs_iter_write(out, &from, &sd.pos, 0);
  595. if (ret <= 0)
  596. break;
  597. sd.num_spliced += ret;
  598. sd.total_len -= ret;
  599. *ppos = sd.pos;
  600. /* dismiss the fully eaten buffers, adjust the partial one */
  601. tail = pipe->tail;
  602. while (ret) {
  603. struct pipe_buffer *buf = &pipe->bufs[tail & mask];
  604. if (ret >= buf->len) {
  605. ret -= buf->len;
  606. buf->len = 0;
  607. pipe_buf_release(pipe, buf);
  608. tail++;
  609. pipe->tail = tail;
  610. if (pipe->files)
  611. sd.need_wakeup = true;
  612. } else {
  613. buf->offset += ret;
  614. buf->len -= ret;
  615. ret = 0;
  616. }
  617. }
  618. }
  619. done:
  620. kfree(array);
  621. splice_from_pipe_end(pipe, &sd);
  622. pipe_unlock(pipe);
  623. if (sd.num_spliced)
  624. ret = sd.num_spliced;
  625. return ret;
  626. }
  627. EXPORT_SYMBOL_NS(iter_file_splice_write, ANDROID_GKI_VFS_EXPORT_ONLY);
  628. /**
  629. * generic_splice_sendpage - splice data from a pipe to a socket
  630. * @pipe: pipe to splice from
  631. * @out: socket to write to
  632. * @ppos: position in @out
  633. * @len: number of bytes to splice
  634. * @flags: splice modifier flags
  635. *
  636. * Description:
  637. * Will send @len bytes from the pipe to a network socket. No data copying
  638. * is involved.
  639. *
  640. */
  641. ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
  642. loff_t *ppos, size_t len, unsigned int flags)
  643. {
  644. return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
  645. }
  646. EXPORT_SYMBOL(generic_splice_sendpage);
  647. static int warn_unsupported(struct file *file, const char *op)
  648. {
  649. pr_debug_ratelimited(
  650. "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
  651. op, file, current->pid, current->comm);
  652. return -EINVAL;
  653. }
  654. /*
  655. * Attempt to initiate a splice from pipe to file.
  656. */
  657. static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
  658. loff_t *ppos, size_t len, unsigned int flags)
  659. {
  660. if (unlikely(!out->f_op->splice_write))
  661. return warn_unsupported(out, "write");
  662. return out->f_op->splice_write(pipe, out, ppos, len, flags);
  663. }
  664. /*
  665. * Attempt to initiate a splice from a file to a pipe.
  666. */
  667. static long do_splice_to(struct file *in, loff_t *ppos,
  668. struct pipe_inode_info *pipe, size_t len,
  669. unsigned int flags)
  670. {
  671. int ret;
  672. if (unlikely(!(in->f_mode & FMODE_READ)))
  673. return -EBADF;
  674. ret = rw_verify_area(READ, in, ppos, len);
  675. if (unlikely(ret < 0))
  676. return ret;
  677. if (unlikely(len > MAX_RW_COUNT))
  678. len = MAX_RW_COUNT;
  679. if (unlikely(!in->f_op->splice_read))
  680. return warn_unsupported(in, "read");
  681. return in->f_op->splice_read(in, ppos, pipe, len, flags);
  682. }
  683. /**
  684. * splice_direct_to_actor - splices data directly between two non-pipes
  685. * @in: file to splice from
  686. * @sd: actor information on where to splice to
  687. * @actor: handles the data splicing
  688. *
  689. * Description:
  690. * This is a special case helper to splice directly between two
  691. * points, without requiring an explicit pipe. Internally an allocated
  692. * pipe is cached in the process, and reused during the lifetime of
  693. * that process.
  694. *
  695. */
  696. ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
  697. splice_direct_actor *actor)
  698. {
  699. struct pipe_inode_info *pipe;
  700. long ret, bytes;
  701. umode_t i_mode;
  702. size_t len;
  703. int i, flags, more;
  704. /*
  705. * We require the input being a regular file, as we don't want to
  706. * randomly drop data for eg socket -> socket splicing. Use the
  707. * piped splicing for that!
  708. */
  709. i_mode = file_inode(in)->i_mode;
  710. if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
  711. return -EINVAL;
  712. /*
  713. * neither in nor out is a pipe, setup an internal pipe attached to
  714. * 'out' and transfer the wanted data from 'in' to 'out' through that
  715. */
  716. pipe = current->splice_pipe;
  717. if (unlikely(!pipe)) {
  718. pipe = alloc_pipe_info();
  719. if (!pipe)
  720. return -ENOMEM;
  721. /*
  722. * We don't have an immediate reader, but we'll read the stuff
  723. * out of the pipe right after the splice_to_pipe(). So set
  724. * PIPE_READERS appropriately.
  725. */
  726. pipe->readers = 1;
  727. current->splice_pipe = pipe;
  728. }
  729. /*
  730. * Do the splice.
  731. */
  732. ret = 0;
  733. bytes = 0;
  734. len = sd->total_len;
  735. flags = sd->flags;
  736. /*
  737. * Don't block on output, we have to drain the direct pipe.
  738. */
  739. sd->flags &= ~SPLICE_F_NONBLOCK;
  740. more = sd->flags & SPLICE_F_MORE;
  741. WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
  742. while (len) {
  743. unsigned int p_space;
  744. size_t read_len;
  745. loff_t pos = sd->pos, prev_pos = pos;
  746. /* Don't try to read more the pipe has space for. */
  747. p_space = pipe->max_usage -
  748. pipe_occupancy(pipe->head, pipe->tail);
  749. read_len = min_t(size_t, len, p_space << PAGE_SHIFT);
  750. ret = do_splice_to(in, &pos, pipe, read_len, flags);
  751. if (unlikely(ret <= 0))
  752. goto out_release;
  753. read_len = ret;
  754. sd->total_len = read_len;
  755. /*
  756. * If more data is pending, set SPLICE_F_MORE
  757. * If this is the last data and SPLICE_F_MORE was not set
  758. * initially, clears it.
  759. */
  760. if (read_len < len)
  761. sd->flags |= SPLICE_F_MORE;
  762. else if (!more)
  763. sd->flags &= ~SPLICE_F_MORE;
  764. /*
  765. * NOTE: nonblocking mode only applies to the input. We
  766. * must not do the output in nonblocking mode as then we
  767. * could get stuck data in the internal pipe:
  768. */
  769. ret = actor(pipe, sd);
  770. if (unlikely(ret <= 0)) {
  771. sd->pos = prev_pos;
  772. goto out_release;
  773. }
  774. bytes += ret;
  775. len -= ret;
  776. sd->pos = pos;
  777. if (ret < read_len) {
  778. sd->pos = prev_pos + ret;
  779. goto out_release;
  780. }
  781. }
  782. done:
  783. pipe->tail = pipe->head = 0;
  784. file_accessed(in);
  785. return bytes;
  786. out_release:
  787. /*
  788. * If we did an incomplete transfer we must release
  789. * the pipe buffers in question:
  790. */
  791. for (i = 0; i < pipe->ring_size; i++) {
  792. struct pipe_buffer *buf = &pipe->bufs[i];
  793. if (buf->ops)
  794. pipe_buf_release(pipe, buf);
  795. }
  796. if (!bytes)
  797. bytes = ret;
  798. goto done;
  799. }
  800. EXPORT_SYMBOL(splice_direct_to_actor);
  801. static int direct_splice_actor(struct pipe_inode_info *pipe,
  802. struct splice_desc *sd)
  803. {
  804. struct file *file = sd->u.file;
  805. return do_splice_from(pipe, file, sd->opos, sd->total_len,
  806. sd->flags);
  807. }
  808. /**
  809. * do_splice_direct - splices data directly between two files
  810. * @in: file to splice from
  811. * @ppos: input file offset
  812. * @out: file to splice to
  813. * @opos: output file offset
  814. * @len: number of bytes to splice
  815. * @flags: splice modifier flags
  816. *
  817. * Description:
  818. * For use by do_sendfile(). splice can easily emulate sendfile, but
  819. * doing it in the application would incur an extra system call
  820. * (splice in + splice out, as compared to just sendfile()). So this helper
  821. * can splice directly through a process-private pipe.
  822. *
  823. */
  824. long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
  825. loff_t *opos, size_t len, unsigned int flags)
  826. {
  827. struct splice_desc sd = {
  828. .len = len,
  829. .total_len = len,
  830. .flags = flags,
  831. .pos = *ppos,
  832. .u.file = out,
  833. .opos = opos,
  834. };
  835. long ret;
  836. if (unlikely(!(out->f_mode & FMODE_WRITE)))
  837. return -EBADF;
  838. if (unlikely(out->f_flags & O_APPEND))
  839. return -EINVAL;
  840. ret = rw_verify_area(WRITE, out, opos, len);
  841. if (unlikely(ret < 0))
  842. return ret;
  843. ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
  844. if (ret > 0)
  845. *ppos = sd.pos;
  846. return ret;
  847. }
  848. EXPORT_SYMBOL(do_splice_direct);
  849. static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
  850. {
  851. for (;;) {
  852. if (unlikely(!pipe->readers)) {
  853. send_sig(SIGPIPE, current, 0);
  854. return -EPIPE;
  855. }
  856. if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
  857. return 0;
  858. if (flags & SPLICE_F_NONBLOCK)
  859. return -EAGAIN;
  860. if (signal_pending(current))
  861. return -ERESTARTSYS;
  862. pipe_wait_writable(pipe);
  863. }
  864. }
  865. static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
  866. struct pipe_inode_info *opipe,
  867. size_t len, unsigned int flags);
  868. /*
  869. * Determine where to splice to/from.
  870. */
  871. long do_splice(struct file *in, loff_t *off_in, struct file *out,
  872. loff_t *off_out, size_t len, unsigned int flags)
  873. {
  874. struct pipe_inode_info *ipipe;
  875. struct pipe_inode_info *opipe;
  876. loff_t offset;
  877. long ret;
  878. if (unlikely(!(in->f_mode & FMODE_READ) ||
  879. !(out->f_mode & FMODE_WRITE)))
  880. return -EBADF;
  881. ipipe = get_pipe_info(in, true);
  882. opipe = get_pipe_info(out, true);
  883. if (ipipe && opipe) {
  884. if (off_in || off_out)
  885. return -ESPIPE;
  886. /* Splicing to self would be fun, but... */
  887. if (ipipe == opipe)
  888. return -EINVAL;
  889. if ((in->f_flags | out->f_flags) & O_NONBLOCK)
  890. flags |= SPLICE_F_NONBLOCK;
  891. return splice_pipe_to_pipe(ipipe, opipe, len, flags);
  892. }
  893. if (ipipe) {
  894. if (off_in)
  895. return -ESPIPE;
  896. if (off_out) {
  897. if (!(out->f_mode & FMODE_PWRITE))
  898. return -EINVAL;
  899. offset = *off_out;
  900. } else {
  901. offset = out->f_pos;
  902. }
  903. if (unlikely(out->f_flags & O_APPEND))
  904. return -EINVAL;
  905. ret = rw_verify_area(WRITE, out, &offset, len);
  906. if (unlikely(ret < 0))
  907. return ret;
  908. if (in->f_flags & O_NONBLOCK)
  909. flags |= SPLICE_F_NONBLOCK;
  910. file_start_write(out);
  911. ret = do_splice_from(ipipe, out, &offset, len, flags);
  912. file_end_write(out);
  913. if (!off_out)
  914. out->f_pos = offset;
  915. else
  916. *off_out = offset;
  917. return ret;
  918. }
  919. if (opipe) {
  920. if (off_out)
  921. return -ESPIPE;
  922. if (off_in) {
  923. if (!(in->f_mode & FMODE_PREAD))
  924. return -EINVAL;
  925. offset = *off_in;
  926. } else {
  927. offset = in->f_pos;
  928. }
  929. if (out->f_flags & O_NONBLOCK)
  930. flags |= SPLICE_F_NONBLOCK;
  931. pipe_lock(opipe);
  932. ret = wait_for_space(opipe, flags);
  933. if (!ret) {
  934. unsigned int p_space;
  935. /* Don't try to read more the pipe has space for. */
  936. p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail);
  937. len = min_t(size_t, len, p_space << PAGE_SHIFT);
  938. ret = do_splice_to(in, &offset, opipe, len, flags);
  939. }
  940. pipe_unlock(opipe);
  941. if (ret > 0)
  942. wakeup_pipe_readers(opipe);
  943. if (!off_in)
  944. in->f_pos = offset;
  945. else
  946. *off_in = offset;
  947. return ret;
  948. }
  949. return -EINVAL;
  950. }
  951. static long __do_splice(struct file *in, loff_t __user *off_in,
  952. struct file *out, loff_t __user *off_out,
  953. size_t len, unsigned int flags)
  954. {
  955. struct pipe_inode_info *ipipe;
  956. struct pipe_inode_info *opipe;
  957. loff_t offset, *__off_in = NULL, *__off_out = NULL;
  958. long ret;
  959. ipipe = get_pipe_info(in, true);
  960. opipe = get_pipe_info(out, true);
  961. if (ipipe && off_in)
  962. return -ESPIPE;
  963. if (opipe && off_out)
  964. return -ESPIPE;
  965. if (off_out) {
  966. if (copy_from_user(&offset, off_out, sizeof(loff_t)))
  967. return -EFAULT;
  968. __off_out = &offset;
  969. }
  970. if (off_in) {
  971. if (copy_from_user(&offset, off_in, sizeof(loff_t)))
  972. return -EFAULT;
  973. __off_in = &offset;
  974. }
  975. ret = do_splice(in, __off_in, out, __off_out, len, flags);
  976. if (ret < 0)
  977. return ret;
  978. if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
  979. return -EFAULT;
  980. if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
  981. return -EFAULT;
  982. return ret;
  983. }
  984. static int iter_to_pipe(struct iov_iter *from,
  985. struct pipe_inode_info *pipe,
  986. unsigned flags)
  987. {
  988. struct pipe_buffer buf = {
  989. .ops = &user_page_pipe_buf_ops,
  990. .flags = flags
  991. };
  992. size_t total = 0;
  993. int ret = 0;
  994. bool failed = false;
  995. while (iov_iter_count(from) && !failed) {
  996. struct page *pages[16];
  997. ssize_t copied;
  998. size_t start;
  999. int n;
  1000. copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
  1001. if (copied <= 0) {
  1002. ret = copied;
  1003. break;
  1004. }
  1005. for (n = 0; copied; n++, start = 0) {
  1006. int size = min_t(int, copied, PAGE_SIZE - start);
  1007. if (!failed) {
  1008. buf.page = pages[n];
  1009. buf.offset = start;
  1010. buf.len = size;
  1011. ret = add_to_pipe(pipe, &buf);
  1012. if (unlikely(ret < 0)) {
  1013. failed = true;
  1014. } else {
  1015. iov_iter_advance(from, ret);
  1016. total += ret;
  1017. }
  1018. } else {
  1019. put_page(pages[n]);
  1020. }
  1021. copied -= size;
  1022. }
  1023. }
  1024. return total ? total : ret;
  1025. }
  1026. static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
  1027. struct splice_desc *sd)
  1028. {
  1029. int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
  1030. return n == sd->len ? n : -EFAULT;
  1031. }
  1032. /*
  1033. * For lack of a better implementation, implement vmsplice() to userspace
  1034. * as a simple copy of the pipes pages to the user iov.
  1035. */
  1036. static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
  1037. unsigned int flags)
  1038. {
  1039. struct pipe_inode_info *pipe = get_pipe_info(file, true);
  1040. struct splice_desc sd = {
  1041. .total_len = iov_iter_count(iter),
  1042. .flags = flags,
  1043. .u.data = iter
  1044. };
  1045. long ret = 0;
  1046. if (!pipe)
  1047. return -EBADF;
  1048. if (sd.total_len) {
  1049. pipe_lock(pipe);
  1050. ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
  1051. pipe_unlock(pipe);
  1052. }
  1053. return ret;
  1054. }
  1055. /*
  1056. * vmsplice splices a user address range into a pipe. It can be thought of
  1057. * as splice-from-memory, where the regular splice is splice-from-file (or
  1058. * to file). In both cases the output is a pipe, naturally.
  1059. */
  1060. static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
  1061. unsigned int flags)
  1062. {
  1063. struct pipe_inode_info *pipe;
  1064. long ret = 0;
  1065. unsigned buf_flag = 0;
  1066. if (flags & SPLICE_F_GIFT)
  1067. buf_flag = PIPE_BUF_FLAG_GIFT;
  1068. pipe = get_pipe_info(file, true);
  1069. if (!pipe)
  1070. return -EBADF;
  1071. pipe_lock(pipe);
  1072. ret = wait_for_space(pipe, flags);
  1073. if (!ret)
  1074. ret = iter_to_pipe(iter, pipe, buf_flag);
  1075. pipe_unlock(pipe);
  1076. if (ret > 0)
  1077. wakeup_pipe_readers(pipe);
  1078. return ret;
  1079. }
  1080. static int vmsplice_type(struct fd f, int *type)
  1081. {
  1082. if (!f.file)
  1083. return -EBADF;
  1084. if (f.file->f_mode & FMODE_WRITE) {
  1085. *type = WRITE;
  1086. } else if (f.file->f_mode & FMODE_READ) {
  1087. *type = READ;
  1088. } else {
  1089. fdput(f);
  1090. return -EBADF;
  1091. }
  1092. return 0;
  1093. }
  1094. /*
  1095. * Note that vmsplice only really supports true splicing _from_ user memory
  1096. * to a pipe, not the other way around. Splicing from user memory is a simple
  1097. * operation that can be supported without any funky alignment restrictions
  1098. * or nasty vm tricks. We simply map in the user memory and fill them into
  1099. * a pipe. The reverse isn't quite as easy, though. There are two possible
  1100. * solutions for that:
  1101. *
  1102. * - memcpy() the data internally, at which point we might as well just
  1103. * do a regular read() on the buffer anyway.
  1104. * - Lots of nasty vm tricks, that are neither fast nor flexible (it
  1105. * has restriction limitations on both ends of the pipe).
  1106. *
  1107. * Currently we punt and implement it as a normal copy, see pipe_to_user().
  1108. *
  1109. */
  1110. SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
  1111. unsigned long, nr_segs, unsigned int, flags)
  1112. {
  1113. struct iovec iovstack[UIO_FASTIOV];
  1114. struct iovec *iov = iovstack;
  1115. struct iov_iter iter;
  1116. ssize_t error;
  1117. struct fd f;
  1118. int type;
  1119. if (unlikely(flags & ~SPLICE_F_ALL))
  1120. return -EINVAL;
  1121. f = fdget(fd);
  1122. error = vmsplice_type(f, &type);
  1123. if (error)
  1124. return error;
  1125. error = import_iovec(type, uiov, nr_segs,
  1126. ARRAY_SIZE(iovstack), &iov, &iter);
  1127. if (error < 0)
  1128. goto out_fdput;
  1129. if (!iov_iter_count(&iter))
  1130. error = 0;
  1131. else if (iov_iter_rw(&iter) == WRITE)
  1132. error = vmsplice_to_pipe(f.file, &iter, flags);
  1133. else
  1134. error = vmsplice_to_user(f.file, &iter, flags);
  1135. kfree(iov);
  1136. out_fdput:
  1137. fdput(f);
  1138. return error;
  1139. }
  1140. SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
  1141. int, fd_out, loff_t __user *, off_out,
  1142. size_t, len, unsigned int, flags)
  1143. {
  1144. struct fd in, out;
  1145. long error;
  1146. if (unlikely(!len))
  1147. return 0;
  1148. if (unlikely(flags & ~SPLICE_F_ALL))
  1149. return -EINVAL;
  1150. error = -EBADF;
  1151. in = fdget(fd_in);
  1152. if (in.file) {
  1153. out = fdget(fd_out);
  1154. if (out.file) {
  1155. error = __do_splice(in.file, off_in, out.file, off_out,
  1156. len, flags);
  1157. fdput(out);
  1158. }
  1159. fdput(in);
  1160. }
  1161. return error;
  1162. }
  1163. /*
  1164. * Make sure there's data to read. Wait for input if we can, otherwise
  1165. * return an appropriate error.
  1166. */
  1167. static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
  1168. {
  1169. int ret;
  1170. /*
  1171. * Check the pipe occupancy without the inode lock first. This function
  1172. * is speculative anyways, so missing one is ok.
  1173. */
  1174. if (!pipe_empty(pipe->head, pipe->tail))
  1175. return 0;
  1176. ret = 0;
  1177. pipe_lock(pipe);
  1178. while (pipe_empty(pipe->head, pipe->tail)) {
  1179. if (signal_pending(current)) {
  1180. ret = -ERESTARTSYS;
  1181. break;
  1182. }
  1183. if (!pipe->writers)
  1184. break;
  1185. if (flags & SPLICE_F_NONBLOCK) {
  1186. ret = -EAGAIN;
  1187. break;
  1188. }
  1189. pipe_wait_readable(pipe);
  1190. }
  1191. pipe_unlock(pipe);
  1192. return ret;
  1193. }
  1194. /*
  1195. * Make sure there's writeable room. Wait for room if we can, otherwise
  1196. * return an appropriate error.
  1197. */
  1198. static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
  1199. {
  1200. int ret;
  1201. /*
  1202. * Check pipe occupancy without the inode lock first. This function
  1203. * is speculative anyways, so missing one is ok.
  1204. */
  1205. if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
  1206. return 0;
  1207. ret = 0;
  1208. pipe_lock(pipe);
  1209. while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
  1210. if (!pipe->readers) {
  1211. send_sig(SIGPIPE, current, 0);
  1212. ret = -EPIPE;
  1213. break;
  1214. }
  1215. if (flags & SPLICE_F_NONBLOCK) {
  1216. ret = -EAGAIN;
  1217. break;
  1218. }
  1219. if (signal_pending(current)) {
  1220. ret = -ERESTARTSYS;
  1221. break;
  1222. }
  1223. pipe_wait_writable(pipe);
  1224. }
  1225. pipe_unlock(pipe);
  1226. return ret;
  1227. }
  1228. /*
  1229. * Splice contents of ipipe to opipe.
  1230. */
  1231. static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
  1232. struct pipe_inode_info *opipe,
  1233. size_t len, unsigned int flags)
  1234. {
  1235. struct pipe_buffer *ibuf, *obuf;
  1236. unsigned int i_head, o_head;
  1237. unsigned int i_tail, o_tail;
  1238. unsigned int i_mask, o_mask;
  1239. int ret = 0;
  1240. bool input_wakeup = false;
  1241. retry:
  1242. ret = ipipe_prep(ipipe, flags);
  1243. if (ret)
  1244. return ret;
  1245. ret = opipe_prep(opipe, flags);
  1246. if (ret)
  1247. return ret;
  1248. /*
  1249. * Potential ABBA deadlock, work around it by ordering lock
  1250. * grabbing by pipe info address. Otherwise two different processes
  1251. * could deadlock (one doing tee from A -> B, the other from B -> A).
  1252. */
  1253. pipe_double_lock(ipipe, opipe);
  1254. i_tail = ipipe->tail;
  1255. i_mask = ipipe->ring_size - 1;
  1256. o_head = opipe->head;
  1257. o_mask = opipe->ring_size - 1;
  1258. do {
  1259. size_t o_len;
  1260. if (!opipe->readers) {
  1261. send_sig(SIGPIPE, current, 0);
  1262. if (!ret)
  1263. ret = -EPIPE;
  1264. break;
  1265. }
  1266. i_head = ipipe->head;
  1267. o_tail = opipe->tail;
  1268. if (pipe_empty(i_head, i_tail) && !ipipe->writers)
  1269. break;
  1270. /*
  1271. * Cannot make any progress, because either the input
  1272. * pipe is empty or the output pipe is full.
  1273. */
  1274. if (pipe_empty(i_head, i_tail) ||
  1275. pipe_full(o_head, o_tail, opipe->max_usage)) {
  1276. /* Already processed some buffers, break */
  1277. if (ret)
  1278. break;
  1279. if (flags & SPLICE_F_NONBLOCK) {
  1280. ret = -EAGAIN;
  1281. break;
  1282. }
  1283. /*
  1284. * We raced with another reader/writer and haven't
  1285. * managed to process any buffers. A zero return
  1286. * value means EOF, so retry instead.
  1287. */
  1288. pipe_unlock(ipipe);
  1289. pipe_unlock(opipe);
  1290. goto retry;
  1291. }
  1292. ibuf = &ipipe->bufs[i_tail & i_mask];
  1293. obuf = &opipe->bufs[o_head & o_mask];
  1294. if (len >= ibuf->len) {
  1295. /*
  1296. * Simply move the whole buffer from ipipe to opipe
  1297. */
  1298. *obuf = *ibuf;
  1299. ibuf->ops = NULL;
  1300. i_tail++;
  1301. ipipe->tail = i_tail;
  1302. input_wakeup = true;
  1303. o_len = obuf->len;
  1304. o_head++;
  1305. opipe->head = o_head;
  1306. } else {
  1307. /*
  1308. * Get a reference to this pipe buffer,
  1309. * so we can copy the contents over.
  1310. */
  1311. if (!pipe_buf_get(ipipe, ibuf)) {
  1312. if (ret == 0)
  1313. ret = -EFAULT;
  1314. break;
  1315. }
  1316. *obuf = *ibuf;
  1317. /*
  1318. * Don't inherit the gift and merge flags, we need to
  1319. * prevent multiple steals of this page.
  1320. */
  1321. obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
  1322. obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
  1323. obuf->len = len;
  1324. ibuf->offset += len;
  1325. ibuf->len -= len;
  1326. o_len = len;
  1327. o_head++;
  1328. opipe->head = o_head;
  1329. }
  1330. ret += o_len;
  1331. len -= o_len;
  1332. } while (len);
  1333. pipe_unlock(ipipe);
  1334. pipe_unlock(opipe);
  1335. /*
  1336. * If we put data in the output pipe, wakeup any potential readers.
  1337. */
  1338. if (ret > 0)
  1339. wakeup_pipe_readers(opipe);
  1340. if (input_wakeup)
  1341. wakeup_pipe_writers(ipipe);
  1342. return ret;
  1343. }
  1344. /*
  1345. * Link contents of ipipe to opipe.
  1346. */
  1347. static int link_pipe(struct pipe_inode_info *ipipe,
  1348. struct pipe_inode_info *opipe,
  1349. size_t len, unsigned int flags)
  1350. {
  1351. struct pipe_buffer *ibuf, *obuf;
  1352. unsigned int i_head, o_head;
  1353. unsigned int i_tail, o_tail;
  1354. unsigned int i_mask, o_mask;
  1355. int ret = 0;
  1356. /*
  1357. * Potential ABBA deadlock, work around it by ordering lock
  1358. * grabbing by pipe info address. Otherwise two different processes
  1359. * could deadlock (one doing tee from A -> B, the other from B -> A).
  1360. */
  1361. pipe_double_lock(ipipe, opipe);
  1362. i_tail = ipipe->tail;
  1363. i_mask = ipipe->ring_size - 1;
  1364. o_head = opipe->head;
  1365. o_mask = opipe->ring_size - 1;
  1366. do {
  1367. if (!opipe->readers) {
  1368. send_sig(SIGPIPE, current, 0);
  1369. if (!ret)
  1370. ret = -EPIPE;
  1371. break;
  1372. }
  1373. i_head = ipipe->head;
  1374. o_tail = opipe->tail;
  1375. /*
  1376. * If we have iterated all input buffers or run out of
  1377. * output room, break.
  1378. */
  1379. if (pipe_empty(i_head, i_tail) ||
  1380. pipe_full(o_head, o_tail, opipe->max_usage))
  1381. break;
  1382. ibuf = &ipipe->bufs[i_tail & i_mask];
  1383. obuf = &opipe->bufs[o_head & o_mask];
  1384. /*
  1385. * Get a reference to this pipe buffer,
  1386. * so we can copy the contents over.
  1387. */
  1388. if (!pipe_buf_get(ipipe, ibuf)) {
  1389. if (ret == 0)
  1390. ret = -EFAULT;
  1391. break;
  1392. }
  1393. *obuf = *ibuf;
  1394. /*
  1395. * Don't inherit the gift and merge flag, we need to prevent
  1396. * multiple steals of this page.
  1397. */
  1398. obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
  1399. obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
  1400. if (obuf->len > len)
  1401. obuf->len = len;
  1402. ret += obuf->len;
  1403. len -= obuf->len;
  1404. o_head++;
  1405. opipe->head = o_head;
  1406. i_tail++;
  1407. } while (len);
  1408. pipe_unlock(ipipe);
  1409. pipe_unlock(opipe);
  1410. /*
  1411. * If we put data in the output pipe, wakeup any potential readers.
  1412. */
  1413. if (ret > 0)
  1414. wakeup_pipe_readers(opipe);
  1415. return ret;
  1416. }
  1417. /*
  1418. * This is a tee(1) implementation that works on pipes. It doesn't copy
  1419. * any data, it simply references the 'in' pages on the 'out' pipe.
  1420. * The 'flags' used are the SPLICE_F_* variants, currently the only
  1421. * applicable one is SPLICE_F_NONBLOCK.
  1422. */
  1423. long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
  1424. {
  1425. struct pipe_inode_info *ipipe = get_pipe_info(in, true);
  1426. struct pipe_inode_info *opipe = get_pipe_info(out, true);
  1427. int ret = -EINVAL;
  1428. if (unlikely(!(in->f_mode & FMODE_READ) ||
  1429. !(out->f_mode & FMODE_WRITE)))
  1430. return -EBADF;
  1431. /*
  1432. * Duplicate the contents of ipipe to opipe without actually
  1433. * copying the data.
  1434. */
  1435. if (ipipe && opipe && ipipe != opipe) {
  1436. if ((in->f_flags | out->f_flags) & O_NONBLOCK)
  1437. flags |= SPLICE_F_NONBLOCK;
  1438. /*
  1439. * Keep going, unless we encounter an error. The ipipe/opipe
  1440. * ordering doesn't really matter.
  1441. */
  1442. ret = ipipe_prep(ipipe, flags);
  1443. if (!ret) {
  1444. ret = opipe_prep(opipe, flags);
  1445. if (!ret)
  1446. ret = link_pipe(ipipe, opipe, len, flags);
  1447. }
  1448. }
  1449. return ret;
  1450. }
  1451. SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
  1452. {
  1453. struct fd in, out;
  1454. int error;
  1455. if (unlikely(flags & ~SPLICE_F_ALL))
  1456. return -EINVAL;
  1457. if (unlikely(!len))
  1458. return 0;
  1459. error = -EBADF;
  1460. in = fdget(fdin);
  1461. if (in.file) {
  1462. out = fdget(fdout);
  1463. if (out.file) {
  1464. error = do_tee(in.file, out.file, len, flags);
  1465. fdput(out);
  1466. }
  1467. fdput(in);
  1468. }
  1469. return error;
  1470. }