splice.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557
  1. /*
  2. * "splice": joining two ropes together by interweaving their strands.
  3. *
  4. * This is the "extended pipe" functionality, where a pipe is used as
  5. * an arbitrary in-memory buffer. Think of a pipe as a small kernel
  6. * buffer that you can use to transfer data from one end to the other.
  7. *
  8. * The traditional unix read/write is extended with a "splice()" operation
  9. * that transfers data buffers to or from a pipe buffer.
  10. *
  11. * Named by Larry McVoy, original implementation from Linus, extended by
  12. * Jens to support splicing to files, network, direct splicing, etc and
  13. * fixing lots of bugs.
  14. *
  15. * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
  16. * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
  17. * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  18. *
  19. */
  20. #include <linux/fs.h>
  21. #include <linux/file.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/pipe_fs_i.h>
  24. #include <linux/mm_inline.h>
  25. #include <linux/swap.h>
  26. #include <linux/writeback.h>
  27. #include <linux/buffer_head.h>
  28. #include <linux/module.h>
  29. #include <linux/syscalls.h>
  30. #include <linux/uio.h>
  31. struct partial_page {
  32. unsigned int offset;
  33. unsigned int len;
  34. };
  35. /*
  36. * Passed to splice_to_pipe
  37. */
  38. struct splice_pipe_desc {
  39. struct page **pages; /* page map */
  40. struct partial_page *partial; /* pages[] may not be contig */
  41. int nr_pages; /* number of pages in map */
  42. unsigned int flags; /* splice flags */
  43. const struct pipe_buf_operations *ops;/* ops associated with output pipe */
  44. };
  45. /*
  46. * Attempt to steal a page from a pipe buffer. This should perhaps go into
  47. * a vm helper function, it's already simplified quite a bit by the
  48. * addition of remove_mapping(). If success is returned, the caller may
  49. * attempt to reuse this page for another destination.
  50. */
  51. static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
  52. struct pipe_buffer *buf)
  53. {
  54. struct page *page = buf->page;
  55. struct address_space *mapping;
  56. lock_page(page);
  57. mapping = page_mapping(page);
  58. if (mapping) {
  59. WARN_ON(!PageUptodate(page));
  60. /*
  61. * At least for ext2 with nobh option, we need to wait on
  62. * writeback completing on this page, since we'll remove it
  63. * from the pagecache. Otherwise truncate wont wait on the
  64. * page, allowing the disk blocks to be reused by someone else
  65. * before we actually wrote our data to them. fs corruption
  66. * ensues.
  67. */
  68. wait_on_page_writeback(page);
  69. if (PagePrivate(page))
  70. try_to_release_page(page, GFP_KERNEL);
  71. /*
  72. * If we succeeded in removing the mapping, set LRU flag
  73. * and return good.
  74. */
  75. if (remove_mapping(mapping, page)) {
  76. buf->flags |= PIPE_BUF_FLAG_LRU;
  77. return 0;
  78. }
  79. }
  80. /*
  81. * Raced with truncate or failed to remove page from current
  82. * address space, unlock and return failure.
  83. */
  84. unlock_page(page);
  85. return 1;
  86. }
  87. static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
  88. struct pipe_buffer *buf)
  89. {
  90. page_cache_release(buf->page);
  91. buf->flags &= ~PIPE_BUF_FLAG_LRU;
  92. }
  93. static int page_cache_pipe_buf_pin(struct pipe_inode_info *pipe,
  94. struct pipe_buffer *buf)
  95. {
  96. struct page *page = buf->page;
  97. int err;
  98. if (!PageUptodate(page)) {
  99. lock_page(page);
  100. /*
  101. * Page got truncated/unhashed. This will cause a 0-byte
  102. * splice, if this is the first page.
  103. */
  104. if (!page->mapping) {
  105. err = -ENODATA;
  106. goto error;
  107. }
  108. /*
  109. * Uh oh, read-error from disk.
  110. */
  111. if (!PageUptodate(page)) {
  112. err = -EIO;
  113. goto error;
  114. }
  115. /*
  116. * Page is ok afterall, we are done.
  117. */
  118. unlock_page(page);
  119. }
  120. return 0;
  121. error:
  122. unlock_page(page);
  123. return err;
  124. }
  125. static const struct pipe_buf_operations page_cache_pipe_buf_ops = {
  126. .can_merge = 0,
  127. .map = generic_pipe_buf_map,
  128. .unmap = generic_pipe_buf_unmap,
  129. .pin = page_cache_pipe_buf_pin,
  130. .release = page_cache_pipe_buf_release,
  131. .steal = page_cache_pipe_buf_steal,
  132. .get = generic_pipe_buf_get,
  133. };
  134. static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
  135. struct pipe_buffer *buf)
  136. {
  137. if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
  138. return 1;
  139. buf->flags |= PIPE_BUF_FLAG_LRU;
  140. return generic_pipe_buf_steal(pipe, buf);
  141. }
  142. static const struct pipe_buf_operations user_page_pipe_buf_ops = {
  143. .can_merge = 0,
  144. .map = generic_pipe_buf_map,
  145. .unmap = generic_pipe_buf_unmap,
  146. .pin = generic_pipe_buf_pin,
  147. .release = page_cache_pipe_buf_release,
  148. .steal = user_page_pipe_buf_steal,
  149. .get = generic_pipe_buf_get,
  150. };
  151. /*
  152. * Pipe output worker. This sets up our pipe format with the page cache
  153. * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
  154. */
  155. static ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
  156. struct splice_pipe_desc *spd)
  157. {
  158. int ret, do_wakeup, page_nr;
  159. ret = 0;
  160. do_wakeup = 0;
  161. page_nr = 0;
  162. if (pipe->inode)
  163. mutex_lock(&pipe->inode->i_mutex);
  164. for (;;) {
  165. if (!pipe->readers) {
  166. send_sig(SIGPIPE, current, 0);
  167. if (!ret)
  168. ret = -EPIPE;
  169. break;
  170. }
  171. if (pipe->nrbufs < PIPE_BUFFERS) {
  172. int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
  173. struct pipe_buffer *buf = pipe->bufs + newbuf;
  174. buf->page = spd->pages[page_nr];
  175. buf->offset = spd->partial[page_nr].offset;
  176. buf->len = spd->partial[page_nr].len;
  177. buf->ops = spd->ops;
  178. if (spd->flags & SPLICE_F_GIFT)
  179. buf->flags |= PIPE_BUF_FLAG_GIFT;
  180. pipe->nrbufs++;
  181. page_nr++;
  182. ret += buf->len;
  183. if (pipe->inode)
  184. do_wakeup = 1;
  185. if (!--spd->nr_pages)
  186. break;
  187. if (pipe->nrbufs < PIPE_BUFFERS)
  188. continue;
  189. break;
  190. }
  191. if (spd->flags & SPLICE_F_NONBLOCK) {
  192. if (!ret)
  193. ret = -EAGAIN;
  194. break;
  195. }
  196. if (signal_pending(current)) {
  197. if (!ret)
  198. ret = -ERESTARTSYS;
  199. break;
  200. }
  201. if (do_wakeup) {
  202. smp_mb();
  203. if (waitqueue_active(&pipe->wait))
  204. wake_up_interruptible_sync(&pipe->wait);
  205. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  206. do_wakeup = 0;
  207. }
  208. pipe->waiting_writers++;
  209. pipe_wait(pipe);
  210. pipe->waiting_writers--;
  211. }
  212. if (pipe->inode)
  213. mutex_unlock(&pipe->inode->i_mutex);
  214. if (do_wakeup) {
  215. smp_mb();
  216. if (waitqueue_active(&pipe->wait))
  217. wake_up_interruptible(&pipe->wait);
  218. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  219. }
  220. while (page_nr < spd->nr_pages)
  221. page_cache_release(spd->pages[page_nr++]);
  222. return ret;
  223. }
  224. static int
  225. __generic_file_splice_read(struct file *in, loff_t *ppos,
  226. struct pipe_inode_info *pipe, size_t len,
  227. unsigned int flags)
  228. {
  229. struct address_space *mapping = in->f_mapping;
  230. unsigned int loff, nr_pages;
  231. struct page *pages[PIPE_BUFFERS];
  232. struct partial_page partial[PIPE_BUFFERS];
  233. struct page *page;
  234. pgoff_t index, end_index;
  235. loff_t isize;
  236. size_t total_len;
  237. int error, page_nr;
  238. struct splice_pipe_desc spd = {
  239. .pages = pages,
  240. .partial = partial,
  241. .flags = flags,
  242. .ops = &page_cache_pipe_buf_ops,
  243. };
  244. index = *ppos >> PAGE_CACHE_SHIFT;
  245. loff = *ppos & ~PAGE_CACHE_MASK;
  246. nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  247. if (nr_pages > PIPE_BUFFERS)
  248. nr_pages = PIPE_BUFFERS;
  249. /*
  250. * Initiate read-ahead on this page range. however, don't call into
  251. * read-ahead if this is a non-zero offset (we are likely doing small
  252. * chunk splice and the page is already there) for a single page.
  253. */
  254. if (!loff || nr_pages > 1)
  255. page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
  256. /*
  257. * Now fill in the holes:
  258. */
  259. error = 0;
  260. total_len = 0;
  261. /*
  262. * Lookup the (hopefully) full range of pages we need.
  263. */
  264. spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
  265. /*
  266. * If find_get_pages_contig() returned fewer pages than we needed,
  267. * allocate the rest.
  268. */
  269. index += spd.nr_pages;
  270. while (spd.nr_pages < nr_pages) {
  271. /*
  272. * Page could be there, find_get_pages_contig() breaks on
  273. * the first hole.
  274. */
  275. page = find_get_page(mapping, index);
  276. if (!page) {
  277. /*
  278. * Make sure the read-ahead engine is notified
  279. * about this failure.
  280. */
  281. handle_ra_miss(mapping, &in->f_ra, index);
  282. /*
  283. * page didn't exist, allocate one.
  284. */
  285. page = page_cache_alloc_cold(mapping);
  286. if (!page)
  287. break;
  288. error = add_to_page_cache_lru(page, mapping, index,
  289. GFP_KERNEL);
  290. if (unlikely(error)) {
  291. page_cache_release(page);
  292. if (error == -EEXIST)
  293. continue;
  294. break;
  295. }
  296. /*
  297. * add_to_page_cache() locks the page, unlock it
  298. * to avoid convoluting the logic below even more.
  299. */
  300. unlock_page(page);
  301. }
  302. pages[spd.nr_pages++] = page;
  303. index++;
  304. }
  305. /*
  306. * Now loop over the map and see if we need to start IO on any
  307. * pages, fill in the partial map, etc.
  308. */
  309. index = *ppos >> PAGE_CACHE_SHIFT;
  310. nr_pages = spd.nr_pages;
  311. spd.nr_pages = 0;
  312. for (page_nr = 0; page_nr < nr_pages; page_nr++) {
  313. unsigned int this_len;
  314. if (!len)
  315. break;
  316. /*
  317. * this_len is the max we'll use from this page
  318. */
  319. this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
  320. page = pages[page_nr];
  321. /*
  322. * If the page isn't uptodate, we may need to start io on it
  323. */
  324. if (!PageUptodate(page)) {
  325. /*
  326. * If in nonblock mode then dont block on waiting
  327. * for an in-flight io page
  328. */
  329. if (flags & SPLICE_F_NONBLOCK)
  330. break;
  331. lock_page(page);
  332. /*
  333. * page was truncated, stop here. if this isn't the
  334. * first page, we'll just complete what we already
  335. * added
  336. */
  337. if (!page->mapping) {
  338. unlock_page(page);
  339. break;
  340. }
  341. /*
  342. * page was already under io and is now done, great
  343. */
  344. if (PageUptodate(page)) {
  345. unlock_page(page);
  346. goto fill_it;
  347. }
  348. /*
  349. * need to read in the page
  350. */
  351. error = mapping->a_ops->readpage(in, page);
  352. if (unlikely(error)) {
  353. /*
  354. * We really should re-lookup the page here,
  355. * but it complicates things a lot. Instead
  356. * lets just do what we already stored, and
  357. * we'll get it the next time we are called.
  358. */
  359. if (error == AOP_TRUNCATED_PAGE)
  360. error = 0;
  361. break;
  362. }
  363. /*
  364. * i_size must be checked after ->readpage().
  365. */
  366. isize = i_size_read(mapping->host);
  367. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  368. if (unlikely(!isize || index > end_index))
  369. break;
  370. /*
  371. * if this is the last page, see if we need to shrink
  372. * the length and stop
  373. */
  374. if (end_index == index) {
  375. loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
  376. if (total_len + loff > isize)
  377. break;
  378. /*
  379. * force quit after adding this page
  380. */
  381. len = this_len;
  382. this_len = min(this_len, loff);
  383. loff = 0;
  384. }
  385. }
  386. fill_it:
  387. partial[page_nr].offset = loff;
  388. partial[page_nr].len = this_len;
  389. len -= this_len;
  390. total_len += this_len;
  391. loff = 0;
  392. spd.nr_pages++;
  393. index++;
  394. }
  395. /*
  396. * Release any pages at the end, if we quit early. 'i' is how far
  397. * we got, 'nr_pages' is how many pages are in the map.
  398. */
  399. while (page_nr < nr_pages)
  400. page_cache_release(pages[page_nr++]);
  401. if (spd.nr_pages)
  402. return splice_to_pipe(pipe, &spd);
  403. return error;
  404. }
  405. /**
  406. * generic_file_splice_read - splice data from file to a pipe
  407. * @in: file to splice from
  408. * @pipe: pipe to splice to
  409. * @len: number of bytes to splice
  410. * @flags: splice modifier flags
  411. *
  412. * Will read pages from given file and fill them into a pipe.
  413. */
  414. ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
  415. struct pipe_inode_info *pipe, size_t len,
  416. unsigned int flags)
  417. {
  418. ssize_t spliced;
  419. int ret;
  420. ret = 0;
  421. spliced = 0;
  422. while (len) {
  423. ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
  424. if (ret < 0)
  425. break;
  426. else if (!ret) {
  427. if (spliced)
  428. break;
  429. if (flags & SPLICE_F_NONBLOCK) {
  430. ret = -EAGAIN;
  431. break;
  432. }
  433. }
  434. *ppos += ret;
  435. len -= ret;
  436. spliced += ret;
  437. }
  438. if (spliced)
  439. return spliced;
  440. return ret;
  441. }
  442. EXPORT_SYMBOL(generic_file_splice_read);
  443. /*
  444. * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
  445. * using sendpage(). Return the number of bytes sent.
  446. */
  447. static int pipe_to_sendpage(struct pipe_inode_info *pipe,
  448. struct pipe_buffer *buf, struct splice_desc *sd)
  449. {
  450. struct file *file = sd->file;
  451. loff_t pos = sd->pos;
  452. int ret, more;
  453. ret = buf->ops->pin(pipe, buf);
  454. if (!ret) {
  455. more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
  456. ret = file->f_op->sendpage(file, buf->page, buf->offset,
  457. sd->len, &pos, more);
  458. }
  459. return ret;
  460. }
  461. /*
  462. * This is a little more tricky than the file -> pipe splicing. There are
  463. * basically three cases:
  464. *
  465. * - Destination page already exists in the address space and there
  466. * are users of it. For that case we have no other option that
  467. * copying the data. Tough luck.
  468. * - Destination page already exists in the address space, but there
  469. * are no users of it. Make sure it's uptodate, then drop it. Fall
  470. * through to last case.
  471. * - Destination page does not exist, we can add the pipe page to
  472. * the page cache and avoid the copy.
  473. *
  474. * If asked to move pages to the output file (SPLICE_F_MOVE is set in
  475. * sd->flags), we attempt to migrate pages from the pipe to the output
  476. * file address space page cache. This is possible if no one else has
  477. * the pipe page referenced outside of the pipe and page cache. If
  478. * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
  479. * a new page in the output file page cache and fill/dirty that.
  480. */
  481. static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
  482. struct splice_desc *sd)
  483. {
  484. struct file *file = sd->file;
  485. struct address_space *mapping = file->f_mapping;
  486. unsigned int offset, this_len;
  487. struct page *page;
  488. pgoff_t index;
  489. int ret;
  490. /*
  491. * make sure the data in this buffer is uptodate
  492. */
  493. ret = buf->ops->pin(pipe, buf);
  494. if (unlikely(ret))
  495. return ret;
  496. index = sd->pos >> PAGE_CACHE_SHIFT;
  497. offset = sd->pos & ~PAGE_CACHE_MASK;
  498. this_len = sd->len;
  499. if (this_len + offset > PAGE_CACHE_SIZE)
  500. this_len = PAGE_CACHE_SIZE - offset;
  501. find_page:
  502. page = find_lock_page(mapping, index);
  503. if (!page) {
  504. ret = -ENOMEM;
  505. page = page_cache_alloc_cold(mapping);
  506. if (unlikely(!page))
  507. goto out_ret;
  508. /*
  509. * This will also lock the page
  510. */
  511. ret = add_to_page_cache_lru(page, mapping, index,
  512. GFP_KERNEL);
  513. if (unlikely(ret))
  514. goto out;
  515. }
  516. ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
  517. if (unlikely(ret)) {
  518. loff_t isize = i_size_read(mapping->host);
  519. if (ret != AOP_TRUNCATED_PAGE)
  520. unlock_page(page);
  521. page_cache_release(page);
  522. if (ret == AOP_TRUNCATED_PAGE)
  523. goto find_page;
  524. /*
  525. * prepare_write() may have instantiated a few blocks
  526. * outside i_size. Trim these off again.
  527. */
  528. if (sd->pos + this_len > isize)
  529. vmtruncate(mapping->host, isize);
  530. goto out_ret;
  531. }
  532. if (buf->page != page) {
  533. /*
  534. * Careful, ->map() uses KM_USER0!
  535. */
  536. char *src = buf->ops->map(pipe, buf, 1);
  537. char *dst = kmap_atomic(page, KM_USER1);
  538. memcpy(dst + offset, src + buf->offset, this_len);
  539. flush_dcache_page(page);
  540. kunmap_atomic(dst, KM_USER1);
  541. buf->ops->unmap(pipe, buf, src);
  542. }
  543. ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
  544. if (ret) {
  545. if (ret == AOP_TRUNCATED_PAGE) {
  546. page_cache_release(page);
  547. goto find_page;
  548. }
  549. if (ret < 0)
  550. goto out;
  551. /*
  552. * Partial write has happened, so 'ret' already initialized by
  553. * number of bytes written, Where is nothing we have to do here.
  554. */
  555. } else
  556. ret = this_len;
  557. /*
  558. * Return the number of bytes written and mark page as
  559. * accessed, we are now done!
  560. */
  561. mark_page_accessed(page);
  562. balance_dirty_pages_ratelimited(mapping);
  563. out:
  564. page_cache_release(page);
  565. unlock_page(page);
  566. out_ret:
  567. return ret;
  568. }
  569. /*
  570. * Pipe input worker. Most of this logic works like a regular pipe, the
  571. * key here is the 'actor' worker passed in that actually moves the data
  572. * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
  573. */
  574. ssize_t __splice_from_pipe(struct pipe_inode_info *pipe,
  575. struct file *out, loff_t *ppos, size_t len,
  576. unsigned int flags, splice_actor *actor)
  577. {
  578. int ret, do_wakeup, err;
  579. struct splice_desc sd;
  580. ret = 0;
  581. do_wakeup = 0;
  582. sd.total_len = len;
  583. sd.flags = flags;
  584. sd.file = out;
  585. sd.pos = *ppos;
  586. for (;;) {
  587. if (pipe->nrbufs) {
  588. struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
  589. const struct pipe_buf_operations *ops = buf->ops;
  590. sd.len = buf->len;
  591. if (sd.len > sd.total_len)
  592. sd.len = sd.total_len;
  593. err = actor(pipe, buf, &sd);
  594. if (err <= 0) {
  595. if (!ret && err != -ENODATA)
  596. ret = err;
  597. break;
  598. }
  599. ret += err;
  600. buf->offset += err;
  601. buf->len -= err;
  602. sd.len -= err;
  603. sd.pos += err;
  604. sd.total_len -= err;
  605. if (sd.len)
  606. continue;
  607. if (!buf->len) {
  608. buf->ops = NULL;
  609. ops->release(pipe, buf);
  610. pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1);
  611. pipe->nrbufs--;
  612. if (pipe->inode)
  613. do_wakeup = 1;
  614. }
  615. if (!sd.total_len)
  616. break;
  617. }
  618. if (pipe->nrbufs)
  619. continue;
  620. if (!pipe->writers)
  621. break;
  622. if (!pipe->waiting_writers) {
  623. if (ret)
  624. break;
  625. }
  626. if (flags & SPLICE_F_NONBLOCK) {
  627. if (!ret)
  628. ret = -EAGAIN;
  629. break;
  630. }
  631. if (signal_pending(current)) {
  632. if (!ret)
  633. ret = -ERESTARTSYS;
  634. break;
  635. }
  636. if (do_wakeup) {
  637. smp_mb();
  638. if (waitqueue_active(&pipe->wait))
  639. wake_up_interruptible_sync(&pipe->wait);
  640. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  641. do_wakeup = 0;
  642. }
  643. pipe_wait(pipe);
  644. }
  645. if (do_wakeup) {
  646. smp_mb();
  647. if (waitqueue_active(&pipe->wait))
  648. wake_up_interruptible(&pipe->wait);
  649. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  650. }
  651. return ret;
  652. }
  653. EXPORT_SYMBOL(__splice_from_pipe);
  654. ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
  655. loff_t *ppos, size_t len, unsigned int flags,
  656. splice_actor *actor)
  657. {
  658. ssize_t ret;
  659. struct inode *inode = out->f_mapping->host;
  660. /*
  661. * The actor worker might be calling ->prepare_write and
  662. * ->commit_write. Most of the time, these expect i_mutex to
  663. * be held. Since this may result in an ABBA deadlock with
  664. * pipe->inode, we have to order lock acquiry here.
  665. */
  666. inode_double_lock(inode, pipe->inode);
  667. ret = __splice_from_pipe(pipe, out, ppos, len, flags, actor);
  668. inode_double_unlock(inode, pipe->inode);
  669. return ret;
  670. }
  671. /**
  672. * generic_file_splice_write_nolock - generic_file_splice_write without mutexes
  673. * @pipe: pipe info
  674. * @out: file to write to
  675. * @len: number of bytes to splice
  676. * @flags: splice modifier flags
  677. *
  678. * Will either move or copy pages (determined by @flags options) from
  679. * the given pipe inode to the given file. The caller is responsible
  680. * for acquiring i_mutex on both inodes.
  681. *
  682. */
  683. ssize_t
  684. generic_file_splice_write_nolock(struct pipe_inode_info *pipe, struct file *out,
  685. loff_t *ppos, size_t len, unsigned int flags)
  686. {
  687. struct address_space *mapping = out->f_mapping;
  688. struct inode *inode = mapping->host;
  689. ssize_t ret;
  690. int err;
  691. err = remove_suid(out->f_path.dentry);
  692. if (unlikely(err))
  693. return err;
  694. ret = __splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
  695. if (ret > 0) {
  696. *ppos += ret;
  697. /*
  698. * If file or inode is SYNC and we actually wrote some data,
  699. * sync it.
  700. */
  701. if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
  702. err = generic_osync_inode(inode, mapping,
  703. OSYNC_METADATA|OSYNC_DATA);
  704. if (err)
  705. ret = err;
  706. }
  707. }
  708. return ret;
  709. }
  710. EXPORT_SYMBOL(generic_file_splice_write_nolock);
  711. /**
  712. * generic_file_splice_write - splice data from a pipe to a file
  713. * @pipe: pipe info
  714. * @out: file to write to
  715. * @len: number of bytes to splice
  716. * @flags: splice modifier flags
  717. *
  718. * Will either move or copy pages (determined by @flags options) from
  719. * the given pipe inode to the given file.
  720. *
  721. */
  722. ssize_t
  723. generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
  724. loff_t *ppos, size_t len, unsigned int flags)
  725. {
  726. struct address_space *mapping = out->f_mapping;
  727. struct inode *inode = mapping->host;
  728. ssize_t ret;
  729. int err;
  730. err = should_remove_suid(out->f_path.dentry);
  731. if (unlikely(err)) {
  732. mutex_lock(&inode->i_mutex);
  733. err = __remove_suid(out->f_path.dentry, err);
  734. mutex_unlock(&inode->i_mutex);
  735. if (err)
  736. return err;
  737. }
  738. ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
  739. if (ret > 0) {
  740. *ppos += ret;
  741. /*
  742. * If file or inode is SYNC and we actually wrote some data,
  743. * sync it.
  744. */
  745. if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
  746. mutex_lock(&inode->i_mutex);
  747. err = generic_osync_inode(inode, mapping,
  748. OSYNC_METADATA|OSYNC_DATA);
  749. mutex_unlock(&inode->i_mutex);
  750. if (err)
  751. ret = err;
  752. }
  753. }
  754. return ret;
  755. }
  756. EXPORT_SYMBOL(generic_file_splice_write);
  757. /**
  758. * generic_splice_sendpage - splice data from a pipe to a socket
  759. * @inode: pipe inode
  760. * @out: socket to write to
  761. * @len: number of bytes to splice
  762. * @flags: splice modifier flags
  763. *
  764. * Will send @len bytes from the pipe to a network socket. No data copying
  765. * is involved.
  766. *
  767. */
  768. ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
  769. loff_t *ppos, size_t len, unsigned int flags)
  770. {
  771. return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
  772. }
  773. EXPORT_SYMBOL(generic_splice_sendpage);
  774. /*
  775. * Attempt to initiate a splice from pipe to file.
  776. */
  777. static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
  778. loff_t *ppos, size_t len, unsigned int flags)
  779. {
  780. int ret;
  781. if (unlikely(!out->f_op || !out->f_op->splice_write))
  782. return -EINVAL;
  783. if (unlikely(!(out->f_mode & FMODE_WRITE)))
  784. return -EBADF;
  785. ret = rw_verify_area(WRITE, out, ppos, len);
  786. if (unlikely(ret < 0))
  787. return ret;
  788. return out->f_op->splice_write(pipe, out, ppos, len, flags);
  789. }
  790. /*
  791. * Attempt to initiate a splice from a file to a pipe.
  792. */
  793. static long do_splice_to(struct file *in, loff_t *ppos,
  794. struct pipe_inode_info *pipe, size_t len,
  795. unsigned int flags)
  796. {
  797. loff_t isize, left;
  798. int ret;
  799. if (unlikely(!in->f_op || !in->f_op->splice_read))
  800. return -EINVAL;
  801. if (unlikely(!(in->f_mode & FMODE_READ)))
  802. return -EBADF;
  803. ret = rw_verify_area(READ, in, ppos, len);
  804. if (unlikely(ret < 0))
  805. return ret;
  806. isize = i_size_read(in->f_mapping->host);
  807. if (unlikely(*ppos >= isize))
  808. return 0;
  809. left = isize - *ppos;
  810. if (unlikely(left < len))
  811. len = left;
  812. return in->f_op->splice_read(in, ppos, pipe, len, flags);
  813. }
  814. long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
  815. size_t len, unsigned int flags)
  816. {
  817. struct pipe_inode_info *pipe;
  818. long ret, bytes;
  819. loff_t out_off;
  820. umode_t i_mode;
  821. int i;
  822. /*
  823. * We require the input being a regular file, as we don't want to
  824. * randomly drop data for eg socket -> socket splicing. Use the
  825. * piped splicing for that!
  826. */
  827. i_mode = in->f_path.dentry->d_inode->i_mode;
  828. if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
  829. return -EINVAL;
  830. /*
  831. * neither in nor out is a pipe, setup an internal pipe attached to
  832. * 'out' and transfer the wanted data from 'in' to 'out' through that
  833. */
  834. pipe = current->splice_pipe;
  835. if (unlikely(!pipe)) {
  836. pipe = alloc_pipe_info(NULL);
  837. if (!pipe)
  838. return -ENOMEM;
  839. /*
  840. * We don't have an immediate reader, but we'll read the stuff
  841. * out of the pipe right after the splice_to_pipe(). So set
  842. * PIPE_READERS appropriately.
  843. */
  844. pipe->readers = 1;
  845. current->splice_pipe = pipe;
  846. }
  847. /*
  848. * Do the splice.
  849. */
  850. ret = 0;
  851. bytes = 0;
  852. out_off = 0;
  853. while (len) {
  854. size_t read_len, max_read_len;
  855. /*
  856. * Do at most PIPE_BUFFERS pages worth of transfer:
  857. */
  858. max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
  859. ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
  860. if (unlikely(ret < 0))
  861. goto out_release;
  862. read_len = ret;
  863. /*
  864. * NOTE: nonblocking mode only applies to the input. We
  865. * must not do the output in nonblocking mode as then we
  866. * could get stuck data in the internal pipe:
  867. */
  868. ret = do_splice_from(pipe, out, &out_off, read_len,
  869. flags & ~SPLICE_F_NONBLOCK);
  870. if (unlikely(ret < 0))
  871. goto out_release;
  872. bytes += ret;
  873. len -= ret;
  874. /*
  875. * In nonblocking mode, if we got back a short read then
  876. * that was due to either an IO error or due to the
  877. * pagecache entry not being there. In the IO error case
  878. * the _next_ splice attempt will produce a clean IO error
  879. * return value (not a short read), so in both cases it's
  880. * correct to break out of the loop here:
  881. */
  882. if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
  883. break;
  884. }
  885. pipe->nrbufs = pipe->curbuf = 0;
  886. return bytes;
  887. out_release:
  888. /*
  889. * If we did an incomplete transfer we must release
  890. * the pipe buffers in question:
  891. */
  892. for (i = 0; i < PIPE_BUFFERS; i++) {
  893. struct pipe_buffer *buf = pipe->bufs + i;
  894. if (buf->ops) {
  895. buf->ops->release(pipe, buf);
  896. buf->ops = NULL;
  897. }
  898. }
  899. pipe->nrbufs = pipe->curbuf = 0;
  900. /*
  901. * If we transferred some data, return the number of bytes:
  902. */
  903. if (bytes > 0)
  904. return bytes;
  905. return ret;
  906. }
  907. EXPORT_SYMBOL(do_splice_direct);
  908. /*
  909. * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
  910. * location, so checking ->i_pipe is not enough to verify that this is a
  911. * pipe.
  912. */
  913. static inline struct pipe_inode_info *pipe_info(struct inode *inode)
  914. {
  915. if (S_ISFIFO(inode->i_mode))
  916. return inode->i_pipe;
  917. return NULL;
  918. }
  919. /*
  920. * Determine where to splice to/from.
  921. */
  922. static long do_splice(struct file *in, loff_t __user *off_in,
  923. struct file *out, loff_t __user *off_out,
  924. size_t len, unsigned int flags)
  925. {
  926. struct pipe_inode_info *pipe;
  927. loff_t offset, *off;
  928. long ret;
  929. pipe = pipe_info(in->f_path.dentry->d_inode);
  930. if (pipe) {
  931. if (off_in)
  932. return -ESPIPE;
  933. if (off_out) {
  934. if (out->f_op->llseek == no_llseek)
  935. return -EINVAL;
  936. if (copy_from_user(&offset, off_out, sizeof(loff_t)))
  937. return -EFAULT;
  938. off = &offset;
  939. } else
  940. off = &out->f_pos;
  941. ret = do_splice_from(pipe, out, off, len, flags);
  942. if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
  943. ret = -EFAULT;
  944. return ret;
  945. }
  946. pipe = pipe_info(out->f_path.dentry->d_inode);
  947. if (pipe) {
  948. if (off_out)
  949. return -ESPIPE;
  950. if (off_in) {
  951. if (in->f_op->llseek == no_llseek)
  952. return -EINVAL;
  953. if (copy_from_user(&offset, off_in, sizeof(loff_t)))
  954. return -EFAULT;
  955. off = &offset;
  956. } else
  957. off = &in->f_pos;
  958. ret = do_splice_to(in, off, pipe, len, flags);
  959. if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
  960. ret = -EFAULT;
  961. return ret;
  962. }
  963. return -EINVAL;
  964. }
  965. /*
  966. * Map an iov into an array of pages and offset/length tupples. With the
  967. * partial_page structure, we can map several non-contiguous ranges into
  968. * our ones pages[] map instead of splitting that operation into pieces.
  969. * Could easily be exported as a generic helper for other users, in which
  970. * case one would probably want to add a 'max_nr_pages' parameter as well.
  971. */
  972. static int get_iovec_page_array(const struct iovec __user *iov,
  973. unsigned int nr_vecs, struct page **pages,
  974. struct partial_page *partial, int aligned)
  975. {
  976. int buffers = 0, error = 0;
  977. /*
  978. * It's ok to take the mmap_sem for reading, even
  979. * across a "get_user()".
  980. */
  981. down_read(&current->mm->mmap_sem);
  982. while (nr_vecs) {
  983. unsigned long off, npages;
  984. void __user *base;
  985. size_t len;
  986. int i;
  987. /*
  988. * Get user address base and length for this iovec.
  989. */
  990. error = get_user(base, &iov->iov_base);
  991. if (unlikely(error))
  992. break;
  993. error = get_user(len, &iov->iov_len);
  994. if (unlikely(error))
  995. break;
  996. /*
  997. * Sanity check this iovec. 0 read succeeds.
  998. */
  999. if (unlikely(!len))
  1000. break;
  1001. error = -EFAULT;
  1002. if (unlikely(!base))
  1003. break;
  1004. /*
  1005. * Get this base offset and number of pages, then map
  1006. * in the user pages.
  1007. */
  1008. off = (unsigned long) base & ~PAGE_MASK;
  1009. /*
  1010. * If asked for alignment, the offset must be zero and the
  1011. * length a multiple of the PAGE_SIZE.
  1012. */
  1013. error = -EINVAL;
  1014. if (aligned && (off || len & ~PAGE_MASK))
  1015. break;
  1016. npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1017. if (npages > PIPE_BUFFERS - buffers)
  1018. npages = PIPE_BUFFERS - buffers;
  1019. error = get_user_pages(current, current->mm,
  1020. (unsigned long) base, npages, 0, 0,
  1021. &pages[buffers], NULL);
  1022. if (unlikely(error <= 0))
  1023. break;
  1024. /*
  1025. * Fill this contiguous range into the partial page map.
  1026. */
  1027. for (i = 0; i < error; i++) {
  1028. const int plen = min_t(size_t, len, PAGE_SIZE - off);
  1029. partial[buffers].offset = off;
  1030. partial[buffers].len = plen;
  1031. off = 0;
  1032. len -= plen;
  1033. buffers++;
  1034. }
  1035. /*
  1036. * We didn't complete this iov, stop here since it probably
  1037. * means we have to move some of this into a pipe to
  1038. * be able to continue.
  1039. */
  1040. if (len)
  1041. break;
  1042. /*
  1043. * Don't continue if we mapped fewer pages than we asked for,
  1044. * or if we mapped the max number of pages that we have
  1045. * room for.
  1046. */
  1047. if (error < npages || buffers == PIPE_BUFFERS)
  1048. break;
  1049. nr_vecs--;
  1050. iov++;
  1051. }
  1052. up_read(&current->mm->mmap_sem);
  1053. if (buffers)
  1054. return buffers;
  1055. return error;
  1056. }
  1057. /*
  1058. * vmsplice splices a user address range into a pipe. It can be thought of
  1059. * as splice-from-memory, where the regular splice is splice-from-file (or
  1060. * to file). In both cases the output is a pipe, naturally.
  1061. *
  1062. * Note that vmsplice only supports splicing _from_ user memory to a pipe,
  1063. * not the other way around. Splicing from user memory is a simple operation
  1064. * that can be supported without any funky alignment restrictions or nasty
  1065. * vm tricks. We simply map in the user memory and fill them into a pipe.
  1066. * The reverse isn't quite as easy, though. There are two possible solutions
  1067. * for that:
  1068. *
  1069. * - memcpy() the data internally, at which point we might as well just
  1070. * do a regular read() on the buffer anyway.
  1071. * - Lots of nasty vm tricks, that are neither fast nor flexible (it
  1072. * has restriction limitations on both ends of the pipe).
  1073. *
  1074. * Alas, it isn't here.
  1075. *
  1076. */
  1077. static long do_vmsplice(struct file *file, const struct iovec __user *iov,
  1078. unsigned long nr_segs, unsigned int flags)
  1079. {
  1080. struct pipe_inode_info *pipe;
  1081. struct page *pages[PIPE_BUFFERS];
  1082. struct partial_page partial[PIPE_BUFFERS];
  1083. struct splice_pipe_desc spd = {
  1084. .pages = pages,
  1085. .partial = partial,
  1086. .flags = flags,
  1087. .ops = &user_page_pipe_buf_ops,
  1088. };
  1089. pipe = pipe_info(file->f_path.dentry->d_inode);
  1090. if (!pipe)
  1091. return -EBADF;
  1092. if (unlikely(nr_segs > UIO_MAXIOV))
  1093. return -EINVAL;
  1094. else if (unlikely(!nr_segs))
  1095. return 0;
  1096. spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
  1097. flags & SPLICE_F_GIFT);
  1098. if (spd.nr_pages <= 0)
  1099. return spd.nr_pages;
  1100. return splice_to_pipe(pipe, &spd);
  1101. }
  1102. asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
  1103. unsigned long nr_segs, unsigned int flags)
  1104. {
  1105. struct file *file;
  1106. long error;
  1107. int fput;
  1108. error = -EBADF;
  1109. file = fget_light(fd, &fput);
  1110. if (file) {
  1111. if (file->f_mode & FMODE_WRITE)
  1112. error = do_vmsplice(file, iov, nr_segs, flags);
  1113. fput_light(file, fput);
  1114. }
  1115. return error;
  1116. }
  1117. asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
  1118. int fd_out, loff_t __user *off_out,
  1119. size_t len, unsigned int flags)
  1120. {
  1121. long error;
  1122. struct file *in, *out;
  1123. int fput_in, fput_out;
  1124. if (unlikely(!len))
  1125. return 0;
  1126. error = -EBADF;
  1127. in = fget_light(fd_in, &fput_in);
  1128. if (in) {
  1129. if (in->f_mode & FMODE_READ) {
  1130. out = fget_light(fd_out, &fput_out);
  1131. if (out) {
  1132. if (out->f_mode & FMODE_WRITE)
  1133. error = do_splice(in, off_in,
  1134. out, off_out,
  1135. len, flags);
  1136. fput_light(out, fput_out);
  1137. }
  1138. }
  1139. fput_light(in, fput_in);
  1140. }
  1141. return error;
  1142. }
  1143. /*
  1144. * Make sure there's data to read. Wait for input if we can, otherwise
  1145. * return an appropriate error.
  1146. */
  1147. static int link_ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
  1148. {
  1149. int ret;
  1150. /*
  1151. * Check ->nrbufs without the inode lock first. This function
  1152. * is speculative anyways, so missing one is ok.
  1153. */
  1154. if (pipe->nrbufs)
  1155. return 0;
  1156. ret = 0;
  1157. mutex_lock(&pipe->inode->i_mutex);
  1158. while (!pipe->nrbufs) {
  1159. if (signal_pending(current)) {
  1160. ret = -ERESTARTSYS;
  1161. break;
  1162. }
  1163. if (!pipe->writers)
  1164. break;
  1165. if (!pipe->waiting_writers) {
  1166. if (flags & SPLICE_F_NONBLOCK) {
  1167. ret = -EAGAIN;
  1168. break;
  1169. }
  1170. }
  1171. pipe_wait(pipe);
  1172. }
  1173. mutex_unlock(&pipe->inode->i_mutex);
  1174. return ret;
  1175. }
  1176. /*
  1177. * Make sure there's writeable room. Wait for room if we can, otherwise
  1178. * return an appropriate error.
  1179. */
  1180. static int link_opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
  1181. {
  1182. int ret;
  1183. /*
  1184. * Check ->nrbufs without the inode lock first. This function
  1185. * is speculative anyways, so missing one is ok.
  1186. */
  1187. if (pipe->nrbufs < PIPE_BUFFERS)
  1188. return 0;
  1189. ret = 0;
  1190. mutex_lock(&pipe->inode->i_mutex);
  1191. while (pipe->nrbufs >= PIPE_BUFFERS) {
  1192. if (!pipe->readers) {
  1193. send_sig(SIGPIPE, current, 0);
  1194. ret = -EPIPE;
  1195. break;
  1196. }
  1197. if (flags & SPLICE_F_NONBLOCK) {
  1198. ret = -EAGAIN;
  1199. break;
  1200. }
  1201. if (signal_pending(current)) {
  1202. ret = -ERESTARTSYS;
  1203. break;
  1204. }
  1205. pipe->waiting_writers++;
  1206. pipe_wait(pipe);
  1207. pipe->waiting_writers--;
  1208. }
  1209. mutex_unlock(&pipe->inode->i_mutex);
  1210. return ret;
  1211. }
  1212. /*
  1213. * Link contents of ipipe to opipe.
  1214. */
  1215. static int link_pipe(struct pipe_inode_info *ipipe,
  1216. struct pipe_inode_info *opipe,
  1217. size_t len, unsigned int flags)
  1218. {
  1219. struct pipe_buffer *ibuf, *obuf;
  1220. int ret = 0, i = 0, nbuf;
  1221. /*
  1222. * Potential ABBA deadlock, work around it by ordering lock
  1223. * grabbing by inode address. Otherwise two different processes
  1224. * could deadlock (one doing tee from A -> B, the other from B -> A).
  1225. */
  1226. inode_double_lock(ipipe->inode, opipe->inode);
  1227. do {
  1228. if (!opipe->readers) {
  1229. send_sig(SIGPIPE, current, 0);
  1230. if (!ret)
  1231. ret = -EPIPE;
  1232. break;
  1233. }
  1234. /*
  1235. * If we have iterated all input buffers or ran out of
  1236. * output room, break.
  1237. */
  1238. if (i >= ipipe->nrbufs || opipe->nrbufs >= PIPE_BUFFERS)
  1239. break;
  1240. ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
  1241. nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
  1242. /*
  1243. * Get a reference to this pipe buffer,
  1244. * so we can copy the contents over.
  1245. */
  1246. ibuf->ops->get(ipipe, ibuf);
  1247. obuf = opipe->bufs + nbuf;
  1248. *obuf = *ibuf;
  1249. /*
  1250. * Don't inherit the gift flag, we need to
  1251. * prevent multiple steals of this page.
  1252. */
  1253. obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
  1254. if (obuf->len > len)
  1255. obuf->len = len;
  1256. opipe->nrbufs++;
  1257. ret += obuf->len;
  1258. len -= obuf->len;
  1259. i++;
  1260. } while (len);
  1261. inode_double_unlock(ipipe->inode, opipe->inode);
  1262. /*
  1263. * If we put data in the output pipe, wakeup any potential readers.
  1264. */
  1265. if (ret > 0) {
  1266. smp_mb();
  1267. if (waitqueue_active(&opipe->wait))
  1268. wake_up_interruptible(&opipe->wait);
  1269. kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
  1270. }
  1271. return ret;
  1272. }
  1273. /*
  1274. * This is a tee(1) implementation that works on pipes. It doesn't copy
  1275. * any data, it simply references the 'in' pages on the 'out' pipe.
  1276. * The 'flags' used are the SPLICE_F_* variants, currently the only
  1277. * applicable one is SPLICE_F_NONBLOCK.
  1278. */
  1279. static long do_tee(struct file *in, struct file *out, size_t len,
  1280. unsigned int flags)
  1281. {
  1282. struct pipe_inode_info *ipipe = pipe_info(in->f_path.dentry->d_inode);
  1283. struct pipe_inode_info *opipe = pipe_info(out->f_path.dentry->d_inode);
  1284. int ret = -EINVAL;
  1285. /*
  1286. * Duplicate the contents of ipipe to opipe without actually
  1287. * copying the data.
  1288. */
  1289. if (ipipe && opipe && ipipe != opipe) {
  1290. /*
  1291. * Keep going, unless we encounter an error. The ipipe/opipe
  1292. * ordering doesn't really matter.
  1293. */
  1294. ret = link_ipipe_prep(ipipe, flags);
  1295. if (!ret) {
  1296. ret = link_opipe_prep(opipe, flags);
  1297. if (!ret) {
  1298. ret = link_pipe(ipipe, opipe, len, flags);
  1299. if (!ret && (flags & SPLICE_F_NONBLOCK))
  1300. ret = -EAGAIN;
  1301. }
  1302. }
  1303. }
  1304. return ret;
  1305. }
  1306. asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
  1307. {
  1308. struct file *in;
  1309. int error, fput_in;
  1310. if (unlikely(!len))
  1311. return 0;
  1312. error = -EBADF;
  1313. in = fget_light(fdin, &fput_in);
  1314. if (in) {
  1315. if (in->f_mode & FMODE_READ) {
  1316. int fput_out;
  1317. struct file *out = fget_light(fdout, &fput_out);
  1318. if (out) {
  1319. if (out->f_mode & FMODE_WRITE)
  1320. error = do_tee(in, out, len, flags);
  1321. fput_light(out, fput_out);
  1322. }
  1323. }
  1324. fput_light(in, fput_in);
  1325. }
  1326. return error;
  1327. }