pipe.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/fs/pipe.c
  4. *
  5. * Copyright (C) 1991, 1992, 1999 Linus Torvalds
  6. */
  7. #include <linux/mm.h>
  8. #include <linux/file.h>
  9. #include <linux/poll.h>
  10. #include <linux/slab.h>
  11. #include <linux/module.h>
  12. #include <linux/init.h>
  13. #include <linux/fs.h>
  14. #include <linux/log2.h>
  15. #include <linux/mount.h>
  16. #include <linux/pseudo_fs.h>
  17. #include <linux/magic.h>
  18. #include <linux/pipe_fs_i.h>
  19. #include <linux/uio.h>
  20. #include <linux/highmem.h>
  21. #include <linux/pagemap.h>
  22. #include <linux/audit.h>
  23. #include <linux/syscalls.h>
  24. #include <linux/fcntl.h>
  25. #include <linux/memcontrol.h>
  26. #include <linux/watch_queue.h>
  27. #include <linux/uaccess.h>
  28. #include <asm/ioctls.h>
  29. #include "internal.h"
  30. /*
  31. * New pipe buffers will be restricted to this size while the user is exceeding
  32. * their pipe buffer quota. The general pipe use case needs at least two
  33. * buffers: one for data yet to be read, and one for new data. If this is less
  34. * than two, then a write to a non-empty pipe may block even if the pipe is not
  35. * full. This can occur with GNU make jobserver or similar uses of pipes as
  36. * semaphores: multiple processes may be waiting to write tokens back to the
  37. * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
  38. *
  39. * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
  40. * own risk, namely: pipe writes to non-full pipes may block until the pipe is
  41. * emptied.
  42. */
  43. #define PIPE_MIN_DEF_BUFFERS 2
  44. /*
  45. * The max size that a non-root user is allowed to grow the pipe. Can
  46. * be set by root in /proc/sys/fs/pipe-max-size
  47. */
  48. unsigned int pipe_max_size = 1048576;
  49. /* Maximum allocatable pages per user. Hard limit is unset by default, soft
  50. * matches default values.
  51. */
  52. unsigned long pipe_user_pages_hard;
  53. unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
  54. /*
  55. * We use head and tail indices that aren't masked off, except at the point of
  56. * dereference, but rather they're allowed to wrap naturally. This means there
  57. * isn't a dead spot in the buffer, but the ring has to be a power of two and
  58. * <= 2^31.
  59. * -- David Howells 2019-09-23.
  60. *
  61. * Reads with count = 0 should always return 0.
  62. * -- Julian Bradfield 1999-06-07.
  63. *
  64. * FIFOs and Pipes now generate SIGIO for both readers and writers.
  65. * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
  66. *
  67. * pipe_read & write cleanup
  68. * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
  69. */
  70. static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
  71. {
  72. if (pipe->files)
  73. mutex_lock_nested(&pipe->mutex, subclass);
  74. }
  75. void pipe_lock(struct pipe_inode_info *pipe)
  76. {
  77. /*
  78. * pipe_lock() nests non-pipe inode locks (for writing to a file)
  79. */
  80. pipe_lock_nested(pipe, I_MUTEX_PARENT);
  81. }
  82. EXPORT_SYMBOL(pipe_lock);
  83. void pipe_unlock(struct pipe_inode_info *pipe)
  84. {
  85. if (pipe->files)
  86. mutex_unlock(&pipe->mutex);
  87. }
  88. EXPORT_SYMBOL(pipe_unlock);
  89. static inline void __pipe_lock(struct pipe_inode_info *pipe)
  90. {
  91. mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
  92. }
  93. static inline void __pipe_unlock(struct pipe_inode_info *pipe)
  94. {
  95. mutex_unlock(&pipe->mutex);
  96. }
  97. void pipe_double_lock(struct pipe_inode_info *pipe1,
  98. struct pipe_inode_info *pipe2)
  99. {
  100. BUG_ON(pipe1 == pipe2);
  101. if (pipe1 < pipe2) {
  102. pipe_lock_nested(pipe1, I_MUTEX_PARENT);
  103. pipe_lock_nested(pipe2, I_MUTEX_CHILD);
  104. } else {
  105. pipe_lock_nested(pipe2, I_MUTEX_PARENT);
  106. pipe_lock_nested(pipe1, I_MUTEX_CHILD);
  107. }
  108. }
  109. static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
  110. struct pipe_buffer *buf)
  111. {
  112. struct page *page = buf->page;
  113. /*
  114. * If nobody else uses this page, and we don't already have a
  115. * temporary page, let's keep track of it as a one-deep
  116. * allocation cache. (Otherwise just release our reference to it)
  117. */
  118. if (page_count(page) == 1 && !pipe->tmp_page)
  119. pipe->tmp_page = page;
  120. else
  121. put_page(page);
  122. }
  123. static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
  124. struct pipe_buffer *buf)
  125. {
  126. struct page *page = buf->page;
  127. if (page_count(page) != 1)
  128. return false;
  129. memcg_kmem_uncharge_page(page, 0);
  130. __SetPageLocked(page);
  131. return true;
  132. }
  133. /**
  134. * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
  135. * @pipe: the pipe that the buffer belongs to
  136. * @buf: the buffer to attempt to steal
  137. *
  138. * Description:
  139. * This function attempts to steal the &struct page attached to
  140. * @buf. If successful, this function returns 0 and returns with
  141. * the page locked. The caller may then reuse the page for whatever
  142. * he wishes; the typical use is insertion into a different file
  143. * page cache.
  144. */
  145. bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
  146. struct pipe_buffer *buf)
  147. {
  148. struct page *page = buf->page;
  149. /*
  150. * A reference of one is golden, that means that the owner of this
  151. * page is the only one holding a reference to it. lock the page
  152. * and return OK.
  153. */
  154. if (page_count(page) == 1) {
  155. lock_page(page);
  156. return true;
  157. }
  158. return false;
  159. }
  160. EXPORT_SYMBOL(generic_pipe_buf_try_steal);
  161. /**
  162. * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
  163. * @pipe: the pipe that the buffer belongs to
  164. * @buf: the buffer to get a reference to
  165. *
  166. * Description:
  167. * This function grabs an extra reference to @buf. It's used in
  168. * in the tee() system call, when we duplicate the buffers in one
  169. * pipe into another.
  170. */
  171. bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
  172. {
  173. return try_get_page(buf->page);
  174. }
  175. EXPORT_SYMBOL(generic_pipe_buf_get);
  176. /**
  177. * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
  178. * @pipe: the pipe that the buffer belongs to
  179. * @buf: the buffer to put a reference to
  180. *
  181. * Description:
  182. * This function releases a reference to @buf.
  183. */
  184. void generic_pipe_buf_release(struct pipe_inode_info *pipe,
  185. struct pipe_buffer *buf)
  186. {
  187. put_page(buf->page);
  188. }
  189. EXPORT_SYMBOL(generic_pipe_buf_release);
  190. static const struct pipe_buf_operations anon_pipe_buf_ops = {
  191. .release = anon_pipe_buf_release,
  192. .try_steal = anon_pipe_buf_try_steal,
  193. .get = generic_pipe_buf_get,
  194. };
  195. /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
  196. static inline bool pipe_readable(const struct pipe_inode_info *pipe)
  197. {
  198. unsigned int head = READ_ONCE(pipe->head);
  199. unsigned int tail = READ_ONCE(pipe->tail);
  200. unsigned int writers = READ_ONCE(pipe->writers);
  201. return !pipe_empty(head, tail) || !writers;
  202. }
  203. static ssize_t
  204. pipe_read(struct kiocb *iocb, struct iov_iter *to)
  205. {
  206. size_t total_len = iov_iter_count(to);
  207. struct file *filp = iocb->ki_filp;
  208. struct pipe_inode_info *pipe = filp->private_data;
  209. bool was_full, wake_next_reader = false;
  210. ssize_t ret;
  211. /* Null read succeeds. */
  212. if (unlikely(total_len == 0))
  213. return 0;
  214. ret = 0;
  215. __pipe_lock(pipe);
  216. /*
  217. * We only wake up writers if the pipe was full when we started
  218. * reading in order to avoid unnecessary wakeups.
  219. *
  220. * But when we do wake up writers, we do so using a sync wakeup
  221. * (WF_SYNC), because we want them to get going and generate more
  222. * data for us.
  223. */
  224. was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
  225. for (;;) {
  226. /* Read ->head with a barrier vs post_one_notification() */
  227. unsigned int head = smp_load_acquire(&pipe->head);
  228. unsigned int tail = pipe->tail;
  229. unsigned int mask = pipe->ring_size - 1;
  230. #ifdef CONFIG_WATCH_QUEUE
  231. if (pipe->note_loss) {
  232. struct watch_notification n;
  233. if (total_len < 8) {
  234. if (ret == 0)
  235. ret = -ENOBUFS;
  236. break;
  237. }
  238. n.type = WATCH_TYPE_META;
  239. n.subtype = WATCH_META_LOSS_NOTIFICATION;
  240. n.info = watch_sizeof(n);
  241. if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
  242. if (ret == 0)
  243. ret = -EFAULT;
  244. break;
  245. }
  246. ret += sizeof(n);
  247. total_len -= sizeof(n);
  248. pipe->note_loss = false;
  249. }
  250. #endif
  251. if (!pipe_empty(head, tail)) {
  252. struct pipe_buffer *buf = &pipe->bufs[tail & mask];
  253. size_t chars = buf->len;
  254. size_t written;
  255. int error;
  256. if (chars > total_len) {
  257. if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
  258. if (ret == 0)
  259. ret = -ENOBUFS;
  260. break;
  261. }
  262. chars = total_len;
  263. }
  264. error = pipe_buf_confirm(pipe, buf);
  265. if (error) {
  266. if (!ret)
  267. ret = error;
  268. break;
  269. }
  270. written = copy_page_to_iter(buf->page, buf->offset, chars, to);
  271. if (unlikely(written < chars)) {
  272. if (!ret)
  273. ret = -EFAULT;
  274. break;
  275. }
  276. ret += chars;
  277. buf->offset += chars;
  278. buf->len -= chars;
  279. /* Was it a packet buffer? Clean up and exit */
  280. if (buf->flags & PIPE_BUF_FLAG_PACKET) {
  281. total_len = chars;
  282. buf->len = 0;
  283. }
  284. if (!buf->len) {
  285. pipe_buf_release(pipe, buf);
  286. spin_lock_irq(&pipe->rd_wait.lock);
  287. #ifdef CONFIG_WATCH_QUEUE
  288. if (buf->flags & PIPE_BUF_FLAG_LOSS)
  289. pipe->note_loss = true;
  290. #endif
  291. tail++;
  292. pipe->tail = tail;
  293. spin_unlock_irq(&pipe->rd_wait.lock);
  294. }
  295. total_len -= chars;
  296. if (!total_len)
  297. break; /* common path: read succeeded */
  298. if (!pipe_empty(head, tail)) /* More to do? */
  299. continue;
  300. }
  301. if (!pipe->writers)
  302. break;
  303. if (ret)
  304. break;
  305. if (filp->f_flags & O_NONBLOCK) {
  306. ret = -EAGAIN;
  307. break;
  308. }
  309. __pipe_unlock(pipe);
  310. /*
  311. * We only get here if we didn't actually read anything.
  312. *
  313. * However, we could have seen (and removed) a zero-sized
  314. * pipe buffer, and might have made space in the buffers
  315. * that way.
  316. *
  317. * You can't make zero-sized pipe buffers by doing an empty
  318. * write (not even in packet mode), but they can happen if
  319. * the writer gets an EFAULT when trying to fill a buffer
  320. * that already got allocated and inserted in the buffer
  321. * array.
  322. *
  323. * So we still need to wake up any pending writers in the
  324. * _very_ unlikely case that the pipe was full, but we got
  325. * no data.
  326. */
  327. if (unlikely(was_full)) {
  328. wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
  329. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  330. }
  331. /*
  332. * But because we didn't read anything, at this point we can
  333. * just return directly with -ERESTARTSYS if we're interrupted,
  334. * since we've done any required wakeups and there's no need
  335. * to mark anything accessed. And we've dropped the lock.
  336. */
  337. if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
  338. return -ERESTARTSYS;
  339. __pipe_lock(pipe);
  340. was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
  341. wake_next_reader = true;
  342. }
  343. if (pipe_empty(pipe->head, pipe->tail))
  344. wake_next_reader = false;
  345. __pipe_unlock(pipe);
  346. if (was_full) {
  347. wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
  348. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  349. }
  350. if (wake_next_reader)
  351. wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
  352. if (ret > 0)
  353. file_accessed(filp);
  354. return ret;
  355. }
  356. static inline int is_packetized(struct file *file)
  357. {
  358. return (file->f_flags & O_DIRECT) != 0;
  359. }
  360. /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
  361. static inline bool pipe_writable(const struct pipe_inode_info *pipe)
  362. {
  363. unsigned int head = READ_ONCE(pipe->head);
  364. unsigned int tail = READ_ONCE(pipe->tail);
  365. unsigned int max_usage = READ_ONCE(pipe->max_usage);
  366. return !pipe_full(head, tail, max_usage) ||
  367. !READ_ONCE(pipe->readers);
  368. }
  369. static ssize_t
  370. pipe_write(struct kiocb *iocb, struct iov_iter *from)
  371. {
  372. struct file *filp = iocb->ki_filp;
  373. struct pipe_inode_info *pipe = filp->private_data;
  374. unsigned int head;
  375. ssize_t ret = 0;
  376. size_t total_len = iov_iter_count(from);
  377. ssize_t chars;
  378. bool was_empty = false;
  379. bool wake_next_writer = false;
  380. /* Null write succeeds. */
  381. if (unlikely(total_len == 0))
  382. return 0;
  383. __pipe_lock(pipe);
  384. if (!pipe->readers) {
  385. send_sig(SIGPIPE, current, 0);
  386. ret = -EPIPE;
  387. goto out;
  388. }
  389. #ifdef CONFIG_WATCH_QUEUE
  390. if (pipe->watch_queue) {
  391. ret = -EXDEV;
  392. goto out;
  393. }
  394. #endif
  395. /*
  396. * Epoll nonsensically wants a wakeup whether the pipe
  397. * was already empty or not.
  398. *
  399. * If it wasn't empty we try to merge new data into
  400. * the last buffer.
  401. *
  402. * That naturally merges small writes, but it also
  403. * page-aligns the rest of the writes for large writes
  404. * spanning multiple pages.
  405. */
  406. head = pipe->head;
  407. was_empty = true;
  408. chars = total_len & (PAGE_SIZE-1);
  409. if (chars && !pipe_empty(head, pipe->tail)) {
  410. unsigned int mask = pipe->ring_size - 1;
  411. struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
  412. int offset = buf->offset + buf->len;
  413. if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
  414. offset + chars <= PAGE_SIZE) {
  415. ret = pipe_buf_confirm(pipe, buf);
  416. if (ret)
  417. goto out;
  418. ret = copy_page_from_iter(buf->page, offset, chars, from);
  419. if (unlikely(ret < chars)) {
  420. ret = -EFAULT;
  421. goto out;
  422. }
  423. buf->len += ret;
  424. if (!iov_iter_count(from))
  425. goto out;
  426. }
  427. }
  428. for (;;) {
  429. if (!pipe->readers) {
  430. send_sig(SIGPIPE, current, 0);
  431. if (!ret)
  432. ret = -EPIPE;
  433. break;
  434. }
  435. head = pipe->head;
  436. if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
  437. unsigned int mask = pipe->ring_size - 1;
  438. struct pipe_buffer *buf = &pipe->bufs[head & mask];
  439. struct page *page = pipe->tmp_page;
  440. int copied;
  441. if (!page) {
  442. page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
  443. if (unlikely(!page)) {
  444. ret = ret ? : -ENOMEM;
  445. break;
  446. }
  447. pipe->tmp_page = page;
  448. }
  449. /* Allocate a slot in the ring in advance and attach an
  450. * empty buffer. If we fault or otherwise fail to use
  451. * it, either the reader will consume it or it'll still
  452. * be there for the next write.
  453. */
  454. spin_lock_irq(&pipe->rd_wait.lock);
  455. head = pipe->head;
  456. if (pipe_full(head, pipe->tail, pipe->max_usage)) {
  457. spin_unlock_irq(&pipe->rd_wait.lock);
  458. continue;
  459. }
  460. pipe->head = head + 1;
  461. spin_unlock_irq(&pipe->rd_wait.lock);
  462. /* Insert it into the buffer array */
  463. buf = &pipe->bufs[head & mask];
  464. buf->page = page;
  465. buf->ops = &anon_pipe_buf_ops;
  466. buf->offset = 0;
  467. buf->len = 0;
  468. if (is_packetized(filp))
  469. buf->flags = PIPE_BUF_FLAG_PACKET;
  470. else
  471. buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
  472. pipe->tmp_page = NULL;
  473. copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
  474. if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
  475. if (!ret)
  476. ret = -EFAULT;
  477. break;
  478. }
  479. ret += copied;
  480. buf->offset = 0;
  481. buf->len = copied;
  482. if (!iov_iter_count(from))
  483. break;
  484. }
  485. if (!pipe_full(head, pipe->tail, pipe->max_usage))
  486. continue;
  487. /* Wait for buffer space to become available. */
  488. if (filp->f_flags & O_NONBLOCK) {
  489. if (!ret)
  490. ret = -EAGAIN;
  491. break;
  492. }
  493. if (signal_pending(current)) {
  494. if (!ret)
  495. ret = -ERESTARTSYS;
  496. break;
  497. }
  498. /*
  499. * We're going to release the pipe lock and wait for more
  500. * space. We wake up any readers if necessary, and then
  501. * after waiting we need to re-check whether the pipe
  502. * become empty while we dropped the lock.
  503. */
  504. __pipe_unlock(pipe);
  505. if (was_empty) {
  506. wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
  507. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  508. }
  509. wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
  510. __pipe_lock(pipe);
  511. was_empty = pipe_empty(pipe->head, pipe->tail);
  512. wake_next_writer = true;
  513. }
  514. out:
  515. if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
  516. wake_next_writer = false;
  517. __pipe_unlock(pipe);
  518. /*
  519. * If we do do a wakeup event, we do a 'sync' wakeup, because we
  520. * want the reader to start processing things asap, rather than
  521. * leave the data pending.
  522. *
  523. * This is particularly important for small writes, because of
  524. * how (for example) the GNU make jobserver uses small writes to
  525. * wake up pending jobs
  526. */
  527. if (was_empty) {
  528. wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
  529. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  530. }
  531. if (wake_next_writer)
  532. wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
  533. if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
  534. int err = file_update_time(filp);
  535. if (err)
  536. ret = err;
  537. sb_end_write(file_inode(filp)->i_sb);
  538. }
  539. return ret;
  540. }
  541. static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
  542. {
  543. struct pipe_inode_info *pipe = filp->private_data;
  544. int count, head, tail, mask;
  545. switch (cmd) {
  546. case FIONREAD:
  547. __pipe_lock(pipe);
  548. count = 0;
  549. head = pipe->head;
  550. tail = pipe->tail;
  551. mask = pipe->ring_size - 1;
  552. while (tail != head) {
  553. count += pipe->bufs[tail & mask].len;
  554. tail++;
  555. }
  556. __pipe_unlock(pipe);
  557. return put_user(count, (int __user *)arg);
  558. #ifdef CONFIG_WATCH_QUEUE
  559. case IOC_WATCH_QUEUE_SET_SIZE: {
  560. int ret;
  561. __pipe_lock(pipe);
  562. ret = watch_queue_set_size(pipe, arg);
  563. __pipe_unlock(pipe);
  564. return ret;
  565. }
  566. case IOC_WATCH_QUEUE_SET_FILTER:
  567. return watch_queue_set_filter(
  568. pipe, (struct watch_notification_filter __user *)arg);
  569. #endif
  570. default:
  571. return -ENOIOCTLCMD;
  572. }
  573. }
  574. /* No kernel lock held - fine */
  575. static __poll_t
  576. pipe_poll(struct file *filp, poll_table *wait)
  577. {
  578. __poll_t mask;
  579. struct pipe_inode_info *pipe = filp->private_data;
  580. unsigned int head, tail;
  581. /*
  582. * Reading pipe state only -- no need for acquiring the semaphore.
  583. *
  584. * But because this is racy, the code has to add the
  585. * entry to the poll table _first_ ..
  586. */
  587. if (filp->f_mode & FMODE_READ)
  588. poll_wait(filp, &pipe->rd_wait, wait);
  589. if (filp->f_mode & FMODE_WRITE)
  590. poll_wait(filp, &pipe->wr_wait, wait);
  591. /*
  592. * .. and only then can you do the racy tests. That way,
  593. * if something changes and you got it wrong, the poll
  594. * table entry will wake you up and fix it.
  595. */
  596. head = READ_ONCE(pipe->head);
  597. tail = READ_ONCE(pipe->tail);
  598. mask = 0;
  599. if (filp->f_mode & FMODE_READ) {
  600. if (!pipe_empty(head, tail))
  601. mask |= EPOLLIN | EPOLLRDNORM;
  602. if (!pipe->writers && filp->f_version != pipe->w_counter)
  603. mask |= EPOLLHUP;
  604. }
  605. if (filp->f_mode & FMODE_WRITE) {
  606. if (!pipe_full(head, tail, pipe->max_usage))
  607. mask |= EPOLLOUT | EPOLLWRNORM;
  608. /*
  609. * Most Unices do not set EPOLLERR for FIFOs but on Linux they
  610. * behave exactly like pipes for poll().
  611. */
  612. if (!pipe->readers)
  613. mask |= EPOLLERR;
  614. }
  615. return mask;
  616. }
  617. static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
  618. {
  619. int kill = 0;
  620. spin_lock(&inode->i_lock);
  621. if (!--pipe->files) {
  622. inode->i_pipe = NULL;
  623. kill = 1;
  624. }
  625. spin_unlock(&inode->i_lock);
  626. if (kill)
  627. free_pipe_info(pipe);
  628. }
  629. static int
  630. pipe_release(struct inode *inode, struct file *file)
  631. {
  632. struct pipe_inode_info *pipe = file->private_data;
  633. __pipe_lock(pipe);
  634. if (file->f_mode & FMODE_READ)
  635. pipe->readers--;
  636. if (file->f_mode & FMODE_WRITE)
  637. pipe->writers--;
  638. /* Was that the last reader or writer, but not the other side? */
  639. if (!pipe->readers != !pipe->writers) {
  640. wake_up_interruptible_all(&pipe->rd_wait);
  641. wake_up_interruptible_all(&pipe->wr_wait);
  642. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  643. kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
  644. }
  645. __pipe_unlock(pipe);
  646. put_pipe_info(inode, pipe);
  647. return 0;
  648. }
  649. static int
  650. pipe_fasync(int fd, struct file *filp, int on)
  651. {
  652. struct pipe_inode_info *pipe = filp->private_data;
  653. int retval = 0;
  654. __pipe_lock(pipe);
  655. if (filp->f_mode & FMODE_READ)
  656. retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
  657. if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
  658. retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
  659. if (retval < 0 && (filp->f_mode & FMODE_READ))
  660. /* this can happen only if on == T */
  661. fasync_helper(-1, filp, 0, &pipe->fasync_readers);
  662. }
  663. __pipe_unlock(pipe);
  664. return retval;
  665. }
  666. unsigned long account_pipe_buffers(struct user_struct *user,
  667. unsigned long old, unsigned long new)
  668. {
  669. return atomic_long_add_return(new - old, &user->pipe_bufs);
  670. }
  671. bool too_many_pipe_buffers_soft(unsigned long user_bufs)
  672. {
  673. unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
  674. return soft_limit && user_bufs > soft_limit;
  675. }
  676. bool too_many_pipe_buffers_hard(unsigned long user_bufs)
  677. {
  678. unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
  679. return hard_limit && user_bufs > hard_limit;
  680. }
  681. bool pipe_is_unprivileged_user(void)
  682. {
  683. return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
  684. }
  685. struct pipe_inode_info *alloc_pipe_info(void)
  686. {
  687. struct pipe_inode_info *pipe;
  688. unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
  689. struct user_struct *user = get_current_user();
  690. unsigned long user_bufs;
  691. unsigned int max_size = READ_ONCE(pipe_max_size);
  692. pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
  693. if (pipe == NULL)
  694. goto out_free_uid;
  695. if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
  696. pipe_bufs = max_size >> PAGE_SHIFT;
  697. user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
  698. if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
  699. user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
  700. pipe_bufs = PIPE_MIN_DEF_BUFFERS;
  701. }
  702. if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
  703. goto out_revert_acct;
  704. pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
  705. GFP_KERNEL_ACCOUNT);
  706. if (pipe->bufs) {
  707. init_waitqueue_head(&pipe->rd_wait);
  708. init_waitqueue_head(&pipe->wr_wait);
  709. pipe->r_counter = pipe->w_counter = 1;
  710. pipe->max_usage = pipe_bufs;
  711. pipe->ring_size = pipe_bufs;
  712. pipe->nr_accounted = pipe_bufs;
  713. pipe->user = user;
  714. mutex_init(&pipe->mutex);
  715. return pipe;
  716. }
  717. out_revert_acct:
  718. (void) account_pipe_buffers(user, pipe_bufs, 0);
  719. kfree(pipe);
  720. out_free_uid:
  721. free_uid(user);
  722. return NULL;
  723. }
  724. void free_pipe_info(struct pipe_inode_info *pipe)
  725. {
  726. int i;
  727. #ifdef CONFIG_WATCH_QUEUE
  728. if (pipe->watch_queue)
  729. watch_queue_clear(pipe->watch_queue);
  730. #endif
  731. (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
  732. free_uid(pipe->user);
  733. for (i = 0; i < pipe->ring_size; i++) {
  734. struct pipe_buffer *buf = pipe->bufs + i;
  735. if (buf->ops)
  736. pipe_buf_release(pipe, buf);
  737. }
  738. #ifdef CONFIG_WATCH_QUEUE
  739. if (pipe->watch_queue)
  740. put_watch_queue(pipe->watch_queue);
  741. #endif
  742. if (pipe->tmp_page)
  743. __free_page(pipe->tmp_page);
  744. kfree(pipe->bufs);
  745. kfree(pipe);
  746. }
  747. static struct vfsmount *pipe_mnt __read_mostly;
  748. /*
  749. * pipefs_dname() is called from d_path().
  750. */
  751. static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
  752. {
  753. return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
  754. d_inode(dentry)->i_ino);
  755. }
  756. static const struct dentry_operations pipefs_dentry_operations = {
  757. .d_dname = pipefs_dname,
  758. };
  759. static struct inode * get_pipe_inode(void)
  760. {
  761. struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
  762. struct pipe_inode_info *pipe;
  763. if (!inode)
  764. goto fail_inode;
  765. inode->i_ino = get_next_ino();
  766. pipe = alloc_pipe_info();
  767. if (!pipe)
  768. goto fail_iput;
  769. inode->i_pipe = pipe;
  770. pipe->files = 2;
  771. pipe->readers = pipe->writers = 1;
  772. inode->i_fop = &pipefifo_fops;
  773. /*
  774. * Mark the inode dirty from the very beginning,
  775. * that way it will never be moved to the dirty
  776. * list because "mark_inode_dirty()" will think
  777. * that it already _is_ on the dirty list.
  778. */
  779. inode->i_state = I_DIRTY;
  780. inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
  781. inode->i_uid = current_fsuid();
  782. inode->i_gid = current_fsgid();
  783. inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
  784. return inode;
  785. fail_iput:
  786. iput(inode);
  787. fail_inode:
  788. return NULL;
  789. }
  790. int create_pipe_files(struct file **res, int flags)
  791. {
  792. struct inode *inode = get_pipe_inode();
  793. struct file *f;
  794. int error;
  795. if (!inode)
  796. return -ENFILE;
  797. if (flags & O_NOTIFICATION_PIPE) {
  798. error = watch_queue_init(inode->i_pipe);
  799. if (error) {
  800. free_pipe_info(inode->i_pipe);
  801. iput(inode);
  802. return error;
  803. }
  804. }
  805. f = alloc_file_pseudo(inode, pipe_mnt, "",
  806. O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
  807. &pipefifo_fops);
  808. if (IS_ERR(f)) {
  809. free_pipe_info(inode->i_pipe);
  810. iput(inode);
  811. return PTR_ERR(f);
  812. }
  813. f->private_data = inode->i_pipe;
  814. res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
  815. &pipefifo_fops);
  816. if (IS_ERR(res[0])) {
  817. put_pipe_info(inode, inode->i_pipe);
  818. fput(f);
  819. return PTR_ERR(res[0]);
  820. }
  821. res[0]->private_data = inode->i_pipe;
  822. res[1] = f;
  823. stream_open(inode, res[0]);
  824. stream_open(inode, res[1]);
  825. return 0;
  826. }
  827. static int __do_pipe_flags(int *fd, struct file **files, int flags)
  828. {
  829. int error;
  830. int fdw, fdr;
  831. if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
  832. return -EINVAL;
  833. error = create_pipe_files(files, flags);
  834. if (error)
  835. return error;
  836. error = get_unused_fd_flags(flags);
  837. if (error < 0)
  838. goto err_read_pipe;
  839. fdr = error;
  840. error = get_unused_fd_flags(flags);
  841. if (error < 0)
  842. goto err_fdr;
  843. fdw = error;
  844. audit_fd_pair(fdr, fdw);
  845. fd[0] = fdr;
  846. fd[1] = fdw;
  847. return 0;
  848. err_fdr:
  849. put_unused_fd(fdr);
  850. err_read_pipe:
  851. fput(files[0]);
  852. fput(files[1]);
  853. return error;
  854. }
  855. int do_pipe_flags(int *fd, int flags)
  856. {
  857. struct file *files[2];
  858. int error = __do_pipe_flags(fd, files, flags);
  859. if (!error) {
  860. fd_install(fd[0], files[0]);
  861. fd_install(fd[1], files[1]);
  862. }
  863. return error;
  864. }
  865. /*
  866. * sys_pipe() is the normal C calling standard for creating
  867. * a pipe. It's not the way Unix traditionally does this, though.
  868. */
  869. static int do_pipe2(int __user *fildes, int flags)
  870. {
  871. struct file *files[2];
  872. int fd[2];
  873. int error;
  874. error = __do_pipe_flags(fd, files, flags);
  875. if (!error) {
  876. if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
  877. fput(files[0]);
  878. fput(files[1]);
  879. put_unused_fd(fd[0]);
  880. put_unused_fd(fd[1]);
  881. error = -EFAULT;
  882. } else {
  883. fd_install(fd[0], files[0]);
  884. fd_install(fd[1], files[1]);
  885. }
  886. }
  887. return error;
  888. }
  889. SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
  890. {
  891. return do_pipe2(fildes, flags);
  892. }
  893. SYSCALL_DEFINE1(pipe, int __user *, fildes)
  894. {
  895. return do_pipe2(fildes, 0);
  896. }
  897. /*
  898. * This is the stupid "wait for pipe to be readable or writable"
  899. * model.
  900. *
  901. * See pipe_read/write() for the proper kind of exclusive wait,
  902. * but that requires that we wake up any other readers/writers
  903. * if we then do not end up reading everything (ie the whole
  904. * "wake_next_reader/writer" logic in pipe_read/write()).
  905. */
  906. void pipe_wait_readable(struct pipe_inode_info *pipe)
  907. {
  908. pipe_unlock(pipe);
  909. wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
  910. pipe_lock(pipe);
  911. }
  912. void pipe_wait_writable(struct pipe_inode_info *pipe)
  913. {
  914. pipe_unlock(pipe);
  915. wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
  916. pipe_lock(pipe);
  917. }
  918. /*
  919. * This depends on both the wait (here) and the wakeup (wake_up_partner)
  920. * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
  921. * race with the count check and waitqueue prep.
  922. *
  923. * Normally in order to avoid races, you'd do the prepare_to_wait() first,
  924. * then check the condition you're waiting for, and only then sleep. But
  925. * because of the pipe lock, we can check the condition before being on
  926. * the wait queue.
  927. *
  928. * We use the 'rd_wait' waitqueue for pipe partner waiting.
  929. */
  930. static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
  931. {
  932. DEFINE_WAIT(rdwait);
  933. int cur = *cnt;
  934. while (cur == *cnt) {
  935. prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
  936. pipe_unlock(pipe);
  937. schedule();
  938. finish_wait(&pipe->rd_wait, &rdwait);
  939. pipe_lock(pipe);
  940. if (signal_pending(current))
  941. break;
  942. }
  943. return cur == *cnt ? -ERESTARTSYS : 0;
  944. }
  945. static void wake_up_partner(struct pipe_inode_info *pipe)
  946. {
  947. wake_up_interruptible_all(&pipe->rd_wait);
  948. }
  949. static int fifo_open(struct inode *inode, struct file *filp)
  950. {
  951. struct pipe_inode_info *pipe;
  952. bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
  953. int ret;
  954. filp->f_version = 0;
  955. spin_lock(&inode->i_lock);
  956. if (inode->i_pipe) {
  957. pipe = inode->i_pipe;
  958. pipe->files++;
  959. spin_unlock(&inode->i_lock);
  960. } else {
  961. spin_unlock(&inode->i_lock);
  962. pipe = alloc_pipe_info();
  963. if (!pipe)
  964. return -ENOMEM;
  965. pipe->files = 1;
  966. spin_lock(&inode->i_lock);
  967. if (unlikely(inode->i_pipe)) {
  968. inode->i_pipe->files++;
  969. spin_unlock(&inode->i_lock);
  970. free_pipe_info(pipe);
  971. pipe = inode->i_pipe;
  972. } else {
  973. inode->i_pipe = pipe;
  974. spin_unlock(&inode->i_lock);
  975. }
  976. }
  977. filp->private_data = pipe;
  978. /* OK, we have a pipe and it's pinned down */
  979. __pipe_lock(pipe);
  980. /* We can only do regular read/write on fifos */
  981. stream_open(inode, filp);
  982. switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
  983. case FMODE_READ:
  984. /*
  985. * O_RDONLY
  986. * POSIX.1 says that O_NONBLOCK means return with the FIFO
  987. * opened, even when there is no process writing the FIFO.
  988. */
  989. pipe->r_counter++;
  990. if (pipe->readers++ == 0)
  991. wake_up_partner(pipe);
  992. if (!is_pipe && !pipe->writers) {
  993. if ((filp->f_flags & O_NONBLOCK)) {
  994. /* suppress EPOLLHUP until we have
  995. * seen a writer */
  996. filp->f_version = pipe->w_counter;
  997. } else {
  998. if (wait_for_partner(pipe, &pipe->w_counter))
  999. goto err_rd;
  1000. }
  1001. }
  1002. break;
  1003. case FMODE_WRITE:
  1004. /*
  1005. * O_WRONLY
  1006. * POSIX.1 says that O_NONBLOCK means return -1 with
  1007. * errno=ENXIO when there is no process reading the FIFO.
  1008. */
  1009. ret = -ENXIO;
  1010. if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
  1011. goto err;
  1012. pipe->w_counter++;
  1013. if (!pipe->writers++)
  1014. wake_up_partner(pipe);
  1015. if (!is_pipe && !pipe->readers) {
  1016. if (wait_for_partner(pipe, &pipe->r_counter))
  1017. goto err_wr;
  1018. }
  1019. break;
  1020. case FMODE_READ | FMODE_WRITE:
  1021. /*
  1022. * O_RDWR
  1023. * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
  1024. * This implementation will NEVER block on a O_RDWR open, since
  1025. * the process can at least talk to itself.
  1026. */
  1027. pipe->readers++;
  1028. pipe->writers++;
  1029. pipe->r_counter++;
  1030. pipe->w_counter++;
  1031. if (pipe->readers == 1 || pipe->writers == 1)
  1032. wake_up_partner(pipe);
  1033. break;
  1034. default:
  1035. ret = -EINVAL;
  1036. goto err;
  1037. }
  1038. /* Ok! */
  1039. __pipe_unlock(pipe);
  1040. return 0;
  1041. err_rd:
  1042. if (!--pipe->readers)
  1043. wake_up_interruptible(&pipe->wr_wait);
  1044. ret = -ERESTARTSYS;
  1045. goto err;
  1046. err_wr:
  1047. if (!--pipe->writers)
  1048. wake_up_interruptible_all(&pipe->rd_wait);
  1049. ret = -ERESTARTSYS;
  1050. goto err;
  1051. err:
  1052. __pipe_unlock(pipe);
  1053. put_pipe_info(inode, pipe);
  1054. return ret;
  1055. }
  1056. const struct file_operations pipefifo_fops = {
  1057. .open = fifo_open,
  1058. .llseek = no_llseek,
  1059. .read_iter = pipe_read,
  1060. .write_iter = pipe_write,
  1061. .poll = pipe_poll,
  1062. .unlocked_ioctl = pipe_ioctl,
  1063. .release = pipe_release,
  1064. .fasync = pipe_fasync,
  1065. .splice_write = iter_file_splice_write,
  1066. };
  1067. /*
  1068. * Currently we rely on the pipe array holding a power-of-2 number
  1069. * of pages. Returns 0 on error.
  1070. */
  1071. unsigned int round_pipe_size(unsigned long size)
  1072. {
  1073. if (size > (1U << 31))
  1074. return 0;
  1075. /* Minimum pipe size, as required by POSIX */
  1076. if (size < PAGE_SIZE)
  1077. return PAGE_SIZE;
  1078. return roundup_pow_of_two(size);
  1079. }
  1080. /*
  1081. * Resize the pipe ring to a number of slots.
  1082. */
  1083. int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
  1084. {
  1085. struct pipe_buffer *bufs;
  1086. unsigned int head, tail, mask, n;
  1087. /*
  1088. * We can shrink the pipe, if arg is greater than the ring occupancy.
  1089. * Since we don't expect a lot of shrink+grow operations, just free and
  1090. * allocate again like we would do for growing. If the pipe currently
  1091. * contains more buffers than arg, then return busy.
  1092. */
  1093. mask = pipe->ring_size - 1;
  1094. head = pipe->head;
  1095. tail = pipe->tail;
  1096. n = pipe_occupancy(pipe->head, pipe->tail);
  1097. if (nr_slots < n)
  1098. return -EBUSY;
  1099. bufs = kcalloc(nr_slots, sizeof(*bufs),
  1100. GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
  1101. if (unlikely(!bufs))
  1102. return -ENOMEM;
  1103. /*
  1104. * The pipe array wraps around, so just start the new one at zero
  1105. * and adjust the indices.
  1106. */
  1107. if (n > 0) {
  1108. unsigned int h = head & mask;
  1109. unsigned int t = tail & mask;
  1110. if (h > t) {
  1111. memcpy(bufs, pipe->bufs + t,
  1112. n * sizeof(struct pipe_buffer));
  1113. } else {
  1114. unsigned int tsize = pipe->ring_size - t;
  1115. if (h > 0)
  1116. memcpy(bufs + tsize, pipe->bufs,
  1117. h * sizeof(struct pipe_buffer));
  1118. memcpy(bufs, pipe->bufs + t,
  1119. tsize * sizeof(struct pipe_buffer));
  1120. }
  1121. }
  1122. head = n;
  1123. tail = 0;
  1124. kfree(pipe->bufs);
  1125. pipe->bufs = bufs;
  1126. pipe->ring_size = nr_slots;
  1127. if (pipe->max_usage > nr_slots)
  1128. pipe->max_usage = nr_slots;
  1129. pipe->tail = tail;
  1130. pipe->head = head;
  1131. /* This might have made more room for writers */
  1132. wake_up_interruptible(&pipe->wr_wait);
  1133. return 0;
  1134. }
  1135. /*
  1136. * Allocate a new array of pipe buffers and copy the info over. Returns the
  1137. * pipe size if successful, or return -ERROR on error.
  1138. */
  1139. static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
  1140. {
  1141. unsigned long user_bufs;
  1142. unsigned int nr_slots, size;
  1143. long ret = 0;
  1144. #ifdef CONFIG_WATCH_QUEUE
  1145. if (pipe->watch_queue)
  1146. return -EBUSY;
  1147. #endif
  1148. size = round_pipe_size(arg);
  1149. nr_slots = size >> PAGE_SHIFT;
  1150. if (!nr_slots)
  1151. return -EINVAL;
  1152. /*
  1153. * If trying to increase the pipe capacity, check that an
  1154. * unprivileged user is not trying to exceed various limits
  1155. * (soft limit check here, hard limit check just below).
  1156. * Decreasing the pipe capacity is always permitted, even
  1157. * if the user is currently over a limit.
  1158. */
  1159. if (nr_slots > pipe->max_usage &&
  1160. size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
  1161. return -EPERM;
  1162. user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
  1163. if (nr_slots > pipe->max_usage &&
  1164. (too_many_pipe_buffers_hard(user_bufs) ||
  1165. too_many_pipe_buffers_soft(user_bufs)) &&
  1166. pipe_is_unprivileged_user()) {
  1167. ret = -EPERM;
  1168. goto out_revert_acct;
  1169. }
  1170. ret = pipe_resize_ring(pipe, nr_slots);
  1171. if (ret < 0)
  1172. goto out_revert_acct;
  1173. pipe->max_usage = nr_slots;
  1174. pipe->nr_accounted = nr_slots;
  1175. return pipe->max_usage * PAGE_SIZE;
  1176. out_revert_acct:
  1177. (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
  1178. return ret;
  1179. }
  1180. /*
  1181. * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
  1182. * location, so checking ->i_pipe is not enough to verify that this is a
  1183. * pipe.
  1184. */
  1185. struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
  1186. {
  1187. struct pipe_inode_info *pipe = file->private_data;
  1188. if (file->f_op != &pipefifo_fops || !pipe)
  1189. return NULL;
  1190. #ifdef CONFIG_WATCH_QUEUE
  1191. if (for_splice && pipe->watch_queue)
  1192. return NULL;
  1193. #endif
  1194. return pipe;
  1195. }
  1196. long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
  1197. {
  1198. struct pipe_inode_info *pipe;
  1199. long ret;
  1200. pipe = get_pipe_info(file, false);
  1201. if (!pipe)
  1202. return -EBADF;
  1203. __pipe_lock(pipe);
  1204. switch (cmd) {
  1205. case F_SETPIPE_SZ:
  1206. ret = pipe_set_size(pipe, arg);
  1207. break;
  1208. case F_GETPIPE_SZ:
  1209. ret = pipe->max_usage * PAGE_SIZE;
  1210. break;
  1211. default:
  1212. ret = -EINVAL;
  1213. break;
  1214. }
  1215. __pipe_unlock(pipe);
  1216. return ret;
  1217. }
  1218. static const struct super_operations pipefs_ops = {
  1219. .destroy_inode = free_inode_nonrcu,
  1220. .statfs = simple_statfs,
  1221. };
  1222. /*
  1223. * pipefs should _never_ be mounted by userland - too much of security hassle,
  1224. * no real gain from having the whole whorehouse mounted. So we don't need
  1225. * any operations on the root directory. However, we need a non-trivial
  1226. * d_name - pipe: will go nicely and kill the special-casing in procfs.
  1227. */
  1228. static int pipefs_init_fs_context(struct fs_context *fc)
  1229. {
  1230. struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
  1231. if (!ctx)
  1232. return -ENOMEM;
  1233. ctx->ops = &pipefs_ops;
  1234. ctx->dops = &pipefs_dentry_operations;
  1235. return 0;
  1236. }
  1237. static struct file_system_type pipe_fs_type = {
  1238. .name = "pipefs",
  1239. .init_fs_context = pipefs_init_fs_context,
  1240. .kill_sb = kill_anon_super,
  1241. };
  1242. static int __init init_pipe_fs(void)
  1243. {
  1244. int err = register_filesystem(&pipe_fs_type);
  1245. if (!err) {
  1246. pipe_mnt = kern_mount(&pipe_fs_type);
  1247. if (IS_ERR(pipe_mnt)) {
  1248. err = PTR_ERR(pipe_mnt);
  1249. unregister_filesystem(&pipe_fs_type);
  1250. }
  1251. }
  1252. return err;
  1253. }
  1254. fs_initcall(init_pipe_fs);