eventfd.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * fs/eventfd.c
  4. *
  5. * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
  6. *
  7. */
  8. #include <linux/file.h>
  9. #include <linux/poll.h>
  10. #include <linux/init.h>
  11. #include <linux/fs.h>
  12. #include <linux/sched/signal.h>
  13. #include <linux/kernel.h>
  14. #include <linux/slab.h>
  15. #include <linux/list.h>
  16. #include <linux/spinlock.h>
  17. #include <linux/anon_inodes.h>
  18. #include <linux/syscalls.h>
  19. #include <linux/export.h>
  20. #include <linux/kref.h>
  21. #include <linux/eventfd.h>
  22. #include <linux/proc_fs.h>
  23. #include <linux/seq_file.h>
  24. #include <linux/idr.h>
  25. #include <linux/uio.h>
  26. DEFINE_PER_CPU(int, eventfd_wake_count);
  27. static DEFINE_IDA(eventfd_ida);
  28. struct eventfd_ctx {
  29. struct kref kref;
  30. wait_queue_head_t wqh;
  31. /*
  32. * Every time that a write(2) is performed on an eventfd, the
  33. * value of the __u64 being written is added to "count" and a
  34. * wakeup is performed on "wqh". A read(2) will return the "count"
  35. * value to userspace, and will reset "count" to zero. The kernel
  36. * side eventfd_signal() also, adds to the "count" counter and
  37. * issue a wakeup.
  38. */
  39. __u64 count;
  40. unsigned int flags;
  41. int id;
  42. };
  43. /**
  44. * eventfd_signal - Adds @n to the eventfd counter.
  45. * @ctx: [in] Pointer to the eventfd context.
  46. * @n: [in] Value of the counter to be added to the eventfd internal counter.
  47. * The value cannot be negative.
  48. *
  49. * This function is supposed to be called by the kernel in paths that do not
  50. * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX
  51. * value, and we signal this as overflow condition by returning a EPOLLERR
  52. * to poll(2).
  53. *
  54. * Returns the amount by which the counter was incremented. This will be less
  55. * than @n if the counter has overflowed.
  56. */
  57. __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n)
  58. {
  59. unsigned long flags;
  60. /*
  61. * Deadlock or stack overflow issues can happen if we recurse here
  62. * through waitqueue wakeup handlers. If the caller users potentially
  63. * nested waitqueues with custom wakeup handlers, then it should
  64. * check eventfd_signal_count() before calling this function. If
  65. * it returns true, the eventfd_signal() call should be deferred to a
  66. * safe context.
  67. */
  68. if (WARN_ON_ONCE(this_cpu_read(eventfd_wake_count)))
  69. return 0;
  70. spin_lock_irqsave(&ctx->wqh.lock, flags);
  71. this_cpu_inc(eventfd_wake_count);
  72. if (ULLONG_MAX - ctx->count < n)
  73. n = ULLONG_MAX - ctx->count;
  74. ctx->count += n;
  75. if (waitqueue_active(&ctx->wqh))
  76. wake_up_locked_poll(&ctx->wqh, EPOLLIN);
  77. this_cpu_dec(eventfd_wake_count);
  78. spin_unlock_irqrestore(&ctx->wqh.lock, flags);
  79. return n;
  80. }
  81. EXPORT_SYMBOL_GPL(eventfd_signal);
  82. static void eventfd_free_ctx(struct eventfd_ctx *ctx)
  83. {
  84. if (ctx->id >= 0)
  85. ida_simple_remove(&eventfd_ida, ctx->id);
  86. kfree(ctx);
  87. }
  88. static void eventfd_free(struct kref *kref)
  89. {
  90. struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref);
  91. eventfd_free_ctx(ctx);
  92. }
  93. /**
  94. * eventfd_ctx_put - Releases a reference to the internal eventfd context.
  95. * @ctx: [in] Pointer to eventfd context.
  96. *
  97. * The eventfd context reference must have been previously acquired either
  98. * with eventfd_ctx_fdget() or eventfd_ctx_fileget().
  99. */
  100. void eventfd_ctx_put(struct eventfd_ctx *ctx)
  101. {
  102. kref_put(&ctx->kref, eventfd_free);
  103. }
  104. EXPORT_SYMBOL_GPL(eventfd_ctx_put);
  105. static int eventfd_release(struct inode *inode, struct file *file)
  106. {
  107. struct eventfd_ctx *ctx = file->private_data;
  108. wake_up_poll(&ctx->wqh, EPOLLHUP);
  109. eventfd_ctx_put(ctx);
  110. return 0;
  111. }
  112. static __poll_t eventfd_poll(struct file *file, poll_table *wait)
  113. {
  114. struct eventfd_ctx *ctx = file->private_data;
  115. __poll_t events = 0;
  116. u64 count;
  117. poll_wait(file, &ctx->wqh, wait);
  118. /*
  119. * All writes to ctx->count occur within ctx->wqh.lock. This read
  120. * can be done outside ctx->wqh.lock because we know that poll_wait
  121. * takes that lock (through add_wait_queue) if our caller will sleep.
  122. *
  123. * The read _can_ therefore seep into add_wait_queue's critical
  124. * section, but cannot move above it! add_wait_queue's spin_lock acts
  125. * as an acquire barrier and ensures that the read be ordered properly
  126. * against the writes. The following CAN happen and is safe:
  127. *
  128. * poll write
  129. * ----------------- ------------
  130. * lock ctx->wqh.lock (in poll_wait)
  131. * count = ctx->count
  132. * __add_wait_queue
  133. * unlock ctx->wqh.lock
  134. * lock ctx->qwh.lock
  135. * ctx->count += n
  136. * if (waitqueue_active)
  137. * wake_up_locked_poll
  138. * unlock ctx->qwh.lock
  139. * eventfd_poll returns 0
  140. *
  141. * but the following, which would miss a wakeup, cannot happen:
  142. *
  143. * poll write
  144. * ----------------- ------------
  145. * count = ctx->count (INVALID!)
  146. * lock ctx->qwh.lock
  147. * ctx->count += n
  148. * **waitqueue_active is false**
  149. * **no wake_up_locked_poll!**
  150. * unlock ctx->qwh.lock
  151. * lock ctx->wqh.lock (in poll_wait)
  152. * __add_wait_queue
  153. * unlock ctx->wqh.lock
  154. * eventfd_poll returns 0
  155. */
  156. count = READ_ONCE(ctx->count);
  157. if (count > 0)
  158. events |= EPOLLIN;
  159. if (count == ULLONG_MAX)
  160. events |= EPOLLERR;
  161. if (ULLONG_MAX - 1 > count)
  162. events |= EPOLLOUT;
  163. return events;
  164. }
  165. static void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
  166. {
  167. *cnt = (ctx->flags & EFD_SEMAPHORE) ? 1 : ctx->count;
  168. ctx->count -= *cnt;
  169. }
  170. /**
  171. * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue.
  172. * @ctx: [in] Pointer to eventfd context.
  173. * @wait: [in] Wait queue to be removed.
  174. * @cnt: [out] Pointer to the 64-bit counter value.
  175. *
  176. * Returns %0 if successful, or the following error codes:
  177. *
  178. * -EAGAIN : The operation would have blocked.
  179. *
  180. * This is used to atomically remove a wait queue entry from the eventfd wait
  181. * queue head, and read/reset the counter value.
  182. */
  183. int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait,
  184. __u64 *cnt)
  185. {
  186. unsigned long flags;
  187. spin_lock_irqsave(&ctx->wqh.lock, flags);
  188. eventfd_ctx_do_read(ctx, cnt);
  189. __remove_wait_queue(&ctx->wqh, wait);
  190. if (*cnt != 0 && waitqueue_active(&ctx->wqh))
  191. wake_up_locked_poll(&ctx->wqh, EPOLLOUT);
  192. spin_unlock_irqrestore(&ctx->wqh.lock, flags);
  193. return *cnt != 0 ? 0 : -EAGAIN;
  194. }
  195. EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue);
  196. static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to)
  197. {
  198. struct file *file = iocb->ki_filp;
  199. struct eventfd_ctx *ctx = file->private_data;
  200. __u64 ucnt = 0;
  201. DECLARE_WAITQUEUE(wait, current);
  202. if (iov_iter_count(to) < sizeof(ucnt))
  203. return -EINVAL;
  204. spin_lock_irq(&ctx->wqh.lock);
  205. if (!ctx->count) {
  206. if ((file->f_flags & O_NONBLOCK) ||
  207. (iocb->ki_flags & IOCB_NOWAIT)) {
  208. spin_unlock_irq(&ctx->wqh.lock);
  209. return -EAGAIN;
  210. }
  211. __add_wait_queue(&ctx->wqh, &wait);
  212. for (;;) {
  213. set_current_state(TASK_INTERRUPTIBLE);
  214. if (ctx->count)
  215. break;
  216. if (signal_pending(current)) {
  217. __remove_wait_queue(&ctx->wqh, &wait);
  218. __set_current_state(TASK_RUNNING);
  219. spin_unlock_irq(&ctx->wqh.lock);
  220. return -ERESTARTSYS;
  221. }
  222. spin_unlock_irq(&ctx->wqh.lock);
  223. schedule();
  224. spin_lock_irq(&ctx->wqh.lock);
  225. }
  226. __remove_wait_queue(&ctx->wqh, &wait);
  227. __set_current_state(TASK_RUNNING);
  228. }
  229. eventfd_ctx_do_read(ctx, &ucnt);
  230. if (waitqueue_active(&ctx->wqh))
  231. wake_up_locked_poll(&ctx->wqh, EPOLLOUT);
  232. spin_unlock_irq(&ctx->wqh.lock);
  233. if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt)))
  234. return -EFAULT;
  235. return sizeof(ucnt);
  236. }
  237. static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
  238. loff_t *ppos)
  239. {
  240. struct eventfd_ctx *ctx = file->private_data;
  241. ssize_t res;
  242. __u64 ucnt;
  243. DECLARE_WAITQUEUE(wait, current);
  244. if (count < sizeof(ucnt))
  245. return -EINVAL;
  246. if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
  247. return -EFAULT;
  248. if (ucnt == ULLONG_MAX)
  249. return -EINVAL;
  250. spin_lock_irq(&ctx->wqh.lock);
  251. res = -EAGAIN;
  252. if (ULLONG_MAX - ctx->count > ucnt)
  253. res = sizeof(ucnt);
  254. else if (!(file->f_flags & O_NONBLOCK)) {
  255. __add_wait_queue(&ctx->wqh, &wait);
  256. for (res = 0;;) {
  257. set_current_state(TASK_INTERRUPTIBLE);
  258. if (ULLONG_MAX - ctx->count > ucnt) {
  259. res = sizeof(ucnt);
  260. break;
  261. }
  262. if (signal_pending(current)) {
  263. res = -ERESTARTSYS;
  264. break;
  265. }
  266. spin_unlock_irq(&ctx->wqh.lock);
  267. schedule();
  268. spin_lock_irq(&ctx->wqh.lock);
  269. }
  270. __remove_wait_queue(&ctx->wqh, &wait);
  271. __set_current_state(TASK_RUNNING);
  272. }
  273. if (likely(res > 0)) {
  274. ctx->count += ucnt;
  275. if (waitqueue_active(&ctx->wqh))
  276. wake_up_locked_poll(&ctx->wqh, EPOLLIN);
  277. }
  278. spin_unlock_irq(&ctx->wqh.lock);
  279. return res;
  280. }
  281. #ifdef CONFIG_PROC_FS
  282. static void eventfd_show_fdinfo(struct seq_file *m, struct file *f)
  283. {
  284. struct eventfd_ctx *ctx = f->private_data;
  285. spin_lock_irq(&ctx->wqh.lock);
  286. seq_printf(m, "eventfd-count: %16llx\n",
  287. (unsigned long long)ctx->count);
  288. spin_unlock_irq(&ctx->wqh.lock);
  289. seq_printf(m, "eventfd-id: %d\n", ctx->id);
  290. }
  291. #endif
  292. static const struct file_operations eventfd_fops = {
  293. #ifdef CONFIG_PROC_FS
  294. .show_fdinfo = eventfd_show_fdinfo,
  295. #endif
  296. .release = eventfd_release,
  297. .poll = eventfd_poll,
  298. .read_iter = eventfd_read,
  299. .write = eventfd_write,
  300. .llseek = noop_llseek,
  301. };
  302. /**
  303. * eventfd_fget - Acquire a reference of an eventfd file descriptor.
  304. * @fd: [in] Eventfd file descriptor.
  305. *
  306. * Returns a pointer to the eventfd file structure in case of success, or the
  307. * following error pointer:
  308. *
  309. * -EBADF : Invalid @fd file descriptor.
  310. * -EINVAL : The @fd file descriptor is not an eventfd file.
  311. */
  312. struct file *eventfd_fget(int fd)
  313. {
  314. struct file *file;
  315. file = fget(fd);
  316. if (!file)
  317. return ERR_PTR(-EBADF);
  318. if (file->f_op != &eventfd_fops) {
  319. fput(file);
  320. return ERR_PTR(-EINVAL);
  321. }
  322. return file;
  323. }
  324. EXPORT_SYMBOL_GPL(eventfd_fget);
  325. /**
  326. * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context.
  327. * @fd: [in] Eventfd file descriptor.
  328. *
  329. * Returns a pointer to the internal eventfd context, otherwise the error
  330. * pointers returned by the following functions:
  331. *
  332. * eventfd_fget
  333. */
  334. struct eventfd_ctx *eventfd_ctx_fdget(int fd)
  335. {
  336. struct eventfd_ctx *ctx;
  337. struct fd f = fdget(fd);
  338. if (!f.file)
  339. return ERR_PTR(-EBADF);
  340. ctx = eventfd_ctx_fileget(f.file);
  341. fdput(f);
  342. return ctx;
  343. }
  344. EXPORT_SYMBOL_GPL(eventfd_ctx_fdget);
  345. /**
  346. * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context.
  347. * @file: [in] Eventfd file pointer.
  348. *
  349. * Returns a pointer to the internal eventfd context, otherwise the error
  350. * pointer:
  351. *
  352. * -EINVAL : The @fd file descriptor is not an eventfd file.
  353. */
  354. struct eventfd_ctx *eventfd_ctx_fileget(struct file *file)
  355. {
  356. struct eventfd_ctx *ctx;
  357. if (file->f_op != &eventfd_fops)
  358. return ERR_PTR(-EINVAL);
  359. ctx = file->private_data;
  360. kref_get(&ctx->kref);
  361. return ctx;
  362. }
  363. EXPORT_SYMBOL_GPL(eventfd_ctx_fileget);
  364. static int do_eventfd(unsigned int count, int flags)
  365. {
  366. struct eventfd_ctx *ctx;
  367. struct file *file;
  368. int fd;
  369. /* Check the EFD_* constants for consistency. */
  370. BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC);
  371. BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK);
  372. if (flags & ~EFD_FLAGS_SET)
  373. return -EINVAL;
  374. ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
  375. if (!ctx)
  376. return -ENOMEM;
  377. kref_init(&ctx->kref);
  378. init_waitqueue_head(&ctx->wqh);
  379. ctx->count = count;
  380. ctx->flags = flags;
  381. ctx->id = ida_simple_get(&eventfd_ida, 0, 0, GFP_KERNEL);
  382. flags &= EFD_SHARED_FCNTL_FLAGS;
  383. flags |= O_RDWR;
  384. fd = get_unused_fd_flags(flags);
  385. if (fd < 0)
  386. goto err;
  387. file = anon_inode_getfile("[eventfd]", &eventfd_fops, ctx, flags);
  388. if (IS_ERR(file)) {
  389. put_unused_fd(fd);
  390. fd = PTR_ERR(file);
  391. goto err;
  392. }
  393. file->f_mode |= FMODE_NOWAIT;
  394. fd_install(fd, file);
  395. return fd;
  396. err:
  397. eventfd_free_ctx(ctx);
  398. return fd;
  399. }
  400. SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags)
  401. {
  402. return do_eventfd(count, flags);
  403. }
  404. SYSCALL_DEFINE1(eventfd, unsigned int, count)
  405. {
  406. return do_eventfd(count, 0);
  407. }