eventpoll.c 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714
  1. /*
  2. * fs/eventpoll.c ( Efficent event polling implementation )
  3. * Copyright (C) 2001,...,2006 Davide Libenzi
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * Davide Libenzi <davidel@xmailserver.org>
  11. *
  12. */
  13. #include <linux/module.h>
  14. #include <linux/init.h>
  15. #include <linux/kernel.h>
  16. #include <linux/sched.h>
  17. #include <linux/fs.h>
  18. #include <linux/file.h>
  19. #include <linux/signal.h>
  20. #include <linux/errno.h>
  21. #include <linux/mm.h>
  22. #include <linux/slab.h>
  23. #include <linux/poll.h>
  24. #include <linux/smp_lock.h>
  25. #include <linux/string.h>
  26. #include <linux/list.h>
  27. #include <linux/hash.h>
  28. #include <linux/spinlock.h>
  29. #include <linux/syscalls.h>
  30. #include <linux/rwsem.h>
  31. #include <linux/rbtree.h>
  32. #include <linux/wait.h>
  33. #include <linux/eventpoll.h>
  34. #include <linux/mount.h>
  35. #include <linux/bitops.h>
  36. #include <linux/mutex.h>
  37. #include <asm/uaccess.h>
  38. #include <asm/system.h>
  39. #include <asm/io.h>
  40. #include <asm/mman.h>
  41. #include <asm/atomic.h>
  42. #include <asm/semaphore.h>
  43. /*
  44. * LOCKING:
  45. * There are three level of locking required by epoll :
  46. *
  47. * 1) epmutex (mutex)
  48. * 2) ep->sem (rw_semaphore)
  49. * 3) ep->lock (rw_lock)
  50. *
  51. * The acquire order is the one listed above, from 1 to 3.
  52. * We need a spinlock (ep->lock) because we manipulate objects
  53. * from inside the poll callback, that might be triggered from
  54. * a wake_up() that in turn might be called from IRQ context.
  55. * So we can't sleep inside the poll callback and hence we need
  56. * a spinlock. During the event transfer loop (from kernel to
  57. * user space) we could end up sleeping due a copy_to_user(), so
  58. * we need a lock that will allow us to sleep. This lock is a
  59. * read-write semaphore (ep->sem). It is acquired on read during
  60. * the event transfer loop and in write during epoll_ctl(EPOLL_CTL_DEL)
  61. * and during eventpoll_release_file(). Then we also need a global
  62. * semaphore to serialize eventpoll_release_file() and ep_free().
  63. * This semaphore is acquired by ep_free() during the epoll file
  64. * cleanup path and it is also acquired by eventpoll_release_file()
  65. * if a file has been pushed inside an epoll set and it is then
  66. * close()d without a previous call toepoll_ctl(EPOLL_CTL_DEL).
  67. * It is possible to drop the "ep->sem" and to use the global
  68. * semaphore "epmutex" (together with "ep->lock") to have it working,
  69. * but having "ep->sem" will make the interface more scalable.
  70. * Events that require holding "epmutex" are very rare, while for
  71. * normal operations the epoll private "ep->sem" will guarantee
  72. * a greater scalability.
  73. */
  74. #define EVENTPOLLFS_MAGIC 0x03111965 /* My birthday should work for this :) */
  75. #define DEBUG_EPOLL 0
  76. #if DEBUG_EPOLL > 0
  77. #define DPRINTK(x) printk x
  78. #define DNPRINTK(n, x) do { if ((n) <= DEBUG_EPOLL) printk x; } while (0)
  79. #else /* #if DEBUG_EPOLL > 0 */
  80. #define DPRINTK(x) (void) 0
  81. #define DNPRINTK(n, x) (void) 0
  82. #endif /* #if DEBUG_EPOLL > 0 */
  83. #define DEBUG_EPI 0
  84. #if DEBUG_EPI != 0
  85. #define EPI_SLAB_DEBUG (SLAB_DEBUG_FREE | SLAB_RED_ZONE /* | SLAB_POISON */)
  86. #else /* #if DEBUG_EPI != 0 */
  87. #define EPI_SLAB_DEBUG 0
  88. #endif /* #if DEBUG_EPI != 0 */
  89. /* Epoll private bits inside the event mask */
  90. #define EP_PRIVATE_BITS (EPOLLONESHOT | EPOLLET)
  91. /* Maximum number of poll wake up nests we are allowing */
  92. #define EP_MAX_POLLWAKE_NESTS 4
  93. /* Maximum msec timeout value storeable in a long int */
  94. #define EP_MAX_MSTIMEO min(1000ULL * MAX_SCHEDULE_TIMEOUT / HZ, (LONG_MAX - 999ULL) / HZ)
  95. #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
  96. struct epoll_filefd {
  97. struct file *file;
  98. int fd;
  99. };
  100. /*
  101. * Node that is linked into the "wake_task_list" member of the "struct poll_safewake".
  102. * It is used to keep track on all tasks that are currently inside the wake_up() code
  103. * to 1) short-circuit the one coming from the same task and same wait queue head
  104. * ( loop ) 2) allow a maximum number of epoll descriptors inclusion nesting
  105. * 3) let go the ones coming from other tasks.
  106. */
  107. struct wake_task_node {
  108. struct list_head llink;
  109. struct task_struct *task;
  110. wait_queue_head_t *wq;
  111. };
  112. /*
  113. * This is used to implement the safe poll wake up avoiding to reenter
  114. * the poll callback from inside wake_up().
  115. */
  116. struct poll_safewake {
  117. struct list_head wake_task_list;
  118. spinlock_t lock;
  119. };
  120. /*
  121. * This structure is stored inside the "private_data" member of the file
  122. * structure and rapresent the main data sructure for the eventpoll
  123. * interface.
  124. */
  125. struct eventpoll {
  126. /* Protect the this structure access */
  127. rwlock_t lock;
  128. /*
  129. * This semaphore is used to ensure that files are not removed
  130. * while epoll is using them. This is read-held during the event
  131. * collection loop and it is write-held during the file cleanup
  132. * path, the epoll file exit code and the ctl operations.
  133. */
  134. struct rw_semaphore sem;
  135. /* Wait queue used by sys_epoll_wait() */
  136. wait_queue_head_t wq;
  137. /* Wait queue used by file->poll() */
  138. wait_queue_head_t poll_wait;
  139. /* List of ready file descriptors */
  140. struct list_head rdllist;
  141. /* RB-Tree root used to store monitored fd structs */
  142. struct rb_root rbr;
  143. };
  144. /* Wait structure used by the poll hooks */
  145. struct eppoll_entry {
  146. /* List header used to link this structure to the "struct epitem" */
  147. struct list_head llink;
  148. /* The "base" pointer is set to the container "struct epitem" */
  149. void *base;
  150. /*
  151. * Wait queue item that will be linked to the target file wait
  152. * queue head.
  153. */
  154. wait_queue_t wait;
  155. /* The wait queue head that linked the "wait" wait queue item */
  156. wait_queue_head_t *whead;
  157. };
  158. /*
  159. * Each file descriptor added to the eventpoll interface will
  160. * have an entry of this type linked to the hash.
  161. */
  162. struct epitem {
  163. /* RB-Tree node used to link this structure to the eventpoll rb-tree */
  164. struct rb_node rbn;
  165. /* List header used to link this structure to the eventpoll ready list */
  166. struct list_head rdllink;
  167. /* The file descriptor information this item refers to */
  168. struct epoll_filefd ffd;
  169. /* Number of active wait queue attached to poll operations */
  170. int nwait;
  171. /* List containing poll wait queues */
  172. struct list_head pwqlist;
  173. /* The "container" of this item */
  174. struct eventpoll *ep;
  175. /* The structure that describe the interested events and the source fd */
  176. struct epoll_event event;
  177. /*
  178. * Used to keep track of the usage count of the structure. This avoids
  179. * that the structure will desappear from underneath our processing.
  180. */
  181. atomic_t usecnt;
  182. /* List header used to link this item to the "struct file" items list */
  183. struct list_head fllink;
  184. /* List header used to link the item to the transfer list */
  185. struct list_head txlink;
  186. /*
  187. * This is used during the collection/transfer of events to userspace
  188. * to pin items empty events set.
  189. */
  190. unsigned int revents;
  191. };
  192. /* Wrapper struct used by poll queueing */
  193. struct ep_pqueue {
  194. poll_table pt;
  195. struct epitem *epi;
  196. };
  197. static void ep_poll_safewake_init(struct poll_safewake *psw);
  198. static void ep_poll_safewake(struct poll_safewake *psw, wait_queue_head_t *wq);
  199. static int ep_getfd(int *efd, struct inode **einode, struct file **efile,
  200. struct eventpoll *ep);
  201. static int ep_alloc(struct eventpoll **pep);
  202. static void ep_free(struct eventpoll *ep);
  203. static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd);
  204. static void ep_use_epitem(struct epitem *epi);
  205. static void ep_release_epitem(struct epitem *epi);
  206. static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
  207. poll_table *pt);
  208. static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi);
  209. static int ep_insert(struct eventpoll *ep, struct epoll_event *event,
  210. struct file *tfile, int fd);
  211. static int ep_modify(struct eventpoll *ep, struct epitem *epi,
  212. struct epoll_event *event);
  213. static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi);
  214. static int ep_unlink(struct eventpoll *ep, struct epitem *epi);
  215. static int ep_remove(struct eventpoll *ep, struct epitem *epi);
  216. static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *key);
  217. static int ep_eventpoll_close(struct inode *inode, struct file *file);
  218. static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait);
  219. static int ep_collect_ready_items(struct eventpoll *ep,
  220. struct list_head *txlist, int maxevents);
  221. static int ep_send_events(struct eventpoll *ep, struct list_head *txlist,
  222. struct epoll_event __user *events);
  223. static void ep_reinject_items(struct eventpoll *ep, struct list_head *txlist);
  224. static int ep_events_transfer(struct eventpoll *ep,
  225. struct epoll_event __user *events,
  226. int maxevents);
  227. static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
  228. int maxevents, long timeout);
  229. static int eventpollfs_delete_dentry(struct dentry *dentry);
  230. static struct inode *ep_eventpoll_inode(void);
  231. static int eventpollfs_get_sb(struct file_system_type *fs_type,
  232. int flags, const char *dev_name,
  233. void *data, struct vfsmount *mnt);
  234. /*
  235. * This semaphore is used to serialize ep_free() and eventpoll_release_file().
  236. */
  237. static struct mutex epmutex;
  238. /* Safe wake up implementation */
  239. static struct poll_safewake psw;
  240. /* Slab cache used to allocate "struct epitem" */
  241. static struct kmem_cache *epi_cache __read_mostly;
  242. /* Slab cache used to allocate "struct eppoll_entry" */
  243. static struct kmem_cache *pwq_cache __read_mostly;
  244. /* Virtual fs used to allocate inodes for eventpoll files */
  245. static struct vfsmount *eventpoll_mnt __read_mostly;
  246. /* File callbacks that implement the eventpoll file behaviour */
  247. static const struct file_operations eventpoll_fops = {
  248. .release = ep_eventpoll_close,
  249. .poll = ep_eventpoll_poll
  250. };
  251. /*
  252. * This is used to register the virtual file system from where
  253. * eventpoll inodes are allocated.
  254. */
  255. static struct file_system_type eventpoll_fs_type = {
  256. .name = "eventpollfs",
  257. .get_sb = eventpollfs_get_sb,
  258. .kill_sb = kill_anon_super,
  259. };
  260. /* Very basic directory entry operations for the eventpoll virtual file system */
  261. static struct dentry_operations eventpollfs_dentry_operations = {
  262. .d_delete = eventpollfs_delete_dentry,
  263. };
  264. /* Fast test to see if the file is an evenpoll file */
  265. static inline int is_file_epoll(struct file *f)
  266. {
  267. return f->f_op == &eventpoll_fops;
  268. }
  269. /* Setup the structure that is used as key for the rb-tree */
  270. static inline void ep_set_ffd(struct epoll_filefd *ffd,
  271. struct file *file, int fd)
  272. {
  273. ffd->file = file;
  274. ffd->fd = fd;
  275. }
  276. /* Compare rb-tree keys */
  277. static inline int ep_cmp_ffd(struct epoll_filefd *p1,
  278. struct epoll_filefd *p2)
  279. {
  280. return (p1->file > p2->file ? +1:
  281. (p1->file < p2->file ? -1 : p1->fd - p2->fd));
  282. }
  283. /* Special initialization for the rb-tree node to detect linkage */
  284. static inline void ep_rb_initnode(struct rb_node *n)
  285. {
  286. rb_set_parent(n, n);
  287. }
  288. /* Removes a node from the rb-tree and marks it for a fast is-linked check */
  289. static inline void ep_rb_erase(struct rb_node *n, struct rb_root *r)
  290. {
  291. rb_erase(n, r);
  292. rb_set_parent(n, n);
  293. }
  294. /* Fast check to verify that the item is linked to the main rb-tree */
  295. static inline int ep_rb_linked(struct rb_node *n)
  296. {
  297. return rb_parent(n) != n;
  298. }
  299. /*
  300. * Remove the item from the list and perform its initialization.
  301. * This is useful for us because we can test if the item is linked
  302. * using "ep_is_linked(p)".
  303. */
  304. static inline void ep_list_del(struct list_head *p)
  305. {
  306. list_del(p);
  307. INIT_LIST_HEAD(p);
  308. }
  309. /* Tells us if the item is currently linked */
  310. static inline int ep_is_linked(struct list_head *p)
  311. {
  312. return !list_empty(p);
  313. }
  314. /* Get the "struct epitem" from a wait queue pointer */
  315. static inline struct epitem * ep_item_from_wait(wait_queue_t *p)
  316. {
  317. return container_of(p, struct eppoll_entry, wait)->base;
  318. }
  319. /* Get the "struct epitem" from an epoll queue wrapper */
  320. static inline struct epitem * ep_item_from_epqueue(poll_table *p)
  321. {
  322. return container_of(p, struct ep_pqueue, pt)->epi;
  323. }
  324. /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
  325. static inline int ep_op_hash_event(int op)
  326. {
  327. return op != EPOLL_CTL_DEL;
  328. }
  329. /* Initialize the poll safe wake up structure */
  330. static void ep_poll_safewake_init(struct poll_safewake *psw)
  331. {
  332. INIT_LIST_HEAD(&psw->wake_task_list);
  333. spin_lock_init(&psw->lock);
  334. }
  335. /*
  336. * Perform a safe wake up of the poll wait list. The problem is that
  337. * with the new callback'd wake up system, it is possible that the
  338. * poll callback is reentered from inside the call to wake_up() done
  339. * on the poll wait queue head. The rule is that we cannot reenter the
  340. * wake up code from the same task more than EP_MAX_POLLWAKE_NESTS times,
  341. * and we cannot reenter the same wait queue head at all. This will
  342. * enable to have a hierarchy of epoll file descriptor of no more than
  343. * EP_MAX_POLLWAKE_NESTS deep. We need the irq version of the spin lock
  344. * because this one gets called by the poll callback, that in turn is called
  345. * from inside a wake_up(), that might be called from irq context.
  346. */
  347. static void ep_poll_safewake(struct poll_safewake *psw, wait_queue_head_t *wq)
  348. {
  349. int wake_nests = 0;
  350. unsigned long flags;
  351. struct task_struct *this_task = current;
  352. struct list_head *lsthead = &psw->wake_task_list, *lnk;
  353. struct wake_task_node *tncur;
  354. struct wake_task_node tnode;
  355. spin_lock_irqsave(&psw->lock, flags);
  356. /* Try to see if the current task is already inside this wakeup call */
  357. list_for_each(lnk, lsthead) {
  358. tncur = list_entry(lnk, struct wake_task_node, llink);
  359. if (tncur->wq == wq ||
  360. (tncur->task == this_task && ++wake_nests > EP_MAX_POLLWAKE_NESTS)) {
  361. /*
  362. * Ops ... loop detected or maximum nest level reached.
  363. * We abort this wake by breaking the cycle itself.
  364. */
  365. spin_unlock_irqrestore(&psw->lock, flags);
  366. return;
  367. }
  368. }
  369. /* Add the current task to the list */
  370. tnode.task = this_task;
  371. tnode.wq = wq;
  372. list_add(&tnode.llink, lsthead);
  373. spin_unlock_irqrestore(&psw->lock, flags);
  374. /* Do really wake up now */
  375. wake_up(wq);
  376. /* Remove the current task from the list */
  377. spin_lock_irqsave(&psw->lock, flags);
  378. list_del(&tnode.llink);
  379. spin_unlock_irqrestore(&psw->lock, flags);
  380. }
  381. /*
  382. * This is called from eventpoll_release() to unlink files from the eventpoll
  383. * interface. We need to have this facility to cleanup correctly files that are
  384. * closed without being removed from the eventpoll interface.
  385. */
  386. void eventpoll_release_file(struct file *file)
  387. {
  388. struct list_head *lsthead = &file->f_ep_links;
  389. struct eventpoll *ep;
  390. struct epitem *epi;
  391. /*
  392. * We don't want to get "file->f_ep_lock" because it is not
  393. * necessary. It is not necessary because we're in the "struct file"
  394. * cleanup path, and this means that noone is using this file anymore.
  395. * The only hit might come from ep_free() but by holding the semaphore
  396. * will correctly serialize the operation. We do need to acquire
  397. * "ep->sem" after "epmutex" because ep_remove() requires it when called
  398. * from anywhere but ep_free().
  399. */
  400. mutex_lock(&epmutex);
  401. while (!list_empty(lsthead)) {
  402. epi = list_entry(lsthead->next, struct epitem, fllink);
  403. ep = epi->ep;
  404. ep_list_del(&epi->fllink);
  405. down_write(&ep->sem);
  406. ep_remove(ep, epi);
  407. up_write(&ep->sem);
  408. }
  409. mutex_unlock(&epmutex);
  410. }
  411. /*
  412. * It opens an eventpoll file descriptor by suggesting a storage of "size"
  413. * file descriptors. The size parameter is just an hint about how to size
  414. * data structures. It won't prevent the user to store more than "size"
  415. * file descriptors inside the epoll interface. It is the kernel part of
  416. * the userspace epoll_create(2).
  417. */
  418. asmlinkage long sys_epoll_create(int size)
  419. {
  420. int error, fd = -1;
  421. struct eventpoll *ep;
  422. struct inode *inode;
  423. struct file *file;
  424. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d)\n",
  425. current, size));
  426. /*
  427. * Sanity check on the size parameter, and create the internal data
  428. * structure ( "struct eventpoll" ).
  429. */
  430. error = -EINVAL;
  431. if (size <= 0 || (error = ep_alloc(&ep)) != 0)
  432. goto eexit_1;
  433. /*
  434. * Creates all the items needed to setup an eventpoll file. That is,
  435. * a file structure, and inode and a free file descriptor.
  436. */
  437. error = ep_getfd(&fd, &inode, &file, ep);
  438. if (error)
  439. goto eexit_2;
  440. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d) = %d\n",
  441. current, size, fd));
  442. return fd;
  443. eexit_2:
  444. ep_free(ep);
  445. kfree(ep);
  446. eexit_1:
  447. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d) = %d\n",
  448. current, size, error));
  449. return error;
  450. }
  451. /*
  452. * The following function implements the controller interface for
  453. * the eventpoll file that enables the insertion/removal/change of
  454. * file descriptors inside the interest set. It represents
  455. * the kernel part of the user space epoll_ctl(2).
  456. */
  457. asmlinkage long
  458. sys_epoll_ctl(int epfd, int op, int fd, struct epoll_event __user *event)
  459. {
  460. int error;
  461. struct file *file, *tfile;
  462. struct eventpoll *ep;
  463. struct epitem *epi;
  464. struct epoll_event epds;
  465. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %p)\n",
  466. current, epfd, op, fd, event));
  467. error = -EFAULT;
  468. if (ep_op_hash_event(op) &&
  469. copy_from_user(&epds, event, sizeof(struct epoll_event)))
  470. goto eexit_1;
  471. /* Get the "struct file *" for the eventpoll file */
  472. error = -EBADF;
  473. file = fget(epfd);
  474. if (!file)
  475. goto eexit_1;
  476. /* Get the "struct file *" for the target file */
  477. tfile = fget(fd);
  478. if (!tfile)
  479. goto eexit_2;
  480. /* The target file descriptor must support poll */
  481. error = -EPERM;
  482. if (!tfile->f_op || !tfile->f_op->poll)
  483. goto eexit_3;
  484. /*
  485. * We have to check that the file structure underneath the file descriptor
  486. * the user passed to us _is_ an eventpoll file. And also we do not permit
  487. * adding an epoll file descriptor inside itself.
  488. */
  489. error = -EINVAL;
  490. if (file == tfile || !is_file_epoll(file))
  491. goto eexit_3;
  492. /*
  493. * At this point it is safe to assume that the "private_data" contains
  494. * our own data structure.
  495. */
  496. ep = file->private_data;
  497. down_write(&ep->sem);
  498. /* Try to lookup the file inside our hash table */
  499. epi = ep_find(ep, tfile, fd);
  500. error = -EINVAL;
  501. switch (op) {
  502. case EPOLL_CTL_ADD:
  503. if (!epi) {
  504. epds.events |= POLLERR | POLLHUP;
  505. error = ep_insert(ep, &epds, tfile, fd);
  506. } else
  507. error = -EEXIST;
  508. break;
  509. case EPOLL_CTL_DEL:
  510. if (epi)
  511. error = ep_remove(ep, epi);
  512. else
  513. error = -ENOENT;
  514. break;
  515. case EPOLL_CTL_MOD:
  516. if (epi) {
  517. epds.events |= POLLERR | POLLHUP;
  518. error = ep_modify(ep, epi, &epds);
  519. } else
  520. error = -ENOENT;
  521. break;
  522. }
  523. /*
  524. * The function ep_find() increments the usage count of the structure
  525. * so, if this is not NULL, we need to release it.
  526. */
  527. if (epi)
  528. ep_release_epitem(epi);
  529. up_write(&ep->sem);
  530. eexit_3:
  531. fput(tfile);
  532. eexit_2:
  533. fput(file);
  534. eexit_1:
  535. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %p) = %d\n",
  536. current, epfd, op, fd, event, error));
  537. return error;
  538. }
  539. /*
  540. * Implement the event wait interface for the eventpoll file. It is the kernel
  541. * part of the user space epoll_wait(2).
  542. */
  543. asmlinkage long sys_epoll_wait(int epfd, struct epoll_event __user *events,
  544. int maxevents, int timeout)
  545. {
  546. int error;
  547. struct file *file;
  548. struct eventpoll *ep;
  549. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d)\n",
  550. current, epfd, events, maxevents, timeout));
  551. /* The maximum number of event must be greater than zero */
  552. if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
  553. return -EINVAL;
  554. /* Verify that the area passed by the user is writeable */
  555. if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event))) {
  556. error = -EFAULT;
  557. goto eexit_1;
  558. }
  559. /* Get the "struct file *" for the eventpoll file */
  560. error = -EBADF;
  561. file = fget(epfd);
  562. if (!file)
  563. goto eexit_1;
  564. /*
  565. * We have to check that the file structure underneath the fd
  566. * the user passed to us _is_ an eventpoll file.
  567. */
  568. error = -EINVAL;
  569. if (!is_file_epoll(file))
  570. goto eexit_2;
  571. /*
  572. * At this point it is safe to assume that the "private_data" contains
  573. * our own data structure.
  574. */
  575. ep = file->private_data;
  576. /* Time to fish for events ... */
  577. error = ep_poll(ep, events, maxevents, timeout);
  578. eexit_2:
  579. fput(file);
  580. eexit_1:
  581. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d) = %d\n",
  582. current, epfd, events, maxevents, timeout, error));
  583. return error;
  584. }
  585. #ifdef TIF_RESTORE_SIGMASK
  586. /*
  587. * Implement the event wait interface for the eventpoll file. It is the kernel
  588. * part of the user space epoll_pwait(2).
  589. */
  590. asmlinkage long sys_epoll_pwait(int epfd, struct epoll_event __user *events,
  591. int maxevents, int timeout, const sigset_t __user *sigmask,
  592. size_t sigsetsize)
  593. {
  594. int error;
  595. sigset_t ksigmask, sigsaved;
  596. /*
  597. * If the caller wants a certain signal mask to be set during the wait,
  598. * we apply it here.
  599. */
  600. if (sigmask) {
  601. if (sigsetsize != sizeof(sigset_t))
  602. return -EINVAL;
  603. if (copy_from_user(&ksigmask, sigmask, sizeof(ksigmask)))
  604. return -EFAULT;
  605. sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP));
  606. sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved);
  607. }
  608. error = sys_epoll_wait(epfd, events, maxevents, timeout);
  609. /*
  610. * If we changed the signal mask, we need to restore the original one.
  611. * In case we've got a signal while waiting, we do not restore the
  612. * signal mask yet, and we allow do_signal() to deliver the signal on
  613. * the way back to userspace, before the signal mask is restored.
  614. */
  615. if (sigmask) {
  616. if (error == -EINTR) {
  617. memcpy(&current->saved_sigmask, &sigsaved,
  618. sizeof(sigsaved));
  619. set_thread_flag(TIF_RESTORE_SIGMASK);
  620. } else
  621. sigprocmask(SIG_SETMASK, &sigsaved, NULL);
  622. }
  623. return error;
  624. }
  625. #endif /* #ifdef TIF_RESTORE_SIGMASK */
  626. /*
  627. * Creates the file descriptor to be used by the epoll interface.
  628. */
  629. static int ep_getfd(int *efd, struct inode **einode, struct file **efile,
  630. struct eventpoll *ep)
  631. {
  632. struct qstr this;
  633. char name[32];
  634. struct dentry *dentry;
  635. struct inode *inode;
  636. struct file *file;
  637. int error, fd;
  638. /* Get an ready to use file */
  639. error = -ENFILE;
  640. file = get_empty_filp();
  641. if (!file)
  642. goto eexit_1;
  643. /* Allocates an inode from the eventpoll file system */
  644. inode = ep_eventpoll_inode();
  645. if (IS_ERR(inode)) {
  646. error = PTR_ERR(inode);
  647. goto eexit_2;
  648. }
  649. /* Allocates a free descriptor to plug the file onto */
  650. error = get_unused_fd();
  651. if (error < 0)
  652. goto eexit_3;
  653. fd = error;
  654. /*
  655. * Link the inode to a directory entry by creating a unique name
  656. * using the inode number.
  657. */
  658. error = -ENOMEM;
  659. sprintf(name, "[%lu]", inode->i_ino);
  660. this.name = name;
  661. this.len = strlen(name);
  662. this.hash = inode->i_ino;
  663. dentry = d_alloc(eventpoll_mnt->mnt_sb->s_root, &this);
  664. if (!dentry)
  665. goto eexit_4;
  666. dentry->d_op = &eventpollfs_dentry_operations;
  667. d_add(dentry, inode);
  668. file->f_path.mnt = mntget(eventpoll_mnt);
  669. file->f_path.dentry = dentry;
  670. file->f_mapping = inode->i_mapping;
  671. file->f_pos = 0;
  672. file->f_flags = O_RDONLY;
  673. file->f_op = &eventpoll_fops;
  674. file->f_mode = FMODE_READ;
  675. file->f_version = 0;
  676. file->private_data = ep;
  677. /* Install the new setup file into the allocated fd. */
  678. fd_install(fd, file);
  679. *efd = fd;
  680. *einode = inode;
  681. *efile = file;
  682. return 0;
  683. eexit_4:
  684. put_unused_fd(fd);
  685. eexit_3:
  686. iput(inode);
  687. eexit_2:
  688. put_filp(file);
  689. eexit_1:
  690. return error;
  691. }
  692. static int ep_alloc(struct eventpoll **pep)
  693. {
  694. struct eventpoll *ep = kzalloc(sizeof(*ep), GFP_KERNEL);
  695. if (!ep)
  696. return -ENOMEM;
  697. rwlock_init(&ep->lock);
  698. init_rwsem(&ep->sem);
  699. init_waitqueue_head(&ep->wq);
  700. init_waitqueue_head(&ep->poll_wait);
  701. INIT_LIST_HEAD(&ep->rdllist);
  702. ep->rbr = RB_ROOT;
  703. *pep = ep;
  704. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_alloc() ep=%p\n",
  705. current, ep));
  706. return 0;
  707. }
  708. static void ep_free(struct eventpoll *ep)
  709. {
  710. struct rb_node *rbp;
  711. struct epitem *epi;
  712. /* We need to release all tasks waiting for these file */
  713. if (waitqueue_active(&ep->poll_wait))
  714. ep_poll_safewake(&psw, &ep->poll_wait);
  715. /*
  716. * We need to lock this because we could be hit by
  717. * eventpoll_release_file() while we're freeing the "struct eventpoll".
  718. * We do not need to hold "ep->sem" here because the epoll file
  719. * is on the way to be removed and no one has references to it
  720. * anymore. The only hit might come from eventpoll_release_file() but
  721. * holding "epmutex" is sufficent here.
  722. */
  723. mutex_lock(&epmutex);
  724. /*
  725. * Walks through the whole tree by unregistering poll callbacks.
  726. */
  727. for (rbp = rb_first(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  728. epi = rb_entry(rbp, struct epitem, rbn);
  729. ep_unregister_pollwait(ep, epi);
  730. }
  731. /*
  732. * Walks through the whole hash by freeing each "struct epitem". At this
  733. * point we are sure no poll callbacks will be lingering around, and also by
  734. * write-holding "sem" we can be sure that no file cleanup code will hit
  735. * us during this operation. So we can avoid the lock on "ep->lock".
  736. */
  737. while ((rbp = rb_first(&ep->rbr)) != 0) {
  738. epi = rb_entry(rbp, struct epitem, rbn);
  739. ep_remove(ep, epi);
  740. }
  741. mutex_unlock(&epmutex);
  742. }
  743. /*
  744. * Search the file inside the eventpoll hash. It add usage count to
  745. * the returned item, so the caller must call ep_release_epitem()
  746. * after finished using the "struct epitem".
  747. */
  748. static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
  749. {
  750. int kcmp;
  751. unsigned long flags;
  752. struct rb_node *rbp;
  753. struct epitem *epi, *epir = NULL;
  754. struct epoll_filefd ffd;
  755. ep_set_ffd(&ffd, file, fd);
  756. read_lock_irqsave(&ep->lock, flags);
  757. for (rbp = ep->rbr.rb_node; rbp; ) {
  758. epi = rb_entry(rbp, struct epitem, rbn);
  759. kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
  760. if (kcmp > 0)
  761. rbp = rbp->rb_right;
  762. else if (kcmp < 0)
  763. rbp = rbp->rb_left;
  764. else {
  765. ep_use_epitem(epi);
  766. epir = epi;
  767. break;
  768. }
  769. }
  770. read_unlock_irqrestore(&ep->lock, flags);
  771. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_find(%p) -> %p\n",
  772. current, file, epir));
  773. return epir;
  774. }
  775. /*
  776. * Increment the usage count of the "struct epitem" making it sure
  777. * that the user will have a valid pointer to reference.
  778. */
  779. static void ep_use_epitem(struct epitem *epi)
  780. {
  781. atomic_inc(&epi->usecnt);
  782. }
  783. /*
  784. * Decrement ( release ) the usage count by signaling that the user
  785. * has finished using the structure. It might lead to freeing the
  786. * structure itself if the count goes to zero.
  787. */
  788. static void ep_release_epitem(struct epitem *epi)
  789. {
  790. if (atomic_dec_and_test(&epi->usecnt))
  791. kmem_cache_free(epi_cache, epi);
  792. }
  793. /*
  794. * This is the callback that is used to add our wait queue to the
  795. * target file wakeup lists.
  796. */
  797. static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
  798. poll_table *pt)
  799. {
  800. struct epitem *epi = ep_item_from_epqueue(pt);
  801. struct eppoll_entry *pwq;
  802. if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {
  803. init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
  804. pwq->whead = whead;
  805. pwq->base = epi;
  806. add_wait_queue(whead, &pwq->wait);
  807. list_add_tail(&pwq->llink, &epi->pwqlist);
  808. epi->nwait++;
  809. } else {
  810. /* We have to signal that an error occurred */
  811. epi->nwait = -1;
  812. }
  813. }
  814. static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
  815. {
  816. int kcmp;
  817. struct rb_node **p = &ep->rbr.rb_node, *parent = NULL;
  818. struct epitem *epic;
  819. while (*p) {
  820. parent = *p;
  821. epic = rb_entry(parent, struct epitem, rbn);
  822. kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);
  823. if (kcmp > 0)
  824. p = &parent->rb_right;
  825. else
  826. p = &parent->rb_left;
  827. }
  828. rb_link_node(&epi->rbn, parent, p);
  829. rb_insert_color(&epi->rbn, &ep->rbr);
  830. }
  831. static int ep_insert(struct eventpoll *ep, struct epoll_event *event,
  832. struct file *tfile, int fd)
  833. {
  834. int error, revents, pwake = 0;
  835. unsigned long flags;
  836. struct epitem *epi;
  837. struct ep_pqueue epq;
  838. error = -ENOMEM;
  839. if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))
  840. goto eexit_1;
  841. /* Item initialization follow here ... */
  842. ep_rb_initnode(&epi->rbn);
  843. INIT_LIST_HEAD(&epi->rdllink);
  844. INIT_LIST_HEAD(&epi->fllink);
  845. INIT_LIST_HEAD(&epi->txlink);
  846. INIT_LIST_HEAD(&epi->pwqlist);
  847. epi->ep = ep;
  848. ep_set_ffd(&epi->ffd, tfile, fd);
  849. epi->event = *event;
  850. atomic_set(&epi->usecnt, 1);
  851. epi->nwait = 0;
  852. /* Initialize the poll table using the queue callback */
  853. epq.epi = epi;
  854. init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
  855. /*
  856. * Attach the item to the poll hooks and get current event bits.
  857. * We can safely use the file* here because its usage count has
  858. * been increased by the caller of this function.
  859. */
  860. revents = tfile->f_op->poll(tfile, &epq.pt);
  861. /*
  862. * We have to check if something went wrong during the poll wait queue
  863. * install process. Namely an allocation for a wait queue failed due
  864. * high memory pressure.
  865. */
  866. if (epi->nwait < 0)
  867. goto eexit_2;
  868. /* Add the current item to the list of active epoll hook for this file */
  869. spin_lock(&tfile->f_ep_lock);
  870. list_add_tail(&epi->fllink, &tfile->f_ep_links);
  871. spin_unlock(&tfile->f_ep_lock);
  872. /* We have to drop the new item inside our item list to keep track of it */
  873. write_lock_irqsave(&ep->lock, flags);
  874. /* Add the current item to the rb-tree */
  875. ep_rbtree_insert(ep, epi);
  876. /* If the file is already "ready" we drop it inside the ready list */
  877. if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) {
  878. list_add_tail(&epi->rdllink, &ep->rdllist);
  879. /* Notify waiting tasks that events are available */
  880. if (waitqueue_active(&ep->wq))
  881. __wake_up_locked(&ep->wq, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE);
  882. if (waitqueue_active(&ep->poll_wait))
  883. pwake++;
  884. }
  885. write_unlock_irqrestore(&ep->lock, flags);
  886. /* We have to call this outside the lock */
  887. if (pwake)
  888. ep_poll_safewake(&psw, &ep->poll_wait);
  889. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_insert(%p, %p, %d)\n",
  890. current, ep, tfile, fd));
  891. return 0;
  892. eexit_2:
  893. ep_unregister_pollwait(ep, epi);
  894. /*
  895. * We need to do this because an event could have been arrived on some
  896. * allocated wait queue.
  897. */
  898. write_lock_irqsave(&ep->lock, flags);
  899. if (ep_is_linked(&epi->rdllink))
  900. ep_list_del(&epi->rdllink);
  901. write_unlock_irqrestore(&ep->lock, flags);
  902. kmem_cache_free(epi_cache, epi);
  903. eexit_1:
  904. return error;
  905. }
  906. /*
  907. * Modify the interest event mask by dropping an event if the new mask
  908. * has a match in the current file status.
  909. */
  910. static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event)
  911. {
  912. int pwake = 0;
  913. unsigned int revents;
  914. unsigned long flags;
  915. /*
  916. * Set the new event interest mask before calling f_op->poll(), otherwise
  917. * a potential race might occur. In fact if we do this operation inside
  918. * the lock, an event might happen between the f_op->poll() call and the
  919. * new event set registering.
  920. */
  921. epi->event.events = event->events;
  922. /*
  923. * Get current event bits. We can safely use the file* here because
  924. * its usage count has been increased by the caller of this function.
  925. */
  926. revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL);
  927. write_lock_irqsave(&ep->lock, flags);
  928. /* Copy the data member from inside the lock */
  929. epi->event.data = event->data;
  930. /*
  931. * If the item is not linked to the hash it means that it's on its
  932. * way toward the removal. Do nothing in this case.
  933. */
  934. if (ep_rb_linked(&epi->rbn)) {
  935. /*
  936. * If the item is "hot" and it is not registered inside the ready
  937. * list, push it inside. If the item is not "hot" and it is currently
  938. * registered inside the ready list, unlink it.
  939. */
  940. if (revents & event->events) {
  941. if (!ep_is_linked(&epi->rdllink)) {
  942. list_add_tail(&epi->rdllink, &ep->rdllist);
  943. /* Notify waiting tasks that events are available */
  944. if (waitqueue_active(&ep->wq))
  945. __wake_up_locked(&ep->wq, TASK_UNINTERRUPTIBLE |
  946. TASK_INTERRUPTIBLE);
  947. if (waitqueue_active(&ep->poll_wait))
  948. pwake++;
  949. }
  950. }
  951. }
  952. write_unlock_irqrestore(&ep->lock, flags);
  953. /* We have to call this outside the lock */
  954. if (pwake)
  955. ep_poll_safewake(&psw, &ep->poll_wait);
  956. return 0;
  957. }
  958. /*
  959. * This function unregister poll callbacks from the associated file descriptor.
  960. * Since this must be called without holding "ep->lock" the atomic exchange trick
  961. * will protect us from multiple unregister.
  962. */
  963. static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
  964. {
  965. int nwait;
  966. struct list_head *lsthead = &epi->pwqlist;
  967. struct eppoll_entry *pwq;
  968. /* This is called without locks, so we need the atomic exchange */
  969. nwait = xchg(&epi->nwait, 0);
  970. if (nwait) {
  971. while (!list_empty(lsthead)) {
  972. pwq = list_entry(lsthead->next, struct eppoll_entry, llink);
  973. ep_list_del(&pwq->llink);
  974. remove_wait_queue(pwq->whead, &pwq->wait);
  975. kmem_cache_free(pwq_cache, pwq);
  976. }
  977. }
  978. }
  979. /*
  980. * Unlink the "struct epitem" from all places it might have been hooked up.
  981. * This function must be called with write IRQ lock on "ep->lock".
  982. */
  983. static int ep_unlink(struct eventpoll *ep, struct epitem *epi)
  984. {
  985. int error;
  986. /*
  987. * It can happen that this one is called for an item already unlinked.
  988. * The check protect us from doing a double unlink ( crash ).
  989. */
  990. error = -ENOENT;
  991. if (!ep_rb_linked(&epi->rbn))
  992. goto eexit_1;
  993. /*
  994. * Clear the event mask for the unlinked item. This will avoid item
  995. * notifications to be sent after the unlink operation from inside
  996. * the kernel->userspace event transfer loop.
  997. */
  998. epi->event.events = 0;
  999. /*
  1000. * At this point is safe to do the job, unlink the item from our rb-tree.
  1001. * This operation togheter with the above check closes the door to
  1002. * double unlinks.
  1003. */
  1004. ep_rb_erase(&epi->rbn, &ep->rbr);
  1005. /*
  1006. * If the item we are going to remove is inside the ready file descriptors
  1007. * we want to remove it from this list to avoid stale events.
  1008. */
  1009. if (ep_is_linked(&epi->rdllink))
  1010. ep_list_del(&epi->rdllink);
  1011. error = 0;
  1012. eexit_1:
  1013. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_unlink(%p, %p) = %d\n",
  1014. current, ep, epi->ffd.file, error));
  1015. return error;
  1016. }
  1017. /*
  1018. * Removes a "struct epitem" from the eventpoll hash and deallocates
  1019. * all the associated resources.
  1020. */
  1021. static int ep_remove(struct eventpoll *ep, struct epitem *epi)
  1022. {
  1023. int error;
  1024. unsigned long flags;
  1025. struct file *file = epi->ffd.file;
  1026. /*
  1027. * Removes poll wait queue hooks. We _have_ to do this without holding
  1028. * the "ep->lock" otherwise a deadlock might occur. This because of the
  1029. * sequence of the lock acquisition. Here we do "ep->lock" then the wait
  1030. * queue head lock when unregistering the wait queue. The wakeup callback
  1031. * will run by holding the wait queue head lock and will call our callback
  1032. * that will try to get "ep->lock".
  1033. */
  1034. ep_unregister_pollwait(ep, epi);
  1035. /* Remove the current item from the list of epoll hooks */
  1036. spin_lock(&file->f_ep_lock);
  1037. if (ep_is_linked(&epi->fllink))
  1038. ep_list_del(&epi->fllink);
  1039. spin_unlock(&file->f_ep_lock);
  1040. /* We need to acquire the write IRQ lock before calling ep_unlink() */
  1041. write_lock_irqsave(&ep->lock, flags);
  1042. /* Really unlink the item from the hash */
  1043. error = ep_unlink(ep, epi);
  1044. write_unlock_irqrestore(&ep->lock, flags);
  1045. if (error)
  1046. goto eexit_1;
  1047. /* At this point it is safe to free the eventpoll item */
  1048. ep_release_epitem(epi);
  1049. error = 0;
  1050. eexit_1:
  1051. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_remove(%p, %p) = %d\n",
  1052. current, ep, file, error));
  1053. return error;
  1054. }
  1055. /*
  1056. * This is the callback that is passed to the wait queue wakeup
  1057. * machanism. It is called by the stored file descriptors when they
  1058. * have events to report.
  1059. */
  1060. static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *key)
  1061. {
  1062. int pwake = 0;
  1063. unsigned long flags;
  1064. struct epitem *epi = ep_item_from_wait(wait);
  1065. struct eventpoll *ep = epi->ep;
  1066. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: poll_callback(%p) epi=%p ep=%p\n",
  1067. current, epi->ffd.file, epi, ep));
  1068. write_lock_irqsave(&ep->lock, flags);
  1069. /*
  1070. * If the event mask does not contain any poll(2) event, we consider the
  1071. * descriptor to be disabled. This condition is likely the effect of the
  1072. * EPOLLONESHOT bit that disables the descriptor when an event is received,
  1073. * until the next EPOLL_CTL_MOD will be issued.
  1074. */
  1075. if (!(epi->event.events & ~EP_PRIVATE_BITS))
  1076. goto is_disabled;
  1077. /* If this file is already in the ready list we exit soon */
  1078. if (ep_is_linked(&epi->rdllink))
  1079. goto is_linked;
  1080. list_add_tail(&epi->rdllink, &ep->rdllist);
  1081. is_linked:
  1082. /*
  1083. * Wake up ( if active ) both the eventpoll wait list and the ->poll()
  1084. * wait list.
  1085. */
  1086. if (waitqueue_active(&ep->wq))
  1087. __wake_up_locked(&ep->wq, TASK_UNINTERRUPTIBLE |
  1088. TASK_INTERRUPTIBLE);
  1089. if (waitqueue_active(&ep->poll_wait))
  1090. pwake++;
  1091. is_disabled:
  1092. write_unlock_irqrestore(&ep->lock, flags);
  1093. /* We have to call this outside the lock */
  1094. if (pwake)
  1095. ep_poll_safewake(&psw, &ep->poll_wait);
  1096. return 1;
  1097. }
  1098. static int ep_eventpoll_close(struct inode *inode, struct file *file)
  1099. {
  1100. struct eventpoll *ep = file->private_data;
  1101. if (ep) {
  1102. ep_free(ep);
  1103. kfree(ep);
  1104. }
  1105. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: close() ep=%p\n", current, ep));
  1106. return 0;
  1107. }
  1108. static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait)
  1109. {
  1110. unsigned int pollflags = 0;
  1111. unsigned long flags;
  1112. struct eventpoll *ep = file->private_data;
  1113. /* Insert inside our poll wait queue */
  1114. poll_wait(file, &ep->poll_wait, wait);
  1115. /* Check our condition */
  1116. read_lock_irqsave(&ep->lock, flags);
  1117. if (!list_empty(&ep->rdllist))
  1118. pollflags = POLLIN | POLLRDNORM;
  1119. read_unlock_irqrestore(&ep->lock, flags);
  1120. return pollflags;
  1121. }
  1122. /*
  1123. * Since we have to release the lock during the __copy_to_user() operation and
  1124. * during the f_op->poll() call, we try to collect the maximum number of items
  1125. * by reducing the irqlock/irqunlock switching rate.
  1126. */
  1127. static int ep_collect_ready_items(struct eventpoll *ep, struct list_head *txlist, int maxevents)
  1128. {
  1129. int nepi;
  1130. unsigned long flags;
  1131. struct list_head *lsthead = &ep->rdllist, *lnk;
  1132. struct epitem *epi;
  1133. write_lock_irqsave(&ep->lock, flags);
  1134. for (nepi = 0, lnk = lsthead->next; lnk != lsthead && nepi < maxevents;) {
  1135. epi = list_entry(lnk, struct epitem, rdllink);
  1136. lnk = lnk->next;
  1137. /* If this file is already in the ready list we exit soon */
  1138. if (!ep_is_linked(&epi->txlink)) {
  1139. /*
  1140. * This is initialized in this way so that the default
  1141. * behaviour of the reinjecting code will be to push back
  1142. * the item inside the ready list.
  1143. */
  1144. epi->revents = epi->event.events;
  1145. /* Link the ready item into the transfer list */
  1146. list_add(&epi->txlink, txlist);
  1147. nepi++;
  1148. /*
  1149. * Unlink the item from the ready list.
  1150. */
  1151. ep_list_del(&epi->rdllink);
  1152. }
  1153. }
  1154. write_unlock_irqrestore(&ep->lock, flags);
  1155. return nepi;
  1156. }
  1157. /*
  1158. * This function is called without holding the "ep->lock" since the call to
  1159. * __copy_to_user() might sleep, and also f_op->poll() might reenable the IRQ
  1160. * because of the way poll() is traditionally implemented in Linux.
  1161. */
  1162. static int ep_send_events(struct eventpoll *ep, struct list_head *txlist,
  1163. struct epoll_event __user *events)
  1164. {
  1165. int eventcnt = 0;
  1166. unsigned int revents;
  1167. struct list_head *lnk;
  1168. struct epitem *epi;
  1169. /*
  1170. * We can loop without lock because this is a task private list.
  1171. * The test done during the collection loop will guarantee us that
  1172. * another task will not try to collect this file. Also, items
  1173. * cannot vanish during the loop because we are holding "sem".
  1174. */
  1175. list_for_each(lnk, txlist) {
  1176. epi = list_entry(lnk, struct epitem, txlink);
  1177. /*
  1178. * Get the ready file event set. We can safely use the file
  1179. * because we are holding the "sem" in read and this will
  1180. * guarantee that both the file and the item will not vanish.
  1181. */
  1182. revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL);
  1183. /*
  1184. * Set the return event set for the current file descriptor.
  1185. * Note that only the task task was successfully able to link
  1186. * the item to its "txlist" will write this field.
  1187. */
  1188. epi->revents = revents & epi->event.events;
  1189. if (epi->revents) {
  1190. if (__put_user(epi->revents,
  1191. &events[eventcnt].events) ||
  1192. __put_user(epi->event.data,
  1193. &events[eventcnt].data))
  1194. return -EFAULT;
  1195. if (epi->event.events & EPOLLONESHOT)
  1196. epi->event.events &= EP_PRIVATE_BITS;
  1197. eventcnt++;
  1198. }
  1199. }
  1200. return eventcnt;
  1201. }
  1202. /*
  1203. * Walk through the transfer list we collected with ep_collect_ready_items()
  1204. * and, if 1) the item is still "alive" 2) its event set is not empty 3) it's
  1205. * not already linked, links it to the ready list. Same as above, we are holding
  1206. * "sem" so items cannot vanish underneath our nose.
  1207. */
  1208. static void ep_reinject_items(struct eventpoll *ep, struct list_head *txlist)
  1209. {
  1210. int ricnt = 0, pwake = 0;
  1211. unsigned long flags;
  1212. struct epitem *epi;
  1213. write_lock_irqsave(&ep->lock, flags);
  1214. while (!list_empty(txlist)) {
  1215. epi = list_entry(txlist->next, struct epitem, txlink);
  1216. /* Unlink the current item from the transfer list */
  1217. ep_list_del(&epi->txlink);
  1218. /*
  1219. * If the item is no more linked to the interest set, we don't
  1220. * have to push it inside the ready list because the following
  1221. * ep_release_epitem() is going to drop it. Also, if the current
  1222. * item is set to have an Edge Triggered behaviour, we don't have
  1223. * to push it back either.
  1224. */
  1225. if (ep_rb_linked(&epi->rbn) && !(epi->event.events & EPOLLET) &&
  1226. (epi->revents & epi->event.events) && !ep_is_linked(&epi->rdllink)) {
  1227. list_add_tail(&epi->rdllink, &ep->rdllist);
  1228. ricnt++;
  1229. }
  1230. }
  1231. if (ricnt) {
  1232. /*
  1233. * Wake up ( if active ) both the eventpoll wait list and the ->poll()
  1234. * wait list.
  1235. */
  1236. if (waitqueue_active(&ep->wq))
  1237. __wake_up_locked(&ep->wq, TASK_UNINTERRUPTIBLE |
  1238. TASK_INTERRUPTIBLE);
  1239. if (waitqueue_active(&ep->poll_wait))
  1240. pwake++;
  1241. }
  1242. write_unlock_irqrestore(&ep->lock, flags);
  1243. /* We have to call this outside the lock */
  1244. if (pwake)
  1245. ep_poll_safewake(&psw, &ep->poll_wait);
  1246. }
  1247. /*
  1248. * Perform the transfer of events to user space.
  1249. */
  1250. static int ep_events_transfer(struct eventpoll *ep,
  1251. struct epoll_event __user *events, int maxevents)
  1252. {
  1253. int eventcnt = 0;
  1254. struct list_head txlist;
  1255. INIT_LIST_HEAD(&txlist);
  1256. /*
  1257. * We need to lock this because we could be hit by
  1258. * eventpoll_release_file() and epoll_ctl(EPOLL_CTL_DEL).
  1259. */
  1260. down_read(&ep->sem);
  1261. /* Collect/extract ready items */
  1262. if (ep_collect_ready_items(ep, &txlist, maxevents) > 0) {
  1263. /* Build result set in userspace */
  1264. eventcnt = ep_send_events(ep, &txlist, events);
  1265. /* Reinject ready items into the ready list */
  1266. ep_reinject_items(ep, &txlist);
  1267. }
  1268. up_read(&ep->sem);
  1269. return eventcnt;
  1270. }
  1271. static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
  1272. int maxevents, long timeout)
  1273. {
  1274. int res, eavail;
  1275. unsigned long flags;
  1276. long jtimeout;
  1277. wait_queue_t wait;
  1278. /*
  1279. * Calculate the timeout by checking for the "infinite" value ( -1 )
  1280. * and the overflow condition. The passed timeout is in milliseconds,
  1281. * that why (t * HZ) / 1000.
  1282. */
  1283. jtimeout = (timeout < 0 || timeout >= EP_MAX_MSTIMEO) ?
  1284. MAX_SCHEDULE_TIMEOUT : (timeout * HZ + 999) / 1000;
  1285. retry:
  1286. write_lock_irqsave(&ep->lock, flags);
  1287. res = 0;
  1288. if (list_empty(&ep->rdllist)) {
  1289. /*
  1290. * We don't have any available event to return to the caller.
  1291. * We need to sleep here, and we will be wake up by
  1292. * ep_poll_callback() when events will become available.
  1293. */
  1294. init_waitqueue_entry(&wait, current);
  1295. __add_wait_queue(&ep->wq, &wait);
  1296. for (;;) {
  1297. /*
  1298. * We don't want to sleep if the ep_poll_callback() sends us
  1299. * a wakeup in between. That's why we set the task state
  1300. * to TASK_INTERRUPTIBLE before doing the checks.
  1301. */
  1302. set_current_state(TASK_INTERRUPTIBLE);
  1303. if (!list_empty(&ep->rdllist) || !jtimeout)
  1304. break;
  1305. if (signal_pending(current)) {
  1306. res = -EINTR;
  1307. break;
  1308. }
  1309. write_unlock_irqrestore(&ep->lock, flags);
  1310. jtimeout = schedule_timeout(jtimeout);
  1311. write_lock_irqsave(&ep->lock, flags);
  1312. }
  1313. __remove_wait_queue(&ep->wq, &wait);
  1314. set_current_state(TASK_RUNNING);
  1315. }
  1316. /* Is it worth to try to dig for events ? */
  1317. eavail = !list_empty(&ep->rdllist);
  1318. write_unlock_irqrestore(&ep->lock, flags);
  1319. /*
  1320. * Try to transfer events to user space. In case we get 0 events and
  1321. * there's still timeout left over, we go trying again in search of
  1322. * more luck.
  1323. */
  1324. if (!res && eavail &&
  1325. !(res = ep_events_transfer(ep, events, maxevents)) && jtimeout)
  1326. goto retry;
  1327. return res;
  1328. }
  1329. static int eventpollfs_delete_dentry(struct dentry *dentry)
  1330. {
  1331. return 1;
  1332. }
  1333. static struct inode *ep_eventpoll_inode(void)
  1334. {
  1335. int error = -ENOMEM;
  1336. struct inode *inode = new_inode(eventpoll_mnt->mnt_sb);
  1337. if (!inode)
  1338. goto eexit_1;
  1339. inode->i_fop = &eventpoll_fops;
  1340. /*
  1341. * Mark the inode dirty from the very beginning,
  1342. * that way it will never be moved to the dirty
  1343. * list because mark_inode_dirty() will think
  1344. * that it already _is_ on the dirty list.
  1345. */
  1346. inode->i_state = I_DIRTY;
  1347. inode->i_mode = S_IRUSR | S_IWUSR;
  1348. inode->i_uid = current->fsuid;
  1349. inode->i_gid = current->fsgid;
  1350. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1351. return inode;
  1352. eexit_1:
  1353. return ERR_PTR(error);
  1354. }
  1355. static int
  1356. eventpollfs_get_sb(struct file_system_type *fs_type, int flags,
  1357. const char *dev_name, void *data, struct vfsmount *mnt)
  1358. {
  1359. return get_sb_pseudo(fs_type, "eventpoll:", NULL, EVENTPOLLFS_MAGIC,
  1360. mnt);
  1361. }
  1362. static int __init eventpoll_init(void)
  1363. {
  1364. int error;
  1365. mutex_init(&epmutex);
  1366. /* Initialize the structure used to perform safe poll wait head wake ups */
  1367. ep_poll_safewake_init(&psw);
  1368. /* Allocates slab cache used to allocate "struct epitem" items */
  1369. epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),
  1370. 0, SLAB_HWCACHE_ALIGN|EPI_SLAB_DEBUG|SLAB_PANIC,
  1371. NULL, NULL);
  1372. /* Allocates slab cache used to allocate "struct eppoll_entry" */
  1373. pwq_cache = kmem_cache_create("eventpoll_pwq",
  1374. sizeof(struct eppoll_entry), 0,
  1375. EPI_SLAB_DEBUG|SLAB_PANIC, NULL, NULL);
  1376. /*
  1377. * Register the virtual file system that will be the source of inodes
  1378. * for the eventpoll files
  1379. */
  1380. error = register_filesystem(&eventpoll_fs_type);
  1381. if (error)
  1382. goto epanic;
  1383. /* Mount the above commented virtual file system */
  1384. eventpoll_mnt = kern_mount(&eventpoll_fs_type);
  1385. error = PTR_ERR(eventpoll_mnt);
  1386. if (IS_ERR(eventpoll_mnt))
  1387. goto epanic;
  1388. DNPRINTK(3, (KERN_INFO "[%p] eventpoll: successfully initialized.\n",
  1389. current));
  1390. return 0;
  1391. epanic:
  1392. panic("eventpoll_init() failed\n");
  1393. }
  1394. static void __exit eventpoll_exit(void)
  1395. {
  1396. /* Undo all operations done inside eventpoll_init() */
  1397. unregister_filesystem(&eventpoll_fs_type);
  1398. mntput(eventpoll_mnt);
  1399. kmem_cache_destroy(pwq_cache);
  1400. kmem_cache_destroy(epi_cache);
  1401. }
  1402. module_init(eventpoll_init);
  1403. module_exit(eventpoll_exit);
  1404. MODULE_LICENSE("GPL");