eventpoll.c 66 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * fs/eventpoll.c (Efficient event retrieval implementation)
  4. * Copyright (C) 2001,...,2009 Davide Libenzi
  5. *
  6. * Davide Libenzi <davidel@xmailserver.org>
  7. */
  8. #include <linux/init.h>
  9. #include <linux/kernel.h>
  10. #include <linux/sched/signal.h>
  11. #include <linux/fs.h>
  12. #include <linux/file.h>
  13. #include <linux/signal.h>
  14. #include <linux/errno.h>
  15. #include <linux/mm.h>
  16. #include <linux/slab.h>
  17. #include <linux/poll.h>
  18. #include <linux/string.h>
  19. #include <linux/list.h>
  20. #include <linux/hash.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/syscalls.h>
  23. #include <linux/rbtree.h>
  24. #include <linux/wait.h>
  25. #include <linux/eventpoll.h>
  26. #include <linux/mount.h>
  27. #include <linux/bitops.h>
  28. #include <linux/mutex.h>
  29. #include <linux/anon_inodes.h>
  30. #include <linux/device.h>
  31. #include <linux/freezer.h>
  32. #include <linux/uaccess.h>
  33. #include <asm/io.h>
  34. #include <asm/mman.h>
  35. #include <linux/atomic.h>
  36. #include <linux/proc_fs.h>
  37. #include <linux/seq_file.h>
  38. #include <linux/compat.h>
  39. #include <linux/rculist.h>
  40. #include <net/busy_poll.h>
  41. #include <trace/hooks/fs.h>
  42. /*
  43. * LOCKING:
  44. * There are three level of locking required by epoll :
  45. *
  46. * 1) epmutex (mutex)
  47. * 2) ep->mtx (mutex)
  48. * 3) ep->lock (rwlock)
  49. *
  50. * The acquire order is the one listed above, from 1 to 3.
  51. * We need a rwlock (ep->lock) because we manipulate objects
  52. * from inside the poll callback, that might be triggered from
  53. * a wake_up() that in turn might be called from IRQ context.
  54. * So we can't sleep inside the poll callback and hence we need
  55. * a spinlock. During the event transfer loop (from kernel to
  56. * user space) we could end up sleeping due a copy_to_user(), so
  57. * we need a lock that will allow us to sleep. This lock is a
  58. * mutex (ep->mtx). It is acquired during the event transfer loop,
  59. * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
  60. * Then we also need a global mutex to serialize eventpoll_release_file()
  61. * and ep_free().
  62. * This mutex is acquired by ep_free() during the epoll file
  63. * cleanup path and it is also acquired by eventpoll_release_file()
  64. * if a file has been pushed inside an epoll set and it is then
  65. * close()d without a previous call to epoll_ctl(EPOLL_CTL_DEL).
  66. * It is also acquired when inserting an epoll fd onto another epoll
  67. * fd. We do this so that we walk the epoll tree and ensure that this
  68. * insertion does not create a cycle of epoll file descriptors, which
  69. * could lead to deadlock. We need a global mutex to prevent two
  70. * simultaneous inserts (A into B and B into A) from racing and
  71. * constructing a cycle without either insert observing that it is
  72. * going to.
  73. * It is necessary to acquire multiple "ep->mtx"es at once in the
  74. * case when one epoll fd is added to another. In this case, we
  75. * always acquire the locks in the order of nesting (i.e. after
  76. * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
  77. * before e2->mtx). Since we disallow cycles of epoll file
  78. * descriptors, this ensures that the mutexes are well-ordered. In
  79. * order to communicate this nesting to lockdep, when walking a tree
  80. * of epoll file descriptors, we use the current recursion depth as
  81. * the lockdep subkey.
  82. * It is possible to drop the "ep->mtx" and to use the global
  83. * mutex "epmutex" (together with "ep->lock") to have it working,
  84. * but having "ep->mtx" will make the interface more scalable.
  85. * Events that require holding "epmutex" are very rare, while for
  86. * normal operations the epoll private "ep->mtx" will guarantee
  87. * a better scalability.
  88. */
  89. /* Epoll private bits inside the event mask */
  90. #define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
  91. #define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)
  92. #define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \
  93. EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)
  94. /* Maximum number of nesting allowed inside epoll sets */
  95. #define EP_MAX_NESTS 4
  96. #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
  97. #define EP_UNACTIVE_PTR ((void *) -1L)
  98. #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))
  99. struct epoll_filefd {
  100. struct file *file;
  101. int fd;
  102. } __packed;
  103. /*
  104. * Structure used to track possible nested calls, for too deep recursions
  105. * and loop cycles.
  106. */
  107. struct nested_call_node {
  108. struct list_head llink;
  109. void *cookie;
  110. void *ctx;
  111. };
  112. /*
  113. * This structure is used as collector for nested calls, to check for
  114. * maximum recursion dept and loop cycles.
  115. */
  116. struct nested_calls {
  117. struct list_head tasks_call_list;
  118. spinlock_t lock;
  119. };
  120. /*
  121. * Each file descriptor added to the eventpoll interface will
  122. * have an entry of this type linked to the "rbr" RB tree.
  123. * Avoid increasing the size of this struct, there can be many thousands
  124. * of these on a server and we do not want this to take another cache line.
  125. */
  126. struct epitem {
  127. union {
  128. /* RB tree node links this structure to the eventpoll RB tree */
  129. struct rb_node rbn;
  130. /* Used to free the struct epitem */
  131. struct rcu_head rcu;
  132. };
  133. /* List header used to link this structure to the eventpoll ready list */
  134. struct list_head rdllink;
  135. /*
  136. * Works together "struct eventpoll"->ovflist in keeping the
  137. * single linked chain of items.
  138. */
  139. struct epitem *next;
  140. /* The file descriptor information this item refers to */
  141. struct epoll_filefd ffd;
  142. /* Number of active wait queue attached to poll operations */
  143. int nwait;
  144. /* List containing poll wait queues */
  145. struct list_head pwqlist;
  146. /* The "container" of this item */
  147. struct eventpoll *ep;
  148. /* List header used to link this item to the "struct file" items list */
  149. struct list_head fllink;
  150. /* wakeup_source used when EPOLLWAKEUP is set */
  151. struct wakeup_source __rcu *ws;
  152. /* The structure that describe the interested events and the source fd */
  153. struct epoll_event event;
  154. };
  155. /*
  156. * This structure is stored inside the "private_data" member of the file
  157. * structure and represents the main data structure for the eventpoll
  158. * interface.
  159. */
  160. struct eventpoll {
  161. /*
  162. * This mutex is used to ensure that files are not removed
  163. * while epoll is using them. This is held during the event
  164. * collection loop, the file cleanup path, the epoll file exit
  165. * code and the ctl operations.
  166. */
  167. struct mutex mtx;
  168. /* Wait queue used by sys_epoll_wait() */
  169. wait_queue_head_t wq;
  170. /* Wait queue used by file->poll() */
  171. wait_queue_head_t poll_wait;
  172. /* List of ready file descriptors */
  173. struct list_head rdllist;
  174. /* Lock which protects rdllist and ovflist */
  175. rwlock_t lock;
  176. /* RB tree root used to store monitored fd structs */
  177. struct rb_root_cached rbr;
  178. /*
  179. * This is a single linked list that chains all the "struct epitem" that
  180. * happened while transferring ready events to userspace w/out
  181. * holding ->lock.
  182. */
  183. struct epitem *ovflist;
  184. /* wakeup_source used when ep_scan_ready_list is running */
  185. struct wakeup_source *ws;
  186. /* The user that created the eventpoll descriptor */
  187. struct user_struct *user;
  188. struct file *file;
  189. /* used to optimize loop detection check */
  190. u64 gen;
  191. #ifdef CONFIG_NET_RX_BUSY_POLL
  192. /* used to track busy poll napi_id */
  193. unsigned int napi_id;
  194. #endif
  195. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  196. /* tracks wakeup nests for lockdep validation */
  197. u8 nests;
  198. #endif
  199. };
  200. /* Wait structure used by the poll hooks */
  201. struct eppoll_entry {
  202. /* List header used to link this structure to the "struct epitem" */
  203. struct list_head llink;
  204. /* The "base" pointer is set to the container "struct epitem" */
  205. struct epitem *base;
  206. /*
  207. * Wait queue item that will be linked to the target file wait
  208. * queue head.
  209. */
  210. wait_queue_entry_t wait;
  211. /* The wait queue head that linked the "wait" wait queue item */
  212. wait_queue_head_t *whead;
  213. };
  214. /* Wrapper struct used by poll queueing */
  215. struct ep_pqueue {
  216. poll_table pt;
  217. struct epitem *epi;
  218. };
  219. /* Used by the ep_send_events() function as callback private data */
  220. struct ep_send_events_data {
  221. int maxevents;
  222. struct epoll_event __user *events;
  223. int res;
  224. };
  225. /*
  226. * Configuration options available inside /proc/sys/fs/epoll/
  227. */
  228. /* Maximum number of epoll watched descriptors, per user */
  229. static long max_user_watches __read_mostly;
  230. /*
  231. * This mutex is used to serialize ep_free() and eventpoll_release_file().
  232. */
  233. static DEFINE_MUTEX(epmutex);
  234. static u64 loop_check_gen = 0;
  235. /* Used to check for epoll file descriptor inclusion loops */
  236. static struct nested_calls poll_loop_ncalls;
  237. /* Slab cache used to allocate "struct epitem" */
  238. static struct kmem_cache *epi_cache __read_mostly;
  239. /* Slab cache used to allocate "struct eppoll_entry" */
  240. static struct kmem_cache *pwq_cache __read_mostly;
  241. /*
  242. * List of files with newly added links, where we may need to limit the number
  243. * of emanating paths. Protected by the epmutex.
  244. */
  245. static LIST_HEAD(tfile_check_list);
  246. #ifdef CONFIG_SYSCTL
  247. #include <linux/sysctl.h>
  248. static long long_zero;
  249. static long long_max = LONG_MAX;
  250. struct ctl_table epoll_table[] = {
  251. {
  252. .procname = "max_user_watches",
  253. .data = &max_user_watches,
  254. .maxlen = sizeof(max_user_watches),
  255. .mode = 0644,
  256. .proc_handler = proc_doulongvec_minmax,
  257. .extra1 = &long_zero,
  258. .extra2 = &long_max,
  259. },
  260. { }
  261. };
  262. #endif /* CONFIG_SYSCTL */
  263. static const struct file_operations eventpoll_fops;
  264. static inline int is_file_epoll(struct file *f)
  265. {
  266. return f->f_op == &eventpoll_fops;
  267. }
  268. /* Setup the structure that is used as key for the RB tree */
  269. static inline void ep_set_ffd(struct epoll_filefd *ffd,
  270. struct file *file, int fd)
  271. {
  272. ffd->file = file;
  273. ffd->fd = fd;
  274. }
  275. /* Compare RB tree keys */
  276. static inline int ep_cmp_ffd(struct epoll_filefd *p1,
  277. struct epoll_filefd *p2)
  278. {
  279. return (p1->file > p2->file ? +1:
  280. (p1->file < p2->file ? -1 : p1->fd - p2->fd));
  281. }
  282. /* Tells us if the item is currently linked */
  283. static inline int ep_is_linked(struct epitem *epi)
  284. {
  285. return !list_empty(&epi->rdllink);
  286. }
  287. static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)
  288. {
  289. return container_of(p, struct eppoll_entry, wait);
  290. }
  291. /* Get the "struct epitem" from a wait queue pointer */
  292. static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
  293. {
  294. return container_of(p, struct eppoll_entry, wait)->base;
  295. }
  296. /* Get the "struct epitem" from an epoll queue wrapper */
  297. static inline struct epitem *ep_item_from_epqueue(poll_table *p)
  298. {
  299. return container_of(p, struct ep_pqueue, pt)->epi;
  300. }
  301. /* Initialize the poll safe wake up structure */
  302. static void ep_nested_calls_init(struct nested_calls *ncalls)
  303. {
  304. INIT_LIST_HEAD(&ncalls->tasks_call_list);
  305. spin_lock_init(&ncalls->lock);
  306. }
  307. /**
  308. * ep_events_available - Checks if ready events might be available.
  309. *
  310. * @ep: Pointer to the eventpoll context.
  311. *
  312. * Returns: Returns a value different than zero if ready events are available,
  313. * or zero otherwise.
  314. */
  315. static inline int ep_events_available(struct eventpoll *ep)
  316. {
  317. return !list_empty_careful(&ep->rdllist) ||
  318. READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
  319. }
  320. #ifdef CONFIG_NET_RX_BUSY_POLL
  321. static bool ep_busy_loop_end(void *p, unsigned long start_time)
  322. {
  323. struct eventpoll *ep = p;
  324. return ep_events_available(ep) || busy_loop_timeout(start_time);
  325. }
  326. /*
  327. * Busy poll if globally on and supporting sockets found && no events,
  328. * busy loop will return if need_resched or ep_events_available.
  329. *
  330. * we must do our busy polling with irqs enabled
  331. */
  332. static void ep_busy_loop(struct eventpoll *ep, int nonblock)
  333. {
  334. unsigned int napi_id = READ_ONCE(ep->napi_id);
  335. if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())
  336. napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep);
  337. }
  338. static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
  339. {
  340. if (ep->napi_id)
  341. ep->napi_id = 0;
  342. }
  343. /*
  344. * Set epoll busy poll NAPI ID from sk.
  345. */
  346. static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
  347. {
  348. struct eventpoll *ep;
  349. unsigned int napi_id;
  350. struct socket *sock;
  351. struct sock *sk;
  352. int err;
  353. if (!net_busy_loop_on())
  354. return;
  355. sock = sock_from_file(epi->ffd.file, &err);
  356. if (!sock)
  357. return;
  358. sk = sock->sk;
  359. if (!sk)
  360. return;
  361. napi_id = READ_ONCE(sk->sk_napi_id);
  362. ep = epi->ep;
  363. /* Non-NAPI IDs can be rejected
  364. * or
  365. * Nothing to do if we already have this ID
  366. */
  367. if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)
  368. return;
  369. /* record NAPI ID for use in next busy poll */
  370. ep->napi_id = napi_id;
  371. }
  372. #else
  373. static inline void ep_busy_loop(struct eventpoll *ep, int nonblock)
  374. {
  375. }
  376. static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
  377. {
  378. }
  379. static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
  380. {
  381. }
  382. #endif /* CONFIG_NET_RX_BUSY_POLL */
  383. /**
  384. * ep_call_nested - Perform a bound (possibly) nested call, by checking
  385. * that the recursion limit is not exceeded, and that
  386. * the same nested call (by the meaning of same cookie) is
  387. * no re-entered.
  388. *
  389. * @ncalls: Pointer to the nested_calls structure to be used for this call.
  390. * @nproc: Nested call core function pointer.
  391. * @priv: Opaque data to be passed to the @nproc callback.
  392. * @cookie: Cookie to be used to identify this nested call.
  393. * @ctx: This instance context.
  394. *
  395. * Returns: Returns the code returned by the @nproc callback, or -1 if
  396. * the maximum recursion limit has been exceeded.
  397. */
  398. static int ep_call_nested(struct nested_calls *ncalls,
  399. int (*nproc)(void *, void *, int), void *priv,
  400. void *cookie, void *ctx)
  401. {
  402. int error, call_nests = 0;
  403. unsigned long flags;
  404. struct list_head *lsthead = &ncalls->tasks_call_list;
  405. struct nested_call_node *tncur;
  406. struct nested_call_node tnode;
  407. spin_lock_irqsave(&ncalls->lock, flags);
  408. /*
  409. * Try to see if the current task is already inside this wakeup call.
  410. * We use a list here, since the population inside this set is always
  411. * very much limited.
  412. */
  413. list_for_each_entry(tncur, lsthead, llink) {
  414. if (tncur->ctx == ctx &&
  415. (tncur->cookie == cookie || ++call_nests > EP_MAX_NESTS)) {
  416. /*
  417. * Ops ... loop detected or maximum nest level reached.
  418. * We abort this wake by breaking the cycle itself.
  419. */
  420. error = -1;
  421. goto out_unlock;
  422. }
  423. }
  424. /* Add the current task and cookie to the list */
  425. tnode.ctx = ctx;
  426. tnode.cookie = cookie;
  427. list_add(&tnode.llink, lsthead);
  428. spin_unlock_irqrestore(&ncalls->lock, flags);
  429. /* Call the nested function */
  430. error = (*nproc)(priv, cookie, call_nests);
  431. /* Remove the current task from the list */
  432. spin_lock_irqsave(&ncalls->lock, flags);
  433. list_del(&tnode.llink);
  434. out_unlock:
  435. spin_unlock_irqrestore(&ncalls->lock, flags);
  436. return error;
  437. }
  438. /*
  439. * As described in commit 0ccf831cb lockdep: annotate epoll
  440. * the use of wait queues used by epoll is done in a very controlled
  441. * manner. Wake ups can nest inside each other, but are never done
  442. * with the same locking. For example:
  443. *
  444. * dfd = socket(...);
  445. * efd1 = epoll_create();
  446. * efd2 = epoll_create();
  447. * epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);
  448. * epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);
  449. *
  450. * When a packet arrives to the device underneath "dfd", the net code will
  451. * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
  452. * callback wakeup entry on that queue, and the wake_up() performed by the
  453. * "dfd" net code will end up in ep_poll_callback(). At this point epoll
  454. * (efd1) notices that it may have some event ready, so it needs to wake up
  455. * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
  456. * that ends up in another wake_up(), after having checked about the
  457. * recursion constraints. That are, no more than EP_MAX_POLLWAKE_NESTS, to
  458. * avoid stack blasting.
  459. *
  460. * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle
  461. * this special case of epoll.
  462. */
  463. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  464. static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi)
  465. {
  466. struct eventpoll *ep_src;
  467. unsigned long flags;
  468. u8 nests = 0;
  469. /*
  470. * To set the subclass or nesting level for spin_lock_irqsave_nested()
  471. * it might be natural to create a per-cpu nest count. However, since
  472. * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can
  473. * schedule() in the -rt kernel, the per-cpu variable are no longer
  474. * protected. Thus, we are introducing a per eventpoll nest field.
  475. * If we are not being call from ep_poll_callback(), epi is NULL and
  476. * we are at the first level of nesting, 0. Otherwise, we are being
  477. * called from ep_poll_callback() and if a previous wakeup source is
  478. * not an epoll file itself, we are at depth 1 since the wakeup source
  479. * is depth 0. If the wakeup source is a previous epoll file in the
  480. * wakeup chain then we use its nests value and record ours as
  481. * nests + 1. The previous epoll file nests value is stable since its
  482. * already holding its own poll_wait.lock.
  483. */
  484. if (epi) {
  485. if ((is_file_epoll(epi->ffd.file))) {
  486. ep_src = epi->ffd.file->private_data;
  487. nests = ep_src->nests;
  488. } else {
  489. nests = 1;
  490. }
  491. }
  492. spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests);
  493. ep->nests = nests + 1;
  494. wake_up_locked_poll(&ep->poll_wait, EPOLLIN);
  495. ep->nests = 0;
  496. spin_unlock_irqrestore(&ep->poll_wait.lock, flags);
  497. }
  498. #else
  499. static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi)
  500. {
  501. wake_up_poll(&ep->poll_wait, EPOLLIN);
  502. }
  503. #endif
  504. static void ep_remove_wait_queue(struct eppoll_entry *pwq)
  505. {
  506. wait_queue_head_t *whead;
  507. rcu_read_lock();
  508. /*
  509. * If it is cleared by POLLFREE, it should be rcu-safe.
  510. * If we read NULL we need a barrier paired with
  511. * smp_store_release() in ep_poll_callback(), otherwise
  512. * we rely on whead->lock.
  513. */
  514. whead = smp_load_acquire(&pwq->whead);
  515. if (whead)
  516. remove_wait_queue(whead, &pwq->wait);
  517. rcu_read_unlock();
  518. }
  519. /*
  520. * This function unregisters poll callbacks from the associated file
  521. * descriptor. Must be called with "mtx" held (or "epmutex" if called from
  522. * ep_free).
  523. */
  524. static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
  525. {
  526. struct list_head *lsthead = &epi->pwqlist;
  527. struct eppoll_entry *pwq;
  528. while (!list_empty(lsthead)) {
  529. pwq = list_first_entry(lsthead, struct eppoll_entry, llink);
  530. list_del(&pwq->llink);
  531. ep_remove_wait_queue(pwq);
  532. kmem_cache_free(pwq_cache, pwq);
  533. }
  534. }
  535. /* call only when ep->mtx is held */
  536. static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)
  537. {
  538. return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));
  539. }
  540. /* call only when ep->mtx is held */
  541. static inline void ep_pm_stay_awake(struct epitem *epi)
  542. {
  543. struct wakeup_source *ws = ep_wakeup_source(epi);
  544. if (ws)
  545. __pm_stay_awake(ws);
  546. }
  547. static inline bool ep_has_wakeup_source(struct epitem *epi)
  548. {
  549. return rcu_access_pointer(epi->ws) ? true : false;
  550. }
  551. /* call when ep->mtx cannot be held (ep_poll_callback) */
  552. static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
  553. {
  554. struct wakeup_source *ws;
  555. rcu_read_lock();
  556. ws = rcu_dereference(epi->ws);
  557. if (ws)
  558. __pm_stay_awake(ws);
  559. rcu_read_unlock();
  560. }
  561. /**
  562. * ep_scan_ready_list - Scans the ready list in a way that makes possible for
  563. * the scan code, to call f_op->poll(). Also allows for
  564. * O(NumReady) performance.
  565. *
  566. * @ep: Pointer to the epoll private data structure.
  567. * @sproc: Pointer to the scan callback.
  568. * @priv: Private opaque data passed to the @sproc callback.
  569. * @depth: The current depth of recursive f_op->poll calls.
  570. * @ep_locked: caller already holds ep->mtx
  571. *
  572. * Returns: The same integer error code returned by the @sproc callback.
  573. */
  574. static __poll_t ep_scan_ready_list(struct eventpoll *ep,
  575. __poll_t (*sproc)(struct eventpoll *,
  576. struct list_head *, void *),
  577. void *priv, int depth, bool ep_locked)
  578. {
  579. __poll_t res;
  580. struct epitem *epi, *nepi;
  581. LIST_HEAD(txlist);
  582. lockdep_assert_irqs_enabled();
  583. /*
  584. * We need to lock this because we could be hit by
  585. * eventpoll_release_file() and epoll_ctl().
  586. */
  587. if (!ep_locked)
  588. mutex_lock_nested(&ep->mtx, depth);
  589. /*
  590. * Steal the ready list, and re-init the original one to the
  591. * empty list. Also, set ep->ovflist to NULL so that events
  592. * happening while looping w/out locks, are not lost. We cannot
  593. * have the poll callback to queue directly on ep->rdllist,
  594. * because we want the "sproc" callback to be able to do it
  595. * in a lockless way.
  596. */
  597. write_lock_irq(&ep->lock);
  598. list_splice_init(&ep->rdllist, &txlist);
  599. WRITE_ONCE(ep->ovflist, NULL);
  600. write_unlock_irq(&ep->lock);
  601. /*
  602. * Now call the callback function.
  603. */
  604. res = (*sproc)(ep, &txlist, priv);
  605. write_lock_irq(&ep->lock);
  606. /*
  607. * During the time we spent inside the "sproc" callback, some
  608. * other events might have been queued by the poll callback.
  609. * We re-insert them inside the main ready-list here.
  610. */
  611. for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
  612. nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
  613. /*
  614. * We need to check if the item is already in the list.
  615. * During the "sproc" callback execution time, items are
  616. * queued into ->ovflist but the "txlist" might already
  617. * contain them, and the list_splice() below takes care of them.
  618. */
  619. if (!ep_is_linked(epi)) {
  620. /*
  621. * ->ovflist is LIFO, so we have to reverse it in order
  622. * to keep in FIFO.
  623. */
  624. list_add(&epi->rdllink, &ep->rdllist);
  625. ep_pm_stay_awake(epi);
  626. }
  627. }
  628. /*
  629. * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
  630. * releasing the lock, events will be queued in the normal way inside
  631. * ep->rdllist.
  632. */
  633. WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
  634. /*
  635. * Quickly re-inject items left on "txlist".
  636. */
  637. list_splice(&txlist, &ep->rdllist);
  638. __pm_relax(ep->ws);
  639. if (!list_empty(&ep->rdllist)) {
  640. if (waitqueue_active(&ep->wq))
  641. wake_up(&ep->wq);
  642. }
  643. write_unlock_irq(&ep->lock);
  644. if (!ep_locked)
  645. mutex_unlock(&ep->mtx);
  646. return res;
  647. }
  648. static void epi_rcu_free(struct rcu_head *head)
  649. {
  650. struct epitem *epi = container_of(head, struct epitem, rcu);
  651. kmem_cache_free(epi_cache, epi);
  652. }
  653. /*
  654. * Removes a "struct epitem" from the eventpoll RB tree and deallocates
  655. * all the associated resources. Must be called with "mtx" held.
  656. */
  657. static int ep_remove(struct eventpoll *ep, struct epitem *epi)
  658. {
  659. struct file *file = epi->ffd.file;
  660. lockdep_assert_irqs_enabled();
  661. /*
  662. * Removes poll wait queue hooks.
  663. */
  664. ep_unregister_pollwait(ep, epi);
  665. /* Remove the current item from the list of epoll hooks */
  666. spin_lock(&file->f_lock);
  667. list_del_rcu(&epi->fllink);
  668. spin_unlock(&file->f_lock);
  669. rb_erase_cached(&epi->rbn, &ep->rbr);
  670. write_lock_irq(&ep->lock);
  671. if (ep_is_linked(epi))
  672. list_del_init(&epi->rdllink);
  673. write_unlock_irq(&ep->lock);
  674. wakeup_source_unregister(ep_wakeup_source(epi));
  675. /*
  676. * At this point it is safe to free the eventpoll item. Use the union
  677. * field epi->rcu, since we are trying to minimize the size of
  678. * 'struct epitem'. The 'rbn' field is no longer in use. Protected by
  679. * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
  680. * use of the rbn field.
  681. */
  682. call_rcu(&epi->rcu, epi_rcu_free);
  683. atomic_long_dec(&ep->user->epoll_watches);
  684. return 0;
  685. }
  686. static void ep_free(struct eventpoll *ep)
  687. {
  688. struct rb_node *rbp;
  689. struct epitem *epi;
  690. /* We need to release all tasks waiting for these file */
  691. if (waitqueue_active(&ep->poll_wait))
  692. ep_poll_safewake(ep, NULL);
  693. /*
  694. * We need to lock this because we could be hit by
  695. * eventpoll_release_file() while we're freeing the "struct eventpoll".
  696. * We do not need to hold "ep->mtx" here because the epoll file
  697. * is on the way to be removed and no one has references to it
  698. * anymore. The only hit might come from eventpoll_release_file() but
  699. * holding "epmutex" is sufficient here.
  700. */
  701. mutex_lock(&epmutex);
  702. /*
  703. * Walks through the whole tree by unregistering poll callbacks.
  704. */
  705. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  706. epi = rb_entry(rbp, struct epitem, rbn);
  707. ep_unregister_pollwait(ep, epi);
  708. cond_resched();
  709. }
  710. /*
  711. * Walks through the whole tree by freeing each "struct epitem". At this
  712. * point we are sure no poll callbacks will be lingering around, and also by
  713. * holding "epmutex" we can be sure that no file cleanup code will hit
  714. * us during this operation. So we can avoid the lock on "ep->lock".
  715. * We do not need to lock ep->mtx, either, we only do it to prevent
  716. * a lockdep warning.
  717. */
  718. mutex_lock(&ep->mtx);
  719. while ((rbp = rb_first_cached(&ep->rbr)) != NULL) {
  720. epi = rb_entry(rbp, struct epitem, rbn);
  721. ep_remove(ep, epi);
  722. cond_resched();
  723. }
  724. mutex_unlock(&ep->mtx);
  725. mutex_unlock(&epmutex);
  726. mutex_destroy(&ep->mtx);
  727. free_uid(ep->user);
  728. wakeup_source_unregister(ep->ws);
  729. kfree(ep);
  730. }
  731. static int ep_eventpoll_release(struct inode *inode, struct file *file)
  732. {
  733. struct eventpoll *ep = file->private_data;
  734. if (ep)
  735. ep_free(ep);
  736. return 0;
  737. }
  738. static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
  739. void *priv);
  740. static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
  741. poll_table *pt);
  742. /*
  743. * Differs from ep_eventpoll_poll() in that internal callers already have
  744. * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
  745. * is correctly annotated.
  746. */
  747. static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,
  748. int depth)
  749. {
  750. struct eventpoll *ep;
  751. bool locked;
  752. pt->_key = epi->event.events;
  753. if (!is_file_epoll(epi->ffd.file))
  754. return vfs_poll(epi->ffd.file, pt) & epi->event.events;
  755. ep = epi->ffd.file->private_data;
  756. poll_wait(epi->ffd.file, &ep->poll_wait, pt);
  757. locked = pt && (pt->_qproc == ep_ptable_queue_proc);
  758. return ep_scan_ready_list(epi->ffd.file->private_data,
  759. ep_read_events_proc, &depth, depth,
  760. locked) & epi->event.events;
  761. }
  762. static __poll_t ep_read_events_proc(struct eventpoll *ep, struct list_head *head,
  763. void *priv)
  764. {
  765. struct epitem *epi, *tmp;
  766. poll_table pt;
  767. int depth = *(int *)priv;
  768. init_poll_funcptr(&pt, NULL);
  769. depth++;
  770. list_for_each_entry_safe(epi, tmp, head, rdllink) {
  771. if (ep_item_poll(epi, &pt, depth)) {
  772. return EPOLLIN | EPOLLRDNORM;
  773. } else {
  774. /*
  775. * Item has been dropped into the ready list by the poll
  776. * callback, but it's not actually ready, as far as
  777. * caller requested events goes. We can remove it here.
  778. */
  779. __pm_relax(ep_wakeup_source(epi));
  780. list_del_init(&epi->rdllink);
  781. }
  782. }
  783. return 0;
  784. }
  785. static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
  786. {
  787. struct eventpoll *ep = file->private_data;
  788. int depth = 0;
  789. /* Insert inside our poll wait queue */
  790. poll_wait(file, &ep->poll_wait, wait);
  791. /*
  792. * Proceed to find out if wanted events are really available inside
  793. * the ready list.
  794. */
  795. return ep_scan_ready_list(ep, ep_read_events_proc,
  796. &depth, depth, false);
  797. }
  798. #ifdef CONFIG_PROC_FS
  799. static void ep_show_fdinfo(struct seq_file *m, struct file *f)
  800. {
  801. struct eventpoll *ep = f->private_data;
  802. struct rb_node *rbp;
  803. mutex_lock(&ep->mtx);
  804. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  805. struct epitem *epi = rb_entry(rbp, struct epitem, rbn);
  806. struct inode *inode = file_inode(epi->ffd.file);
  807. seq_printf(m, "tfd: %8d events: %8x data: %16llx "
  808. " pos:%lli ino:%lx sdev:%x\n",
  809. epi->ffd.fd, epi->event.events,
  810. (long long)epi->event.data,
  811. (long long)epi->ffd.file->f_pos,
  812. inode->i_ino, inode->i_sb->s_dev);
  813. if (seq_has_overflowed(m))
  814. break;
  815. }
  816. mutex_unlock(&ep->mtx);
  817. }
  818. #endif
  819. /* File callbacks that implement the eventpoll file behaviour */
  820. static const struct file_operations eventpoll_fops = {
  821. #ifdef CONFIG_PROC_FS
  822. .show_fdinfo = ep_show_fdinfo,
  823. #endif
  824. .release = ep_eventpoll_release,
  825. .poll = ep_eventpoll_poll,
  826. .llseek = noop_llseek,
  827. };
  828. /*
  829. * This is called from eventpoll_release() to unlink files from the eventpoll
  830. * interface. We need to have this facility to cleanup correctly files that are
  831. * closed without being removed from the eventpoll interface.
  832. */
  833. void eventpoll_release_file(struct file *file)
  834. {
  835. struct eventpoll *ep;
  836. struct epitem *epi, *next;
  837. /*
  838. * We don't want to get "file->f_lock" because it is not
  839. * necessary. It is not necessary because we're in the "struct file"
  840. * cleanup path, and this means that no one is using this file anymore.
  841. * So, for example, epoll_ctl() cannot hit here since if we reach this
  842. * point, the file counter already went to zero and fget() would fail.
  843. * The only hit might come from ep_free() but by holding the mutex
  844. * will correctly serialize the operation. We do need to acquire
  845. * "ep->mtx" after "epmutex" because ep_remove() requires it when called
  846. * from anywhere but ep_free().
  847. *
  848. * Besides, ep_remove() acquires the lock, so we can't hold it here.
  849. */
  850. mutex_lock(&epmutex);
  851. list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {
  852. ep = epi->ep;
  853. mutex_lock_nested(&ep->mtx, 0);
  854. ep_remove(ep, epi);
  855. mutex_unlock(&ep->mtx);
  856. }
  857. mutex_unlock(&epmutex);
  858. }
  859. static int ep_alloc(struct eventpoll **pep)
  860. {
  861. int error;
  862. struct user_struct *user;
  863. struct eventpoll *ep;
  864. user = get_current_user();
  865. error = -ENOMEM;
  866. ep = kzalloc(sizeof(*ep), GFP_KERNEL);
  867. if (unlikely(!ep))
  868. goto free_uid;
  869. mutex_init(&ep->mtx);
  870. rwlock_init(&ep->lock);
  871. init_waitqueue_head(&ep->wq);
  872. init_waitqueue_head(&ep->poll_wait);
  873. INIT_LIST_HEAD(&ep->rdllist);
  874. ep->rbr = RB_ROOT_CACHED;
  875. ep->ovflist = EP_UNACTIVE_PTR;
  876. ep->user = user;
  877. *pep = ep;
  878. return 0;
  879. free_uid:
  880. free_uid(user);
  881. return error;
  882. }
  883. /*
  884. * Search the file inside the eventpoll tree. The RB tree operations
  885. * are protected by the "mtx" mutex, and ep_find() must be called with
  886. * "mtx" held.
  887. */
  888. static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
  889. {
  890. int kcmp;
  891. struct rb_node *rbp;
  892. struct epitem *epi, *epir = NULL;
  893. struct epoll_filefd ffd;
  894. ep_set_ffd(&ffd, file, fd);
  895. for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
  896. epi = rb_entry(rbp, struct epitem, rbn);
  897. kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
  898. if (kcmp > 0)
  899. rbp = rbp->rb_right;
  900. else if (kcmp < 0)
  901. rbp = rbp->rb_left;
  902. else {
  903. epir = epi;
  904. break;
  905. }
  906. }
  907. return epir;
  908. }
  909. #ifdef CONFIG_KCMP
  910. static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
  911. {
  912. struct rb_node *rbp;
  913. struct epitem *epi;
  914. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  915. epi = rb_entry(rbp, struct epitem, rbn);
  916. if (epi->ffd.fd == tfd) {
  917. if (toff == 0)
  918. return epi;
  919. else
  920. toff--;
  921. }
  922. cond_resched();
  923. }
  924. return NULL;
  925. }
  926. struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
  927. unsigned long toff)
  928. {
  929. struct file *file_raw;
  930. struct eventpoll *ep;
  931. struct epitem *epi;
  932. if (!is_file_epoll(file))
  933. return ERR_PTR(-EINVAL);
  934. ep = file->private_data;
  935. mutex_lock(&ep->mtx);
  936. epi = ep_find_tfd(ep, tfd, toff);
  937. if (epi)
  938. file_raw = epi->ffd.file;
  939. else
  940. file_raw = ERR_PTR(-ENOENT);
  941. mutex_unlock(&ep->mtx);
  942. return file_raw;
  943. }
  944. #endif /* CONFIG_KCMP */
  945. /**
  946. * Adds a new entry to the tail of the list in a lockless way, i.e.
  947. * multiple CPUs are allowed to call this function concurrently.
  948. *
  949. * Beware: it is necessary to prevent any other modifications of the
  950. * existing list until all changes are completed, in other words
  951. * concurrent list_add_tail_lockless() calls should be protected
  952. * with a read lock, where write lock acts as a barrier which
  953. * makes sure all list_add_tail_lockless() calls are fully
  954. * completed.
  955. *
  956. * Also an element can be locklessly added to the list only in one
  957. * direction i.e. either to the tail either to the head, otherwise
  958. * concurrent access will corrupt the list.
  959. *
  960. * Returns %false if element has been already added to the list, %true
  961. * otherwise.
  962. */
  963. static inline bool list_add_tail_lockless(struct list_head *new,
  964. struct list_head *head)
  965. {
  966. struct list_head *prev;
  967. /*
  968. * This is simple 'new->next = head' operation, but cmpxchg()
  969. * is used in order to detect that same element has been just
  970. * added to the list from another CPU: the winner observes
  971. * new->next == new.
  972. */
  973. if (cmpxchg(&new->next, new, head) != new)
  974. return false;
  975. /*
  976. * Initially ->next of a new element must be updated with the head
  977. * (we are inserting to the tail) and only then pointers are atomically
  978. * exchanged. XCHG guarantees memory ordering, thus ->next should be
  979. * updated before pointers are actually swapped and pointers are
  980. * swapped before prev->next is updated.
  981. */
  982. prev = xchg(&head->prev, new);
  983. /*
  984. * It is safe to modify prev->next and new->prev, because a new element
  985. * is added only to the tail and new->next is updated before XCHG.
  986. */
  987. prev->next = new;
  988. new->prev = prev;
  989. return true;
  990. }
  991. /**
  992. * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
  993. * i.e. multiple CPUs are allowed to call this function concurrently.
  994. *
  995. * Returns %false if epi element has been already chained, %true otherwise.
  996. */
  997. static inline bool chain_epi_lockless(struct epitem *epi)
  998. {
  999. struct eventpoll *ep = epi->ep;
  1000. /* Fast preliminary check */
  1001. if (epi->next != EP_UNACTIVE_PTR)
  1002. return false;
  1003. /* Check that the same epi has not been just chained from another CPU */
  1004. if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)
  1005. return false;
  1006. /* Atomically exchange tail */
  1007. epi->next = xchg(&ep->ovflist, epi);
  1008. return true;
  1009. }
  1010. /*
  1011. * This is the callback that is passed to the wait queue wakeup
  1012. * mechanism. It is called by the stored file descriptors when they
  1013. * have events to report.
  1014. *
  1015. * This callback takes a read lock in order not to content with concurrent
  1016. * events from another file descriptors, thus all modifications to ->rdllist
  1017. * or ->ovflist are lockless. Read lock is paired with the write lock from
  1018. * ep_scan_ready_list(), which stops all list modifications and guarantees
  1019. * that lists state is seen correctly.
  1020. *
  1021. * Another thing worth to mention is that ep_poll_callback() can be called
  1022. * concurrently for the same @epi from different CPUs if poll table was inited
  1023. * with several wait queues entries. Plural wakeup from different CPUs of a
  1024. * single wait queue is serialized by wq.lock, but the case when multiple wait
  1025. * queues are used should be detected accordingly. This is detected using
  1026. * cmpxchg() operation.
  1027. */
  1028. static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  1029. {
  1030. int pwake = 0;
  1031. struct epitem *epi = ep_item_from_wait(wait);
  1032. struct eventpoll *ep = epi->ep;
  1033. __poll_t pollflags = key_to_poll(key);
  1034. unsigned long flags;
  1035. int ewake = 0;
  1036. read_lock_irqsave(&ep->lock, flags);
  1037. ep_set_busy_poll_napi_id(epi);
  1038. /*
  1039. * If the event mask does not contain any poll(2) event, we consider the
  1040. * descriptor to be disabled. This condition is likely the effect of the
  1041. * EPOLLONESHOT bit that disables the descriptor when an event is received,
  1042. * until the next EPOLL_CTL_MOD will be issued.
  1043. */
  1044. if (!(epi->event.events & ~EP_PRIVATE_BITS))
  1045. goto out_unlock;
  1046. /*
  1047. * Check the events coming with the callback. At this stage, not
  1048. * every device reports the events in the "key" parameter of the
  1049. * callback. We need to be able to handle both cases here, hence the
  1050. * test for "key" != NULL before the event match test.
  1051. */
  1052. if (pollflags && !(pollflags & epi->event.events))
  1053. goto out_unlock;
  1054. /*
  1055. * If we are transferring events to userspace, we can hold no locks
  1056. * (because we're accessing user memory, and because of linux f_op->poll()
  1057. * semantics). All the events that happen during that period of time are
  1058. * chained in ep->ovflist and requeued later on.
  1059. */
  1060. if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {
  1061. if (chain_epi_lockless(epi))
  1062. ep_pm_stay_awake_rcu(epi);
  1063. } else if (!ep_is_linked(epi)) {
  1064. /* In the usual case, add event to ready list. */
  1065. if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))
  1066. ep_pm_stay_awake_rcu(epi);
  1067. }
  1068. /*
  1069. * Wake up ( if active ) both the eventpoll wait list and the ->poll()
  1070. * wait list.
  1071. */
  1072. if (waitqueue_active(&ep->wq)) {
  1073. if ((epi->event.events & EPOLLEXCLUSIVE) &&
  1074. !(pollflags & POLLFREE)) {
  1075. switch (pollflags & EPOLLINOUT_BITS) {
  1076. case EPOLLIN:
  1077. if (epi->event.events & EPOLLIN)
  1078. ewake = 1;
  1079. break;
  1080. case EPOLLOUT:
  1081. if (epi->event.events & EPOLLOUT)
  1082. ewake = 1;
  1083. break;
  1084. case 0:
  1085. ewake = 1;
  1086. break;
  1087. }
  1088. }
  1089. wake_up(&ep->wq);
  1090. }
  1091. if (waitqueue_active(&ep->poll_wait))
  1092. pwake++;
  1093. out_unlock:
  1094. read_unlock_irqrestore(&ep->lock, flags);
  1095. /* We have to call this outside the lock */
  1096. if (pwake)
  1097. ep_poll_safewake(ep, epi);
  1098. if (!(epi->event.events & EPOLLEXCLUSIVE))
  1099. ewake = 1;
  1100. if (pollflags & POLLFREE) {
  1101. /*
  1102. * If we race with ep_remove_wait_queue() it can miss
  1103. * ->whead = NULL and do another remove_wait_queue() after
  1104. * us, so we can't use __remove_wait_queue().
  1105. */
  1106. list_del_init(&wait->entry);
  1107. /*
  1108. * ->whead != NULL protects us from the race with ep_free()
  1109. * or ep_remove(), ep_remove_wait_queue() takes whead->lock
  1110. * held by the caller. Once we nullify it, nothing protects
  1111. * ep/epi or even wait.
  1112. */
  1113. smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
  1114. }
  1115. return ewake;
  1116. }
  1117. /*
  1118. * This is the callback that is used to add our wait queue to the
  1119. * target file wakeup lists.
  1120. */
  1121. static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
  1122. poll_table *pt)
  1123. {
  1124. struct epitem *epi = ep_item_from_epqueue(pt);
  1125. struct eppoll_entry *pwq;
  1126. if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {
  1127. init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
  1128. pwq->whead = whead;
  1129. pwq->base = epi;
  1130. if (epi->event.events & EPOLLEXCLUSIVE)
  1131. add_wait_queue_exclusive(whead, &pwq->wait);
  1132. else
  1133. add_wait_queue(whead, &pwq->wait);
  1134. list_add_tail(&pwq->llink, &epi->pwqlist);
  1135. epi->nwait++;
  1136. } else {
  1137. /* We have to signal that an error occurred */
  1138. epi->nwait = -1;
  1139. }
  1140. }
  1141. static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
  1142. {
  1143. int kcmp;
  1144. struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL;
  1145. struct epitem *epic;
  1146. bool leftmost = true;
  1147. while (*p) {
  1148. parent = *p;
  1149. epic = rb_entry(parent, struct epitem, rbn);
  1150. kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);
  1151. if (kcmp > 0) {
  1152. p = &parent->rb_right;
  1153. leftmost = false;
  1154. } else
  1155. p = &parent->rb_left;
  1156. }
  1157. rb_link_node(&epi->rbn, parent, p);
  1158. rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);
  1159. }
  1160. #define PATH_ARR_SIZE 5
  1161. /*
  1162. * These are the number paths of length 1 to 5, that we are allowing to emanate
  1163. * from a single file of interest. For example, we allow 1000 paths of length
  1164. * 1, to emanate from each file of interest. This essentially represents the
  1165. * potential wakeup paths, which need to be limited in order to avoid massive
  1166. * uncontrolled wakeup storms. The common use case should be a single ep which
  1167. * is connected to n file sources. In this case each file source has 1 path
  1168. * of length 1. Thus, the numbers below should be more than sufficient. These
  1169. * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify
  1170. * and delete can't add additional paths. Protected by the epmutex.
  1171. */
  1172. static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };
  1173. static int path_count[PATH_ARR_SIZE];
  1174. static int path_count_inc(int nests)
  1175. {
  1176. /* Allow an arbitrary number of depth 1 paths */
  1177. if (nests == 0)
  1178. return 0;
  1179. if (++path_count[nests] > path_limits[nests])
  1180. return -1;
  1181. return 0;
  1182. }
  1183. static void path_count_init(void)
  1184. {
  1185. int i;
  1186. for (i = 0; i < PATH_ARR_SIZE; i++)
  1187. path_count[i] = 0;
  1188. }
  1189. static int reverse_path_check_proc(void *priv, void *cookie, int call_nests)
  1190. {
  1191. int error = 0;
  1192. struct file *file = priv;
  1193. struct file *child_file;
  1194. struct epitem *epi;
  1195. /* CTL_DEL can remove links here, but that can't increase our count */
  1196. rcu_read_lock();
  1197. list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) {
  1198. child_file = epi->ep->file;
  1199. if (is_file_epoll(child_file)) {
  1200. if (list_empty(&child_file->f_ep_links)) {
  1201. if (path_count_inc(call_nests)) {
  1202. error = -1;
  1203. break;
  1204. }
  1205. } else {
  1206. error = ep_call_nested(&poll_loop_ncalls,
  1207. reverse_path_check_proc,
  1208. child_file, child_file,
  1209. current);
  1210. }
  1211. if (error != 0)
  1212. break;
  1213. } else {
  1214. printk(KERN_ERR "reverse_path_check_proc: "
  1215. "file is not an ep!\n");
  1216. }
  1217. }
  1218. rcu_read_unlock();
  1219. return error;
  1220. }
  1221. /**
  1222. * reverse_path_check - The tfile_check_list is list of file *, which have
  1223. * links that are proposed to be newly added. We need to
  1224. * make sure that those added links don't add too many
  1225. * paths such that we will spend all our time waking up
  1226. * eventpoll objects.
  1227. *
  1228. * Returns: Returns zero if the proposed links don't create too many paths,
  1229. * -1 otherwise.
  1230. */
  1231. static int reverse_path_check(void)
  1232. {
  1233. int error = 0;
  1234. struct file *current_file;
  1235. /* let's call this for all tfiles */
  1236. list_for_each_entry(current_file, &tfile_check_list, f_tfile_llink) {
  1237. path_count_init();
  1238. error = ep_call_nested(&poll_loop_ncalls,
  1239. reverse_path_check_proc, current_file,
  1240. current_file, current);
  1241. if (error)
  1242. break;
  1243. }
  1244. return error;
  1245. }
  1246. static int ep_create_wakeup_source(struct epitem *epi)
  1247. {
  1248. struct name_snapshot n;
  1249. struct wakeup_source *ws;
  1250. char ws_name[64];
  1251. strlcpy(ws_name, "eventpoll", sizeof(ws_name));
  1252. trace_android_vh_ep_create_wakeup_source(ws_name, sizeof(ws_name));
  1253. if (!epi->ep->ws) {
  1254. epi->ep->ws = wakeup_source_register(NULL, ws_name);
  1255. if (!epi->ep->ws)
  1256. return -ENOMEM;
  1257. }
  1258. take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);
  1259. strlcpy(ws_name, n.name.name, sizeof(ws_name));
  1260. trace_android_vh_ep_create_wakeup_source(ws_name, sizeof(ws_name));
  1261. ws = wakeup_source_register(NULL, ws_name);
  1262. release_dentry_name_snapshot(&n);
  1263. if (!ws)
  1264. return -ENOMEM;
  1265. rcu_assign_pointer(epi->ws, ws);
  1266. return 0;
  1267. }
  1268. /* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */
  1269. static noinline void ep_destroy_wakeup_source(struct epitem *epi)
  1270. {
  1271. struct wakeup_source *ws = ep_wakeup_source(epi);
  1272. RCU_INIT_POINTER(epi->ws, NULL);
  1273. /*
  1274. * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is
  1275. * used internally by wakeup_source_remove, too (called by
  1276. * wakeup_source_unregister), so we cannot use call_rcu
  1277. */
  1278. synchronize_rcu();
  1279. wakeup_source_unregister(ws);
  1280. }
  1281. /*
  1282. * Must be called with "mtx" held.
  1283. */
  1284. static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
  1285. struct file *tfile, int fd, int full_check)
  1286. {
  1287. int error, pwake = 0;
  1288. __poll_t revents;
  1289. long user_watches;
  1290. struct epitem *epi;
  1291. struct ep_pqueue epq;
  1292. lockdep_assert_irqs_enabled();
  1293. user_watches = atomic_long_read(&ep->user->epoll_watches);
  1294. if (unlikely(user_watches >= max_user_watches))
  1295. return -ENOSPC;
  1296. if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))
  1297. return -ENOMEM;
  1298. /* Item initialization follow here ... */
  1299. INIT_LIST_HEAD(&epi->rdllink);
  1300. INIT_LIST_HEAD(&epi->fllink);
  1301. INIT_LIST_HEAD(&epi->pwqlist);
  1302. epi->ep = ep;
  1303. ep_set_ffd(&epi->ffd, tfile, fd);
  1304. epi->event = *event;
  1305. epi->nwait = 0;
  1306. epi->next = EP_UNACTIVE_PTR;
  1307. if (epi->event.events & EPOLLWAKEUP) {
  1308. error = ep_create_wakeup_source(epi);
  1309. if (error)
  1310. goto error_create_wakeup_source;
  1311. } else {
  1312. RCU_INIT_POINTER(epi->ws, NULL);
  1313. }
  1314. /* Add the current item to the list of active epoll hook for this file */
  1315. spin_lock(&tfile->f_lock);
  1316. list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);
  1317. spin_unlock(&tfile->f_lock);
  1318. /*
  1319. * Add the current item to the RB tree. All RB tree operations are
  1320. * protected by "mtx", and ep_insert() is called with "mtx" held.
  1321. */
  1322. ep_rbtree_insert(ep, epi);
  1323. /* now check if we've created too many backpaths */
  1324. error = -EINVAL;
  1325. if (full_check && reverse_path_check())
  1326. goto error_remove_epi;
  1327. /* Initialize the poll table using the queue callback */
  1328. epq.epi = epi;
  1329. init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
  1330. /*
  1331. * Attach the item to the poll hooks and get current event bits.
  1332. * We can safely use the file* here because its usage count has
  1333. * been increased by the caller of this function. Note that after
  1334. * this operation completes, the poll callback can start hitting
  1335. * the new item.
  1336. */
  1337. revents = ep_item_poll(epi, &epq.pt, 1);
  1338. /*
  1339. * We have to check if something went wrong during the poll wait queue
  1340. * install process. Namely an allocation for a wait queue failed due
  1341. * high memory pressure.
  1342. */
  1343. error = -ENOMEM;
  1344. if (epi->nwait < 0)
  1345. goto error_unregister;
  1346. /* We have to drop the new item inside our item list to keep track of it */
  1347. write_lock_irq(&ep->lock);
  1348. /* record NAPI ID of new item if present */
  1349. ep_set_busy_poll_napi_id(epi);
  1350. /* If the file is already "ready" we drop it inside the ready list */
  1351. if (revents && !ep_is_linked(epi)) {
  1352. list_add_tail(&epi->rdllink, &ep->rdllist);
  1353. ep_pm_stay_awake(epi);
  1354. /* Notify waiting tasks that events are available */
  1355. if (waitqueue_active(&ep->wq))
  1356. wake_up(&ep->wq);
  1357. if (waitqueue_active(&ep->poll_wait))
  1358. pwake++;
  1359. }
  1360. write_unlock_irq(&ep->lock);
  1361. atomic_long_inc(&ep->user->epoll_watches);
  1362. /* We have to call this outside the lock */
  1363. if (pwake)
  1364. ep_poll_safewake(ep, NULL);
  1365. return 0;
  1366. error_unregister:
  1367. ep_unregister_pollwait(ep, epi);
  1368. error_remove_epi:
  1369. spin_lock(&tfile->f_lock);
  1370. list_del_rcu(&epi->fllink);
  1371. spin_unlock(&tfile->f_lock);
  1372. rb_erase_cached(&epi->rbn, &ep->rbr);
  1373. /*
  1374. * We need to do this because an event could have been arrived on some
  1375. * allocated wait queue. Note that we don't care about the ep->ovflist
  1376. * list, since that is used/cleaned only inside a section bound by "mtx".
  1377. * And ep_insert() is called with "mtx" held.
  1378. */
  1379. write_lock_irq(&ep->lock);
  1380. if (ep_is_linked(epi))
  1381. list_del_init(&epi->rdllink);
  1382. write_unlock_irq(&ep->lock);
  1383. wakeup_source_unregister(ep_wakeup_source(epi));
  1384. error_create_wakeup_source:
  1385. kmem_cache_free(epi_cache, epi);
  1386. return error;
  1387. }
  1388. /*
  1389. * Modify the interest event mask by dropping an event if the new mask
  1390. * has a match in the current file status. Must be called with "mtx" held.
  1391. */
  1392. static int ep_modify(struct eventpoll *ep, struct epitem *epi,
  1393. const struct epoll_event *event)
  1394. {
  1395. int pwake = 0;
  1396. poll_table pt;
  1397. lockdep_assert_irqs_enabled();
  1398. init_poll_funcptr(&pt, NULL);
  1399. /*
  1400. * Set the new event interest mask before calling f_op->poll();
  1401. * otherwise we might miss an event that happens between the
  1402. * f_op->poll() call and the new event set registering.
  1403. */
  1404. epi->event.events = event->events; /* need barrier below */
  1405. epi->event.data = event->data; /* protected by mtx */
  1406. if (epi->event.events & EPOLLWAKEUP) {
  1407. if (!ep_has_wakeup_source(epi))
  1408. ep_create_wakeup_source(epi);
  1409. } else if (ep_has_wakeup_source(epi)) {
  1410. ep_destroy_wakeup_source(epi);
  1411. }
  1412. /*
  1413. * The following barrier has two effects:
  1414. *
  1415. * 1) Flush epi changes above to other CPUs. This ensures
  1416. * we do not miss events from ep_poll_callback if an
  1417. * event occurs immediately after we call f_op->poll().
  1418. * We need this because we did not take ep->lock while
  1419. * changing epi above (but ep_poll_callback does take
  1420. * ep->lock).
  1421. *
  1422. * 2) We also need to ensure we do not miss _past_ events
  1423. * when calling f_op->poll(). This barrier also
  1424. * pairs with the barrier in wq_has_sleeper (see
  1425. * comments for wq_has_sleeper).
  1426. *
  1427. * This barrier will now guarantee ep_poll_callback or f_op->poll
  1428. * (or both) will notice the readiness of an item.
  1429. */
  1430. smp_mb();
  1431. /*
  1432. * Get current event bits. We can safely use the file* here because
  1433. * its usage count has been increased by the caller of this function.
  1434. * If the item is "hot" and it is not registered inside the ready
  1435. * list, push it inside.
  1436. */
  1437. if (ep_item_poll(epi, &pt, 1)) {
  1438. write_lock_irq(&ep->lock);
  1439. if (!ep_is_linked(epi)) {
  1440. list_add_tail(&epi->rdllink, &ep->rdllist);
  1441. ep_pm_stay_awake(epi);
  1442. /* Notify waiting tasks that events are available */
  1443. if (waitqueue_active(&ep->wq))
  1444. wake_up(&ep->wq);
  1445. if (waitqueue_active(&ep->poll_wait))
  1446. pwake++;
  1447. }
  1448. write_unlock_irq(&ep->lock);
  1449. }
  1450. /* We have to call this outside the lock */
  1451. if (pwake)
  1452. ep_poll_safewake(ep, NULL);
  1453. return 0;
  1454. }
  1455. static __poll_t ep_send_events_proc(struct eventpoll *ep, struct list_head *head,
  1456. void *priv)
  1457. {
  1458. struct ep_send_events_data *esed = priv;
  1459. __poll_t revents;
  1460. struct epitem *epi, *tmp;
  1461. struct epoll_event __user *uevent = esed->events;
  1462. struct wakeup_source *ws;
  1463. poll_table pt;
  1464. init_poll_funcptr(&pt, NULL);
  1465. esed->res = 0;
  1466. /*
  1467. * We can loop without lock because we are passed a task private list.
  1468. * Items cannot vanish during the loop because ep_scan_ready_list() is
  1469. * holding "mtx" during this call.
  1470. */
  1471. lockdep_assert_held(&ep->mtx);
  1472. list_for_each_entry_safe(epi, tmp, head, rdllink) {
  1473. if (esed->res >= esed->maxevents)
  1474. break;
  1475. /*
  1476. * Activate ep->ws before deactivating epi->ws to prevent
  1477. * triggering auto-suspend here (in case we reactive epi->ws
  1478. * below).
  1479. *
  1480. * This could be rearranged to delay the deactivation of epi->ws
  1481. * instead, but then epi->ws would temporarily be out of sync
  1482. * with ep_is_linked().
  1483. */
  1484. ws = ep_wakeup_source(epi);
  1485. if (ws) {
  1486. if (ws->active)
  1487. __pm_stay_awake(ep->ws);
  1488. __pm_relax(ws);
  1489. }
  1490. list_del_init(&epi->rdllink);
  1491. /*
  1492. * If the event mask intersect the caller-requested one,
  1493. * deliver the event to userspace. Again, ep_scan_ready_list()
  1494. * is holding ep->mtx, so no operations coming from userspace
  1495. * can change the item.
  1496. */
  1497. revents = ep_item_poll(epi, &pt, 1);
  1498. if (!revents)
  1499. continue;
  1500. if (__put_user(revents, &uevent->events) ||
  1501. __put_user(epi->event.data, &uevent->data)) {
  1502. list_add(&epi->rdllink, head);
  1503. ep_pm_stay_awake(epi);
  1504. if (!esed->res)
  1505. esed->res = -EFAULT;
  1506. return 0;
  1507. }
  1508. esed->res++;
  1509. uevent++;
  1510. if (epi->event.events & EPOLLONESHOT)
  1511. epi->event.events &= EP_PRIVATE_BITS;
  1512. else if (!(epi->event.events & EPOLLET)) {
  1513. /*
  1514. * If this file has been added with Level
  1515. * Trigger mode, we need to insert back inside
  1516. * the ready list, so that the next call to
  1517. * epoll_wait() will check again the events
  1518. * availability. At this point, no one can insert
  1519. * into ep->rdllist besides us. The epoll_ctl()
  1520. * callers are locked out by
  1521. * ep_scan_ready_list() holding "mtx" and the
  1522. * poll callback will queue them in ep->ovflist.
  1523. */
  1524. list_add_tail(&epi->rdllink, &ep->rdllist);
  1525. ep_pm_stay_awake(epi);
  1526. }
  1527. }
  1528. return 0;
  1529. }
  1530. static int ep_send_events(struct eventpoll *ep,
  1531. struct epoll_event __user *events, int maxevents)
  1532. {
  1533. struct ep_send_events_data esed;
  1534. esed.maxevents = maxevents;
  1535. esed.events = events;
  1536. ep_scan_ready_list(ep, ep_send_events_proc, &esed, 0, false);
  1537. return esed.res;
  1538. }
  1539. static inline struct timespec64 ep_set_mstimeout(long ms)
  1540. {
  1541. struct timespec64 now, ts = {
  1542. .tv_sec = ms / MSEC_PER_SEC,
  1543. .tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC),
  1544. };
  1545. ktime_get_ts64(&now);
  1546. return timespec64_add_safe(now, ts);
  1547. }
  1548. /**
  1549. * ep_poll - Retrieves ready events, and delivers them to the caller supplied
  1550. * event buffer.
  1551. *
  1552. * @ep: Pointer to the eventpoll context.
  1553. * @events: Pointer to the userspace buffer where the ready events should be
  1554. * stored.
  1555. * @maxevents: Size (in terms of number of events) of the caller event buffer.
  1556. * @timeout: Maximum timeout for the ready events fetch operation, in
  1557. * milliseconds. If the @timeout is zero, the function will not block,
  1558. * while if the @timeout is less than zero, the function will block
  1559. * until at least one event has been retrieved (or an error
  1560. * occurred).
  1561. *
  1562. * Returns: Returns the number of ready events which have been fetched, or an
  1563. * error code, in case of error.
  1564. */
  1565. static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
  1566. int maxevents, long timeout)
  1567. {
  1568. int res = 0, eavail, timed_out = 0;
  1569. u64 slack = 0;
  1570. wait_queue_entry_t wait;
  1571. ktime_t expires, *to = NULL;
  1572. lockdep_assert_irqs_enabled();
  1573. if (timeout > 0) {
  1574. struct timespec64 end_time = ep_set_mstimeout(timeout);
  1575. slack = select_estimate_accuracy(&end_time);
  1576. to = &expires;
  1577. *to = timespec64_to_ktime(end_time);
  1578. } else if (timeout == 0) {
  1579. /*
  1580. * Avoid the unnecessary trip to the wait queue loop, if the
  1581. * caller specified a non blocking operation. We still need
  1582. * lock because we could race and not see an epi being added
  1583. * to the ready list while in irq callback. Thus incorrectly
  1584. * returning 0 back to userspace.
  1585. */
  1586. timed_out = 1;
  1587. write_lock_irq(&ep->lock);
  1588. eavail = ep_events_available(ep);
  1589. write_unlock_irq(&ep->lock);
  1590. goto send_events;
  1591. }
  1592. fetch_events:
  1593. if (!ep_events_available(ep))
  1594. ep_busy_loop(ep, timed_out);
  1595. eavail = ep_events_available(ep);
  1596. if (eavail)
  1597. goto send_events;
  1598. /*
  1599. * Busy poll timed out. Drop NAPI ID for now, we can add
  1600. * it back in when we have moved a socket with a valid NAPI
  1601. * ID onto the ready list.
  1602. */
  1603. ep_reset_busy_poll_napi_id(ep);
  1604. do {
  1605. /*
  1606. * Internally init_wait() uses autoremove_wake_function(),
  1607. * thus wait entry is removed from the wait queue on each
  1608. * wakeup. Why it is important? In case of several waiters
  1609. * each new wakeup will hit the next waiter, giving it the
  1610. * chance to harvest new event. Otherwise wakeup can be
  1611. * lost. This is also good performance-wise, because on
  1612. * normal wakeup path no need to call __remove_wait_queue()
  1613. * explicitly, thus ep->lock is not taken, which halts the
  1614. * event delivery.
  1615. */
  1616. init_wait(&wait);
  1617. write_lock_irq(&ep->lock);
  1618. /*
  1619. * Barrierless variant, waitqueue_active() is called under
  1620. * the same lock on wakeup ep_poll_callback() side, so it
  1621. * is safe to avoid an explicit barrier.
  1622. */
  1623. __set_current_state(TASK_INTERRUPTIBLE);
  1624. /*
  1625. * Do the final check under the lock. ep_scan_ready_list()
  1626. * plays with two lists (->rdllist and ->ovflist) and there
  1627. * is always a race when both lists are empty for short
  1628. * period of time although events are pending, so lock is
  1629. * important.
  1630. */
  1631. eavail = ep_events_available(ep);
  1632. if (!eavail) {
  1633. if (signal_pending(current))
  1634. res = -EINTR;
  1635. else
  1636. __add_wait_queue_exclusive(&ep->wq, &wait);
  1637. }
  1638. write_unlock_irq(&ep->lock);
  1639. if (!eavail && !res)
  1640. timed_out = !freezable_schedule_hrtimeout_range(to, slack,
  1641. HRTIMER_MODE_ABS);
  1642. /*
  1643. * We were woken up, thus go and try to harvest some events.
  1644. * If timed out and still on the wait queue, recheck eavail
  1645. * carefully under lock, below.
  1646. */
  1647. eavail = 1;
  1648. } while (0);
  1649. __set_current_state(TASK_RUNNING);
  1650. if (!list_empty_careful(&wait.entry)) {
  1651. write_lock_irq(&ep->lock);
  1652. /*
  1653. * If the thread timed out and is not on the wait queue, it
  1654. * means that the thread was woken up after its timeout expired
  1655. * before it could reacquire the lock. Thus, when wait.entry is
  1656. * empty, it needs to harvest events.
  1657. */
  1658. if (timed_out)
  1659. eavail = list_empty(&wait.entry);
  1660. __remove_wait_queue(&ep->wq, &wait);
  1661. write_unlock_irq(&ep->lock);
  1662. }
  1663. send_events:
  1664. if (fatal_signal_pending(current)) {
  1665. /*
  1666. * Always short-circuit for fatal signals to allow
  1667. * threads to make a timely exit without the chance of
  1668. * finding more events available and fetching
  1669. * repeatedly.
  1670. */
  1671. res = -EINTR;
  1672. }
  1673. /*
  1674. * Try to transfer events to user space. In case we get 0 events and
  1675. * there's still timeout left over, we go trying again in search of
  1676. * more luck.
  1677. */
  1678. if (!res && eavail &&
  1679. !(res = ep_send_events(ep, events, maxevents)) && !timed_out)
  1680. goto fetch_events;
  1681. return res;
  1682. }
  1683. /**
  1684. * ep_loop_check_proc - Callback function to be passed to the @ep_call_nested()
  1685. * API, to verify that adding an epoll file inside another
  1686. * epoll structure, does not violate the constraints, in
  1687. * terms of closed loops, or too deep chains (which can
  1688. * result in excessive stack usage).
  1689. *
  1690. * @priv: Pointer to the epoll file to be currently checked.
  1691. * @cookie: Original cookie for this call. This is the top-of-the-chain epoll
  1692. * data structure pointer.
  1693. * @call_nests: Current dept of the @ep_call_nested() call stack.
  1694. *
  1695. * Returns: Returns zero if adding the epoll @file inside current epoll
  1696. * structure @ep does not violate the constraints, or -1 otherwise.
  1697. */
  1698. static int ep_loop_check_proc(void *priv, void *cookie, int call_nests)
  1699. {
  1700. int error = 0;
  1701. struct file *file = priv;
  1702. struct eventpoll *ep = file->private_data;
  1703. struct eventpoll *ep_tovisit;
  1704. struct rb_node *rbp;
  1705. struct epitem *epi;
  1706. mutex_lock_nested(&ep->mtx, call_nests + 1);
  1707. ep->gen = loop_check_gen;
  1708. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  1709. epi = rb_entry(rbp, struct epitem, rbn);
  1710. if (unlikely(is_file_epoll(epi->ffd.file))) {
  1711. ep_tovisit = epi->ffd.file->private_data;
  1712. if (ep_tovisit->gen == loop_check_gen)
  1713. continue;
  1714. error = ep_call_nested(&poll_loop_ncalls,
  1715. ep_loop_check_proc, epi->ffd.file,
  1716. ep_tovisit, current);
  1717. if (error != 0)
  1718. break;
  1719. } else {
  1720. /*
  1721. * If we've reached a file that is not associated with
  1722. * an ep, then we need to check if the newly added
  1723. * links are going to add too many wakeup paths. We do
  1724. * this by adding it to the tfile_check_list, if it's
  1725. * not already there, and calling reverse_path_check()
  1726. * during ep_insert().
  1727. */
  1728. if (list_empty(&epi->ffd.file->f_tfile_llink)) {
  1729. if (get_file_rcu(epi->ffd.file))
  1730. list_add(&epi->ffd.file->f_tfile_llink,
  1731. &tfile_check_list);
  1732. }
  1733. }
  1734. }
  1735. mutex_unlock(&ep->mtx);
  1736. return error;
  1737. }
  1738. /**
  1739. * ep_loop_check - Performs a check to verify that adding an epoll file (@file)
  1740. * another epoll file (represented by @ep) does not create
  1741. * closed loops or too deep chains.
  1742. *
  1743. * @ep: Pointer to the epoll private data structure.
  1744. * @file: Pointer to the epoll file to be checked.
  1745. *
  1746. * Returns: Returns zero if adding the epoll @file inside current epoll
  1747. * structure @ep does not violate the constraints, or -1 otherwise.
  1748. */
  1749. static int ep_loop_check(struct eventpoll *ep, struct file *file)
  1750. {
  1751. return ep_call_nested(&poll_loop_ncalls,
  1752. ep_loop_check_proc, file, ep, current);
  1753. }
  1754. static void clear_tfile_check_list(void)
  1755. {
  1756. struct file *file;
  1757. /* first clear the tfile_check_list */
  1758. while (!list_empty(&tfile_check_list)) {
  1759. file = list_first_entry(&tfile_check_list, struct file,
  1760. f_tfile_llink);
  1761. list_del_init(&file->f_tfile_llink);
  1762. fput(file);
  1763. }
  1764. INIT_LIST_HEAD(&tfile_check_list);
  1765. }
  1766. /*
  1767. * Open an eventpoll file descriptor.
  1768. */
  1769. static int do_epoll_create(int flags)
  1770. {
  1771. int error, fd;
  1772. struct eventpoll *ep = NULL;
  1773. struct file *file;
  1774. /* Check the EPOLL_* constant for consistency. */
  1775. BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);
  1776. if (flags & ~EPOLL_CLOEXEC)
  1777. return -EINVAL;
  1778. /*
  1779. * Create the internal data structure ("struct eventpoll").
  1780. */
  1781. error = ep_alloc(&ep);
  1782. if (error < 0)
  1783. return error;
  1784. /*
  1785. * Creates all the items needed to setup an eventpoll file. That is,
  1786. * a file structure and a free file descriptor.
  1787. */
  1788. fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));
  1789. if (fd < 0) {
  1790. error = fd;
  1791. goto out_free_ep;
  1792. }
  1793. file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
  1794. O_RDWR | (flags & O_CLOEXEC));
  1795. if (IS_ERR(file)) {
  1796. error = PTR_ERR(file);
  1797. goto out_free_fd;
  1798. }
  1799. ep->file = file;
  1800. fd_install(fd, file);
  1801. return fd;
  1802. out_free_fd:
  1803. put_unused_fd(fd);
  1804. out_free_ep:
  1805. ep_free(ep);
  1806. return error;
  1807. }
  1808. SYSCALL_DEFINE1(epoll_create1, int, flags)
  1809. {
  1810. return do_epoll_create(flags);
  1811. }
  1812. SYSCALL_DEFINE1(epoll_create, int, size)
  1813. {
  1814. if (size <= 0)
  1815. return -EINVAL;
  1816. return do_epoll_create(0);
  1817. }
  1818. static inline int epoll_mutex_lock(struct mutex *mutex, int depth,
  1819. bool nonblock)
  1820. {
  1821. if (!nonblock) {
  1822. mutex_lock_nested(mutex, depth);
  1823. return 0;
  1824. }
  1825. if (mutex_trylock(mutex))
  1826. return 0;
  1827. return -EAGAIN;
  1828. }
  1829. int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
  1830. bool nonblock)
  1831. {
  1832. int error;
  1833. int full_check = 0;
  1834. struct fd f, tf;
  1835. struct eventpoll *ep;
  1836. struct epitem *epi;
  1837. struct eventpoll *tep = NULL;
  1838. error = -EBADF;
  1839. f = fdget(epfd);
  1840. if (!f.file)
  1841. goto error_return;
  1842. /* Get the "struct file *" for the target file */
  1843. tf = fdget(fd);
  1844. if (!tf.file)
  1845. goto error_fput;
  1846. /* The target file descriptor must support poll */
  1847. error = -EPERM;
  1848. if (!file_can_poll(tf.file))
  1849. goto error_tgt_fput;
  1850. /* Check if EPOLLWAKEUP is allowed */
  1851. if (ep_op_has_event(op))
  1852. ep_take_care_of_epollwakeup(epds);
  1853. /*
  1854. * We have to check that the file structure underneath the file descriptor
  1855. * the user passed to us _is_ an eventpoll file. And also we do not permit
  1856. * adding an epoll file descriptor inside itself.
  1857. */
  1858. error = -EINVAL;
  1859. if (f.file == tf.file || !is_file_epoll(f.file))
  1860. goto error_tgt_fput;
  1861. /*
  1862. * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
  1863. * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
  1864. * Also, we do not currently supported nested exclusive wakeups.
  1865. */
  1866. if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) {
  1867. if (op == EPOLL_CTL_MOD)
  1868. goto error_tgt_fput;
  1869. if (op == EPOLL_CTL_ADD && (is_file_epoll(tf.file) ||
  1870. (epds->events & ~EPOLLEXCLUSIVE_OK_BITS)))
  1871. goto error_tgt_fput;
  1872. }
  1873. /*
  1874. * At this point it is safe to assume that the "private_data" contains
  1875. * our own data structure.
  1876. */
  1877. ep = f.file->private_data;
  1878. /*
  1879. * When we insert an epoll file descriptor, inside another epoll file
  1880. * descriptor, there is the change of creating closed loops, which are
  1881. * better be handled here, than in more critical paths. While we are
  1882. * checking for loops we also determine the list of files reachable
  1883. * and hang them on the tfile_check_list, so we can check that we
  1884. * haven't created too many possible wakeup paths.
  1885. *
  1886. * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when
  1887. * the epoll file descriptor is attaching directly to a wakeup source,
  1888. * unless the epoll file descriptor is nested. The purpose of taking the
  1889. * 'epmutex' on add is to prevent complex toplogies such as loops and
  1890. * deep wakeup paths from forming in parallel through multiple
  1891. * EPOLL_CTL_ADD operations.
  1892. */
  1893. error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
  1894. if (error)
  1895. goto error_tgt_fput;
  1896. if (op == EPOLL_CTL_ADD) {
  1897. if (!list_empty(&f.file->f_ep_links) ||
  1898. ep->gen == loop_check_gen ||
  1899. is_file_epoll(tf.file)) {
  1900. mutex_unlock(&ep->mtx);
  1901. error = epoll_mutex_lock(&epmutex, 0, nonblock);
  1902. if (error)
  1903. goto error_tgt_fput;
  1904. loop_check_gen++;
  1905. full_check = 1;
  1906. if (is_file_epoll(tf.file)) {
  1907. error = -ELOOP;
  1908. if (ep_loop_check(ep, tf.file) != 0)
  1909. goto error_tgt_fput;
  1910. } else {
  1911. get_file(tf.file);
  1912. list_add(&tf.file->f_tfile_llink,
  1913. &tfile_check_list);
  1914. }
  1915. error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
  1916. if (error)
  1917. goto error_tgt_fput;
  1918. if (is_file_epoll(tf.file)) {
  1919. tep = tf.file->private_data;
  1920. error = epoll_mutex_lock(&tep->mtx, 1, nonblock);
  1921. if (error) {
  1922. mutex_unlock(&ep->mtx);
  1923. goto error_tgt_fput;
  1924. }
  1925. }
  1926. }
  1927. }
  1928. /*
  1929. * Try to lookup the file inside our RB tree, Since we grabbed "mtx"
  1930. * above, we can be sure to be able to use the item looked up by
  1931. * ep_find() till we release the mutex.
  1932. */
  1933. epi = ep_find(ep, tf.file, fd);
  1934. error = -EINVAL;
  1935. switch (op) {
  1936. case EPOLL_CTL_ADD:
  1937. if (!epi) {
  1938. epds->events |= EPOLLERR | EPOLLHUP;
  1939. error = ep_insert(ep, epds, tf.file, fd, full_check);
  1940. } else
  1941. error = -EEXIST;
  1942. break;
  1943. case EPOLL_CTL_DEL:
  1944. if (epi)
  1945. error = ep_remove(ep, epi);
  1946. else
  1947. error = -ENOENT;
  1948. break;
  1949. case EPOLL_CTL_MOD:
  1950. if (epi) {
  1951. if (!(epi->event.events & EPOLLEXCLUSIVE)) {
  1952. epds->events |= EPOLLERR | EPOLLHUP;
  1953. error = ep_modify(ep, epi, epds);
  1954. }
  1955. } else
  1956. error = -ENOENT;
  1957. break;
  1958. }
  1959. if (tep != NULL)
  1960. mutex_unlock(&tep->mtx);
  1961. mutex_unlock(&ep->mtx);
  1962. error_tgt_fput:
  1963. if (full_check) {
  1964. clear_tfile_check_list();
  1965. loop_check_gen++;
  1966. mutex_unlock(&epmutex);
  1967. }
  1968. fdput(tf);
  1969. error_fput:
  1970. fdput(f);
  1971. error_return:
  1972. return error;
  1973. }
  1974. /*
  1975. * The following function implements the controller interface for
  1976. * the eventpoll file that enables the insertion/removal/change of
  1977. * file descriptors inside the interest set.
  1978. */
  1979. SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
  1980. struct epoll_event __user *, event)
  1981. {
  1982. struct epoll_event epds;
  1983. if (ep_op_has_event(op) &&
  1984. copy_from_user(&epds, event, sizeof(struct epoll_event)))
  1985. return -EFAULT;
  1986. return do_epoll_ctl(epfd, op, fd, &epds, false);
  1987. }
  1988. /*
  1989. * Implement the event wait interface for the eventpoll file. It is the kernel
  1990. * part of the user space epoll_wait(2).
  1991. */
  1992. static int do_epoll_wait(int epfd, struct epoll_event __user *events,
  1993. int maxevents, int timeout)
  1994. {
  1995. int error;
  1996. struct fd f;
  1997. struct eventpoll *ep;
  1998. /* The maximum number of event must be greater than zero */
  1999. if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
  2000. return -EINVAL;
  2001. /* Verify that the area passed by the user is writeable */
  2002. if (!access_ok(events, maxevents * sizeof(struct epoll_event)))
  2003. return -EFAULT;
  2004. /* Get the "struct file *" for the eventpoll file */
  2005. f = fdget(epfd);
  2006. if (!f.file)
  2007. return -EBADF;
  2008. /*
  2009. * We have to check that the file structure underneath the fd
  2010. * the user passed to us _is_ an eventpoll file.
  2011. */
  2012. error = -EINVAL;
  2013. if (!is_file_epoll(f.file))
  2014. goto error_fput;
  2015. /*
  2016. * At this point it is safe to assume that the "private_data" contains
  2017. * our own data structure.
  2018. */
  2019. ep = f.file->private_data;
  2020. /* Time to fish for events ... */
  2021. error = ep_poll(ep, events, maxevents, timeout);
  2022. error_fput:
  2023. fdput(f);
  2024. return error;
  2025. }
  2026. SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
  2027. int, maxevents, int, timeout)
  2028. {
  2029. return do_epoll_wait(epfd, events, maxevents, timeout);
  2030. }
  2031. /*
  2032. * Implement the event wait interface for the eventpoll file. It is the kernel
  2033. * part of the user space epoll_pwait(2).
  2034. */
  2035. SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events,
  2036. int, maxevents, int, timeout, const sigset_t __user *, sigmask,
  2037. size_t, sigsetsize)
  2038. {
  2039. int error;
  2040. /*
  2041. * If the caller wants a certain signal mask to be set during the wait,
  2042. * we apply it here.
  2043. */
  2044. error = set_user_sigmask(sigmask, sigsetsize);
  2045. if (error)
  2046. return error;
  2047. error = do_epoll_wait(epfd, events, maxevents, timeout);
  2048. restore_saved_sigmask_unless(error == -EINTR);
  2049. return error;
  2050. }
  2051. #ifdef CONFIG_COMPAT
  2052. COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd,
  2053. struct epoll_event __user *, events,
  2054. int, maxevents, int, timeout,
  2055. const compat_sigset_t __user *, sigmask,
  2056. compat_size_t, sigsetsize)
  2057. {
  2058. long err;
  2059. /*
  2060. * If the caller wants a certain signal mask to be set during the wait,
  2061. * we apply it here.
  2062. */
  2063. err = set_compat_user_sigmask(sigmask, sigsetsize);
  2064. if (err)
  2065. return err;
  2066. err = do_epoll_wait(epfd, events, maxevents, timeout);
  2067. restore_saved_sigmask_unless(err == -EINTR);
  2068. return err;
  2069. }
  2070. #endif
  2071. static int __init eventpoll_init(void)
  2072. {
  2073. struct sysinfo si;
  2074. si_meminfo(&si);
  2075. /*
  2076. * Allows top 4% of lomem to be allocated for epoll watches (per user).
  2077. */
  2078. max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) /
  2079. EP_ITEM_COST;
  2080. BUG_ON(max_user_watches < 0);
  2081. /*
  2082. * Initialize the structure used to perform epoll file descriptor
  2083. * inclusion loops checks.
  2084. */
  2085. ep_nested_calls_init(&poll_loop_ncalls);
  2086. /*
  2087. * We can have many thousands of epitems, so prevent this from
  2088. * using an extra cache line on 64-bit (and smaller) CPUs
  2089. */
  2090. BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128);
  2091. /* Allocates slab cache used to allocate "struct epitem" items */
  2092. epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),
  2093. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
  2094. /* Allocates slab cache used to allocate "struct eppoll_entry" */
  2095. pwq_cache = kmem_cache_create("eventpoll_pwq",
  2096. sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
  2097. return 0;
  2098. }
  2099. fs_initcall(eventpoll_init);