watch_queue.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Watch queue and general notification mechanism, built on pipes
  3. *
  4. * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. *
  7. * See Documentation/watch_queue.rst
  8. */
  9. #define pr_fmt(fmt) "watchq: " fmt
  10. #include <linux/module.h>
  11. #include <linux/init.h>
  12. #include <linux/sched.h>
  13. #include <linux/slab.h>
  14. #include <linux/printk.h>
  15. #include <linux/miscdevice.h>
  16. #include <linux/fs.h>
  17. #include <linux/mm.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/poll.h>
  20. #include <linux/uaccess.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/file.h>
  23. #include <linux/security.h>
  24. #include <linux/cred.h>
  25. #include <linux/sched/signal.h>
  26. #include <linux/watch_queue.h>
  27. #include <linux/pipe_fs_i.h>
  28. MODULE_DESCRIPTION("Watch queue");
  29. MODULE_AUTHOR("Red Hat, Inc.");
  30. MODULE_LICENSE("GPL");
  31. #define WATCH_QUEUE_NOTE_SIZE 128
  32. #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE)
  33. static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe,
  34. struct pipe_buffer *buf)
  35. {
  36. struct watch_queue *wqueue = (struct watch_queue *)buf->private;
  37. struct page *page;
  38. unsigned int bit;
  39. /* We need to work out which note within the page this refers to, but
  40. * the note might have been maximum size, so merely ANDing the offset
  41. * off doesn't work. OTOH, the note must've been more than zero size.
  42. */
  43. bit = buf->offset + buf->len;
  44. if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0)
  45. bit -= WATCH_QUEUE_NOTE_SIZE;
  46. bit /= WATCH_QUEUE_NOTE_SIZE;
  47. page = buf->page;
  48. bit += page->index;
  49. set_bit(bit, wqueue->notes_bitmap);
  50. generic_pipe_buf_release(pipe, buf);
  51. }
  52. // No try_steal function => no stealing
  53. #define watch_queue_pipe_buf_try_steal NULL
  54. /* New data written to a pipe may be appended to a buffer with this type. */
  55. static const struct pipe_buf_operations watch_queue_pipe_buf_ops = {
  56. .release = watch_queue_pipe_buf_release,
  57. .try_steal = watch_queue_pipe_buf_try_steal,
  58. .get = generic_pipe_buf_get,
  59. };
  60. /*
  61. * Post a notification to a watch queue.
  62. */
  63. static bool post_one_notification(struct watch_queue *wqueue,
  64. struct watch_notification *n)
  65. {
  66. void *p;
  67. struct pipe_inode_info *pipe = wqueue->pipe;
  68. struct pipe_buffer *buf;
  69. struct page *page;
  70. unsigned int head, tail, mask, note, offset, len;
  71. bool done = false;
  72. if (!pipe)
  73. return false;
  74. spin_lock_irq(&pipe->rd_wait.lock);
  75. if (wqueue->defunct)
  76. goto out;
  77. mask = pipe->ring_size - 1;
  78. head = pipe->head;
  79. tail = pipe->tail;
  80. if (pipe_full(head, tail, pipe->ring_size))
  81. goto lost;
  82. note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);
  83. if (note >= wqueue->nr_notes)
  84. goto lost;
  85. page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];
  86. offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE;
  87. get_page(page);
  88. len = n->info & WATCH_INFO_LENGTH;
  89. p = kmap_atomic(page);
  90. memcpy(p + offset, n, len);
  91. kunmap_atomic(p);
  92. buf = &pipe->bufs[head & mask];
  93. buf->page = page;
  94. buf->private = (unsigned long)wqueue;
  95. buf->ops = &watch_queue_pipe_buf_ops;
  96. buf->offset = offset;
  97. buf->len = len;
  98. buf->flags = PIPE_BUF_FLAG_WHOLE;
  99. smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */
  100. if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {
  101. spin_unlock_irq(&pipe->rd_wait.lock);
  102. BUG();
  103. }
  104. wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
  105. done = true;
  106. out:
  107. spin_unlock_irq(&pipe->rd_wait.lock);
  108. if (done)
  109. kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  110. return done;
  111. lost:
  112. buf = &pipe->bufs[(head - 1) & mask];
  113. buf->flags |= PIPE_BUF_FLAG_LOSS;
  114. goto out;
  115. }
  116. /*
  117. * Apply filter rules to a notification.
  118. */
  119. static bool filter_watch_notification(const struct watch_filter *wf,
  120. const struct watch_notification *n)
  121. {
  122. const struct watch_type_filter *wt;
  123. unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8;
  124. unsigned int st_index = n->subtype / st_bits;
  125. unsigned int st_bit = 1U << (n->subtype % st_bits);
  126. int i;
  127. if (!test_bit(n->type, wf->type_filter))
  128. return false;
  129. for (i = 0; i < wf->nr_filters; i++) {
  130. wt = &wf->filters[i];
  131. if (n->type == wt->type &&
  132. (wt->subtype_filter[st_index] & st_bit) &&
  133. (n->info & wt->info_mask) == wt->info_filter)
  134. return true;
  135. }
  136. return false; /* If there is a filter, the default is to reject. */
  137. }
  138. /**
  139. * __post_watch_notification - Post an event notification
  140. * @wlist: The watch list to post the event to.
  141. * @n: The notification record to post.
  142. * @cred: The creds of the process that triggered the notification.
  143. * @id: The ID to match on the watch.
  144. *
  145. * Post a notification of an event into a set of watch queues and let the users
  146. * know.
  147. *
  148. * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and
  149. * should be in units of sizeof(*n).
  150. */
  151. void __post_watch_notification(struct watch_list *wlist,
  152. struct watch_notification *n,
  153. const struct cred *cred,
  154. u64 id)
  155. {
  156. const struct watch_filter *wf;
  157. struct watch_queue *wqueue;
  158. struct watch *watch;
  159. if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) {
  160. WARN_ON(1);
  161. return;
  162. }
  163. rcu_read_lock();
  164. hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) {
  165. if (watch->id != id)
  166. continue;
  167. n->info &= ~WATCH_INFO_ID;
  168. n->info |= watch->info_id;
  169. wqueue = rcu_dereference(watch->queue);
  170. wf = rcu_dereference(wqueue->filter);
  171. if (wf && !filter_watch_notification(wf, n))
  172. continue;
  173. if (security_post_notification(watch->cred, cred, n) < 0)
  174. continue;
  175. post_one_notification(wqueue, n);
  176. }
  177. rcu_read_unlock();
  178. }
  179. EXPORT_SYMBOL(__post_watch_notification);
  180. /*
  181. * Allocate sufficient pages to preallocation for the requested number of
  182. * notifications.
  183. */
  184. long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes)
  185. {
  186. struct watch_queue *wqueue = pipe->watch_queue;
  187. struct page **pages;
  188. unsigned long *bitmap;
  189. unsigned long user_bufs;
  190. unsigned int bmsize;
  191. int ret, i, nr_pages;
  192. if (!wqueue)
  193. return -ENODEV;
  194. if (wqueue->notes)
  195. return -EBUSY;
  196. if (nr_notes < 1 ||
  197. nr_notes > 512) /* TODO: choose a better hard limit */
  198. return -EINVAL;
  199. nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1);
  200. nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE;
  201. user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages);
  202. if (nr_pages > pipe->max_usage &&
  203. (too_many_pipe_buffers_hard(user_bufs) ||
  204. too_many_pipe_buffers_soft(user_bufs)) &&
  205. pipe_is_unprivileged_user()) {
  206. ret = -EPERM;
  207. goto error;
  208. }
  209. nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;
  210. ret = pipe_resize_ring(pipe, roundup_pow_of_two(nr_notes));
  211. if (ret < 0)
  212. goto error;
  213. pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL);
  214. if (!pages)
  215. goto error;
  216. for (i = 0; i < nr_pages; i++) {
  217. pages[i] = alloc_page(GFP_KERNEL);
  218. if (!pages[i])
  219. goto error_p;
  220. pages[i]->index = i * WATCH_QUEUE_NOTES_PER_PAGE;
  221. }
  222. bmsize = (nr_notes + BITS_PER_LONG - 1) / BITS_PER_LONG;
  223. bmsize *= sizeof(unsigned long);
  224. bitmap = kmalloc(bmsize, GFP_KERNEL);
  225. if (!bitmap)
  226. goto error_p;
  227. memset(bitmap, 0xff, bmsize);
  228. wqueue->notes = pages;
  229. wqueue->notes_bitmap = bitmap;
  230. wqueue->nr_pages = nr_pages;
  231. wqueue->nr_notes = nr_notes;
  232. return 0;
  233. error_p:
  234. while (--i >= 0)
  235. __free_page(pages[i]);
  236. kfree(pages);
  237. error:
  238. (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted);
  239. return ret;
  240. }
  241. /*
  242. * Set the filter on a watch queue.
  243. */
  244. long watch_queue_set_filter(struct pipe_inode_info *pipe,
  245. struct watch_notification_filter __user *_filter)
  246. {
  247. struct watch_notification_type_filter *tf;
  248. struct watch_notification_filter filter;
  249. struct watch_type_filter *q;
  250. struct watch_filter *wfilter;
  251. struct watch_queue *wqueue = pipe->watch_queue;
  252. int ret, nr_filter = 0, i;
  253. if (!wqueue)
  254. return -ENODEV;
  255. if (!_filter) {
  256. /* Remove the old filter */
  257. wfilter = NULL;
  258. goto set;
  259. }
  260. /* Grab the user's filter specification */
  261. if (copy_from_user(&filter, _filter, sizeof(filter)) != 0)
  262. return -EFAULT;
  263. if (filter.nr_filters == 0 ||
  264. filter.nr_filters > 16 ||
  265. filter.__reserved != 0)
  266. return -EINVAL;
  267. tf = memdup_user(_filter->filters, filter.nr_filters * sizeof(*tf));
  268. if (IS_ERR(tf))
  269. return PTR_ERR(tf);
  270. ret = -EINVAL;
  271. for (i = 0; i < filter.nr_filters; i++) {
  272. if ((tf[i].info_filter & ~tf[i].info_mask) ||
  273. tf[i].info_mask & WATCH_INFO_LENGTH)
  274. goto err_filter;
  275. /* Ignore any unknown types */
  276. if (tf[i].type >= WATCH_TYPE__NR)
  277. continue;
  278. nr_filter++;
  279. }
  280. /* Now we need to build the internal filter from only the relevant
  281. * user-specified filters.
  282. */
  283. ret = -ENOMEM;
  284. wfilter = kzalloc(struct_size(wfilter, filters, nr_filter), GFP_KERNEL);
  285. if (!wfilter)
  286. goto err_filter;
  287. wfilter->nr_filters = nr_filter;
  288. q = wfilter->filters;
  289. for (i = 0; i < filter.nr_filters; i++) {
  290. if (tf[i].type >= WATCH_TYPE__NR)
  291. continue;
  292. q->type = tf[i].type;
  293. q->info_filter = tf[i].info_filter;
  294. q->info_mask = tf[i].info_mask;
  295. q->subtype_filter[0] = tf[i].subtype_filter[0];
  296. __set_bit(q->type, wfilter->type_filter);
  297. q++;
  298. }
  299. kfree(tf);
  300. set:
  301. pipe_lock(pipe);
  302. wfilter = rcu_replace_pointer(wqueue->filter, wfilter,
  303. lockdep_is_held(&pipe->mutex));
  304. pipe_unlock(pipe);
  305. if (wfilter)
  306. kfree_rcu(wfilter, rcu);
  307. return 0;
  308. err_filter:
  309. kfree(tf);
  310. return ret;
  311. }
  312. static void __put_watch_queue(struct kref *kref)
  313. {
  314. struct watch_queue *wqueue =
  315. container_of(kref, struct watch_queue, usage);
  316. struct watch_filter *wfilter;
  317. int i;
  318. for (i = 0; i < wqueue->nr_pages; i++)
  319. __free_page(wqueue->notes[i]);
  320. kfree(wqueue->notes);
  321. bitmap_free(wqueue->notes_bitmap);
  322. wfilter = rcu_access_pointer(wqueue->filter);
  323. if (wfilter)
  324. kfree_rcu(wfilter, rcu);
  325. kfree_rcu(wqueue, rcu);
  326. }
  327. /**
  328. * put_watch_queue - Dispose of a ref on a watchqueue.
  329. * @wqueue: The watch queue to unref.
  330. */
  331. void put_watch_queue(struct watch_queue *wqueue)
  332. {
  333. kref_put(&wqueue->usage, __put_watch_queue);
  334. }
  335. EXPORT_SYMBOL(put_watch_queue);
  336. static void free_watch(struct rcu_head *rcu)
  337. {
  338. struct watch *watch = container_of(rcu, struct watch, rcu);
  339. put_watch_queue(rcu_access_pointer(watch->queue));
  340. atomic_dec(&watch->cred->user->nr_watches);
  341. put_cred(watch->cred);
  342. kfree(watch);
  343. }
  344. static void __put_watch(struct kref *kref)
  345. {
  346. struct watch *watch = container_of(kref, struct watch, usage);
  347. call_rcu(&watch->rcu, free_watch);
  348. }
  349. /*
  350. * Discard a watch.
  351. */
  352. static void put_watch(struct watch *watch)
  353. {
  354. kref_put(&watch->usage, __put_watch);
  355. }
  356. /**
  357. * init_watch_queue - Initialise a watch
  358. * @watch: The watch to initialise.
  359. * @wqueue: The queue to assign.
  360. *
  361. * Initialise a watch and set the watch queue.
  362. */
  363. void init_watch(struct watch *watch, struct watch_queue *wqueue)
  364. {
  365. kref_init(&watch->usage);
  366. INIT_HLIST_NODE(&watch->list_node);
  367. INIT_HLIST_NODE(&watch->queue_node);
  368. rcu_assign_pointer(watch->queue, wqueue);
  369. }
  370. /**
  371. * add_watch_to_object - Add a watch on an object to a watch list
  372. * @watch: The watch to add
  373. * @wlist: The watch list to add to
  374. *
  375. * @watch->queue must have been set to point to the queue to post notifications
  376. * to and the watch list of the object to be watched. @watch->cred must also
  377. * have been set to the appropriate credentials and a ref taken on them.
  378. *
  379. * The caller must pin the queue and the list both and must hold the list
  380. * locked against racing watch additions/removals.
  381. */
  382. int add_watch_to_object(struct watch *watch, struct watch_list *wlist)
  383. {
  384. struct watch_queue *wqueue = rcu_access_pointer(watch->queue);
  385. struct watch *w;
  386. hlist_for_each_entry(w, &wlist->watchers, list_node) {
  387. struct watch_queue *wq = rcu_access_pointer(w->queue);
  388. if (wqueue == wq && watch->id == w->id)
  389. return -EBUSY;
  390. }
  391. watch->cred = get_current_cred();
  392. rcu_assign_pointer(watch->watch_list, wlist);
  393. if (atomic_inc_return(&watch->cred->user->nr_watches) >
  394. task_rlimit(current, RLIMIT_NOFILE)) {
  395. atomic_dec(&watch->cred->user->nr_watches);
  396. put_cred(watch->cred);
  397. return -EAGAIN;
  398. }
  399. spin_lock_bh(&wqueue->lock);
  400. kref_get(&wqueue->usage);
  401. kref_get(&watch->usage);
  402. hlist_add_head(&watch->queue_node, &wqueue->watches);
  403. spin_unlock_bh(&wqueue->lock);
  404. hlist_add_head(&watch->list_node, &wlist->watchers);
  405. return 0;
  406. }
  407. EXPORT_SYMBOL(add_watch_to_object);
  408. /**
  409. * remove_watch_from_object - Remove a watch or all watches from an object.
  410. * @wlist: The watch list to remove from
  411. * @wq: The watch queue of interest (ignored if @all is true)
  412. * @id: The ID of the watch to remove (ignored if @all is true)
  413. * @all: True to remove all objects
  414. *
  415. * Remove a specific watch or all watches from an object. A notification is
  416. * sent to the watcher to tell them that this happened.
  417. */
  418. int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq,
  419. u64 id, bool all)
  420. {
  421. struct watch_notification_removal n;
  422. struct watch_queue *wqueue;
  423. struct watch *watch;
  424. int ret = -EBADSLT;
  425. rcu_read_lock();
  426. again:
  427. spin_lock(&wlist->lock);
  428. hlist_for_each_entry(watch, &wlist->watchers, list_node) {
  429. if (all ||
  430. (watch->id == id && rcu_access_pointer(watch->queue) == wq))
  431. goto found;
  432. }
  433. spin_unlock(&wlist->lock);
  434. goto out;
  435. found:
  436. ret = 0;
  437. hlist_del_init_rcu(&watch->list_node);
  438. rcu_assign_pointer(watch->watch_list, NULL);
  439. spin_unlock(&wlist->lock);
  440. /* We now own the reference on watch that used to belong to wlist. */
  441. n.watch.type = WATCH_TYPE_META;
  442. n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION;
  443. n.watch.info = watch->info_id | watch_sizeof(n.watch);
  444. n.id = id;
  445. if (id != 0)
  446. n.watch.info = watch->info_id | watch_sizeof(n);
  447. wqueue = rcu_dereference(watch->queue);
  448. /* We don't need the watch list lock for the next bit as RCU is
  449. * protecting *wqueue from deallocation.
  450. */
  451. if (wqueue) {
  452. post_one_notification(wqueue, &n.watch);
  453. spin_lock_bh(&wqueue->lock);
  454. if (!hlist_unhashed(&watch->queue_node)) {
  455. hlist_del_init_rcu(&watch->queue_node);
  456. put_watch(watch);
  457. }
  458. spin_unlock_bh(&wqueue->lock);
  459. }
  460. if (wlist->release_watch) {
  461. void (*release_watch)(struct watch *);
  462. release_watch = wlist->release_watch;
  463. rcu_read_unlock();
  464. (*release_watch)(watch);
  465. rcu_read_lock();
  466. }
  467. put_watch(watch);
  468. if (all && !hlist_empty(&wlist->watchers))
  469. goto again;
  470. out:
  471. rcu_read_unlock();
  472. return ret;
  473. }
  474. EXPORT_SYMBOL(remove_watch_from_object);
  475. /*
  476. * Remove all the watches that are contributory to a queue. This has the
  477. * potential to race with removal of the watches by the destruction of the
  478. * objects being watched or with the distribution of notifications.
  479. */
  480. void watch_queue_clear(struct watch_queue *wqueue)
  481. {
  482. struct watch_list *wlist;
  483. struct watch *watch;
  484. bool release;
  485. rcu_read_lock();
  486. spin_lock_bh(&wqueue->lock);
  487. /* Prevent new notifications from being stored. */
  488. wqueue->defunct = true;
  489. while (!hlist_empty(&wqueue->watches)) {
  490. watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);
  491. hlist_del_init_rcu(&watch->queue_node);
  492. /* We now own a ref on the watch. */
  493. spin_unlock_bh(&wqueue->lock);
  494. /* We can't do the next bit under the queue lock as we need to
  495. * get the list lock - which would cause a deadlock if someone
  496. * was removing from the opposite direction at the same time or
  497. * posting a notification.
  498. */
  499. wlist = rcu_dereference(watch->watch_list);
  500. if (wlist) {
  501. void (*release_watch)(struct watch *);
  502. spin_lock(&wlist->lock);
  503. release = !hlist_unhashed(&watch->list_node);
  504. if (release) {
  505. hlist_del_init_rcu(&watch->list_node);
  506. rcu_assign_pointer(watch->watch_list, NULL);
  507. /* We now own a second ref on the watch. */
  508. }
  509. release_watch = wlist->release_watch;
  510. spin_unlock(&wlist->lock);
  511. if (release) {
  512. if (release_watch) {
  513. rcu_read_unlock();
  514. /* This might need to call dput(), so
  515. * we have to drop all the locks.
  516. */
  517. (*release_watch)(watch);
  518. rcu_read_lock();
  519. }
  520. put_watch(watch);
  521. }
  522. }
  523. put_watch(watch);
  524. spin_lock_bh(&wqueue->lock);
  525. }
  526. spin_unlock_bh(&wqueue->lock);
  527. rcu_read_unlock();
  528. }
  529. /**
  530. * get_watch_queue - Get a watch queue from its file descriptor.
  531. * @fd: The fd to query.
  532. */
  533. struct watch_queue *get_watch_queue(int fd)
  534. {
  535. struct pipe_inode_info *pipe;
  536. struct watch_queue *wqueue = ERR_PTR(-EINVAL);
  537. struct fd f;
  538. f = fdget(fd);
  539. if (f.file) {
  540. pipe = get_pipe_info(f.file, false);
  541. if (pipe && pipe->watch_queue) {
  542. wqueue = pipe->watch_queue;
  543. kref_get(&wqueue->usage);
  544. }
  545. fdput(f);
  546. }
  547. return wqueue;
  548. }
  549. EXPORT_SYMBOL(get_watch_queue);
  550. /*
  551. * Initialise a watch queue
  552. */
  553. int watch_queue_init(struct pipe_inode_info *pipe)
  554. {
  555. struct watch_queue *wqueue;
  556. wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);
  557. if (!wqueue)
  558. return -ENOMEM;
  559. wqueue->pipe = pipe;
  560. kref_init(&wqueue->usage);
  561. spin_lock_init(&wqueue->lock);
  562. INIT_HLIST_HEAD(&wqueue->watches);
  563. pipe->watch_queue = wqueue;
  564. return 0;
  565. }