wait_bit.c 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * The implementation of the wait_bit*() and related waiting APIs:
  4. */
  5. #include "sched.h"
  6. #define WAIT_TABLE_BITS 8
  7. #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
  8. static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
  9. wait_queue_head_t *bit_waitqueue(void *word, int bit)
  10. {
  11. const int shift = BITS_PER_LONG == 32 ? 5 : 6;
  12. unsigned long val = (unsigned long)word << shift | bit;
  13. return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
  14. }
  15. EXPORT_SYMBOL(bit_waitqueue);
  16. int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
  17. {
  18. struct wait_bit_key *key = arg;
  19. struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  20. if (wait_bit->key.flags != key->flags ||
  21. wait_bit->key.bit_nr != key->bit_nr ||
  22. test_bit(key->bit_nr, key->flags))
  23. return 0;
  24. return autoremove_wake_function(wq_entry, mode, sync, key);
  25. }
  26. EXPORT_SYMBOL(wake_bit_function);
  27. /*
  28. * To allow interruptible waiting and asynchronous (i.e. nonblocking)
  29. * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
  30. * permitted return codes. Nonzero return codes halt waiting and return.
  31. */
  32. int __sched
  33. __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  34. wait_bit_action_f *action, unsigned mode)
  35. {
  36. int ret = 0;
  37. do {
  38. prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
  39. if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
  40. ret = (*action)(&wbq_entry->key, mode);
  41. } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
  42. finish_wait(wq_head, &wbq_entry->wq_entry);
  43. return ret;
  44. }
  45. EXPORT_SYMBOL(__wait_on_bit);
  46. int __sched out_of_line_wait_on_bit(void *word, int bit,
  47. wait_bit_action_f *action, unsigned mode)
  48. {
  49. struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  50. DEFINE_WAIT_BIT(wq_entry, word, bit);
  51. return __wait_on_bit(wq_head, &wq_entry, action, mode);
  52. }
  53. EXPORT_SYMBOL(out_of_line_wait_on_bit);
  54. int __sched out_of_line_wait_on_bit_timeout(
  55. void *word, int bit, wait_bit_action_f *action,
  56. unsigned mode, unsigned long timeout)
  57. {
  58. struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  59. DEFINE_WAIT_BIT(wq_entry, word, bit);
  60. wq_entry.key.timeout = jiffies + timeout;
  61. return __wait_on_bit(wq_head, &wq_entry, action, mode);
  62. }
  63. EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
  64. int __sched
  65. __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  66. wait_bit_action_f *action, unsigned mode)
  67. {
  68. int ret = 0;
  69. for (;;) {
  70. prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
  71. if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
  72. ret = action(&wbq_entry->key, mode);
  73. /*
  74. * See the comment in prepare_to_wait_event().
  75. * finish_wait() does not necessarily takes wwq_head->lock,
  76. * but test_and_set_bit() implies mb() which pairs with
  77. * smp_mb__after_atomic() before wake_up_page().
  78. */
  79. if (ret)
  80. finish_wait(wq_head, &wbq_entry->wq_entry);
  81. }
  82. if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
  83. if (!ret)
  84. finish_wait(wq_head, &wbq_entry->wq_entry);
  85. return 0;
  86. } else if (ret) {
  87. return ret;
  88. }
  89. }
  90. }
  91. EXPORT_SYMBOL(__wait_on_bit_lock);
  92. int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
  93. wait_bit_action_f *action, unsigned mode)
  94. {
  95. struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  96. DEFINE_WAIT_BIT(wq_entry, word, bit);
  97. return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
  98. }
  99. EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
  100. void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
  101. {
  102. struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
  103. if (waitqueue_active(wq_head))
  104. __wake_up(wq_head, TASK_NORMAL, 1, &key);
  105. }
  106. EXPORT_SYMBOL(__wake_up_bit);
  107. /**
  108. * wake_up_bit - wake up a waiter on a bit
  109. * @word: the word being waited on, a kernel virtual address
  110. * @bit: the bit of the word being waited on
  111. *
  112. * There is a standard hashed waitqueue table for generic use. This
  113. * is the part of the hashtable's accessor API that wakes up waiters
  114. * on a bit. For instance, if one were to have waiters on a bitflag,
  115. * one would call wake_up_bit() after clearing the bit.
  116. *
  117. * In order for this to function properly, as it uses waitqueue_active()
  118. * internally, some kind of memory barrier must be done prior to calling
  119. * this. Typically, this will be smp_mb__after_atomic(), but in some
  120. * cases where bitflags are manipulated non-atomically under a lock, one
  121. * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
  122. * because spin_unlock() does not guarantee a memory barrier.
  123. */
  124. void wake_up_bit(void *word, int bit)
  125. {
  126. __wake_up_bit(bit_waitqueue(word, bit), word, bit);
  127. }
  128. EXPORT_SYMBOL(wake_up_bit);
  129. wait_queue_head_t *__var_waitqueue(void *p)
  130. {
  131. return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS);
  132. }
  133. EXPORT_SYMBOL(__var_waitqueue);
  134. static int
  135. var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,
  136. int sync, void *arg)
  137. {
  138. struct wait_bit_key *key = arg;
  139. struct wait_bit_queue_entry *wbq_entry =
  140. container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  141. if (wbq_entry->key.flags != key->flags ||
  142. wbq_entry->key.bit_nr != key->bit_nr)
  143. return 0;
  144. return autoremove_wake_function(wq_entry, mode, sync, key);
  145. }
  146. void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags)
  147. {
  148. *wbq_entry = (struct wait_bit_queue_entry){
  149. .key = {
  150. .flags = (var),
  151. .bit_nr = -1,
  152. },
  153. .wq_entry = {
  154. .flags = flags,
  155. .private = current,
  156. .func = var_wake_function,
  157. .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry),
  158. },
  159. };
  160. }
  161. EXPORT_SYMBOL(init_wait_var_entry);
  162. void wake_up_var(void *var)
  163. {
  164. __wake_up_bit(__var_waitqueue(var), var, -1);
  165. }
  166. EXPORT_SYMBOL(wake_up_var);
  167. __sched int bit_wait(struct wait_bit_key *word, int mode)
  168. {
  169. schedule();
  170. if (signal_pending_state(mode, current))
  171. return -EINTR;
  172. return 0;
  173. }
  174. EXPORT_SYMBOL(bit_wait);
  175. __sched int bit_wait_io(struct wait_bit_key *word, int mode)
  176. {
  177. io_schedule();
  178. if (signal_pending_state(mode, current))
  179. return -EINTR;
  180. return 0;
  181. }
  182. EXPORT_SYMBOL(bit_wait_io);
  183. __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
  184. {
  185. unsigned long now = READ_ONCE(jiffies);
  186. if (time_after_eq(now, word->timeout))
  187. return -EAGAIN;
  188. schedule_timeout(word->timeout - now);
  189. if (signal_pending_state(mode, current))
  190. return -EINTR;
  191. return 0;
  192. }
  193. EXPORT_SYMBOL_GPL(bit_wait_timeout);
  194. __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
  195. {
  196. unsigned long now = READ_ONCE(jiffies);
  197. if (time_after_eq(now, word->timeout))
  198. return -EAGAIN;
  199. io_schedule_timeout(word->timeout - now);
  200. if (signal_pending_state(mode, current))
  201. return -EINTR;
  202. return 0;
  203. }
  204. EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
  205. void __init wait_bit_init(void)
  206. {
  207. int i;
  208. for (i = 0; i < WAIT_TABLE_SIZE; i++)
  209. init_waitqueue_head(bit_wait_table + i);
  210. }