completion.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Generic wait-for-completion handler;
  4. *
  5. * It differs from semaphores in that their default case is the opposite,
  6. * wait_for_completion default blocks whereas semaphore default non-block. The
  7. * interface also makes it easy to 'complete' multiple waiting threads,
  8. * something which isn't entirely natural for semaphores.
  9. *
  10. * But more importantly, the primitive documents the usage. Semaphores would
  11. * typically be used for exclusion which gives rise to priority inversion.
  12. * Waiting for completion is a typically sync point, but not an exclusion point.
  13. */
  14. #include "sched.h"
  15. /**
  16. * complete: - signals a single thread waiting on this completion
  17. * @x: holds the state of this particular completion
  18. *
  19. * This will wake up a single thread waiting on this completion. Threads will be
  20. * awakened in the same order in which they were queued.
  21. *
  22. * See also complete_all(), wait_for_completion() and related routines.
  23. *
  24. * If this function wakes up a task, it executes a full memory barrier before
  25. * accessing the task state.
  26. */
  27. void complete(struct completion *x)
  28. {
  29. unsigned long flags;
  30. raw_spin_lock_irqsave(&x->wait.lock, flags);
  31. if (x->done != UINT_MAX)
  32. x->done++;
  33. swake_up_locked(&x->wait);
  34. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  35. }
  36. EXPORT_SYMBOL(complete);
  37. /**
  38. * complete_all: - signals all threads waiting on this completion
  39. * @x: holds the state of this particular completion
  40. *
  41. * This will wake up all threads waiting on this particular completion event.
  42. *
  43. * If this function wakes up a task, it executes a full memory barrier before
  44. * accessing the task state.
  45. *
  46. * Since complete_all() sets the completion of @x permanently to done
  47. * to allow multiple waiters to finish, a call to reinit_completion()
  48. * must be used on @x if @x is to be used again. The code must make
  49. * sure that all waiters have woken and finished before reinitializing
  50. * @x. Also note that the function completion_done() can not be used
  51. * to know if there are still waiters after complete_all() has been called.
  52. */
  53. void complete_all(struct completion *x)
  54. {
  55. unsigned long flags;
  56. lockdep_assert_RT_in_threaded_ctx();
  57. raw_spin_lock_irqsave(&x->wait.lock, flags);
  58. x->done = UINT_MAX;
  59. swake_up_all_locked(&x->wait);
  60. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  61. }
  62. EXPORT_SYMBOL(complete_all);
  63. static inline long __sched
  64. do_wait_for_common(struct completion *x,
  65. long (*action)(long), long timeout, int state)
  66. {
  67. if (!x->done) {
  68. DECLARE_SWAITQUEUE(wait);
  69. do {
  70. if (signal_pending_state(state, current)) {
  71. timeout = -ERESTARTSYS;
  72. break;
  73. }
  74. __prepare_to_swait(&x->wait, &wait);
  75. __set_current_state(state);
  76. raw_spin_unlock_irq(&x->wait.lock);
  77. timeout = action(timeout);
  78. raw_spin_lock_irq(&x->wait.lock);
  79. } while (!x->done && timeout);
  80. __finish_swait(&x->wait, &wait);
  81. if (!x->done)
  82. return timeout;
  83. }
  84. if (x->done != UINT_MAX)
  85. x->done--;
  86. return timeout ?: 1;
  87. }
  88. static inline long __sched
  89. __wait_for_common(struct completion *x,
  90. long (*action)(long), long timeout, int state)
  91. {
  92. might_sleep();
  93. complete_acquire(x);
  94. raw_spin_lock_irq(&x->wait.lock);
  95. timeout = do_wait_for_common(x, action, timeout, state);
  96. raw_spin_unlock_irq(&x->wait.lock);
  97. complete_release(x);
  98. return timeout;
  99. }
  100. static long __sched
  101. wait_for_common(struct completion *x, long timeout, int state)
  102. {
  103. return __wait_for_common(x, schedule_timeout, timeout, state);
  104. }
  105. static long __sched
  106. wait_for_common_io(struct completion *x, long timeout, int state)
  107. {
  108. return __wait_for_common(x, io_schedule_timeout, timeout, state);
  109. }
  110. /**
  111. * wait_for_completion: - waits for completion of a task
  112. * @x: holds the state of this particular completion
  113. *
  114. * This waits to be signaled for completion of a specific task. It is NOT
  115. * interruptible and there is no timeout.
  116. *
  117. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  118. * and interrupt capability. Also see complete().
  119. */
  120. void __sched wait_for_completion(struct completion *x)
  121. {
  122. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  123. }
  124. EXPORT_SYMBOL(wait_for_completion);
  125. /**
  126. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  127. * @x: holds the state of this particular completion
  128. * @timeout: timeout value in jiffies
  129. *
  130. * This waits for either a completion of a specific task to be signaled or for a
  131. * specified timeout to expire. The timeout is in jiffies. It is not
  132. * interruptible.
  133. *
  134. * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
  135. * till timeout) if completed.
  136. */
  137. unsigned long __sched
  138. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  139. {
  140. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  141. }
  142. EXPORT_SYMBOL(wait_for_completion_timeout);
  143. /**
  144. * wait_for_completion_io: - waits for completion of a task
  145. * @x: holds the state of this particular completion
  146. *
  147. * This waits to be signaled for completion of a specific task. It is NOT
  148. * interruptible and there is no timeout. The caller is accounted as waiting
  149. * for IO (which traditionally means blkio only).
  150. */
  151. void __sched wait_for_completion_io(struct completion *x)
  152. {
  153. wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  154. }
  155. EXPORT_SYMBOL(wait_for_completion_io);
  156. /**
  157. * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
  158. * @x: holds the state of this particular completion
  159. * @timeout: timeout value in jiffies
  160. *
  161. * This waits for either a completion of a specific task to be signaled or for a
  162. * specified timeout to expire. The timeout is in jiffies. It is not
  163. * interruptible. The caller is accounted as waiting for IO (which traditionally
  164. * means blkio only).
  165. *
  166. * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
  167. * till timeout) if completed.
  168. */
  169. unsigned long __sched
  170. wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
  171. {
  172. return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
  173. }
  174. EXPORT_SYMBOL(wait_for_completion_io_timeout);
  175. /**
  176. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  177. * @x: holds the state of this particular completion
  178. *
  179. * This waits for completion of a specific task to be signaled. It is
  180. * interruptible.
  181. *
  182. * Return: -ERESTARTSYS if interrupted, 0 if completed.
  183. */
  184. int __sched wait_for_completion_interruptible(struct completion *x)
  185. {
  186. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  187. if (t == -ERESTARTSYS)
  188. return t;
  189. return 0;
  190. }
  191. EXPORT_SYMBOL(wait_for_completion_interruptible);
  192. /**
  193. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  194. * @x: holds the state of this particular completion
  195. * @timeout: timeout value in jiffies
  196. *
  197. * This waits for either a completion of a specific task to be signaled or for a
  198. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  199. *
  200. * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
  201. * or number of jiffies left till timeout) if completed.
  202. */
  203. long __sched
  204. wait_for_completion_interruptible_timeout(struct completion *x,
  205. unsigned long timeout)
  206. {
  207. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  208. }
  209. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  210. /**
  211. * wait_for_completion_killable: - waits for completion of a task (killable)
  212. * @x: holds the state of this particular completion
  213. *
  214. * This waits to be signaled for completion of a specific task. It can be
  215. * interrupted by a kill signal.
  216. *
  217. * Return: -ERESTARTSYS if interrupted, 0 if completed.
  218. */
  219. int __sched wait_for_completion_killable(struct completion *x)
  220. {
  221. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  222. if (t == -ERESTARTSYS)
  223. return t;
  224. return 0;
  225. }
  226. EXPORT_SYMBOL(wait_for_completion_killable);
  227. /**
  228. * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
  229. * @x: holds the state of this particular completion
  230. * @timeout: timeout value in jiffies
  231. *
  232. * This waits for either a completion of a specific task to be
  233. * signaled or for a specified timeout to expire. It can be
  234. * interrupted by a kill signal. The timeout is in jiffies.
  235. *
  236. * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
  237. * or number of jiffies left till timeout) if completed.
  238. */
  239. long __sched
  240. wait_for_completion_killable_timeout(struct completion *x,
  241. unsigned long timeout)
  242. {
  243. return wait_for_common(x, timeout, TASK_KILLABLE);
  244. }
  245. EXPORT_SYMBOL(wait_for_completion_killable_timeout);
  246. /**
  247. * try_wait_for_completion - try to decrement a completion without blocking
  248. * @x: completion structure
  249. *
  250. * Return: 0 if a decrement cannot be done without blocking
  251. * 1 if a decrement succeeded.
  252. *
  253. * If a completion is being used as a counting completion,
  254. * attempt to decrement the counter without blocking. This
  255. * enables us to avoid waiting if the resource the completion
  256. * is protecting is not available.
  257. */
  258. bool try_wait_for_completion(struct completion *x)
  259. {
  260. unsigned long flags;
  261. bool ret = true;
  262. /*
  263. * Since x->done will need to be locked only
  264. * in the non-blocking case, we check x->done
  265. * first without taking the lock so we can
  266. * return early in the blocking case.
  267. */
  268. if (!READ_ONCE(x->done))
  269. return false;
  270. raw_spin_lock_irqsave(&x->wait.lock, flags);
  271. if (!x->done)
  272. ret = false;
  273. else if (x->done != UINT_MAX)
  274. x->done--;
  275. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  276. return ret;
  277. }
  278. EXPORT_SYMBOL(try_wait_for_completion);
  279. /**
  280. * completion_done - Test to see if a completion has any waiters
  281. * @x: completion structure
  282. *
  283. * Return: 0 if there are waiters (wait_for_completion() in progress)
  284. * 1 if there are no waiters.
  285. *
  286. * Note, this will always return true if complete_all() was called on @X.
  287. */
  288. bool completion_done(struct completion *x)
  289. {
  290. unsigned long flags;
  291. if (!READ_ONCE(x->done))
  292. return false;
  293. /*
  294. * If ->done, we need to wait for complete() to release ->wait.lock
  295. * otherwise we can end up freeing the completion before complete()
  296. * is done referencing it.
  297. */
  298. raw_spin_lock_irqsave(&x->wait.lock, flags);
  299. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  300. return true;
  301. }
  302. EXPORT_SYMBOL(completion_done);