closure.c 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Asynchronous refcounty things
  4. *
  5. * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
  6. * Copyright 2012 Google, Inc.
  7. */
  8. #include <linux/debugfs.h>
  9. #include <linux/module.h>
  10. #include <linux/seq_file.h>
  11. #include <linux/sched/debug.h>
  12. #include "closure.h"
  13. static inline void closure_put_after_sub(struct closure *cl, int flags)
  14. {
  15. int r = flags & CLOSURE_REMAINING_MASK;
  16. BUG_ON(flags & CLOSURE_GUARD_MASK);
  17. BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR));
  18. if (!r) {
  19. if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {
  20. atomic_set(&cl->remaining,
  21. CLOSURE_REMAINING_INITIALIZER);
  22. closure_queue(cl);
  23. } else {
  24. struct closure *parent = cl->parent;
  25. closure_fn *destructor = cl->fn;
  26. closure_debug_destroy(cl);
  27. if (destructor)
  28. destructor(cl);
  29. if (parent)
  30. closure_put(parent);
  31. }
  32. }
  33. }
  34. /* For clearing flags with the same atomic op as a put */
  35. void closure_sub(struct closure *cl, int v)
  36. {
  37. closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining));
  38. }
  39. /*
  40. * closure_put - decrement a closure's refcount
  41. */
  42. void closure_put(struct closure *cl)
  43. {
  44. closure_put_after_sub(cl, atomic_dec_return(&cl->remaining));
  45. }
  46. /*
  47. * closure_wake_up - wake up all closures on a wait list, without memory barrier
  48. */
  49. void __closure_wake_up(struct closure_waitlist *wait_list)
  50. {
  51. struct llist_node *list;
  52. struct closure *cl, *t;
  53. struct llist_node *reverse = NULL;
  54. list = llist_del_all(&wait_list->list);
  55. /* We first reverse the list to preserve FIFO ordering and fairness */
  56. reverse = llist_reverse_order(list);
  57. /* Then do the wakeups */
  58. llist_for_each_entry_safe(cl, t, reverse, list) {
  59. closure_set_waiting(cl, 0);
  60. closure_sub(cl, CLOSURE_WAITING + 1);
  61. }
  62. }
  63. /**
  64. * closure_wait - add a closure to a waitlist
  65. * @waitlist: will own a ref on @cl, which will be released when
  66. * closure_wake_up() is called on @waitlist.
  67. * @cl: closure pointer.
  68. *
  69. */
  70. bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)
  71. {
  72. if (atomic_read(&cl->remaining) & CLOSURE_WAITING)
  73. return false;
  74. closure_set_waiting(cl, _RET_IP_);
  75. atomic_add(CLOSURE_WAITING + 1, &cl->remaining);
  76. llist_add(&cl->list, &waitlist->list);
  77. return true;
  78. }
  79. struct closure_syncer {
  80. struct task_struct *task;
  81. int done;
  82. };
  83. static void closure_sync_fn(struct closure *cl)
  84. {
  85. struct closure_syncer *s = cl->s;
  86. struct task_struct *p;
  87. rcu_read_lock();
  88. p = READ_ONCE(s->task);
  89. s->done = 1;
  90. wake_up_process(p);
  91. rcu_read_unlock();
  92. }
  93. void __sched __closure_sync(struct closure *cl)
  94. {
  95. struct closure_syncer s = { .task = current };
  96. cl->s = &s;
  97. continue_at(cl, closure_sync_fn, NULL);
  98. while (1) {
  99. set_current_state(TASK_UNINTERRUPTIBLE);
  100. if (s.done)
  101. break;
  102. schedule();
  103. }
  104. __set_current_state(TASK_RUNNING);
  105. }
  106. #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
  107. static LIST_HEAD(closure_list);
  108. static DEFINE_SPINLOCK(closure_list_lock);
  109. void closure_debug_create(struct closure *cl)
  110. {
  111. unsigned long flags;
  112. BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);
  113. cl->magic = CLOSURE_MAGIC_ALIVE;
  114. spin_lock_irqsave(&closure_list_lock, flags);
  115. list_add(&cl->all, &closure_list);
  116. spin_unlock_irqrestore(&closure_list_lock, flags);
  117. }
  118. void closure_debug_destroy(struct closure *cl)
  119. {
  120. unsigned long flags;
  121. BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);
  122. cl->magic = CLOSURE_MAGIC_DEAD;
  123. spin_lock_irqsave(&closure_list_lock, flags);
  124. list_del(&cl->all);
  125. spin_unlock_irqrestore(&closure_list_lock, flags);
  126. }
  127. static struct dentry *closure_debug;
  128. static int debug_show(struct seq_file *f, void *data)
  129. {
  130. struct closure *cl;
  131. spin_lock_irq(&closure_list_lock);
  132. list_for_each_entry(cl, &closure_list, all) {
  133. int r = atomic_read(&cl->remaining);
  134. seq_printf(f, "%p: %pS -> %pS p %p r %i ",
  135. cl, (void *) cl->ip, cl->fn, cl->parent,
  136. r & CLOSURE_REMAINING_MASK);
  137. seq_printf(f, "%s%s\n",
  138. test_bit(WORK_STRUCT_PENDING_BIT,
  139. work_data_bits(&cl->work)) ? "Q" : "",
  140. r & CLOSURE_RUNNING ? "R" : "");
  141. if (r & CLOSURE_WAITING)
  142. seq_printf(f, " W %pS\n",
  143. (void *) cl->waiting_on);
  144. seq_printf(f, "\n");
  145. }
  146. spin_unlock_irq(&closure_list_lock);
  147. return 0;
  148. }
  149. DEFINE_SHOW_ATTRIBUTE(debug);
  150. void __init closure_debug_init(void)
  151. {
  152. if (!IS_ERR_OR_NULL(bcache_debug))
  153. /*
  154. * it is unnecessary to check return value of
  155. * debugfs_create_file(), we should not care
  156. * about this.
  157. */
  158. closure_debug = debugfs_create_file(
  159. "closures", 0400, bcache_debug, NULL, &debug_fops);
  160. }
  161. #endif
  162. MODULE_AUTHOR("Kent Overstreet <koverstreet@google.com>");
  163. MODULE_LICENSE("GPL");