dma-fence-chain.c 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * fence-chain: chain fences together in a timeline
  4. *
  5. * Copyright (C) 2018 Advanced Micro Devices, Inc.
  6. * Authors:
  7. * Christian König <christian.koenig@amd.com>
  8. */
  9. #include <linux/dma-fence-chain.h>
  10. static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
  11. /**
  12. * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
  13. * @chain: chain node to get the previous node from
  14. *
  15. * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
  16. * chain node.
  17. */
  18. static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
  19. {
  20. struct dma_fence *prev;
  21. rcu_read_lock();
  22. prev = dma_fence_get_rcu_safe(&chain->prev);
  23. rcu_read_unlock();
  24. return prev;
  25. }
  26. /**
  27. * dma_fence_chain_walk - chain walking function
  28. * @fence: current chain node
  29. *
  30. * Walk the chain to the next node. Returns the next fence or NULL if we are at
  31. * the end of the chain. Garbage collects chain nodes which are already
  32. * signaled.
  33. */
  34. struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
  35. {
  36. struct dma_fence_chain *chain, *prev_chain;
  37. struct dma_fence *prev, *replacement, *tmp;
  38. chain = to_dma_fence_chain(fence);
  39. if (!chain) {
  40. dma_fence_put(fence);
  41. return NULL;
  42. }
  43. while ((prev = dma_fence_chain_get_prev(chain))) {
  44. prev_chain = to_dma_fence_chain(prev);
  45. if (prev_chain) {
  46. if (!dma_fence_is_signaled(prev_chain->fence))
  47. break;
  48. replacement = dma_fence_chain_get_prev(prev_chain);
  49. } else {
  50. if (!dma_fence_is_signaled(prev))
  51. break;
  52. replacement = NULL;
  53. }
  54. tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
  55. prev, replacement);
  56. if (tmp == prev)
  57. dma_fence_put(tmp);
  58. else
  59. dma_fence_put(replacement);
  60. dma_fence_put(prev);
  61. }
  62. dma_fence_put(fence);
  63. return prev;
  64. }
  65. EXPORT_SYMBOL(dma_fence_chain_walk);
  66. /**
  67. * dma_fence_chain_find_seqno - find fence chain node by seqno
  68. * @pfence: pointer to the chain node where to start
  69. * @seqno: the sequence number to search for
  70. *
  71. * Advance the fence pointer to the chain node which will signal this sequence
  72. * number. If no sequence number is provided then this is a no-op.
  73. *
  74. * Returns EINVAL if the fence is not a chain node or the sequence number has
  75. * not yet advanced far enough.
  76. */
  77. int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
  78. {
  79. struct dma_fence_chain *chain;
  80. if (!seqno)
  81. return 0;
  82. chain = to_dma_fence_chain(*pfence);
  83. if (!chain || chain->base.seqno < seqno)
  84. return -EINVAL;
  85. dma_fence_chain_for_each(*pfence, &chain->base) {
  86. if ((*pfence)->context != chain->base.context ||
  87. to_dma_fence_chain(*pfence)->prev_seqno < seqno)
  88. break;
  89. }
  90. dma_fence_put(&chain->base);
  91. return 0;
  92. }
  93. EXPORT_SYMBOL(dma_fence_chain_find_seqno);
  94. static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
  95. {
  96. return "dma_fence_chain";
  97. }
  98. static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
  99. {
  100. return "unbound";
  101. }
  102. static void dma_fence_chain_irq_work(struct irq_work *work)
  103. {
  104. struct dma_fence_chain *chain;
  105. chain = container_of(work, typeof(*chain), work);
  106. /* Try to rearm the callback */
  107. if (!dma_fence_chain_enable_signaling(&chain->base))
  108. /* Ok, we are done. No more unsignaled fences left */
  109. dma_fence_signal(&chain->base);
  110. dma_fence_put(&chain->base);
  111. }
  112. static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
  113. {
  114. struct dma_fence_chain *chain;
  115. chain = container_of(cb, typeof(*chain), cb);
  116. irq_work_queue(&chain->work);
  117. dma_fence_put(f);
  118. }
  119. static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
  120. {
  121. struct dma_fence_chain *head = to_dma_fence_chain(fence);
  122. dma_fence_get(&head->base);
  123. dma_fence_chain_for_each(fence, &head->base) {
  124. struct dma_fence_chain *chain = to_dma_fence_chain(fence);
  125. struct dma_fence *f = chain ? chain->fence : fence;
  126. dma_fence_get(f);
  127. if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
  128. dma_fence_put(fence);
  129. return true;
  130. }
  131. dma_fence_put(f);
  132. }
  133. dma_fence_put(&head->base);
  134. return false;
  135. }
  136. static bool dma_fence_chain_signaled(struct dma_fence *fence)
  137. {
  138. dma_fence_chain_for_each(fence, fence) {
  139. struct dma_fence_chain *chain = to_dma_fence_chain(fence);
  140. struct dma_fence *f = chain ? chain->fence : fence;
  141. if (!dma_fence_is_signaled(f)) {
  142. dma_fence_put(fence);
  143. return false;
  144. }
  145. }
  146. return true;
  147. }
  148. static void dma_fence_chain_release(struct dma_fence *fence)
  149. {
  150. struct dma_fence_chain *chain = to_dma_fence_chain(fence);
  151. struct dma_fence *prev;
  152. /* Manually unlink the chain as much as possible to avoid recursion
  153. * and potential stack overflow.
  154. */
  155. while ((prev = rcu_dereference_protected(chain->prev, true))) {
  156. struct dma_fence_chain *prev_chain;
  157. if (kref_read(&prev->refcount) > 1)
  158. break;
  159. prev_chain = to_dma_fence_chain(prev);
  160. if (!prev_chain)
  161. break;
  162. /* No need for atomic operations since we hold the last
  163. * reference to prev_chain.
  164. */
  165. chain->prev = prev_chain->prev;
  166. RCU_INIT_POINTER(prev_chain->prev, NULL);
  167. dma_fence_put(prev);
  168. }
  169. dma_fence_put(prev);
  170. dma_fence_put(chain->fence);
  171. dma_fence_free(fence);
  172. }
  173. const struct dma_fence_ops dma_fence_chain_ops = {
  174. .use_64bit_seqno = true,
  175. .get_driver_name = dma_fence_chain_get_driver_name,
  176. .get_timeline_name = dma_fence_chain_get_timeline_name,
  177. .enable_signaling = dma_fence_chain_enable_signaling,
  178. .signaled = dma_fence_chain_signaled,
  179. .release = dma_fence_chain_release,
  180. };
  181. EXPORT_SYMBOL(dma_fence_chain_ops);
  182. /**
  183. * dma_fence_chain_init - initialize a fence chain
  184. * @chain: the chain node to initialize
  185. * @prev: the previous fence
  186. * @fence: the current fence
  187. * @seqno: the sequence number to use for the fence chain
  188. *
  189. * Initialize a new chain node and either start a new chain or add the node to
  190. * the existing chain of the previous fence.
  191. */
  192. void dma_fence_chain_init(struct dma_fence_chain *chain,
  193. struct dma_fence *prev,
  194. struct dma_fence *fence,
  195. uint64_t seqno)
  196. {
  197. struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
  198. uint64_t context;
  199. spin_lock_init(&chain->lock);
  200. rcu_assign_pointer(chain->prev, prev);
  201. chain->fence = fence;
  202. chain->prev_seqno = 0;
  203. init_irq_work(&chain->work, dma_fence_chain_irq_work);
  204. /* Try to reuse the context of the previous chain node. */
  205. if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
  206. context = prev->context;
  207. chain->prev_seqno = prev->seqno;
  208. } else {
  209. context = dma_fence_context_alloc(1);
  210. /* Make sure that we always have a valid sequence number. */
  211. if (prev_chain)
  212. seqno = max(prev->seqno, seqno);
  213. }
  214. dma_fence_init(&chain->base, &dma_fence_chain_ops,
  215. &chain->lock, context, seqno);
  216. }
  217. EXPORT_SYMBOL(dma_fence_chain_init);