nouveau_fence.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. /*
  2. * Copyright (C) 2007 Ben Skeggs.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining
  6. * a copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sublicense, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * The above copyright notice and this permission notice (including the
  14. * next paragraph) shall be included in all copies or substantial
  15. * portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20. * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21. * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22. * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24. *
  25. */
  26. #include <linux/ktime.h>
  27. #include <linux/hrtimer.h>
  28. #include <linux/sched/signal.h>
  29. #include <trace/events/dma_fence.h>
  30. #include <nvif/cl826e.h>
  31. #include <nvif/notify.h>
  32. #include <nvif/event.h>
  33. #include "nouveau_drv.h"
  34. #include "nouveau_dma.h"
  35. #include "nouveau_fence.h"
  36. static const struct dma_fence_ops nouveau_fence_ops_uevent;
  37. static const struct dma_fence_ops nouveau_fence_ops_legacy;
  38. static inline struct nouveau_fence *
  39. from_fence(struct dma_fence *fence)
  40. {
  41. return container_of(fence, struct nouveau_fence, base);
  42. }
  43. static inline struct nouveau_fence_chan *
  44. nouveau_fctx(struct nouveau_fence *fence)
  45. {
  46. return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
  47. }
  48. static int
  49. nouveau_fence_signal(struct nouveau_fence *fence)
  50. {
  51. int drop = 0;
  52. dma_fence_signal_locked(&fence->base);
  53. list_del(&fence->head);
  54. rcu_assign_pointer(fence->channel, NULL);
  55. if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
  56. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  57. if (!--fctx->notify_ref)
  58. drop = 1;
  59. }
  60. dma_fence_put(&fence->base);
  61. return drop;
  62. }
  63. static struct nouveau_fence *
  64. nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
  65. {
  66. if (fence->ops != &nouveau_fence_ops_legacy &&
  67. fence->ops != &nouveau_fence_ops_uevent)
  68. return NULL;
  69. if (fence->context < drm->chan.context_base ||
  70. fence->context >= drm->chan.context_base + drm->chan.nr)
  71. return NULL;
  72. return from_fence(fence);
  73. }
  74. void
  75. nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
  76. {
  77. struct nouveau_fence *fence;
  78. spin_lock_irq(&fctx->lock);
  79. while (!list_empty(&fctx->pending)) {
  80. fence = list_entry(fctx->pending.next, typeof(*fence), head);
  81. if (error)
  82. dma_fence_set_error(&fence->base, error);
  83. if (nouveau_fence_signal(fence))
  84. nvif_notify_put(&fctx->notify);
  85. }
  86. spin_unlock_irq(&fctx->lock);
  87. }
  88. void
  89. nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
  90. {
  91. nouveau_fence_context_kill(fctx, 0);
  92. nvif_notify_dtor(&fctx->notify);
  93. fctx->dead = 1;
  94. /*
  95. * Ensure that all accesses to fence->channel complete before freeing
  96. * the channel.
  97. */
  98. synchronize_rcu();
  99. }
  100. static void
  101. nouveau_fence_context_put(struct kref *fence_ref)
  102. {
  103. kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
  104. }
  105. void
  106. nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
  107. {
  108. kref_put(&fctx->fence_ref, nouveau_fence_context_put);
  109. }
  110. static int
  111. nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
  112. {
  113. struct nouveau_fence *fence;
  114. int drop = 0;
  115. u32 seq = fctx->read(chan);
  116. while (!list_empty(&fctx->pending)) {
  117. fence = list_entry(fctx->pending.next, typeof(*fence), head);
  118. if ((int)(seq - fence->base.seqno) < 0)
  119. break;
  120. drop |= nouveau_fence_signal(fence);
  121. }
  122. return drop;
  123. }
  124. static int
  125. nouveau_fence_wait_uevent_handler(struct nvif_notify *notify)
  126. {
  127. struct nouveau_fence_chan *fctx =
  128. container_of(notify, typeof(*fctx), notify);
  129. unsigned long flags;
  130. int ret = NVIF_NOTIFY_KEEP;
  131. spin_lock_irqsave(&fctx->lock, flags);
  132. if (!list_empty(&fctx->pending)) {
  133. struct nouveau_fence *fence;
  134. struct nouveau_channel *chan;
  135. fence = list_entry(fctx->pending.next, typeof(*fence), head);
  136. chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
  137. if (nouveau_fence_update(chan, fctx))
  138. ret = NVIF_NOTIFY_DROP;
  139. }
  140. spin_unlock_irqrestore(&fctx->lock, flags);
  141. return ret;
  142. }
  143. void
  144. nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
  145. {
  146. struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
  147. struct nouveau_cli *cli = (void *)chan->user.client;
  148. int ret;
  149. INIT_LIST_HEAD(&fctx->flip);
  150. INIT_LIST_HEAD(&fctx->pending);
  151. spin_lock_init(&fctx->lock);
  152. fctx->context = chan->drm->chan.context_base + chan->chid;
  153. if (chan == chan->drm->cechan)
  154. strcpy(fctx->name, "copy engine channel");
  155. else if (chan == chan->drm->channel)
  156. strcpy(fctx->name, "generic kernel channel");
  157. else
  158. strcpy(fctx->name, nvxx_client(&cli->base)->name);
  159. kref_init(&fctx->fence_ref);
  160. if (!priv->uevent)
  161. return;
  162. ret = nvif_notify_ctor(&chan->user, "fenceNonStallIntr",
  163. nouveau_fence_wait_uevent_handler,
  164. false, NV826E_V0_NTFY_NON_STALL_INTERRUPT,
  165. &(struct nvif_notify_uevent_req) { },
  166. sizeof(struct nvif_notify_uevent_req),
  167. sizeof(struct nvif_notify_uevent_rep),
  168. &fctx->notify);
  169. WARN_ON(ret);
  170. }
  171. int
  172. nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
  173. {
  174. struct nouveau_fence_chan *fctx = chan->fence;
  175. struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
  176. int ret;
  177. fence->channel = chan;
  178. fence->timeout = jiffies + (15 * HZ);
  179. if (priv->uevent)
  180. dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
  181. &fctx->lock, fctx->context, ++fctx->sequence);
  182. else
  183. dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
  184. &fctx->lock, fctx->context, ++fctx->sequence);
  185. kref_get(&fctx->fence_ref);
  186. trace_dma_fence_emit(&fence->base);
  187. ret = fctx->emit(fence);
  188. if (!ret) {
  189. dma_fence_get(&fence->base);
  190. spin_lock_irq(&fctx->lock);
  191. if (nouveau_fence_update(chan, fctx))
  192. nvif_notify_put(&fctx->notify);
  193. list_add_tail(&fence->head, &fctx->pending);
  194. spin_unlock_irq(&fctx->lock);
  195. }
  196. return ret;
  197. }
  198. bool
  199. nouveau_fence_done(struct nouveau_fence *fence)
  200. {
  201. if (fence->base.ops == &nouveau_fence_ops_legacy ||
  202. fence->base.ops == &nouveau_fence_ops_uevent) {
  203. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  204. struct nouveau_channel *chan;
  205. unsigned long flags;
  206. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
  207. return true;
  208. spin_lock_irqsave(&fctx->lock, flags);
  209. chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
  210. if (chan && nouveau_fence_update(chan, fctx))
  211. nvif_notify_put(&fctx->notify);
  212. spin_unlock_irqrestore(&fctx->lock, flags);
  213. }
  214. return dma_fence_is_signaled(&fence->base);
  215. }
  216. static long
  217. nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
  218. {
  219. struct nouveau_fence *fence = from_fence(f);
  220. unsigned long sleep_time = NSEC_PER_MSEC / 1000;
  221. unsigned long t = jiffies, timeout = t + wait;
  222. while (!nouveau_fence_done(fence)) {
  223. ktime_t kt;
  224. t = jiffies;
  225. if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
  226. __set_current_state(TASK_RUNNING);
  227. return 0;
  228. }
  229. __set_current_state(intr ? TASK_INTERRUPTIBLE :
  230. TASK_UNINTERRUPTIBLE);
  231. kt = sleep_time;
  232. schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
  233. sleep_time *= 2;
  234. if (sleep_time > NSEC_PER_MSEC)
  235. sleep_time = NSEC_PER_MSEC;
  236. if (intr && signal_pending(current))
  237. return -ERESTARTSYS;
  238. }
  239. __set_current_state(TASK_RUNNING);
  240. return timeout - t;
  241. }
  242. static int
  243. nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
  244. {
  245. int ret = 0;
  246. while (!nouveau_fence_done(fence)) {
  247. if (time_after_eq(jiffies, fence->timeout)) {
  248. ret = -EBUSY;
  249. break;
  250. }
  251. __set_current_state(intr ?
  252. TASK_INTERRUPTIBLE :
  253. TASK_UNINTERRUPTIBLE);
  254. if (intr && signal_pending(current)) {
  255. ret = -ERESTARTSYS;
  256. break;
  257. }
  258. }
  259. __set_current_state(TASK_RUNNING);
  260. return ret;
  261. }
  262. int
  263. nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
  264. {
  265. long ret;
  266. if (!lazy)
  267. return nouveau_fence_wait_busy(fence, intr);
  268. ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
  269. if (ret < 0)
  270. return ret;
  271. else if (!ret)
  272. return -EBUSY;
  273. else
  274. return 0;
  275. }
  276. int
  277. nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
  278. {
  279. struct nouveau_fence_chan *fctx = chan->fence;
  280. struct dma_fence *fence;
  281. struct dma_resv *resv = nvbo->bo.base.resv;
  282. struct dma_resv_list *fobj;
  283. struct nouveau_fence *f;
  284. int ret = 0, i;
  285. if (!exclusive) {
  286. ret = dma_resv_reserve_shared(resv, 1);
  287. if (ret)
  288. return ret;
  289. }
  290. fobj = dma_resv_get_list(resv);
  291. fence = dma_resv_get_excl(resv);
  292. if (fence && (!exclusive || !fobj || !fobj->shared_count)) {
  293. struct nouveau_channel *prev = NULL;
  294. bool must_wait = true;
  295. f = nouveau_local_fence(fence, chan->drm);
  296. if (f) {
  297. rcu_read_lock();
  298. prev = rcu_dereference(f->channel);
  299. if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
  300. must_wait = false;
  301. rcu_read_unlock();
  302. }
  303. if (must_wait)
  304. ret = dma_fence_wait(fence, intr);
  305. return ret;
  306. }
  307. if (!exclusive || !fobj)
  308. return ret;
  309. for (i = 0; i < fobj->shared_count && !ret; ++i) {
  310. struct nouveau_channel *prev = NULL;
  311. bool must_wait = true;
  312. fence = rcu_dereference_protected(fobj->shared[i],
  313. dma_resv_held(resv));
  314. f = nouveau_local_fence(fence, chan->drm);
  315. if (f) {
  316. rcu_read_lock();
  317. prev = rcu_dereference(f->channel);
  318. if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0))
  319. must_wait = false;
  320. rcu_read_unlock();
  321. }
  322. if (must_wait)
  323. ret = dma_fence_wait(fence, intr);
  324. }
  325. return ret;
  326. }
  327. void
  328. nouveau_fence_unref(struct nouveau_fence **pfence)
  329. {
  330. if (*pfence)
  331. dma_fence_put(&(*pfence)->base);
  332. *pfence = NULL;
  333. }
  334. int
  335. nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
  336. struct nouveau_fence **pfence)
  337. {
  338. struct nouveau_fence *fence;
  339. int ret = 0;
  340. if (unlikely(!chan->fence))
  341. return -ENODEV;
  342. fence = kzalloc(sizeof(*fence), GFP_KERNEL);
  343. if (!fence)
  344. return -ENOMEM;
  345. ret = nouveau_fence_emit(fence, chan);
  346. if (ret)
  347. nouveau_fence_unref(&fence);
  348. *pfence = fence;
  349. return ret;
  350. }
  351. static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
  352. {
  353. return "nouveau";
  354. }
  355. static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
  356. {
  357. struct nouveau_fence *fence = from_fence(f);
  358. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  359. return !fctx->dead ? fctx->name : "dead channel";
  360. }
  361. /*
  362. * In an ideal world, read would not assume the channel context is still alive.
  363. * This function may be called from another device, running into free memory as a
  364. * result. The drm node should still be there, so we can derive the index from
  365. * the fence context.
  366. */
  367. static bool nouveau_fence_is_signaled(struct dma_fence *f)
  368. {
  369. struct nouveau_fence *fence = from_fence(f);
  370. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  371. struct nouveau_channel *chan;
  372. bool ret = false;
  373. rcu_read_lock();
  374. chan = rcu_dereference(fence->channel);
  375. if (chan)
  376. ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
  377. rcu_read_unlock();
  378. return ret;
  379. }
  380. static bool nouveau_fence_no_signaling(struct dma_fence *f)
  381. {
  382. struct nouveau_fence *fence = from_fence(f);
  383. /*
  384. * caller should have a reference on the fence,
  385. * else fence could get freed here
  386. */
  387. WARN_ON(kref_read(&fence->base.refcount) <= 1);
  388. /*
  389. * This needs uevents to work correctly, but dma_fence_add_callback relies on
  390. * being able to enable signaling. It will still get signaled eventually,
  391. * just not right away.
  392. */
  393. if (nouveau_fence_is_signaled(f)) {
  394. list_del(&fence->head);
  395. dma_fence_put(&fence->base);
  396. return false;
  397. }
  398. return true;
  399. }
  400. static void nouveau_fence_release(struct dma_fence *f)
  401. {
  402. struct nouveau_fence *fence = from_fence(f);
  403. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  404. kref_put(&fctx->fence_ref, nouveau_fence_context_put);
  405. dma_fence_free(&fence->base);
  406. }
  407. static const struct dma_fence_ops nouveau_fence_ops_legacy = {
  408. .get_driver_name = nouveau_fence_get_get_driver_name,
  409. .get_timeline_name = nouveau_fence_get_timeline_name,
  410. .enable_signaling = nouveau_fence_no_signaling,
  411. .signaled = nouveau_fence_is_signaled,
  412. .wait = nouveau_fence_wait_legacy,
  413. .release = nouveau_fence_release
  414. };
  415. static bool nouveau_fence_enable_signaling(struct dma_fence *f)
  416. {
  417. struct nouveau_fence *fence = from_fence(f);
  418. struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  419. bool ret;
  420. if (!fctx->notify_ref++)
  421. nvif_notify_get(&fctx->notify);
  422. ret = nouveau_fence_no_signaling(f);
  423. if (ret)
  424. set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
  425. else if (!--fctx->notify_ref)
  426. nvif_notify_put(&fctx->notify);
  427. return ret;
  428. }
  429. static const struct dma_fence_ops nouveau_fence_ops_uevent = {
  430. .get_driver_name = nouveau_fence_get_get_driver_name,
  431. .get_timeline_name = nouveau_fence_get_timeline_name,
  432. .enable_signaling = nouveau_fence_enable_signaling,
  433. .signaled = nouveau_fence_is_signaled,
  434. .release = nouveau_fence_release
  435. };