st-dma-fence.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574
  1. /* SPDX-License-Identifier: MIT */
  2. /*
  3. * Copyright © 2019 Intel Corporation
  4. */
  5. #include <linux/delay.h>
  6. #include <linux/dma-fence.h>
  7. #include <linux/kernel.h>
  8. #include <linux/kthread.h>
  9. #include <linux/sched/signal.h>
  10. #include <linux/slab.h>
  11. #include <linux/spinlock.h>
  12. #include "selftest.h"
  13. static struct kmem_cache *slab_fences;
  14. static struct mock_fence {
  15. struct dma_fence base;
  16. struct spinlock lock;
  17. } *to_mock_fence(struct dma_fence *f) {
  18. return container_of(f, struct mock_fence, base);
  19. }
  20. static const char *mock_name(struct dma_fence *f)
  21. {
  22. return "mock";
  23. }
  24. static void mock_fence_release(struct dma_fence *f)
  25. {
  26. kmem_cache_free(slab_fences, to_mock_fence(f));
  27. }
  28. struct wait_cb {
  29. struct dma_fence_cb cb;
  30. struct task_struct *task;
  31. };
  32. static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
  33. {
  34. wake_up_process(container_of(cb, struct wait_cb, cb)->task);
  35. }
  36. static long mock_wait(struct dma_fence *f, bool intr, long timeout)
  37. {
  38. const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
  39. struct wait_cb cb = { .task = current };
  40. if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
  41. return timeout;
  42. while (timeout) {
  43. set_current_state(state);
  44. if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
  45. break;
  46. if (signal_pending_state(state, current))
  47. break;
  48. timeout = schedule_timeout(timeout);
  49. }
  50. __set_current_state(TASK_RUNNING);
  51. if (!dma_fence_remove_callback(f, &cb.cb))
  52. return timeout;
  53. if (signal_pending_state(state, current))
  54. return -ERESTARTSYS;
  55. return -ETIME;
  56. }
  57. static const struct dma_fence_ops mock_ops = {
  58. .get_driver_name = mock_name,
  59. .get_timeline_name = mock_name,
  60. .wait = mock_wait,
  61. .release = mock_fence_release,
  62. };
  63. static struct dma_fence *mock_fence(void)
  64. {
  65. struct mock_fence *f;
  66. f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
  67. if (!f)
  68. return NULL;
  69. spin_lock_init(&f->lock);
  70. dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
  71. return &f->base;
  72. }
  73. static int sanitycheck(void *arg)
  74. {
  75. struct dma_fence *f;
  76. f = mock_fence();
  77. if (!f)
  78. return -ENOMEM;
  79. dma_fence_signal(f);
  80. dma_fence_put(f);
  81. return 0;
  82. }
  83. static int test_signaling(void *arg)
  84. {
  85. struct dma_fence *f;
  86. int err = -EINVAL;
  87. f = mock_fence();
  88. if (!f)
  89. return -ENOMEM;
  90. if (dma_fence_is_signaled(f)) {
  91. pr_err("Fence unexpectedly signaled on creation\n");
  92. goto err_free;
  93. }
  94. if (dma_fence_signal(f)) {
  95. pr_err("Fence reported being already signaled\n");
  96. goto err_free;
  97. }
  98. if (!dma_fence_is_signaled(f)) {
  99. pr_err("Fence not reporting signaled\n");
  100. goto err_free;
  101. }
  102. if (!dma_fence_signal(f)) {
  103. pr_err("Fence reported not being already signaled\n");
  104. goto err_free;
  105. }
  106. err = 0;
  107. err_free:
  108. dma_fence_put(f);
  109. return err;
  110. }
  111. struct simple_cb {
  112. struct dma_fence_cb cb;
  113. bool seen;
  114. };
  115. static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
  116. {
  117. smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
  118. }
  119. static int test_add_callback(void *arg)
  120. {
  121. struct simple_cb cb = {};
  122. struct dma_fence *f;
  123. int err = -EINVAL;
  124. f = mock_fence();
  125. if (!f)
  126. return -ENOMEM;
  127. if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
  128. pr_err("Failed to add callback, fence already signaled!\n");
  129. goto err_free;
  130. }
  131. dma_fence_signal(f);
  132. if (!cb.seen) {
  133. pr_err("Callback failed!\n");
  134. goto err_free;
  135. }
  136. err = 0;
  137. err_free:
  138. dma_fence_put(f);
  139. return err;
  140. }
  141. static int test_late_add_callback(void *arg)
  142. {
  143. struct simple_cb cb = {};
  144. struct dma_fence *f;
  145. int err = -EINVAL;
  146. f = mock_fence();
  147. if (!f)
  148. return -ENOMEM;
  149. dma_fence_signal(f);
  150. if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
  151. pr_err("Added callback, but fence was already signaled!\n");
  152. goto err_free;
  153. }
  154. dma_fence_signal(f);
  155. if (cb.seen) {
  156. pr_err("Callback called after failed attachment !\n");
  157. goto err_free;
  158. }
  159. err = 0;
  160. err_free:
  161. dma_fence_put(f);
  162. return err;
  163. }
  164. static int test_rm_callback(void *arg)
  165. {
  166. struct simple_cb cb = {};
  167. struct dma_fence *f;
  168. int err = -EINVAL;
  169. f = mock_fence();
  170. if (!f)
  171. return -ENOMEM;
  172. if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
  173. pr_err("Failed to add callback, fence already signaled!\n");
  174. goto err_free;
  175. }
  176. if (!dma_fence_remove_callback(f, &cb.cb)) {
  177. pr_err("Failed to remove callback!\n");
  178. goto err_free;
  179. }
  180. dma_fence_signal(f);
  181. if (cb.seen) {
  182. pr_err("Callback still signaled after removal!\n");
  183. goto err_free;
  184. }
  185. err = 0;
  186. err_free:
  187. dma_fence_put(f);
  188. return err;
  189. }
  190. static int test_late_rm_callback(void *arg)
  191. {
  192. struct simple_cb cb = {};
  193. struct dma_fence *f;
  194. int err = -EINVAL;
  195. f = mock_fence();
  196. if (!f)
  197. return -ENOMEM;
  198. if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
  199. pr_err("Failed to add callback, fence already signaled!\n");
  200. goto err_free;
  201. }
  202. dma_fence_signal(f);
  203. if (!cb.seen) {
  204. pr_err("Callback failed!\n");
  205. goto err_free;
  206. }
  207. if (dma_fence_remove_callback(f, &cb.cb)) {
  208. pr_err("Callback removal succeed after being executed!\n");
  209. goto err_free;
  210. }
  211. err = 0;
  212. err_free:
  213. dma_fence_put(f);
  214. return err;
  215. }
  216. static int test_status(void *arg)
  217. {
  218. struct dma_fence *f;
  219. int err = -EINVAL;
  220. f = mock_fence();
  221. if (!f)
  222. return -ENOMEM;
  223. if (dma_fence_get_status(f)) {
  224. pr_err("Fence unexpectedly has signaled status on creation\n");
  225. goto err_free;
  226. }
  227. dma_fence_signal(f);
  228. if (!dma_fence_get_status(f)) {
  229. pr_err("Fence not reporting signaled status\n");
  230. goto err_free;
  231. }
  232. err = 0;
  233. err_free:
  234. dma_fence_put(f);
  235. return err;
  236. }
  237. static int test_error(void *arg)
  238. {
  239. struct dma_fence *f;
  240. int err = -EINVAL;
  241. f = mock_fence();
  242. if (!f)
  243. return -ENOMEM;
  244. dma_fence_set_error(f, -EIO);
  245. if (dma_fence_get_status(f)) {
  246. pr_err("Fence unexpectedly has error status before signal\n");
  247. goto err_free;
  248. }
  249. dma_fence_signal(f);
  250. if (dma_fence_get_status(f) != -EIO) {
  251. pr_err("Fence not reporting error status, got %d\n",
  252. dma_fence_get_status(f));
  253. goto err_free;
  254. }
  255. err = 0;
  256. err_free:
  257. dma_fence_put(f);
  258. return err;
  259. }
  260. static int test_wait(void *arg)
  261. {
  262. struct dma_fence *f;
  263. int err = -EINVAL;
  264. f = mock_fence();
  265. if (!f)
  266. return -ENOMEM;
  267. if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
  268. pr_err("Wait reported complete before being signaled\n");
  269. goto err_free;
  270. }
  271. dma_fence_signal(f);
  272. if (dma_fence_wait_timeout(f, false, 0) != 0) {
  273. pr_err("Wait reported incomplete after being signaled\n");
  274. goto err_free;
  275. }
  276. err = 0;
  277. err_free:
  278. dma_fence_signal(f);
  279. dma_fence_put(f);
  280. return err;
  281. }
  282. struct wait_timer {
  283. struct timer_list timer;
  284. struct dma_fence *f;
  285. };
  286. static void wait_timer(struct timer_list *timer)
  287. {
  288. struct wait_timer *wt = from_timer(wt, timer, timer);
  289. dma_fence_signal(wt->f);
  290. }
  291. static int test_wait_timeout(void *arg)
  292. {
  293. struct wait_timer wt;
  294. int err = -EINVAL;
  295. timer_setup_on_stack(&wt.timer, wait_timer, 0);
  296. wt.f = mock_fence();
  297. if (!wt.f)
  298. return -ENOMEM;
  299. if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
  300. pr_err("Wait reported complete before being signaled\n");
  301. goto err_free;
  302. }
  303. mod_timer(&wt.timer, jiffies + 1);
  304. if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
  305. if (timer_pending(&wt.timer)) {
  306. pr_notice("Timer did not fire within the jiffie!\n");
  307. err = 0; /* not our fault! */
  308. } else {
  309. pr_err("Wait reported incomplete after timeout\n");
  310. }
  311. goto err_free;
  312. }
  313. err = 0;
  314. err_free:
  315. del_timer_sync(&wt.timer);
  316. destroy_timer_on_stack(&wt.timer);
  317. dma_fence_signal(wt.f);
  318. dma_fence_put(wt.f);
  319. return err;
  320. }
  321. static int test_stub(void *arg)
  322. {
  323. struct dma_fence *f[64];
  324. int err = -EINVAL;
  325. int i;
  326. for (i = 0; i < ARRAY_SIZE(f); i++) {
  327. f[i] = dma_fence_get_stub();
  328. if (!dma_fence_is_signaled(f[i])) {
  329. pr_err("Obtained unsignaled stub fence!\n");
  330. goto err;
  331. }
  332. }
  333. err = 0;
  334. err:
  335. while (i--)
  336. dma_fence_put(f[i]);
  337. return err;
  338. }
  339. /* Now off to the races! */
  340. struct race_thread {
  341. struct dma_fence __rcu **fences;
  342. struct task_struct *task;
  343. bool before;
  344. int id;
  345. };
  346. static void __wait_for_callbacks(struct dma_fence *f)
  347. {
  348. spin_lock_irq(f->lock);
  349. spin_unlock_irq(f->lock);
  350. }
  351. static int thread_signal_callback(void *arg)
  352. {
  353. const struct race_thread *t = arg;
  354. unsigned long pass = 0;
  355. unsigned long miss = 0;
  356. int err = 0;
  357. while (!err && !kthread_should_stop()) {
  358. struct dma_fence *f1, *f2;
  359. struct simple_cb cb;
  360. f1 = mock_fence();
  361. if (!f1) {
  362. err = -ENOMEM;
  363. break;
  364. }
  365. rcu_assign_pointer(t->fences[t->id], f1);
  366. smp_wmb();
  367. rcu_read_lock();
  368. do {
  369. f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);
  370. } while (!f2 && !kthread_should_stop());
  371. rcu_read_unlock();
  372. if (t->before)
  373. dma_fence_signal(f1);
  374. smp_store_mb(cb.seen, false);
  375. if (!f2 || dma_fence_add_callback(f2, &cb.cb, simple_callback))
  376. miss++, cb.seen = true;
  377. if (!t->before)
  378. dma_fence_signal(f1);
  379. if (!cb.seen) {
  380. dma_fence_wait(f2, false);
  381. __wait_for_callbacks(f2);
  382. }
  383. if (!READ_ONCE(cb.seen)) {
  384. pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
  385. t->id, pass, miss,
  386. t->before ? "before" : "after",
  387. dma_fence_is_signaled(f2) ? "yes" : "no");
  388. err = -EINVAL;
  389. }
  390. dma_fence_put(f2);
  391. rcu_assign_pointer(t->fences[t->id], NULL);
  392. smp_wmb();
  393. dma_fence_put(f1);
  394. pass++;
  395. }
  396. pr_info("%s[%d] completed %lu passes, %lu misses\n",
  397. __func__, t->id, pass, miss);
  398. return err;
  399. }
  400. static int race_signal_callback(void *arg)
  401. {
  402. struct dma_fence __rcu *f[2] = {};
  403. int ret = 0;
  404. int pass;
  405. for (pass = 0; !ret && pass <= 1; pass++) {
  406. struct race_thread t[2];
  407. int i;
  408. for (i = 0; i < ARRAY_SIZE(t); i++) {
  409. t[i].fences = f;
  410. t[i].id = i;
  411. t[i].before = pass;
  412. t[i].task = kthread_run(thread_signal_callback, &t[i],
  413. "dma-fence:%d", i);
  414. get_task_struct(t[i].task);
  415. }
  416. msleep(50);
  417. for (i = 0; i < ARRAY_SIZE(t); i++) {
  418. int err;
  419. err = kthread_stop(t[i].task);
  420. if (err && !ret)
  421. ret = err;
  422. put_task_struct(t[i].task);
  423. }
  424. }
  425. return ret;
  426. }
  427. int dma_fence(void)
  428. {
  429. static const struct subtest tests[] = {
  430. SUBTEST(sanitycheck),
  431. SUBTEST(test_signaling),
  432. SUBTEST(test_add_callback),
  433. SUBTEST(test_late_add_callback),
  434. SUBTEST(test_rm_callback),
  435. SUBTEST(test_late_rm_callback),
  436. SUBTEST(test_status),
  437. SUBTEST(test_error),
  438. SUBTEST(test_wait),
  439. SUBTEST(test_wait_timeout),
  440. SUBTEST(test_stub),
  441. SUBTEST(race_signal_callback),
  442. };
  443. int ret;
  444. pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
  445. slab_fences = KMEM_CACHE(mock_fence,
  446. SLAB_TYPESAFE_BY_RCU |
  447. SLAB_HWCACHE_ALIGN);
  448. if (!slab_fences)
  449. return -ENOMEM;
  450. ret = subtests(tests, NULL);
  451. kmem_cache_destroy(slab_fences);
  452. return ret;
  453. }