v3d_sched.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /* Copyright (C) 2018 Broadcom */
  3. /**
  4. * DOC: Broadcom V3D scheduling
  5. *
  6. * The shared DRM GPU scheduler is used to coordinate submitting jobs
  7. * to the hardware. Each DRM fd (roughly a client process) gets its
  8. * own scheduler entity, which will process jobs in order. The GPU
  9. * scheduler will round-robin between clients to submit the next job.
  10. *
  11. * For simplicity, and in order to keep latency low for interactive
  12. * jobs when bulk background jobs are queued up, we submit a new job
  13. * to the HW only when it has completed the last one, instead of
  14. * filling up the CT[01]Q FIFOs with jobs. Similarly, we use
  15. * v3d_job_dependency() to manage the dependency between bin and
  16. * render, instead of having the clients submit jobs using the HW's
  17. * semaphores to interlock between them.
  18. */
  19. #include <linux/kthread.h>
  20. #include "v3d_drv.h"
  21. #include "v3d_regs.h"
  22. #include "v3d_trace.h"
  23. static struct v3d_job *
  24. to_v3d_job(struct drm_sched_job *sched_job)
  25. {
  26. return container_of(sched_job, struct v3d_job, base);
  27. }
  28. static struct v3d_bin_job *
  29. to_bin_job(struct drm_sched_job *sched_job)
  30. {
  31. return container_of(sched_job, struct v3d_bin_job, base.base);
  32. }
  33. static struct v3d_render_job *
  34. to_render_job(struct drm_sched_job *sched_job)
  35. {
  36. return container_of(sched_job, struct v3d_render_job, base.base);
  37. }
  38. static struct v3d_tfu_job *
  39. to_tfu_job(struct drm_sched_job *sched_job)
  40. {
  41. return container_of(sched_job, struct v3d_tfu_job, base.base);
  42. }
  43. static struct v3d_csd_job *
  44. to_csd_job(struct drm_sched_job *sched_job)
  45. {
  46. return container_of(sched_job, struct v3d_csd_job, base.base);
  47. }
  48. static void
  49. v3d_job_free(struct drm_sched_job *sched_job)
  50. {
  51. struct v3d_job *job = to_v3d_job(sched_job);
  52. drm_sched_job_cleanup(sched_job);
  53. v3d_job_put(job);
  54. }
  55. /**
  56. * Returns the fences that the job depends on, one by one.
  57. *
  58. * If placed in the scheduler's .dependency method, the corresponding
  59. * .run_job won't be called until all of them have been signaled.
  60. */
  61. static struct dma_fence *
  62. v3d_job_dependency(struct drm_sched_job *sched_job,
  63. struct drm_sched_entity *s_entity)
  64. {
  65. struct v3d_job *job = to_v3d_job(sched_job);
  66. /* XXX: Wait on a fence for switching the GMP if necessary,
  67. * and then do so.
  68. */
  69. if (!xa_empty(&job->deps))
  70. return xa_erase(&job->deps, job->last_dep++);
  71. return NULL;
  72. }
  73. static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
  74. {
  75. struct v3d_bin_job *job = to_bin_job(sched_job);
  76. struct v3d_dev *v3d = job->base.v3d;
  77. struct drm_device *dev = &v3d->drm;
  78. struct dma_fence *fence;
  79. unsigned long irqflags;
  80. if (unlikely(job->base.base.s_fence->finished.error))
  81. return NULL;
  82. /* Lock required around bin_job update vs
  83. * v3d_overflow_mem_work().
  84. */
  85. spin_lock_irqsave(&v3d->job_lock, irqflags);
  86. v3d->bin_job = job;
  87. /* Clear out the overflow allocation, so we don't
  88. * reuse the overflow attached to a previous job.
  89. */
  90. V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
  91. spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  92. v3d_invalidate_caches(v3d);
  93. fence = v3d_fence_create(v3d, V3D_BIN);
  94. if (IS_ERR(fence))
  95. return NULL;
  96. if (job->base.irq_fence)
  97. dma_fence_put(job->base.irq_fence);
  98. job->base.irq_fence = dma_fence_get(fence);
  99. trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
  100. job->start, job->end);
  101. /* Set the current and end address of the control list.
  102. * Writing the end register is what starts the job.
  103. */
  104. if (job->qma) {
  105. V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);
  106. V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);
  107. }
  108. if (job->qts) {
  109. V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
  110. V3D_CLE_CT0QTS_ENABLE |
  111. job->qts);
  112. }
  113. V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);
  114. V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);
  115. return fence;
  116. }
  117. static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
  118. {
  119. struct v3d_render_job *job = to_render_job(sched_job);
  120. struct v3d_dev *v3d = job->base.v3d;
  121. struct drm_device *dev = &v3d->drm;
  122. struct dma_fence *fence;
  123. if (unlikely(job->base.base.s_fence->finished.error))
  124. return NULL;
  125. v3d->render_job = job;
  126. /* Can we avoid this flush? We need to be careful of
  127. * scheduling, though -- imagine job0 rendering to texture and
  128. * job1 reading, and them being executed as bin0, bin1,
  129. * render0, render1, so that render1's flush at bin time
  130. * wasn't enough.
  131. */
  132. v3d_invalidate_caches(v3d);
  133. fence = v3d_fence_create(v3d, V3D_RENDER);
  134. if (IS_ERR(fence))
  135. return NULL;
  136. if (job->base.irq_fence)
  137. dma_fence_put(job->base.irq_fence);
  138. job->base.irq_fence = dma_fence_get(fence);
  139. trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
  140. job->start, job->end);
  141. /* XXX: Set the QCFG */
  142. /* Set the current and end address of the control list.
  143. * Writing the end register is what starts the job.
  144. */
  145. V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);
  146. V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);
  147. return fence;
  148. }
  149. static struct dma_fence *
  150. v3d_tfu_job_run(struct drm_sched_job *sched_job)
  151. {
  152. struct v3d_tfu_job *job = to_tfu_job(sched_job);
  153. struct v3d_dev *v3d = job->base.v3d;
  154. struct drm_device *dev = &v3d->drm;
  155. struct dma_fence *fence;
  156. fence = v3d_fence_create(v3d, V3D_TFU);
  157. if (IS_ERR(fence))
  158. return NULL;
  159. v3d->tfu_job = job;
  160. if (job->base.irq_fence)
  161. dma_fence_put(job->base.irq_fence);
  162. job->base.irq_fence = dma_fence_get(fence);
  163. trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
  164. V3D_WRITE(V3D_TFU_IIA, job->args.iia);
  165. V3D_WRITE(V3D_TFU_IIS, job->args.iis);
  166. V3D_WRITE(V3D_TFU_ICA, job->args.ica);
  167. V3D_WRITE(V3D_TFU_IUA, job->args.iua);
  168. V3D_WRITE(V3D_TFU_IOA, job->args.ioa);
  169. V3D_WRITE(V3D_TFU_IOS, job->args.ios);
  170. V3D_WRITE(V3D_TFU_COEF0, job->args.coef[0]);
  171. if (job->args.coef[0] & V3D_TFU_COEF0_USECOEF) {
  172. V3D_WRITE(V3D_TFU_COEF1, job->args.coef[1]);
  173. V3D_WRITE(V3D_TFU_COEF2, job->args.coef[2]);
  174. V3D_WRITE(V3D_TFU_COEF3, job->args.coef[3]);
  175. }
  176. /* ICFG kicks off the job. */
  177. V3D_WRITE(V3D_TFU_ICFG, job->args.icfg | V3D_TFU_ICFG_IOC);
  178. return fence;
  179. }
  180. static struct dma_fence *
  181. v3d_csd_job_run(struct drm_sched_job *sched_job)
  182. {
  183. struct v3d_csd_job *job = to_csd_job(sched_job);
  184. struct v3d_dev *v3d = job->base.v3d;
  185. struct drm_device *dev = &v3d->drm;
  186. struct dma_fence *fence;
  187. int i;
  188. v3d->csd_job = job;
  189. v3d_invalidate_caches(v3d);
  190. fence = v3d_fence_create(v3d, V3D_CSD);
  191. if (IS_ERR(fence))
  192. return NULL;
  193. if (job->base.irq_fence)
  194. dma_fence_put(job->base.irq_fence);
  195. job->base.irq_fence = dma_fence_get(fence);
  196. trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
  197. for (i = 1; i <= 6; i++)
  198. V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0 + 4 * i, job->args.cfg[i]);
  199. /* CFG0 write kicks off the job. */
  200. V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0, job->args.cfg[0]);
  201. return fence;
  202. }
  203. static struct dma_fence *
  204. v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
  205. {
  206. struct v3d_job *job = to_v3d_job(sched_job);
  207. struct v3d_dev *v3d = job->v3d;
  208. v3d_clean_caches(v3d);
  209. return NULL;
  210. }
  211. static void
  212. v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
  213. {
  214. enum v3d_queue q;
  215. mutex_lock(&v3d->reset_lock);
  216. /* block scheduler */
  217. for (q = 0; q < V3D_MAX_QUEUES; q++)
  218. drm_sched_stop(&v3d->queue[q].sched, sched_job);
  219. if (sched_job)
  220. drm_sched_increase_karma(sched_job);
  221. /* get the GPU back into the init state */
  222. v3d_reset(v3d);
  223. for (q = 0; q < V3D_MAX_QUEUES; q++)
  224. drm_sched_resubmit_jobs(&v3d->queue[q].sched);
  225. /* Unblock schedulers and restart their jobs. */
  226. for (q = 0; q < V3D_MAX_QUEUES; q++) {
  227. drm_sched_start(&v3d->queue[q].sched, true);
  228. }
  229. mutex_unlock(&v3d->reset_lock);
  230. }
  231. /* If the current address or return address have changed, then the GPU
  232. * has probably made progress and we should delay the reset. This
  233. * could fail if the GPU got in an infinite loop in the CL, but that
  234. * is pretty unlikely outside of an i-g-t testcase.
  235. */
  236. static void
  237. v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q,
  238. u32 *timedout_ctca, u32 *timedout_ctra)
  239. {
  240. struct v3d_job *job = to_v3d_job(sched_job);
  241. struct v3d_dev *v3d = job->v3d;
  242. u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q));
  243. u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q));
  244. if (*timedout_ctca != ctca || *timedout_ctra != ctra) {
  245. *timedout_ctca = ctca;
  246. *timedout_ctra = ctra;
  247. return;
  248. }
  249. v3d_gpu_reset_for_timeout(v3d, sched_job);
  250. }
  251. static void
  252. v3d_bin_job_timedout(struct drm_sched_job *sched_job)
  253. {
  254. struct v3d_bin_job *job = to_bin_job(sched_job);
  255. v3d_cl_job_timedout(sched_job, V3D_BIN,
  256. &job->timedout_ctca, &job->timedout_ctra);
  257. }
  258. static void
  259. v3d_render_job_timedout(struct drm_sched_job *sched_job)
  260. {
  261. struct v3d_render_job *job = to_render_job(sched_job);
  262. v3d_cl_job_timedout(sched_job, V3D_RENDER,
  263. &job->timedout_ctca, &job->timedout_ctra);
  264. }
  265. static void
  266. v3d_generic_job_timedout(struct drm_sched_job *sched_job)
  267. {
  268. struct v3d_job *job = to_v3d_job(sched_job);
  269. v3d_gpu_reset_for_timeout(job->v3d, sched_job);
  270. }
  271. static void
  272. v3d_csd_job_timedout(struct drm_sched_job *sched_job)
  273. {
  274. struct v3d_csd_job *job = to_csd_job(sched_job);
  275. struct v3d_dev *v3d = job->base.v3d;
  276. u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4);
  277. /* If we've made progress, skip reset and let the timer get
  278. * rearmed.
  279. */
  280. if (job->timedout_batches != batches) {
  281. job->timedout_batches = batches;
  282. return;
  283. }
  284. v3d_gpu_reset_for_timeout(v3d, sched_job);
  285. }
  286. static const struct drm_sched_backend_ops v3d_bin_sched_ops = {
  287. .dependency = v3d_job_dependency,
  288. .run_job = v3d_bin_job_run,
  289. .timedout_job = v3d_bin_job_timedout,
  290. .free_job = v3d_job_free,
  291. };
  292. static const struct drm_sched_backend_ops v3d_render_sched_ops = {
  293. .dependency = v3d_job_dependency,
  294. .run_job = v3d_render_job_run,
  295. .timedout_job = v3d_render_job_timedout,
  296. .free_job = v3d_job_free,
  297. };
  298. static const struct drm_sched_backend_ops v3d_tfu_sched_ops = {
  299. .dependency = v3d_job_dependency,
  300. .run_job = v3d_tfu_job_run,
  301. .timedout_job = v3d_generic_job_timedout,
  302. .free_job = v3d_job_free,
  303. };
  304. static const struct drm_sched_backend_ops v3d_csd_sched_ops = {
  305. .dependency = v3d_job_dependency,
  306. .run_job = v3d_csd_job_run,
  307. .timedout_job = v3d_csd_job_timedout,
  308. .free_job = v3d_job_free
  309. };
  310. static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
  311. .dependency = v3d_job_dependency,
  312. .run_job = v3d_cache_clean_job_run,
  313. .timedout_job = v3d_generic_job_timedout,
  314. .free_job = v3d_job_free
  315. };
  316. int
  317. v3d_sched_init(struct v3d_dev *v3d)
  318. {
  319. int hw_jobs_limit = 1;
  320. int job_hang_limit = 0;
  321. int hang_limit_ms = 500;
  322. int ret;
  323. ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
  324. &v3d_bin_sched_ops,
  325. hw_jobs_limit, job_hang_limit,
  326. msecs_to_jiffies(hang_limit_ms),
  327. "v3d_bin");
  328. if (ret) {
  329. dev_err(v3d->drm.dev, "Failed to create bin scheduler: %d.", ret);
  330. return ret;
  331. }
  332. ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
  333. &v3d_render_sched_ops,
  334. hw_jobs_limit, job_hang_limit,
  335. msecs_to_jiffies(hang_limit_ms),
  336. "v3d_render");
  337. if (ret) {
  338. dev_err(v3d->drm.dev, "Failed to create render scheduler: %d.",
  339. ret);
  340. v3d_sched_fini(v3d);
  341. return ret;
  342. }
  343. ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,
  344. &v3d_tfu_sched_ops,
  345. hw_jobs_limit, job_hang_limit,
  346. msecs_to_jiffies(hang_limit_ms),
  347. "v3d_tfu");
  348. if (ret) {
  349. dev_err(v3d->drm.dev, "Failed to create TFU scheduler: %d.",
  350. ret);
  351. v3d_sched_fini(v3d);
  352. return ret;
  353. }
  354. if (v3d_has_csd(v3d)) {
  355. ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,
  356. &v3d_csd_sched_ops,
  357. hw_jobs_limit, job_hang_limit,
  358. msecs_to_jiffies(hang_limit_ms),
  359. "v3d_csd");
  360. if (ret) {
  361. dev_err(v3d->drm.dev, "Failed to create CSD scheduler: %d.",
  362. ret);
  363. v3d_sched_fini(v3d);
  364. return ret;
  365. }
  366. ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,
  367. &v3d_cache_clean_sched_ops,
  368. hw_jobs_limit, job_hang_limit,
  369. msecs_to_jiffies(hang_limit_ms),
  370. "v3d_cache_clean");
  371. if (ret) {
  372. dev_err(v3d->drm.dev, "Failed to create CACHE_CLEAN scheduler: %d.",
  373. ret);
  374. v3d_sched_fini(v3d);
  375. return ret;
  376. }
  377. }
  378. return 0;
  379. }
  380. void
  381. v3d_sched_fini(struct v3d_dev *v3d)
  382. {
  383. enum v3d_queue q;
  384. for (q = 0; q < V3D_MAX_QUEUES; q++) {
  385. if (v3d->queue[q].sched.ready)
  386. drm_sched_fini(&v3d->queue[q].sched);
  387. }
  388. }