sched_main.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922
  1. /*
  2. * Copyright 2015 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. */
  23. /**
  24. * DOC: Overview
  25. *
  26. * The GPU scheduler provides entities which allow userspace to push jobs
  27. * into software queues which are then scheduled on a hardware run queue.
  28. * The software queues have a priority among them. The scheduler selects the entities
  29. * from the run queue using a FIFO. The scheduler provides dependency handling
  30. * features among jobs. The driver is supposed to provide callback functions for
  31. * backend operations to the scheduler like submitting a job to hardware run queue,
  32. * returning the dependencies of a job etc.
  33. *
  34. * The organisation of the scheduler is the following:
  35. *
  36. * 1. Each hw run queue has one scheduler
  37. * 2. Each scheduler has multiple run queues with different priorities
  38. * (e.g., HIGH_HW,HIGH_SW, KERNEL, NORMAL)
  39. * 3. Each scheduler run queue has a queue of entities to schedule
  40. * 4. Entities themselves maintain a queue of jobs that will be scheduled on
  41. * the hardware.
  42. *
  43. * The jobs in a entity are always scheduled in the order that they were pushed.
  44. */
  45. #include <linux/kthread.h>
  46. #include <linux/wait.h>
  47. #include <linux/sched.h>
  48. #include <linux/completion.h>
  49. #include <uapi/linux/sched/types.h>
  50. #include <drm/drm_print.h>
  51. #include <drm/gpu_scheduler.h>
  52. #include <drm/spsc_queue.h>
  53. #define CREATE_TRACE_POINTS
  54. #include "gpu_scheduler_trace.h"
  55. #define to_drm_sched_job(sched_job) \
  56. container_of((sched_job), struct drm_sched_job, queue_node)
  57. static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
  58. /**
  59. * drm_sched_rq_init - initialize a given run queue struct
  60. *
  61. * @rq: scheduler run queue
  62. *
  63. * Initializes a scheduler runqueue.
  64. */
  65. static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
  66. struct drm_sched_rq *rq)
  67. {
  68. spin_lock_init(&rq->lock);
  69. INIT_LIST_HEAD(&rq->entities);
  70. rq->current_entity = NULL;
  71. rq->sched = sched;
  72. }
  73. /**
  74. * drm_sched_rq_add_entity - add an entity
  75. *
  76. * @rq: scheduler run queue
  77. * @entity: scheduler entity
  78. *
  79. * Adds a scheduler entity to the run queue.
  80. */
  81. void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
  82. struct drm_sched_entity *entity)
  83. {
  84. if (!list_empty(&entity->list))
  85. return;
  86. spin_lock(&rq->lock);
  87. atomic_inc(&rq->sched->score);
  88. list_add_tail(&entity->list, &rq->entities);
  89. spin_unlock(&rq->lock);
  90. }
  91. /**
  92. * drm_sched_rq_remove_entity - remove an entity
  93. *
  94. * @rq: scheduler run queue
  95. * @entity: scheduler entity
  96. *
  97. * Removes a scheduler entity from the run queue.
  98. */
  99. void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
  100. struct drm_sched_entity *entity)
  101. {
  102. if (list_empty(&entity->list))
  103. return;
  104. spin_lock(&rq->lock);
  105. atomic_dec(&rq->sched->score);
  106. list_del_init(&entity->list);
  107. if (rq->current_entity == entity)
  108. rq->current_entity = NULL;
  109. spin_unlock(&rq->lock);
  110. }
  111. /**
  112. * drm_sched_rq_select_entity - Select an entity which could provide a job to run
  113. *
  114. * @rq: scheduler run queue to check.
  115. *
  116. * Try to find a ready entity, returns NULL if none found.
  117. */
  118. static struct drm_sched_entity *
  119. drm_sched_rq_select_entity(struct drm_sched_rq *rq)
  120. {
  121. struct drm_sched_entity *entity;
  122. spin_lock(&rq->lock);
  123. entity = rq->current_entity;
  124. if (entity) {
  125. list_for_each_entry_continue(entity, &rq->entities, list) {
  126. if (drm_sched_entity_is_ready(entity)) {
  127. rq->current_entity = entity;
  128. reinit_completion(&entity->entity_idle);
  129. spin_unlock(&rq->lock);
  130. return entity;
  131. }
  132. }
  133. }
  134. list_for_each_entry(entity, &rq->entities, list) {
  135. if (drm_sched_entity_is_ready(entity)) {
  136. rq->current_entity = entity;
  137. reinit_completion(&entity->entity_idle);
  138. spin_unlock(&rq->lock);
  139. return entity;
  140. }
  141. if (entity == rq->current_entity)
  142. break;
  143. }
  144. spin_unlock(&rq->lock);
  145. return NULL;
  146. }
  147. /**
  148. * drm_sched_dependency_optimized
  149. *
  150. * @fence: the dependency fence
  151. * @entity: the entity which depends on the above fence
  152. *
  153. * Returns true if the dependency can be optimized and false otherwise
  154. */
  155. bool drm_sched_dependency_optimized(struct dma_fence* fence,
  156. struct drm_sched_entity *entity)
  157. {
  158. struct drm_gpu_scheduler *sched = entity->rq->sched;
  159. struct drm_sched_fence *s_fence;
  160. if (!fence || dma_fence_is_signaled(fence))
  161. return false;
  162. if (fence->context == entity->fence_context)
  163. return true;
  164. s_fence = to_drm_sched_fence(fence);
  165. if (s_fence && s_fence->sched == sched)
  166. return true;
  167. return false;
  168. }
  169. EXPORT_SYMBOL(drm_sched_dependency_optimized);
  170. /**
  171. * drm_sched_start_timeout - start timeout for reset worker
  172. *
  173. * @sched: scheduler instance to start the worker for
  174. *
  175. * Start the timeout for the given scheduler.
  176. */
  177. static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched)
  178. {
  179. if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
  180. !list_empty(&sched->ring_mirror_list))
  181. schedule_delayed_work(&sched->work_tdr, sched->timeout);
  182. }
  183. /**
  184. * drm_sched_fault - immediately start timeout handler
  185. *
  186. * @sched: scheduler where the timeout handling should be started.
  187. *
  188. * Start timeout handling immediately when the driver detects a hardware fault.
  189. */
  190. void drm_sched_fault(struct drm_gpu_scheduler *sched)
  191. {
  192. mod_delayed_work(system_wq, &sched->work_tdr, 0);
  193. }
  194. EXPORT_SYMBOL(drm_sched_fault);
  195. /**
  196. * drm_sched_suspend_timeout - Suspend scheduler job timeout
  197. *
  198. * @sched: scheduler instance for which to suspend the timeout
  199. *
  200. * Suspend the delayed work timeout for the scheduler. This is done by
  201. * modifying the delayed work timeout to an arbitrary large value,
  202. * MAX_SCHEDULE_TIMEOUT in this case.
  203. *
  204. * Returns the timeout remaining
  205. *
  206. */
  207. unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched)
  208. {
  209. unsigned long sched_timeout, now = jiffies;
  210. sched_timeout = sched->work_tdr.timer.expires;
  211. /*
  212. * Modify the timeout to an arbitrarily large value. This also prevents
  213. * the timeout to be restarted when new submissions arrive
  214. */
  215. if (mod_delayed_work(system_wq, &sched->work_tdr, MAX_SCHEDULE_TIMEOUT)
  216. && time_after(sched_timeout, now))
  217. return sched_timeout - now;
  218. else
  219. return sched->timeout;
  220. }
  221. EXPORT_SYMBOL(drm_sched_suspend_timeout);
  222. /**
  223. * drm_sched_resume_timeout - Resume scheduler job timeout
  224. *
  225. * @sched: scheduler instance for which to resume the timeout
  226. * @remaining: remaining timeout
  227. *
  228. * Resume the delayed work timeout for the scheduler.
  229. */
  230. void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
  231. unsigned long remaining)
  232. {
  233. spin_lock(&sched->job_list_lock);
  234. if (list_empty(&sched->ring_mirror_list))
  235. cancel_delayed_work(&sched->work_tdr);
  236. else
  237. mod_delayed_work(system_wq, &sched->work_tdr, remaining);
  238. spin_unlock(&sched->job_list_lock);
  239. }
  240. EXPORT_SYMBOL(drm_sched_resume_timeout);
  241. static void drm_sched_job_begin(struct drm_sched_job *s_job)
  242. {
  243. struct drm_gpu_scheduler *sched = s_job->sched;
  244. spin_lock(&sched->job_list_lock);
  245. list_add_tail(&s_job->node, &sched->ring_mirror_list);
  246. drm_sched_start_timeout(sched);
  247. spin_unlock(&sched->job_list_lock);
  248. }
  249. static void drm_sched_job_timedout(struct work_struct *work)
  250. {
  251. struct drm_gpu_scheduler *sched;
  252. struct drm_sched_job *job;
  253. sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
  254. /* Protects against concurrent deletion in drm_sched_get_cleanup_job */
  255. spin_lock(&sched->job_list_lock);
  256. job = list_first_entry_or_null(&sched->ring_mirror_list,
  257. struct drm_sched_job, node);
  258. if (job) {
  259. /*
  260. * Remove the bad job so it cannot be freed by concurrent
  261. * drm_sched_cleanup_jobs. It will be reinserted back after sched->thread
  262. * is parked at which point it's safe.
  263. */
  264. list_del_init(&job->node);
  265. spin_unlock(&sched->job_list_lock);
  266. job->sched->ops->timedout_job(job);
  267. /*
  268. * Guilty job did complete and hence needs to be manually removed
  269. * See drm_sched_stop doc.
  270. */
  271. if (sched->free_guilty) {
  272. job->sched->ops->free_job(job);
  273. sched->free_guilty = false;
  274. }
  275. } else {
  276. spin_unlock(&sched->job_list_lock);
  277. }
  278. spin_lock(&sched->job_list_lock);
  279. drm_sched_start_timeout(sched);
  280. spin_unlock(&sched->job_list_lock);
  281. }
  282. /**
  283. * drm_sched_increase_karma - Update sched_entity guilty flag
  284. *
  285. * @bad: The job guilty of time out
  286. *
  287. * Increment on every hang caused by the 'bad' job. If this exceeds the hang
  288. * limit of the scheduler then the respective sched entity is marked guilty and
  289. * jobs from it will not be scheduled further
  290. */
  291. void drm_sched_increase_karma(struct drm_sched_job *bad)
  292. {
  293. int i;
  294. struct drm_sched_entity *tmp;
  295. struct drm_sched_entity *entity;
  296. struct drm_gpu_scheduler *sched = bad->sched;
  297. /* don't increase @bad's karma if it's from KERNEL RQ,
  298. * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
  299. * corrupt but keep in mind that kernel jobs always considered good.
  300. */
  301. if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
  302. atomic_inc(&bad->karma);
  303. for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
  304. i++) {
  305. struct drm_sched_rq *rq = &sched->sched_rq[i];
  306. spin_lock(&rq->lock);
  307. list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
  308. if (bad->s_fence->scheduled.context ==
  309. entity->fence_context) {
  310. if (atomic_read(&bad->karma) >
  311. bad->sched->hang_limit)
  312. if (entity->guilty)
  313. atomic_set(entity->guilty, 1);
  314. break;
  315. }
  316. }
  317. spin_unlock(&rq->lock);
  318. if (&entity->list != &rq->entities)
  319. break;
  320. }
  321. }
  322. }
  323. EXPORT_SYMBOL(drm_sched_increase_karma);
  324. /**
  325. * drm_sched_stop - stop the scheduler
  326. *
  327. * @sched: scheduler instance
  328. * @bad: job which caused the time out
  329. *
  330. * Stop the scheduler and also removes and frees all completed jobs.
  331. * Note: bad job will not be freed as it might be used later and so it's
  332. * callers responsibility to release it manually if it's not part of the
  333. * mirror list any more.
  334. *
  335. */
  336. void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
  337. {
  338. struct drm_sched_job *s_job, *tmp;
  339. kthread_park(sched->thread);
  340. /*
  341. * Reinsert back the bad job here - now it's safe as
  342. * drm_sched_get_cleanup_job cannot race against us and release the
  343. * bad job at this point - we parked (waited for) any in progress
  344. * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
  345. * now until the scheduler thread is unparked.
  346. */
  347. if (bad && bad->sched == sched)
  348. /*
  349. * Add at the head of the queue to reflect it was the earliest
  350. * job extracted.
  351. */
  352. list_add(&bad->node, &sched->ring_mirror_list);
  353. /*
  354. * Iterate the job list from later to earlier one and either deactive
  355. * their HW callbacks or remove them from mirror list if they already
  356. * signaled.
  357. * This iteration is thread safe as sched thread is stopped.
  358. */
  359. list_for_each_entry_safe_reverse(s_job, tmp, &sched->ring_mirror_list, node) {
  360. if (s_job->s_fence->parent &&
  361. dma_fence_remove_callback(s_job->s_fence->parent,
  362. &s_job->cb)) {
  363. atomic_dec(&sched->hw_rq_count);
  364. } else {
  365. /*
  366. * remove job from ring_mirror_list.
  367. * Locking here is for concurrent resume timeout
  368. */
  369. spin_lock(&sched->job_list_lock);
  370. list_del_init(&s_job->node);
  371. spin_unlock(&sched->job_list_lock);
  372. /*
  373. * Wait for job's HW fence callback to finish using s_job
  374. * before releasing it.
  375. *
  376. * Job is still alive so fence refcount at least 1
  377. */
  378. dma_fence_wait(&s_job->s_fence->finished, false);
  379. /*
  380. * We must keep bad job alive for later use during
  381. * recovery by some of the drivers but leave a hint
  382. * that the guilty job must be released.
  383. */
  384. if (bad != s_job)
  385. sched->ops->free_job(s_job);
  386. else
  387. sched->free_guilty = true;
  388. }
  389. }
  390. /*
  391. * Stop pending timer in flight as we rearm it in drm_sched_start. This
  392. * avoids the pending timeout work in progress to fire right away after
  393. * this TDR finished and before the newly restarted jobs had a
  394. * chance to complete.
  395. */
  396. cancel_delayed_work(&sched->work_tdr);
  397. }
  398. EXPORT_SYMBOL(drm_sched_stop);
  399. /**
  400. * drm_sched_job_recovery - recover jobs after a reset
  401. *
  402. * @sched: scheduler instance
  403. * @full_recovery: proceed with complete sched restart
  404. *
  405. */
  406. void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
  407. {
  408. struct drm_sched_job *s_job, *tmp;
  409. int r;
  410. /*
  411. * Locking the list is not required here as the sched thread is parked
  412. * so no new jobs are being inserted or removed. Also concurrent
  413. * GPU recovers can't run in parallel.
  414. */
  415. list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
  416. struct dma_fence *fence = s_job->s_fence->parent;
  417. atomic_inc(&sched->hw_rq_count);
  418. if (!full_recovery)
  419. continue;
  420. if (fence) {
  421. r = dma_fence_add_callback(fence, &s_job->cb,
  422. drm_sched_process_job);
  423. if (r == -ENOENT)
  424. drm_sched_process_job(fence, &s_job->cb);
  425. else if (r)
  426. DRM_ERROR("fence add callback failed (%d)\n",
  427. r);
  428. } else
  429. drm_sched_process_job(NULL, &s_job->cb);
  430. }
  431. if (full_recovery) {
  432. spin_lock(&sched->job_list_lock);
  433. drm_sched_start_timeout(sched);
  434. spin_unlock(&sched->job_list_lock);
  435. }
  436. kthread_unpark(sched->thread);
  437. }
  438. EXPORT_SYMBOL(drm_sched_start);
  439. /**
  440. * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
  441. *
  442. * @sched: scheduler instance
  443. *
  444. */
  445. void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
  446. {
  447. struct drm_sched_job *s_job, *tmp;
  448. uint64_t guilty_context;
  449. bool found_guilty = false;
  450. struct dma_fence *fence;
  451. list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
  452. struct drm_sched_fence *s_fence = s_job->s_fence;
  453. if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
  454. found_guilty = true;
  455. guilty_context = s_job->s_fence->scheduled.context;
  456. }
  457. if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
  458. dma_fence_set_error(&s_fence->finished, -ECANCELED);
  459. dma_fence_put(s_job->s_fence->parent);
  460. fence = sched->ops->run_job(s_job);
  461. if (IS_ERR_OR_NULL(fence)) {
  462. if (IS_ERR(fence))
  463. dma_fence_set_error(&s_fence->finished, PTR_ERR(fence));
  464. s_job->s_fence->parent = NULL;
  465. } else {
  466. s_job->s_fence->parent = fence;
  467. }
  468. }
  469. }
  470. EXPORT_SYMBOL(drm_sched_resubmit_jobs);
  471. /**
  472. * drm_sched_job_init - init a scheduler job
  473. *
  474. * @job: scheduler job to init
  475. * @entity: scheduler entity to use
  476. * @owner: job owner for debugging
  477. *
  478. * Refer to drm_sched_entity_push_job() documentation
  479. * for locking considerations.
  480. *
  481. * Returns 0 for success, negative error code otherwise.
  482. */
  483. int drm_sched_job_init(struct drm_sched_job *job,
  484. struct drm_sched_entity *entity,
  485. void *owner)
  486. {
  487. struct drm_gpu_scheduler *sched;
  488. drm_sched_entity_select_rq(entity);
  489. if (!entity->rq)
  490. return -ENOENT;
  491. sched = entity->rq->sched;
  492. job->sched = sched;
  493. job->entity = entity;
  494. job->s_priority = entity->rq - sched->sched_rq;
  495. job->s_fence = drm_sched_fence_create(entity, owner);
  496. if (!job->s_fence)
  497. return -ENOMEM;
  498. job->id = atomic64_inc_return(&sched->job_id_count);
  499. INIT_LIST_HEAD(&job->node);
  500. return 0;
  501. }
  502. EXPORT_SYMBOL(drm_sched_job_init);
  503. /**
  504. * drm_sched_job_cleanup - clean up scheduler job resources
  505. *
  506. * @job: scheduler job to clean up
  507. */
  508. void drm_sched_job_cleanup(struct drm_sched_job *job)
  509. {
  510. dma_fence_put(&job->s_fence->finished);
  511. job->s_fence = NULL;
  512. }
  513. EXPORT_SYMBOL(drm_sched_job_cleanup);
  514. /**
  515. * drm_sched_ready - is the scheduler ready
  516. *
  517. * @sched: scheduler instance
  518. *
  519. * Return true if we can push more jobs to the hw, otherwise false.
  520. */
  521. static bool drm_sched_ready(struct drm_gpu_scheduler *sched)
  522. {
  523. return atomic_read(&sched->hw_rq_count) <
  524. sched->hw_submission_limit;
  525. }
  526. /**
  527. * drm_sched_wakeup - Wake up the scheduler when it is ready
  528. *
  529. * @sched: scheduler instance
  530. *
  531. */
  532. void drm_sched_wakeup(struct drm_gpu_scheduler *sched)
  533. {
  534. if (drm_sched_ready(sched))
  535. wake_up_interruptible(&sched->wake_up_worker);
  536. }
  537. /**
  538. * drm_sched_select_entity - Select next entity to process
  539. *
  540. * @sched: scheduler instance
  541. *
  542. * Returns the entity to process or NULL if none are found.
  543. */
  544. static struct drm_sched_entity *
  545. drm_sched_select_entity(struct drm_gpu_scheduler *sched)
  546. {
  547. struct drm_sched_entity *entity;
  548. int i;
  549. if (!drm_sched_ready(sched))
  550. return NULL;
  551. /* Kernel run queue has higher priority than normal run queue*/
  552. for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
  553. entity = drm_sched_rq_select_entity(&sched->sched_rq[i]);
  554. if (entity)
  555. break;
  556. }
  557. return entity;
  558. }
  559. /**
  560. * drm_sched_process_job - process a job
  561. *
  562. * @f: fence
  563. * @cb: fence callbacks
  564. *
  565. * Called after job has finished execution.
  566. */
  567. static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
  568. {
  569. struct drm_sched_job *s_job = container_of(cb, struct drm_sched_job, cb);
  570. struct drm_sched_fence *s_fence = s_job->s_fence;
  571. struct drm_gpu_scheduler *sched = s_fence->sched;
  572. atomic_dec(&sched->hw_rq_count);
  573. atomic_dec(&sched->score);
  574. trace_drm_sched_process_job(s_fence);
  575. dma_fence_get(&s_fence->finished);
  576. drm_sched_fence_finished(s_fence);
  577. dma_fence_put(&s_fence->finished);
  578. wake_up_interruptible(&sched->wake_up_worker);
  579. }
  580. /**
  581. * drm_sched_get_cleanup_job - fetch the next finished job to be destroyed
  582. *
  583. * @sched: scheduler instance
  584. *
  585. * Returns the next finished job from the mirror list (if there is one)
  586. * ready for it to be destroyed.
  587. */
  588. static struct drm_sched_job *
  589. drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
  590. {
  591. struct drm_sched_job *job;
  592. /*
  593. * Don't destroy jobs while the timeout worker is running OR thread
  594. * is being parked and hence assumed to not touch ring_mirror_list
  595. */
  596. if ((sched->timeout != MAX_SCHEDULE_TIMEOUT &&
  597. !cancel_delayed_work(&sched->work_tdr)) ||
  598. kthread_should_park())
  599. return NULL;
  600. spin_lock(&sched->job_list_lock);
  601. job = list_first_entry_or_null(&sched->ring_mirror_list,
  602. struct drm_sched_job, node);
  603. if (job && dma_fence_is_signaled(&job->s_fence->finished)) {
  604. /* remove job from ring_mirror_list */
  605. list_del_init(&job->node);
  606. } else {
  607. job = NULL;
  608. /* queue timeout for next job */
  609. drm_sched_start_timeout(sched);
  610. }
  611. spin_unlock(&sched->job_list_lock);
  612. return job;
  613. }
  614. /**
  615. * drm_sched_pick_best - Get a drm sched from a sched_list with the least load
  616. * @sched_list: list of drm_gpu_schedulers
  617. * @num_sched_list: number of drm_gpu_schedulers in the sched_list
  618. *
  619. * Returns pointer of the sched with the least load or NULL if none of the
  620. * drm_gpu_schedulers are ready
  621. */
  622. struct drm_gpu_scheduler *
  623. drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
  624. unsigned int num_sched_list)
  625. {
  626. struct drm_gpu_scheduler *sched, *picked_sched = NULL;
  627. int i;
  628. unsigned int min_score = UINT_MAX, num_score;
  629. for (i = 0; i < num_sched_list; ++i) {
  630. sched = sched_list[i];
  631. if (!sched->ready) {
  632. DRM_WARN("scheduler %s is not ready, skipping",
  633. sched->name);
  634. continue;
  635. }
  636. num_score = atomic_read(&sched->score);
  637. if (num_score < min_score) {
  638. min_score = num_score;
  639. picked_sched = sched;
  640. }
  641. }
  642. return picked_sched;
  643. }
  644. EXPORT_SYMBOL(drm_sched_pick_best);
  645. /**
  646. * drm_sched_blocked - check if the scheduler is blocked
  647. *
  648. * @sched: scheduler instance
  649. *
  650. * Returns true if blocked, otherwise false.
  651. */
  652. static bool drm_sched_blocked(struct drm_gpu_scheduler *sched)
  653. {
  654. if (kthread_should_park()) {
  655. kthread_parkme();
  656. return true;
  657. }
  658. return false;
  659. }
  660. /**
  661. * drm_sched_main - main scheduler thread
  662. *
  663. * @param: scheduler instance
  664. *
  665. * Returns 0.
  666. */
  667. static int drm_sched_main(void *param)
  668. {
  669. struct drm_gpu_scheduler *sched = (struct drm_gpu_scheduler *)param;
  670. int r;
  671. sched_set_fifo_low(current);
  672. while (!kthread_should_stop()) {
  673. struct drm_sched_entity *entity = NULL;
  674. struct drm_sched_fence *s_fence;
  675. struct drm_sched_job *sched_job;
  676. struct dma_fence *fence;
  677. struct drm_sched_job *cleanup_job = NULL;
  678. wait_event_interruptible(sched->wake_up_worker,
  679. (cleanup_job = drm_sched_get_cleanup_job(sched)) ||
  680. (!drm_sched_blocked(sched) &&
  681. (entity = drm_sched_select_entity(sched))) ||
  682. kthread_should_stop());
  683. if (cleanup_job) {
  684. sched->ops->free_job(cleanup_job);
  685. /* queue timeout for next job */
  686. drm_sched_start_timeout(sched);
  687. }
  688. if (!entity)
  689. continue;
  690. sched_job = drm_sched_entity_pop_job(entity);
  691. complete(&entity->entity_idle);
  692. if (!sched_job)
  693. continue;
  694. s_fence = sched_job->s_fence;
  695. atomic_inc(&sched->hw_rq_count);
  696. drm_sched_job_begin(sched_job);
  697. trace_drm_run_job(sched_job, entity);
  698. fence = sched->ops->run_job(sched_job);
  699. drm_sched_fence_scheduled(s_fence);
  700. if (!IS_ERR_OR_NULL(fence)) {
  701. s_fence->parent = dma_fence_get(fence);
  702. r = dma_fence_add_callback(fence, &sched_job->cb,
  703. drm_sched_process_job);
  704. if (r == -ENOENT)
  705. drm_sched_process_job(fence, &sched_job->cb);
  706. else if (r)
  707. DRM_ERROR("fence add callback failed (%d)\n",
  708. r);
  709. dma_fence_put(fence);
  710. } else {
  711. if (IS_ERR(fence))
  712. dma_fence_set_error(&s_fence->finished, PTR_ERR(fence));
  713. drm_sched_process_job(NULL, &sched_job->cb);
  714. }
  715. wake_up(&sched->job_scheduled);
  716. }
  717. return 0;
  718. }
  719. /**
  720. * drm_sched_init - Init a gpu scheduler instance
  721. *
  722. * @sched: scheduler instance
  723. * @ops: backend operations for this scheduler
  724. * @hw_submission: number of hw submissions that can be in flight
  725. * @hang_limit: number of times to allow a job to hang before dropping it
  726. * @timeout: timeout value in jiffies for the scheduler
  727. * @name: name used for debugging
  728. *
  729. * Return 0 on success, otherwise error code.
  730. */
  731. int drm_sched_init(struct drm_gpu_scheduler *sched,
  732. const struct drm_sched_backend_ops *ops,
  733. unsigned hw_submission,
  734. unsigned hang_limit,
  735. long timeout,
  736. const char *name)
  737. {
  738. int i, ret;
  739. sched->ops = ops;
  740. sched->hw_submission_limit = hw_submission;
  741. sched->name = name;
  742. sched->timeout = timeout;
  743. sched->hang_limit = hang_limit;
  744. for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; i++)
  745. drm_sched_rq_init(sched, &sched->sched_rq[i]);
  746. init_waitqueue_head(&sched->wake_up_worker);
  747. init_waitqueue_head(&sched->job_scheduled);
  748. INIT_LIST_HEAD(&sched->ring_mirror_list);
  749. spin_lock_init(&sched->job_list_lock);
  750. atomic_set(&sched->hw_rq_count, 0);
  751. INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout);
  752. atomic_set(&sched->score, 0);
  753. atomic64_set(&sched->job_id_count, 0);
  754. /* Each scheduler will run on a seperate kernel thread */
  755. sched->thread = kthread_run(drm_sched_main, sched, sched->name);
  756. if (IS_ERR(sched->thread)) {
  757. ret = PTR_ERR(sched->thread);
  758. sched->thread = NULL;
  759. DRM_ERROR("Failed to create scheduler for %s.\n", name);
  760. return ret;
  761. }
  762. sched->ready = true;
  763. return 0;
  764. }
  765. EXPORT_SYMBOL(drm_sched_init);
  766. /**
  767. * drm_sched_fini - Destroy a gpu scheduler
  768. *
  769. * @sched: scheduler instance
  770. *
  771. * Tears down and cleans up the scheduler.
  772. */
  773. void drm_sched_fini(struct drm_gpu_scheduler *sched)
  774. {
  775. struct drm_sched_entity *s_entity;
  776. int i;
  777. if (sched->thread)
  778. kthread_stop(sched->thread);
  779. for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
  780. struct drm_sched_rq *rq = &sched->sched_rq[i];
  781. if (!rq)
  782. continue;
  783. spin_lock(&rq->lock);
  784. list_for_each_entry(s_entity, &rq->entities, list)
  785. /*
  786. * Prevents reinsertion and marks job_queue as idle,
  787. * it will removed from rq in drm_sched_entity_fini
  788. * eventually
  789. */
  790. s_entity->stopped = true;
  791. spin_unlock(&rq->lock);
  792. }
  793. /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */
  794. wake_up_all(&sched->job_scheduled);
  795. /* Confirm no work left behind accessing device structures */
  796. cancel_delayed_work_sync(&sched->work_tdr);
  797. sched->ready = false;
  798. }
  799. EXPORT_SYMBOL(drm_sched_fini);