gpu_scheduler.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350
  1. /*
  2. * Copyright 2015 Advanced Micro Devices, Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. *
  22. */
  23. #ifndef _DRM_GPU_SCHEDULER_H_
  24. #define _DRM_GPU_SCHEDULER_H_
  25. #include <drm/spsc_queue.h>
  26. #include <linux/dma-fence.h>
  27. #include <linux/completion.h>
  28. #define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000)
  29. struct drm_gpu_scheduler;
  30. struct drm_sched_rq;
  31. /* These are often used as an (initial) index
  32. * to an array, and as such should start at 0.
  33. */
  34. enum drm_sched_priority {
  35. DRM_SCHED_PRIORITY_MIN,
  36. DRM_SCHED_PRIORITY_NORMAL,
  37. DRM_SCHED_PRIORITY_HIGH,
  38. DRM_SCHED_PRIORITY_KERNEL,
  39. DRM_SCHED_PRIORITY_COUNT,
  40. DRM_SCHED_PRIORITY_UNSET = -2
  41. };
  42. /**
  43. * struct drm_sched_entity - A wrapper around a job queue (typically
  44. * attached to the DRM file_priv).
  45. *
  46. * @list: used to append this struct to the list of entities in the
  47. * runqueue.
  48. * @rq: runqueue on which this entity is currently scheduled.
  49. * @sched_list: A list of schedulers (drm_gpu_schedulers).
  50. * Jobs from this entity can be scheduled on any scheduler
  51. * on this list.
  52. * @num_sched_list: number of drm_gpu_schedulers in the sched_list.
  53. * @priority: priority of the entity
  54. * @rq_lock: lock to modify the runqueue to which this entity belongs.
  55. * @job_queue: the list of jobs of this entity.
  56. * @fence_seq: a linearly increasing seqno incremented with each
  57. * new &drm_sched_fence which is part of the entity.
  58. * @fence_context: a unique context for all the fences which belong
  59. * to this entity.
  60. * The &drm_sched_fence.scheduled uses the
  61. * fence_context but &drm_sched_fence.finished uses
  62. * fence_context + 1.
  63. * @dependency: the dependency fence of the job which is on the top
  64. * of the job queue.
  65. * @cb: callback for the dependency fence above.
  66. * @guilty: points to ctx's guilty.
  67. * @fini_status: contains the exit status in case the process was signalled.
  68. * @last_scheduled: points to the finished fence of the last scheduled job.
  69. * @last_user: last group leader pushing a job into the entity.
  70. * @stopped: Marks the enity as removed from rq and destined for termination.
  71. * @entity_idle: Signals when enityt is not in use
  72. *
  73. * Entities will emit jobs in order to their corresponding hardware
  74. * ring, and the scheduler will alternate between entities based on
  75. * scheduling policy.
  76. */
  77. struct drm_sched_entity {
  78. struct list_head list;
  79. struct drm_sched_rq *rq;
  80. struct drm_gpu_scheduler **sched_list;
  81. unsigned int num_sched_list;
  82. enum drm_sched_priority priority;
  83. spinlock_t rq_lock;
  84. struct spsc_queue job_queue;
  85. atomic_t fence_seq;
  86. uint64_t fence_context;
  87. struct dma_fence *dependency;
  88. struct dma_fence_cb cb;
  89. atomic_t *guilty;
  90. struct dma_fence *last_scheduled;
  91. struct task_struct *last_user;
  92. bool stopped;
  93. struct completion entity_idle;
  94. };
  95. /**
  96. * struct drm_sched_rq - queue of entities to be scheduled.
  97. *
  98. * @lock: to modify the entities list.
  99. * @sched: the scheduler to which this rq belongs to.
  100. * @entities: list of the entities to be scheduled.
  101. * @current_entity: the entity which is to be scheduled.
  102. *
  103. * Run queue is a set of entities scheduling command submissions for
  104. * one specific ring. It implements the scheduling policy that selects
  105. * the next entity to emit commands from.
  106. */
  107. struct drm_sched_rq {
  108. spinlock_t lock;
  109. struct drm_gpu_scheduler *sched;
  110. struct list_head entities;
  111. struct drm_sched_entity *current_entity;
  112. };
  113. /**
  114. * struct drm_sched_fence - fences corresponding to the scheduling of a job.
  115. */
  116. struct drm_sched_fence {
  117. /**
  118. * @scheduled: this fence is what will be signaled by the scheduler
  119. * when the job is scheduled.
  120. */
  121. struct dma_fence scheduled;
  122. /**
  123. * @finished: this fence is what will be signaled by the scheduler
  124. * when the job is completed.
  125. *
  126. * When setting up an out fence for the job, you should use
  127. * this, since it's available immediately upon
  128. * drm_sched_job_init(), and the fence returned by the driver
  129. * from run_job() won't be created until the dependencies have
  130. * resolved.
  131. */
  132. struct dma_fence finished;
  133. /**
  134. * @parent: the fence returned by &drm_sched_backend_ops.run_job
  135. * when scheduling the job on hardware. We signal the
  136. * &drm_sched_fence.finished fence once parent is signalled.
  137. */
  138. struct dma_fence *parent;
  139. /**
  140. * @sched: the scheduler instance to which the job having this struct
  141. * belongs to.
  142. */
  143. struct drm_gpu_scheduler *sched;
  144. /**
  145. * @lock: the lock used by the scheduled and the finished fences.
  146. */
  147. spinlock_t lock;
  148. /**
  149. * @owner: job owner for debugging
  150. */
  151. void *owner;
  152. };
  153. struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
  154. /**
  155. * struct drm_sched_job - A job to be run by an entity.
  156. *
  157. * @queue_node: used to append this struct to the queue of jobs in an entity.
  158. * @sched: the scheduler instance on which this job is scheduled.
  159. * @s_fence: contains the fences for the scheduling of job.
  160. * @finish_cb: the callback for the finished fence.
  161. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
  162. * @id: a unique id assigned to each job scheduled on the scheduler.
  163. * @karma: increment on every hang caused by this job. If this exceeds the hang
  164. * limit of the scheduler then the job is marked guilty and will not
  165. * be scheduled further.
  166. * @s_priority: the priority of the job.
  167. * @entity: the entity to which this job belongs.
  168. * @cb: the callback for the parent fence in s_fence.
  169. *
  170. * A job is created by the driver using drm_sched_job_init(), and
  171. * should call drm_sched_entity_push_job() once it wants the scheduler
  172. * to schedule the job.
  173. */
  174. struct drm_sched_job {
  175. struct spsc_node queue_node;
  176. struct drm_gpu_scheduler *sched;
  177. struct drm_sched_fence *s_fence;
  178. struct dma_fence_cb finish_cb;
  179. struct list_head node;
  180. uint64_t id;
  181. atomic_t karma;
  182. enum drm_sched_priority s_priority;
  183. struct drm_sched_entity *entity;
  184. struct dma_fence_cb cb;
  185. };
  186. static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
  187. int threshold)
  188. {
  189. return (s_job && atomic_inc_return(&s_job->karma) > threshold);
  190. }
  191. /**
  192. * struct drm_sched_backend_ops
  193. *
  194. * Define the backend operations called by the scheduler,
  195. * these functions should be implemented in driver side.
  196. */
  197. struct drm_sched_backend_ops {
  198. /**
  199. * @dependency: Called when the scheduler is considering scheduling
  200. * this job next, to get another struct dma_fence for this job to
  201. * block on. Once it returns NULL, run_job() may be called.
  202. */
  203. struct dma_fence *(*dependency)(struct drm_sched_job *sched_job,
  204. struct drm_sched_entity *s_entity);
  205. /**
  206. * @run_job: Called to execute the job once all of the dependencies
  207. * have been resolved. This may be called multiple times, if
  208. * timedout_job() has happened and drm_sched_job_recovery()
  209. * decides to try it again.
  210. */
  211. struct dma_fence *(*run_job)(struct drm_sched_job *sched_job);
  212. /**
  213. * @timedout_job: Called when a job has taken too long to execute,
  214. * to trigger GPU recovery.
  215. */
  216. void (*timedout_job)(struct drm_sched_job *sched_job);
  217. /**
  218. * @free_job: Called once the job's finished fence has been signaled
  219. * and it's time to clean it up.
  220. */
  221. void (*free_job)(struct drm_sched_job *sched_job);
  222. };
  223. /**
  224. * struct drm_gpu_scheduler
  225. *
  226. * @ops: backend operations provided by the driver.
  227. * @hw_submission_limit: the max size of the hardware queue.
  228. * @timeout: the time after which a job is removed from the scheduler.
  229. * @name: name of the ring for which this scheduler is being used.
  230. * @sched_rq: priority wise array of run queues.
  231. * @wake_up_worker: the wait queue on which the scheduler sleeps until a job
  232. * is ready to be scheduled.
  233. * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler
  234. * waits on this wait queue until all the scheduled jobs are
  235. * finished.
  236. * @hw_rq_count: the number of jobs currently in the hardware queue.
  237. * @job_id_count: used to assign unique id to the each job.
  238. * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the
  239. * timeout interval is over.
  240. * @thread: the kthread on which the scheduler which run.
  241. * @ring_mirror_list: the list of jobs which are currently in the job queue.
  242. * @job_list_lock: lock to protect the ring_mirror_list.
  243. * @hang_limit: once the hangs by a job crosses this limit then it is marked
  244. * guilty and it will be considered for scheduling further.
  245. * @score: score to help loadbalancer pick a idle sched
  246. * @ready: marks if the underlying HW is ready to work
  247. * @free_guilty: A hit to time out handler to free the guilty job.
  248. *
  249. * One scheduler is implemented for each hardware ring.
  250. */
  251. struct drm_gpu_scheduler {
  252. const struct drm_sched_backend_ops *ops;
  253. uint32_t hw_submission_limit;
  254. long timeout;
  255. const char *name;
  256. struct drm_sched_rq sched_rq[DRM_SCHED_PRIORITY_COUNT];
  257. wait_queue_head_t wake_up_worker;
  258. wait_queue_head_t job_scheduled;
  259. atomic_t hw_rq_count;
  260. atomic64_t job_id_count;
  261. struct delayed_work work_tdr;
  262. struct task_struct *thread;
  263. struct list_head ring_mirror_list;
  264. spinlock_t job_list_lock;
  265. int hang_limit;
  266. atomic_t score;
  267. bool ready;
  268. bool free_guilty;
  269. };
  270. int drm_sched_init(struct drm_gpu_scheduler *sched,
  271. const struct drm_sched_backend_ops *ops,
  272. uint32_t hw_submission, unsigned hang_limit, long timeout,
  273. const char *name);
  274. void drm_sched_fini(struct drm_gpu_scheduler *sched);
  275. int drm_sched_job_init(struct drm_sched_job *job,
  276. struct drm_sched_entity *entity,
  277. void *owner);
  278. void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
  279. struct drm_gpu_scheduler **sched_list,
  280. unsigned int num_sched_list);
  281. void drm_sched_job_cleanup(struct drm_sched_job *job);
  282. void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
  283. void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
  284. void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
  285. void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
  286. void drm_sched_increase_karma(struct drm_sched_job *bad);
  287. bool drm_sched_dependency_optimized(struct dma_fence* fence,
  288. struct drm_sched_entity *entity);
  289. void drm_sched_fault(struct drm_gpu_scheduler *sched);
  290. void drm_sched_job_kickout(struct drm_sched_job *s_job);
  291. void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
  292. struct drm_sched_entity *entity);
  293. void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
  294. struct drm_sched_entity *entity);
  295. int drm_sched_entity_init(struct drm_sched_entity *entity,
  296. enum drm_sched_priority priority,
  297. struct drm_gpu_scheduler **sched_list,
  298. unsigned int num_sched_list,
  299. atomic_t *guilty);
  300. long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
  301. void drm_sched_entity_fini(struct drm_sched_entity *entity);
  302. void drm_sched_entity_destroy(struct drm_sched_entity *entity);
  303. void drm_sched_entity_select_rq(struct drm_sched_entity *entity);
  304. struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity);
  305. void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
  306. struct drm_sched_entity *entity);
  307. void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
  308. enum drm_sched_priority priority);
  309. bool drm_sched_entity_is_ready(struct drm_sched_entity *entity);
  310. struct drm_sched_fence *drm_sched_fence_create(
  311. struct drm_sched_entity *s_entity, void *owner);
  312. void drm_sched_fence_scheduled(struct drm_sched_fence *fence);
  313. void drm_sched_fence_finished(struct drm_sched_fence *fence);
  314. unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
  315. void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
  316. unsigned long remaining);
  317. struct drm_gpu_scheduler *
  318. drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
  319. unsigned int num_sched_list);
  320. #endif