v3d_drv.h 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /* Copyright (C) 2015-2018 Broadcom */
  3. #include <linux/delay.h>
  4. #include <linux/mutex.h>
  5. #include <linux/spinlock_types.h>
  6. #include <linux/workqueue.h>
  7. #include <drm/drm_encoder.h>
  8. #include <drm/drm_gem.h>
  9. #include <drm/drm_gem_shmem_helper.h>
  10. #include <drm/gpu_scheduler.h>
  11. #include "uapi/drm/v3d_drm.h"
  12. struct clk;
  13. struct platform_device;
  14. struct reset_control;
  15. #define GMP_GRANULARITY (128 * 1024)
  16. /* Enum for each of the V3D queues. */
  17. enum v3d_queue {
  18. V3D_BIN,
  19. V3D_RENDER,
  20. V3D_TFU,
  21. V3D_CSD,
  22. V3D_CACHE_CLEAN,
  23. };
  24. #define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1)
  25. struct v3d_queue_state {
  26. struct drm_gpu_scheduler sched;
  27. u64 fence_context;
  28. u64 emit_seqno;
  29. };
  30. struct v3d_dev {
  31. struct drm_device drm;
  32. /* Short representation (e.g. 33, 41) of the V3D tech version
  33. * and revision.
  34. */
  35. int ver;
  36. bool single_irq_line;
  37. void __iomem *hub_regs;
  38. void __iomem *core_regs[3];
  39. void __iomem *bridge_regs;
  40. void __iomem *gca_regs;
  41. struct clk *clk;
  42. struct reset_control *reset;
  43. /* Virtual and DMA addresses of the single shared page table. */
  44. volatile u32 *pt;
  45. dma_addr_t pt_paddr;
  46. /* Virtual and DMA addresses of the MMU's scratch page. When
  47. * a read or write is invalid in the MMU, it will be
  48. * redirected here.
  49. */
  50. void *mmu_scratch;
  51. dma_addr_t mmu_scratch_paddr;
  52. /* virtual address bits from V3D to the MMU. */
  53. int va_width;
  54. /* Number of V3D cores. */
  55. u32 cores;
  56. /* Allocator managing the address space. All units are in
  57. * number of pages.
  58. */
  59. struct drm_mm mm;
  60. spinlock_t mm_lock;
  61. struct work_struct overflow_mem_work;
  62. struct v3d_bin_job *bin_job;
  63. struct v3d_render_job *render_job;
  64. struct v3d_tfu_job *tfu_job;
  65. struct v3d_csd_job *csd_job;
  66. struct v3d_queue_state queue[V3D_MAX_QUEUES];
  67. /* Spinlock used to synchronize the overflow memory
  68. * management against bin job submission.
  69. */
  70. spinlock_t job_lock;
  71. /* Protects bo_stats */
  72. struct mutex bo_lock;
  73. /* Lock taken when resetting the GPU, to keep multiple
  74. * processes from trying to park the scheduler threads and
  75. * reset at once.
  76. */
  77. struct mutex reset_lock;
  78. /* Lock taken when creating and pushing the GPU scheduler
  79. * jobs, to keep the sched-fence seqnos in order.
  80. */
  81. struct mutex sched_lock;
  82. /* Lock taken during a cache clean and when initiating an L2
  83. * flush, to keep L2 flushes from interfering with the
  84. * synchronous L2 cleans.
  85. */
  86. struct mutex cache_clean_lock;
  87. struct {
  88. u32 num_allocated;
  89. u32 pages_allocated;
  90. } bo_stats;
  91. };
  92. static inline struct v3d_dev *
  93. to_v3d_dev(struct drm_device *dev)
  94. {
  95. return container_of(dev, struct v3d_dev, drm);
  96. }
  97. static inline bool
  98. v3d_has_csd(struct v3d_dev *v3d)
  99. {
  100. return v3d->ver >= 41;
  101. }
  102. #define v3d_to_pdev(v3d) to_platform_device((v3d)->drm.dev)
  103. /* The per-fd struct, which tracks the MMU mappings. */
  104. struct v3d_file_priv {
  105. struct v3d_dev *v3d;
  106. struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
  107. };
  108. struct v3d_bo {
  109. struct drm_gem_shmem_object base;
  110. struct drm_mm_node node;
  111. /* List entry for the BO's position in
  112. * v3d_render_job->unref_list
  113. */
  114. struct list_head unref_head;
  115. };
  116. static inline struct v3d_bo *
  117. to_v3d_bo(struct drm_gem_object *bo)
  118. {
  119. return (struct v3d_bo *)bo;
  120. }
  121. struct v3d_fence {
  122. struct dma_fence base;
  123. struct drm_device *dev;
  124. /* v3d seqno for signaled() test */
  125. u64 seqno;
  126. enum v3d_queue queue;
  127. };
  128. static inline struct v3d_fence *
  129. to_v3d_fence(struct dma_fence *fence)
  130. {
  131. return (struct v3d_fence *)fence;
  132. }
  133. #define V3D_READ(offset) readl(v3d->hub_regs + offset)
  134. #define V3D_WRITE(offset, val) writel(val, v3d->hub_regs + offset)
  135. #define V3D_BRIDGE_READ(offset) readl(v3d->bridge_regs + offset)
  136. #define V3D_BRIDGE_WRITE(offset, val) writel(val, v3d->bridge_regs + offset)
  137. #define V3D_GCA_READ(offset) readl(v3d->gca_regs + offset)
  138. #define V3D_GCA_WRITE(offset, val) writel(val, v3d->gca_regs + offset)
  139. #define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset)
  140. #define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset)
  141. struct v3d_job {
  142. struct drm_sched_job base;
  143. struct kref refcount;
  144. struct v3d_dev *v3d;
  145. /* This is the array of BOs that were looked up at the start
  146. * of submission.
  147. */
  148. struct drm_gem_object **bo;
  149. u32 bo_count;
  150. /* Array of struct dma_fence * to block on before submitting this job.
  151. */
  152. struct xarray deps;
  153. unsigned long last_dep;
  154. /* v3d fence to be signaled by IRQ handler when the job is complete. */
  155. struct dma_fence *irq_fence;
  156. /* scheduler fence for when the job is considered complete and
  157. * the BO reservations can be released.
  158. */
  159. struct dma_fence *done_fence;
  160. /* Callback for the freeing of the job on refcount going to 0. */
  161. void (*free)(struct kref *ref);
  162. };
  163. struct v3d_bin_job {
  164. struct v3d_job base;
  165. /* GPU virtual addresses of the start/end of the CL job. */
  166. u32 start, end;
  167. u32 timedout_ctca, timedout_ctra;
  168. /* Corresponding render job, for attaching our overflow memory. */
  169. struct v3d_render_job *render;
  170. /* Submitted tile memory allocation start/size, tile state. */
  171. u32 qma, qms, qts;
  172. };
  173. struct v3d_render_job {
  174. struct v3d_job base;
  175. /* GPU virtual addresses of the start/end of the CL job. */
  176. u32 start, end;
  177. u32 timedout_ctca, timedout_ctra;
  178. /* List of overflow BOs used in the job that need to be
  179. * released once the job is complete.
  180. */
  181. struct list_head unref_list;
  182. };
  183. struct v3d_tfu_job {
  184. struct v3d_job base;
  185. struct drm_v3d_submit_tfu args;
  186. };
  187. struct v3d_csd_job {
  188. struct v3d_job base;
  189. u32 timedout_batches;
  190. struct drm_v3d_submit_csd args;
  191. };
  192. /**
  193. * __wait_for - magic wait macro
  194. *
  195. * Macro to help avoid open coding check/wait/timeout patterns. Note that it's
  196. * important that we check the condition again after having timed out, since the
  197. * timeout could be due to preemption or similar and we've never had a chance to
  198. * check the condition before the timeout.
  199. */
  200. #define __wait_for(OP, COND, US, Wmin, Wmax) ({ \
  201. const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (US)); \
  202. long wait__ = (Wmin); /* recommended min for usleep is 10 us */ \
  203. int ret__; \
  204. might_sleep(); \
  205. for (;;) { \
  206. const bool expired__ = ktime_after(ktime_get_raw(), end__); \
  207. OP; \
  208. /* Guarantee COND check prior to timeout */ \
  209. barrier(); \
  210. if (COND) { \
  211. ret__ = 0; \
  212. break; \
  213. } \
  214. if (expired__) { \
  215. ret__ = -ETIMEDOUT; \
  216. break; \
  217. } \
  218. usleep_range(wait__, wait__ * 2); \
  219. if (wait__ < (Wmax)) \
  220. wait__ <<= 1; \
  221. } \
  222. ret__; \
  223. })
  224. #define _wait_for(COND, US, Wmin, Wmax) __wait_for(, (COND), (US), (Wmin), \
  225. (Wmax))
  226. #define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)
  227. static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
  228. {
  229. /* nsecs_to_jiffies64() does not guard against overflow */
  230. if (NSEC_PER_SEC % HZ &&
  231. div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
  232. return MAX_JIFFY_OFFSET;
  233. return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
  234. }
  235. /* v3d_bo.c */
  236. struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size);
  237. void v3d_free_object(struct drm_gem_object *gem_obj);
  238. struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
  239. size_t size);
  240. int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
  241. struct drm_file *file_priv);
  242. int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
  243. struct drm_file *file_priv);
  244. int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
  245. struct drm_file *file_priv);
  246. struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
  247. struct dma_buf_attachment *attach,
  248. struct sg_table *sgt);
  249. /* v3d_debugfs.c */
  250. void v3d_debugfs_init(struct drm_minor *minor);
  251. /* v3d_fence.c */
  252. extern const struct dma_fence_ops v3d_fence_ops;
  253. struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue);
  254. /* v3d_gem.c */
  255. int v3d_gem_init(struct drm_device *dev);
  256. void v3d_gem_destroy(struct drm_device *dev);
  257. int v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
  258. struct drm_file *file_priv);
  259. int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
  260. struct drm_file *file_priv);
  261. int v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
  262. struct drm_file *file_priv);
  263. int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
  264. struct drm_file *file_priv);
  265. void v3d_job_put(struct v3d_job *job);
  266. void v3d_reset(struct v3d_dev *v3d);
  267. void v3d_invalidate_caches(struct v3d_dev *v3d);
  268. void v3d_clean_caches(struct v3d_dev *v3d);
  269. /* v3d_irq.c */
  270. int v3d_irq_init(struct v3d_dev *v3d);
  271. void v3d_irq_enable(struct v3d_dev *v3d);
  272. void v3d_irq_disable(struct v3d_dev *v3d);
  273. void v3d_irq_reset(struct v3d_dev *v3d);
  274. /* v3d_mmu.c */
  275. int v3d_mmu_get_offset(struct drm_file *file_priv, struct v3d_bo *bo,
  276. u32 *offset);
  277. int v3d_mmu_set_page_table(struct v3d_dev *v3d);
  278. void v3d_mmu_insert_ptes(struct v3d_bo *bo);
  279. void v3d_mmu_remove_ptes(struct v3d_bo *bo);
  280. /* v3d_sched.c */
  281. int v3d_sched_init(struct v3d_dev *v3d);
  282. void v3d_sched_fini(struct v3d_dev *v3d);