vc4_v3d.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2014 The Linux Foundation. All rights reserved.
  4. * Copyright (C) 2013 Red Hat
  5. * Author: Rob Clark <robdclark@gmail.com>
  6. */
  7. #include <linux/clk.h>
  8. #include <linux/component.h>
  9. #include <linux/platform_device.h>
  10. #include <linux/pm_runtime.h>
  11. #include <drm/drm_irq.h>
  12. #include "vc4_drv.h"
  13. #include "vc4_regs.h"
  14. static const struct debugfs_reg32 v3d_regs[] = {
  15. VC4_REG32(V3D_IDENT0),
  16. VC4_REG32(V3D_IDENT1),
  17. VC4_REG32(V3D_IDENT2),
  18. VC4_REG32(V3D_SCRATCH),
  19. VC4_REG32(V3D_L2CACTL),
  20. VC4_REG32(V3D_SLCACTL),
  21. VC4_REG32(V3D_INTCTL),
  22. VC4_REG32(V3D_INTENA),
  23. VC4_REG32(V3D_INTDIS),
  24. VC4_REG32(V3D_CT0CS),
  25. VC4_REG32(V3D_CT1CS),
  26. VC4_REG32(V3D_CT0EA),
  27. VC4_REG32(V3D_CT1EA),
  28. VC4_REG32(V3D_CT0CA),
  29. VC4_REG32(V3D_CT1CA),
  30. VC4_REG32(V3D_CT00RA0),
  31. VC4_REG32(V3D_CT01RA0),
  32. VC4_REG32(V3D_CT0LC),
  33. VC4_REG32(V3D_CT1LC),
  34. VC4_REG32(V3D_CT0PC),
  35. VC4_REG32(V3D_CT1PC),
  36. VC4_REG32(V3D_PCS),
  37. VC4_REG32(V3D_BFC),
  38. VC4_REG32(V3D_RFC),
  39. VC4_REG32(V3D_BPCA),
  40. VC4_REG32(V3D_BPCS),
  41. VC4_REG32(V3D_BPOA),
  42. VC4_REG32(V3D_BPOS),
  43. VC4_REG32(V3D_BXCF),
  44. VC4_REG32(V3D_SQRSV0),
  45. VC4_REG32(V3D_SQRSV1),
  46. VC4_REG32(V3D_SQCNTL),
  47. VC4_REG32(V3D_SRQPC),
  48. VC4_REG32(V3D_SRQUA),
  49. VC4_REG32(V3D_SRQUL),
  50. VC4_REG32(V3D_SRQCS),
  51. VC4_REG32(V3D_VPACNTL),
  52. VC4_REG32(V3D_VPMBASE),
  53. VC4_REG32(V3D_PCTRC),
  54. VC4_REG32(V3D_PCTRE),
  55. VC4_REG32(V3D_PCTR(0)),
  56. VC4_REG32(V3D_PCTRS(0)),
  57. VC4_REG32(V3D_PCTR(1)),
  58. VC4_REG32(V3D_PCTRS(1)),
  59. VC4_REG32(V3D_PCTR(2)),
  60. VC4_REG32(V3D_PCTRS(2)),
  61. VC4_REG32(V3D_PCTR(3)),
  62. VC4_REG32(V3D_PCTRS(3)),
  63. VC4_REG32(V3D_PCTR(4)),
  64. VC4_REG32(V3D_PCTRS(4)),
  65. VC4_REG32(V3D_PCTR(5)),
  66. VC4_REG32(V3D_PCTRS(5)),
  67. VC4_REG32(V3D_PCTR(6)),
  68. VC4_REG32(V3D_PCTRS(6)),
  69. VC4_REG32(V3D_PCTR(7)),
  70. VC4_REG32(V3D_PCTRS(7)),
  71. VC4_REG32(V3D_PCTR(8)),
  72. VC4_REG32(V3D_PCTRS(8)),
  73. VC4_REG32(V3D_PCTR(9)),
  74. VC4_REG32(V3D_PCTRS(9)),
  75. VC4_REG32(V3D_PCTR(10)),
  76. VC4_REG32(V3D_PCTRS(10)),
  77. VC4_REG32(V3D_PCTR(11)),
  78. VC4_REG32(V3D_PCTRS(11)),
  79. VC4_REG32(V3D_PCTR(12)),
  80. VC4_REG32(V3D_PCTRS(12)),
  81. VC4_REG32(V3D_PCTR(13)),
  82. VC4_REG32(V3D_PCTRS(13)),
  83. VC4_REG32(V3D_PCTR(14)),
  84. VC4_REG32(V3D_PCTRS(14)),
  85. VC4_REG32(V3D_PCTR(15)),
  86. VC4_REG32(V3D_PCTRS(15)),
  87. VC4_REG32(V3D_DBGE),
  88. VC4_REG32(V3D_FDBGO),
  89. VC4_REG32(V3D_FDBGB),
  90. VC4_REG32(V3D_FDBGR),
  91. VC4_REG32(V3D_FDBGS),
  92. VC4_REG32(V3D_ERRSTAT),
  93. };
  94. static int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused)
  95. {
  96. struct drm_info_node *node = (struct drm_info_node *)m->private;
  97. struct drm_device *dev = node->minor->dev;
  98. struct vc4_dev *vc4 = to_vc4_dev(dev);
  99. int ret = vc4_v3d_pm_get(vc4);
  100. if (ret == 0) {
  101. uint32_t ident1 = V3D_READ(V3D_IDENT1);
  102. uint32_t nslc = VC4_GET_FIELD(ident1, V3D_IDENT1_NSLC);
  103. uint32_t tups = VC4_GET_FIELD(ident1, V3D_IDENT1_TUPS);
  104. uint32_t qups = VC4_GET_FIELD(ident1, V3D_IDENT1_QUPS);
  105. seq_printf(m, "Revision: %d\n",
  106. VC4_GET_FIELD(ident1, V3D_IDENT1_REV));
  107. seq_printf(m, "Slices: %d\n", nslc);
  108. seq_printf(m, "TMUs: %d\n", nslc * tups);
  109. seq_printf(m, "QPUs: %d\n", nslc * qups);
  110. seq_printf(m, "Semaphores: %d\n",
  111. VC4_GET_FIELD(ident1, V3D_IDENT1_NSEM));
  112. vc4_v3d_pm_put(vc4);
  113. }
  114. return 0;
  115. }
  116. /**
  117. * Wraps pm_runtime_get_sync() in a refcount, so that we can reliably
  118. * get the pm_runtime refcount to 0 in vc4_reset().
  119. */
  120. int
  121. vc4_v3d_pm_get(struct vc4_dev *vc4)
  122. {
  123. mutex_lock(&vc4->power_lock);
  124. if (vc4->power_refcount++ == 0) {
  125. int ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
  126. if (ret < 0) {
  127. vc4->power_refcount--;
  128. mutex_unlock(&vc4->power_lock);
  129. return ret;
  130. }
  131. }
  132. mutex_unlock(&vc4->power_lock);
  133. return 0;
  134. }
  135. void
  136. vc4_v3d_pm_put(struct vc4_dev *vc4)
  137. {
  138. mutex_lock(&vc4->power_lock);
  139. if (--vc4->power_refcount == 0) {
  140. pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
  141. pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
  142. }
  143. mutex_unlock(&vc4->power_lock);
  144. }
  145. static void vc4_v3d_init_hw(struct drm_device *dev)
  146. {
  147. struct vc4_dev *vc4 = to_vc4_dev(dev);
  148. /* Take all the memory that would have been reserved for user
  149. * QPU programs, since we don't have an interface for running
  150. * them, anyway.
  151. */
  152. V3D_WRITE(V3D_VPMBASE, 0);
  153. }
  154. int vc4_v3d_get_bin_slot(struct vc4_dev *vc4)
  155. {
  156. struct drm_device *dev = &vc4->base;
  157. unsigned long irqflags;
  158. int slot;
  159. uint64_t seqno = 0;
  160. struct vc4_exec_info *exec;
  161. try_again:
  162. spin_lock_irqsave(&vc4->job_lock, irqflags);
  163. slot = ffs(~vc4->bin_alloc_used);
  164. if (slot != 0) {
  165. /* Switch from ffs() bit index to a 0-based index. */
  166. slot--;
  167. vc4->bin_alloc_used |= BIT(slot);
  168. spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  169. return slot;
  170. }
  171. /* Couldn't find an open slot. Wait for render to complete
  172. * and try again.
  173. */
  174. exec = vc4_last_render_job(vc4);
  175. if (exec)
  176. seqno = exec->seqno;
  177. spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  178. if (seqno) {
  179. int ret = vc4_wait_for_seqno(dev, seqno, ~0ull, true);
  180. if (ret == 0)
  181. goto try_again;
  182. return ret;
  183. }
  184. return -ENOMEM;
  185. }
  186. /**
  187. * bin_bo_alloc() - allocates the memory that will be used for
  188. * tile binning.
  189. *
  190. * The binner has a limitation that the addresses in the tile state
  191. * buffer that point into the tile alloc buffer or binner overflow
  192. * memory only have 28 bits (256MB), and the top 4 on the bus for
  193. * tile alloc references end up coming from the tile state buffer's
  194. * address.
  195. *
  196. * To work around this, we allocate a single large buffer while V3D is
  197. * in use, make sure that it has the top 4 bits constant across its
  198. * entire extent, and then put the tile state, tile alloc, and binner
  199. * overflow memory inside that buffer.
  200. *
  201. * This creates a limitation where we may not be able to execute a job
  202. * if it doesn't fit within the buffer that we allocated up front.
  203. * However, it turns out that 16MB is "enough for anybody", and
  204. * real-world applications run into allocation failures from the
  205. * overall CMA pool before they make scenes complicated enough to run
  206. * out of bin space.
  207. */
  208. static int bin_bo_alloc(struct vc4_dev *vc4)
  209. {
  210. struct vc4_v3d *v3d = vc4->v3d;
  211. uint32_t size = 16 * 1024 * 1024;
  212. int ret = 0;
  213. struct list_head list;
  214. if (!v3d)
  215. return -ENODEV;
  216. /* We may need to try allocating more than once to get a BO
  217. * that doesn't cross 256MB. Track the ones we've allocated
  218. * that failed so far, so that we can free them when we've got
  219. * one that succeeded (if we freed them right away, our next
  220. * allocation would probably be the same chunk of memory).
  221. */
  222. INIT_LIST_HEAD(&list);
  223. while (true) {
  224. struct vc4_bo *bo = vc4_bo_create(&vc4->base, size, true,
  225. VC4_BO_TYPE_BIN);
  226. if (IS_ERR(bo)) {
  227. ret = PTR_ERR(bo);
  228. dev_err(&v3d->pdev->dev,
  229. "Failed to allocate memory for tile binning: "
  230. "%d. You may need to enable CMA or give it "
  231. "more memory.",
  232. ret);
  233. break;
  234. }
  235. /* Check if this BO won't trigger the addressing bug. */
  236. if ((bo->base.paddr & 0xf0000000) ==
  237. ((bo->base.paddr + bo->base.base.size - 1) & 0xf0000000)) {
  238. vc4->bin_bo = bo;
  239. /* Set up for allocating 512KB chunks of
  240. * binner memory. The biggest allocation we
  241. * need to do is for the initial tile alloc +
  242. * tile state buffer. We can render to a
  243. * maximum of ((2048*2048) / (32*32) = 4096
  244. * tiles in a frame (until we do floating
  245. * point rendering, at which point it would be
  246. * 8192). Tile state is 48b/tile (rounded to
  247. * a page), and tile alloc is 32b/tile
  248. * (rounded to a page), plus a page of extra,
  249. * for a total of 320kb for our worst-case.
  250. * We choose 512kb so that it divides evenly
  251. * into our 16MB, and the rest of the 512kb
  252. * will be used as storage for the overflow
  253. * from the initial 32b CL per bin.
  254. */
  255. vc4->bin_alloc_size = 512 * 1024;
  256. vc4->bin_alloc_used = 0;
  257. vc4->bin_alloc_overflow = 0;
  258. WARN_ON_ONCE(sizeof(vc4->bin_alloc_used) * 8 !=
  259. bo->base.base.size / vc4->bin_alloc_size);
  260. kref_init(&vc4->bin_bo_kref);
  261. /* Enable the out-of-memory interrupt to set our
  262. * newly-allocated binner BO, potentially from an
  263. * already-pending-but-masked interrupt.
  264. */
  265. V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM);
  266. break;
  267. }
  268. /* Put it on the list to free later, and try again. */
  269. list_add(&bo->unref_head, &list);
  270. }
  271. /* Free all the BOs we allocated but didn't choose. */
  272. while (!list_empty(&list)) {
  273. struct vc4_bo *bo = list_last_entry(&list,
  274. struct vc4_bo, unref_head);
  275. list_del(&bo->unref_head);
  276. drm_gem_object_put(&bo->base.base);
  277. }
  278. return ret;
  279. }
  280. int vc4_v3d_bin_bo_get(struct vc4_dev *vc4, bool *used)
  281. {
  282. int ret = 0;
  283. mutex_lock(&vc4->bin_bo_lock);
  284. if (used && *used)
  285. goto complete;
  286. if (vc4->bin_bo)
  287. kref_get(&vc4->bin_bo_kref);
  288. else
  289. ret = bin_bo_alloc(vc4);
  290. if (ret == 0 && used)
  291. *used = true;
  292. complete:
  293. mutex_unlock(&vc4->bin_bo_lock);
  294. return ret;
  295. }
  296. static void bin_bo_release(struct kref *ref)
  297. {
  298. struct vc4_dev *vc4 = container_of(ref, struct vc4_dev, bin_bo_kref);
  299. if (WARN_ON_ONCE(!vc4->bin_bo))
  300. return;
  301. drm_gem_object_put(&vc4->bin_bo->base.base);
  302. vc4->bin_bo = NULL;
  303. }
  304. void vc4_v3d_bin_bo_put(struct vc4_dev *vc4)
  305. {
  306. mutex_lock(&vc4->bin_bo_lock);
  307. kref_put(&vc4->bin_bo_kref, bin_bo_release);
  308. mutex_unlock(&vc4->bin_bo_lock);
  309. }
  310. #ifdef CONFIG_PM
  311. static int vc4_v3d_runtime_suspend(struct device *dev)
  312. {
  313. struct vc4_v3d *v3d = dev_get_drvdata(dev);
  314. struct vc4_dev *vc4 = v3d->vc4;
  315. vc4_irq_uninstall(&vc4->base);
  316. clk_disable_unprepare(v3d->clk);
  317. return 0;
  318. }
  319. static int vc4_v3d_runtime_resume(struct device *dev)
  320. {
  321. struct vc4_v3d *v3d = dev_get_drvdata(dev);
  322. struct vc4_dev *vc4 = v3d->vc4;
  323. int ret;
  324. ret = clk_prepare_enable(v3d->clk);
  325. if (ret != 0)
  326. return ret;
  327. vc4_v3d_init_hw(&vc4->base);
  328. /* We disabled the IRQ as part of vc4_irq_uninstall in suspend. */
  329. enable_irq(vc4->base.irq);
  330. vc4_irq_postinstall(&vc4->base);
  331. return 0;
  332. }
  333. #endif
  334. static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
  335. {
  336. struct platform_device *pdev = to_platform_device(dev);
  337. struct drm_device *drm = dev_get_drvdata(master);
  338. struct vc4_dev *vc4 = to_vc4_dev(drm);
  339. struct vc4_v3d *v3d = NULL;
  340. int ret;
  341. v3d = devm_kzalloc(&pdev->dev, sizeof(*v3d), GFP_KERNEL);
  342. if (!v3d)
  343. return -ENOMEM;
  344. dev_set_drvdata(dev, v3d);
  345. v3d->pdev = pdev;
  346. v3d->regs = vc4_ioremap_regs(pdev, 0);
  347. if (IS_ERR(v3d->regs))
  348. return PTR_ERR(v3d->regs);
  349. v3d->regset.base = v3d->regs;
  350. v3d->regset.regs = v3d_regs;
  351. v3d->regset.nregs = ARRAY_SIZE(v3d_regs);
  352. vc4->v3d = v3d;
  353. v3d->vc4 = vc4;
  354. v3d->clk = devm_clk_get(dev, NULL);
  355. if (IS_ERR(v3d->clk)) {
  356. int ret = PTR_ERR(v3d->clk);
  357. if (ret == -ENOENT) {
  358. /* bcm2835 didn't have a clock reference in the DT. */
  359. ret = 0;
  360. v3d->clk = NULL;
  361. } else {
  362. if (ret != -EPROBE_DEFER)
  363. dev_err(dev, "Failed to get V3D clock: %d\n",
  364. ret);
  365. return ret;
  366. }
  367. }
  368. if (V3D_READ(V3D_IDENT0) != V3D_EXPECTED_IDENT0) {
  369. DRM_ERROR("V3D_IDENT0 read 0x%08x instead of 0x%08x\n",
  370. V3D_READ(V3D_IDENT0), V3D_EXPECTED_IDENT0);
  371. return -EINVAL;
  372. }
  373. ret = clk_prepare_enable(v3d->clk);
  374. if (ret != 0)
  375. return ret;
  376. /* Reset the binner overflow address/size at setup, to be sure
  377. * we don't reuse an old one.
  378. */
  379. V3D_WRITE(V3D_BPOA, 0);
  380. V3D_WRITE(V3D_BPOS, 0);
  381. vc4_v3d_init_hw(drm);
  382. ret = drm_irq_install(drm, platform_get_irq(pdev, 0));
  383. if (ret) {
  384. DRM_ERROR("Failed to install IRQ handler\n");
  385. return ret;
  386. }
  387. pm_runtime_set_active(dev);
  388. pm_runtime_use_autosuspend(dev);
  389. pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */
  390. pm_runtime_enable(dev);
  391. vc4_debugfs_add_file(drm, "v3d_ident", vc4_v3d_debugfs_ident, NULL);
  392. vc4_debugfs_add_regset32(drm, "v3d_regs", &v3d->regset);
  393. return 0;
  394. }
  395. static void vc4_v3d_unbind(struct device *dev, struct device *master,
  396. void *data)
  397. {
  398. struct drm_device *drm = dev_get_drvdata(master);
  399. struct vc4_dev *vc4 = to_vc4_dev(drm);
  400. pm_runtime_disable(dev);
  401. drm_irq_uninstall(drm);
  402. /* Disable the binner's overflow memory address, so the next
  403. * driver probe (if any) doesn't try to reuse our old
  404. * allocation.
  405. */
  406. V3D_WRITE(V3D_BPOA, 0);
  407. V3D_WRITE(V3D_BPOS, 0);
  408. vc4->v3d = NULL;
  409. }
  410. static const struct dev_pm_ops vc4_v3d_pm_ops = {
  411. SET_RUNTIME_PM_OPS(vc4_v3d_runtime_suspend, vc4_v3d_runtime_resume, NULL)
  412. };
  413. static const struct component_ops vc4_v3d_ops = {
  414. .bind = vc4_v3d_bind,
  415. .unbind = vc4_v3d_unbind,
  416. };
  417. static int vc4_v3d_dev_probe(struct platform_device *pdev)
  418. {
  419. return component_add(&pdev->dev, &vc4_v3d_ops);
  420. }
  421. static int vc4_v3d_dev_remove(struct platform_device *pdev)
  422. {
  423. component_del(&pdev->dev, &vc4_v3d_ops);
  424. return 0;
  425. }
  426. const struct of_device_id vc4_v3d_dt_match[] = {
  427. { .compatible = "brcm,bcm2835-v3d" },
  428. { .compatible = "brcm,cygnus-v3d" },
  429. { .compatible = "brcm,vc4-v3d" },
  430. {}
  431. };
  432. struct platform_driver vc4_v3d_driver = {
  433. .probe = vc4_v3d_dev_probe,
  434. .remove = vc4_v3d_dev_remove,
  435. .driver = {
  436. .name = "vc4_v3d",
  437. .of_match_table = vc4_v3d_dt_match,
  438. .pm = &vc4_v3d_pm_ops,
  439. },
  440. };