lima_device.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534
  1. // SPDX-License-Identifier: GPL-2.0 OR MIT
  2. /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3. #include <linux/regulator/consumer.h>
  4. #include <linux/reset.h>
  5. #include <linux/clk.h>
  6. #include <linux/dma-mapping.h>
  7. #include <linux/platform_device.h>
  8. #include "lima_device.h"
  9. #include "lima_gp.h"
  10. #include "lima_pp.h"
  11. #include "lima_mmu.h"
  12. #include "lima_pmu.h"
  13. #include "lima_l2_cache.h"
  14. #include "lima_dlbu.h"
  15. #include "lima_bcast.h"
  16. #include "lima_vm.h"
  17. struct lima_ip_desc {
  18. char *name;
  19. char *irq_name;
  20. bool must_have[lima_gpu_num];
  21. int offset[lima_gpu_num];
  22. int (*init)(struct lima_ip *ip);
  23. void (*fini)(struct lima_ip *ip);
  24. int (*resume)(struct lima_ip *ip);
  25. void (*suspend)(struct lima_ip *ip);
  26. };
  27. #define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \
  28. [lima_ip_##ipname] = { \
  29. .name = #ipname, \
  30. .irq_name = irq, \
  31. .must_have = { \
  32. [lima_gpu_mali400] = mst0, \
  33. [lima_gpu_mali450] = mst1, \
  34. }, \
  35. .offset = { \
  36. [lima_gpu_mali400] = off0, \
  37. [lima_gpu_mali450] = off1, \
  38. }, \
  39. .init = lima_##func##_init, \
  40. .fini = lima_##func##_fini, \
  41. .resume = lima_##func##_resume, \
  42. .suspend = lima_##func##_suspend, \
  43. }
  44. static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {
  45. LIMA_IP_DESC(pmu, false, false, 0x02000, 0x02000, pmu, "pmu"),
  46. LIMA_IP_DESC(l2_cache0, true, true, 0x01000, 0x10000, l2_cache, NULL),
  47. LIMA_IP_DESC(l2_cache1, false, true, -1, 0x01000, l2_cache, NULL),
  48. LIMA_IP_DESC(l2_cache2, false, false, -1, 0x11000, l2_cache, NULL),
  49. LIMA_IP_DESC(gp, true, true, 0x00000, 0x00000, gp, "gp"),
  50. LIMA_IP_DESC(pp0, true, true, 0x08000, 0x08000, pp, "pp0"),
  51. LIMA_IP_DESC(pp1, false, false, 0x0A000, 0x0A000, pp, "pp1"),
  52. LIMA_IP_DESC(pp2, false, false, 0x0C000, 0x0C000, pp, "pp2"),
  53. LIMA_IP_DESC(pp3, false, false, 0x0E000, 0x0E000, pp, "pp3"),
  54. LIMA_IP_DESC(pp4, false, false, -1, 0x28000, pp, "pp4"),
  55. LIMA_IP_DESC(pp5, false, false, -1, 0x2A000, pp, "pp5"),
  56. LIMA_IP_DESC(pp6, false, false, -1, 0x2C000, pp, "pp6"),
  57. LIMA_IP_DESC(pp7, false, false, -1, 0x2E000, pp, "pp7"),
  58. LIMA_IP_DESC(gpmmu, true, true, 0x03000, 0x03000, mmu, "gpmmu"),
  59. LIMA_IP_DESC(ppmmu0, true, true, 0x04000, 0x04000, mmu, "ppmmu0"),
  60. LIMA_IP_DESC(ppmmu1, false, false, 0x05000, 0x05000, mmu, "ppmmu1"),
  61. LIMA_IP_DESC(ppmmu2, false, false, 0x06000, 0x06000, mmu, "ppmmu2"),
  62. LIMA_IP_DESC(ppmmu3, false, false, 0x07000, 0x07000, mmu, "ppmmu3"),
  63. LIMA_IP_DESC(ppmmu4, false, false, -1, 0x1C000, mmu, "ppmmu4"),
  64. LIMA_IP_DESC(ppmmu5, false, false, -1, 0x1D000, mmu, "ppmmu5"),
  65. LIMA_IP_DESC(ppmmu6, false, false, -1, 0x1E000, mmu, "ppmmu6"),
  66. LIMA_IP_DESC(ppmmu7, false, false, -1, 0x1F000, mmu, "ppmmu7"),
  67. LIMA_IP_DESC(dlbu, false, true, -1, 0x14000, dlbu, NULL),
  68. LIMA_IP_DESC(bcast, false, true, -1, 0x13000, bcast, NULL),
  69. LIMA_IP_DESC(pp_bcast, false, true, -1, 0x16000, pp_bcast, "pp"),
  70. LIMA_IP_DESC(ppmmu_bcast, false, true, -1, 0x15000, mmu, NULL),
  71. };
  72. const char *lima_ip_name(struct lima_ip *ip)
  73. {
  74. return lima_ip_desc[ip->id].name;
  75. }
  76. static int lima_clk_enable(struct lima_device *dev)
  77. {
  78. int err;
  79. err = clk_prepare_enable(dev->clk_bus);
  80. if (err)
  81. return err;
  82. err = clk_prepare_enable(dev->clk_gpu);
  83. if (err)
  84. goto error_out0;
  85. if (dev->reset) {
  86. err = reset_control_deassert(dev->reset);
  87. if (err) {
  88. dev_err(dev->dev,
  89. "reset controller deassert failed %d\n", err);
  90. goto error_out1;
  91. }
  92. }
  93. return 0;
  94. error_out1:
  95. clk_disable_unprepare(dev->clk_gpu);
  96. error_out0:
  97. clk_disable_unprepare(dev->clk_bus);
  98. return err;
  99. }
  100. static void lima_clk_disable(struct lima_device *dev)
  101. {
  102. if (dev->reset)
  103. reset_control_assert(dev->reset);
  104. clk_disable_unprepare(dev->clk_gpu);
  105. clk_disable_unprepare(dev->clk_bus);
  106. }
  107. static int lima_clk_init(struct lima_device *dev)
  108. {
  109. int err;
  110. dev->clk_bus = devm_clk_get(dev->dev, "bus");
  111. if (IS_ERR(dev->clk_bus)) {
  112. err = PTR_ERR(dev->clk_bus);
  113. if (err != -EPROBE_DEFER)
  114. dev_err(dev->dev, "get bus clk failed %d\n", err);
  115. dev->clk_bus = NULL;
  116. return err;
  117. }
  118. dev->clk_gpu = devm_clk_get(dev->dev, "core");
  119. if (IS_ERR(dev->clk_gpu)) {
  120. err = PTR_ERR(dev->clk_gpu);
  121. if (err != -EPROBE_DEFER)
  122. dev_err(dev->dev, "get core clk failed %d\n", err);
  123. dev->clk_gpu = NULL;
  124. return err;
  125. }
  126. dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);
  127. if (IS_ERR(dev->reset)) {
  128. err = PTR_ERR(dev->reset);
  129. if (err != -EPROBE_DEFER)
  130. dev_err(dev->dev, "get reset controller failed %d\n",
  131. err);
  132. dev->reset = NULL;
  133. return err;
  134. }
  135. return lima_clk_enable(dev);
  136. }
  137. static void lima_clk_fini(struct lima_device *dev)
  138. {
  139. lima_clk_disable(dev);
  140. }
  141. static int lima_regulator_enable(struct lima_device *dev)
  142. {
  143. int ret;
  144. if (!dev->regulator)
  145. return 0;
  146. ret = regulator_enable(dev->regulator);
  147. if (ret < 0) {
  148. dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
  149. return ret;
  150. }
  151. return 0;
  152. }
  153. static void lima_regulator_disable(struct lima_device *dev)
  154. {
  155. if (dev->regulator)
  156. regulator_disable(dev->regulator);
  157. }
  158. static int lima_regulator_init(struct lima_device *dev)
  159. {
  160. int ret;
  161. dev->regulator = devm_regulator_get_optional(dev->dev, "mali");
  162. if (IS_ERR(dev->regulator)) {
  163. ret = PTR_ERR(dev->regulator);
  164. dev->regulator = NULL;
  165. if (ret == -ENODEV)
  166. return 0;
  167. if (ret != -EPROBE_DEFER)
  168. dev_err(dev->dev, "failed to get regulator: %d\n", ret);
  169. return ret;
  170. }
  171. return lima_regulator_enable(dev);
  172. }
  173. static void lima_regulator_fini(struct lima_device *dev)
  174. {
  175. lima_regulator_disable(dev);
  176. }
  177. static int lima_init_ip(struct lima_device *dev, int index)
  178. {
  179. struct platform_device *pdev = to_platform_device(dev->dev);
  180. struct lima_ip_desc *desc = lima_ip_desc + index;
  181. struct lima_ip *ip = dev->ip + index;
  182. const char *irq_name = desc->irq_name;
  183. int offset = desc->offset[dev->id];
  184. bool must = desc->must_have[dev->id];
  185. int err;
  186. if (offset < 0)
  187. return 0;
  188. ip->dev = dev;
  189. ip->id = index;
  190. ip->iomem = dev->iomem + offset;
  191. if (irq_name) {
  192. err = must ? platform_get_irq_byname(pdev, irq_name) :
  193. platform_get_irq_byname_optional(pdev, irq_name);
  194. if (err < 0)
  195. goto out;
  196. ip->irq = err;
  197. }
  198. err = desc->init(ip);
  199. if (!err) {
  200. ip->present = true;
  201. return 0;
  202. }
  203. out:
  204. return must ? err : 0;
  205. }
  206. static void lima_fini_ip(struct lima_device *ldev, int index)
  207. {
  208. struct lima_ip_desc *desc = lima_ip_desc + index;
  209. struct lima_ip *ip = ldev->ip + index;
  210. if (ip->present)
  211. desc->fini(ip);
  212. }
  213. static int lima_resume_ip(struct lima_device *ldev, int index)
  214. {
  215. struct lima_ip_desc *desc = lima_ip_desc + index;
  216. struct lima_ip *ip = ldev->ip + index;
  217. int ret = 0;
  218. if (ip->present)
  219. ret = desc->resume(ip);
  220. return ret;
  221. }
  222. static void lima_suspend_ip(struct lima_device *ldev, int index)
  223. {
  224. struct lima_ip_desc *desc = lima_ip_desc + index;
  225. struct lima_ip *ip = ldev->ip + index;
  226. if (ip->present)
  227. desc->suspend(ip);
  228. }
  229. static int lima_init_gp_pipe(struct lima_device *dev)
  230. {
  231. struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
  232. int err;
  233. pipe->ldev = dev;
  234. err = lima_sched_pipe_init(pipe, "gp");
  235. if (err)
  236. return err;
  237. pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0;
  238. pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu;
  239. pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp;
  240. err = lima_gp_pipe_init(dev);
  241. if (err) {
  242. lima_sched_pipe_fini(pipe);
  243. return err;
  244. }
  245. return 0;
  246. }
  247. static void lima_fini_gp_pipe(struct lima_device *dev)
  248. {
  249. struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
  250. lima_gp_pipe_fini(dev);
  251. lima_sched_pipe_fini(pipe);
  252. }
  253. static int lima_init_pp_pipe(struct lima_device *dev)
  254. {
  255. struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
  256. int err, i;
  257. pipe->ldev = dev;
  258. err = lima_sched_pipe_init(pipe, "pp");
  259. if (err)
  260. return err;
  261. for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) {
  262. struct lima_ip *pp = dev->ip + lima_ip_pp0 + i;
  263. struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i;
  264. struct lima_ip *l2_cache;
  265. if (dev->id == lima_gpu_mali400)
  266. l2_cache = dev->ip + lima_ip_l2_cache0;
  267. else
  268. l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2);
  269. if (pp->present && ppmmu->present && l2_cache->present) {
  270. pipe->mmu[pipe->num_mmu++] = ppmmu;
  271. pipe->processor[pipe->num_processor++] = pp;
  272. if (!pipe->l2_cache[i >> 2])
  273. pipe->l2_cache[pipe->num_l2_cache++] = l2_cache;
  274. }
  275. }
  276. if (dev->ip[lima_ip_bcast].present) {
  277. pipe->bcast_processor = dev->ip + lima_ip_pp_bcast;
  278. pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast;
  279. }
  280. err = lima_pp_pipe_init(dev);
  281. if (err) {
  282. lima_sched_pipe_fini(pipe);
  283. return err;
  284. }
  285. return 0;
  286. }
  287. static void lima_fini_pp_pipe(struct lima_device *dev)
  288. {
  289. struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
  290. lima_pp_pipe_fini(dev);
  291. lima_sched_pipe_fini(pipe);
  292. }
  293. int lima_device_init(struct lima_device *ldev)
  294. {
  295. struct platform_device *pdev = to_platform_device(ldev->dev);
  296. int err, i;
  297. dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
  298. dma_set_max_seg_size(ldev->dev, UINT_MAX);
  299. err = lima_clk_init(ldev);
  300. if (err)
  301. return err;
  302. err = lima_regulator_init(ldev);
  303. if (err)
  304. goto err_out0;
  305. ldev->empty_vm = lima_vm_create(ldev);
  306. if (!ldev->empty_vm) {
  307. err = -ENOMEM;
  308. goto err_out1;
  309. }
  310. ldev->va_start = 0;
  311. if (ldev->id == lima_gpu_mali450) {
  312. ldev->va_end = LIMA_VA_RESERVE_START;
  313. ldev->dlbu_cpu = dma_alloc_wc(
  314. ldev->dev, LIMA_PAGE_SIZE,
  315. &ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN);
  316. if (!ldev->dlbu_cpu) {
  317. err = -ENOMEM;
  318. goto err_out2;
  319. }
  320. } else
  321. ldev->va_end = LIMA_VA_RESERVE_END;
  322. ldev->iomem = devm_platform_ioremap_resource(pdev, 0);
  323. if (IS_ERR(ldev->iomem)) {
  324. dev_err(ldev->dev, "fail to ioremap iomem\n");
  325. err = PTR_ERR(ldev->iomem);
  326. goto err_out3;
  327. }
  328. for (i = 0; i < lima_ip_num; i++) {
  329. err = lima_init_ip(ldev, i);
  330. if (err)
  331. goto err_out4;
  332. }
  333. err = lima_init_gp_pipe(ldev);
  334. if (err)
  335. goto err_out4;
  336. err = lima_init_pp_pipe(ldev);
  337. if (err)
  338. goto err_out5;
  339. ldev->dump.magic = LIMA_DUMP_MAGIC;
  340. ldev->dump.version_major = LIMA_DUMP_MAJOR;
  341. ldev->dump.version_minor = LIMA_DUMP_MINOR;
  342. INIT_LIST_HEAD(&ldev->error_task_list);
  343. mutex_init(&ldev->error_task_list_lock);
  344. dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus));
  345. dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu));
  346. return 0;
  347. err_out5:
  348. lima_fini_gp_pipe(ldev);
  349. err_out4:
  350. while (--i >= 0)
  351. lima_fini_ip(ldev, i);
  352. err_out3:
  353. if (ldev->dlbu_cpu)
  354. dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
  355. ldev->dlbu_cpu, ldev->dlbu_dma);
  356. err_out2:
  357. lima_vm_put(ldev->empty_vm);
  358. err_out1:
  359. lima_regulator_fini(ldev);
  360. err_out0:
  361. lima_clk_fini(ldev);
  362. return err;
  363. }
  364. void lima_device_fini(struct lima_device *ldev)
  365. {
  366. int i;
  367. struct lima_sched_error_task *et, *tmp;
  368. list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
  369. list_del(&et->list);
  370. kvfree(et);
  371. }
  372. mutex_destroy(&ldev->error_task_list_lock);
  373. lima_fini_pp_pipe(ldev);
  374. lima_fini_gp_pipe(ldev);
  375. for (i = lima_ip_num - 1; i >= 0; i--)
  376. lima_fini_ip(ldev, i);
  377. if (ldev->dlbu_cpu)
  378. dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
  379. ldev->dlbu_cpu, ldev->dlbu_dma);
  380. lima_vm_put(ldev->empty_vm);
  381. lima_regulator_fini(ldev);
  382. lima_clk_fini(ldev);
  383. }
  384. int lima_device_resume(struct device *dev)
  385. {
  386. struct lima_device *ldev = dev_get_drvdata(dev);
  387. int i, err;
  388. err = lima_clk_enable(ldev);
  389. if (err) {
  390. dev_err(dev, "resume clk fail %d\n", err);
  391. return err;
  392. }
  393. err = lima_regulator_enable(ldev);
  394. if (err) {
  395. dev_err(dev, "resume regulator fail %d\n", err);
  396. goto err_out0;
  397. }
  398. for (i = 0; i < lima_ip_num; i++) {
  399. err = lima_resume_ip(ldev, i);
  400. if (err) {
  401. dev_err(dev, "resume ip %d fail\n", i);
  402. goto err_out1;
  403. }
  404. }
  405. err = lima_devfreq_resume(&ldev->devfreq);
  406. if (err) {
  407. dev_err(dev, "devfreq resume fail\n");
  408. goto err_out1;
  409. }
  410. return 0;
  411. err_out1:
  412. while (--i >= 0)
  413. lima_suspend_ip(ldev, i);
  414. lima_regulator_disable(ldev);
  415. err_out0:
  416. lima_clk_disable(ldev);
  417. return err;
  418. }
  419. int lima_device_suspend(struct device *dev)
  420. {
  421. struct lima_device *ldev = dev_get_drvdata(dev);
  422. int i, err;
  423. /* check any task running */
  424. for (i = 0; i < lima_pipe_num; i++) {
  425. if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
  426. return -EBUSY;
  427. }
  428. err = lima_devfreq_suspend(&ldev->devfreq);
  429. if (err) {
  430. dev_err(dev, "devfreq suspend fail\n");
  431. return err;
  432. }
  433. for (i = lima_ip_num - 1; i >= 0; i--)
  434. lima_suspend_ip(ldev, i);
  435. lima_regulator_disable(ldev);
  436. lima_clk_disable(ldev);
  437. return 0;
  438. }