exynos_drm_g2d.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2012 Samsung Electronics Co.Ltd
  4. * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
  5. */
  6. #include <linux/clk.h>
  7. #include <linux/component.h>
  8. #include <linux/delay.h>
  9. #include <linux/dma-mapping.h>
  10. #include <linux/err.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/io.h>
  13. #include <linux/kernel.h>
  14. #include <linux/of.h>
  15. #include <linux/platform_device.h>
  16. #include <linux/pm_runtime.h>
  17. #include <linux/slab.h>
  18. #include <linux/uaccess.h>
  19. #include <linux/workqueue.h>
  20. #include <drm/drm_file.h>
  21. #include <drm/exynos_drm.h>
  22. #include "exynos_drm_drv.h"
  23. #include "exynos_drm_g2d.h"
  24. #include "exynos_drm_gem.h"
  25. #define G2D_HW_MAJOR_VER 4
  26. #define G2D_HW_MINOR_VER 1
  27. /* vaild register range set from user: 0x0104 ~ 0x0880 */
  28. #define G2D_VALID_START 0x0104
  29. #define G2D_VALID_END 0x0880
  30. /* general registers */
  31. #define G2D_SOFT_RESET 0x0000
  32. #define G2D_INTEN 0x0004
  33. #define G2D_INTC_PEND 0x000C
  34. #define G2D_DMA_SFR_BASE_ADDR 0x0080
  35. #define G2D_DMA_COMMAND 0x0084
  36. #define G2D_DMA_STATUS 0x008C
  37. #define G2D_DMA_HOLD_CMD 0x0090
  38. /* command registers */
  39. #define G2D_BITBLT_START 0x0100
  40. /* registers for base address */
  41. #define G2D_SRC_BASE_ADDR 0x0304
  42. #define G2D_SRC_STRIDE 0x0308
  43. #define G2D_SRC_COLOR_MODE 0x030C
  44. #define G2D_SRC_LEFT_TOP 0x0310
  45. #define G2D_SRC_RIGHT_BOTTOM 0x0314
  46. #define G2D_SRC_PLANE2_BASE_ADDR 0x0318
  47. #define G2D_DST_BASE_ADDR 0x0404
  48. #define G2D_DST_STRIDE 0x0408
  49. #define G2D_DST_COLOR_MODE 0x040C
  50. #define G2D_DST_LEFT_TOP 0x0410
  51. #define G2D_DST_RIGHT_BOTTOM 0x0414
  52. #define G2D_DST_PLANE2_BASE_ADDR 0x0418
  53. #define G2D_PAT_BASE_ADDR 0x0500
  54. #define G2D_MSK_BASE_ADDR 0x0520
  55. /* G2D_SOFT_RESET */
  56. #define G2D_SFRCLEAR (1 << 1)
  57. #define G2D_R (1 << 0)
  58. /* G2D_INTEN */
  59. #define G2D_INTEN_ACF (1 << 3)
  60. #define G2D_INTEN_UCF (1 << 2)
  61. #define G2D_INTEN_GCF (1 << 1)
  62. #define G2D_INTEN_SCF (1 << 0)
  63. /* G2D_INTC_PEND */
  64. #define G2D_INTP_ACMD_FIN (1 << 3)
  65. #define G2D_INTP_UCMD_FIN (1 << 2)
  66. #define G2D_INTP_GCMD_FIN (1 << 1)
  67. #define G2D_INTP_SCMD_FIN (1 << 0)
  68. /* G2D_DMA_COMMAND */
  69. #define G2D_DMA_HALT (1 << 2)
  70. #define G2D_DMA_CONTINUE (1 << 1)
  71. #define G2D_DMA_START (1 << 0)
  72. /* G2D_DMA_STATUS */
  73. #define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)
  74. #define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)
  75. #define G2D_DMA_DONE (1 << 0)
  76. #define G2D_DMA_LIST_DONE_COUNT_OFFSET 17
  77. /* G2D_DMA_HOLD_CMD */
  78. #define G2D_USER_HOLD (1 << 2)
  79. #define G2D_LIST_HOLD (1 << 1)
  80. #define G2D_BITBLT_HOLD (1 << 0)
  81. /* G2D_BITBLT_START */
  82. #define G2D_START_CASESEL (1 << 2)
  83. #define G2D_START_NHOLT (1 << 1)
  84. #define G2D_START_BITBLT (1 << 0)
  85. /* buffer color format */
  86. #define G2D_FMT_XRGB8888 0
  87. #define G2D_FMT_ARGB8888 1
  88. #define G2D_FMT_RGB565 2
  89. #define G2D_FMT_XRGB1555 3
  90. #define G2D_FMT_ARGB1555 4
  91. #define G2D_FMT_XRGB4444 5
  92. #define G2D_FMT_ARGB4444 6
  93. #define G2D_FMT_PACKED_RGB888 7
  94. #define G2D_FMT_A8 11
  95. #define G2D_FMT_L8 12
  96. /* buffer valid length */
  97. #define G2D_LEN_MIN 1
  98. #define G2D_LEN_MAX 8000
  99. #define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)
  100. #define G2D_CMDLIST_NUM 64
  101. #define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
  102. #define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
  103. /* maximum buffer pool size of userptr is 64MB as default */
  104. #define MAX_POOL (64 * 1024 * 1024)
  105. enum {
  106. BUF_TYPE_GEM = 1,
  107. BUF_TYPE_USERPTR,
  108. };
  109. enum g2d_reg_type {
  110. REG_TYPE_NONE = -1,
  111. REG_TYPE_SRC,
  112. REG_TYPE_SRC_PLANE2,
  113. REG_TYPE_DST,
  114. REG_TYPE_DST_PLANE2,
  115. REG_TYPE_PAT,
  116. REG_TYPE_MSK,
  117. MAX_REG_TYPE_NR
  118. };
  119. enum g2d_flag_bits {
  120. /*
  121. * If set, suspends the runqueue worker after the currently
  122. * processed node is finished.
  123. */
  124. G2D_BIT_SUSPEND_RUNQUEUE,
  125. /*
  126. * If set, indicates that the engine is currently busy.
  127. */
  128. G2D_BIT_ENGINE_BUSY,
  129. };
  130. /* cmdlist data structure */
  131. struct g2d_cmdlist {
  132. u32 head;
  133. unsigned long data[G2D_CMDLIST_DATA_NUM];
  134. u32 last; /* last data offset */
  135. };
  136. /*
  137. * A structure of buffer description
  138. *
  139. * @format: color format
  140. * @stride: buffer stride/pitch in bytes
  141. * @left_x: the x coordinates of left top corner
  142. * @top_y: the y coordinates of left top corner
  143. * @right_x: the x coordinates of right bottom corner
  144. * @bottom_y: the y coordinates of right bottom corner
  145. *
  146. */
  147. struct g2d_buf_desc {
  148. unsigned int format;
  149. unsigned int stride;
  150. unsigned int left_x;
  151. unsigned int top_y;
  152. unsigned int right_x;
  153. unsigned int bottom_y;
  154. };
  155. /*
  156. * A structure of buffer information
  157. *
  158. * @map_nr: manages the number of mapped buffers
  159. * @reg_types: stores regitster type in the order of requested command
  160. * @handles: stores buffer handle in its reg_type position
  161. * @types: stores buffer type in its reg_type position
  162. * @descs: stores buffer description in its reg_type position
  163. *
  164. */
  165. struct g2d_buf_info {
  166. unsigned int map_nr;
  167. enum g2d_reg_type reg_types[MAX_REG_TYPE_NR];
  168. void *obj[MAX_REG_TYPE_NR];
  169. unsigned int types[MAX_REG_TYPE_NR];
  170. struct g2d_buf_desc descs[MAX_REG_TYPE_NR];
  171. };
  172. struct drm_exynos_pending_g2d_event {
  173. struct drm_pending_event base;
  174. struct drm_exynos_g2d_event event;
  175. };
  176. struct g2d_cmdlist_userptr {
  177. struct list_head list;
  178. dma_addr_t dma_addr;
  179. unsigned long userptr;
  180. unsigned long size;
  181. struct frame_vector *vec;
  182. struct sg_table *sgt;
  183. atomic_t refcount;
  184. bool in_pool;
  185. bool out_of_list;
  186. };
  187. struct g2d_cmdlist_node {
  188. struct list_head list;
  189. struct g2d_cmdlist *cmdlist;
  190. dma_addr_t dma_addr;
  191. struct g2d_buf_info buf_info;
  192. struct drm_exynos_pending_g2d_event *event;
  193. };
  194. struct g2d_runqueue_node {
  195. struct list_head list;
  196. struct list_head run_cmdlist;
  197. struct list_head event_list;
  198. struct drm_file *filp;
  199. pid_t pid;
  200. struct completion complete;
  201. int async;
  202. };
  203. struct g2d_data {
  204. struct device *dev;
  205. void *dma_priv;
  206. struct clk *gate_clk;
  207. void __iomem *regs;
  208. int irq;
  209. struct workqueue_struct *g2d_workq;
  210. struct work_struct runqueue_work;
  211. struct drm_device *drm_dev;
  212. unsigned long flags;
  213. /* cmdlist */
  214. struct g2d_cmdlist_node *cmdlist_node;
  215. struct list_head free_cmdlist;
  216. struct mutex cmdlist_mutex;
  217. dma_addr_t cmdlist_pool;
  218. void *cmdlist_pool_virt;
  219. unsigned long cmdlist_dma_attrs;
  220. /* runqueue*/
  221. struct g2d_runqueue_node *runqueue_node;
  222. struct list_head runqueue;
  223. struct mutex runqueue_mutex;
  224. struct kmem_cache *runqueue_slab;
  225. unsigned long current_pool;
  226. unsigned long max_pool;
  227. };
  228. static inline void g2d_hw_reset(struct g2d_data *g2d)
  229. {
  230. writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
  231. clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
  232. }
  233. static int g2d_init_cmdlist(struct g2d_data *g2d)
  234. {
  235. struct device *dev = g2d->dev;
  236. struct g2d_cmdlist_node *node;
  237. int nr;
  238. int ret;
  239. struct g2d_buf_info *buf_info;
  240. g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
  241. g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
  242. G2D_CMDLIST_POOL_SIZE,
  243. &g2d->cmdlist_pool, GFP_KERNEL,
  244. g2d->cmdlist_dma_attrs);
  245. if (!g2d->cmdlist_pool_virt) {
  246. dev_err(dev, "failed to allocate dma memory\n");
  247. return -ENOMEM;
  248. }
  249. node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
  250. if (!node) {
  251. ret = -ENOMEM;
  252. goto err;
  253. }
  254. for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
  255. unsigned int i;
  256. node[nr].cmdlist =
  257. g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
  258. node[nr].dma_addr =
  259. g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
  260. buf_info = &node[nr].buf_info;
  261. for (i = 0; i < MAX_REG_TYPE_NR; i++)
  262. buf_info->reg_types[i] = REG_TYPE_NONE;
  263. list_add_tail(&node[nr].list, &g2d->free_cmdlist);
  264. }
  265. return 0;
  266. err:
  267. dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
  268. g2d->cmdlist_pool_virt,
  269. g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
  270. return ret;
  271. }
  272. static void g2d_fini_cmdlist(struct g2d_data *g2d)
  273. {
  274. kfree(g2d->cmdlist_node);
  275. if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
  276. dma_free_attrs(to_dma_dev(g2d->drm_dev),
  277. G2D_CMDLIST_POOL_SIZE,
  278. g2d->cmdlist_pool_virt,
  279. g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
  280. }
  281. }
  282. static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
  283. {
  284. struct device *dev = g2d->dev;
  285. struct g2d_cmdlist_node *node;
  286. mutex_lock(&g2d->cmdlist_mutex);
  287. if (list_empty(&g2d->free_cmdlist)) {
  288. dev_err(dev, "there is no free cmdlist\n");
  289. mutex_unlock(&g2d->cmdlist_mutex);
  290. return NULL;
  291. }
  292. node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
  293. list);
  294. list_del_init(&node->list);
  295. mutex_unlock(&g2d->cmdlist_mutex);
  296. return node;
  297. }
  298. static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
  299. {
  300. mutex_lock(&g2d->cmdlist_mutex);
  301. list_move_tail(&node->list, &g2d->free_cmdlist);
  302. mutex_unlock(&g2d->cmdlist_mutex);
  303. }
  304. static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
  305. struct g2d_cmdlist_node *node)
  306. {
  307. struct g2d_cmdlist_node *lnode;
  308. if (list_empty(&file_priv->inuse_cmdlist))
  309. goto add_to_list;
  310. /* this links to base address of new cmdlist */
  311. lnode = list_entry(file_priv->inuse_cmdlist.prev,
  312. struct g2d_cmdlist_node, list);
  313. lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
  314. add_to_list:
  315. list_add_tail(&node->list, &file_priv->inuse_cmdlist);
  316. if (node->event)
  317. list_add_tail(&node->event->base.link, &file_priv->event_list);
  318. }
  319. static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
  320. void *obj,
  321. bool force)
  322. {
  323. struct g2d_cmdlist_userptr *g2d_userptr = obj;
  324. struct page **pages;
  325. if (!obj)
  326. return;
  327. if (force)
  328. goto out;
  329. atomic_dec(&g2d_userptr->refcount);
  330. if (atomic_read(&g2d_userptr->refcount) > 0)
  331. return;
  332. if (g2d_userptr->in_pool)
  333. return;
  334. out:
  335. dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt,
  336. DMA_BIDIRECTIONAL, 0);
  337. pages = frame_vector_pages(g2d_userptr->vec);
  338. if (!IS_ERR(pages)) {
  339. int i;
  340. for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++)
  341. set_page_dirty_lock(pages[i]);
  342. }
  343. put_vaddr_frames(g2d_userptr->vec);
  344. frame_vector_destroy(g2d_userptr->vec);
  345. if (!g2d_userptr->out_of_list)
  346. list_del_init(&g2d_userptr->list);
  347. sg_free_table(g2d_userptr->sgt);
  348. kfree(g2d_userptr->sgt);
  349. kfree(g2d_userptr);
  350. }
  351. static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
  352. unsigned long userptr,
  353. unsigned long size,
  354. struct drm_file *filp,
  355. void **obj)
  356. {
  357. struct drm_exynos_file_private *file_priv = filp->driver_priv;
  358. struct g2d_cmdlist_userptr *g2d_userptr;
  359. struct sg_table *sgt;
  360. unsigned long start, end;
  361. unsigned int npages, offset;
  362. int ret;
  363. if (!size) {
  364. DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
  365. return ERR_PTR(-EINVAL);
  366. }
  367. /* check if userptr already exists in userptr_list. */
  368. list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
  369. if (g2d_userptr->userptr == userptr) {
  370. /*
  371. * also check size because there could be same address
  372. * and different size.
  373. */
  374. if (g2d_userptr->size == size) {
  375. atomic_inc(&g2d_userptr->refcount);
  376. *obj = g2d_userptr;
  377. return &g2d_userptr->dma_addr;
  378. }
  379. /*
  380. * at this moment, maybe g2d dma is accessing this
  381. * g2d_userptr memory region so just remove this
  382. * g2d_userptr object from userptr_list not to be
  383. * referred again and also except it the userptr
  384. * pool to be released after the dma access completion.
  385. */
  386. g2d_userptr->out_of_list = true;
  387. g2d_userptr->in_pool = false;
  388. list_del_init(&g2d_userptr->list);
  389. break;
  390. }
  391. }
  392. g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
  393. if (!g2d_userptr)
  394. return ERR_PTR(-ENOMEM);
  395. atomic_set(&g2d_userptr->refcount, 1);
  396. g2d_userptr->size = size;
  397. start = userptr & PAGE_MASK;
  398. offset = userptr & ~PAGE_MASK;
  399. end = PAGE_ALIGN(userptr + size);
  400. npages = (end - start) >> PAGE_SHIFT;
  401. g2d_userptr->vec = frame_vector_create(npages);
  402. if (!g2d_userptr->vec) {
  403. ret = -ENOMEM;
  404. goto err_free;
  405. }
  406. ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
  407. g2d_userptr->vec);
  408. if (ret != npages) {
  409. DRM_DEV_ERROR(g2d->dev,
  410. "failed to get user pages from userptr.\n");
  411. if (ret < 0)
  412. goto err_destroy_framevec;
  413. ret = -EFAULT;
  414. goto err_put_framevec;
  415. }
  416. if (frame_vector_to_pages(g2d_userptr->vec) < 0) {
  417. ret = -EFAULT;
  418. goto err_put_framevec;
  419. }
  420. sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
  421. if (!sgt) {
  422. ret = -ENOMEM;
  423. goto err_put_framevec;
  424. }
  425. ret = sg_alloc_table_from_pages(sgt,
  426. frame_vector_pages(g2d_userptr->vec),
  427. npages, offset, size, GFP_KERNEL);
  428. if (ret < 0) {
  429. DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
  430. goto err_free_sgt;
  431. }
  432. g2d_userptr->sgt = sgt;
  433. ret = dma_map_sgtable(to_dma_dev(g2d->drm_dev), sgt,
  434. DMA_BIDIRECTIONAL, 0);
  435. if (ret) {
  436. DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
  437. goto err_sg_free_table;
  438. }
  439. g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
  440. g2d_userptr->userptr = userptr;
  441. list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
  442. if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
  443. g2d->current_pool += npages << PAGE_SHIFT;
  444. g2d_userptr->in_pool = true;
  445. }
  446. *obj = g2d_userptr;
  447. return &g2d_userptr->dma_addr;
  448. err_sg_free_table:
  449. sg_free_table(sgt);
  450. err_free_sgt:
  451. kfree(sgt);
  452. err_put_framevec:
  453. put_vaddr_frames(g2d_userptr->vec);
  454. err_destroy_framevec:
  455. frame_vector_destroy(g2d_userptr->vec);
  456. err_free:
  457. kfree(g2d_userptr);
  458. return ERR_PTR(ret);
  459. }
  460. static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
  461. {
  462. struct drm_exynos_file_private *file_priv = filp->driver_priv;
  463. struct g2d_cmdlist_userptr *g2d_userptr, *n;
  464. list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
  465. if (g2d_userptr->in_pool)
  466. g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
  467. g2d->current_pool = 0;
  468. }
  469. static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
  470. {
  471. enum g2d_reg_type reg_type;
  472. switch (reg_offset) {
  473. case G2D_SRC_BASE_ADDR:
  474. case G2D_SRC_STRIDE:
  475. case G2D_SRC_COLOR_MODE:
  476. case G2D_SRC_LEFT_TOP:
  477. case G2D_SRC_RIGHT_BOTTOM:
  478. reg_type = REG_TYPE_SRC;
  479. break;
  480. case G2D_SRC_PLANE2_BASE_ADDR:
  481. reg_type = REG_TYPE_SRC_PLANE2;
  482. break;
  483. case G2D_DST_BASE_ADDR:
  484. case G2D_DST_STRIDE:
  485. case G2D_DST_COLOR_MODE:
  486. case G2D_DST_LEFT_TOP:
  487. case G2D_DST_RIGHT_BOTTOM:
  488. reg_type = REG_TYPE_DST;
  489. break;
  490. case G2D_DST_PLANE2_BASE_ADDR:
  491. reg_type = REG_TYPE_DST_PLANE2;
  492. break;
  493. case G2D_PAT_BASE_ADDR:
  494. reg_type = REG_TYPE_PAT;
  495. break;
  496. case G2D_MSK_BASE_ADDR:
  497. reg_type = REG_TYPE_MSK;
  498. break;
  499. default:
  500. reg_type = REG_TYPE_NONE;
  501. DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
  502. reg_offset);
  503. break;
  504. }
  505. return reg_type;
  506. }
  507. static unsigned long g2d_get_buf_bpp(unsigned int format)
  508. {
  509. unsigned long bpp;
  510. switch (format) {
  511. case G2D_FMT_XRGB8888:
  512. case G2D_FMT_ARGB8888:
  513. bpp = 4;
  514. break;
  515. case G2D_FMT_RGB565:
  516. case G2D_FMT_XRGB1555:
  517. case G2D_FMT_ARGB1555:
  518. case G2D_FMT_XRGB4444:
  519. case G2D_FMT_ARGB4444:
  520. bpp = 2;
  521. break;
  522. case G2D_FMT_PACKED_RGB888:
  523. bpp = 3;
  524. break;
  525. default:
  526. bpp = 1;
  527. break;
  528. }
  529. return bpp;
  530. }
  531. static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
  532. struct g2d_buf_desc *buf_desc,
  533. enum g2d_reg_type reg_type,
  534. unsigned long size)
  535. {
  536. int width, height;
  537. unsigned long bpp, last_pos;
  538. /*
  539. * check source and destination buffers only.
  540. * so the others are always valid.
  541. */
  542. if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
  543. return true;
  544. /* This check also makes sure that right_x > left_x. */
  545. width = (int)buf_desc->right_x - (int)buf_desc->left_x;
  546. if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
  547. DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
  548. return false;
  549. }
  550. /* This check also makes sure that bottom_y > top_y. */
  551. height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
  552. if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
  553. DRM_DEV_ERROR(g2d->dev,
  554. "height[%d] is out of range!\n", height);
  555. return false;
  556. }
  557. bpp = g2d_get_buf_bpp(buf_desc->format);
  558. /* Compute the position of the last byte that the engine accesses. */
  559. last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
  560. (unsigned long)buf_desc->stride +
  561. (unsigned long)buf_desc->right_x * bpp - 1;
  562. /*
  563. * Since right_x > left_x and bottom_y > top_y we already know
  564. * that the first_pos < last_pos (first_pos being the position
  565. * of the first byte the engine accesses), it just remains to
  566. * check if last_pos is smaller then the buffer size.
  567. */
  568. if (last_pos >= size) {
  569. DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
  570. "is out of range [%lu]!\n", last_pos, size);
  571. return false;
  572. }
  573. return true;
  574. }
  575. static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
  576. struct g2d_cmdlist_node *node,
  577. struct drm_device *drm_dev,
  578. struct drm_file *file)
  579. {
  580. struct g2d_cmdlist *cmdlist = node->cmdlist;
  581. struct g2d_buf_info *buf_info = &node->buf_info;
  582. int offset;
  583. int ret;
  584. int i;
  585. for (i = 0; i < buf_info->map_nr; i++) {
  586. struct g2d_buf_desc *buf_desc;
  587. enum g2d_reg_type reg_type;
  588. int reg_pos;
  589. unsigned long handle;
  590. dma_addr_t *addr;
  591. reg_pos = cmdlist->last - 2 * (i + 1);
  592. offset = cmdlist->data[reg_pos];
  593. handle = cmdlist->data[reg_pos + 1];
  594. reg_type = g2d_get_reg_type(g2d, offset);
  595. if (reg_type == REG_TYPE_NONE) {
  596. ret = -EFAULT;
  597. goto err;
  598. }
  599. buf_desc = &buf_info->descs[reg_type];
  600. if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
  601. struct exynos_drm_gem *exynos_gem;
  602. exynos_gem = exynos_drm_gem_get(file, handle);
  603. if (!exynos_gem) {
  604. ret = -EFAULT;
  605. goto err;
  606. }
  607. if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
  608. reg_type, exynos_gem->size)) {
  609. exynos_drm_gem_put(exynos_gem);
  610. ret = -EFAULT;
  611. goto err;
  612. }
  613. addr = &exynos_gem->dma_addr;
  614. buf_info->obj[reg_type] = exynos_gem;
  615. } else {
  616. struct drm_exynos_g2d_userptr g2d_userptr;
  617. if (copy_from_user(&g2d_userptr, (void __user *)handle,
  618. sizeof(struct drm_exynos_g2d_userptr))) {
  619. ret = -EFAULT;
  620. goto err;
  621. }
  622. if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
  623. reg_type,
  624. g2d_userptr.size)) {
  625. ret = -EFAULT;
  626. goto err;
  627. }
  628. addr = g2d_userptr_get_dma_addr(g2d,
  629. g2d_userptr.userptr,
  630. g2d_userptr.size,
  631. file,
  632. &buf_info->obj[reg_type]);
  633. if (IS_ERR(addr)) {
  634. ret = -EFAULT;
  635. goto err;
  636. }
  637. }
  638. cmdlist->data[reg_pos + 1] = *addr;
  639. buf_info->reg_types[i] = reg_type;
  640. }
  641. return 0;
  642. err:
  643. buf_info->map_nr = i;
  644. return ret;
  645. }
  646. static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
  647. struct g2d_cmdlist_node *node,
  648. struct drm_file *filp)
  649. {
  650. struct g2d_buf_info *buf_info = &node->buf_info;
  651. int i;
  652. for (i = 0; i < buf_info->map_nr; i++) {
  653. struct g2d_buf_desc *buf_desc;
  654. enum g2d_reg_type reg_type;
  655. void *obj;
  656. reg_type = buf_info->reg_types[i];
  657. buf_desc = &buf_info->descs[reg_type];
  658. obj = buf_info->obj[reg_type];
  659. if (buf_info->types[reg_type] == BUF_TYPE_GEM)
  660. exynos_drm_gem_put(obj);
  661. else
  662. g2d_userptr_put_dma_addr(g2d, obj, false);
  663. buf_info->reg_types[i] = REG_TYPE_NONE;
  664. buf_info->obj[reg_type] = NULL;
  665. buf_info->types[reg_type] = 0;
  666. memset(buf_desc, 0x00, sizeof(*buf_desc));
  667. }
  668. buf_info->map_nr = 0;
  669. }
  670. static void g2d_dma_start(struct g2d_data *g2d,
  671. struct g2d_runqueue_node *runqueue_node)
  672. {
  673. struct g2d_cmdlist_node *node =
  674. list_first_entry(&runqueue_node->run_cmdlist,
  675. struct g2d_cmdlist_node, list);
  676. set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
  677. writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
  678. writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
  679. }
  680. static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
  681. {
  682. struct g2d_runqueue_node *runqueue_node;
  683. if (list_empty(&g2d->runqueue))
  684. return NULL;
  685. runqueue_node = list_first_entry(&g2d->runqueue,
  686. struct g2d_runqueue_node, list);
  687. list_del_init(&runqueue_node->list);
  688. return runqueue_node;
  689. }
  690. static void g2d_free_runqueue_node(struct g2d_data *g2d,
  691. struct g2d_runqueue_node *runqueue_node)
  692. {
  693. struct g2d_cmdlist_node *node;
  694. mutex_lock(&g2d->cmdlist_mutex);
  695. /*
  696. * commands in run_cmdlist have been completed so unmap all gem
  697. * objects in each command node so that they are unreferenced.
  698. */
  699. list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
  700. g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
  701. list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
  702. mutex_unlock(&g2d->cmdlist_mutex);
  703. kmem_cache_free(g2d->runqueue_slab, runqueue_node);
  704. }
  705. /**
  706. * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
  707. * @g2d: G2D state object
  708. * @file: if not zero, only remove items with this DRM file
  709. *
  710. * Has to be called under runqueue lock.
  711. */
  712. static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
  713. {
  714. struct g2d_runqueue_node *node, *n;
  715. if (list_empty(&g2d->runqueue))
  716. return;
  717. list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
  718. if (file && node->filp != file)
  719. continue;
  720. list_del_init(&node->list);
  721. g2d_free_runqueue_node(g2d, node);
  722. }
  723. }
  724. static void g2d_runqueue_worker(struct work_struct *work)
  725. {
  726. struct g2d_data *g2d = container_of(work, struct g2d_data,
  727. runqueue_work);
  728. struct g2d_runqueue_node *runqueue_node;
  729. /*
  730. * The engine is busy and the completion of the current node is going
  731. * to poke the runqueue worker, so nothing to do here.
  732. */
  733. if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
  734. return;
  735. mutex_lock(&g2d->runqueue_mutex);
  736. runqueue_node = g2d->runqueue_node;
  737. g2d->runqueue_node = NULL;
  738. if (runqueue_node) {
  739. pm_runtime_mark_last_busy(g2d->dev);
  740. pm_runtime_put_autosuspend(g2d->dev);
  741. complete(&runqueue_node->complete);
  742. if (runqueue_node->async)
  743. g2d_free_runqueue_node(g2d, runqueue_node);
  744. }
  745. if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
  746. g2d->runqueue_node = g2d_get_runqueue_node(g2d);
  747. if (g2d->runqueue_node) {
  748. pm_runtime_get_sync(g2d->dev);
  749. g2d_dma_start(g2d, g2d->runqueue_node);
  750. }
  751. }
  752. mutex_unlock(&g2d->runqueue_mutex);
  753. }
  754. static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
  755. {
  756. struct drm_device *drm_dev = g2d->drm_dev;
  757. struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
  758. struct drm_exynos_pending_g2d_event *e;
  759. struct timespec64 now;
  760. if (list_empty(&runqueue_node->event_list))
  761. return;
  762. e = list_first_entry(&runqueue_node->event_list,
  763. struct drm_exynos_pending_g2d_event, base.link);
  764. ktime_get_ts64(&now);
  765. e->event.tv_sec = now.tv_sec;
  766. e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
  767. e->event.cmdlist_no = cmdlist_no;
  768. drm_send_event(drm_dev, &e->base);
  769. }
  770. static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
  771. {
  772. struct g2d_data *g2d = dev_id;
  773. u32 pending;
  774. pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
  775. if (pending)
  776. writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
  777. if (pending & G2D_INTP_GCMD_FIN) {
  778. u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
  779. cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
  780. G2D_DMA_LIST_DONE_COUNT_OFFSET;
  781. g2d_finish_event(g2d, cmdlist_no);
  782. writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
  783. if (!(pending & G2D_INTP_ACMD_FIN)) {
  784. writel_relaxed(G2D_DMA_CONTINUE,
  785. g2d->regs + G2D_DMA_COMMAND);
  786. }
  787. }
  788. if (pending & G2D_INTP_ACMD_FIN) {
  789. clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
  790. queue_work(g2d->g2d_workq, &g2d->runqueue_work);
  791. }
  792. return IRQ_HANDLED;
  793. }
  794. /**
  795. * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
  796. * @g2d: G2D state object
  797. * @file: if not zero, only wait if the current runqueue node belongs
  798. * to the DRM file
  799. *
  800. * Should the engine not become idle after a 100ms timeout, a hardware
  801. * reset is issued.
  802. */
  803. static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
  804. {
  805. struct device *dev = g2d->dev;
  806. struct g2d_runqueue_node *runqueue_node = NULL;
  807. unsigned int tries = 10;
  808. mutex_lock(&g2d->runqueue_mutex);
  809. /* If no node is currently processed, we have nothing to do. */
  810. if (!g2d->runqueue_node)
  811. goto out;
  812. runqueue_node = g2d->runqueue_node;
  813. /* Check if the currently processed item belongs to us. */
  814. if (file && runqueue_node->filp != file)
  815. goto out;
  816. mutex_unlock(&g2d->runqueue_mutex);
  817. /* Wait for the G2D engine to finish. */
  818. while (tries-- && (g2d->runqueue_node == runqueue_node))
  819. mdelay(10);
  820. mutex_lock(&g2d->runqueue_mutex);
  821. if (g2d->runqueue_node != runqueue_node)
  822. goto out;
  823. dev_err(dev, "wait timed out, resetting engine...\n");
  824. g2d_hw_reset(g2d);
  825. /*
  826. * After the hardware reset of the engine we are going to loose
  827. * the IRQ which triggers the PM runtime put().
  828. * So do this manually here.
  829. */
  830. pm_runtime_mark_last_busy(dev);
  831. pm_runtime_put_autosuspend(dev);
  832. complete(&runqueue_node->complete);
  833. if (runqueue_node->async)
  834. g2d_free_runqueue_node(g2d, runqueue_node);
  835. out:
  836. mutex_unlock(&g2d->runqueue_mutex);
  837. }
  838. static int g2d_check_reg_offset(struct g2d_data *g2d,
  839. struct g2d_cmdlist_node *node,
  840. int nr, bool for_addr)
  841. {
  842. struct g2d_cmdlist *cmdlist = node->cmdlist;
  843. int reg_offset;
  844. int index;
  845. int i;
  846. for (i = 0; i < nr; i++) {
  847. struct g2d_buf_info *buf_info = &node->buf_info;
  848. struct g2d_buf_desc *buf_desc;
  849. enum g2d_reg_type reg_type;
  850. unsigned long value;
  851. index = cmdlist->last - 2 * (i + 1);
  852. reg_offset = cmdlist->data[index] & ~0xfffff000;
  853. if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
  854. goto err;
  855. if (reg_offset % 4)
  856. goto err;
  857. switch (reg_offset) {
  858. case G2D_SRC_BASE_ADDR:
  859. case G2D_SRC_PLANE2_BASE_ADDR:
  860. case G2D_DST_BASE_ADDR:
  861. case G2D_DST_PLANE2_BASE_ADDR:
  862. case G2D_PAT_BASE_ADDR:
  863. case G2D_MSK_BASE_ADDR:
  864. if (!for_addr)
  865. goto err;
  866. reg_type = g2d_get_reg_type(g2d, reg_offset);
  867. /* check userptr buffer type. */
  868. if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
  869. buf_info->types[reg_type] = BUF_TYPE_USERPTR;
  870. cmdlist->data[index] &= ~G2D_BUF_USERPTR;
  871. } else
  872. buf_info->types[reg_type] = BUF_TYPE_GEM;
  873. break;
  874. case G2D_SRC_STRIDE:
  875. case G2D_DST_STRIDE:
  876. if (for_addr)
  877. goto err;
  878. reg_type = g2d_get_reg_type(g2d, reg_offset);
  879. buf_desc = &buf_info->descs[reg_type];
  880. buf_desc->stride = cmdlist->data[index + 1];
  881. break;
  882. case G2D_SRC_COLOR_MODE:
  883. case G2D_DST_COLOR_MODE:
  884. if (for_addr)
  885. goto err;
  886. reg_type = g2d_get_reg_type(g2d, reg_offset);
  887. buf_desc = &buf_info->descs[reg_type];
  888. value = cmdlist->data[index + 1];
  889. buf_desc->format = value & 0xf;
  890. break;
  891. case G2D_SRC_LEFT_TOP:
  892. case G2D_DST_LEFT_TOP:
  893. if (for_addr)
  894. goto err;
  895. reg_type = g2d_get_reg_type(g2d, reg_offset);
  896. buf_desc = &buf_info->descs[reg_type];
  897. value = cmdlist->data[index + 1];
  898. buf_desc->left_x = value & 0x1fff;
  899. buf_desc->top_y = (value & 0x1fff0000) >> 16;
  900. break;
  901. case G2D_SRC_RIGHT_BOTTOM:
  902. case G2D_DST_RIGHT_BOTTOM:
  903. if (for_addr)
  904. goto err;
  905. reg_type = g2d_get_reg_type(g2d, reg_offset);
  906. buf_desc = &buf_info->descs[reg_type];
  907. value = cmdlist->data[index + 1];
  908. buf_desc->right_x = value & 0x1fff;
  909. buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
  910. break;
  911. default:
  912. if (for_addr)
  913. goto err;
  914. break;
  915. }
  916. }
  917. return 0;
  918. err:
  919. dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
  920. return -EINVAL;
  921. }
  922. /* ioctl functions */
  923. int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
  924. struct drm_file *file)
  925. {
  926. struct drm_exynos_g2d_get_ver *ver = data;
  927. ver->major = G2D_HW_MAJOR_VER;
  928. ver->minor = G2D_HW_MINOR_VER;
  929. return 0;
  930. }
  931. int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
  932. struct drm_file *file)
  933. {
  934. struct drm_exynos_file_private *file_priv = file->driver_priv;
  935. struct exynos_drm_private *priv = drm_dev->dev_private;
  936. struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
  937. struct drm_exynos_g2d_set_cmdlist *req = data;
  938. struct drm_exynos_g2d_cmd *cmd;
  939. struct drm_exynos_pending_g2d_event *e;
  940. struct g2d_cmdlist_node *node;
  941. struct g2d_cmdlist *cmdlist;
  942. int size;
  943. int ret;
  944. node = g2d_get_cmdlist(g2d);
  945. if (!node)
  946. return -ENOMEM;
  947. /*
  948. * To avoid an integer overflow for the later size computations, we
  949. * enforce a maximum number of submitted commands here. This limit is
  950. * sufficient for all conceivable usage cases of the G2D.
  951. */
  952. if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
  953. req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
  954. dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
  955. return -EINVAL;
  956. }
  957. node->event = NULL;
  958. if (req->event_type != G2D_EVENT_NOT) {
  959. e = kzalloc(sizeof(*node->event), GFP_KERNEL);
  960. if (!e) {
  961. ret = -ENOMEM;
  962. goto err;
  963. }
  964. e->event.base.type = DRM_EXYNOS_G2D_EVENT;
  965. e->event.base.length = sizeof(e->event);
  966. e->event.user_data = req->user_data;
  967. ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);
  968. if (ret) {
  969. kfree(e);
  970. goto err;
  971. }
  972. node->event = e;
  973. }
  974. cmdlist = node->cmdlist;
  975. cmdlist->last = 0;
  976. /*
  977. * If don't clear SFR registers, the cmdlist is affected by register
  978. * values of previous cmdlist. G2D hw executes SFR clear command and
  979. * a next command at the same time then the next command is ignored and
  980. * is executed rightly from next next command, so needs a dummy command
  981. * to next command of SFR clear command.
  982. */
  983. cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
  984. cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
  985. cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
  986. cmdlist->data[cmdlist->last++] = 0;
  987. /*
  988. * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
  989. * and GCF bit should be set to INTEN register if user wants
  990. * G2D interrupt event once current command list execution is
  991. * finished.
  992. * Otherwise only ACF bit should be set to INTEN register so
  993. * that one interrupt is occurred after all command lists
  994. * have been completed.
  995. */
  996. if (node->event) {
  997. cmdlist->data[cmdlist->last++] = G2D_INTEN;
  998. cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
  999. cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
  1000. cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
  1001. } else {
  1002. cmdlist->data[cmdlist->last++] = G2D_INTEN;
  1003. cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
  1004. }
  1005. /*
  1006. * Check the size of cmdlist. The 2 that is added last comes from
  1007. * the implicit G2D_BITBLT_START that is appended once we have
  1008. * checked all the submitted commands.
  1009. */
  1010. size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
  1011. if (size > G2D_CMDLIST_DATA_NUM) {
  1012. dev_err(g2d->dev, "cmdlist size is too big\n");
  1013. ret = -EINVAL;
  1014. goto err_free_event;
  1015. }
  1016. cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
  1017. if (copy_from_user(cmdlist->data + cmdlist->last,
  1018. (void __user *)cmd,
  1019. sizeof(*cmd) * req->cmd_nr)) {
  1020. ret = -EFAULT;
  1021. goto err_free_event;
  1022. }
  1023. cmdlist->last += req->cmd_nr * 2;
  1024. ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
  1025. if (ret < 0)
  1026. goto err_free_event;
  1027. node->buf_info.map_nr = req->cmd_buf_nr;
  1028. if (req->cmd_buf_nr) {
  1029. struct drm_exynos_g2d_cmd *cmd_buf;
  1030. cmd_buf = (struct drm_exynos_g2d_cmd *)
  1031. (unsigned long)req->cmd_buf;
  1032. if (copy_from_user(cmdlist->data + cmdlist->last,
  1033. (void __user *)cmd_buf,
  1034. sizeof(*cmd_buf) * req->cmd_buf_nr)) {
  1035. ret = -EFAULT;
  1036. goto err_free_event;
  1037. }
  1038. cmdlist->last += req->cmd_buf_nr * 2;
  1039. ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
  1040. if (ret < 0)
  1041. goto err_free_event;
  1042. ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
  1043. if (ret < 0)
  1044. goto err_unmap;
  1045. }
  1046. cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
  1047. cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
  1048. /* head */
  1049. cmdlist->head = cmdlist->last / 2;
  1050. /* tail */
  1051. cmdlist->data[cmdlist->last] = 0;
  1052. g2d_add_cmdlist_to_inuse(file_priv, node);
  1053. return 0;
  1054. err_unmap:
  1055. g2d_unmap_cmdlist_gem(g2d, node, file);
  1056. err_free_event:
  1057. if (node->event)
  1058. drm_event_cancel_free(drm_dev, &node->event->base);
  1059. err:
  1060. g2d_put_cmdlist(g2d, node);
  1061. return ret;
  1062. }
  1063. int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
  1064. struct drm_file *file)
  1065. {
  1066. struct drm_exynos_file_private *file_priv = file->driver_priv;
  1067. struct exynos_drm_private *priv = drm_dev->dev_private;
  1068. struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
  1069. struct drm_exynos_g2d_exec *req = data;
  1070. struct g2d_runqueue_node *runqueue_node;
  1071. struct list_head *run_cmdlist;
  1072. struct list_head *event_list;
  1073. runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
  1074. if (!runqueue_node)
  1075. return -ENOMEM;
  1076. run_cmdlist = &runqueue_node->run_cmdlist;
  1077. event_list = &runqueue_node->event_list;
  1078. INIT_LIST_HEAD(run_cmdlist);
  1079. INIT_LIST_HEAD(event_list);
  1080. init_completion(&runqueue_node->complete);
  1081. runqueue_node->async = req->async;
  1082. list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
  1083. list_splice_init(&file_priv->event_list, event_list);
  1084. if (list_empty(run_cmdlist)) {
  1085. dev_err(g2d->dev, "there is no inuse cmdlist\n");
  1086. kmem_cache_free(g2d->runqueue_slab, runqueue_node);
  1087. return -EPERM;
  1088. }
  1089. mutex_lock(&g2d->runqueue_mutex);
  1090. runqueue_node->pid = current->pid;
  1091. runqueue_node->filp = file;
  1092. list_add_tail(&runqueue_node->list, &g2d->runqueue);
  1093. mutex_unlock(&g2d->runqueue_mutex);
  1094. /* Let the runqueue know that there is work to do. */
  1095. queue_work(g2d->g2d_workq, &g2d->runqueue_work);
  1096. if (runqueue_node->async)
  1097. goto out;
  1098. wait_for_completion(&runqueue_node->complete);
  1099. g2d_free_runqueue_node(g2d, runqueue_node);
  1100. out:
  1101. return 0;
  1102. }
  1103. int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
  1104. {
  1105. struct drm_exynos_file_private *file_priv = file->driver_priv;
  1106. INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
  1107. INIT_LIST_HEAD(&file_priv->event_list);
  1108. INIT_LIST_HEAD(&file_priv->userptr_list);
  1109. return 0;
  1110. }
  1111. void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
  1112. {
  1113. struct drm_exynos_file_private *file_priv = file->driver_priv;
  1114. struct exynos_drm_private *priv = drm_dev->dev_private;
  1115. struct g2d_data *g2d;
  1116. struct g2d_cmdlist_node *node, *n;
  1117. if (!priv->g2d_dev)
  1118. return;
  1119. g2d = dev_get_drvdata(priv->g2d_dev);
  1120. /* Remove the runqueue nodes that belong to us. */
  1121. mutex_lock(&g2d->runqueue_mutex);
  1122. g2d_remove_runqueue_nodes(g2d, file);
  1123. mutex_unlock(&g2d->runqueue_mutex);
  1124. /*
  1125. * Wait for the runqueue worker to finish its current node.
  1126. * After this the engine should no longer be accessing any
  1127. * memory belonging to us.
  1128. */
  1129. g2d_wait_finish(g2d, file);
  1130. /*
  1131. * Even after the engine is idle, there might still be stale cmdlists
  1132. * (i.e. cmdlisst which we submitted but never executed) around, with
  1133. * their corresponding GEM/userptr buffers.
  1134. * Properly unmap these buffers here.
  1135. */
  1136. mutex_lock(&g2d->cmdlist_mutex);
  1137. list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
  1138. g2d_unmap_cmdlist_gem(g2d, node, file);
  1139. list_move_tail(&node->list, &g2d->free_cmdlist);
  1140. }
  1141. mutex_unlock(&g2d->cmdlist_mutex);
  1142. /* release all g2d_userptr in pool. */
  1143. g2d_userptr_free_all(g2d, file);
  1144. }
  1145. static int g2d_bind(struct device *dev, struct device *master, void *data)
  1146. {
  1147. struct g2d_data *g2d = dev_get_drvdata(dev);
  1148. struct drm_device *drm_dev = data;
  1149. struct exynos_drm_private *priv = drm_dev->dev_private;
  1150. int ret;
  1151. g2d->drm_dev = drm_dev;
  1152. /* allocate dma-aware cmdlist buffer. */
  1153. ret = g2d_init_cmdlist(g2d);
  1154. if (ret < 0) {
  1155. dev_err(dev, "cmdlist init failed\n");
  1156. return ret;
  1157. }
  1158. ret = exynos_drm_register_dma(drm_dev, dev, &g2d->dma_priv);
  1159. if (ret < 0) {
  1160. dev_err(dev, "failed to enable iommu.\n");
  1161. g2d_fini_cmdlist(g2d);
  1162. return ret;
  1163. }
  1164. priv->g2d_dev = dev;
  1165. dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
  1166. G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
  1167. return 0;
  1168. }
  1169. static void g2d_unbind(struct device *dev, struct device *master, void *data)
  1170. {
  1171. struct g2d_data *g2d = dev_get_drvdata(dev);
  1172. struct drm_device *drm_dev = data;
  1173. struct exynos_drm_private *priv = drm_dev->dev_private;
  1174. /* Suspend operation and wait for engine idle. */
  1175. set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
  1176. g2d_wait_finish(g2d, NULL);
  1177. priv->g2d_dev = NULL;
  1178. cancel_work_sync(&g2d->runqueue_work);
  1179. exynos_drm_unregister_dma(g2d->drm_dev, dev, &g2d->dma_priv);
  1180. }
  1181. static const struct component_ops g2d_component_ops = {
  1182. .bind = g2d_bind,
  1183. .unbind = g2d_unbind,
  1184. };
  1185. static int g2d_probe(struct platform_device *pdev)
  1186. {
  1187. struct device *dev = &pdev->dev;
  1188. struct resource *res;
  1189. struct g2d_data *g2d;
  1190. int ret;
  1191. g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
  1192. if (!g2d)
  1193. return -ENOMEM;
  1194. g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
  1195. sizeof(struct g2d_runqueue_node), 0, 0, NULL);
  1196. if (!g2d->runqueue_slab)
  1197. return -ENOMEM;
  1198. g2d->dev = dev;
  1199. g2d->g2d_workq = create_singlethread_workqueue("g2d");
  1200. if (!g2d->g2d_workq) {
  1201. dev_err(dev, "failed to create workqueue\n");
  1202. ret = -EINVAL;
  1203. goto err_destroy_slab;
  1204. }
  1205. INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
  1206. INIT_LIST_HEAD(&g2d->free_cmdlist);
  1207. INIT_LIST_HEAD(&g2d->runqueue);
  1208. mutex_init(&g2d->cmdlist_mutex);
  1209. mutex_init(&g2d->runqueue_mutex);
  1210. g2d->gate_clk = devm_clk_get(dev, "fimg2d");
  1211. if (IS_ERR(g2d->gate_clk)) {
  1212. dev_err(dev, "failed to get gate clock\n");
  1213. ret = PTR_ERR(g2d->gate_clk);
  1214. goto err_destroy_workqueue;
  1215. }
  1216. pm_runtime_use_autosuspend(dev);
  1217. pm_runtime_set_autosuspend_delay(dev, 2000);
  1218. pm_runtime_enable(dev);
  1219. clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
  1220. clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
  1221. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1222. g2d->regs = devm_ioremap_resource(dev, res);
  1223. if (IS_ERR(g2d->regs)) {
  1224. ret = PTR_ERR(g2d->regs);
  1225. goto err_put_clk;
  1226. }
  1227. g2d->irq = platform_get_irq(pdev, 0);
  1228. if (g2d->irq < 0) {
  1229. ret = g2d->irq;
  1230. goto err_put_clk;
  1231. }
  1232. ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,
  1233. "drm_g2d", g2d);
  1234. if (ret < 0) {
  1235. dev_err(dev, "irq request failed\n");
  1236. goto err_put_clk;
  1237. }
  1238. g2d->max_pool = MAX_POOL;
  1239. platform_set_drvdata(pdev, g2d);
  1240. ret = component_add(dev, &g2d_component_ops);
  1241. if (ret < 0) {
  1242. dev_err(dev, "failed to register drm g2d device\n");
  1243. goto err_put_clk;
  1244. }
  1245. return 0;
  1246. err_put_clk:
  1247. pm_runtime_disable(dev);
  1248. err_destroy_workqueue:
  1249. destroy_workqueue(g2d->g2d_workq);
  1250. err_destroy_slab:
  1251. kmem_cache_destroy(g2d->runqueue_slab);
  1252. return ret;
  1253. }
  1254. static int g2d_remove(struct platform_device *pdev)
  1255. {
  1256. struct g2d_data *g2d = platform_get_drvdata(pdev);
  1257. component_del(&pdev->dev, &g2d_component_ops);
  1258. /* There should be no locking needed here. */
  1259. g2d_remove_runqueue_nodes(g2d, NULL);
  1260. pm_runtime_dont_use_autosuspend(&pdev->dev);
  1261. pm_runtime_disable(&pdev->dev);
  1262. g2d_fini_cmdlist(g2d);
  1263. destroy_workqueue(g2d->g2d_workq);
  1264. kmem_cache_destroy(g2d->runqueue_slab);
  1265. return 0;
  1266. }
  1267. #ifdef CONFIG_PM_SLEEP
  1268. static int g2d_suspend(struct device *dev)
  1269. {
  1270. struct g2d_data *g2d = dev_get_drvdata(dev);
  1271. /*
  1272. * Suspend the runqueue worker operation and wait until the G2D
  1273. * engine is idle.
  1274. */
  1275. set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
  1276. g2d_wait_finish(g2d, NULL);
  1277. flush_work(&g2d->runqueue_work);
  1278. return 0;
  1279. }
  1280. static int g2d_resume(struct device *dev)
  1281. {
  1282. struct g2d_data *g2d = dev_get_drvdata(dev);
  1283. clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
  1284. queue_work(g2d->g2d_workq, &g2d->runqueue_work);
  1285. return 0;
  1286. }
  1287. #endif
  1288. #ifdef CONFIG_PM
  1289. static int g2d_runtime_suspend(struct device *dev)
  1290. {
  1291. struct g2d_data *g2d = dev_get_drvdata(dev);
  1292. clk_disable_unprepare(g2d->gate_clk);
  1293. return 0;
  1294. }
  1295. static int g2d_runtime_resume(struct device *dev)
  1296. {
  1297. struct g2d_data *g2d = dev_get_drvdata(dev);
  1298. int ret;
  1299. ret = clk_prepare_enable(g2d->gate_clk);
  1300. if (ret < 0)
  1301. dev_warn(dev, "failed to enable clock.\n");
  1302. return ret;
  1303. }
  1304. #endif
  1305. static const struct dev_pm_ops g2d_pm_ops = {
  1306. SET_SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
  1307. SET_RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
  1308. };
  1309. static const struct of_device_id exynos_g2d_match[] = {
  1310. { .compatible = "samsung,exynos5250-g2d" },
  1311. { .compatible = "samsung,exynos4212-g2d" },
  1312. {},
  1313. };
  1314. MODULE_DEVICE_TABLE(of, exynos_g2d_match);
  1315. struct platform_driver g2d_driver = {
  1316. .probe = g2d_probe,
  1317. .remove = g2d_remove,
  1318. .driver = {
  1319. .name = "exynos-drm-g2d",
  1320. .owner = THIS_MODULE,
  1321. .pm = &g2d_pm_ops,
  1322. .of_match_table = exynos_g2d_match,
  1323. },
  1324. };