vmwgfx_blit.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508
  1. // SPDX-License-Identifier: GPL-2.0 OR MIT
  2. /**************************************************************************
  3. *
  4. * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
  5. * All Rights Reserved.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the
  9. * "Software"), to deal in the Software without restriction, including
  10. * without limitation the rights to use, copy, modify, merge, publish,
  11. * distribute, sub license, and/or sell copies of the Software, and to
  12. * permit persons to whom the Software is furnished to do so, subject to
  13. * the following conditions:
  14. *
  15. * The above copyright notice and this permission notice (including the
  16. * next paragraph) shall be included in all copies or substantial portions
  17. * of the Software.
  18. *
  19. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26. *
  27. **************************************************************************/
  28. #include "vmwgfx_drv.h"
  29. #include <linux/highmem.h>
  30. /*
  31. * Template that implements find_first_diff() for a generic
  32. * unsigned integer type. @size and return value are in bytes.
  33. */
  34. #define VMW_FIND_FIRST_DIFF(_type) \
  35. static size_t vmw_find_first_diff_ ## _type \
  36. (const _type * dst, const _type * src, size_t size)\
  37. { \
  38. size_t i; \
  39. \
  40. for (i = 0; i < size; i += sizeof(_type)) { \
  41. if (*dst++ != *src++) \
  42. break; \
  43. } \
  44. \
  45. return i; \
  46. }
  47. /*
  48. * Template that implements find_last_diff() for a generic
  49. * unsigned integer type. Pointers point to the item following the
  50. * *end* of the area to be examined. @size and return value are in
  51. * bytes.
  52. */
  53. #define VMW_FIND_LAST_DIFF(_type) \
  54. static ssize_t vmw_find_last_diff_ ## _type( \
  55. const _type * dst, const _type * src, size_t size) \
  56. { \
  57. while (size) { \
  58. if (*--dst != *--src) \
  59. break; \
  60. \
  61. size -= sizeof(_type); \
  62. } \
  63. return size; \
  64. }
  65. /*
  66. * Instantiate find diff functions for relevant unsigned integer sizes,
  67. * assuming that wider integers are faster (including aligning) up to the
  68. * architecture native width, which is assumed to be 32 bit unless
  69. * CONFIG_64BIT is defined.
  70. */
  71. VMW_FIND_FIRST_DIFF(u8);
  72. VMW_FIND_LAST_DIFF(u8);
  73. VMW_FIND_FIRST_DIFF(u16);
  74. VMW_FIND_LAST_DIFF(u16);
  75. VMW_FIND_FIRST_DIFF(u32);
  76. VMW_FIND_LAST_DIFF(u32);
  77. #ifdef CONFIG_64BIT
  78. VMW_FIND_FIRST_DIFF(u64);
  79. VMW_FIND_LAST_DIFF(u64);
  80. #endif
  81. /* We use size aligned copies. This computes (addr - align(addr)) */
  82. #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
  83. /*
  84. * Template to compute find_first_diff() for a certain integer type
  85. * including a head copy for alignment, and adjustment of parameters
  86. * for tail find or increased resolution find using an unsigned integer find
  87. * of smaller width. If finding is complete, and resolution is sufficient,
  88. * the macro executes a return statement. Otherwise it falls through.
  89. */
  90. #define VMW_TRY_FIND_FIRST_DIFF(_type) \
  91. do { \
  92. unsigned int spill = SPILL(dst, _type); \
  93. size_t diff_offs; \
  94. \
  95. if (spill && spill == SPILL(src, _type) && \
  96. sizeof(_type) - spill <= size) { \
  97. spill = sizeof(_type) - spill; \
  98. diff_offs = vmw_find_first_diff_u8(dst, src, spill); \
  99. if (diff_offs < spill) \
  100. return round_down(offset + diff_offs, granularity); \
  101. \
  102. dst += spill; \
  103. src += spill; \
  104. size -= spill; \
  105. offset += spill; \
  106. spill = 0; \
  107. } \
  108. if (!spill && !SPILL(src, _type)) { \
  109. size_t to_copy = size & ~(sizeof(_type) - 1); \
  110. \
  111. diff_offs = vmw_find_first_diff_ ## _type \
  112. ((_type *) dst, (_type *) src, to_copy); \
  113. if (diff_offs >= size || granularity == sizeof(_type)) \
  114. return (offset + diff_offs); \
  115. \
  116. dst += diff_offs; \
  117. src += diff_offs; \
  118. size -= diff_offs; \
  119. offset += diff_offs; \
  120. } \
  121. } while (0) \
  122. /**
  123. * vmw_find_first_diff - find the first difference between dst and src
  124. *
  125. * @dst: The destination address
  126. * @src: The source address
  127. * @size: Number of bytes to compare
  128. * @granularity: The granularity needed for the return value in bytes.
  129. * return: The offset from find start where the first difference was
  130. * encountered in bytes. If no difference was found, the function returns
  131. * a value >= @size.
  132. */
  133. static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
  134. size_t granularity)
  135. {
  136. size_t offset = 0;
  137. /*
  138. * Try finding with large integers if alignment allows, or we can
  139. * fix it. Fall through if we need better resolution or alignment
  140. * was bad.
  141. */
  142. #ifdef CONFIG_64BIT
  143. VMW_TRY_FIND_FIRST_DIFF(u64);
  144. #endif
  145. VMW_TRY_FIND_FIRST_DIFF(u32);
  146. VMW_TRY_FIND_FIRST_DIFF(u16);
  147. return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
  148. granularity);
  149. }
  150. /*
  151. * Template to compute find_last_diff() for a certain integer type
  152. * including a tail copy for alignment, and adjustment of parameters
  153. * for head find or increased resolution find using an unsigned integer find
  154. * of smaller width. If finding is complete, and resolution is sufficient,
  155. * the macro executes a return statement. Otherwise it falls through.
  156. */
  157. #define VMW_TRY_FIND_LAST_DIFF(_type) \
  158. do { \
  159. unsigned int spill = SPILL(dst, _type); \
  160. ssize_t location; \
  161. ssize_t diff_offs; \
  162. \
  163. if (spill && spill <= size && spill == SPILL(src, _type)) { \
  164. diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
  165. if (diff_offs) { \
  166. location = size - spill + diff_offs - 1; \
  167. return round_down(location, granularity); \
  168. } \
  169. \
  170. dst -= spill; \
  171. src -= spill; \
  172. size -= spill; \
  173. spill = 0; \
  174. } \
  175. if (!spill && !SPILL(src, _type)) { \
  176. size_t to_copy = round_down(size, sizeof(_type)); \
  177. \
  178. diff_offs = vmw_find_last_diff_ ## _type \
  179. ((_type *) dst, (_type *) src, to_copy); \
  180. location = size - to_copy + diff_offs - sizeof(_type); \
  181. if (location < 0 || granularity == sizeof(_type)) \
  182. return location; \
  183. \
  184. dst -= to_copy - diff_offs; \
  185. src -= to_copy - diff_offs; \
  186. size -= to_copy - diff_offs; \
  187. } \
  188. } while (0)
  189. /**
  190. * vmw_find_last_diff - find the last difference between dst and src
  191. *
  192. * @dst: The destination address
  193. * @src: The source address
  194. * @size: Number of bytes to compare
  195. * @granularity: The granularity needed for the return value in bytes.
  196. * return: The offset from find start where the last difference was
  197. * encountered in bytes, or a negative value if no difference was found.
  198. */
  199. static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
  200. size_t granularity)
  201. {
  202. dst += size;
  203. src += size;
  204. #ifdef CONFIG_64BIT
  205. VMW_TRY_FIND_LAST_DIFF(u64);
  206. #endif
  207. VMW_TRY_FIND_LAST_DIFF(u32);
  208. VMW_TRY_FIND_LAST_DIFF(u16);
  209. return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
  210. granularity);
  211. }
  212. /**
  213. * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
  214. * struct vmw_diff_cpy.
  215. *
  216. * @diff: The struct vmw_diff_cpy closure argument (unused).
  217. * @dest: The copy destination.
  218. * @src: The copy source.
  219. * @n: Number of bytes to copy.
  220. */
  221. void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
  222. {
  223. memcpy(dest, src, n);
  224. }
  225. /**
  226. * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
  227. *
  228. * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
  229. * @diff_offs: The offset from @diff->line_offset where the difference was
  230. * found.
  231. */
  232. static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
  233. {
  234. size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
  235. struct drm_rect *rect = &diff->rect;
  236. rect->x1 = min_t(int, rect->x1, offs);
  237. rect->x2 = max_t(int, rect->x2, offs + 1);
  238. rect->y1 = min_t(int, rect->y1, diff->line);
  239. rect->y2 = max_t(int, rect->y2, diff->line + 1);
  240. }
  241. /**
  242. * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
  243. *
  244. * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
  245. * @dest: The copy destination.
  246. * @src: The copy source.
  247. * @n: Number of bytes to copy.
  248. *
  249. * In order to correctly track the modified content, the field @diff->line must
  250. * be pre-loaded with the current line number, the field @diff->line_offset must
  251. * be pre-loaded with the line offset in bytes where the copy starts, and
  252. * finally the field @diff->cpp need to be preloaded with the number of bytes
  253. * per unit in the horizontal direction of the area we're examining.
  254. * Typically bytes per pixel.
  255. * This is needed to know the needed granularity of the difference computing
  256. * operations. A higher cpp generally leads to faster execution at the cost of
  257. * bounding box width precision.
  258. */
  259. void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
  260. size_t n)
  261. {
  262. ssize_t csize, byte_len;
  263. if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
  264. return;
  265. /* TODO: Possibly use a single vmw_find_first_diff per line? */
  266. csize = vmw_find_first_diff(dest, src, n, diff->cpp);
  267. if (csize < n) {
  268. vmw_adjust_rect(diff, csize);
  269. byte_len = diff->cpp;
  270. /*
  271. * Starting from where first difference was found, find
  272. * location of last difference, and then copy.
  273. */
  274. diff->line_offset += csize;
  275. dest += csize;
  276. src += csize;
  277. n -= csize;
  278. csize = vmw_find_last_diff(dest, src, n, diff->cpp);
  279. if (csize >= 0) {
  280. byte_len += csize;
  281. vmw_adjust_rect(diff, csize);
  282. }
  283. memcpy(dest, src, byte_len);
  284. }
  285. diff->line_offset += n;
  286. }
  287. /**
  288. * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
  289. *
  290. * @mapped_dst: Already mapped destination page index in @dst_pages.
  291. * @dst_addr: Kernel virtual address of mapped destination page.
  292. * @dst_pages: Array of destination bo pages.
  293. * @dst_num_pages: Number of destination bo pages.
  294. * @dst_prot: Destination bo page protection.
  295. * @mapped_src: Already mapped source page index in @dst_pages.
  296. * @src_addr: Kernel virtual address of mapped source page.
  297. * @src_pages: Array of source bo pages.
  298. * @src_num_pages: Number of source bo pages.
  299. * @src_prot: Source bo page protection.
  300. * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
  301. */
  302. struct vmw_bo_blit_line_data {
  303. u32 mapped_dst;
  304. u8 *dst_addr;
  305. struct page **dst_pages;
  306. u32 dst_num_pages;
  307. pgprot_t dst_prot;
  308. u32 mapped_src;
  309. u8 *src_addr;
  310. struct page **src_pages;
  311. u32 src_num_pages;
  312. pgprot_t src_prot;
  313. struct vmw_diff_cpy *diff;
  314. };
  315. /**
  316. * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
  317. *
  318. * @d: Blit data as described above.
  319. * @dst_offset: Destination copy start offset from start of bo.
  320. * @src_offset: Source copy start offset from start of bo.
  321. * @bytes_to_copy: Number of bytes to copy in this line.
  322. */
  323. static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
  324. u32 dst_offset,
  325. u32 src_offset,
  326. u32 bytes_to_copy)
  327. {
  328. struct vmw_diff_cpy *diff = d->diff;
  329. while (bytes_to_copy) {
  330. u32 copy_size = bytes_to_copy;
  331. u32 dst_page = dst_offset >> PAGE_SHIFT;
  332. u32 src_page = src_offset >> PAGE_SHIFT;
  333. u32 dst_page_offset = dst_offset & ~PAGE_MASK;
  334. u32 src_page_offset = src_offset & ~PAGE_MASK;
  335. bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
  336. bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
  337. unmap_dst);
  338. copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
  339. copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
  340. if (unmap_src) {
  341. kunmap_atomic(d->src_addr);
  342. d->src_addr = NULL;
  343. }
  344. if (unmap_dst) {
  345. kunmap_atomic(d->dst_addr);
  346. d->dst_addr = NULL;
  347. }
  348. if (!d->dst_addr) {
  349. if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
  350. return -EINVAL;
  351. d->dst_addr =
  352. kmap_atomic_prot(d->dst_pages[dst_page],
  353. d->dst_prot);
  354. if (!d->dst_addr)
  355. return -ENOMEM;
  356. d->mapped_dst = dst_page;
  357. }
  358. if (!d->src_addr) {
  359. if (WARN_ON_ONCE(src_page >= d->src_num_pages))
  360. return -EINVAL;
  361. d->src_addr =
  362. kmap_atomic_prot(d->src_pages[src_page],
  363. d->src_prot);
  364. if (!d->src_addr)
  365. return -ENOMEM;
  366. d->mapped_src = src_page;
  367. }
  368. diff->do_cpy(diff, d->dst_addr + dst_page_offset,
  369. d->src_addr + src_page_offset, copy_size);
  370. bytes_to_copy -= copy_size;
  371. dst_offset += copy_size;
  372. src_offset += copy_size;
  373. }
  374. return 0;
  375. }
  376. /**
  377. * ttm_bo_cpu_blit - in-kernel cpu blit.
  378. *
  379. * @dst: Destination buffer object.
  380. * @dst_offset: Destination offset of blit start in bytes.
  381. * @dst_stride: Destination stride in bytes.
  382. * @src: Source buffer object.
  383. * @src_offset: Source offset of blit start in bytes.
  384. * @src_stride: Source stride in bytes.
  385. * @w: Width of blit.
  386. * @h: Height of blit.
  387. * return: Zero on success. Negative error value on failure. Will print out
  388. * kernel warnings on caller bugs.
  389. *
  390. * Performs a CPU blit from one buffer object to another avoiding a full
  391. * bo vmap which may exhaust- or fragment vmalloc space.
  392. * On supported architectures (x86), we're using kmap_atomic which avoids
  393. * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
  394. * reference already set-up mappings.
  395. *
  396. * Neither of the buffer objects may be placed in PCI memory
  397. * (Fixed memory in TTM terminology) when using this function.
  398. */
  399. int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
  400. u32 dst_offset, u32 dst_stride,
  401. struct ttm_buffer_object *src,
  402. u32 src_offset, u32 src_stride,
  403. u32 w, u32 h,
  404. struct vmw_diff_cpy *diff)
  405. {
  406. struct ttm_operation_ctx ctx = {
  407. .interruptible = false,
  408. .no_wait_gpu = false
  409. };
  410. u32 j, initial_line = dst_offset / dst_stride;
  411. struct vmw_bo_blit_line_data d;
  412. int ret = 0;
  413. /* Buffer objects need to be either pinned or reserved: */
  414. if (!(dst->mem.placement & TTM_PL_FLAG_NO_EVICT))
  415. dma_resv_assert_held(dst->base.resv);
  416. if (!(src->mem.placement & TTM_PL_FLAG_NO_EVICT))
  417. dma_resv_assert_held(src->base.resv);
  418. if (!ttm_tt_is_populated(dst->ttm)) {
  419. ret = dst->bdev->driver->ttm_tt_populate(dst->bdev, dst->ttm, &ctx);
  420. if (ret)
  421. return ret;
  422. }
  423. if (!ttm_tt_is_populated(src->ttm)) {
  424. ret = src->bdev->driver->ttm_tt_populate(src->bdev, src->ttm, &ctx);
  425. if (ret)
  426. return ret;
  427. }
  428. d.mapped_dst = 0;
  429. d.mapped_src = 0;
  430. d.dst_addr = NULL;
  431. d.src_addr = NULL;
  432. d.dst_pages = dst->ttm->pages;
  433. d.src_pages = src->ttm->pages;
  434. d.dst_num_pages = dst->num_pages;
  435. d.src_num_pages = src->num_pages;
  436. d.dst_prot = ttm_io_prot(dst->mem.placement, PAGE_KERNEL);
  437. d.src_prot = ttm_io_prot(src->mem.placement, PAGE_KERNEL);
  438. d.diff = diff;
  439. for (j = 0; j < h; ++j) {
  440. diff->line = j + initial_line;
  441. diff->line_offset = dst_offset % dst_stride;
  442. ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w);
  443. if (ret)
  444. goto out;
  445. dst_offset += dst_stride;
  446. src_offset += src_stride;
  447. }
  448. out:
  449. if (d.src_addr)
  450. kunmap_atomic(d.src_addr);
  451. if (d.dst_addr)
  452. kunmap_atomic(d.dst_addr);
  453. return ret;
  454. }