SkBlitRow_D32.cpp 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314
  1. /*
  2. * Copyright 2011 Google Inc.
  3. *
  4. * Use of this source code is governed by a BSD-style license that can be
  5. * found in the LICENSE file.
  6. */
  7. #include "include/private/SkColorData.h"
  8. #include "src/core/SkBlitRow.h"
  9. #include "src/core/SkOpts.h"
  10. #include "src/core/SkUtils.h"
  11. // Everyone agrees memcpy() is the best way to do this.
  12. static void blit_row_s32_opaque(SkPMColor* dst,
  13. const SkPMColor* src,
  14. int count,
  15. U8CPU alpha) {
  16. SkASSERT(255 == alpha);
  17. memcpy(dst, src, count * sizeof(SkPMColor));
  18. }
  19. // We have SSE2, NEON, and portable implementations of
  20. // blit_row_s32_blend() and blit_row_s32a_blend().
  21. // TODO(mtklein): can we do better in NEON than 2 pixels at a time?
  22. #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
  23. #include <emmintrin.h>
  24. static inline __m128i SkPMLerp_SSE2(const __m128i& src,
  25. const __m128i& dst,
  26. const unsigned src_scale) {
  27. // Computes dst + (((src - dst)*src_scale)>>8)
  28. const __m128i mask = _mm_set1_epi32(0x00FF00FF);
  29. // Unpack the 16x8-bit source into 2 8x16-bit splayed halves.
  30. __m128i src_rb = _mm_and_si128(mask, src);
  31. __m128i src_ag = _mm_srli_epi16(src, 8);
  32. __m128i dst_rb = _mm_and_si128(mask, dst);
  33. __m128i dst_ag = _mm_srli_epi16(dst, 8);
  34. // Compute scaled differences.
  35. __m128i diff_rb = _mm_sub_epi16(src_rb, dst_rb);
  36. __m128i diff_ag = _mm_sub_epi16(src_ag, dst_ag);
  37. __m128i s = _mm_set1_epi16(src_scale);
  38. diff_rb = _mm_mullo_epi16(diff_rb, s);
  39. diff_ag = _mm_mullo_epi16(diff_ag, s);
  40. // Pack the differences back together.
  41. diff_rb = _mm_srli_epi16(diff_rb, 8);
  42. diff_ag = _mm_andnot_si128(mask, diff_ag);
  43. __m128i diff = _mm_or_si128(diff_rb, diff_ag);
  44. // Add difference to destination.
  45. return _mm_add_epi8(dst, diff);
  46. }
  47. static void blit_row_s32_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  48. SkASSERT(alpha <= 255);
  49. auto src4 = (const __m128i*)src;
  50. auto dst4 = ( __m128i*)dst;
  51. while (count >= 4) {
  52. _mm_storeu_si128(dst4, SkPMLerp_SSE2(_mm_loadu_si128(src4),
  53. _mm_loadu_si128(dst4),
  54. SkAlpha255To256(alpha)));
  55. src4++;
  56. dst4++;
  57. count -= 4;
  58. }
  59. src = (const SkPMColor*)src4;
  60. dst = ( SkPMColor*)dst4;
  61. while (count --> 0) {
  62. *dst = SkPMLerp(*src, *dst, SkAlpha255To256(alpha));
  63. src++;
  64. dst++;
  65. }
  66. }
  67. static inline __m128i SkBlendARGB32_SSE2(const __m128i& src,
  68. const __m128i& dst,
  69. const unsigned aa) {
  70. unsigned alpha = SkAlpha255To256(aa);
  71. __m128i src_scale = _mm_set1_epi16(alpha);
  72. // SkAlphaMulInv256(SkGetPackedA32(src), src_scale)
  73. __m128i dst_scale = _mm_srli_epi32(src, 24);
  74. // High words in dst_scale are 0, so it's safe to multiply with 16-bit src_scale.
  75. dst_scale = _mm_mullo_epi16(dst_scale, src_scale);
  76. dst_scale = _mm_sub_epi32(_mm_set1_epi32(0xFFFF), dst_scale);
  77. dst_scale = _mm_add_epi32(dst_scale, _mm_srli_epi32(dst_scale, 8));
  78. dst_scale = _mm_srli_epi32(dst_scale, 8);
  79. // Duplicate scales into 2x16-bit pattern per pixel.
  80. dst_scale = _mm_shufflelo_epi16(dst_scale, _MM_SHUFFLE(2, 2, 0, 0));
  81. dst_scale = _mm_shufflehi_epi16(dst_scale, _MM_SHUFFLE(2, 2, 0, 0));
  82. const __m128i mask = _mm_set1_epi32(0x00FF00FF);
  83. // Unpack the 16x8-bit source/destination into 2 8x16-bit splayed halves.
  84. __m128i src_rb = _mm_and_si128(mask, src);
  85. __m128i src_ag = _mm_srli_epi16(src, 8);
  86. __m128i dst_rb = _mm_and_si128(mask, dst);
  87. __m128i dst_ag = _mm_srli_epi16(dst, 8);
  88. // Scale them.
  89. src_rb = _mm_mullo_epi16(src_rb, src_scale);
  90. src_ag = _mm_mullo_epi16(src_ag, src_scale);
  91. dst_rb = _mm_mullo_epi16(dst_rb, dst_scale);
  92. dst_ag = _mm_mullo_epi16(dst_ag, dst_scale);
  93. // Add the scaled source and destination.
  94. dst_rb = _mm_add_epi16(src_rb, dst_rb);
  95. dst_ag = _mm_add_epi16(src_ag, dst_ag);
  96. // Unsplay the halves back together.
  97. dst_rb = _mm_srli_epi16(dst_rb, 8);
  98. dst_ag = _mm_andnot_si128(mask, dst_ag);
  99. return _mm_or_si128(dst_rb, dst_ag);
  100. }
  101. static void blit_row_s32a_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  102. SkASSERT(alpha <= 255);
  103. auto src4 = (const __m128i*)src;
  104. auto dst4 = ( __m128i*)dst;
  105. while (count >= 4) {
  106. _mm_storeu_si128(dst4, SkBlendARGB32_SSE2(_mm_loadu_si128(src4),
  107. _mm_loadu_si128(dst4),
  108. alpha));
  109. src4++;
  110. dst4++;
  111. count -= 4;
  112. }
  113. src = (const SkPMColor*)src4;
  114. dst = ( SkPMColor*)dst4;
  115. while (count --> 0) {
  116. *dst = SkBlendARGB32(*src, *dst, alpha);
  117. src++;
  118. dst++;
  119. }
  120. }
  121. #elif defined(SK_ARM_HAS_NEON)
  122. #include <arm_neon.h>
  123. static void blit_row_s32_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  124. SkASSERT(alpha <= 255);
  125. uint16_t src_scale = SkAlpha255To256(alpha);
  126. uint16_t dst_scale = 256 - src_scale;
  127. while (count >= 2) {
  128. uint8x8_t vsrc, vdst, vres;
  129. uint16x8_t vsrc_wide, vdst_wide;
  130. vsrc = vreinterpret_u8_u32(vld1_u32(src));
  131. vdst = vreinterpret_u8_u32(vld1_u32(dst));
  132. vsrc_wide = vmovl_u8(vsrc);
  133. vsrc_wide = vmulq_u16(vsrc_wide, vdupq_n_u16(src_scale));
  134. vdst_wide = vmull_u8(vdst, vdup_n_u8(dst_scale));
  135. vdst_wide += vsrc_wide;
  136. vres = vshrn_n_u16(vdst_wide, 8);
  137. vst1_u32(dst, vreinterpret_u32_u8(vres));
  138. src += 2;
  139. dst += 2;
  140. count -= 2;
  141. }
  142. if (count == 1) {
  143. uint8x8_t vsrc = vdup_n_u8(0), vdst = vdup_n_u8(0), vres;
  144. uint16x8_t vsrc_wide, vdst_wide;
  145. vsrc = vreinterpret_u8_u32(vld1_lane_u32(src, vreinterpret_u32_u8(vsrc), 0));
  146. vdst = vreinterpret_u8_u32(vld1_lane_u32(dst, vreinterpret_u32_u8(vdst), 0));
  147. vsrc_wide = vmovl_u8(vsrc);
  148. vsrc_wide = vmulq_u16(vsrc_wide, vdupq_n_u16(src_scale));
  149. vdst_wide = vmull_u8(vdst, vdup_n_u8(dst_scale));
  150. vdst_wide += vsrc_wide;
  151. vres = vshrn_n_u16(vdst_wide, 8);
  152. vst1_lane_u32(dst, vreinterpret_u32_u8(vres), 0);
  153. }
  154. }
  155. static void blit_row_s32a_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  156. SkASSERT(alpha < 255);
  157. unsigned alpha256 = SkAlpha255To256(alpha);
  158. if (count & 1) {
  159. uint8x8_t vsrc = vdup_n_u8(0), vdst = vdup_n_u8(0), vres;
  160. uint16x8_t vdst_wide, vsrc_wide;
  161. unsigned dst_scale;
  162. vsrc = vreinterpret_u8_u32(vld1_lane_u32(src, vreinterpret_u32_u8(vsrc), 0));
  163. vdst = vreinterpret_u8_u32(vld1_lane_u32(dst, vreinterpret_u32_u8(vdst), 0));
  164. dst_scale = vget_lane_u8(vsrc, 3);
  165. dst_scale = SkAlphaMulInv256(dst_scale, alpha256);
  166. vsrc_wide = vmovl_u8(vsrc);
  167. vsrc_wide = vmulq_n_u16(vsrc_wide, alpha256);
  168. vdst_wide = vmovl_u8(vdst);
  169. vdst_wide = vmulq_n_u16(vdst_wide, dst_scale);
  170. vdst_wide += vsrc_wide;
  171. vres = vshrn_n_u16(vdst_wide, 8);
  172. vst1_lane_u32(dst, vreinterpret_u32_u8(vres), 0);
  173. dst++;
  174. src++;
  175. count--;
  176. }
  177. uint8x8_t alpha_mask;
  178. static const uint8_t alpha_mask_setup[] = {3,3,3,3,7,7,7,7};
  179. alpha_mask = vld1_u8(alpha_mask_setup);
  180. while (count) {
  181. uint8x8_t vsrc, vdst, vres, vsrc_alphas;
  182. uint16x8_t vdst_wide, vsrc_wide, vsrc_scale, vdst_scale;
  183. __builtin_prefetch(src+32);
  184. __builtin_prefetch(dst+32);
  185. vsrc = vreinterpret_u8_u32(vld1_u32(src));
  186. vdst = vreinterpret_u8_u32(vld1_u32(dst));
  187. vsrc_scale = vdupq_n_u16(alpha256);
  188. vsrc_alphas = vtbl1_u8(vsrc, alpha_mask);
  189. vdst_scale = vmovl_u8(vsrc_alphas);
  190. // Calculate SkAlphaMulInv256(vdst_scale, vsrc_scale).
  191. // A 16-bit lane would overflow if we used 0xFFFF here,
  192. // so use an approximation with 0xFF00 that is off by 1,
  193. // and add back 1 after to get the correct value.
  194. // This is valid if alpha256 <= 255.
  195. vdst_scale = vmlsq_u16(vdupq_n_u16(0xFF00), vdst_scale, vsrc_scale);
  196. vdst_scale = vsraq_n_u16(vdst_scale, vdst_scale, 8);
  197. vdst_scale = vsraq_n_u16(vdupq_n_u16(1), vdst_scale, 8);
  198. vsrc_wide = vmovl_u8(vsrc);
  199. vsrc_wide *= vsrc_scale;
  200. vdst_wide = vmovl_u8(vdst);
  201. vdst_wide *= vdst_scale;
  202. vdst_wide += vsrc_wide;
  203. vres = vshrn_n_u16(vdst_wide, 8);
  204. vst1_u32(dst, vreinterpret_u32_u8(vres));
  205. src += 2;
  206. dst += 2;
  207. count -= 2;
  208. }
  209. }
  210. #else
  211. static void blit_row_s32_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  212. SkASSERT(alpha <= 255);
  213. while (count --> 0) {
  214. *dst = SkPMLerp(*src, *dst, SkAlpha255To256(alpha));
  215. src++;
  216. dst++;
  217. }
  218. }
  219. static void blit_row_s32a_blend(SkPMColor* dst, const SkPMColor* src, int count, U8CPU alpha) {
  220. SkASSERT(alpha <= 255);
  221. while (count --> 0) {
  222. *dst = SkBlendARGB32(*src, *dst, alpha);
  223. src++;
  224. dst++;
  225. }
  226. }
  227. #endif
  228. SkBlitRow::Proc32 SkBlitRow::Factory32(unsigned flags) {
  229. static const SkBlitRow::Proc32 kProcs[] = {
  230. blit_row_s32_opaque,
  231. blit_row_s32_blend,
  232. nullptr, // blit_row_s32a_opaque is in SkOpts
  233. blit_row_s32a_blend
  234. };
  235. SkASSERT(flags < SK_ARRAY_COUNT(kProcs));
  236. flags &= SK_ARRAY_COUNT(kProcs) - 1; // just to be safe
  237. return flags == 2 ? SkOpts::blit_row_s32a_opaque
  238. : kProcs[flags];
  239. }
  240. void SkBlitRow::Color32(SkPMColor dst[], const SkPMColor src[], int count, SkPMColor color) {
  241. switch (SkGetPackedA32(color)) {
  242. case 0: memmove(dst, src, count * sizeof(SkPMColor)); return;
  243. case 255: sk_memset32(dst, color, count); return;
  244. }
  245. return SkOpts::blit_row_color32(dst, src, count, color);
  246. }