paint_canvas_video_renderer.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_
  5. #define MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include "base/memory/ref_counted.h"
  9. #include "base/threading/thread_checker.h"
  10. #include "base/timer/timer.h"
  11. #include "cc/paint/paint_canvas.h"
  12. #include "cc/paint/paint_flags.h"
  13. #include "cc/paint/paint_image.h"
  14. #include "gpu/command_buffer/common/mailbox.h"
  15. #include "media/base/media_export.h"
  16. #include "media/base/timestamp_constants.h"
  17. #include "media/base/video_frame.h"
  18. #include "media/base/video_transformation.h"
  19. #include "media/renderers/video_frame_yuv_converter.h"
  20. #include "third_party/abseil-cpp/absl/types/optional.h"
  21. namespace gfx {
  22. class RectF;
  23. }
  24. namespace gpu {
  25. struct Capabilities;
  26. namespace gles2 {
  27. class GLES2Interface;
  28. }
  29. } // namespace gpu
  30. namespace viz {
  31. class RasterContextProvider;
  32. }
  33. namespace media {
  34. class VideoTextureBacking;
  35. // Handles rendering of VideoFrames to PaintCanvases.
  36. class MEDIA_EXPORT PaintCanvasVideoRenderer {
  37. public:
  38. // Specifies the chroma upsampling filter used for pixel formats with chroma
  39. // subsampling (YUV 4:2:0 and YUV 4:2:2).
  40. //
  41. // NOTE: Keep the numeric values in sync with libyuv::FilterMode.
  42. enum FilterMode {
  43. kFilterNone = 0, // Nearest neighbor.
  44. kFilterBilinear = 2, // Bilinear interpolation.
  45. };
  46. PaintCanvasVideoRenderer();
  47. PaintCanvasVideoRenderer(const PaintCanvasVideoRenderer&) = delete;
  48. PaintCanvasVideoRenderer& operator=(const PaintCanvasVideoRenderer&) = delete;
  49. ~PaintCanvasVideoRenderer();
  50. // Paints |video_frame| translated and scaled to |dest_rect| on |canvas|.
  51. //
  52. // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d|
  53. // and |context_support| must be provided.
  54. //
  55. // If |video_frame| is nullptr or an unsupported format, |dest_rect| will be
  56. // painted black.
  57. void Paint(scoped_refptr<VideoFrame> video_frame,
  58. cc::PaintCanvas* canvas,
  59. const gfx::RectF& dest_rect,
  60. cc::PaintFlags& flags,
  61. VideoTransformation video_transformation,
  62. viz::RasterContextProvider* raster_context_provider);
  63. // Paints |video_frame|, scaled to its |video_frame->visible_rect().size()|
  64. // on |canvas|. Note that the origin of |video_frame->visible_rect()| is
  65. // ignored -- the copy is done to the origin of |canvas|.
  66. //
  67. // If the format of |video_frame| is PIXEL_FORMAT_NATIVE_TEXTURE, |context_3d|
  68. // and |context_support| must be provided.
  69. void Copy(scoped_refptr<VideoFrame> video_frame,
  70. cc::PaintCanvas* canvas,
  71. viz::RasterContextProvider* raster_context_provider);
  72. // Convert the contents of |video_frame| to raw RGB pixels. |rgb_pixels|
  73. // should point into a buffer large enough to hold as many 32 bit RGBA pixels
  74. // as are in the visible_rect() area of the frame. |premultiply_alpha|
  75. // indicates whether the R, G, B samples in |rgb_pixels| should be multiplied
  76. // by alpha. |filter| specifies the chroma upsampling filter used for pixel
  77. // formats with chroma subsampling. If chroma planes in the pixel format are
  78. // not subsampled, |filter| is ignored.
  79. //
  80. // NOTE: If |video_frame| doesn't have an alpha plane, all the A samples in
  81. // |rgb_pixels| will be 255 (equivalent to an alpha of 1.0) and therefore the
  82. // value of |premultiply_alpha| has no effect on the R, G, B samples in
  83. // |rgb_pixels|.
  84. static void ConvertVideoFrameToRGBPixels(const media::VideoFrame* video_frame,
  85. void* rgb_pixels,
  86. size_t row_bytes,
  87. bool premultiply_alpha = true,
  88. FilterMode filter = kFilterNone);
  89. // Copy the contents of |video_frame| to |texture| of |destination_gl|.
  90. //
  91. // The format of |video_frame| must be VideoFrame::NATIVE_TEXTURE.
  92. bool CopyVideoFrameTexturesToGLTexture(
  93. viz::RasterContextProvider* raster_context_provider,
  94. gpu::gles2::GLES2Interface* destination_gl,
  95. scoped_refptr<VideoFrame> video_frame,
  96. unsigned int target,
  97. unsigned int texture,
  98. unsigned int internal_format,
  99. unsigned int format,
  100. unsigned int type,
  101. int level,
  102. bool premultiply_alpha,
  103. bool flip_y);
  104. // TODO(776222): Remove this function from PaintCanvasVideoRenderer.
  105. static bool PrepareVideoFrameForWebGL(
  106. viz::RasterContextProvider* raster_context_provider,
  107. gpu::gles2::GLES2Interface* gl,
  108. scoped_refptr<VideoFrame> video_frame,
  109. unsigned int target,
  110. unsigned int texture);
  111. // Copy the CPU-side YUV contents of |video_frame| to texture |texture| in
  112. // context |destination_gl|.
  113. // |level|, |internal_format|, |type| specify target texture |texture|.
  114. // The format of |video_frame| must be mappable.
  115. // |context_3d| has a GrContext that may be used during the copy.
  116. // CorrectLastImageDimensions() ensures that the source texture will be
  117. // cropped to |visible_rect|. Returns true on success.
  118. bool CopyVideoFrameYUVDataToGLTexture(
  119. viz::RasterContextProvider* raster_context_provider,
  120. gpu::gles2::GLES2Interface* destination_gl,
  121. scoped_refptr<VideoFrame> video_frame,
  122. unsigned int target,
  123. unsigned int texture,
  124. unsigned int internal_format,
  125. unsigned int format,
  126. unsigned int type,
  127. int level,
  128. bool premultiply_alpha,
  129. bool flip_y);
  130. // Calls texImage2D where the texture image data source is the contents of
  131. // |video_frame|. Texture |texture| needs to be created and bound to |target|
  132. // before this call and the binding is active upon return.
  133. // This is an optimization of WebGL |video_frame| TexImage2D implementation
  134. // for specific combinations of |video_frame| and |texture| formats; e.g. if
  135. // |frame format| is Y16, optimizes conversion of normalized 16-bit content
  136. // and calls texImage2D to |texture|. |level|, |internal_format|, |format| and
  137. // |type| are WebGL texImage2D parameters.
  138. // Returns false if there is no implementation for given parameters.
  139. static bool TexImage2D(unsigned target,
  140. unsigned texture,
  141. gpu::gles2::GLES2Interface* gl,
  142. const gpu::Capabilities& gpu_capabilities,
  143. VideoFrame* video_frame,
  144. int level,
  145. int internalformat,
  146. unsigned format,
  147. unsigned type,
  148. bool flip_y,
  149. bool premultiply_alpha);
  150. // Calls texSubImage2D where the texture image data source is the contents of
  151. // |video_frame|.
  152. // This is an optimization of WebGL |video_frame| TexSubImage2D implementation
  153. // for specific combinations of |video_frame| and texture |format| and |type|;
  154. // e.g. if |frame format| is Y16, converts unsigned 16-bit value to target
  155. // |format| and calls WebGL texSubImage2D. |level|, |format|, |type|,
  156. // |xoffset| and |yoffset| are texSubImage2D parameters.
  157. // Returns false if there is no implementation for given parameters.
  158. static bool TexSubImage2D(unsigned target,
  159. gpu::gles2::GLES2Interface* gl,
  160. VideoFrame* video_frame,
  161. int level,
  162. unsigned format,
  163. unsigned type,
  164. int xoffset,
  165. int yoffset,
  166. bool flip_y,
  167. bool premultiply_alpha);
  168. // In general, We hold the most recently painted frame to increase the
  169. // performance for the case that the same frame needs to be painted
  170. // repeatedly. Call this function if you are sure the most recent frame will
  171. // never be painted again, so we can release the resource.
  172. void ResetCache();
  173. // Used for unit test.
  174. gfx::Size LastImageDimensionsForTesting();
  175. private:
  176. // This structure wraps information extracted out of a VideoFrame and/or
  177. // constructed out of it. The various calls in PaintCanvasVideoRenderer must
  178. // not keep a reference to the VideoFrame so necessary data is extracted out
  179. // of it.
  180. struct Cache {
  181. explicit Cache(int frame_id);
  182. ~Cache();
  183. // VideoFrame::unique_id() of the videoframe used to generate the cache.
  184. int frame_id;
  185. // A PaintImage that can be used to draw into a PaintCanvas. This is sized
  186. // to the visible size of the VideoFrame. Its contents are generated lazily.
  187. cc::PaintImage paint_image;
  188. // The backing for the source texture. This is also responsible for managing
  189. // the lifetime of the texture.
  190. sk_sp<VideoTextureBacking> texture_backing;
  191. // The GL texture ID used in non-OOP code path.
  192. // This is only set if the VideoFrame was texture-backed.
  193. uint32_t source_texture = 0;
  194. // The allocated size of VideoFrame texture.
  195. // This is only set if the VideoFrame was texture-backed.
  196. gfx::Size coded_size;
  197. // The visible subrect of |coded_size| that represents the logical contents
  198. // of the frame after cropping.
  199. // This is only set if the VideoFrame was texture-backed.
  200. gfx::Rect visible_rect;
  201. // True if the underlying resource was created with a top left origin.
  202. bool texture_origin_is_top_left = true;
  203. // Used to allow recycling of the previous shared image. This requires that
  204. // no external users have access to this resource via SkImage. Returns true
  205. // if the existing resource can be recycled.
  206. bool Recycle();
  207. };
  208. // Update the cache holding the most-recently-painted frame. Returns false
  209. // if the image couldn't be updated.
  210. bool UpdateLastImage(scoped_refptr<VideoFrame> video_frame,
  211. viz::RasterContextProvider* raster_context_provider,
  212. bool allow_wrap_texture);
  213. bool PrepareVideoFrame(scoped_refptr<VideoFrame> video_frame,
  214. viz::RasterContextProvider* raster_context_provider,
  215. const gpu::MailboxHolder& dest_holder);
  216. bool UploadVideoFrameToGLTexture(
  217. viz::RasterContextProvider* raster_context_provider,
  218. gpu::gles2::GLES2Interface* destination_gl,
  219. scoped_refptr<VideoFrame> video_frame,
  220. unsigned int target,
  221. unsigned int texture,
  222. unsigned int internal_format,
  223. unsigned int format,
  224. unsigned int type,
  225. bool flip_y);
  226. bool CacheBackingWrapsTexture() const;
  227. absl::optional<Cache> cache_;
  228. // If |cache_| is not used for a while, it's deleted to save memory.
  229. base::DelayTimer cache_deleting_timer_;
  230. // Stable paint image id to provide to draw image calls.
  231. cc::PaintImage::Id renderer_stable_id_;
  232. // Used for DCHECKs to ensure method calls executed in the correct thread.
  233. base::ThreadChecker thread_checker_;
  234. struct YUVTextureCache {
  235. YUVTextureCache();
  236. ~YUVTextureCache();
  237. void Reset();
  238. // The ContextProvider that holds the texture.
  239. scoped_refptr<viz::RasterContextProvider> raster_context_provider;
  240. // The size of the texture.
  241. gfx::Size size;
  242. // The shared image backing the texture.
  243. gpu::Mailbox mailbox;
  244. // Used to perform YUV->RGB conversion on video frames. Internally caches
  245. // shared images that are created to upload CPU video frame data to the GPU.
  246. VideoFrameYUVConverter yuv_converter;
  247. // A SyncToken after last usage, used for reusing or destroying texture and
  248. // shared image.
  249. gpu::SyncToken sync_token;
  250. };
  251. YUVTextureCache yuv_cache_;
  252. };
  253. } // namespace media
  254. #endif // MEDIA_RENDERERS_PAINT_CANVAS_VIDEO_RENDERER_H_