video_resource_updater.h 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_
  5. #define MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <vector>
  10. #include "base/memory/raw_ptr.h"
  11. #include "base/memory/ref_counted.h"
  12. #include "base/memory/weak_ptr.h"
  13. #include "base/trace_event/memory_dump_provider.h"
  14. #include "base/unguessable_token.h"
  15. #include "components/viz/common/resources/release_callback.h"
  16. #include "components/viz/common/resources/resource_format.h"
  17. #include "components/viz/common/resources/resource_id.h"
  18. #include "components/viz/common/resources/transferable_resource.h"
  19. #include "gpu/command_buffer/client/gles2_interface.h"
  20. #include "media/base/media_export.h"
  21. #include "ui/gfx/buffer_types.h"
  22. #include "ui/gfx/geometry/size.h"
  23. namespace gfx {
  24. class Rect;
  25. class Transform;
  26. } // namespace gfx
  27. namespace viz {
  28. class ClientResourceProvider;
  29. class ContextProvider;
  30. class RasterContextProvider;
  31. class CompositorRenderPass;
  32. class SharedBitmapReporter;
  33. } // namespace viz
  34. namespace gfx {
  35. class MaskFilterInfo;
  36. }
  37. namespace media {
  38. class PaintCanvasVideoRenderer;
  39. class VideoFrame;
  40. // Specifies what type of data is contained in the mailboxes, as well as how
  41. // many mailboxes will be present.
  42. enum class VideoFrameResourceType {
  43. NONE,
  44. YUV,
  45. RGB,
  46. RGBA_PREMULTIPLIED,
  47. RGBA,
  48. STREAM_TEXTURE,
  49. // The VideoFrame is merely a hint to compositor that a hole must be made
  50. // transparent so the video underlay will be visible.
  51. // Used by Chromecast only.
  52. VIDEO_HOLE,
  53. };
  54. class MEDIA_EXPORT VideoFrameExternalResources {
  55. public:
  56. VideoFrameResourceType type = VideoFrameResourceType::NONE;
  57. std::vector<viz::TransferableResource> resources;
  58. std::vector<viz::ReleaseCallback> release_callbacks;
  59. // Used by hardware textures which do not return values in the 0-1 range.
  60. // After a lookup, subtract offset and multiply by multiplier.
  61. float offset = 0.f;
  62. float multiplier = 1.f;
  63. uint32_t bits_per_channel = 8;
  64. VideoFrameExternalResources();
  65. VideoFrameExternalResources(VideoFrameExternalResources&& other);
  66. VideoFrameExternalResources& operator=(VideoFrameExternalResources&& other);
  67. ~VideoFrameExternalResources();
  68. };
  69. // VideoResourceUpdater is used by the video system to produce frame content as
  70. // resources consumable by the display compositor.
  71. class MEDIA_EXPORT VideoResourceUpdater
  72. : public base::trace_event::MemoryDumpProvider {
  73. public:
  74. // For GPU compositing |context_provider| should be provided and for software
  75. // compositing |shared_bitmap_reporter| should be provided. If there is a
  76. // non-null |context_provider| we assume GPU compositing.
  77. VideoResourceUpdater(viz::ContextProvider* context_provider,
  78. viz::RasterContextProvider* raster_context_provider,
  79. viz::SharedBitmapReporter* shared_bitmap_reporter,
  80. viz::ClientResourceProvider* resource_provider,
  81. bool use_stream_video_draw_quad,
  82. bool use_gpu_memory_buffer_resources,
  83. bool use_r16_texture,
  84. int max_resource_size);
  85. VideoResourceUpdater(const VideoResourceUpdater&) = delete;
  86. VideoResourceUpdater& operator=(const VideoResourceUpdater&) = delete;
  87. ~VideoResourceUpdater() override;
  88. // For each CompositorFrame the following sequence is expected:
  89. // 1. ObtainFrameResources(): Import resources for the next video frame with
  90. // viz::ClientResourceProvider. This will reuse existing GPU or
  91. // SharedMemory buffers if possible, otherwise it will allocate new ones.
  92. // 2. AppendQuads(): Add DrawQuads to CompositorFrame for video.
  93. // 3. ReleaseFrameResources(): After the CompositorFrame has been submitted,
  94. // remove imported resources from viz::ClientResourceProvider.
  95. void ObtainFrameResources(scoped_refptr<VideoFrame> video_frame);
  96. void ReleaseFrameResources();
  97. // Appends a quad representing |frame| to |render_pass|.
  98. // At most one quad is expected to be appended, this is enforced by the users
  99. // of this class (e.g: VideoFrameSubmitter). Producing only one quad will
  100. // allow viz to optimize compositing when the only content changing per-frame
  101. // is the video.
  102. void AppendQuads(viz::CompositorRenderPass* render_pass,
  103. scoped_refptr<VideoFrame> frame,
  104. gfx::Transform transform,
  105. gfx::Rect quad_rect,
  106. gfx::Rect visible_quad_rect,
  107. const gfx::MaskFilterInfo& mask_filter_info,
  108. absl::optional<gfx::Rect> clip_rect,
  109. bool context_opaque,
  110. float draw_opacity,
  111. int sorting_context_id);
  112. // TODO(kylechar): This is only public for testing, make private.
  113. VideoFrameExternalResources CreateExternalResourcesFromVideoFrame(
  114. scoped_refptr<VideoFrame> video_frame);
  115. viz::ResourceFormat YuvResourceFormat(int bits_per_channel);
  116. private:
  117. class PlaneResource;
  118. class HardwarePlaneResource;
  119. class SoftwarePlaneResource;
  120. // A resource that will be embedded in a DrawQuad in the next CompositorFrame.
  121. // Each video plane will correspond to one FrameResource.
  122. struct FrameResource {
  123. FrameResource();
  124. FrameResource(viz::ResourceId id, const gfx::Size& size);
  125. viz::ResourceId id;
  126. gfx::Size size_in_pixels;
  127. };
  128. bool software_compositor() const {
  129. return context_provider_ == nullptr && raster_context_provider_ == nullptr;
  130. }
  131. // Obtain a resource of the right format by either recycling an
  132. // unreferenced but appropriately formatted resource, or by
  133. // allocating a new resource.
  134. // Additionally, if the |unique_id| and |plane_index| match, then
  135. // it is assumed that the resource has the right data already and will only be
  136. // used for reading, and so is returned even if it is still referenced.
  137. // Passing -1 for |plane_index| avoids returning referenced
  138. // resources.
  139. PlaneResource* RecycleOrAllocateResource(const gfx::Size& resource_size,
  140. viz::ResourceFormat resource_format,
  141. const gfx::ColorSpace& color_space,
  142. int unique_id,
  143. int plane_index);
  144. PlaneResource* AllocateResource(const gfx::Size& plane_size,
  145. viz::ResourceFormat format,
  146. const gfx::ColorSpace& color_space);
  147. // Create a copy of a texture-backed source video frame in a new GL_TEXTURE_2D
  148. // texture. This is used when there are multiple GPU threads (Android WebView)
  149. // and the source video frame texture can't be used on the output GL context.
  150. // https://crbug.com/582170
  151. void CopyHardwarePlane(VideoFrame* video_frame,
  152. const gfx::ColorSpace& resource_color_space,
  153. const gpu::MailboxHolder& mailbox_holder,
  154. VideoFrameExternalResources* external_resources);
  155. // Get resources ready to be appended into DrawQuads. This is used for GPU
  156. // compositing most of the time, except for the cases mentioned in
  157. // CreateForSoftwarePlanes().
  158. VideoFrameExternalResources CreateForHardwarePlanes(
  159. scoped_refptr<VideoFrame> video_frame);
  160. // Get resources ready to be appended into DrawQuads. This is always used for
  161. // software compositing. This is also used for GPU compositing when the input
  162. // video frame has no textures.
  163. VideoFrameExternalResources CreateForSoftwarePlanes(
  164. scoped_refptr<VideoFrame> video_frame);
  165. gpu::gles2::GLES2Interface* ContextGL();
  166. void RecycleResource(uint32_t plane_resource_id,
  167. const gpu::SyncToken& sync_token,
  168. bool lost_resource);
  169. void ReturnTexture(scoped_refptr<VideoFrame> video_frame,
  170. const gpu::SyncToken& sync_token,
  171. bool lost_resource);
  172. // base::trace_event::MemoryDumpProvider implementation.
  173. bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
  174. base::trace_event::ProcessMemoryDump* pmd) override;
  175. const raw_ptr<viz::ContextProvider> context_provider_;
  176. const raw_ptr<viz::RasterContextProvider> raster_context_provider_;
  177. const raw_ptr<viz::SharedBitmapReporter> shared_bitmap_reporter_;
  178. const raw_ptr<viz::ClientResourceProvider> resource_provider_;
  179. const bool use_stream_video_draw_quad_;
  180. const bool use_gpu_memory_buffer_resources_;
  181. // TODO(crbug.com/759456): Remove after r16 is used without the flag.
  182. const bool use_r16_texture_;
  183. const int max_resource_size_;
  184. const int tracing_id_;
  185. std::unique_ptr<PaintCanvasVideoRenderer> video_renderer_;
  186. uint32_t next_plane_resource_id_ = 1;
  187. // Temporary pixel buffer when converting between formats.
  188. std::unique_ptr<uint8_t[]> upload_pixels_;
  189. size_t upload_pixels_size_ = 0;
  190. VideoFrameResourceType frame_resource_type_;
  191. float frame_resource_offset_;
  192. float frame_resource_multiplier_;
  193. uint32_t frame_bits_per_channel_;
  194. // Resources that will be placed into quads by the next call to
  195. // AppendDrawQuads().
  196. std::vector<FrameResource> frame_resources_;
  197. // If the video resource is a hole punching VideoFrame sent by Chromecast,
  198. // the VideoFrame carries an |overlay_plane_id_| to activate the video
  199. // overlay, but there is no video content to display within VideoFrame.
  200. base::UnguessableToken overlay_plane_id_;
  201. // Resources allocated by VideoResourceUpdater. Used to recycle resources so
  202. // we can reduce the number of allocations and data transfers.
  203. std::vector<std::unique_ptr<PlaneResource>> all_resources_;
  204. base::WeakPtrFactory<VideoResourceUpdater> weak_ptr_factory_{this};
  205. };
  206. } // namespace media
  207. #endif // MEDIA_RENDERERS_VIDEO_RESOURCE_UPDATER_H_