camera_video_frame_renderer.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. // Copyright 2022 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/capture_mode/camera_video_frame_renderer.h"
  5. #include <cmath>
  6. #include <iterator>
  7. #include <memory>
  8. #include <vector>
  9. #include "base/check.h"
  10. #include "cc/trees/layer_tree_frame_sink.h"
  11. #include "components/viz/common/hit_test/hit_test_region_list.h"
  12. #include "components/viz/common/resources/returned_resource.h"
  13. #include "media/renderers/video_resource_updater.h"
  14. #include "ui/aura/env.h"
  15. #include "ui/aura/window_tree_host.h"
  16. #include "ui/compositor/compositor.h"
  17. #include "ui/compositor/layer.h"
  18. #include "ui/compositor/layer_type.h"
  19. #include "ui/gfx/geometry/dip_util.h"
  20. #include "ui/gfx/geometry/size_conversions.h"
  21. namespace ash {
  22. namespace {
  23. ui::ContextFactory* GetContextFactory() {
  24. return aura::Env::GetInstance()->context_factory();
  25. }
  26. } // namespace
  27. CameraVideoFrameRenderer::CameraVideoFrameRenderer(
  28. mojo::Remote<video_capture::mojom::VideoSource> camera_video_source,
  29. const media::VideoCaptureFormat& capture_format,
  30. bool should_flip_frames_horizontally)
  31. : host_window_(/*delegate=*/nullptr),
  32. video_frame_handler_(/*delegate=*/this,
  33. std::move(camera_video_source),
  34. capture_format),
  35. context_provider_(GetContextFactory()->SharedMainThreadContextProvider()),
  36. raster_context_provider_(
  37. GetContextFactory()->SharedMainThreadRasterContextProvider()),
  38. should_flip_frames_horizontally_(should_flip_frames_horizontally) {
  39. host_window_.set_owned_by_parent(false);
  40. host_window_.Init(ui::LAYER_SOLID_COLOR);
  41. host_window_.layer()->SetColor(SK_ColorDKGRAY);
  42. host_window_.SetName("CameraVideoFramesHost");
  43. }
  44. CameraVideoFrameRenderer::~CameraVideoFrameRenderer() {
  45. if (layer_tree_frame_sink_)
  46. layer_tree_frame_sink_->DetachFromClient();
  47. client_resource_provider_.ShutdownAndReleaseAllResources();
  48. }
  49. void CameraVideoFrameRenderer::Initialize() {
  50. DCHECK(!layer_tree_frame_sink_);
  51. DCHECK(host_window_.parent())
  52. << "Before calling Initialize(), host_window_ must be added to the "
  53. "window hierarchy first.";
  54. layer_tree_frame_sink_ = host_window_.CreateLayerTreeFrameSink();
  55. layer_tree_frame_sink_->BindToClient(this);
  56. const int max_texture_size =
  57. raster_context_provider_->ContextCapabilities().max_texture_size;
  58. video_resource_updater_ = std::make_unique<media::VideoResourceUpdater>(
  59. context_provider_.get(), raster_context_provider_.get(),
  60. layer_tree_frame_sink_.get(), &client_resource_provider_,
  61. /*use_stream_video_draw_quad=*/false,
  62. /*use_gpu_memory_buffer_resources=*/false,
  63. /*use_r16_texture=*/false, max_texture_size);
  64. video_frame_handler_.StartHandlingFrames();
  65. }
  66. void CameraVideoFrameRenderer::OnCameraVideoFrame(
  67. scoped_refptr<media::VideoFrame> frame) {
  68. DCHECK(layer_tree_frame_sink_);
  69. DCHECK(video_resource_updater_);
  70. current_video_frame_ = std::move(frame);
  71. }
  72. void CameraVideoFrameRenderer::OnBeginFrameSourcePausedChanged(bool paused) {}
  73. bool CameraVideoFrameRenderer::OnBeginFrameDerivedImpl(
  74. const viz::BeginFrameArgs& args) {
  75. viz::BeginFrameAck current_begin_frame_ack(args, false);
  76. if (pending_compositor_frame_ack_ || !current_video_frame_) {
  77. // TODO(afakhry): If latency becomes an issue, instead of calling
  78. // `DidNotProduceFrame()` immediately, we can wait within a deadline for
  79. // a new video frame, and call `DidNotProduceFrame()` if the deadline
  80. // expires and no video frame was received.
  81. layer_tree_frame_sink_->DidNotProduceFrame(
  82. current_begin_frame_ack, cc::FrameSkippedReason::kWaitingOnMain);
  83. return false;
  84. }
  85. // We move the `current_video_frame_` below into `CreateCompositorFrame()`.
  86. // However, `on_video_frame_rendered_for_test_` (if valid) should be called
  87. // after the compositor frame has been submitted, so we extend the lifetime of
  88. // frame by keeping ref-counted ptr to it here.
  89. scoped_refptr<media::VideoFrame> frame_for_test;
  90. if (on_video_frame_rendered_for_test_)
  91. frame_for_test = current_video_frame_;
  92. pending_compositor_frame_ack_ = true;
  93. video_resource_updater_->ObtainFrameResources(current_video_frame_);
  94. layer_tree_frame_sink_->SubmitCompositorFrame(
  95. CreateCompositorFrame(current_begin_frame_ack,
  96. std::move(current_video_frame_)),
  97. /*hit_test_data_changed=*/false);
  98. video_resource_updater_->ReleaseFrameResources();
  99. if (on_video_frame_rendered_for_test_) {
  100. DCHECK(frame_for_test);
  101. std::move(on_video_frame_rendered_for_test_).Run(std::move(frame_for_test));
  102. }
  103. return true;
  104. }
  105. void CameraVideoFrameRenderer::SetBeginFrameSource(
  106. viz::BeginFrameSource* source) {
  107. if (source == begin_frame_source_)
  108. return;
  109. if (begin_frame_source_)
  110. begin_frame_source_->RemoveObserver(this);
  111. begin_frame_source_ = source;
  112. if (begin_frame_source_)
  113. begin_frame_source_->AddObserver(this);
  114. }
  115. absl::optional<viz::HitTestRegionList>
  116. CameraVideoFrameRenderer::BuildHitTestData() {
  117. return absl::nullopt;
  118. }
  119. void CameraVideoFrameRenderer::ReclaimResources(
  120. std::vector<viz::ReturnedResource> resources) {
  121. client_resource_provider_.ReceiveReturnsFromParent(std::move(resources));
  122. }
  123. void CameraVideoFrameRenderer::SetTreeActivationCallback(
  124. base::RepeatingClosure callback) {}
  125. void CameraVideoFrameRenderer::DidReceiveCompositorFrameAck() {
  126. pending_compositor_frame_ack_ = false;
  127. }
  128. void CameraVideoFrameRenderer::DidPresentCompositorFrame(
  129. uint32_t frame_token,
  130. const viz::FrameTimingDetails& details) {}
  131. void CameraVideoFrameRenderer::DidLoseLayerTreeFrameSink() {}
  132. void CameraVideoFrameRenderer::OnDraw(const gfx::Transform& transform,
  133. const gfx::Rect& viewport,
  134. bool resourceless_software_draw,
  135. bool skip_draw) {}
  136. void CameraVideoFrameRenderer::SetMemoryPolicy(
  137. const cc::ManagedMemoryPolicy& policy) {}
  138. void CameraVideoFrameRenderer::SetExternalTilePriorityConstraints(
  139. const gfx::Rect& viewport_rect,
  140. const gfx::Transform& transform) {}
  141. viz::CompositorFrame CameraVideoFrameRenderer::CreateCompositorFrame(
  142. const viz::BeginFrameAck& begin_frame_ack,
  143. scoped_refptr<media::VideoFrame> video_frame) {
  144. viz::CompositorFrame compositor_frame;
  145. compositor_frame.metadata.frame_token = ++compositor_frame_token_generator_;
  146. compositor_frame.metadata.begin_frame_ack = begin_frame_ack;
  147. if (video_frame->ColorSpace().IsValid()) {
  148. compositor_frame.metadata.content_color_usage =
  149. video_frame->ColorSpace().GetContentColorUsage();
  150. }
  151. compositor_frame.metadata.begin_frame_ack.has_damage = true;
  152. compositor_frame.metadata.may_contain_video = true;
  153. const float dsf = host_window_.GetHost()->device_scale_factor();
  154. compositor_frame.metadata.device_scale_factor = dsf;
  155. // In our current implementation the `host_window_` is always expected to be
  156. // a square.
  157. const gfx::Size host_size_pixels = gfx::ToRoundedSize(
  158. gfx::ConvertSizeToPixels(host_window_.bounds().size(), dsf));
  159. DCHECK_EQ(host_size_pixels.width(), host_size_pixels.height());
  160. // Calculate the compositor frame size based on the video frame size, such
  161. // that the video frame is scaled to fit the size of the `host_window_` in
  162. // pixels.
  163. const gfx::Size video_frame_size = video_frame->natural_size();
  164. gfx::Size compositor_frame_size;
  165. // Pick the smallest side of the video frame and scale it to fit the
  166. // corresponding length of the `host_window_`. Scale the other side
  167. // maintaining the same aspect ratio. This ensures that we don't end up with
  168. // a compositor frame with one side shorter than the corresponding side of the
  169. // `host_window_`.
  170. if (video_frame_size.height() <= video_frame_size.width()) {
  171. const float scale = static_cast<float>(host_size_pixels.height()) /
  172. video_frame_size.height();
  173. compositor_frame_size.SetSize(std::ceilf(video_frame_size.width() * scale),
  174. host_size_pixels.height());
  175. } else {
  176. const float scale =
  177. static_cast<float>(host_size_pixels.width()) / video_frame_size.width();
  178. compositor_frame_size.SetSize(
  179. host_size_pixels.width(),
  180. std::ceilf(video_frame_size.height() * scale));
  181. }
  182. DCHECK(!compositor_frame_size.IsEmpty());
  183. DCHECK(compositor_frame_size.width() == host_size_pixels.width() ||
  184. compositor_frame_size.height() == host_size_pixels.height());
  185. // A change in the size of the compositor frame means we need to identify a
  186. // new surface to submit the compositor frame to since the surface size is
  187. // different.
  188. if (compositor_frame_size != last_compositor_frame_size_pixels_ ||
  189. dsf != last_compositor_frame_dsf_) {
  190. last_compositor_frame_size_pixels_ = compositor_frame_size;
  191. last_compositor_frame_dsf_ = dsf;
  192. host_window_.AllocateLocalSurfaceId();
  193. }
  194. const gfx::Rect quad_rect(compositor_frame_size);
  195. auto render_pass =
  196. viz::CompositorRenderPass::Create(/*shared_quad_state_list_size=*/1u,
  197. /*quad_list_size=*/1u);
  198. render_pass->SetNew(viz::CompositorRenderPassId{1}, quad_rect, quad_rect,
  199. gfx::Transform());
  200. // If the camera should behave like a mirror, we should flip the frames around
  201. // the Y axis (by scaling by -1 in X).
  202. gfx::Transform transform;
  203. if (should_flip_frames_horizontally_) {
  204. transform.Scale(-1, 1);
  205. transform.Translate(-compositor_frame_size.width(), 0);
  206. }
  207. // Center the compositor frame horizontally and vertically inside
  208. // `host_window_`.
  209. int x_offset =
  210. -(compositor_frame_size.width() - host_size_pixels.width()) / 2;
  211. const int y_offset =
  212. -(compositor_frame_size.height() - host_size_pixels.height()) / 2;
  213. // When the frame is flipped horizontally around Y, we offset in the opposite
  214. // direction.
  215. if (should_flip_frames_horizontally_)
  216. x_offset *= -1;
  217. transform.Translate(x_offset, y_offset);
  218. const bool context_opaque = media::IsOpaque(video_frame->format());
  219. // Note that `video_frame`'s ownership is moved into `AppendQuads()`. Do not
  220. // access after the `std::move()` below.
  221. video_resource_updater_->AppendQuads(
  222. render_pass.get(), std::move(video_frame), transform, quad_rect,
  223. /*visible_quad_rect=*/quad_rect,
  224. /*mask_filter_info=*/gfx::MaskFilterInfo(),
  225. /*clip_rect=*/absl::nullopt, context_opaque,
  226. /*draw_opacity=*/1.0f, /*sorting_context_id=*/0);
  227. compositor_frame.render_pass_list.emplace_back(std::move(render_pass));
  228. const auto& quad_list = compositor_frame.render_pass_list.back()->quad_list;
  229. DCHECK_EQ(quad_list.size(), 1u);
  230. const viz::DrawQuad::Resources& resources = quad_list.front()->resources;
  231. std::vector<viz::ResourceId> resource_ids;
  232. resource_ids.reserve(resources.count);
  233. for (uint32_t i = 0; i < resources.count; ++i)
  234. resource_ids.push_back(resources.ids[i]);
  235. std::vector<viz::TransferableResource> resource_list;
  236. client_resource_provider_.PrepareSendToParent(resource_ids, &resource_list,
  237. raster_context_provider_.get());
  238. compositor_frame.resource_list = std::move(resource_list);
  239. return compositor_frame;
  240. }
  241. } // namespace ash