// Copyright 2022 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "ash/capture_mode/fake_camera_device.h" #include #include #include "base/bind.h" #include "base/callback_helpers.h" #include "base/location.h" #include "base/memory/ptr_util.h" #include "base/memory/shared_memory_mapping.h" #include "base/memory/unsafe_shared_memory_region.h" #include "base/notreached.h" #include "cc/paint/paint_flags.h" #include "cc/paint/skia_paint_canvas.h" #include "gpu/command_buffer/client/gpu_memory_buffer_manager.h" #include "gpu/ipc/common/surface_handle.h" #include "media/base/video_frame.h" #include "media/base/video_types.h" #include "media/capture/mojom/video_capture_buffer.mojom.h" #include "media/capture/video_capture_types.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" #include "services/video_capture/public/mojom/video_frame_handler.mojom.h" #include "third_party/skia/include/core/SkRect.h" #include "ui/aura/env.h" #include "ui/compositor/compositor.h" #include "ui/gfx/gpu_memory_buffer.h" namespace ash { namespace { // The next ID to be used for a newly created buffer. int g_next_buffer_id = 0; std::unique_ptr CreateGpuMemoryBuffer( const gfx::Size& frame_size) { return aura::Env::GetInstance() ->context_factory() ->GetGpuMemoryBufferManager() ->CreateGpuMemoryBuffer(frame_size, gfx::BufferFormat::BGRA_8888, gfx::BufferUsage::SCANOUT_CPU_READ_WRITE, gpu::kNullSurfaceHandle, /*shutdown_event=*/nullptr); } SkRect GetCircleRect(const gfx::Point& center, int radius) { return SkRect::MakeLTRB(center.x() - radius, center.y() - radius, center.x() + radius, center.y() + radius); } // Draws the content of the produced video frame whose size is the given // `frame_size` on the given `canvas`. void DrawFrameOnCanvas(cc::SkiaPaintCanvas&& canvas, const gfx::Size& frame_size) { cc::PaintFlags flags; flags.setStyle(cc::PaintFlags::kFill_Style); const auto canvas_bounds = SkRect::MakeIWH(frame_size.width(), frame_size.height()); flags.setColor(SK_ColorBLUE); canvas.drawRect(canvas_bounds, flags); flags.setColor(SK_ColorGREEN); const auto canvas_center = gfx::Rect(frame_size).CenterPoint(); const int radius = 20; canvas.drawOval(GetCircleRect(canvas_center, radius), flags); flags.setColor(SK_ColorRED); canvas.drawOval(GetCircleRect(gfx::Point(), radius), flags); } // ----------------------------------------------------------------------------- // BufferStrategy: // Defines an interface for operations that can be done on different buffer // types. class BufferStrategy { public: virtual ~BufferStrategy() = default; // Gets the handle of the concrete buffer implementation that can be sent over // via mojo. virtual media::mojom::VideoBufferHandlePtr GetHandle() const = 0; // Draws the contents of a video frame whose size is `frame_size` on the // buffer backing this object. virtual void DrawFrameOnBuffer(const gfx::Size& frame_size) = 0; }; // ----------------------------------------------------------------------------- // GpuMemoryBufferStrategy: // Defines a concrete implementation of `BufferStrategy` which creates a // `GpuMemoryBuffer` and implements all the operations on it. class GpuMemoryBufferStrategy : public BufferStrategy { public: explicit GpuMemoryBufferStrategy(const gfx::Size& frame_size) : gmb_(CreateGpuMemoryBuffer(frame_size)) { DCHECK(gmb_); } uint8_t* data() { return static_cast(gmb_->memory(0)); } size_t bytes_per_row() { return gmb_->stride(0); } // BufferStrategy: media::mojom::VideoBufferHandlePtr GetHandle() const override { return media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle( gmb_->CloneHandle()); } void DrawFrameOnBuffer(const gfx::Size& frame_size) override { const gfx::Size buffer_size = gmb_->GetSize(); gmb_->Map(); // Clear all the buffer to 0. memset(data(), 0, bytes_per_row() * buffer_size.height()); SkBitmap bitmap; // Create an `SkImageInfo` with color type `kBGRA_8888_SkColorType` which // matches the buffer format `BGRA_8888` of the `gmb_`. This `info` will be // used to read the pixels from `bitmap` into the buffer. const auto info = SkImageInfo::Make(frame_size.width(), frame_size.height(), kBGRA_8888_SkColorType, kPremul_SkAlphaType); bitmap.setInfo(info); bitmap.setPixels(data()); DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size); gmb_->Unmap(); } private: std::unique_ptr gmb_; }; // ----------------------------------------------------------------------------- // SharedMemoryBufferStrategy: // Defines a concrete implementation of `BufferStrategy` which creates an // `UnsafeSharedMemoryRegion` and implements all the operations on it. class SharedMemoryBufferStrategy : public BufferStrategy { public: explicit SharedMemoryBufferStrategy(const gfx::Size& frame_size) : region_(base::UnsafeSharedMemoryRegion::Create( media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_ARGB, frame_size))) { DCHECK(region_.IsValid()); } // BufferStrategy: media::mojom::VideoBufferHandlePtr GetHandle() const override { return media::mojom::VideoBufferHandle::NewUnsafeShmemRegion( region_.Duplicate()); } void DrawFrameOnBuffer(const gfx::Size& frame_size) override { if (!mapping_.IsValid()) mapping_ = region_.Map(); DCHECK(mapping_.IsValid()); uint8_t* buffer_ptr = mapping_.GetMemoryAsSpan().data(); const int buffer_size = mapping_.size(); memset(buffer_ptr, 0, buffer_size); SkBitmap bitmap; bitmap.setInfo( SkImageInfo::MakeN32Premul(frame_size.width(), frame_size.height())); bitmap.setPixels(buffer_ptr); DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size); } private: base::UnsafeSharedMemoryRegion region_; base::WritableSharedMemoryMapping mapping_; }; } // namespace // ----------------------------------------------------------------------------- // FakeCameraDevice::Buffer: class FakeCameraDevice::Buffer { public: Buffer(const Buffer&) = delete; Buffer& operator=(const Buffer&) = delete; ~Buffer() = default; // Creates an instance of this object with the given `buffer_id`. The actual // underlying buffer will depend on the given `buffer_type` and will be big // enough to hold the content of a video frame of size `frame_size`. static std::unique_ptr Create( int buffer_id, media::VideoCaptureBufferType buffer_type, const gfx::Size& frame_size) { switch (buffer_type) { case media::VideoCaptureBufferType::kSharedMemory: return base::WrapUnique(new Buffer( buffer_id, buffer_type, frame_size, std::make_unique(frame_size))); case media::VideoCaptureBufferType::kGpuMemoryBuffer: return base::WrapUnique( new Buffer(buffer_id, buffer_type, frame_size, std::make_unique(frame_size))); default: NOTREACHED(); return nullptr; } } int buffer_id() const { return buffer_id_; } media::VideoCaptureBufferType buffer_type() const { return buffer_type_; } const gfx::Size& frame_size() const { return frame_size_; } void set_is_in_use(bool value) { is_in_use_ = value; } // Returns true if this buffer is not already in use, and can be reused for // the given `buffer_type` and the given `frame_size`. bool CanBeReused(media::VideoCaptureBufferType buffer_type, const gfx::Size& frame_size) const { return !is_in_use_ && buffer_type_ == buffer_type && frame_size_ == frame_size; } // Returns a handle for the underlying buffer that can be sent via mojo. media::mojom::VideoBufferHandlePtr GetHandle() const { return buffer_strategy_->GetHandle(); } // Draw the content of the video frame on the underlying buffer. void DrawFrame() { // buffer_strategy_->CopyBitmapToBuffer( // FakeCameraDevice::GetProducedFrameAsBitmap(frame_size_)); buffer_strategy_->DrawFrameOnBuffer(frame_size_); } private: Buffer(int buffer_id, media::VideoCaptureBufferType buffer_type, const gfx::Size& frame_size, std::unique_ptr strategy) : buffer_id_(buffer_id), buffer_type_(buffer_type), frame_size_(frame_size), buffer_strategy_(std::move(strategy)) {} // The ID of this buffer. const int buffer_id_; // The type of this buffer. Only `kSharedMemory`, and `kGpuMemoryBuffer` are // supported. const media::VideoCaptureBufferType buffer_type_; // The size of the video frame this buffer can hold. const gfx::Size frame_size_; // The strategy object that holds the underlying buffer. const std::unique_ptr buffer_strategy_; // True if this buffer is still in use by the subscriber. bool is_in_use_ = false; }; // ----------------------------------------------------------------------------- // FakeCameraDevice::Subscription: // A fake implementation of the `PushVideoStreamSubscription` mojo API which // is created for a remote implementation of `VideoFrameHandler` that subscribes // to the camera device. class FakeCameraDevice::Subscription : public video_capture::mojom::PushVideoStreamSubscription { public: Subscription( FakeCameraDevice* owner_device, mojo::PendingReceiver subscription, mojo::PendingRemote subscriber) : owner_device_(owner_device), receiver_(this, std::move(subscription)), subscriber_(std::move(subscriber)) { subscriber_.set_disconnect_handler(base::BindOnce( &Subscription::OnSubscriberDisconnected, base::Unretained(this))); } bool is_active() const { return is_active_; } bool is_suspended() const { return is_suspended_; } void OnNewBuffer(int buffer_id, media::mojom::VideoBufferHandlePtr buffer_handle) { DCHECK(is_active_); subscriber_->OnNewBuffer(buffer_id, std::move(buffer_handle)); } void OnBufferRetired(int buffer_id) { DCHECK(is_active_); subscriber_->OnBufferRetired(buffer_id); } void OnFrameReadyInBuffer( video_capture::mojom::ReadyFrameInBufferPtr buffer) { DCHECK(is_active_ && !is_suspended_); subscriber_->OnFrameReadyInBuffer(std::move(buffer), {}); } void OnFrameDropped() { DCHECK(is_active_ && !is_suspended_); subscriber_->OnFrameDropped( media::VideoCaptureFrameDropReason::kBufferPoolMaxBufferCountExceeded); } void TriggerFatalError() { subscriber_->OnError( media::VideoCaptureError::kCrosHalV3DeviceDelegateMojoConnectionError); } // video_capture::mojom::PushVideoStreamSubscription: void Activate() override { is_active_ = true; DCHECK(subscriber_); subscriber_->OnFrameAccessHandlerReady( owner_device_->GetNewAccessHandlerRemote()); owner_device_->OnSubscriptionActivationChanged(this); } void Suspend(SuspendCallback callback) override { is_suspended_ = true; } void Resume() override { is_suspended_ = false; } void GetPhotoState(GetPhotoStateCallback callback) override {} void SetPhotoOptions(media::mojom::PhotoSettingsPtr settings, SetPhotoOptionsCallback callback) override {} void TakePhoto(TakePhotoCallback callback) override {} void Close(CloseCallback callback) override { is_active_ = false; owner_device_->OnSubscriptionActivationChanged(this); } void ProcessFeedback(const media::VideoCaptureFeedback& feedback) override {} private: void OnSubscriberDisconnected() { owner_device_->OnSubscriberDisconnected(this); // `this` is deleted at this point. } // The camera device which owns this object. FakeCameraDevice* const owner_device_; mojo::Receiver receiver_{ this}; // The remote subscriber which implements the `VideoFrameHandler` mojo API. mojo::Remote subscriber_; // True when this subscription is active. Active subscriptions always produce // `OnNewBuffer()` and `OnBufferRetired()` calls regardless of the state of // `is_suspended_`. bool is_active_ = false; // True if this subscription is suspended. Suspended subscriptions don't // produce `OnFrameReadyInBuffer()` nor `OnFrameDropped()` calls. bool is_suspended_ = false; }; // ----------------------------------------------------------------------------- // FakeCameraDevice: FakeCameraDevice::FakeCameraDevice( const media::VideoCaptureDeviceInfo& device_info) : device_info_(device_info) {} FakeCameraDevice::~FakeCameraDevice() = default; // static SkBitmap FakeCameraDevice::GetProducedFrameAsBitmap( const gfx::Size& frame_size) { SkBitmap bitmap; bitmap.allocN32Pixels(frame_size.width(), frame_size.height()); DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size); return bitmap; } void FakeCameraDevice::Bind( mojo::PendingReceiver pending_receiver) { video_source_receiver_set_.Add(this, std::move(pending_receiver)); } void FakeCameraDevice::TriggerFatalError() { for (auto& pair : subscriptions_map_) { auto* subscription = pair.first; subscription->TriggerFatalError(); } } void FakeCameraDevice::CreatePushSubscription( mojo::PendingRemote subscriber, const media::VideoCaptureParams& requested_settings, bool force_reopen_with_new_settings, mojo::PendingReceiver subscription, CreatePushSubscriptionCallback callback) { auto new_subscription = std::make_unique( this, std::move(subscription), std::move(subscriber)); auto* new_subscription_ptr = new_subscription.get(); subscriptions_map_.emplace(new_subscription_ptr, std::move(new_subscription)); DCHECK(requested_settings.IsValid()); const bool has_current_settings = current_settings_.has_value(); if (force_reopen_with_new_settings || !has_current_settings) { RetireAllBuffers(); current_settings_.emplace(requested_settings); OnSubscriptionActivationChanged(/*subscription=*/nullptr); } DCHECK(callback); std::move(callback).Run( video_capture::mojom::CreatePushSubscriptionResultCode::NewSuccessCode( video_capture::mojom::CreatePushSubscriptionSuccessCode:: kCreatedWithRequestedSettings), requested_settings); } void FakeCameraDevice::OnFinishedConsumingBuffer(int32_t buffer_id) { auto iter = buffer_pool_.find(buffer_id); if (iter != buffer_pool_.end()) iter->second->set_is_in_use(false); } mojo::PendingRemote FakeCameraDevice::GetNewAccessHandlerRemote() { mojo::PendingReceiver receiver; auto remote = receiver.InitWithNewPipeAndPassRemote(); access_handler_receiver_set_.Add(this, std::move(receiver)); return remote; } void FakeCameraDevice::OnSubscriberDisconnected(Subscription* subscription) { DCHECK(subscriptions_map_.contains(subscription)); subscriptions_map_.erase(subscription); if (subscriptions_map_.empty()) { frame_timer_.Stop(); current_settings_.reset(); RetireAllBuffers(); } } void FakeCameraDevice::OnSubscriptionActivationChanged( Subscription* subscription) { DCHECK(current_settings_.has_value()); // Newly activated subscriptions should be told about all existing buffers. if (subscription && subscription->is_active()) { for (auto& pair : buffer_pool_) { Buffer* buffer = pair.second.get(); subscription->OnNewBuffer(buffer->buffer_id(), buffer->GetHandle()); } } const auto frame_duration = base::Hertz(current_settings_->requested_format.frame_rate); if (IsAnySubscriptionActive()) { if (!frame_timer_.IsRunning() || frame_timer_.GetCurrentDelay() != frame_duration) { frame_timer_.Start(FROM_HERE, frame_duration, this, &FakeCameraDevice::OnNextFrame); } } else { frame_timer_.Stop(); } } bool FakeCameraDevice::IsAnySubscriptionActive() const { for (const auto& pair : subscriptions_map_) { if (pair.first->is_active()) return true; } return false; } void FakeCameraDevice::OnNextFrame() { DCHECK(IsAnySubscriptionActive()); DCHECK(current_settings_.has_value()); if (start_time_.is_null()) start_time_ = base::Time::Now(); auto* buffer = AllocateOrReuseBuffer(); if (!buffer) return; buffer->DrawFrame(); for (auto& pair : subscriptions_map_) { auto* subscription = pair.first; if (!subscription->is_active() || subscription->is_suspended()) return; media::mojom::VideoFrameInfoPtr info = media::mojom::VideoFrameInfo::New(); info->timestamp = base::Time::Now() - start_time_; info->pixel_format = media::PIXEL_FORMAT_ARGB; info->coded_size = current_settings_->requested_format.frame_size; info->visible_rect = gfx::Rect(info->coded_size); info->is_premapped = false; info->color_space = gfx::ColorSpace(); subscription->OnFrameReadyInBuffer( video_capture::mojom::ReadyFrameInBuffer::New(buffer->buffer_id(), /*frame_feedback_id=*/0, std::move(info))); } } FakeCameraDevice::Buffer* FakeCameraDevice::AllocateOrReuseBuffer() { DCHECK(current_settings_.has_value()); const auto buffer_type = current_settings_->buffer_type; const auto frame_size = current_settings_->requested_format.frame_size; // Can we reuse an existing buffer? for (auto& pair : buffer_pool_) { Buffer* buffer = pair.second.get(); if (buffer->CanBeReused(buffer_type, frame_size)) { buffer->set_is_in_use(true); return buffer; } } if (buffer_pool_.size() >= FakeCameraDevice::kMaxBufferCount) { for (auto& pair : subscriptions_map_) { auto* subscription = pair.first; if (subscription->is_active() && !subscription->is_suspended()) subscription->OnFrameDropped(); } return nullptr; } const int buffer_id = g_next_buffer_id++; auto unique_buffer = Buffer::Create(buffer_id, buffer_type, frame_size); Buffer* buffer_ptr = unique_buffer.get(); buffer_ptr->set_is_in_use(true); buffer_pool_.emplace(buffer_id, std::move(unique_buffer)); for (auto& pair : subscriptions_map_) { auto* subscription = pair.first; if (subscription->is_active()) subscription->OnNewBuffer(buffer_id, buffer_ptr->GetHandle()); } return buffer_ptr; } void FakeCameraDevice::RetireAllBuffers() { for (auto& pair : buffer_pool_) { const int buffer_id = pair.first; for (auto& pair : subscriptions_map_) { auto* subscription = pair.first; if (subscription->is_active()) subscription->OnBufferRetired(buffer_id); } } buffer_pool_.clear(); } } // namespace ash