123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557 |
- // Copyright 2022 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "ash/capture_mode/fake_camera_device.h"
- #include <cstring>
- #include <memory>
- #include "base/bind.h"
- #include "base/callback_helpers.h"
- #include "base/location.h"
- #include "base/memory/ptr_util.h"
- #include "base/memory/shared_memory_mapping.h"
- #include "base/memory/unsafe_shared_memory_region.h"
- #include "base/notreached.h"
- #include "cc/paint/paint_flags.h"
- #include "cc/paint/skia_paint_canvas.h"
- #include "gpu/command_buffer/client/gpu_memory_buffer_manager.h"
- #include "gpu/ipc/common/surface_handle.h"
- #include "media/base/video_frame.h"
- #include "media/base/video_types.h"
- #include "media/capture/mojom/video_capture_buffer.mojom.h"
- #include "media/capture/video_capture_types.h"
- #include "mojo/public/cpp/bindings/receiver.h"
- #include "mojo/public/cpp/bindings/remote.h"
- #include "services/video_capture/public/mojom/video_frame_handler.mojom.h"
- #include "third_party/skia/include/core/SkRect.h"
- #include "ui/aura/env.h"
- #include "ui/compositor/compositor.h"
- #include "ui/gfx/gpu_memory_buffer.h"
- namespace ash {
- namespace {
- // The next ID to be used for a newly created buffer.
- int g_next_buffer_id = 0;
- std::unique_ptr<gfx::GpuMemoryBuffer> CreateGpuMemoryBuffer(
- const gfx::Size& frame_size) {
- return aura::Env::GetInstance()
- ->context_factory()
- ->GetGpuMemoryBufferManager()
- ->CreateGpuMemoryBuffer(frame_size, gfx::BufferFormat::BGRA_8888,
- gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
- gpu::kNullSurfaceHandle,
- /*shutdown_event=*/nullptr);
- }
- SkRect GetCircleRect(const gfx::Point& center, int radius) {
- return SkRect::MakeLTRB(center.x() - radius, center.y() - radius,
- center.x() + radius, center.y() + radius);
- }
- // Draws the content of the produced video frame whose size is the given
- // `frame_size` on the given `canvas`.
- void DrawFrameOnCanvas(cc::SkiaPaintCanvas&& canvas,
- const gfx::Size& frame_size) {
- cc::PaintFlags flags;
- flags.setStyle(cc::PaintFlags::kFill_Style);
- const auto canvas_bounds =
- SkRect::MakeIWH(frame_size.width(), frame_size.height());
- flags.setColor(SK_ColorBLUE);
- canvas.drawRect(canvas_bounds, flags);
- flags.setColor(SK_ColorGREEN);
- const auto canvas_center = gfx::Rect(frame_size).CenterPoint();
- const int radius = 20;
- canvas.drawOval(GetCircleRect(canvas_center, radius), flags);
- flags.setColor(SK_ColorRED);
- canvas.drawOval(GetCircleRect(gfx::Point(), radius), flags);
- }
- // -----------------------------------------------------------------------------
- // BufferStrategy:
- // Defines an interface for operations that can be done on different buffer
- // types.
- class BufferStrategy {
- public:
- virtual ~BufferStrategy() = default;
- // Gets the handle of the concrete buffer implementation that can be sent over
- // via mojo.
- virtual media::mojom::VideoBufferHandlePtr GetHandle() const = 0;
- // Draws the contents of a video frame whose size is `frame_size` on the
- // buffer backing this object.
- virtual void DrawFrameOnBuffer(const gfx::Size& frame_size) = 0;
- };
- // -----------------------------------------------------------------------------
- // GpuMemoryBufferStrategy:
- // Defines a concrete implementation of `BufferStrategy` which creates a
- // `GpuMemoryBuffer` and implements all the operations on it.
- class GpuMemoryBufferStrategy : public BufferStrategy {
- public:
- explicit GpuMemoryBufferStrategy(const gfx::Size& frame_size)
- : gmb_(CreateGpuMemoryBuffer(frame_size)) {
- DCHECK(gmb_);
- }
- uint8_t* data() { return static_cast<uint8_t*>(gmb_->memory(0)); }
- size_t bytes_per_row() { return gmb_->stride(0); }
- // BufferStrategy:
- media::mojom::VideoBufferHandlePtr GetHandle() const override {
- return media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
- gmb_->CloneHandle());
- }
- void DrawFrameOnBuffer(const gfx::Size& frame_size) override {
- const gfx::Size buffer_size = gmb_->GetSize();
- gmb_->Map();
- // Clear all the buffer to 0.
- memset(data(), 0, bytes_per_row() * buffer_size.height());
- SkBitmap bitmap;
- // Create an `SkImageInfo` with color type `kBGRA_8888_SkColorType` which
- // matches the buffer format `BGRA_8888` of the `gmb_`. This `info` will be
- // used to read the pixels from `bitmap` into the buffer.
- const auto info =
- SkImageInfo::Make(frame_size.width(), frame_size.height(),
- kBGRA_8888_SkColorType, kPremul_SkAlphaType);
- bitmap.setInfo(info);
- bitmap.setPixels(data());
- DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
- gmb_->Unmap();
- }
- private:
- std::unique_ptr<gfx::GpuMemoryBuffer> gmb_;
- };
- // -----------------------------------------------------------------------------
- // SharedMemoryBufferStrategy:
- // Defines a concrete implementation of `BufferStrategy` which creates an
- // `UnsafeSharedMemoryRegion` and implements all the operations on it.
- class SharedMemoryBufferStrategy : public BufferStrategy {
- public:
- explicit SharedMemoryBufferStrategy(const gfx::Size& frame_size)
- : region_(base::UnsafeSharedMemoryRegion::Create(
- media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_ARGB,
- frame_size))) {
- DCHECK(region_.IsValid());
- }
- // BufferStrategy:
- media::mojom::VideoBufferHandlePtr GetHandle() const override {
- return media::mojom::VideoBufferHandle::NewUnsafeShmemRegion(
- region_.Duplicate());
- }
- void DrawFrameOnBuffer(const gfx::Size& frame_size) override {
- if (!mapping_.IsValid())
- mapping_ = region_.Map();
- DCHECK(mapping_.IsValid());
- uint8_t* buffer_ptr = mapping_.GetMemoryAsSpan<uint8_t>().data();
- const int buffer_size = mapping_.size();
- memset(buffer_ptr, 0, buffer_size);
- SkBitmap bitmap;
- bitmap.setInfo(
- SkImageInfo::MakeN32Premul(frame_size.width(), frame_size.height()));
- bitmap.setPixels(buffer_ptr);
- DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
- }
- private:
- base::UnsafeSharedMemoryRegion region_;
- base::WritableSharedMemoryMapping mapping_;
- };
- } // namespace
- // -----------------------------------------------------------------------------
- // FakeCameraDevice::Buffer:
- class FakeCameraDevice::Buffer {
- public:
- Buffer(const Buffer&) = delete;
- Buffer& operator=(const Buffer&) = delete;
- ~Buffer() = default;
- // Creates an instance of this object with the given `buffer_id`. The actual
- // underlying buffer will depend on the given `buffer_type` and will be big
- // enough to hold the content of a video frame of size `frame_size`.
- static std::unique_ptr<FakeCameraDevice::Buffer> Create(
- int buffer_id,
- media::VideoCaptureBufferType buffer_type,
- const gfx::Size& frame_size) {
- switch (buffer_type) {
- case media::VideoCaptureBufferType::kSharedMemory:
- return base::WrapUnique(new Buffer(
- buffer_id, buffer_type, frame_size,
- std::make_unique<SharedMemoryBufferStrategy>(frame_size)));
- case media::VideoCaptureBufferType::kGpuMemoryBuffer:
- return base::WrapUnique(
- new Buffer(buffer_id, buffer_type, frame_size,
- std::make_unique<GpuMemoryBufferStrategy>(frame_size)));
- default:
- NOTREACHED();
- return nullptr;
- }
- }
- int buffer_id() const { return buffer_id_; }
- media::VideoCaptureBufferType buffer_type() const { return buffer_type_; }
- const gfx::Size& frame_size() const { return frame_size_; }
- void set_is_in_use(bool value) { is_in_use_ = value; }
- // Returns true if this buffer is not already in use, and can be reused for
- // the given `buffer_type` and the given `frame_size`.
- bool CanBeReused(media::VideoCaptureBufferType buffer_type,
- const gfx::Size& frame_size) const {
- return !is_in_use_ && buffer_type_ == buffer_type &&
- frame_size_ == frame_size;
- }
- // Returns a handle for the underlying buffer that can be sent via mojo.
- media::mojom::VideoBufferHandlePtr GetHandle() const {
- return buffer_strategy_->GetHandle();
- }
- // Draw the content of the video frame on the underlying buffer.
- void DrawFrame() {
- // buffer_strategy_->CopyBitmapToBuffer(
- // FakeCameraDevice::GetProducedFrameAsBitmap(frame_size_));
- buffer_strategy_->DrawFrameOnBuffer(frame_size_);
- }
- private:
- Buffer(int buffer_id,
- media::VideoCaptureBufferType buffer_type,
- const gfx::Size& frame_size,
- std::unique_ptr<BufferStrategy> strategy)
- : buffer_id_(buffer_id),
- buffer_type_(buffer_type),
- frame_size_(frame_size),
- buffer_strategy_(std::move(strategy)) {}
- // The ID of this buffer.
- const int buffer_id_;
- // The type of this buffer. Only `kSharedMemory`, and `kGpuMemoryBuffer` are
- // supported.
- const media::VideoCaptureBufferType buffer_type_;
- // The size of the video frame this buffer can hold.
- const gfx::Size frame_size_;
- // The strategy object that holds the underlying buffer.
- const std::unique_ptr<BufferStrategy> buffer_strategy_;
- // True if this buffer is still in use by the subscriber.
- bool is_in_use_ = false;
- };
- // -----------------------------------------------------------------------------
- // FakeCameraDevice::Subscription:
- // A fake implementation of the `PushVideoStreamSubscription` mojo API which
- // is created for a remote implementation of `VideoFrameHandler` that subscribes
- // to the camera device.
- class FakeCameraDevice::Subscription
- : public video_capture::mojom::PushVideoStreamSubscription {
- public:
- Subscription(
- FakeCameraDevice* owner_device,
- mojo::PendingReceiver<video_capture::mojom::PushVideoStreamSubscription>
- subscription,
- mojo::PendingRemote<video_capture::mojom::VideoFrameHandler> subscriber)
- : owner_device_(owner_device),
- receiver_(this, std::move(subscription)),
- subscriber_(std::move(subscriber)) {
- subscriber_.set_disconnect_handler(base::BindOnce(
- &Subscription::OnSubscriberDisconnected, base::Unretained(this)));
- }
- bool is_active() const { return is_active_; }
- bool is_suspended() const { return is_suspended_; }
- void OnNewBuffer(int buffer_id,
- media::mojom::VideoBufferHandlePtr buffer_handle) {
- DCHECK(is_active_);
- subscriber_->OnNewBuffer(buffer_id, std::move(buffer_handle));
- }
- void OnBufferRetired(int buffer_id) {
- DCHECK(is_active_);
- subscriber_->OnBufferRetired(buffer_id);
- }
- void OnFrameReadyInBuffer(
- video_capture::mojom::ReadyFrameInBufferPtr buffer) {
- DCHECK(is_active_ && !is_suspended_);
- subscriber_->OnFrameReadyInBuffer(std::move(buffer), {});
- }
- void OnFrameDropped() {
- DCHECK(is_active_ && !is_suspended_);
- subscriber_->OnFrameDropped(
- media::VideoCaptureFrameDropReason::kBufferPoolMaxBufferCountExceeded);
- }
- void TriggerFatalError() {
- subscriber_->OnError(
- media::VideoCaptureError::kCrosHalV3DeviceDelegateMojoConnectionError);
- }
- // video_capture::mojom::PushVideoStreamSubscription:
- void Activate() override {
- is_active_ = true;
- DCHECK(subscriber_);
- subscriber_->OnFrameAccessHandlerReady(
- owner_device_->GetNewAccessHandlerRemote());
- owner_device_->OnSubscriptionActivationChanged(this);
- }
- void Suspend(SuspendCallback callback) override { is_suspended_ = true; }
- void Resume() override { is_suspended_ = false; }
- void GetPhotoState(GetPhotoStateCallback callback) override {}
- void SetPhotoOptions(media::mojom::PhotoSettingsPtr settings,
- SetPhotoOptionsCallback callback) override {}
- void TakePhoto(TakePhotoCallback callback) override {}
- void Close(CloseCallback callback) override {
- is_active_ = false;
- owner_device_->OnSubscriptionActivationChanged(this);
- }
- void ProcessFeedback(const media::VideoCaptureFeedback& feedback) override {}
- private:
- void OnSubscriberDisconnected() {
- owner_device_->OnSubscriberDisconnected(this);
- // `this` is deleted at this point.
- }
- // The camera device which owns this object.
- FakeCameraDevice* const owner_device_;
- mojo::Receiver<video_capture::mojom::PushVideoStreamSubscription> receiver_{
- this};
- // The remote subscriber which implements the `VideoFrameHandler` mojo API.
- mojo::Remote<video_capture::mojom::VideoFrameHandler> subscriber_;
- // True when this subscription is active. Active subscriptions always produce
- // `OnNewBuffer()` and `OnBufferRetired()` calls regardless of the state of
- // `is_suspended_`.
- bool is_active_ = false;
- // True if this subscription is suspended. Suspended subscriptions don't
- // produce `OnFrameReadyInBuffer()` nor `OnFrameDropped()` calls.
- bool is_suspended_ = false;
- };
- // -----------------------------------------------------------------------------
- // FakeCameraDevice:
- FakeCameraDevice::FakeCameraDevice(
- const media::VideoCaptureDeviceInfo& device_info)
- : device_info_(device_info) {}
- FakeCameraDevice::~FakeCameraDevice() = default;
- // static
- SkBitmap FakeCameraDevice::GetProducedFrameAsBitmap(
- const gfx::Size& frame_size) {
- SkBitmap bitmap;
- bitmap.allocN32Pixels(frame_size.width(), frame_size.height());
- DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
- return bitmap;
- }
- void FakeCameraDevice::Bind(
- mojo::PendingReceiver<video_capture::mojom::VideoSource> pending_receiver) {
- video_source_receiver_set_.Add(this, std::move(pending_receiver));
- }
- void FakeCameraDevice::TriggerFatalError() {
- for (auto& pair : subscriptions_map_) {
- auto* subscription = pair.first;
- subscription->TriggerFatalError();
- }
- }
- void FakeCameraDevice::CreatePushSubscription(
- mojo::PendingRemote<video_capture::mojom::VideoFrameHandler> subscriber,
- const media::VideoCaptureParams& requested_settings,
- bool force_reopen_with_new_settings,
- mojo::PendingReceiver<video_capture::mojom::PushVideoStreamSubscription>
- subscription,
- CreatePushSubscriptionCallback callback) {
- auto new_subscription = std::make_unique<FakeCameraDevice::Subscription>(
- this, std::move(subscription), std::move(subscriber));
- auto* new_subscription_ptr = new_subscription.get();
- subscriptions_map_.emplace(new_subscription_ptr, std::move(new_subscription));
- DCHECK(requested_settings.IsValid());
- const bool has_current_settings = current_settings_.has_value();
- if (force_reopen_with_new_settings || !has_current_settings) {
- RetireAllBuffers();
- current_settings_.emplace(requested_settings);
- OnSubscriptionActivationChanged(/*subscription=*/nullptr);
- }
- DCHECK(callback);
- std::move(callback).Run(
- video_capture::mojom::CreatePushSubscriptionResultCode::NewSuccessCode(
- video_capture::mojom::CreatePushSubscriptionSuccessCode::
- kCreatedWithRequestedSettings),
- requested_settings);
- }
- void FakeCameraDevice::OnFinishedConsumingBuffer(int32_t buffer_id) {
- auto iter = buffer_pool_.find(buffer_id);
- if (iter != buffer_pool_.end())
- iter->second->set_is_in_use(false);
- }
- mojo::PendingRemote<video_capture::mojom::VideoFrameAccessHandler>
- FakeCameraDevice::GetNewAccessHandlerRemote() {
- mojo::PendingReceiver<video_capture::mojom::VideoFrameAccessHandler> receiver;
- auto remote = receiver.InitWithNewPipeAndPassRemote();
- access_handler_receiver_set_.Add(this, std::move(receiver));
- return remote;
- }
- void FakeCameraDevice::OnSubscriberDisconnected(Subscription* subscription) {
- DCHECK(subscriptions_map_.contains(subscription));
- subscriptions_map_.erase(subscription);
- if (subscriptions_map_.empty()) {
- frame_timer_.Stop();
- current_settings_.reset();
- RetireAllBuffers();
- }
- }
- void FakeCameraDevice::OnSubscriptionActivationChanged(
- Subscription* subscription) {
- DCHECK(current_settings_.has_value());
- // Newly activated subscriptions should be told about all existing buffers.
- if (subscription && subscription->is_active()) {
- for (auto& pair : buffer_pool_) {
- Buffer* buffer = pair.second.get();
- subscription->OnNewBuffer(buffer->buffer_id(), buffer->GetHandle());
- }
- }
- const auto frame_duration =
- base::Hertz(current_settings_->requested_format.frame_rate);
- if (IsAnySubscriptionActive()) {
- if (!frame_timer_.IsRunning() ||
- frame_timer_.GetCurrentDelay() != frame_duration) {
- frame_timer_.Start(FROM_HERE, frame_duration, this,
- &FakeCameraDevice::OnNextFrame);
- }
- } else {
- frame_timer_.Stop();
- }
- }
- bool FakeCameraDevice::IsAnySubscriptionActive() const {
- for (const auto& pair : subscriptions_map_) {
- if (pair.first->is_active())
- return true;
- }
- return false;
- }
- void FakeCameraDevice::OnNextFrame() {
- DCHECK(IsAnySubscriptionActive());
- DCHECK(current_settings_.has_value());
- if (start_time_.is_null())
- start_time_ = base::Time::Now();
- auto* buffer = AllocateOrReuseBuffer();
- if (!buffer)
- return;
- buffer->DrawFrame();
- for (auto& pair : subscriptions_map_) {
- auto* subscription = pair.first;
- if (!subscription->is_active() || subscription->is_suspended())
- return;
- media::mojom::VideoFrameInfoPtr info = media::mojom::VideoFrameInfo::New();
- info->timestamp = base::Time::Now() - start_time_;
- info->pixel_format = media::PIXEL_FORMAT_ARGB;
- info->coded_size = current_settings_->requested_format.frame_size;
- info->visible_rect = gfx::Rect(info->coded_size);
- info->is_premapped = false;
- info->color_space = gfx::ColorSpace();
- subscription->OnFrameReadyInBuffer(
- video_capture::mojom::ReadyFrameInBuffer::New(buffer->buffer_id(),
- /*frame_feedback_id=*/0,
- std::move(info)));
- }
- }
- FakeCameraDevice::Buffer* FakeCameraDevice::AllocateOrReuseBuffer() {
- DCHECK(current_settings_.has_value());
- const auto buffer_type = current_settings_->buffer_type;
- const auto frame_size = current_settings_->requested_format.frame_size;
- // Can we reuse an existing buffer?
- for (auto& pair : buffer_pool_) {
- Buffer* buffer = pair.second.get();
- if (buffer->CanBeReused(buffer_type, frame_size)) {
- buffer->set_is_in_use(true);
- return buffer;
- }
- }
- if (buffer_pool_.size() >= FakeCameraDevice::kMaxBufferCount) {
- for (auto& pair : subscriptions_map_) {
- auto* subscription = pair.first;
- if (subscription->is_active() && !subscription->is_suspended())
- subscription->OnFrameDropped();
- }
- return nullptr;
- }
- const int buffer_id = g_next_buffer_id++;
- auto unique_buffer = Buffer::Create(buffer_id, buffer_type, frame_size);
- Buffer* buffer_ptr = unique_buffer.get();
- buffer_ptr->set_is_in_use(true);
- buffer_pool_.emplace(buffer_id, std::move(unique_buffer));
- for (auto& pair : subscriptions_map_) {
- auto* subscription = pair.first;
- if (subscription->is_active())
- subscription->OnNewBuffer(buffer_id, buffer_ptr->GetHandle());
- }
- return buffer_ptr;
- }
- void FakeCameraDevice::RetireAllBuffers() {
- for (auto& pair : buffer_pool_) {
- const int buffer_id = pair.first;
- for (auto& pair : subscriptions_map_) {
- auto* subscription = pair.first;
- if (subscription->is_active())
- subscription->OnBufferRetired(buffer_id);
- }
- }
- buffer_pool_.clear();
- }
- } // namespace ash
|