fake_camera_device.cc 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. // Copyright 2022 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/capture_mode/fake_camera_device.h"
  5. #include <cstring>
  6. #include <memory>
  7. #include "base/bind.h"
  8. #include "base/callback_helpers.h"
  9. #include "base/location.h"
  10. #include "base/memory/ptr_util.h"
  11. #include "base/memory/shared_memory_mapping.h"
  12. #include "base/memory/unsafe_shared_memory_region.h"
  13. #include "base/notreached.h"
  14. #include "cc/paint/paint_flags.h"
  15. #include "cc/paint/skia_paint_canvas.h"
  16. #include "gpu/command_buffer/client/gpu_memory_buffer_manager.h"
  17. #include "gpu/ipc/common/surface_handle.h"
  18. #include "media/base/video_frame.h"
  19. #include "media/base/video_types.h"
  20. #include "media/capture/mojom/video_capture_buffer.mojom.h"
  21. #include "media/capture/video_capture_types.h"
  22. #include "mojo/public/cpp/bindings/receiver.h"
  23. #include "mojo/public/cpp/bindings/remote.h"
  24. #include "services/video_capture/public/mojom/video_frame_handler.mojom.h"
  25. #include "third_party/skia/include/core/SkRect.h"
  26. #include "ui/aura/env.h"
  27. #include "ui/compositor/compositor.h"
  28. #include "ui/gfx/gpu_memory_buffer.h"
  29. namespace ash {
  30. namespace {
  31. // The next ID to be used for a newly created buffer.
  32. int g_next_buffer_id = 0;
  33. std::unique_ptr<gfx::GpuMemoryBuffer> CreateGpuMemoryBuffer(
  34. const gfx::Size& frame_size) {
  35. return aura::Env::GetInstance()
  36. ->context_factory()
  37. ->GetGpuMemoryBufferManager()
  38. ->CreateGpuMemoryBuffer(frame_size, gfx::BufferFormat::BGRA_8888,
  39. gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
  40. gpu::kNullSurfaceHandle,
  41. /*shutdown_event=*/nullptr);
  42. }
  43. SkRect GetCircleRect(const gfx::Point& center, int radius) {
  44. return SkRect::MakeLTRB(center.x() - radius, center.y() - radius,
  45. center.x() + radius, center.y() + radius);
  46. }
  47. // Draws the content of the produced video frame whose size is the given
  48. // `frame_size` on the given `canvas`.
  49. void DrawFrameOnCanvas(cc::SkiaPaintCanvas&& canvas,
  50. const gfx::Size& frame_size) {
  51. cc::PaintFlags flags;
  52. flags.setStyle(cc::PaintFlags::kFill_Style);
  53. const auto canvas_bounds =
  54. SkRect::MakeIWH(frame_size.width(), frame_size.height());
  55. flags.setColor(SK_ColorBLUE);
  56. canvas.drawRect(canvas_bounds, flags);
  57. flags.setColor(SK_ColorGREEN);
  58. const auto canvas_center = gfx::Rect(frame_size).CenterPoint();
  59. const int radius = 20;
  60. canvas.drawOval(GetCircleRect(canvas_center, radius), flags);
  61. flags.setColor(SK_ColorRED);
  62. canvas.drawOval(GetCircleRect(gfx::Point(), radius), flags);
  63. }
  64. // -----------------------------------------------------------------------------
  65. // BufferStrategy:
  66. // Defines an interface for operations that can be done on different buffer
  67. // types.
  68. class BufferStrategy {
  69. public:
  70. virtual ~BufferStrategy() = default;
  71. // Gets the handle of the concrete buffer implementation that can be sent over
  72. // via mojo.
  73. virtual media::mojom::VideoBufferHandlePtr GetHandle() const = 0;
  74. // Draws the contents of a video frame whose size is `frame_size` on the
  75. // buffer backing this object.
  76. virtual void DrawFrameOnBuffer(const gfx::Size& frame_size) = 0;
  77. };
  78. // -----------------------------------------------------------------------------
  79. // GpuMemoryBufferStrategy:
  80. // Defines a concrete implementation of `BufferStrategy` which creates a
  81. // `GpuMemoryBuffer` and implements all the operations on it.
  82. class GpuMemoryBufferStrategy : public BufferStrategy {
  83. public:
  84. explicit GpuMemoryBufferStrategy(const gfx::Size& frame_size)
  85. : gmb_(CreateGpuMemoryBuffer(frame_size)) {
  86. DCHECK(gmb_);
  87. }
  88. uint8_t* data() { return static_cast<uint8_t*>(gmb_->memory(0)); }
  89. size_t bytes_per_row() { return gmb_->stride(0); }
  90. // BufferStrategy:
  91. media::mojom::VideoBufferHandlePtr GetHandle() const override {
  92. return media::mojom::VideoBufferHandle::NewGpuMemoryBufferHandle(
  93. gmb_->CloneHandle());
  94. }
  95. void DrawFrameOnBuffer(const gfx::Size& frame_size) override {
  96. const gfx::Size buffer_size = gmb_->GetSize();
  97. gmb_->Map();
  98. // Clear all the buffer to 0.
  99. memset(data(), 0, bytes_per_row() * buffer_size.height());
  100. SkBitmap bitmap;
  101. // Create an `SkImageInfo` with color type `kBGRA_8888_SkColorType` which
  102. // matches the buffer format `BGRA_8888` of the `gmb_`. This `info` will be
  103. // used to read the pixels from `bitmap` into the buffer.
  104. const auto info =
  105. SkImageInfo::Make(frame_size.width(), frame_size.height(),
  106. kBGRA_8888_SkColorType, kPremul_SkAlphaType);
  107. bitmap.setInfo(info);
  108. bitmap.setPixels(data());
  109. DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
  110. gmb_->Unmap();
  111. }
  112. private:
  113. std::unique_ptr<gfx::GpuMemoryBuffer> gmb_;
  114. };
  115. // -----------------------------------------------------------------------------
  116. // SharedMemoryBufferStrategy:
  117. // Defines a concrete implementation of `BufferStrategy` which creates an
  118. // `UnsafeSharedMemoryRegion` and implements all the operations on it.
  119. class SharedMemoryBufferStrategy : public BufferStrategy {
  120. public:
  121. explicit SharedMemoryBufferStrategy(const gfx::Size& frame_size)
  122. : region_(base::UnsafeSharedMemoryRegion::Create(
  123. media::VideoFrame::AllocationSize(media::PIXEL_FORMAT_ARGB,
  124. frame_size))) {
  125. DCHECK(region_.IsValid());
  126. }
  127. // BufferStrategy:
  128. media::mojom::VideoBufferHandlePtr GetHandle() const override {
  129. return media::mojom::VideoBufferHandle::NewUnsafeShmemRegion(
  130. region_.Duplicate());
  131. }
  132. void DrawFrameOnBuffer(const gfx::Size& frame_size) override {
  133. if (!mapping_.IsValid())
  134. mapping_ = region_.Map();
  135. DCHECK(mapping_.IsValid());
  136. uint8_t* buffer_ptr = mapping_.GetMemoryAsSpan<uint8_t>().data();
  137. const int buffer_size = mapping_.size();
  138. memset(buffer_ptr, 0, buffer_size);
  139. SkBitmap bitmap;
  140. bitmap.setInfo(
  141. SkImageInfo::MakeN32Premul(frame_size.width(), frame_size.height()));
  142. bitmap.setPixels(buffer_ptr);
  143. DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
  144. }
  145. private:
  146. base::UnsafeSharedMemoryRegion region_;
  147. base::WritableSharedMemoryMapping mapping_;
  148. };
  149. } // namespace
  150. // -----------------------------------------------------------------------------
  151. // FakeCameraDevice::Buffer:
  152. class FakeCameraDevice::Buffer {
  153. public:
  154. Buffer(const Buffer&) = delete;
  155. Buffer& operator=(const Buffer&) = delete;
  156. ~Buffer() = default;
  157. // Creates an instance of this object with the given `buffer_id`. The actual
  158. // underlying buffer will depend on the given `buffer_type` and will be big
  159. // enough to hold the content of a video frame of size `frame_size`.
  160. static std::unique_ptr<FakeCameraDevice::Buffer> Create(
  161. int buffer_id,
  162. media::VideoCaptureBufferType buffer_type,
  163. const gfx::Size& frame_size) {
  164. switch (buffer_type) {
  165. case media::VideoCaptureBufferType::kSharedMemory:
  166. return base::WrapUnique(new Buffer(
  167. buffer_id, buffer_type, frame_size,
  168. std::make_unique<SharedMemoryBufferStrategy>(frame_size)));
  169. case media::VideoCaptureBufferType::kGpuMemoryBuffer:
  170. return base::WrapUnique(
  171. new Buffer(buffer_id, buffer_type, frame_size,
  172. std::make_unique<GpuMemoryBufferStrategy>(frame_size)));
  173. default:
  174. NOTREACHED();
  175. return nullptr;
  176. }
  177. }
  178. int buffer_id() const { return buffer_id_; }
  179. media::VideoCaptureBufferType buffer_type() const { return buffer_type_; }
  180. const gfx::Size& frame_size() const { return frame_size_; }
  181. void set_is_in_use(bool value) { is_in_use_ = value; }
  182. // Returns true if this buffer is not already in use, and can be reused for
  183. // the given `buffer_type` and the given `frame_size`.
  184. bool CanBeReused(media::VideoCaptureBufferType buffer_type,
  185. const gfx::Size& frame_size) const {
  186. return !is_in_use_ && buffer_type_ == buffer_type &&
  187. frame_size_ == frame_size;
  188. }
  189. // Returns a handle for the underlying buffer that can be sent via mojo.
  190. media::mojom::VideoBufferHandlePtr GetHandle() const {
  191. return buffer_strategy_->GetHandle();
  192. }
  193. // Draw the content of the video frame on the underlying buffer.
  194. void DrawFrame() {
  195. // buffer_strategy_->CopyBitmapToBuffer(
  196. // FakeCameraDevice::GetProducedFrameAsBitmap(frame_size_));
  197. buffer_strategy_->DrawFrameOnBuffer(frame_size_);
  198. }
  199. private:
  200. Buffer(int buffer_id,
  201. media::VideoCaptureBufferType buffer_type,
  202. const gfx::Size& frame_size,
  203. std::unique_ptr<BufferStrategy> strategy)
  204. : buffer_id_(buffer_id),
  205. buffer_type_(buffer_type),
  206. frame_size_(frame_size),
  207. buffer_strategy_(std::move(strategy)) {}
  208. // The ID of this buffer.
  209. const int buffer_id_;
  210. // The type of this buffer. Only `kSharedMemory`, and `kGpuMemoryBuffer` are
  211. // supported.
  212. const media::VideoCaptureBufferType buffer_type_;
  213. // The size of the video frame this buffer can hold.
  214. const gfx::Size frame_size_;
  215. // The strategy object that holds the underlying buffer.
  216. const std::unique_ptr<BufferStrategy> buffer_strategy_;
  217. // True if this buffer is still in use by the subscriber.
  218. bool is_in_use_ = false;
  219. };
  220. // -----------------------------------------------------------------------------
  221. // FakeCameraDevice::Subscription:
  222. // A fake implementation of the `PushVideoStreamSubscription` mojo API which
  223. // is created for a remote implementation of `VideoFrameHandler` that subscribes
  224. // to the camera device.
  225. class FakeCameraDevice::Subscription
  226. : public video_capture::mojom::PushVideoStreamSubscription {
  227. public:
  228. Subscription(
  229. FakeCameraDevice* owner_device,
  230. mojo::PendingReceiver<video_capture::mojom::PushVideoStreamSubscription>
  231. subscription,
  232. mojo::PendingRemote<video_capture::mojom::VideoFrameHandler> subscriber)
  233. : owner_device_(owner_device),
  234. receiver_(this, std::move(subscription)),
  235. subscriber_(std::move(subscriber)) {
  236. subscriber_.set_disconnect_handler(base::BindOnce(
  237. &Subscription::OnSubscriberDisconnected, base::Unretained(this)));
  238. }
  239. bool is_active() const { return is_active_; }
  240. bool is_suspended() const { return is_suspended_; }
  241. void OnNewBuffer(int buffer_id,
  242. media::mojom::VideoBufferHandlePtr buffer_handle) {
  243. DCHECK(is_active_);
  244. subscriber_->OnNewBuffer(buffer_id, std::move(buffer_handle));
  245. }
  246. void OnBufferRetired(int buffer_id) {
  247. DCHECK(is_active_);
  248. subscriber_->OnBufferRetired(buffer_id);
  249. }
  250. void OnFrameReadyInBuffer(
  251. video_capture::mojom::ReadyFrameInBufferPtr buffer) {
  252. DCHECK(is_active_ && !is_suspended_);
  253. subscriber_->OnFrameReadyInBuffer(std::move(buffer), {});
  254. }
  255. void OnFrameDropped() {
  256. DCHECK(is_active_ && !is_suspended_);
  257. subscriber_->OnFrameDropped(
  258. media::VideoCaptureFrameDropReason::kBufferPoolMaxBufferCountExceeded);
  259. }
  260. void TriggerFatalError() {
  261. subscriber_->OnError(
  262. media::VideoCaptureError::kCrosHalV3DeviceDelegateMojoConnectionError);
  263. }
  264. // video_capture::mojom::PushVideoStreamSubscription:
  265. void Activate() override {
  266. is_active_ = true;
  267. DCHECK(subscriber_);
  268. subscriber_->OnFrameAccessHandlerReady(
  269. owner_device_->GetNewAccessHandlerRemote());
  270. owner_device_->OnSubscriptionActivationChanged(this);
  271. }
  272. void Suspend(SuspendCallback callback) override { is_suspended_ = true; }
  273. void Resume() override { is_suspended_ = false; }
  274. void GetPhotoState(GetPhotoStateCallback callback) override {}
  275. void SetPhotoOptions(media::mojom::PhotoSettingsPtr settings,
  276. SetPhotoOptionsCallback callback) override {}
  277. void TakePhoto(TakePhotoCallback callback) override {}
  278. void Close(CloseCallback callback) override {
  279. is_active_ = false;
  280. owner_device_->OnSubscriptionActivationChanged(this);
  281. }
  282. void ProcessFeedback(const media::VideoCaptureFeedback& feedback) override {}
  283. private:
  284. void OnSubscriberDisconnected() {
  285. owner_device_->OnSubscriberDisconnected(this);
  286. // `this` is deleted at this point.
  287. }
  288. // The camera device which owns this object.
  289. FakeCameraDevice* const owner_device_;
  290. mojo::Receiver<video_capture::mojom::PushVideoStreamSubscription> receiver_{
  291. this};
  292. // The remote subscriber which implements the `VideoFrameHandler` mojo API.
  293. mojo::Remote<video_capture::mojom::VideoFrameHandler> subscriber_;
  294. // True when this subscription is active. Active subscriptions always produce
  295. // `OnNewBuffer()` and `OnBufferRetired()` calls regardless of the state of
  296. // `is_suspended_`.
  297. bool is_active_ = false;
  298. // True if this subscription is suspended. Suspended subscriptions don't
  299. // produce `OnFrameReadyInBuffer()` nor `OnFrameDropped()` calls.
  300. bool is_suspended_ = false;
  301. };
  302. // -----------------------------------------------------------------------------
  303. // FakeCameraDevice:
  304. FakeCameraDevice::FakeCameraDevice(
  305. const media::VideoCaptureDeviceInfo& device_info)
  306. : device_info_(device_info) {}
  307. FakeCameraDevice::~FakeCameraDevice() = default;
  308. // static
  309. SkBitmap FakeCameraDevice::GetProducedFrameAsBitmap(
  310. const gfx::Size& frame_size) {
  311. SkBitmap bitmap;
  312. bitmap.allocN32Pixels(frame_size.width(), frame_size.height());
  313. DrawFrameOnCanvas(cc::SkiaPaintCanvas(bitmap), frame_size);
  314. return bitmap;
  315. }
  316. void FakeCameraDevice::Bind(
  317. mojo::PendingReceiver<video_capture::mojom::VideoSource> pending_receiver) {
  318. video_source_receiver_set_.Add(this, std::move(pending_receiver));
  319. }
  320. void FakeCameraDevice::TriggerFatalError() {
  321. for (auto& pair : subscriptions_map_) {
  322. auto* subscription = pair.first;
  323. subscription->TriggerFatalError();
  324. }
  325. }
  326. void FakeCameraDevice::CreatePushSubscription(
  327. mojo::PendingRemote<video_capture::mojom::VideoFrameHandler> subscriber,
  328. const media::VideoCaptureParams& requested_settings,
  329. bool force_reopen_with_new_settings,
  330. mojo::PendingReceiver<video_capture::mojom::PushVideoStreamSubscription>
  331. subscription,
  332. CreatePushSubscriptionCallback callback) {
  333. auto new_subscription = std::make_unique<FakeCameraDevice::Subscription>(
  334. this, std::move(subscription), std::move(subscriber));
  335. auto* new_subscription_ptr = new_subscription.get();
  336. subscriptions_map_.emplace(new_subscription_ptr, std::move(new_subscription));
  337. DCHECK(requested_settings.IsValid());
  338. const bool has_current_settings = current_settings_.has_value();
  339. if (force_reopen_with_new_settings || !has_current_settings) {
  340. RetireAllBuffers();
  341. current_settings_.emplace(requested_settings);
  342. OnSubscriptionActivationChanged(/*subscription=*/nullptr);
  343. }
  344. DCHECK(callback);
  345. std::move(callback).Run(
  346. video_capture::mojom::CreatePushSubscriptionResultCode::NewSuccessCode(
  347. video_capture::mojom::CreatePushSubscriptionSuccessCode::
  348. kCreatedWithRequestedSettings),
  349. requested_settings);
  350. }
  351. void FakeCameraDevice::OnFinishedConsumingBuffer(int32_t buffer_id) {
  352. auto iter = buffer_pool_.find(buffer_id);
  353. if (iter != buffer_pool_.end())
  354. iter->second->set_is_in_use(false);
  355. }
  356. mojo::PendingRemote<video_capture::mojom::VideoFrameAccessHandler>
  357. FakeCameraDevice::GetNewAccessHandlerRemote() {
  358. mojo::PendingReceiver<video_capture::mojom::VideoFrameAccessHandler> receiver;
  359. auto remote = receiver.InitWithNewPipeAndPassRemote();
  360. access_handler_receiver_set_.Add(this, std::move(receiver));
  361. return remote;
  362. }
  363. void FakeCameraDevice::OnSubscriberDisconnected(Subscription* subscription) {
  364. DCHECK(subscriptions_map_.contains(subscription));
  365. subscriptions_map_.erase(subscription);
  366. if (subscriptions_map_.empty()) {
  367. frame_timer_.Stop();
  368. current_settings_.reset();
  369. RetireAllBuffers();
  370. }
  371. }
  372. void FakeCameraDevice::OnSubscriptionActivationChanged(
  373. Subscription* subscription) {
  374. DCHECK(current_settings_.has_value());
  375. // Newly activated subscriptions should be told about all existing buffers.
  376. if (subscription && subscription->is_active()) {
  377. for (auto& pair : buffer_pool_) {
  378. Buffer* buffer = pair.second.get();
  379. subscription->OnNewBuffer(buffer->buffer_id(), buffer->GetHandle());
  380. }
  381. }
  382. const auto frame_duration =
  383. base::Hertz(current_settings_->requested_format.frame_rate);
  384. if (IsAnySubscriptionActive()) {
  385. if (!frame_timer_.IsRunning() ||
  386. frame_timer_.GetCurrentDelay() != frame_duration) {
  387. frame_timer_.Start(FROM_HERE, frame_duration, this,
  388. &FakeCameraDevice::OnNextFrame);
  389. }
  390. } else {
  391. frame_timer_.Stop();
  392. }
  393. }
  394. bool FakeCameraDevice::IsAnySubscriptionActive() const {
  395. for (const auto& pair : subscriptions_map_) {
  396. if (pair.first->is_active())
  397. return true;
  398. }
  399. return false;
  400. }
  401. void FakeCameraDevice::OnNextFrame() {
  402. DCHECK(IsAnySubscriptionActive());
  403. DCHECK(current_settings_.has_value());
  404. if (start_time_.is_null())
  405. start_time_ = base::Time::Now();
  406. auto* buffer = AllocateOrReuseBuffer();
  407. if (!buffer)
  408. return;
  409. buffer->DrawFrame();
  410. for (auto& pair : subscriptions_map_) {
  411. auto* subscription = pair.first;
  412. if (!subscription->is_active() || subscription->is_suspended())
  413. return;
  414. media::mojom::VideoFrameInfoPtr info = media::mojom::VideoFrameInfo::New();
  415. info->timestamp = base::Time::Now() - start_time_;
  416. info->pixel_format = media::PIXEL_FORMAT_ARGB;
  417. info->coded_size = current_settings_->requested_format.frame_size;
  418. info->visible_rect = gfx::Rect(info->coded_size);
  419. info->is_premapped = false;
  420. info->color_space = gfx::ColorSpace();
  421. subscription->OnFrameReadyInBuffer(
  422. video_capture::mojom::ReadyFrameInBuffer::New(buffer->buffer_id(),
  423. /*frame_feedback_id=*/0,
  424. std::move(info)));
  425. }
  426. }
  427. FakeCameraDevice::Buffer* FakeCameraDevice::AllocateOrReuseBuffer() {
  428. DCHECK(current_settings_.has_value());
  429. const auto buffer_type = current_settings_->buffer_type;
  430. const auto frame_size = current_settings_->requested_format.frame_size;
  431. // Can we reuse an existing buffer?
  432. for (auto& pair : buffer_pool_) {
  433. Buffer* buffer = pair.second.get();
  434. if (buffer->CanBeReused(buffer_type, frame_size)) {
  435. buffer->set_is_in_use(true);
  436. return buffer;
  437. }
  438. }
  439. if (buffer_pool_.size() >= FakeCameraDevice::kMaxBufferCount) {
  440. for (auto& pair : subscriptions_map_) {
  441. auto* subscription = pair.first;
  442. if (subscription->is_active() && !subscription->is_suspended())
  443. subscription->OnFrameDropped();
  444. }
  445. return nullptr;
  446. }
  447. const int buffer_id = g_next_buffer_id++;
  448. auto unique_buffer = Buffer::Create(buffer_id, buffer_type, frame_size);
  449. Buffer* buffer_ptr = unique_buffer.get();
  450. buffer_ptr->set_is_in_use(true);
  451. buffer_pool_.emplace(buffer_id, std::move(unique_buffer));
  452. for (auto& pair : subscriptions_map_) {
  453. auto* subscription = pair.first;
  454. if (subscription->is_active())
  455. subscription->OnNewBuffer(buffer_id, buffer_ptr->GetHandle());
  456. }
  457. return buffer_ptr;
  458. }
  459. void FakeCameraDevice::RetireAllBuffers() {
  460. for (auto& pair : buffer_pool_) {
  461. const int buffer_id = pair.first;
  462. for (auto& pair : subscriptions_map_) {
  463. auto* subscription = pair.first;
  464. if (subscription->is_active())
  465. subscription->OnBufferRetired(buffer_id);
  466. }
  467. }
  468. buffer_pool_.clear();
  469. }
  470. } // namespace ash