video_frame.h 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_BASE_VIDEO_FRAME_H_
  5. #define MEDIA_BASE_VIDEO_FRAME_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <string>
  10. #include <utility>
  11. #include <vector>
  12. #include "base/callback.h"
  13. #include "base/check_op.h"
  14. #include "base/hash/md5.h"
  15. #include "base/memory/free_deleter.h"
  16. #include "base/memory/raw_ptr.h"
  17. #include "base/memory/ref_counted.h"
  18. #include "base/memory/unsafe_shared_memory_region.h"
  19. #include "base/synchronization/lock.h"
  20. #include "base/thread_annotations.h"
  21. #include "base/time/time.h"
  22. #include "base/unguessable_token.h"
  23. #include "build/build_config.h"
  24. #include "gpu/command_buffer/common/mailbox_holder.h"
  25. #include "gpu/ipc/common/vulkan_ycbcr_info.h"
  26. #include "media/base/video_frame_layout.h"
  27. #include "media/base/video_frame_metadata.h"
  28. #include "media/base/video_types.h"
  29. #include "third_party/abseil-cpp/absl/types/optional.h"
  30. #include "ui/gfx/color_space.h"
  31. #include "ui/gfx/geometry/rect.h"
  32. #include "ui/gfx/geometry/size.h"
  33. #include "ui/gfx/hdr_metadata.h"
  34. #if BUILDFLAG(IS_MAC)
  35. #include <CoreVideo/CVPixelBuffer.h>
  36. #include "base/mac/scoped_cftyperef.h"
  37. #endif // BUILDFLAG(IS_MAC)
  38. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  39. #include "base/files/scoped_file.h"
  40. #endif // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  41. namespace gfx {
  42. class GpuMemoryBuffer;
  43. struct GpuMemoryBufferHandle;
  44. }
  45. namespace media {
  46. class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  47. public:
  48. static constexpr size_t kFrameSizeAlignment = 16;
  49. static constexpr size_t kFrameSizePadding = 16;
  50. static constexpr size_t kFrameAddressAlignment =
  51. VideoFrameLayout::kBufferAddressAlignment;
  52. enum {
  53. kMaxPlanes = 4,
  54. kYPlane = 0,
  55. kARGBPlane = kYPlane,
  56. kUPlane = 1,
  57. kUVPlane = kUPlane,
  58. kVPlane = 2,
  59. kAPlane = 3,
  60. };
  61. // Defines the pixel storage type. Differentiates between directly accessible
  62. // |data_| and pixels that are only indirectly accessible and not via mappable
  63. // memory.
  64. // Note that VideoFrames of any StorageType can also have Texture backing,
  65. // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE.
  66. enum StorageType {
  67. STORAGE_UNKNOWN = 0,
  68. STORAGE_OPAQUE = 1, // We don't know how VideoFrame's pixels are stored.
  69. STORAGE_UNOWNED_MEMORY = 2, // External, non owned data pointers.
  70. STORAGE_OWNED_MEMORY = 3, // VideoFrame has allocated its own data buffer.
  71. STORAGE_SHMEM = 4, // Backed by unsafe (writable) shared memory.
  72. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  73. // TODO(mcasas): Consider turning this type into STORAGE_NATIVE
  74. // based on the idea of using this same enum value for both DMA
  75. // buffers on Linux and CVPixelBuffers on Mac (which currently use
  76. // STORAGE_UNOWNED_MEMORY) and handle it appropriately in all cases.
  77. STORAGE_DMABUFS = 5, // Each plane is stored into a DmaBuf.
  78. #endif
  79. // Backed by a mojo shared buffer. This should only be used by the
  80. // MojoSharedBufferVideoFrame subclass.
  81. STORAGE_MOJO_SHARED_BUFFER = 6,
  82. STORAGE_GPU_MEMORY_BUFFER = 7,
  83. STORAGE_LAST = STORAGE_GPU_MEMORY_BUFFER,
  84. };
  85. // CB to be called on the mailbox backing this frame and its GpuMemoryBuffers
  86. // (if they exist) when the frame is destroyed.
  87. using ReleaseMailboxCB = base::OnceCallback<void(const gpu::SyncToken&)>;
  88. using ReleaseMailboxAndGpuMemoryBufferCB =
  89. base::OnceCallback<void(const gpu::SyncToken&,
  90. std::unique_ptr<gfx::GpuMemoryBuffer>)>;
  91. // Interface representing client operations on a SyncToken, i.e. insert one in
  92. // the GPU Command Buffer and wait for it.
  93. class SyncTokenClient {
  94. public:
  95. SyncTokenClient() = default;
  96. SyncTokenClient(const SyncTokenClient&) = delete;
  97. SyncTokenClient& operator=(const SyncTokenClient&) = delete;
  98. virtual void GenerateSyncToken(gpu::SyncToken* sync_token) = 0;
  99. virtual void WaitSyncToken(const gpu::SyncToken& sync_token) = 0;
  100. protected:
  101. virtual ~SyncTokenClient() = default;
  102. };
  103. VideoFrame() = delete;
  104. VideoFrame(const VideoFrame&) = delete;
  105. VideoFrame& operator=(const VideoFrame&) = delete;
  106. // Returns true if size is valid for a VideoFrame.
  107. static bool IsValidSize(const gfx::Size& coded_size,
  108. const gfx::Rect& visible_rect,
  109. const gfx::Size& natural_size);
  110. // Returns true if and only if the |size| is within limits, i.e., neither
  111. // dimension exceeds limits::kMaxDimension and the total area doesn't exceed
  112. // limits::kMaxCanvas. Prefer the overload that accepts the |coded_size|,
  113. // |visible_rect|, and |natural_size| if trying to validate the VideoFrame.
  114. static bool IsValidCodedSize(const gfx::Size& size);
  115. // Returns true if frame configuration is valid.
  116. static bool IsValidConfig(VideoPixelFormat format,
  117. StorageType storage_type,
  118. const gfx::Size& coded_size,
  119. const gfx::Rect& visible_rect,
  120. const gfx::Size& natural_size);
  121. // Creates a new frame in system memory with given parameters. Buffers for the
  122. // frame are allocated but not initialized. The caller must not make
  123. // assumptions about the actual underlying size(s), but check the returned
  124. // VideoFrame instead.
  125. static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format,
  126. const gfx::Size& coded_size,
  127. const gfx::Rect& visible_rect,
  128. const gfx::Size& natural_size,
  129. base::TimeDelta timestamp);
  130. // Used by Chromecast only.
  131. // Create a new frame that doesn't contain any valid video content. This frame
  132. // is meant to be sent to compositor to inform that the compositor should
  133. // punch a transparent hole so the video underlay will be visible.
  134. static scoped_refptr<VideoFrame> CreateVideoHoleFrame(
  135. const base::UnguessableToken& overlay_plane_id,
  136. const gfx::Size& natural_size,
  137. base::TimeDelta timestamp);
  138. // Offers the same functionality as CreateFrame, and additionally zeroes out
  139. // the initial allocated buffers.
  140. static scoped_refptr<VideoFrame> CreateZeroInitializedFrame(
  141. VideoPixelFormat format,
  142. const gfx::Size& coded_size,
  143. const gfx::Rect& visible_rect,
  144. const gfx::Size& natural_size,
  145. base::TimeDelta timestamp);
  146. // Creates a new frame in system memory with given parameters. Buffers for the
  147. // frame are allocated but not initialized. The caller should specify the
  148. // physical buffer size and strides if needed in |layout| parameter.
  149. static scoped_refptr<VideoFrame> CreateFrameWithLayout(
  150. const VideoFrameLayout& layout,
  151. const gfx::Rect& visible_rect,
  152. const gfx::Size& natural_size,
  153. base::TimeDelta timestamp,
  154. bool zero_initialize_memory);
  155. // Wraps a set of native textures with a VideoFrame.
  156. // |mailbox_holders_release_cb| will be called with a sync token as the
  157. // argument when the VideoFrame is to be destroyed.
  158. static scoped_refptr<VideoFrame> WrapNativeTextures(
  159. VideoPixelFormat format,
  160. const gpu::MailboxHolder (&mailbox_holder)[kMaxPlanes],
  161. ReleaseMailboxCB mailbox_holders_release_cb,
  162. const gfx::Size& coded_size,
  163. const gfx::Rect& visible_rect,
  164. const gfx::Size& natural_size,
  165. base::TimeDelta timestamp);
  166. // Wraps packed image data residing in a memory buffer with a VideoFrame.
  167. // The image data resides in |data| and is assumed to be packed tightly in a
  168. // buffer of logical dimensions |coded_size| with the appropriate bit depth
  169. // and plane count as given by |format|. Returns NULL on failure.
  170. static scoped_refptr<VideoFrame> WrapExternalData(
  171. VideoPixelFormat format,
  172. const gfx::Size& coded_size,
  173. const gfx::Rect& visible_rect,
  174. const gfx::Size& natural_size,
  175. uint8_t* data,
  176. size_t data_size,
  177. base::TimeDelta timestamp);
  178. static scoped_refptr<VideoFrame> WrapExternalDataWithLayout(
  179. const VideoFrameLayout& layout,
  180. const gfx::Rect& visible_rect,
  181. const gfx::Size& natural_size,
  182. uint8_t* data,
  183. size_t data_size,
  184. base::TimeDelta timestamp);
  185. // Wraps external YUV data of the given parameters with a VideoFrame.
  186. // The returned VideoFrame does not own the data passed in.
  187. static scoped_refptr<VideoFrame> WrapExternalYuvData(
  188. VideoPixelFormat format,
  189. const gfx::Size& coded_size,
  190. const gfx::Rect& visible_rect,
  191. const gfx::Size& natural_size,
  192. int32_t y_stride,
  193. int32_t u_stride,
  194. int32_t v_stride,
  195. uint8_t* y_data,
  196. uint8_t* u_data,
  197. uint8_t* v_data,
  198. base::TimeDelta timestamp);
  199. // Wraps external YUV data with VideoFrameLayout. The returned VideoFrame does
  200. // not own the data passed in.
  201. static scoped_refptr<VideoFrame> WrapExternalYuvDataWithLayout(
  202. const VideoFrameLayout& layout,
  203. const gfx::Rect& visible_rect,
  204. const gfx::Size& natural_size,
  205. uint8_t* y_data,
  206. uint8_t* u_data,
  207. uint8_t* v_data,
  208. base::TimeDelta timestamp);
  209. // Wraps external YUVA data of the given parameters with a VideoFrame.
  210. // The returned VideoFrame does not own the data passed in.
  211. static scoped_refptr<VideoFrame> WrapExternalYuvaData(
  212. VideoPixelFormat format,
  213. const gfx::Size& coded_size,
  214. const gfx::Rect& visible_rect,
  215. const gfx::Size& natural_size,
  216. int32_t y_stride,
  217. int32_t u_stride,
  218. int32_t v_stride,
  219. int32_t a_stride,
  220. uint8_t* y_data,
  221. uint8_t* u_data,
  222. uint8_t* v_data,
  223. uint8_t* a_data,
  224. base::TimeDelta timestamp);
  225. // Wraps external NV12 data of the given parameters with a VideoFrame.
  226. // The returned VideoFrame does not own the data passed in.
  227. static scoped_refptr<VideoFrame> WrapExternalYuvData(
  228. VideoPixelFormat format,
  229. const gfx::Size& coded_size,
  230. const gfx::Rect& visible_rect,
  231. const gfx::Size& natural_size,
  232. int32_t y_stride,
  233. int32_t uv_stride,
  234. uint8_t* y_data,
  235. uint8_t* uv_data,
  236. base::TimeDelta timestamp);
  237. // Wraps |gpu_memory_buffer| along with the mailboxes created from
  238. // |gpu_memory_buffer|. This will transfer ownership of |gpu_memory_buffer|
  239. // to the returned VideoFrame. |mailbox_holder_and_gmb_release_cb| will be
  240. // called with a sync token and with |gpu_memory_buffer| as arguments when the
  241. // VideoFrame is to be destroyed.
  242. static scoped_refptr<VideoFrame> WrapExternalGpuMemoryBuffer(
  243. const gfx::Rect& visible_rect,
  244. const gfx::Size& natural_size,
  245. std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
  246. const gpu::MailboxHolder (&mailbox_holders)[kMaxPlanes],
  247. ReleaseMailboxAndGpuMemoryBufferCB mailbox_holder_and_gmb_release_cb,
  248. base::TimeDelta timestamp);
  249. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  250. // Wraps provided dmabufs
  251. // (https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html) with a
  252. // VideoFrame. The frame will take ownership of |dmabuf_fds|, and will
  253. // automatically close() them on destruction. Callers can call
  254. // media::DuplicateFDs() if they need to retain a copy of the FDs for
  255. // themselves. Note that the FDs are consumed even in case of failure.
  256. // The image data is only accessible via dmabuf fds, which are usually passed
  257. // directly to a hardware device and/or to another process, or can also be
  258. // mapped via mmap() for CPU access.
  259. // Returns NULL on failure.
  260. static scoped_refptr<VideoFrame> WrapExternalDmabufs(
  261. const VideoFrameLayout& layout,
  262. const gfx::Rect& visible_rect,
  263. const gfx::Size& natural_size,
  264. std::vector<base::ScopedFD> dmabuf_fds,
  265. base::TimeDelta timestamp);
  266. #endif
  267. #if BUILDFLAG(IS_MAC)
  268. // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
  269. // retained for the lifetime of the VideoFrame and released upon destruction.
  270. // The image data is only accessible via the pixel buffer, which could be
  271. // backed by an IOSurface from another process. All the attributes of the
  272. // VideoFrame are derived from the pixel buffer, with the exception of the
  273. // timestamp. If information is missing or is incompatible (for example, a
  274. // pixel format that has no VideoFrame match), NULL is returned.
  275. // http://crbug.com/401308
  276. static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
  277. CVPixelBufferRef cv_pixel_buffer,
  278. base::TimeDelta timestamp);
  279. // Wraps a provided IOSurface with a VideoFrame. The IOSurface is retained
  280. // and locked for the lifetime of the VideoFrame. This is for unaccelerated
  281. // (CPU-only) access to the IOSurface, and is not efficient. It is the path
  282. // that video capture uses when hardware acceleration is disabled.
  283. // https://crbug.com/1125879
  284. static scoped_refptr<VideoFrame> WrapUnacceleratedIOSurface(
  285. gfx::GpuMemoryBufferHandle handle,
  286. const gfx::Rect& visible_rect,
  287. base::TimeDelta timestamp);
  288. #endif
  289. // Wraps |frame|. |visible_rect| must be a sub rect within
  290. // frame->visible_rect().
  291. static scoped_refptr<VideoFrame> WrapVideoFrame(
  292. scoped_refptr<VideoFrame> frame,
  293. VideoPixelFormat format,
  294. const gfx::Rect& visible_rect,
  295. const gfx::Size& natural_size);
  296. // Creates a frame which indicates end-of-stream.
  297. static scoped_refptr<VideoFrame> CreateEOSFrame();
  298. // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
  299. static scoped_refptr<VideoFrame> CreateColorFrame(const gfx::Size& size,
  300. uint8_t y,
  301. uint8_t u,
  302. uint8_t v,
  303. base::TimeDelta timestamp);
  304. // Allocates YV12 frame based on |size|, and sets its data to the YUV
  305. // equivalent of RGB(0,0,0).
  306. static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
  307. // Allocates YV12A frame based on |size|, and sets its data to the YUVA
  308. // equivalent of RGBA(0,0,0,0).
  309. static scoped_refptr<VideoFrame> CreateTransparentFrame(
  310. const gfx::Size& size);
  311. static size_t NumPlanes(VideoPixelFormat format);
  312. // Returns the required allocation size for a (tightly packed) frame of the
  313. // given coded size and format.
  314. static size_t AllocationSize(VideoPixelFormat format,
  315. const gfx::Size& coded_size);
  316. // Returns |dimensions| adjusted to appropriate boundaries based on |format|.
  317. static gfx::Size DetermineAlignedSize(VideoPixelFormat format,
  318. const gfx::Size& dimensions);
  319. // Returns the plane gfx::Size (in bytes) for a plane of the given coded size
  320. // and format.
  321. static gfx::Size PlaneSize(VideoPixelFormat format,
  322. size_t plane,
  323. const gfx::Size& coded_size);
  324. // Returns the plane gfx::Size (in samples) for a plane of the given coded
  325. // size and format.
  326. static gfx::Size PlaneSizeInSamples(VideoPixelFormat format,
  327. size_t plane,
  328. const gfx::Size& coded_size);
  329. // Returns horizontal bits per pixel for given |plane| and |format|.
  330. static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane);
  331. // Returns bits per pixel for given |plane| and |format|.
  332. static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane);
  333. // Returns the number of bytes per row for the given plane, format, and width.
  334. // The width may be aligned to format requirements.
  335. static size_t RowBytes(size_t plane, VideoPixelFormat format, int width);
  336. // Returns the number of bytes per element for given |plane| and |format|.
  337. static int BytesPerElement(VideoPixelFormat format, size_t plane);
  338. // Calculates strides for each plane based on |format| and |coded_size|.
  339. static std::vector<int32_t> ComputeStrides(VideoPixelFormat format,
  340. const gfx::Size& coded_size);
  341. // Returns the number of rows for the given plane, format, and height.
  342. // The height may be aligned to format requirements.
  343. static size_t Rows(size_t plane, VideoPixelFormat format, int height);
  344. // Returns the number of columns for the given plane, format, and width.
  345. // The width may be aligned to format requirements.
  346. static size_t Columns(size_t plane, VideoPixelFormat format, int width);
  347. // Used to keep a running hash of seen frames. Expects an initialized MD5
  348. // context. Calls MD5Update with the context and the contents of the frame.
  349. static void HashFrameForTesting(base::MD5Context* context,
  350. const VideoFrame& frame);
  351. // Returns true if |frame| is accesible mapped in the VideoFrame memory space.
  352. // static
  353. static bool IsStorageTypeMappable(VideoFrame::StorageType storage_type);
  354. // Returns true if |plane| is a valid plane index for the given |format|.
  355. static bool IsValidPlane(VideoPixelFormat format, size_t plane);
  356. // Returns the pixel size of each subsample for a given |plane| and |format|.
  357. // E.g. 2x2 for the U-plane in PIXEL_FORMAT_I420.
  358. static gfx::Size SampleSize(VideoPixelFormat format, size_t plane);
  359. // Returns a human readable string of StorageType.
  360. static std::string StorageTypeToString(VideoFrame::StorageType storage_type);
  361. // A video frame wrapping external data may be backed by an unsafe shared
  362. // memory region. These methods are used to appropriately transform a
  363. // VideoFrame created with WrapExternalData, WrapExternalYuvaData, etc. The
  364. // storage type of the Video Frame will be changed to STORAGE_SHM. Once the
  365. // backing of a VideoFrame is set, it cannot be changed.
  366. //
  367. // The region is NOT owned by the video frame. Both the region and its
  368. // associated mapping must outlive this instance.
  369. void BackWithSharedMemory(const base::UnsafeSharedMemoryRegion* region);
  370. // As above, but the VideoFrame owns the shared memory region as well as the
  371. // mapping. They will be destroyed with their VideoFrame.
  372. void BackWithOwnedSharedMemory(base::UnsafeSharedMemoryRegion region,
  373. base::WritableSharedMemoryMapping mapping);
  374. // Valid for shared memory backed VideoFrames.
  375. const base::UnsafeSharedMemoryRegion* shm_region() {
  376. DCHECK(IsValidSharedMemoryFrame());
  377. DCHECK(storage_type_ == STORAGE_SHMEM);
  378. return shm_region_;
  379. }
  380. // Returns true if |frame| is accessible and mapped in the VideoFrame memory
  381. // space. If false, clients should refrain from accessing data(),
  382. // visible_data() etc.
  383. bool IsMappable() const;
  384. // Returns true if |frame| has textures with any StorageType and should not be
  385. // accessed via data(), visible_data() etc.
  386. bool HasTextures() const;
  387. // Returns the number of native textures.
  388. size_t NumTextures() const;
  389. // Returns true if the video frame is backed with GpuMemoryBuffer.
  390. bool HasGpuMemoryBuffer() const;
  391. // Gets the GpuMemoryBuffer backing the VideoFrame.
  392. gfx::GpuMemoryBuffer* GetGpuMemoryBuffer() const;
  393. // Returns true if the video frame was created with the given parameters.
  394. bool IsSameAllocation(VideoPixelFormat format,
  395. const gfx::Size& coded_size,
  396. const gfx::Rect& visible_rect,
  397. const gfx::Size& natural_size) const;
  398. // Returns the color space of this frame's content.
  399. gfx::ColorSpace ColorSpace() const;
  400. void set_color_space(const gfx::ColorSpace& color_space) {
  401. color_space_ = color_space;
  402. }
  403. const absl::optional<gfx::HDRMetadata>& hdr_metadata() const {
  404. return hdr_metadata_;
  405. }
  406. void set_hdr_metadata(const absl::optional<gfx::HDRMetadata>& hdr_metadata) {
  407. hdr_metadata_ = hdr_metadata;
  408. }
  409. const VideoFrameLayout& layout() const { return layout_; }
  410. VideoPixelFormat format() const { return layout_.format(); }
  411. StorageType storage_type() const { return storage_type_; }
  412. // The full dimensions of the video frame data.
  413. const gfx::Size& coded_size() const { return layout_.coded_size(); }
  414. // A subsection of [0, 0, coded_size().width(), coded_size.height()]. This
  415. // can be set to "soft-apply" a cropping. It determines the pointers into
  416. // the data returned by visible_data().
  417. const gfx::Rect& visible_rect() const { return visible_rect_; }
  418. // Specifies that the |visible_rect| section of the frame is supposed to be
  419. // scaled to this size when being presented. This can be used to represent
  420. // anamorphic frames, or to "soft-apply" any custom scaling.
  421. const gfx::Size& natural_size() const { return natural_size_; }
  422. int stride(size_t plane) const {
  423. DCHECK(IsValidPlane(format(), plane));
  424. DCHECK_LT(plane, layout_.num_planes());
  425. return layout_.planes()[plane].stride;
  426. }
  427. // Returns the number of bytes per row and number of rows for a given plane.
  428. //
  429. // As opposed to stride(), row_bytes() refers to the bytes representing
  430. // frame data scanlines (coded_size.width() pixels, without stride padding).
  431. int row_bytes(size_t plane) const;
  432. int rows(size_t plane) const;
  433. // Returns the number of columns for a given plane.
  434. int columns(size_t plane) const;
  435. // Returns pointer to the buffer for a given plane, if this is an
  436. // IsMappable() frame type. The memory is owned by VideoFrame object and must
  437. // not be freed by the caller.
  438. const uint8_t* data(size_t plane) const {
  439. DCHECK(IsValidPlane(format(), plane));
  440. DCHECK(IsMappable());
  441. return data_[plane];
  442. }
  443. uint8_t* data(size_t plane) {
  444. DCHECK(IsValidPlane(format(), plane));
  445. DCHECK(IsMappable());
  446. return data_[plane];
  447. }
  448. const absl::optional<gpu::VulkanYCbCrInfo>& ycbcr_info() const {
  449. return wrapped_frame_ ? wrapped_frame_->ycbcr_info() : ycbcr_info_;
  450. }
  451. // Returns pointer to the data in the visible region of the frame, for
  452. // IsMappable() storage types. The returned pointer is offsetted into the
  453. // plane buffer specified by visible_rect().origin(). Memory is owned by
  454. // VideoFrame object and must not be freed by the caller.
  455. const uint8_t* visible_data(size_t plane) const;
  456. uint8_t* visible_data(size_t plane);
  457. // Returns a mailbox holder for a given texture.
  458. // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
  459. // mailbox, the caller must wait for the included sync point.
  460. const gpu::MailboxHolder& mailbox_holder(size_t texture_index) const;
  461. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  462. // Returns a vector containing the backing DmaBufs for this frame. The number
  463. // of returned DmaBufs will be equal or less than the number of planes of
  464. // the frame. If there are less, this means that the last FD contains the
  465. // remaining planes.
  466. // Note that the returned FDs are still owned by the VideoFrame. This means
  467. // that the caller shall not close them, or use them after the VideoFrame is
  468. // destroyed. For such use cases, use media::DuplicateFDs() to obtain your
  469. // own copy of the FDs.
  470. const std::vector<base::ScopedFD>& DmabufFds() const;
  471. // Returns true if |frame| has DmaBufs.
  472. bool HasDmaBufs() const;
  473. // Returns true if both VideoFrames are backed by DMABUF memory and point
  474. // to the same set of DMABUFs, meaning that both frames use the same memory.
  475. bool IsSameDmaBufsAs(const VideoFrame& frame) const;
  476. #endif
  477. #if BUILDFLAG(IS_MAC)
  478. // Returns the backing CVPixelBuffer, if present.
  479. CVPixelBufferRef CvPixelBuffer() const;
  480. #endif
  481. // Sets the mailbox (and GpuMemoryBuffer, if desired) release callback.
  482. //
  483. // The callback may be run from ANY THREAD, and so it is up to the client to
  484. // ensure thread safety.
  485. //
  486. // WARNING: This method is not thread safe; it should only be called if you
  487. // are still the only owner of this VideoFrame.
  488. void SetReleaseMailboxCB(ReleaseMailboxCB release_mailbox_cb);
  489. void SetReleaseMailboxAndGpuMemoryBufferCB(
  490. ReleaseMailboxAndGpuMemoryBufferCB release_mailbox_cb);
  491. // Tests whether a mailbox release callback is configured.
  492. bool HasReleaseMailboxCB() const;
  493. // Adds a callback to be run when the VideoFrame is about to be destroyed.
  494. // The callback may be run from ANY THREAD, and so it is up to the client to
  495. // ensure thread safety. Although read-only access to the members of this
  496. // VideoFrame is permitted while the callback executes (including
  497. // VideoFrameMetadata), clients should not assume the data pointers are
  498. // valid.
  499. void AddDestructionObserver(base::OnceClosure callback);
  500. // Returns a dictionary of optional metadata. This contains information
  501. // associated with the frame that downstream clients might use for frame-level
  502. // logging, quality/performance optimizations, signaling, etc.
  503. const VideoFrameMetadata& metadata() const { return metadata_; }
  504. VideoFrameMetadata& metadata() { return metadata_; }
  505. void set_metadata(const VideoFrameMetadata& metadata) {
  506. metadata_ = metadata;
  507. }
  508. // Resets |metadata_|.
  509. void clear_metadata() { set_metadata(VideoFrameMetadata()); }
  510. // The time span between the current frame and the first frame of the stream.
  511. // This is the media timestamp, and not the reference time.
  512. // See VideoFrameMetadata::REFERENCE_TIME for details.
  513. base::TimeDelta timestamp() const { return timestamp_; }
  514. void set_timestamp(base::TimeDelta timestamp) { timestamp_ = timestamp; }
  515. // It uses |client| to insert a new sync token and potentially waits on an
  516. // older sync token. The final sync point will be used to release this
  517. // VideoFrame. Also returns the new sync token.
  518. // This method is thread safe. Both blink and compositor threads can call it.
  519. gpu::SyncToken UpdateReleaseSyncToken(SyncTokenClient* client);
  520. // Similar to UpdateReleaseSyncToken() but operates on the gpu::SyncToken
  521. // for each plane. This should only be called when a VideoFrame has a single
  522. // owner. I.e., before it has been vended after creation.
  523. gpu::SyncToken UpdateMailboxHolderSyncToken(size_t plane,
  524. SyncTokenClient* client);
  525. // Returns a human-readable string describing |*this|.
  526. std::string AsHumanReadableString() const;
  527. // Unique identifier for this video frame; generated at construction time and
  528. // guaranteed to be unique within a single process.
  529. int unique_id() const { return unique_id_; }
  530. // Returns the number of bits per channel.
  531. size_t BitDepth() const;
  532. // Provide the sampler conversion information for the frame.
  533. void set_ycbcr_info(const absl::optional<gpu::VulkanYCbCrInfo>& ycbcr_info) {
  534. ycbcr_info_ = ycbcr_info;
  535. }
  536. protected:
  537. friend class base::RefCountedThreadSafe<VideoFrame>;
  538. enum class FrameControlType {
  539. kNone,
  540. kEos,
  541. kVideoHole,
  542. };
  543. // Clients must use the static factory/wrapping methods to create a new frame.
  544. // Derived classes should create their own factory/wrapping methods, and use
  545. // this constructor to do basic initialization.
  546. VideoFrame(const VideoFrameLayout& layout,
  547. StorageType storage_type,
  548. const gfx::Rect& visible_rect,
  549. const gfx::Size& natural_size,
  550. base::TimeDelta timestamp,
  551. FrameControlType frame_control_type = FrameControlType::kNone);
  552. virtual ~VideoFrame();
  553. // Creates a summary of the configuration settings provided as parameters.
  554. static std::string ConfigToString(const VideoPixelFormat format,
  555. const VideoFrame::StorageType storage_type,
  556. const gfx::Size& coded_size,
  557. const gfx::Rect& visible_rect,
  558. const gfx::Size& natural_size);
  559. void set_data(size_t plane, uint8_t* ptr) {
  560. DCHECK(IsValidPlane(format(), plane));
  561. DCHECK(ptr);
  562. data_[plane] = ptr;
  563. }
  564. private:
  565. // The constructor of VideoFrame should use IsValidConfigInternal()
  566. // instead of the public IsValidConfig() to check the config, because we can
  567. // create special video frames that won't pass the check by IsValidConfig().
  568. static bool IsValidConfigInternal(VideoPixelFormat format,
  569. FrameControlType frame_control_type,
  570. const gfx::Size& coded_size,
  571. const gfx::Rect& visible_rect,
  572. const gfx::Size& natural_size);
  573. static scoped_refptr<VideoFrame> CreateFrameInternal(
  574. VideoPixelFormat format,
  575. const gfx::Size& coded_size,
  576. const gfx::Rect& visible_rect,
  577. const gfx::Size& natural_size,
  578. base::TimeDelta timestamp,
  579. bool zero_initialize_memory);
  580. // Return the alignment for the whole frame, calculated as the max of the
  581. // alignment for each individual plane.
  582. static gfx::Size CommonAlignment(VideoPixelFormat format);
  583. // Tries to allocate the requisite amount of memory for this frame. Returns
  584. // false if this would cause an out of memory error.
  585. [[nodiscard]] bool AllocateMemory(bool zero_initialize_memory);
  586. // Calculates plane size.
  587. // It first considers buffer size layout_ object provides. If layout's
  588. // number of buffers equals to number of planes, and buffer size is assigned
  589. // (non-zero), it returns buffers' size.
  590. // Otherwise, it uses the first (num_buffers - 1) assigned buffers' size as
  591. // plane size. Then for the rest unassigned planes, calculates their size
  592. // based on format, coded size and stride for the plane.
  593. std::vector<size_t> CalculatePlaneSize() const;
  594. // Returns true iff the frame has a shared memory storage type, and the
  595. // associated regions are valid.
  596. bool IsValidSharedMemoryFrame() const;
  597. // VideFrameLayout (includes format, coded_size, and strides).
  598. const VideoFrameLayout layout_;
  599. // Set by WrapVideoFrame to soft-apply a new set of format, visible rectangle,
  600. // and natural size on |wrapped_frame_|
  601. scoped_refptr<VideoFrame> wrapped_frame_;
  602. // Storage type for the different planes.
  603. StorageType storage_type_; // TODO(mcasas): make const
  604. // Width, height, and offsets of the visible portion of the video frame. Must
  605. // be a subrect of |coded_size_|. Can be odd with respect to the sample
  606. // boundaries, e.g. for formats with subsampled chroma.
  607. const gfx::Rect visible_rect_;
  608. // Width and height of the visible portion of the video frame
  609. // (|visible_rect_.size()|) with aspect ratio taken into account.
  610. const gfx::Size natural_size_;
  611. // Array of data pointers to each plane.
  612. // TODO(mcasas): we don't know on ctor if we own |data_| or not. Change
  613. // to std::unique_ptr<uint8_t, AlignedFreeDeleter> after refactoring
  614. // VideoFrame.
  615. uint8_t* data_[kMaxPlanes];
  616. // Native texture mailboxes, if this is a IsTexture() frame.
  617. gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
  618. ReleaseMailboxAndGpuMemoryBufferCB mailbox_holders_and_gmb_release_cb_;
  619. // Shared memory handle, if this frame is STORAGE_SHMEM. The region pointed
  620. // to is unowned.
  621. raw_ptr<const base::UnsafeSharedMemoryRegion> shm_region_ = nullptr;
  622. // Used if this is a STORAGE_SHMEM frame with owned shared memory. In that
  623. // case, shm_region_ will refer to this region.
  624. base::UnsafeSharedMemoryRegion owned_shm_region_;
  625. base::WritableSharedMemoryMapping owned_shm_mapping_;
  626. // GPU memory buffer, if this frame is STORAGE_GPU_MEMORY_BUFFER.
  627. std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer_;
  628. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  629. class DmabufHolder;
  630. // Dmabufs for the frame, used when storage is STORAGE_DMABUFS. Size is either
  631. // equal or less than the number of planes of the frame. If it is less, then
  632. // the memory area represented by the last FD contains the remaining planes.
  633. // If a STORAGE_DMABUFS frame is wrapped into another, the wrapping frame
  634. // will get an extra reference to the FDs (i.e. no duplication is involved).
  635. // This makes it possible to test whether two VideoFrame instances point to
  636. // the same DMABUF memory by testing for
  637. // (&vf1->DmabufFds() == &vf2->DmabufFds()).
  638. scoped_refptr<DmabufHolder> dmabuf_fds_;
  639. #endif
  640. #if BUILDFLAG(IS_MAC)
  641. // CVPixelBuffer, if this frame is wrapping one.
  642. base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
  643. #endif
  644. std::vector<base::OnceClosure> done_callbacks_;
  645. base::TimeDelta timestamp_;
  646. base::Lock release_sync_token_lock_;
  647. gpu::SyncToken release_sync_token_ GUARDED_BY(release_sync_token_lock_);
  648. VideoFrameMetadata metadata_;
  649. // Generated at construction time.
  650. const int unique_id_;
  651. gfx::ColorSpace color_space_;
  652. absl::optional<gfx::HDRMetadata> hdr_metadata_;
  653. // Sampler conversion information which is used in vulkan context for android.
  654. absl::optional<gpu::VulkanYCbCrInfo> ycbcr_info_;
  655. // Allocation which makes up |data_| planes for self-allocated frames.
  656. std::unique_ptr<uint8_t, base::FreeDeleter> private_data_;
  657. };
  658. } // namespace media
  659. #endif // MEDIA_BASE_VIDEO_FRAME_H_