video_frame_metadata.h 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_BASE_VIDEO_FRAME_METADATA_H_
  5. #define MEDIA_BASE_VIDEO_FRAME_METADATA_H_
  6. #include "base/time/time.h"
  7. #include "base/unguessable_token.h"
  8. #include "build/build_config.h"
  9. #include "media/base/media_export.h"
  10. #include "media/base/video_transformation.h"
  11. #include "media/gpu/buildflags.h"
  12. #include "third_party/abseil-cpp/absl/types/optional.h"
  13. #include "ui/gfx/geometry/rect.h"
  14. namespace media {
  15. // NOTE: When adding new VideoFrameMetadata fields, please ensure you update the
  16. // MergeMetadataFrom() method.
  17. struct MEDIA_EXPORT VideoFrameMetadata {
  18. VideoFrameMetadata();
  19. ~VideoFrameMetadata() = default;
  20. VideoFrameMetadata(const VideoFrameMetadata& other);
  21. // Merges internal values from |metadata_source|.
  22. void MergeMetadataFrom(const VideoFrameMetadata& metadata_source);
  23. // Sources of VideoFrames use this marker to indicate that the associated
  24. // VideoFrame can be overlaid, case in which its contents do not need to be
  25. // further composited but displayed directly.
  26. bool allow_overlay = false;
  27. // Video capture begin/end timestamps. Consumers can use these values for
  28. // dynamic optimizations, logging stats, etc.
  29. absl::optional<base::TimeTicks> capture_begin_time;
  30. absl::optional<base::TimeTicks> capture_end_time;
  31. // A counter that is increased by the producer of video frames each time
  32. // it pushes out a new frame. By looking for gaps in this counter, clients
  33. // can determine whether or not any frames have been dropped on the way from
  34. // the producer between two consecutively received frames. Note that the
  35. // counter may start at arbitrary values, so the absolute value of it has no
  36. // meaning.
  37. absl::optional<int> capture_counter;
  38. // The rectangular region of the frame that has changed since the frame
  39. // with the directly preceding CAPTURE_COUNTER. If that frame was not
  40. // received, typically because it was dropped during transport from the
  41. // producer, clients must assume that the entire frame has changed.
  42. // The rectangle is relative to the full frame data, i.e. [0, 0,
  43. // coded_size().width(), coded_size().height()]. It does not have to be
  44. // fully contained within visible_rect().
  45. absl::optional<gfx::Rect> capture_update_rect;
  46. // For encoded frames, this is the original source size which may be different
  47. // from the encoded size. It's used for the HiDPI tab capture heuristic.
  48. // The size corresponds to the active region if region capture is active,
  49. // or otherwise the full size of the captured source.
  50. absl::optional<gfx::Size> source_size;
  51. // If cropping was applied due to Region Capture to produce this frame,
  52. // then this reflects where the frame's contents originate from in the
  53. // original uncropped frame.
  54. //
  55. // NOTE: May also be nullopt if region capture is enabled but the capture rect
  56. // is in a different coordinate space. For more info, see
  57. // https://crbug.com/1327560.
  58. absl::optional<gfx::Rect> region_capture_rect;
  59. // Whenever cropTo() is called, Blink increments the crop_version and records
  60. // a Promise as associated with that crop_version.
  61. // When Blink observes a frame with this new version or a later one,
  62. // Blink resolves the Promise.
  63. // Frames associated with a source which cannot be cropped will always
  64. // have this value set to zero.
  65. uint32_t crop_version = 0;
  66. // Indicates that mailbox created in one context, is also being used in a
  67. // different context belonging to another share group and video frames are
  68. // using SurfaceTexture to render frames.
  69. // Textures generated from SurfaceTexture can't be shared between contexts
  70. // of different share group and hence this frame must be copied to a new
  71. // texture before use, rather than being used directly.
  72. bool copy_required = false;
  73. // Indicates if the current frame is the End of its current Stream.
  74. bool end_of_stream = false;
  75. // The estimated duration of this frame (i.e., the amount of time between
  76. // the media timestamp of this frame and the next). Note that this is not
  77. // the same information provided by FRAME_RATE as the FRAME_DURATION can
  78. // vary unpredictably for every frame. Consumers can use this to optimize
  79. // playback scheduling, make encoding quality decisions, and/or compute
  80. // frame-level resource utilization stats.
  81. absl::optional<base::TimeDelta> frame_duration;
  82. // Represents either the fixed frame rate, or the maximum frame rate to
  83. // expect from a variable-rate source. This value generally remains the
  84. // same for all frames in the same session.
  85. absl::optional<double> frame_rate;
  86. // This is a boolean that signals that the video capture engine detects
  87. // interactive content. One possible optimization that this signal can help
  88. // with is remote content: adjusting end-to-end latency down to help the
  89. // user better coordinate their actions.
  90. bool interactive_content = false;
  91. // This field represents the local time at which either: 1) the frame was
  92. // generated, if it was done so locally; or 2) the targeted play-out time
  93. // of the frame, if it was generated from a remote source. This value is NOT
  94. // a high-resolution timestamp, and so it should not be used as a
  95. // presentation time; but, instead, it should be used for buffering playback
  96. // and for A/V synchronization purposes.
  97. absl::optional<base::TimeTicks> reference_time;
  98. // Sources of VideoFrames use this marker to indicate that an instance of
  99. // VideoFrameExternalResources produced from the associated video frame
  100. // should use read lock fences.
  101. bool read_lock_fences_enabled = false;
  102. // Indicates that the frame has a rotation and/or flip.
  103. absl::optional<VideoTransformation> transformation;
  104. // Android only: if set, then this frame is not suitable for overlay, even
  105. // if ALLOW_OVERLAY is set. However, it allows us to process the overlay
  106. // to see if it would have been promoted, if it were backed by a SurfaceView
  107. // instead. This lets us figure out when SurfaceViews are appropriate.
  108. bool texture_owner = false;
  109. // Android & Windows only: if set, then this frame's resource would like to
  110. // be notified about its promotability to an overlay.
  111. bool wants_promotion_hint = false;
  112. // Windows only: set when frame is backed by a dcomp surface handle.
  113. bool dcomp_surface = false;
  114. // This video frame comes from protected content.
  115. bool protected_video = false;
  116. // This video frame is protected by hardware. This option is valid only if
  117. // PROTECTED_VIDEO is also set to true.
  118. bool hw_protected = false;
  119. // This video frame's shared image backing can support zero-copy WebGPU
  120. // import.
  121. bool is_webgpu_compatible = false;
  122. #if BUILDFLAG(USE_VAAPI)
  123. // The ID of the VA-API protected session used to decode this frame, if
  124. // applicable. The proper type is VAProtectedSessionID. However, in order to
  125. // avoid including the VA-API headers in this file, we use the underlying
  126. // type. Users of this field are expected to have compile-time assertions to
  127. // ensure it's safe to use this as a VAProtectedSessionID.
  128. //
  129. // Notes on IPC: this field should not be copied to the Mojo version of
  130. // VideoFrameMetadata because it should not cross process boundaries.
  131. absl::optional<unsigned int> hw_va_protected_session_id;
  132. #endif
  133. // An UnguessableToken that identifies VideoOverlayFactory that created
  134. // this VideoFrame. It's used by Cast to help with video hole punch.
  135. absl::optional<base::UnguessableToken> overlay_plane_id;
  136. // Whether this frame was decoded in a power efficient way.
  137. bool power_efficient = false;
  138. // Implemented only for single texture backed frames, true means the origin of
  139. // the texture is top left and false means bottom left.
  140. bool texture_origin_is_top_left = true;
  141. // CompositorFrameMetadata variables associated with this frame. Used for
  142. // remote debugging.
  143. // TODO(crbug.com/832220): Use a customized dictionary value instead of
  144. // using these keys directly.
  145. absl::optional<double> device_scale_factor;
  146. absl::optional<double> page_scale_factor;
  147. absl::optional<double> root_scroll_offset_x;
  148. absl::optional<double> root_scroll_offset_y;
  149. absl::optional<double> top_controls_visible_height;
  150. // If present, this field represents the local time at which the VideoFrame
  151. // was decoded from whichever format it was encoded in. Sometimes only
  152. // DECODE_END_TIME will be present.
  153. absl::optional<base::TimeTicks> decode_begin_time;
  154. absl::optional<base::TimeTicks> decode_end_time;
  155. // If present, this field represents the elapsed time from the submission of
  156. // the encoded packet with the same PTS as this frame to the decoder until
  157. // the decoded frame was ready for presentation.
  158. absl::optional<base::TimeDelta> processing_time;
  159. // The RTP timestamp associated with this video frame. Stored as a double
  160. // since base::DictionaryValue doesn't have a uint32_t type.
  161. //
  162. // https://w3c.github.io/webrtc-pc/#dom-rtcrtpcontributingsource-rtptimestamp
  163. absl::optional<double> rtp_timestamp;
  164. // For video frames coming from a remote source, this is the time the
  165. // encoded frame was received by the platform, i.e., the time at
  166. // which the last packet belonging to this frame was received over the
  167. // network.
  168. absl::optional<base::TimeTicks> receive_time;
  169. // If present, this field represents the duration this frame is ideally
  170. // expected to spend on the screen during playback. Unlike FRAME_DURATION
  171. // this field takes into account current playback rate.
  172. absl::optional<base::TimeDelta> wallclock_frame_duration;
  173. // WebRTC streams only: if present, this field represents the maximum
  174. // composition delay that is allowed for this frame. This is respected
  175. // in a best effort manner.
  176. // This is an experimental feature, see crbug.com/1138888 for more
  177. // information.
  178. absl::optional<int> maximum_composition_delay_in_frames;
  179. };
  180. } // namespace media
  181. #endif // MEDIA_BASE_VIDEO_FRAME_METADATA_H_