frame_sink_video_capture.mojom 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. module viz.mojom;
  5. import "media/capture/mojom/video_capture_buffer.mojom";
  6. import "media/capture/mojom/video_capture_types.mojom";
  7. import "media/mojo/mojom/media_types.mojom";
  8. import "mojo/public/mojom/base/time.mojom";
  9. import "mojo/public/mojom/base/token.mojom";
  10. import "mojo/public/mojom/base/shared_memory.mojom";
  11. import "services/viz/public/mojom/compositing/frame_sink_id.mojom";
  12. import "services/viz/public/mojom/compositing/subtree_capture_id.mojom";
  13. import "skia/public/mojom/bitmap.mojom";
  14. import "ui/gfx/geometry/mojom/geometry.mojom";
  15. import "ui/gfx/mojom/color_space.mojom";
  16. // Provided with each call to FrameSinkVideoConsumer::OnFrameCaptured() so that
  17. // the consumer can notify the capturer the instant it is done consuming the
  18. // frame, and then later (optionally) provide utilization feedback.
  19. interface FrameSinkVideoConsumerFrameCallbacks {
  20. // Informs the capturer that the consumer is done reading the content of the
  21. // video frame.
  22. Done();
  23. // |feedback| contains consumer feedback like resource utilization,
  24. // maximum requested frame-rate and resolution.
  25. ProvideFeedback(media.mojom.VideoCaptureFeedback feedback);
  26. };
  27. // Provided with FrameSinkVideoConsumer::ChangeTarget() to indicate what
  28. // target should be selected for capture.
  29. union VideoCaptureSubTarget {
  30. SubtreeCaptureId subtree_capture_id;
  31. mojo_base.mojom.Token region_capture_crop_id;
  32. };
  33. struct VideoCaptureTarget {
  34. FrameSinkId frame_sink_id;
  35. VideoCaptureSubTarget? sub_target;
  36. };
  37. // Interface for a consumer that receives frames and notifications related to
  38. // capture of the source content. An instance that implements this interface is
  39. // provided to FrameSinkVideoCapturer.Start().
  40. interface FrameSinkVideoConsumer {
  41. // Called to deliver each frame to the consumer. |data| contains the video
  42. // frame's image data, and is valid for reading until the consumer notifies
  43. // the service that it is Done(). |info| is used to interpret the
  44. // format/layout of the data, and also contains the frame timestamps and other
  45. // metadata (the following media::VideoFrameMetadata keys are set:
  46. // CAPTURE_BEGIN_TIME, CAPTURE_END_TIME, CAPTURE_COUNTER, CAPTURE_UPDATE_RECT,
  47. // COLOR_SPACE, FRAME_DURATION, INTERACTIVE_CONTENT, REFERENCE_TIME).
  48. // |content_rect| is the region of the frame that contains the captured
  49. // content, with the rest of the frame having been letterboxed to adhere to
  50. // resolution constraints.
  51. OnFrameCaptured(media.mojom.VideoBufferHandle data,
  52. media.mojom.VideoFrameInfo info,
  53. gfx.mojom.Rect content_rect,
  54. pending_remote<FrameSinkVideoConsumerFrameCallbacks>
  55. callbacks);
  56. // Called to indicate that a new |crop_version| has been set, and that all
  57. // frames which will subsequently be delivered using OnFrameCaptured(),
  58. // will have a crop-version that is greater-than-or-equal-to this one.
  59. // This allows the Promise which cropTo() returned to be resolved promptly,
  60. // even if frames are not being produced over the track at the moment,
  61. // which can happen if the track is cropped to an invisible target, or is
  62. // paused, or is muted.
  63. OnNewCropVersion(uint32 crop_version);
  64. // Called when a frame is *not* delivered because Region Capture was applied,
  65. // but the cropped-to region consists of zero pixels. This is needed because
  66. // the blue border that is normally drawn around the cropped-to region should
  67. // be updated.
  68. OnFrameWithEmptyRegionCapture();
  69. // Indicates that OnFrameCaptured() will not be called again, an end-of-stream
  70. // signal.
  71. OnStopped();
  72. // Called to deliver a log message to the consumer.
  73. // Currently used to populate WebRTC native logs.
  74. OnLog(string message);
  75. };
  76. // Used to express the preference on the type of VideoBufferHandles that the
  77. // consumer would like to obtain from the capturer. By default, the video
  78. // buffers are returned via read only shared memory.
  79. enum BufferFormatPreference {
  80. kDefault = 0,
  81. kPreferGpuMemoryBuffer,
  82. };
  83. // Interface to an implementation that captures the frames of a
  84. // CompositorFrameSink's surface as a video stream. During operation, the
  85. // "target" frame sink can be changed, as needed, to continue capturing the
  86. // desired content (e.g., a new web page in the same browser tab).
  87. //
  88. // The SetXYZ() methods set or change the video capture parameters and
  89. // constraints. They may be called any time before and/or after a Start() call.
  90. // Note that the consumer must not assume any changes have taken effect until
  91. // this is reflected in some later delivered frame.
  92. //
  93. // General usage: A capturer is created, its SetXYZ() methods are called,
  94. // followed by ChangeTarget() and then Start(). Then, later, when video capture
  95. // should suspend, Stop() is called. After that point, Start() can be called
  96. // again to resume capture; or the mojo binding can be dropped to auto-terminate
  97. // the capturer.
  98. interface FrameSinkVideoCapturer {
  99. // Specifies the pixel format to use.
  100. //
  101. // Default, if never called: PIXEL_FORMAT_I420.
  102. SetFormat(media.mojom.VideoPixelFormat format);
  103. // Specifies the maximum rate of capture in terms of a minimum time period
  104. // (min_period = 1/max_frame_rate).
  105. //
  106. // Default, if never called: a very conservative 1/5 second (for 5 FPS
  107. // max). Usually 30-60 FPS is desired for frame sinks that may contain video
  108. // content.
  109. SetMinCapturePeriod(mojo_base.mojom.TimeDelta min_period);
  110. // Specifies the minimum amount of time that must elapse between changing the
  111. // size of video frames. This can be set to zero to disable resize throttling,
  112. // which means each video frame could have a different size. Note that
  113. // setting this period too low can result in erratic frame sizing behaviour;
  114. // consider disabling auto-throttling in such cases.
  115. //
  116. // Default, if never called: 3 seconds.
  117. SetMinSizeChangePeriod(mojo_base.mojom.TimeDelta min_period);
  118. // Specifies the range (if any) of acceptable capture resolutions and whether
  119. // a fixed aspect ratio is required. When |min_size| is equal to |max_size|,
  120. // capture resolution will be held constant. Otherwise, capture resolution
  121. // will dynamically adapt over time as the source content size and system
  122. // resource utilization change. If a fixed aspect ratio is required, the
  123. // aspect ratio of |max_size| is used and the implementation will letterbox
  124. // around the captured content when needed.
  125. //
  126. // Default, if never called: Fixed at 640x360 (standard definition, 16:9).
  127. // Usually, 320x180↔3840x2160 (auto-adapt, 16:9) is desired when the consumer
  128. // can handle variable frame resolutions.
  129. SetResolutionConstraints(gfx.mojom.Size min_size, gfx.mojom.Size max_size,
  130. bool use_fixed_aspect_ratio);
  131. // Specifies whether capture resolution should be automatically adjusted
  132. // within the bounds provided to SetReslutionConstraints in response to
  133. // utilization feedback from the consumer and the buffer pool. When set to
  134. // false, the capturer will always use the highest resolution within
  135. // constraints that doesn't exceed the source size.
  136. //
  137. // Default, if never called: true.
  138. SetAutoThrottlingEnabled(bool enabled);
  139. // Targets a different compositor frame sink. This may be called anytime,
  140. // before or after Start(). If |target| is null, capture will suspend
  141. // until a new frame sink target is set. If |target| is provided, the
  142. // frame sink identifier associated with it must be valid.
  143. // If the |sub_target| is a valid subtree capture id, the capturer will
  144. // capture a render pass associated with a layer subtree under the target
  145. // frame sink, which is identifiable by that
  146. // |subtree_capture_id|. Else, if the |sub_target| is a valid region capture
  147. // crop id, only the region associated with that crop id will be captured.
  148. // Otherwise, the capturer captures the root render pass of the target frame
  149. // sink.
  150. // |crop_version| is a non-decreasing value (for subsequent calls).
  151. // It is increased whenever the crop-target changes (new sub_target).
  152. // Viz annotates frames with this value, and that allows Blink to
  153. // discover when a crop has been applied. Blink will resolve a Promise it
  154. // gave the Web-application when the crop was applied, and this way,
  155. // the application will know when to start consuming cropped frames.
  156. // Note that only tab-capture frames are croppable. Anything else,
  157. // such as window-capture for instance, should always set |crop_version| to 0.
  158. ChangeTarget(VideoCaptureTarget? target, uint32 crop_version);
  159. // Starts emitting video frames to the given |consumer|.
  160. // The caller can express if the |consumer| prefers to receive video buffers
  161. // via read only shared memory or via GPU memory buffers by controlling
  162. // |buffer_format_preference| parameter. If a consumer only supports read only
  163. // shared memory, `BufferFormatPreference::kDefault` value should be used.
  164. Start(pending_remote<FrameSinkVideoConsumer> consumer,
  165. BufferFormatPreference buffer_format_preference);
  166. // Stops capturing "soon." Meaning, no new frame captures will be started, but
  167. // there may already be some in-flight for delivery. Wait for the "end of
  168. // stream" signal provided by FrameSinkVideoConsumer.OnStopped().
  169. Stop();
  170. // Requests the capturer send a duplicate of the last frame. This is used to
  171. // resolve occasional "picture loss" issues consumer-side.
  172. RequestRefreshFrame();
  173. // Creates an overlay to be rendered within each captured video frame. The
  174. // |stacking_index| is an arbitrary value that determines whether to render
  175. // this overlay before/after other overlays. Greater values mean "after" and
  176. // "on top of" those with lesser values. Specifying the same index as an
  177. // existing overlay will cause the existing one to be dropped and replaced
  178. // with a new one.
  179. CreateOverlay(int32 stacking_index,
  180. pending_receiver<FrameSinkVideoCaptureOverlay> receiver);
  181. };
  182. // Control interface for a small image to be composited on top of each captured
  183. // video frame. This allows clients to, for example, have the capturer render
  184. // mouse cursors or debug info boxes on top of the captured content.
  185. interface FrameSinkVideoCaptureOverlay {
  186. // Sets/Changes the overlay |image| and its position and size, relative to the
  187. // source content. |bounds| consists of coordinates where the range [0.0,1.0)
  188. // indicates the relative position+size within the bounds of the source
  189. // content (e.g., 0.0 refers to the top or left edge; 1.0 to just after the
  190. // bottom or right edge). Pass empty |bounds| to temporarily hide the overlay
  191. // until a later call to SetBounds().
  192. SetImageAndBounds(skia.mojom.BitmapN32 image,
  193. gfx.mojom.RectF bounds);
  194. // Changes the bounds of the previously-set image, showing the overlay if
  195. // non-empty bounds are provided, and hiding the overlay otherwise. |bounds|
  196. // has the same semantics as described in ShowImageAt().
  197. SetBounds(gfx.mojom.RectF bounds);
  198. };