video_encode_accelerator.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. // Copyright 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_
  5. #define MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <vector>
  10. #include "base/callback_forward.h"
  11. #include "base/memory/scoped_refptr.h"
  12. #include "base/time/time.h"
  13. #include "media/base/bitrate.h"
  14. #include "media/base/media_export.h"
  15. #include "media/base/svc_scalability_mode.h"
  16. #include "media/base/video_bitrate_allocation.h"
  17. #include "media/base/video_codecs.h"
  18. #include "media/base/video_types.h"
  19. #include "media/video/video_encoder_info.h"
  20. #include "third_party/abseil-cpp/absl/types/optional.h"
  21. namespace media {
  22. class BitstreamBuffer;
  23. class MediaLog;
  24. class VideoFrame;
  25. // Metadata for a H264 bitstream buffer.
  26. // |temporal_idx| indicates the temporal index for this frame.
  27. // |layer_sync| is true iff this frame has |temporal_idx| > 0 and does NOT
  28. // reference any reference buffer containing a frame with
  29. // temporal_idx > 0.
  30. struct MEDIA_EXPORT H264Metadata final {
  31. uint8_t temporal_idx = 0;
  32. bool layer_sync = false;
  33. };
  34. // Metadata for a VP8 bitstream buffer.
  35. // |non_reference| is true iff this frame does not update any reference buffer,
  36. // meaning dropping this frame still results in a decodable
  37. // stream.
  38. // |temporal_idx| indicates the temporal index for this frame.
  39. // |layer_sync| is true iff this frame has |temporal_idx| > 0 and does NOT
  40. // reference any reference buffer containing a frame with
  41. // temporal_idx > 0.
  42. struct MEDIA_EXPORT Vp8Metadata final {
  43. bool non_reference = false;
  44. uint8_t temporal_idx = 0;
  45. bool layer_sync = false;
  46. };
  47. // Metadata for a VP9 bitstream buffer, this struct resembles
  48. // webrtc::CodecSpecificInfoVP9 [1]
  49. // https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/video_coding/include/video_codec_interface.h;l=56;drc=e904161cecbe5e2ca31382e2a62fc776151bb8f2
  50. struct MEDIA_EXPORT Vp9Metadata final {
  51. Vp9Metadata();
  52. ~Vp9Metadata();
  53. Vp9Metadata(const Vp9Metadata&);
  54. // True iff this layer frame is dependent on previously coded frame(s).
  55. bool inter_pic_predicted = false;
  56. // True iff this frame only references TL0 frames.
  57. bool temporal_up_switch = false;
  58. // True iff frame is referenced by upper spatial layer frame.
  59. bool referenced_by_upper_spatial_layers = false;
  60. // True iff frame is dependent on directly lower spatial layer frame.
  61. bool reference_lower_spatial_layers = false;
  62. // True iff frame is last layer frame of picture.
  63. bool end_of_picture = true;
  64. // The temporal index for this frame.
  65. uint8_t temporal_idx = 0;
  66. // The spatial index for this frame.
  67. uint8_t spatial_idx = 0;
  68. // The resolutions of active spatial layers, filled if and only if keyframe or
  69. // the number of active spatial layers is changed.
  70. std::vector<gfx::Size> spatial_layer_resolutions;
  71. // The differences between the picture id of this frame and picture ids
  72. // of reference frames, only be filled for non key frames.
  73. std::vector<uint8_t> p_diffs;
  74. };
  75. // Metadata for an AV1 bitstream buffer.
  76. struct MEDIA_EXPORT Av1Metadata final {
  77. Av1Metadata();
  78. ~Av1Metadata();
  79. Av1Metadata(const Av1Metadata&);
  80. // True iff this layer frame is dependent on previously coded frame(s).
  81. bool inter_pic_predicted = false;
  82. // True iff this frame is a switch point between sequences.
  83. bool switch_frame = false;
  84. // True iff frame is last layer frame of picture.
  85. bool end_of_picture = true;
  86. // The temporal index for this frame.
  87. uint8_t temporal_idx = 0;
  88. // The spatial index for this frame.
  89. uint8_t spatial_idx = 0;
  90. // The resolutions of active spatial layers, filled if and only if keyframe or
  91. // the number of active spatial layers is changed.
  92. std::vector<gfx::Size> spatial_layer_resolutions;
  93. // The differences between the frame number of this frame and frame number
  94. // of referenced frames, only be to filled for non key frames.
  95. std::vector<uint8_t> f_diffs;
  96. };
  97. // Metadata associated with a bitstream buffer.
  98. // |payload_size| is the byte size of the used portion of the buffer.
  99. // |key_frame| is true if this delivered frame is a keyframe.
  100. // |timestamp| is the same timestamp as in VideoFrame passed to Encode().
  101. // |qp| is the quantizer value, default invalid qp value is -1 following
  102. // webrtc::EncodedImage.
  103. struct MEDIA_EXPORT BitstreamBufferMetadata final {
  104. BitstreamBufferMetadata();
  105. BitstreamBufferMetadata(const BitstreamBufferMetadata& other);
  106. BitstreamBufferMetadata& operator=(const BitstreamBufferMetadata& other);
  107. BitstreamBufferMetadata(BitstreamBufferMetadata&& other);
  108. BitstreamBufferMetadata(size_t payload_size_bytes,
  109. bool key_frame,
  110. base::TimeDelta timestamp);
  111. ~BitstreamBufferMetadata();
  112. size_t payload_size_bytes;
  113. bool key_frame;
  114. base::TimeDelta timestamp;
  115. int32_t qp = -1;
  116. // |h264|, |vp8| or |vp9| may be set, but not multiple of them. Presumably,
  117. // it's also possible for none of them to be set.
  118. absl::optional<H264Metadata> h264;
  119. absl::optional<Vp8Metadata> vp8;
  120. absl::optional<Vp9Metadata> vp9;
  121. absl::optional<Av1Metadata> av1;
  122. };
  123. // Video encoder interface.
  124. class MEDIA_EXPORT VideoEncodeAccelerator {
  125. public:
  126. // Bitmask values for supported rate control modes.
  127. enum SupportedRateControlMode : uint8_t {
  128. kNoMode = 0, // for uninitialized profiles only
  129. kConstantMode = 0b0001,
  130. kVariableMode = 0b0010,
  131. };
  132. // Specification of an encoding profile supported by an encoder.
  133. struct MEDIA_EXPORT SupportedProfile {
  134. SupportedProfile();
  135. SupportedProfile(
  136. VideoCodecProfile profile,
  137. const gfx::Size& max_resolution,
  138. uint32_t max_framerate_numerator = 0u,
  139. uint32_t max_framerate_denominator = 1u,
  140. SupportedRateControlMode rc_modes = kConstantMode,
  141. const std::vector<SVCScalabilityMode>& scalability_modes = {});
  142. SupportedProfile(const SupportedProfile& other);
  143. SupportedProfile& operator=(const SupportedProfile& other) = default;
  144. ~SupportedProfile();
  145. VideoCodecProfile profile;
  146. gfx::Size min_resolution;
  147. gfx::Size max_resolution;
  148. uint32_t max_framerate_numerator{0};
  149. uint32_t max_framerate_denominator{0};
  150. SupportedRateControlMode rate_control_modes = kNoMode;
  151. std::vector<SVCScalabilityMode> scalability_modes;
  152. };
  153. using SupportedProfiles = std::vector<SupportedProfile>;
  154. using FlushCallback = base::OnceCallback<void(bool)>;
  155. // Enumeration of potential errors generated by the API.
  156. enum Error {
  157. // An operation was attempted during an incompatible encoder state.
  158. kIllegalStateError,
  159. // Invalid argument was passed to an API method.
  160. kInvalidArgumentError,
  161. // A failure occurred at the GPU process or one of its dependencies.
  162. // Examples of such failures include GPU hardware failures, GPU driver
  163. // failures, GPU library failures, GPU process programming errors, and so
  164. // on.
  165. kPlatformFailureError,
  166. kErrorMax = kPlatformFailureError
  167. };
  168. // A default framerate for all VEA implementations.
  169. enum { kDefaultFramerate = 30 };
  170. // Parameters required for VEA initialization.
  171. struct MEDIA_EXPORT Config {
  172. // Indicates if video content should be treated as a "normal" camera feed
  173. // or as generated (e.g. screen capture).
  174. enum class ContentType { kCamera, kDisplay };
  175. enum class InterLayerPredMode : int {
  176. kOff = 0, // Inter-layer prediction is disabled.
  177. kOn = 1, // Inter-layer prediction is enabled.
  178. kOnKeyPic = 2 // Inter-layer prediction is enabled for key picture.
  179. };
  180. // Indicates the storage type of a video frame provided on Encode().
  181. // kShmem if a video frame has a shared memory.
  182. // kGpuMemoryBuffer if a video frame has a GpuMemoryBuffer.
  183. enum class StorageType { kShmem, kGpuMemoryBuffer };
  184. struct MEDIA_EXPORT SpatialLayer {
  185. // The encoder dimension of the spatial layer.
  186. int32_t width = 0;
  187. int32_t height = 0;
  188. // The bitrate of encoded output stream of the spatial layer in bits per
  189. // second.
  190. uint32_t bitrate_bps = 0u;
  191. uint32_t framerate = 0u;
  192. // The recommended maximum qp value of the spatial layer. VEA can ignore
  193. // this value.
  194. uint8_t max_qp = 0u;
  195. // The number of temporal layers of the spatial layer. The detail of
  196. // the temporal layer structure is up to VideoEncodeAccelerator.
  197. uint8_t num_of_temporal_layers = 0u;
  198. };
  199. Config();
  200. Config(const Config& config);
  201. Config(VideoPixelFormat input_format,
  202. const gfx::Size& input_visible_size,
  203. VideoCodecProfile output_profile,
  204. const Bitrate& bitrate,
  205. absl::optional<uint32_t> initial_framerate = absl::nullopt,
  206. absl::optional<uint32_t> gop_length = absl::nullopt,
  207. absl::optional<uint8_t> h264_output_level = absl::nullopt,
  208. bool is_constrained_h264 = false,
  209. absl::optional<StorageType> storage_type = absl::nullopt,
  210. ContentType content_type = ContentType::kCamera,
  211. const std::vector<SpatialLayer>& spatial_layers = {},
  212. InterLayerPredMode inter_layer_pred = InterLayerPredMode::kOnKeyPic);
  213. ~Config();
  214. std::string AsHumanReadableString() const;
  215. bool HasTemporalLayer() const;
  216. bool HasSpatialLayer() const;
  217. // Frame format of input stream (as would be reported by
  218. // VideoFrame::format() for frames passed to Encode()).
  219. VideoPixelFormat input_format;
  220. // Resolution of input stream (as would be reported by
  221. // VideoFrame::visible_rect().size() for frames passed to Encode()).
  222. gfx::Size input_visible_size;
  223. // Codec profile of encoded output stream.
  224. VideoCodecProfile output_profile;
  225. // Configuration details for the bitrate, indicating the bitrate mode (ex.
  226. // variable or constant) and target bitrate.
  227. Bitrate bitrate;
  228. // Initial encoding framerate in frames per second. This is optional and
  229. // VideoEncodeAccelerator should use |kDefaultFramerate| if not given.
  230. absl::optional<uint32_t> initial_framerate;
  231. // Group of picture length for encoded output stream, indicates the
  232. // distance between two key frames, i.e. IPPPIPPP would be represent as 4.
  233. absl::optional<uint32_t> gop_length;
  234. // Codec level of encoded output stream for H264 only. This value should
  235. // be aligned to the H264 standard definition of SPS.level_idc.
  236. // If this is not given, VideoEncodeAccelerator selects one of proper H.264
  237. // levels for |input_visible_size| and |initial_framerate|.
  238. absl::optional<uint8_t> h264_output_level;
  239. // Indicates baseline profile or constrained baseline profile for H264 only.
  240. bool is_constrained_h264;
  241. // The storage type of video frame provided on Encode().
  242. // If no value is set, VEA doesn't check the storage type of video frame on
  243. // Encode().
  244. // This is kShmem iff a video frame is mapped in user space.
  245. // This is kDmabuf iff a video frame has dmabuf.
  246. absl::optional<StorageType> storage_type;
  247. // Indicates captured video (from a camera) or generated (screen grabber).
  248. // Screen content has a number of special properties such as lack of noise,
  249. // burstiness of motion and requirements for readability of small text in
  250. // bright colors. With this content hint the encoder may choose to optimize
  251. // for the given use case.
  252. ContentType content_type;
  253. // The configuration for spatial layers. This is not empty if and only if
  254. // either spatial or temporal layer encoding is configured. When this is not
  255. // empty, VideoEncodeAccelerator should refer the width, height, bitrate and
  256. // etc. of |spatial_layers|.
  257. std::vector<SpatialLayer> spatial_layers;
  258. // Indicates the inter layer prediction mode for SVC encoding.
  259. InterLayerPredMode inter_layer_pred;
  260. // This flag forces the encoder to use low latency mode, suitable for
  261. // RTC use cases.
  262. bool require_low_delay = true;
  263. };
  264. // Interface for clients that use VideoEncodeAccelerator. These callbacks will
  265. // not be made unless Initialize() has returned successfully.
  266. class MEDIA_EXPORT Client {
  267. public:
  268. // Callback to tell the client what size of frames and buffers to provide
  269. // for input and output. The VEA disclaims use or ownership of all
  270. // previously provided buffers once this callback is made.
  271. // Parameters:
  272. // |input_count| is the number of input VideoFrames required for encoding.
  273. // The client should be prepared to feed at least this many frames into the
  274. // encoder before being returned any input frames, since the encoder may
  275. // need to hold onto some subset of inputs as reference pictures.
  276. // |input_coded_size| is the logical size of the input frames (as reported
  277. // by VideoFrame::coded_size()) to encode, in pixels. The encoder may have
  278. // hardware alignment requirements that make this different from
  279. // |input_visible_size|, as requested in Initialize(), in which case the
  280. // input VideoFrame to Encode() should be padded appropriately.
  281. // |output_buffer_size| is the required size of output buffers for this
  282. // encoder in bytes.
  283. virtual void RequireBitstreamBuffers(unsigned int input_count,
  284. const gfx::Size& input_coded_size,
  285. size_t output_buffer_size) = 0;
  286. // Callback to deliver encoded bitstream buffers. Ownership of the buffer
  287. // is transferred back to the VEA::Client once this callback is made.
  288. // Parameters:
  289. // |bitstream_buffer_id| is the id of the buffer that is ready.
  290. // |metadata| contains data such as payload size and timestamp. See above.
  291. virtual void BitstreamBufferReady(
  292. int32_t bitstream_buffer_id,
  293. const BitstreamBufferMetadata& metadata) = 0;
  294. // Error notification callback. Note that errors in Initialize() will not be
  295. // reported here, but will instead be indicated by a false return value
  296. // there.
  297. virtual void NotifyError(Error error) = 0;
  298. // Call VideoEncoderInfo of the VEA is changed.
  299. virtual void NotifyEncoderInfoChange(const VideoEncoderInfo& info);
  300. protected:
  301. // Clients are not owned by VEA instances and should not be deleted through
  302. // these pointers.
  303. virtual ~Client() {}
  304. };
  305. // Video encoder functions.
  306. // Returns a list of the supported codec profiles of the video encoder. This
  307. // can be called before Initialize().
  308. virtual SupportedProfiles GetSupportedProfiles() = 0;
  309. // Returns a list of the supported codec profiles of the video encoder,
  310. // similar to GetSupportedProfiles(), but this function only populates:
  311. // codec, framerate range and resolution range.
  312. //
  313. // Populating things like SVC modes can take a lot of time and they are
  314. // not always used. See https://crbug.com/1263196
  315. virtual SupportedProfiles GetSupportedProfilesLight();
  316. // Initializes the video encoder with specific configuration. Called once per
  317. // encoder construction. This call is synchronous and returns true iff
  318. // initialization is successful.
  319. // TODO(mcasas): Update to asynchronous, https://crbug.com/744210.
  320. // Parameters:
  321. // |config| contains the initialization parameters.
  322. // |client| is the client of this video encoder. The provided pointer must
  323. // be valid until Destroy() is called.
  324. // |media_log| is used to report error messages.
  325. // TODO(sheu): handle resolution changes. http://crbug.com/249944
  326. virtual bool Initialize(const Config& config,
  327. Client* client,
  328. std::unique_ptr<MediaLog> media_log) = 0;
  329. // Encodes the given frame.
  330. // The storage type of |frame| must be the |storage_type| if it is specified
  331. // in Initialize().
  332. // TODO(crbug.com/895230): Raise an error if the storage types are mismatch.
  333. // Parameters:
  334. // |frame| is the VideoFrame that is to be encoded.
  335. // |force_keyframe| forces the encoding of a keyframe for this frame.
  336. virtual void Encode(scoped_refptr<VideoFrame> frame, bool force_keyframe) = 0;
  337. // Send a bitstream buffer to the encoder to be used for storing future
  338. // encoded output. Each call here with a given |buffer| will cause the buffer
  339. // to be filled once, then returned with BitstreamBufferReady().
  340. // Parameters:
  341. // |buffer| is the bitstream buffer to use for output.
  342. virtual void UseOutputBitstreamBuffer(BitstreamBuffer buffer) = 0;
  343. // Request a change to the encoding parameters. This is only a request,
  344. // fulfilled on a best-effort basis.
  345. // Parameters:
  346. // |bitrate| is the requested new bitrate. The bitrate mode cannot be changed
  347. // using this method and attempting to do so will result in an error.
  348. // Instead, re-create a VideoEncodeAccelerator. |framerate| is the requested
  349. // new framerate, in frames per second.
  350. virtual void RequestEncodingParametersChange(const Bitrate& bitrate,
  351. uint32_t framerate) = 0;
  352. // Request a change to the encoding parameters. This is only a request,
  353. // fulfilled on a best-effort basis. If not implemented, default behavior is
  354. // to get the sum over layers and pass to version with bitrate as uint32_t.
  355. // Parameters:
  356. // |bitrate| is the requested new bitrate, per spatial and temporal layer.
  357. // |framerate| is the requested new framerate, in frames per second.
  358. virtual void RequestEncodingParametersChange(
  359. const VideoBitrateAllocation& bitrate,
  360. uint32_t framerate);
  361. // Destroys the encoder: all pending inputs and outputs are dropped
  362. // immediately and the component is freed. This call may asynchronously free
  363. // system resources, but its client-visible effects are synchronous. After
  364. // this method returns no more callbacks will be made on the client. Deletes
  365. // |this| unconditionally, so make sure to drop all pointers to it!
  366. virtual void Destroy() = 0;
  367. // Flushes the encoder: all pending inputs will be encoded and all bitstreams
  368. // handed back to the client, and afterwards the |flush_callback| will be
  369. // called. The FlushCallback takes a boolean argument: |true| indicates the
  370. // flush is complete; |false| indicates the flush is cancelled due to errors
  371. // or destruction. The client should not invoke Flush() or Encode() while the
  372. // previous Flush() is not finished yet.
  373. virtual void Flush(FlushCallback flush_callback);
  374. // Returns true if the encoder support flush. This method must be called after
  375. // VEA has been initialized.
  376. virtual bool IsFlushSupported();
  377. // Returns true if the encoder supports automatic resize of GPU backed frames
  378. // to the size provided during encoder configuration.
  379. // This method must be called after VEA has been initialized.
  380. virtual bool IsGpuFrameResizeSupported();
  381. protected:
  382. // Do not delete directly; use Destroy() or own it with a scoped_ptr, which
  383. // will Destroy() it properly by default.
  384. virtual ~VideoEncodeAccelerator();
  385. };
  386. MEDIA_EXPORT bool operator==(const VideoEncodeAccelerator::SupportedProfile& l,
  387. const VideoEncodeAccelerator::SupportedProfile& r);
  388. MEDIA_EXPORT bool operator==(const Vp8Metadata& l, const Vp8Metadata& r);
  389. MEDIA_EXPORT bool operator==(const Vp9Metadata& l, const Vp9Metadata& r);
  390. MEDIA_EXPORT bool operator==(const BitstreamBufferMetadata& l,
  391. const BitstreamBufferMetadata& r);
  392. MEDIA_EXPORT bool operator==(
  393. const VideoEncodeAccelerator::Config::SpatialLayer& l,
  394. const VideoEncodeAccelerator::Config::SpatialLayer& r);
  395. MEDIA_EXPORT bool operator==(const VideoEncodeAccelerator::Config& l,
  396. const VideoEncodeAccelerator::Config& r);
  397. MEDIA_EXPORT inline VideoEncodeAccelerator::SupportedRateControlMode operator|(
  398. VideoEncodeAccelerator::SupportedRateControlMode lhs,
  399. VideoEncodeAccelerator::SupportedRateControlMode rhs) {
  400. return static_cast<VideoEncodeAccelerator::SupportedRateControlMode>(
  401. static_cast<uint8_t>(lhs) | static_cast<uint8_t>(rhs));
  402. }
  403. MEDIA_EXPORT inline VideoEncodeAccelerator::SupportedRateControlMode&
  404. operator|=(VideoEncodeAccelerator::SupportedRateControlMode& lhs,
  405. VideoEncodeAccelerator::SupportedRateControlMode rhs) {
  406. lhs = lhs | rhs;
  407. return lhs;
  408. }
  409. MEDIA_EXPORT inline VideoEncodeAccelerator::SupportedRateControlMode operator&(
  410. VideoEncodeAccelerator::SupportedRateControlMode lhs,
  411. VideoEncodeAccelerator::SupportedRateControlMode rhs) {
  412. return static_cast<VideoEncodeAccelerator::SupportedRateControlMode>(
  413. static_cast<uint8_t>(lhs) & static_cast<uint8_t>(rhs));
  414. }
  415. MEDIA_EXPORT inline VideoEncodeAccelerator::SupportedRateControlMode&
  416. operator&=(VideoEncodeAccelerator::SupportedRateControlMode& lhs,
  417. VideoEncodeAccelerator::SupportedRateControlMode rhs) {
  418. lhs = lhs & rhs;
  419. return lhs;
  420. }
  421. } // namespace media
  422. namespace std {
  423. // Specialize std::default_delete so that
  424. // std::unique_ptr<VideoEncodeAccelerator> uses "Destroy()" instead of trying to
  425. // use the destructor.
  426. template <>
  427. struct MEDIA_EXPORT default_delete<media::VideoEncodeAccelerator> {
  428. void operator()(media::VideoEncodeAccelerator* vea) const;
  429. };
  430. } // namespace std
  431. #endif // MEDIA_VIDEO_VIDEO_ENCODE_ACCELERATOR_H_