h264_decoder.h 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_GPU_H264_DECODER_H_
  5. #define MEDIA_GPU_H264_DECODER_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <vector>
  10. #include "base/containers/span.h"
  11. #include "base/memory/ref_counted.h"
  12. #include "media/base/limits.h"
  13. #include "media/base/subsample_entry.h"
  14. #include "media/base/video_types.h"
  15. #include "media/gpu/accelerated_video_decoder.h"
  16. #include "media/gpu/h264_dpb.h"
  17. #include "media/gpu/media_gpu_export.h"
  18. #include "media/video/h264_parser.h"
  19. #include "ui/gfx/geometry/rect.h"
  20. #include "ui/gfx/geometry/size.h"
  21. namespace media {
  22. // Clients of this class are expected to pass H264 Annex-B byte stream
  23. // and are expected to provide an implementation of H264Accelerator for
  24. // offloading final steps of the decoding process.
  25. //
  26. // This class must be created, called and destroyed on a single thread, and
  27. // does nothing internally on any other thread.
  28. class MEDIA_GPU_EXPORT H264Decoder : public AcceleratedVideoDecoder {
  29. public:
  30. class MEDIA_GPU_EXPORT H264Accelerator {
  31. public:
  32. // Methods may return kTryAgain if they need additional data (provided
  33. // independently) in order to proceed. Examples are things like not having
  34. // an appropriate key to decode encrypted content, or needing to wait
  35. // until hardware buffers are available. This is not considered an
  36. // unrecoverable error, but rather a pause to allow an application to
  37. // independently provide the required data. When H264Decoder::Decode()
  38. // is called again, it will attempt to resume processing of the stream
  39. // by calling the same method again.
  40. enum class Status {
  41. // Operation completed successfully.
  42. kOk,
  43. // Operation failed.
  44. kFail,
  45. // Operation failed because some external data is missing. Retry the same
  46. // operation later, once the data has been provided.
  47. kTryAgain,
  48. // Operation is not supported. Used by SetStream() to indicate that the
  49. // Accelerator can not handle this operation.
  50. kNotSupported,
  51. };
  52. H264Accelerator();
  53. H264Accelerator(const H264Accelerator&) = delete;
  54. H264Accelerator& operator=(const H264Accelerator&) = delete;
  55. virtual ~H264Accelerator();
  56. // Create a new H264Picture that the decoder client can use for decoding
  57. // and pass back to this accelerator for decoding or reference.
  58. // When the picture is no longer needed by decoder, it will just drop
  59. // its reference to it, and it may do so at any time.
  60. // Note that this may return nullptr if accelerator is not able to provide
  61. // any new pictures at given time. The decoder is expected to handle
  62. // this situation as normal and return from Decode() with kRanOutOfSurfaces.
  63. virtual scoped_refptr<H264Picture> CreateH264Picture() = 0;
  64. // Provides the raw NALU data for an SPS. The |sps| passed to
  65. // SubmitFrameMetadata() is always the most recent SPS passed to
  66. // ProcessSPS() with the same |seq_parameter_set_id|.
  67. virtual void ProcessSPS(const H264SPS* sps,
  68. base::span<const uint8_t> sps_nalu_data);
  69. // Provides the raw NALU data for a PPS. The |pps| passed to
  70. // SubmitFrameMetadata() is always the most recent PPS passed to
  71. // ProcessPPS() with the same |pic_parameter_set_id|.
  72. virtual void ProcessPPS(const H264PPS* pps,
  73. base::span<const uint8_t> pps_nalu_data);
  74. // Submit metadata for the current frame, providing the current |sps| and
  75. // |pps| for it, |dpb| has to contain all the pictures in DPB for current
  76. // frame, and |ref_pic_p0/b0/b1| as specified in the H264 spec. Note that
  77. // depending on the frame type, either p0, or b0 and b1 are used. |pic|
  78. // contains information about the picture for the current frame.
  79. // Note that this does not run decode in the accelerator and the decoder
  80. // is expected to follow this call with one or more SubmitSlice() calls
  81. // before calling SubmitDecode().
  82. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  83. // the accelerator needs additional data before being able to proceed.
  84. virtual Status SubmitFrameMetadata(
  85. const H264SPS* sps,
  86. const H264PPS* pps,
  87. const H264DPB& dpb,
  88. const H264Picture::Vector& ref_pic_listp0,
  89. const H264Picture::Vector& ref_pic_listb0,
  90. const H264Picture::Vector& ref_pic_listb1,
  91. scoped_refptr<H264Picture> pic) = 0;
  92. // Used for handling CENCv1 streams where the entire slice header, except
  93. // for the NALU type byte, is encrypted. |data| represents the encrypted
  94. // ranges which will include any SEI NALUs along with the encrypted slice
  95. // NALU. |subsamples| specifies what is encrypted and should have just a
  96. // single clear byte for each and the rest is encrypted. |sps_nalu_data|
  97. // and |pps_nalu_data| are the SPS and PPS NALUs respectively.
  98. // |slice_header_out| should have its fields filled in upon successful
  99. // return. Returns kOk if successful, kFail if there are errors, or
  100. // kTryAgain if the accelerator needs additional data before being able to
  101. // proceed.
  102. virtual Status ParseEncryptedSliceHeader(
  103. const std::vector<base::span<const uint8_t>>& data,
  104. const std::vector<SubsampleEntry>& subsamples,
  105. H264SliceHeader* slice_header_out);
  106. // Submit one slice for the current frame, passing the current |pps| and
  107. // |pic| (same as in SubmitFrameMetadata()), the parsed header for the
  108. // current slice in |slice_hdr|, and the reordered |ref_pic_listX|,
  109. // as per H264 spec.
  110. // |data| pointing to the full slice (including the unparsed header) of
  111. // |size| in bytes.
  112. // |subsamples| specifies which part of the slice data is encrypted.
  113. // This must be called one or more times per frame, before SubmitDecode().
  114. // Note that |data| does not have to remain valid after this call returns.
  115. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  116. // the accelerator needs additional data before being able to proceed.
  117. virtual Status SubmitSlice(
  118. const H264PPS* pps,
  119. const H264SliceHeader* slice_hdr,
  120. const H264Picture::Vector& ref_pic_list0,
  121. const H264Picture::Vector& ref_pic_list1,
  122. scoped_refptr<H264Picture> pic,
  123. const uint8_t* data,
  124. size_t size,
  125. const std::vector<SubsampleEntry>& subsamples) = 0;
  126. // Execute the decode in hardware for |pic|, using all the slices and
  127. // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since
  128. // the previous call to SubmitDecode().
  129. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  130. // the accelerator needs additional data before being able to proceed.
  131. virtual Status SubmitDecode(scoped_refptr<H264Picture> pic) = 0;
  132. // Schedule output (display) of |pic|. Note that returning from this
  133. // method does not mean that |pic| has already been outputted (displayed),
  134. // but guarantees that all pictures will be outputted in the same order
  135. // as this method was called for them. Decoder may drop its reference
  136. // to |pic| after calling this method.
  137. // Return true if successful.
  138. virtual bool OutputPicture(scoped_refptr<H264Picture> pic) = 0;
  139. // Reset any current state that may be cached in the accelerator, dropping
  140. // any cached parameters/slices that have not been committed yet.
  141. virtual void Reset() = 0;
  142. // Notifies the accelerator whenever there is a new stream to process.
  143. // |stream| is the data in annex B format, which may include SPS and PPS
  144. // NALUs when there is a configuration change. The first frame must contain
  145. // the SPS and PPS NALUs. SPS and PPS NALUs may not be encrypted.
  146. // |decrypt_config| is the config for decrypting the stream. The accelerator
  147. // should use |decrypt_config| to keep track of the parts of |stream| that
  148. // are encrypted. If kTryAgain is returned, the decoder will retry this call
  149. // later. This method has a default implementation that returns
  150. // kNotSupported.
  151. virtual Status SetStream(base::span<const uint8_t> stream,
  152. const DecryptConfig* decrypt_config);
  153. };
  154. H264Decoder(std::unique_ptr<H264Accelerator> accelerator,
  155. VideoCodecProfile profile,
  156. const VideoColorSpace& container_color_space = VideoColorSpace());
  157. H264Decoder(const H264Decoder&) = delete;
  158. H264Decoder& operator=(const H264Decoder&) = delete;
  159. ~H264Decoder() override;
  160. // AcceleratedVideoDecoder implementation.
  161. void SetStream(int32_t id, const DecoderBuffer& decoder) override;
  162. [[nodiscard]] bool Flush() override;
  163. void Reset() override;
  164. [[nodiscard]] DecodeResult Decode() override;
  165. gfx::Size GetPicSize() const override;
  166. gfx::Rect GetVisibleRect() const override;
  167. VideoCodecProfile GetProfile() const override;
  168. uint8_t GetBitDepth() const override;
  169. VideoChromaSampling GetChromaSampling() const override;
  170. size_t GetRequiredNumOfPictures() const override;
  171. size_t GetNumReferenceFrames() const override;
  172. // Return true if we need to start a new picture.
  173. static bool IsNewPrimaryCodedPicture(const H264Picture* curr_pic,
  174. int curr_pps_id,
  175. const H264SPS* sps,
  176. const H264SliceHeader& slice_hdr);
  177. // Fill a H264Picture in |pic| from given |sps| and |slice_hdr|. Return false
  178. // when there is an error.
  179. static bool FillH264PictureFromSliceHeader(const H264SPS* sps,
  180. const H264SliceHeader& slice_hdr,
  181. H264Picture* pic);
  182. private:
  183. // Internal state of the decoder.
  184. enum class State {
  185. // After initialization, need an SPS.
  186. kNeedStreamMetadata,
  187. // Ready to decode from any point.
  188. kDecoding,
  189. // After Reset(), need a resume point.
  190. kAfterReset,
  191. // The following keep track of what step is next in Decode() processing
  192. // in order to resume properly after H264Decoder::kTryAgain (or another
  193. // retryable error) is returned. The next time Decode() is called the call
  194. // that previously failed will be retried and execution continues from
  195. // there (if possible).
  196. kParseSliceHeader,
  197. kTryPreprocessCurrentSlice,
  198. kEnsurePicture,
  199. kTryNewFrame,
  200. kTryCurrentSlice,
  201. // Error in decode, can't continue.
  202. kError,
  203. };
  204. // Process H264 stream structures.
  205. bool ProcessSPS(int sps_id, bool* need_new_buffers);
  206. // Processes a CENCv1 encrypted slice header and fills in |curr_slice_hdr_|
  207. // with the relevant parsed fields.
  208. H264Accelerator::Status ProcessEncryptedSliceHeader(
  209. const std::vector<SubsampleEntry>& subsamples);
  210. // Process current slice header to discover if we need to start a new picture,
  211. // finishing up the current one.
  212. H264Accelerator::Status PreprocessCurrentSlice();
  213. // Process current slice as a slice of the current picture.
  214. H264Accelerator::Status ProcessCurrentSlice();
  215. // Initialize the current picture according to data in |slice_hdr|.
  216. bool InitCurrPicture(const H264SliceHeader* slice_hdr);
  217. // Initialize |pic| as a "non-existing" picture (see spec) with |frame_num|,
  218. // to be used for frame gap concealment.
  219. bool InitNonexistingPicture(scoped_refptr<H264Picture> pic, int frame_num);
  220. // Calculate picture order counts for |pic| on initialization
  221. // of a new frame (see spec).
  222. bool CalculatePicOrderCounts(scoped_refptr<H264Picture> pic);
  223. // Update PicNum values in pictures stored in DPB on creation of
  224. // a picture with |frame_num|.
  225. void UpdatePicNums(int frame_num);
  226. bool UpdateMaxNumReorderFrames(const H264SPS* sps);
  227. // Prepare reference picture lists for the current frame.
  228. void PrepareRefPicLists();
  229. // Prepare reference picture lists for the given slice.
  230. bool ModifyReferencePicLists(const H264SliceHeader* slice_hdr,
  231. H264Picture::Vector* ref_pic_list0,
  232. H264Picture::Vector* ref_pic_list1);
  233. // Construct initial reference picture lists for use in decoding of
  234. // P and B pictures (see 8.2.4 in spec).
  235. void ConstructReferencePicListsP();
  236. void ConstructReferencePicListsB();
  237. // Helper functions for reference list construction, per spec.
  238. int PicNumF(const H264Picture& pic);
  239. int LongTermPicNumF(const H264Picture& pic);
  240. // Perform the reference picture lists' modification (reordering), as
  241. // specified in spec (8.2.4).
  242. //
  243. // |list| indicates list number and should be either 0 or 1.
  244. bool ModifyReferencePicList(const H264SliceHeader* slice_hdr,
  245. int list,
  246. H264Picture::Vector* ref_pic_listx);
  247. // Perform reference picture memory management operations (marking/unmarking
  248. // of reference pictures, long term picture management, discarding, etc.).
  249. // See 8.2.5 in spec.
  250. bool HandleMemoryManagementOps(scoped_refptr<H264Picture> pic);
  251. bool ReferencePictureMarking(scoped_refptr<H264Picture> pic);
  252. bool SlidingWindowPictureMarking();
  253. // Handle a gap in frame_num in the stream up to |frame_num|, by creating
  254. // "non-existing" pictures (see spec).
  255. bool HandleFrameNumGap(int frame_num);
  256. // Start processing a new frame.
  257. H264Accelerator::Status StartNewFrame(const H264SliceHeader* slice_hdr);
  258. // All data for a frame received, process it and decode.
  259. H264Accelerator::Status FinishPrevFrameIfPresent();
  260. // Called after we are done processing |pic|. Performs all operations to be
  261. // done after decoding, including DPB management, reference picture marking
  262. // and memory management operations.
  263. // This will also output pictures if any have become ready to be outputted
  264. // after processing |pic|.
  265. bool FinishPicture(scoped_refptr<H264Picture> pic);
  266. // Clear DPB contents and remove all surfaces in DPB from *in_use_ list.
  267. // Cleared pictures will be made available for decode, unless they are
  268. // at client waiting to be displayed.
  269. void ClearDPB();
  270. // Commits all pending data for HW decoder and starts HW decoder.
  271. H264Accelerator::Status DecodePicture();
  272. // Notifies client that a picture is ready for output.
  273. bool OutputPic(scoped_refptr<H264Picture> pic);
  274. // Output all pictures in DPB that have not been outputted yet.
  275. bool OutputAllRemainingPics();
  276. // Decoder state.
  277. State state_;
  278. // The colorspace for the h264 container.
  279. const VideoColorSpace container_color_space_;
  280. // Parser in use.
  281. H264Parser parser_;
  282. // Most recent call to SetStream().
  283. const uint8_t* current_stream_ = nullptr;
  284. size_t current_stream_size_ = 0;
  285. // Decrypting config for the most recent data passed to SetStream().
  286. std::unique_ptr<DecryptConfig> current_decrypt_config_;
  287. // Keep track of when SetStream() is called so that
  288. // H264Accelerator::SetStream() can be called.
  289. bool current_stream_has_been_changed_ = false;
  290. // DPB in use.
  291. H264DPB dpb_;
  292. // Current stream buffer id; to be assigned to pictures decoded from it.
  293. int32_t stream_id_ = -1;
  294. // Picture currently being processed/decoded.
  295. scoped_refptr<H264Picture> curr_pic_;
  296. // Reference picture lists, constructed for each frame.
  297. H264Picture::Vector ref_pic_list_p0_;
  298. H264Picture::Vector ref_pic_list_b0_;
  299. H264Picture::Vector ref_pic_list_b1_;
  300. // Global state values, needed in decoding. See spec.
  301. int max_frame_num_;
  302. int max_pic_num_;
  303. int max_long_term_frame_idx_;
  304. size_t max_num_reorder_frames_;
  305. int prev_frame_num_;
  306. int prev_ref_frame_num_;
  307. int prev_frame_num_offset_;
  308. bool prev_has_memmgmnt5_;
  309. // Values related to previously decoded reference picture.
  310. bool prev_ref_has_memmgmnt5_;
  311. int prev_ref_top_field_order_cnt_;
  312. int prev_ref_pic_order_cnt_msb_;
  313. int prev_ref_pic_order_cnt_lsb_;
  314. H264Picture::Field prev_ref_field_;
  315. // Currently active SPS and PPS.
  316. int curr_sps_id_;
  317. int curr_pps_id_;
  318. // Last PPS that was parsed. Used for full sample encryption, which has the
  319. // assumption this is streaming content which does not switch between
  320. // different PPSes in the stream (they are present once in the container for
  321. // the stream).
  322. int last_parsed_pps_id_;
  323. // Current NALU and slice header being processed.
  324. std::unique_ptr<H264NALU> curr_nalu_;
  325. std::unique_ptr<H264SliceHeader> curr_slice_hdr_;
  326. // Encrypted SEI NALUs preceding a fully encrypted slice NALU. We need to
  327. // save these that are part of a single sample so they can all be decrypted
  328. // together.
  329. std::vector<base::span<const uint8_t>> encrypted_sei_nalus_;
  330. std::vector<SubsampleEntry> sei_subsamples_;
  331. // These are absl::nullopt unless get recovery point SEI message after Reset.
  332. // A frame_num of the frame at output order that is correct in content.
  333. absl::optional<int> recovery_frame_num_;
  334. // A value in the recovery point SEI message to compute |recovery_frame_num_|
  335. // later.
  336. absl::optional<int> recovery_frame_cnt_;
  337. // Output picture size.
  338. gfx::Size pic_size_;
  339. // Output visible cropping rect.
  340. gfx::Rect visible_rect_;
  341. // Profile of input bitstream.
  342. VideoCodecProfile profile_;
  343. // Bit depth of input bitstream.
  344. uint8_t bit_depth_ = 0;
  345. // Chroma subsampling format of input bitstream.
  346. VideoChromaSampling chroma_sampling_ = VideoChromaSampling::kUnknown;
  347. // PicOrderCount of the previously outputted frame.
  348. int last_output_poc_;
  349. const std::unique_ptr<H264Accelerator> accelerator_;
  350. };
  351. } // namespace media
  352. #endif // MEDIA_GPU_H264_DECODER_H_