h265_decoder.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_GPU_H265_DECODER_H_
  5. #define MEDIA_GPU_H265_DECODER_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <memory>
  9. #include <vector>
  10. #include "base/containers/span.h"
  11. #include "base/memory/ref_counted.h"
  12. #include "media/base/decrypt_config.h"
  13. #include "media/base/subsample_entry.h"
  14. #include "media/base/video_codecs.h"
  15. #include "media/gpu/accelerated_video_decoder.h"
  16. #include "media/gpu/h265_dpb.h"
  17. #include "media/gpu/media_gpu_export.h"
  18. #include "media/video/h265_parser.h"
  19. #include "ui/gfx/geometry/rect.h"
  20. #include "ui/gfx/geometry/size.h"
  21. namespace media {
  22. // Clients of this class are expected to pass H265 Annex-B byte stream
  23. // and are expected to provide an implementation of H265Accelerator for
  24. // offloading final steps of the decoding process.
  25. //
  26. // This class must be created, called and destroyed on a single thread, and
  27. // does nothing internally on any other thread.
  28. //
  29. // It is expected that when a DecoderBuffer is submitted, that it will contain a
  30. // complete frame of data. Multiple slices per frame are handled. This class can
  31. // also handle multiple frames in a DecoderBuffer, but that condition should
  32. // never actually occur.
  33. class MEDIA_GPU_EXPORT H265Decoder final : public AcceleratedVideoDecoder {
  34. public:
  35. class MEDIA_GPU_EXPORT H265Accelerator {
  36. public:
  37. // Methods may return kTryAgain if they need additional data (provided
  38. // independently) in order to proceed. Examples are things like not having
  39. // an appropriate key to decode encrypted content, or needing to wait
  40. // until hardware buffers are available. This is not considered an
  41. // unrecoverable error, but rather a pause to allow an application to
  42. // independently provide the required data. When H265Decoder::Decode()
  43. // is called again, it will attempt to resume processing of the stream
  44. // by calling the same method again.
  45. enum class Status {
  46. // Operation completed successfully.
  47. kOk,
  48. // Operation failed.
  49. kFail,
  50. // Operation failed because some external data is missing. Retry the same
  51. // operation later, once the data has been provided.
  52. kTryAgain,
  53. // Operation is not supported. Used by SetStream() to indicate that the
  54. // Accelerator can not handle this operation.
  55. kNotSupported,
  56. };
  57. H265Accelerator();
  58. H265Accelerator(const H265Accelerator&) = delete;
  59. H265Accelerator& operator=(const H265Accelerator&) = delete;
  60. virtual ~H265Accelerator();
  61. // Create a new H265Picture that the decoder client can use for decoding
  62. // and pass back to this accelerator for decoding or reference.
  63. // When the picture is no longer needed by decoder, it will just drop
  64. // its reference to it, and it may do so at any time.
  65. // Note that this may return nullptr if accelerator is not able to provide
  66. // any new pictures at given time. The decoder is expected to handle
  67. // this situation as normal and return from Decode() with kRanOutOfSurfaces.
  68. virtual scoped_refptr<H265Picture> CreateH265Picture() = 0;
  69. // Submit metadata for the current frame, providing the current |sps|, |pps|
  70. // and |slice_hdr| for it. |ref_pic_list| contains the set of pictures as
  71. // described in 8.3.2 from the lists RefPicSetLtCurr, RefPicSetLtFoll,
  72. // RefPicSetStCurrBefore, RefPicSetStCurrAfter and RefPicSetStFoll.
  73. // |pic| contains information about the picture for the current frame.
  74. // Note that this does not run decode in the accelerator and the decoder
  75. // is expected to follow this call with one or more SubmitSlice() calls
  76. // before calling SubmitDecode().
  77. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  78. // the accelerator needs additional data before being able to proceed.
  79. virtual Status SubmitFrameMetadata(const H265SPS* sps,
  80. const H265PPS* pps,
  81. const H265SliceHeader* slice_hdr,
  82. const H265Picture::Vector& ref_pic_list,
  83. scoped_refptr<H265Picture> pic) = 0;
  84. // Submit one slice for the current frame, passing the current |pps| and
  85. // |pic| (same as in SubmitFrameMetadata()), the parsed header for the
  86. // current slice in |slice_hdr|, the |ref_pic_listX| and |ref_pic_set_XX|,
  87. // as per H265 spec. |data| pointing to the full slice (including the
  88. // unparsed header) of |size| in bytes.
  89. // |subsamples| specifies which part of the slice data is encrypted.
  90. // This must be called one or more times per frame, before SubmitDecode().
  91. // Note that |data| does not have to remain valid after this call returns.
  92. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  93. // the accelerator needs additional data before being able to proceed.
  94. virtual Status SubmitSlice(
  95. const H265SPS* sps,
  96. const H265PPS* pps,
  97. const H265SliceHeader* slice_hdr,
  98. const H265Picture::Vector& ref_pic_list0,
  99. const H265Picture::Vector& ref_pic_list1,
  100. const H265Picture::Vector& ref_pic_set_lt_curr,
  101. const H265Picture::Vector& ref_pic_set_st_curr_after,
  102. const H265Picture::Vector& ref_pic_set_st_curr_before,
  103. scoped_refptr<H265Picture> pic,
  104. const uint8_t* data,
  105. size_t size,
  106. const std::vector<SubsampleEntry>& subsamples) = 0;
  107. // Execute the decode in hardware for |pic|, using all the slices and
  108. // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since
  109. // the previous call to SubmitDecode().
  110. // Returns kOk if successful, kFail if there are errors, or kTryAgain if
  111. // the accelerator needs additional data before being able to proceed.
  112. virtual Status SubmitDecode(scoped_refptr<H265Picture> pic) = 0;
  113. // Schedule output (display) of |pic|. Note that returning from this
  114. // method does not mean that |pic| has already been outputted (displayed),
  115. // but guarantees that all pictures will be outputted in the same order
  116. // as this method was called for them. Decoder may drop its reference
  117. // to |pic| after calling this method.
  118. // Return true if successful.
  119. virtual bool OutputPicture(scoped_refptr<H265Picture> pic) = 0;
  120. // Reset any current state that may be cached in the accelerator, dropping
  121. // any cached parameters/slices that have not been committed yet.
  122. virtual void Reset() = 0;
  123. // Notifies the accelerator whenever there is a new stream to process.
  124. // |stream| is the data in annex B format, which may include SPS and PPS
  125. // NALUs when there is a configuration change. The first frame must contain
  126. // the SPS and PPS NALUs. SPS and PPS NALUs may not be encrypted.
  127. // |decrypt_config| is the config for decrypting the stream. The accelerator
  128. // should use |decrypt_config| to keep track of the parts of |stream| that
  129. // are encrypted. If kTryAgain is returned, the decoder will retry this call
  130. // later. This method has a default implementation that returns
  131. // kNotSupported.
  132. virtual Status SetStream(base::span<const uint8_t> stream,
  133. const DecryptConfig* decrypt_config);
  134. // Indicates whether the accelerator supports bitstreams with
  135. // specific chroma subsampling format.
  136. virtual bool IsChromaSamplingSupported(VideoChromaSampling format) = 0;
  137. };
  138. H265Decoder(std::unique_ptr<H265Accelerator> accelerator,
  139. VideoCodecProfile profile,
  140. const VideoColorSpace& container_color_space = VideoColorSpace());
  141. H265Decoder(const H265Decoder&) = delete;
  142. H265Decoder& operator=(const H265Decoder&) = delete;
  143. ~H265Decoder() override;
  144. // AcceleratedVideoDecoder implementation.
  145. void SetStream(int32_t id, const DecoderBuffer& decoder) override;
  146. [[nodiscard]] bool Flush() override;
  147. void Reset() override;
  148. [[nodiscard]] DecodeResult Decode() override;
  149. gfx::Size GetPicSize() const override;
  150. gfx::Rect GetVisibleRect() const override;
  151. VideoCodecProfile GetProfile() const override;
  152. uint8_t GetBitDepth() const override;
  153. VideoChromaSampling GetChromaSampling() const override;
  154. size_t GetRequiredNumOfPictures() const override;
  155. size_t GetNumReferenceFrames() const override;
  156. private:
  157. // Internal state of the decoder.
  158. enum State {
  159. // Ready to decode from any point.
  160. kDecoding,
  161. // After Reset(), need a resume point.
  162. kAfterReset,
  163. // The following keep track of what step is next in Decode() processing
  164. // in order to resume properly after H265Decoder::kTryAgain (or another
  165. // retryable error) is returned. The next time Decode() is called the call
  166. // that previously failed will be retried and execution continues from
  167. // there (if possible).
  168. kTryPreprocessCurrentSlice,
  169. kEnsurePicture,
  170. kTryNewFrame,
  171. kTryCurrentSlice,
  172. // Error in decode, can't continue.
  173. kError,
  174. };
  175. // Process H265 stream structures.
  176. bool ProcessPPS(int pps_id, bool* need_new_buffers);
  177. // Process current slice header to discover if we need to start a new picture,
  178. // finishing up the current one.
  179. H265Accelerator::Status PreprocessCurrentSlice();
  180. // Process current slice as a slice of the current picture.
  181. H265Accelerator::Status ProcessCurrentSlice();
  182. // Start processing a new frame. This also generates all the POC and output
  183. // variables for the frame, generates reference picture lists, performs
  184. // reference picture marking, DPB management and picture output.
  185. H265Accelerator::Status StartNewFrame(const H265SliceHeader* slice_hdr);
  186. // All data for a frame received, process it and decode.
  187. H265Accelerator::Status FinishPrevFrameIfPresent();
  188. // Called after we are done processing |pic|.
  189. void FinishPicture(scoped_refptr<H265Picture> pic);
  190. // Commits all pending data for HW decoder and starts HW decoder.
  191. H265Accelerator::Status DecodePicture();
  192. // Notifies client that a picture is ready for output.
  193. bool OutputPic(scoped_refptr<H265Picture> pic);
  194. // Output all pictures in DPB that have not been outputted yet.
  195. bool OutputAllRemainingPics();
  196. // Calculates the picture output flags using |slice_hdr| for |curr_pic_|.
  197. void CalcPicOutputFlags(const H265SliceHeader* slice_hdr);
  198. // Calculates picture order count (POC) using |pps| and|slice_hdr| for
  199. // |curr_pic_|.
  200. void CalcPictureOrderCount(const H265PPS* pps,
  201. const H265SliceHeader* slice_hdr);
  202. // Calculates the POCs for the reference pictures for |curr_pic_| using
  203. // |sps|, |pps| and |slice_hdr| and stores them in the member variables.
  204. // Returns false if bitstream conformance is not maintained, true otherwise.
  205. bool CalcRefPicPocs(const H265SPS* sps,
  206. const H265PPS* pps,
  207. const H265SliceHeader* slice_hdr);
  208. // Builds the reference pictures lists for |curr_pic_| using |sps|, |pps|,
  209. // |slice_hdr| and the member variables calculated in CalcRefPicPocs. Returns
  210. // false if bitstream conformance is not maintained or needed reference
  211. // pictures are missing, true otherwise. At the end of this,
  212. // |ref_pic_list{0,1}| will be populated with the required reference pictures
  213. // for submitting to the accelerator.
  214. bool BuildRefPicLists(const H265SPS* sps,
  215. const H265PPS* pps,
  216. const H265SliceHeader* slice_hdr);
  217. // Performs DPB management operations for |curr_pic_| by removing no longer
  218. // needed entries from the DPB and outputting pictures from the DPB. |sps|
  219. // should be the corresponding SPS for |curr_pic_|.
  220. bool PerformDpbOperations(const H265SPS* sps);
  221. // Decoder state.
  222. State state_;
  223. // The colorspace for the h265 container.
  224. const VideoColorSpace container_color_space_;
  225. // Parser in use.
  226. H265Parser parser_;
  227. // Most recent call to SetStream().
  228. const uint8_t* current_stream_ = nullptr;
  229. size_t current_stream_size_ = 0;
  230. // Decrypting config for the most recent data passed to SetStream().
  231. std::unique_ptr<DecryptConfig> current_decrypt_config_;
  232. // Keep track of when SetStream() is called so that
  233. // H265Accelerator::SetStream() can be called.
  234. bool current_stream_has_been_changed_ = false;
  235. // DPB in use.
  236. H265DPB dpb_;
  237. // Current stream buffer id; to be assigned to pictures decoded from it.
  238. int32_t stream_id_ = -1;
  239. // Picture currently being processed/decoded.
  240. scoped_refptr<H265Picture> curr_pic_;
  241. // Used to identify first picture in decoding order or first picture that
  242. // follows an EOS NALU.
  243. bool first_picture_ = true;
  244. // Global state values, needed in decoding. See spec.
  245. scoped_refptr<H265Picture> prev_tid0_pic_;
  246. int max_pic_order_cnt_lsb_;
  247. bool curr_delta_poc_msb_present_flag_[kMaxDpbSize];
  248. bool foll_delta_poc_msb_present_flag_[kMaxDpbSize];
  249. int num_poc_st_curr_before_;
  250. int num_poc_st_curr_after_;
  251. int num_poc_st_foll_;
  252. int num_poc_lt_curr_;
  253. int num_poc_lt_foll_;
  254. int poc_st_curr_before_[kMaxDpbSize];
  255. int poc_st_curr_after_[kMaxDpbSize];
  256. int poc_st_foll_[kMaxDpbSize];
  257. int poc_lt_curr_[kMaxDpbSize];
  258. int poc_lt_foll_[kMaxDpbSize];
  259. H265Picture::Vector ref_pic_list0_;
  260. H265Picture::Vector ref_pic_list1_;
  261. H265Picture::Vector ref_pic_set_lt_curr_;
  262. H265Picture::Vector ref_pic_set_st_curr_after_;
  263. H265Picture::Vector ref_pic_set_st_curr_before_;
  264. // |ref_pic_list_| is the collection of all pictures from StCurrBefore,
  265. // StCurrAfter, StFoll, LtCurr and LtFoll.
  266. H265Picture::Vector ref_pic_list_;
  267. // Currently active SPS and PPS.
  268. int curr_sps_id_ = -1;
  269. int curr_pps_id_ = -1;
  270. // Current NALU and slice header being processed.
  271. std::unique_ptr<H265NALU> curr_nalu_;
  272. std::unique_ptr<H265SliceHeader> curr_slice_hdr_;
  273. std::unique_ptr<H265SliceHeader> last_slice_hdr_;
  274. // Output picture size.
  275. gfx::Size pic_size_;
  276. // Output visible cropping rect.
  277. gfx::Rect visible_rect_;
  278. // Profile of input bitstream.
  279. VideoCodecProfile profile_;
  280. // Bit depth of input bitstream.
  281. uint8_t bit_depth_ = 0;
  282. // Chroma sampling format of input bitstream
  283. VideoChromaSampling chroma_sampling_ = VideoChromaSampling::kUnknown;
  284. const std::unique_ptr<H265Accelerator> accelerator_;
  285. };
  286. } // namespace media
  287. #endif // MEDIA_GPU_H265_DECODER_H_