vp9_parser.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. //
  5. // This file contains an implementation of a VP9 bitstream parser. The main
  6. // purpose of this parser is to support hardware decode acceleration. Some
  7. // accelerators, e.g. libva which implements VA-API, require the caller
  8. // (chrome) to feed them parsed VP9 frame header.
  9. //
  10. // See media::VP9Decoder for example usage.
  11. //
  12. #ifndef MEDIA_FILTERS_VP9_PARSER_H_
  13. #define MEDIA_FILTERS_VP9_PARSER_H_
  14. #include <stddef.h>
  15. #include <stdint.h>
  16. #include <sys/types.h>
  17. #include <memory>
  18. #include "base/callback.h"
  19. #include "base/containers/circular_deque.h"
  20. #include "base/memory/weak_ptr.h"
  21. #include "media/base/decrypt_config.h"
  22. #include "media/base/media_export.h"
  23. #include "media/base/video_color_space.h"
  24. #include "ui/gfx/geometry/size.h"
  25. namespace media {
  26. const int kVp9MaxProfile = 4;
  27. const int kVp9NumRefFramesLog2 = 3;
  28. const size_t kVp9NumRefFrames = 1 << kVp9NumRefFramesLog2;
  29. const uint8_t kVp9MaxProb = 255;
  30. const size_t kVp9NumRefsPerFrame = 3;
  31. const size_t kVp9NumFrameContextsLog2 = 2;
  32. const size_t kVp9NumFrameContexts = 1 << kVp9NumFrameContextsLog2;
  33. using Vp9Prob = uint8_t;
  34. enum class Vp9ColorSpace {
  35. UNKNOWN = 0,
  36. BT_601 = 1,
  37. BT_709 = 2,
  38. SMPTE_170 = 3,
  39. SMPTE_240 = 4,
  40. BT_2020 = 5,
  41. RESERVED = 6,
  42. SRGB = 7,
  43. };
  44. enum Vp9InterpolationFilter {
  45. EIGHTTAP = 0,
  46. EIGHTTAP_SMOOTH = 1,
  47. EIGHTTAP_SHARP = 2,
  48. BILINEAR = 3,
  49. SWITCHABLE = 4,
  50. };
  51. enum Vp9RefType {
  52. VP9_FRAME_INTRA = 0,
  53. VP9_FRAME_LAST = 1,
  54. VP9_FRAME_GOLDEN = 2,
  55. VP9_FRAME_ALTREF = 3,
  56. VP9_FRAME_MAX = 4,
  57. };
  58. enum Vp9ReferenceMode {
  59. SINGLE_REFERENCE = 0,
  60. COMPOUND_REFERENCE = 1,
  61. REFERENCE_MODE_SELECT = 2,
  62. };
  63. struct MEDIA_EXPORT Vp9SegmentationParams {
  64. static const size_t kNumSegments = 8;
  65. static const size_t kNumTreeProbs = kNumSegments - 1;
  66. static const size_t kNumPredictionProbs = 3;
  67. enum SegmentLevelFeature {
  68. SEG_LVL_ALT_Q = 0,
  69. SEG_LVL_ALT_LF = 1,
  70. SEG_LVL_REF_FRAME = 2,
  71. SEG_LVL_SKIP = 3,
  72. SEG_LVL_MAX
  73. };
  74. bool enabled;
  75. bool update_map;
  76. uint8_t tree_probs[kNumTreeProbs];
  77. bool temporal_update;
  78. uint8_t pred_probs[kNumPredictionProbs];
  79. bool update_data;
  80. bool abs_or_delta_update;
  81. bool feature_enabled[kNumSegments][SEG_LVL_MAX];
  82. int16_t feature_data[kNumSegments][SEG_LVL_MAX];
  83. int16_t y_dequant[kNumSegments][2];
  84. int16_t uv_dequant[kNumSegments][2];
  85. bool FeatureEnabled(size_t seg_id, SegmentLevelFeature feature) const {
  86. return feature_enabled[seg_id][feature];
  87. }
  88. int16_t FeatureData(size_t seg_id, SegmentLevelFeature feature) const {
  89. return feature_data[seg_id][feature];
  90. }
  91. };
  92. struct MEDIA_EXPORT Vp9LoopFilterParams {
  93. static const size_t kNumModeDeltas = 2;
  94. uint8_t level;
  95. uint8_t sharpness;
  96. bool delta_enabled;
  97. bool delta_update;
  98. bool update_ref_deltas[VP9_FRAME_MAX];
  99. int8_t ref_deltas[VP9_FRAME_MAX];
  100. bool update_mode_deltas[kNumModeDeltas];
  101. int8_t mode_deltas[kNumModeDeltas];
  102. // Calculated from above fields.
  103. uint8_t lvl[Vp9SegmentationParams::kNumSegments][VP9_FRAME_MAX]
  104. [kNumModeDeltas];
  105. };
  106. // Members of Vp9FrameHeader will be 0-initialized by Vp9Parser::ParseNextFrame.
  107. struct MEDIA_EXPORT Vp9QuantizationParams {
  108. bool IsLossless() const {
  109. return base_q_idx == 0 && delta_q_y_dc == 0 && delta_q_uv_dc == 0 &&
  110. delta_q_uv_ac == 0;
  111. }
  112. uint8_t base_q_idx;
  113. int8_t delta_q_y_dc;
  114. int8_t delta_q_uv_dc;
  115. int8_t delta_q_uv_ac;
  116. };
  117. // Entropy context for frame parsing
  118. struct MEDIA_EXPORT Vp9FrameContext {
  119. bool IsValid() const;
  120. Vp9Prob tx_probs_8x8[2][1];
  121. Vp9Prob tx_probs_16x16[2][2];
  122. Vp9Prob tx_probs_32x32[2][3];
  123. Vp9Prob coef_probs[4][2][2][6][6][3];
  124. Vp9Prob skip_prob[3];
  125. Vp9Prob inter_mode_probs[7][3];
  126. Vp9Prob interp_filter_probs[4][2];
  127. Vp9Prob is_inter_prob[4];
  128. Vp9Prob comp_mode_prob[5];
  129. Vp9Prob single_ref_prob[5][2];
  130. Vp9Prob comp_ref_prob[5];
  131. Vp9Prob y_mode_probs[4][9];
  132. Vp9Prob uv_mode_probs[10][9];
  133. Vp9Prob partition_probs[16][3];
  134. Vp9Prob mv_joint_probs[3];
  135. Vp9Prob mv_sign_prob[2];
  136. Vp9Prob mv_class_probs[2][10];
  137. Vp9Prob mv_class0_bit_prob[2];
  138. Vp9Prob mv_bits_prob[2][10];
  139. Vp9Prob mv_class0_fr_probs[2][2][3];
  140. Vp9Prob mv_fr_probs[2][3];
  141. Vp9Prob mv_class0_hp_prob[2];
  142. Vp9Prob mv_hp_prob[2];
  143. };
  144. struct MEDIA_EXPORT Vp9CompressedHeader {
  145. enum Vp9TxMode {
  146. ONLY_4X4 = 0,
  147. ALLOW_8X8 = 1,
  148. ALLOW_16X16 = 2,
  149. ALLOW_32X32 = 3,
  150. TX_MODE_SELECT = 4,
  151. TX_MODES = 5,
  152. };
  153. Vp9TxMode tx_mode;
  154. Vp9ReferenceMode reference_mode;
  155. };
  156. // VP9 frame header.
  157. struct MEDIA_EXPORT Vp9FrameHeader {
  158. enum FrameType {
  159. KEYFRAME = 0,
  160. INTERFRAME = 1,
  161. };
  162. bool IsKeyframe() const;
  163. bool IsIntra() const;
  164. bool RefreshFlag(size_t i) const {
  165. return !!(refresh_frame_flags & (1u << i));
  166. }
  167. VideoColorSpace GetColorSpace() const;
  168. uint8_t profile;
  169. bool show_existing_frame;
  170. uint8_t frame_to_show_map_idx;
  171. FrameType frame_type;
  172. bool show_frame;
  173. bool error_resilient_mode;
  174. uint8_t bit_depth;
  175. Vp9ColorSpace color_space;
  176. bool color_range;
  177. uint8_t subsampling_x;
  178. uint8_t subsampling_y;
  179. // The range of frame_width and frame_height is 1..2^16.
  180. uint32_t frame_width;
  181. uint32_t frame_height;
  182. uint32_t render_width;
  183. uint32_t render_height;
  184. bool intra_only;
  185. uint8_t reset_frame_context;
  186. uint8_t refresh_frame_flags;
  187. uint8_t ref_frame_idx[kVp9NumRefsPerFrame];
  188. bool ref_frame_sign_bias[Vp9RefType::VP9_FRAME_MAX];
  189. bool allow_high_precision_mv;
  190. Vp9InterpolationFilter interpolation_filter;
  191. bool refresh_frame_context;
  192. bool frame_parallel_decoding_mode;
  193. uint8_t frame_context_idx;
  194. // |frame_context_idx_to_save_probs| is to be used by save_probs() only, and
  195. // |frame_context_idx| otherwise.
  196. uint8_t frame_context_idx_to_save_probs;
  197. Vp9QuantizationParams quant_params;
  198. uint8_t tile_cols_log2;
  199. uint8_t tile_rows_log2;
  200. // Pointer to the beginning of frame data. It is a responsibility of the
  201. // client of the Vp9Parser to maintain validity of this data while it is
  202. // being used outside of that class.
  203. const uint8_t* data;
  204. // Size of |data| in bytes.
  205. size_t frame_size;
  206. // Size of compressed header in bytes.
  207. size_t header_size_in_bytes;
  208. // Size of uncompressed header in bytes.
  209. size_t uncompressed_header_size;
  210. Vp9CompressedHeader compressed_header;
  211. // Initial frame entropy context after load_probs2(frame_context_idx).
  212. Vp9FrameContext initial_frame_context;
  213. // Current frame entropy context after header parsing.
  214. Vp9FrameContext frame_context;
  215. // Segmentation and loop filter params from uncompressed header
  216. Vp9SegmentationParams segmentation;
  217. Vp9LoopFilterParams loop_filter;
  218. };
  219. // A parser for VP9 bitstream.
  220. class MEDIA_EXPORT Vp9Parser {
  221. public:
  222. // If context update is needed after decoding a frame, the client must
  223. // execute this callback, passing the updated context state.
  224. using ContextRefreshCallback =
  225. base::OnceCallback<void(const Vp9FrameContext&)>;
  226. // ParseNextFrame() return values. See documentation for ParseNextFrame().
  227. enum Result {
  228. kOk,
  229. kInvalidStream,
  230. kEOStream,
  231. kAwaitingRefresh,
  232. };
  233. // The parsing context to keep track of references.
  234. struct ReferenceSlot {
  235. bool initialized;
  236. uint32_t frame_width;
  237. uint32_t frame_height;
  238. uint8_t subsampling_x;
  239. uint8_t subsampling_y;
  240. uint8_t bit_depth;
  241. // More fields for consistency checking.
  242. uint8_t profile;
  243. Vp9ColorSpace color_space;
  244. };
  245. // The parsing context that persists across frames.
  246. class Context {
  247. public:
  248. class MEDIA_EXPORT Vp9FrameContextManager {
  249. public:
  250. Vp9FrameContextManager();
  251. ~Vp9FrameContextManager();
  252. bool initialized() const { return initialized_; }
  253. bool needs_client_update() const { return needs_client_update_; }
  254. const Vp9FrameContext& frame_context() const;
  255. // Resets to uninitialized state.
  256. void Reset();
  257. // Marks this context as requiring an update from parser's client.
  258. void SetNeedsClientUpdate();
  259. // Updates frame context.
  260. void Update(const Vp9FrameContext& frame_context);
  261. // Returns a callback to update frame context at a later time with.
  262. ContextRefreshCallback GetUpdateCb();
  263. private:
  264. // Updates frame context from parser's client.
  265. void UpdateFromClient(const Vp9FrameContext& frame_context);
  266. bool initialized_ = false;
  267. bool needs_client_update_ = false;
  268. Vp9FrameContext frame_context_;
  269. base::WeakPtrFactory<Vp9FrameContextManager> weak_ptr_factory_{this};
  270. };
  271. void Reset();
  272. // Mark |frame_context_idx| as requiring update from the client.
  273. void MarkFrameContextForUpdate(size_t frame_context_idx);
  274. // Update frame context at |frame_context_idx| with the contents of
  275. // |frame_context|.
  276. void UpdateFrameContext(size_t frame_context_idx,
  277. const Vp9FrameContext& frame_context);
  278. // Return ReferenceSlot for frame at |ref_idx|.
  279. const ReferenceSlot& GetRefSlot(size_t ref_idx) const;
  280. // Update contents of ReferenceSlot at |ref_idx| with the contents of
  281. // |ref_slot|.
  282. void UpdateRefSlot(size_t ref_idx, const ReferenceSlot& ref_slot);
  283. const Vp9SegmentationParams& segmentation() const { return segmentation_; }
  284. const Vp9LoopFilterParams& loop_filter() const { return loop_filter_; }
  285. private:
  286. friend class Vp9UncompressedHeaderParser;
  287. friend class Vp9Parser;
  288. friend class Vp9ParserTest;
  289. // Segmentation and loop filter state.
  290. Vp9SegmentationParams segmentation_;
  291. Vp9LoopFilterParams loop_filter_;
  292. // Frame references.
  293. ReferenceSlot ref_slots_[kVp9NumRefFrames];
  294. Vp9FrameContextManager frame_context_managers_[kVp9NumFrameContexts];
  295. };
  296. // See homonymous member variables for information on the parameters.
  297. explicit Vp9Parser(bool parsing_compressed_header);
  298. Vp9Parser(bool parsing_compressed_header, bool needs_external_context_update);
  299. Vp9Parser(const Vp9Parser&) = delete;
  300. Vp9Parser& operator=(const Vp9Parser&) = delete;
  301. ~Vp9Parser();
  302. // Set a new stream buffer to read from, starting at |stream| and of size
  303. // |stream_size| in bytes. |stream| must point to the beginning of a single
  304. // frame or a single superframe, is owned by caller and must remain valid
  305. // until the next call to SetStream(). |spatial_layer_frame_size| may be
  306. // filled if the parsed stream is VP9 SVC. It stands for frame sizes of
  307. // spatial layers. SVC frame might have multiple frames without superframe
  308. // index. The info helps Vp9Parser detecting the beginning of each frame.
  309. void SetStream(const uint8_t* stream,
  310. off_t stream_size,
  311. const std::vector<uint32_t>& spatial_layer_frame_size,
  312. std::unique_ptr<DecryptConfig> stream_config);
  313. void SetStream(const uint8_t* stream,
  314. off_t stream_size,
  315. std::unique_ptr<DecryptConfig> stream_config);
  316. // Parse the next frame in the current stream buffer, filling |fhdr| with
  317. // the parsed frame header and updating current segmentation and loop filter
  318. // state. The necessary frame size to decode |fhdr| fills in |allocate_size|.
  319. // The size can be larger than frame size of |fhdr| in the case of SVC stream.
  320. // Also fills |frame_decrypt_config| _if_ the parser was set to use a super
  321. // frame decrypt config.
  322. // Return kOk if a frame has successfully been parsed,
  323. // kEOStream if there is no more data in the current stream buffer,
  324. // kAwaitingRefresh if this frame awaiting frame context update, or
  325. // kInvalidStream on error.
  326. Result ParseNextFrame(Vp9FrameHeader* fhdr,
  327. gfx::Size* allocate_size,
  328. std::unique_ptr<DecryptConfig>* frame_decrypt_config);
  329. // Perform the same superframe parsing logic, but don't attempt to parse
  330. // the normal frame headers afterwards, and then only return the decrypt
  331. // config, since the frame itself isn't useful for the testing.
  332. // Returns |true| if a frame would have been sent to |ParseUncompressedHeader|
  333. // |false| if there was an error parsing the superframe.
  334. std::unique_ptr<DecryptConfig> NextFrameDecryptContextForTesting();
  335. std::string IncrementIVForTesting(const std::string& iv, uint32_t by);
  336. // Return current parsing context.
  337. const Context& context() const { return context_; }
  338. // Return a ContextRefreshCallback, which, if not null, has to be called with
  339. // the new context state after the frame associated with |frame_context_idx|
  340. // is decoded.
  341. ContextRefreshCallback GetContextRefreshCb(size_t frame_context_idx);
  342. // Clear parser state and return to an initialized state.
  343. void Reset();
  344. private:
  345. // Stores start pointer and size of each frame within the current superframe.
  346. struct FrameInfo {
  347. FrameInfo();
  348. FrameInfo(const FrameInfo& copy_from);
  349. FrameInfo(const uint8_t* ptr, off_t size);
  350. ~FrameInfo();
  351. FrameInfo& operator=(const FrameInfo& copy_from);
  352. bool IsValid() const { return ptr != nullptr; }
  353. void Reset() { ptr = nullptr; }
  354. // Starting address of the frame.
  355. const uint8_t* ptr = nullptr;
  356. // Size of the frame in bytes.
  357. off_t size = 0;
  358. // Necessary height and width to decode the frame.
  359. // This is filled only if the stream is SVC.
  360. gfx::Size allocate_size;
  361. std::unique_ptr<DecryptConfig> decrypt_config;
  362. };
  363. base::circular_deque<FrameInfo> ParseSuperframe();
  364. // Parses a frame in SVC stream with |spatial_layer_frame_size_|.
  365. base::circular_deque<FrameInfo> ParseSVCFrame();
  366. // Returns true and populates |result| with the parsing result if parsing of
  367. // current frame is finished (possibly unsuccessfully). |fhdr| will only be
  368. // populated and valid if |result| is kOk. Otherwise return false, indicating
  369. // that the compressed header must be parsed next.
  370. bool ParseUncompressedHeader(const FrameInfo& frame_info,
  371. Vp9FrameHeader* fhdr,
  372. Result* result,
  373. Vp9Parser::Context* context);
  374. // Returns true if parsing of current frame is finished and |result| will be
  375. // populated with value of parsing result. Otherwise, needs to continue setup
  376. // current frame.
  377. bool ParseCompressedHeader(const FrameInfo& frame_info, Result* result);
  378. int64_t GetQIndex(const Vp9QuantizationParams& quant, size_t segid) const;
  379. // Returns true if the setup to |context_| succeeded.
  380. bool SetupSegmentationDequant();
  381. void SetupLoopFilter();
  382. // Returns true if the setup to |context| succeeded.
  383. void UpdateSlots(Vp9Parser::Context* context);
  384. // Current address in the bitstream buffer.
  385. const uint8_t* stream_;
  386. // Remaining bytes in stream_.
  387. off_t bytes_left_;
  388. // Set on ctor if the client needs VP9Parser to also parse compressed headers,
  389. // otherwise they'll be skipped.
  390. const bool parsing_compressed_header_;
  391. // Set on ctor if the client needs to call the ContextRefreshCallback obtained
  392. // via GetContextRefreshCb() with the updated Vp9FrameContext; otherwise
  393. // VP9Parser will update it internally.
  394. const bool needs_external_context_update_;
  395. // FrameInfo for the remaining frames in the current superframe to be parsed.
  396. base::circular_deque<FrameInfo> frames_;
  397. Context context_;
  398. // Encrypted stream info.
  399. std::unique_ptr<DecryptConfig> stream_decrypt_config_;
  400. // The frame size of each spatial layer.
  401. std::vector<uint32_t> spatial_layer_frame_size_;
  402. FrameInfo curr_frame_info_;
  403. Vp9FrameHeader curr_frame_header_;
  404. };
  405. } // namespace media
  406. #endif // MEDIA_FILTERS_VP9_PARSER_H_