h264_parser.h 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. //
  5. // This file contains an implementation of an H264 Annex-B video stream parser.
  6. #ifndef MEDIA_VIDEO_H264_PARSER_H_
  7. #define MEDIA_VIDEO_H264_PARSER_H_
  8. #include <stddef.h>
  9. #include <stdint.h>
  10. #include <sys/types.h>
  11. #include <map>
  12. #include <memory>
  13. #include <vector>
  14. #include "media/base/media_export.h"
  15. #include "media/base/ranges.h"
  16. #include "media/base/video_codecs.h"
  17. #include "media/base/video_color_space.h"
  18. #include "media/base/video_types.h"
  19. #include "media/video/h264_bit_reader.h"
  20. #include "third_party/abseil-cpp/absl/types/optional.h"
  21. namespace gfx {
  22. class Rect;
  23. class Size;
  24. } // namespace gfx
  25. namespace media {
  26. struct SubsampleEntry;
  27. // For explanations of each struct and its members, see H.264 specification
  28. // at http://www.itu.int/rec/T-REC-H.264.
  29. struct MEDIA_EXPORT H264NALU {
  30. H264NALU();
  31. enum Type {
  32. kUnspecified = 0,
  33. kNonIDRSlice = 1,
  34. kSliceDataA = 2,
  35. kSliceDataB = 3,
  36. kSliceDataC = 4,
  37. kIDRSlice = 5,
  38. kSEIMessage = 6,
  39. kSPS = 7,
  40. kPPS = 8,
  41. kAUD = 9,
  42. kEOSeq = 10,
  43. kEOStream = 11,
  44. kFiller = 12,
  45. kSPSExt = 13,
  46. kPrefix = 14,
  47. kSubsetSPS = 15,
  48. kDPS = 16,
  49. kReserved17 = 17,
  50. kReserved18 = 18,
  51. kCodedSliceAux = 19,
  52. kCodedSliceExtension = 20,
  53. };
  54. // After (without) start code; we don't own the underlying memory
  55. // and a shallow copy should be made when copying this struct.
  56. const uint8_t* data;
  57. off_t size; // From after start code to start code of next NALU (or EOS).
  58. int nal_ref_idc;
  59. int nal_unit_type;
  60. };
  61. enum {
  62. kH264ScalingList4x4Length = 16,
  63. kH264ScalingList8x8Length = 64,
  64. };
  65. struct MEDIA_EXPORT H264SPS {
  66. H264SPS();
  67. enum H264ProfileIDC {
  68. kProfileIDCBaseline = 66,
  69. kProfileIDCConstrainedBaseline = kProfileIDCBaseline,
  70. kProfileIDCMain = 77,
  71. kProfileIDScalableBaseline = 83,
  72. kProfileIDScalableHigh = 86,
  73. kProfileIDCHigh = 100,
  74. kProfileIDHigh10 = 110,
  75. kProfileIDSMultiviewHigh = 118,
  76. kProfileIDHigh422 = 122,
  77. kProfileIDStereoHigh = 128,
  78. kProfileIDHigh444Predictive = 244,
  79. };
  80. enum H264LevelIDC : uint8_t {
  81. kLevelIDC1p0 = 10,
  82. kLevelIDC1B = 9,
  83. kLevelIDC1p1 = 11,
  84. kLevelIDC1p2 = 12,
  85. kLevelIDC1p3 = 13,
  86. kLevelIDC2p0 = 20,
  87. kLevelIDC2p1 = 21,
  88. kLevelIDC2p2 = 22,
  89. kLevelIDC3p0 = 30,
  90. kLevelIDC3p1 = 31,
  91. kLevelIDC3p2 = 32,
  92. kLevelIDC4p0 = 40,
  93. kLevelIDC4p1 = 41,
  94. kLevelIDC4p2 = 42,
  95. kLevelIDC5p0 = 50,
  96. kLevelIDC5p1 = 51,
  97. kLevelIDC5p2 = 52,
  98. kLevelIDC6p0 = 60,
  99. kLevelIDC6p1 = 61,
  100. kLevelIDC6p2 = 62,
  101. };
  102. enum AspectRatioIdc {
  103. kExtendedSar = 255,
  104. };
  105. enum {
  106. // Constants for HRD parameters (spec ch. E.2.2).
  107. kBitRateScaleConstantTerm = 6, // Equation E-37.
  108. kCPBSizeScaleConstantTerm = 4, // Equation E-38.
  109. kDefaultInitialCPBRemovalDelayLength = 24,
  110. kDefaultDPBOutputDelayLength = 24,
  111. kDefaultTimeOffsetLength = 24,
  112. };
  113. int profile_idc;
  114. bool constraint_set0_flag;
  115. bool constraint_set1_flag;
  116. bool constraint_set2_flag;
  117. bool constraint_set3_flag;
  118. bool constraint_set4_flag;
  119. bool constraint_set5_flag;
  120. int level_idc;
  121. int seq_parameter_set_id;
  122. int chroma_format_idc;
  123. bool separate_colour_plane_flag;
  124. int bit_depth_luma_minus8;
  125. int bit_depth_chroma_minus8;
  126. bool qpprime_y_zero_transform_bypass_flag;
  127. bool seq_scaling_matrix_present_flag;
  128. int scaling_list4x4[6][kH264ScalingList4x4Length];
  129. int scaling_list8x8[6][kH264ScalingList8x8Length];
  130. int log2_max_frame_num_minus4;
  131. int pic_order_cnt_type;
  132. int log2_max_pic_order_cnt_lsb_minus4;
  133. bool delta_pic_order_always_zero_flag;
  134. int offset_for_non_ref_pic;
  135. int offset_for_top_to_bottom_field;
  136. int num_ref_frames_in_pic_order_cnt_cycle;
  137. int expected_delta_per_pic_order_cnt_cycle; // calculated
  138. int offset_for_ref_frame[255];
  139. int max_num_ref_frames;
  140. bool gaps_in_frame_num_value_allowed_flag;
  141. int pic_width_in_mbs_minus1;
  142. int pic_height_in_map_units_minus1;
  143. bool frame_mbs_only_flag;
  144. bool mb_adaptive_frame_field_flag;
  145. bool direct_8x8_inference_flag;
  146. bool frame_cropping_flag;
  147. int frame_crop_left_offset;
  148. int frame_crop_right_offset;
  149. int frame_crop_top_offset;
  150. int frame_crop_bottom_offset;
  151. bool vui_parameters_present_flag;
  152. int sar_width; // Set to 0 when not specified.
  153. int sar_height; // Set to 0 when not specified.
  154. bool bitstream_restriction_flag;
  155. int max_num_reorder_frames;
  156. int max_dec_frame_buffering;
  157. bool timing_info_present_flag;
  158. int num_units_in_tick;
  159. int time_scale;
  160. bool fixed_frame_rate_flag;
  161. bool video_signal_type_present_flag;
  162. int video_format;
  163. bool video_full_range_flag;
  164. bool colour_description_present_flag;
  165. int colour_primaries;
  166. int transfer_characteristics;
  167. int matrix_coefficients;
  168. // TODO(posciak): actually parse these instead of ParseAndIgnoreHRDParameters.
  169. bool nal_hrd_parameters_present_flag;
  170. int cpb_cnt_minus1;
  171. int bit_rate_scale;
  172. int cpb_size_scale;
  173. int bit_rate_value_minus1[32];
  174. int cpb_size_value_minus1[32];
  175. bool cbr_flag[32];
  176. int initial_cpb_removal_delay_length_minus_1;
  177. int cpb_removal_delay_length_minus1;
  178. int dpb_output_delay_length_minus1;
  179. int time_offset_length;
  180. bool low_delay_hrd_flag;
  181. int chroma_array_type;
  182. // Get corresponding SPS |level_idc| and |constraint_set3_flag| value from
  183. // requested |profile| and |level| (see Spec A.3.1).
  184. static void GetLevelConfigFromProfileLevel(VideoCodecProfile profile,
  185. uint8_t level,
  186. int* level_idc,
  187. bool* constraint_set3_flag);
  188. // Helpers to compute frequently-used values. These methods return
  189. // absl::nullopt if they encounter integer overflow. They do not verify that
  190. // the results are in-spec for the given profile or level.
  191. absl::optional<gfx::Size> GetCodedSize() const;
  192. absl::optional<gfx::Rect> GetVisibleRect() const;
  193. VideoColorSpace GetColorSpace() const;
  194. VideoChromaSampling GetChromaSampling() const;
  195. // Helper to compute indicated level from parsed SPS data. The value of
  196. // indicated level would be included in H264LevelIDC enum representing the
  197. // level as in name.
  198. uint8_t GetIndicatedLevel() const;
  199. // Helper to check if indicated level is lower than or equal to
  200. // |target_level|.
  201. bool CheckIndicatedLevelWithinTarget(uint8_t target_level) const;
  202. };
  203. struct MEDIA_EXPORT H264PPS {
  204. H264PPS();
  205. int pic_parameter_set_id;
  206. int seq_parameter_set_id;
  207. bool entropy_coding_mode_flag;
  208. bool bottom_field_pic_order_in_frame_present_flag;
  209. int num_slice_groups_minus1;
  210. // TODO(posciak): Slice groups not implemented, could be added at some point.
  211. int num_ref_idx_l0_default_active_minus1;
  212. int num_ref_idx_l1_default_active_minus1;
  213. bool weighted_pred_flag;
  214. int weighted_bipred_idc;
  215. int pic_init_qp_minus26;
  216. int pic_init_qs_minus26;
  217. int chroma_qp_index_offset;
  218. bool deblocking_filter_control_present_flag;
  219. bool constrained_intra_pred_flag;
  220. bool redundant_pic_cnt_present_flag;
  221. bool transform_8x8_mode_flag;
  222. bool pic_scaling_matrix_present_flag;
  223. int scaling_list4x4[6][kH264ScalingList4x4Length];
  224. int scaling_list8x8[6][kH264ScalingList8x8Length];
  225. int second_chroma_qp_index_offset;
  226. };
  227. struct MEDIA_EXPORT H264ModificationOfPicNum {
  228. int modification_of_pic_nums_idc;
  229. union {
  230. int abs_diff_pic_num_minus1;
  231. int long_term_pic_num;
  232. };
  233. };
  234. struct MEDIA_EXPORT H264WeightingFactors {
  235. bool luma_weight_flag;
  236. bool chroma_weight_flag;
  237. int luma_weight[32];
  238. int luma_offset[32];
  239. int chroma_weight[32][2];
  240. int chroma_offset[32][2];
  241. };
  242. struct MEDIA_EXPORT H264DecRefPicMarking {
  243. int memory_mgmnt_control_operation;
  244. int difference_of_pic_nums_minus1;
  245. int long_term_pic_num;
  246. int long_term_frame_idx;
  247. int max_long_term_frame_idx_plus1;
  248. };
  249. struct MEDIA_EXPORT H264SliceHeader {
  250. H264SliceHeader();
  251. enum { kRefListSize = 32, kRefListModSize = kRefListSize };
  252. enum Type {
  253. kPSlice = 0,
  254. kBSlice = 1,
  255. kISlice = 2,
  256. kSPSlice = 3,
  257. kSISlice = 4,
  258. };
  259. bool IsPSlice() const;
  260. bool IsBSlice() const;
  261. bool IsISlice() const;
  262. bool IsSPSlice() const;
  263. bool IsSISlice() const;
  264. bool idr_pic_flag; // from NAL header
  265. int nal_ref_idc; // from NAL header
  266. const uint8_t* nalu_data; // from NAL header
  267. off_t nalu_size; // from NAL header
  268. off_t header_bit_size; // calculated
  269. int first_mb_in_slice;
  270. int slice_type;
  271. int pic_parameter_set_id;
  272. int colour_plane_id; // TODO(posciak): use this! http://crbug.com/139878
  273. int frame_num;
  274. bool field_pic_flag;
  275. bool bottom_field_flag;
  276. int idr_pic_id;
  277. int pic_order_cnt_lsb;
  278. int delta_pic_order_cnt_bottom;
  279. int delta_pic_order_cnt0;
  280. int delta_pic_order_cnt1;
  281. int redundant_pic_cnt;
  282. bool direct_spatial_mv_pred_flag;
  283. bool num_ref_idx_active_override_flag;
  284. int num_ref_idx_l0_active_minus1;
  285. int num_ref_idx_l1_active_minus1;
  286. bool ref_pic_list_modification_flag_l0;
  287. bool ref_pic_list_modification_flag_l1;
  288. H264ModificationOfPicNum ref_list_l0_modifications[kRefListModSize];
  289. H264ModificationOfPicNum ref_list_l1_modifications[kRefListModSize];
  290. int luma_log2_weight_denom;
  291. int chroma_log2_weight_denom;
  292. bool luma_weight_l0_flag;
  293. bool chroma_weight_l0_flag;
  294. H264WeightingFactors pred_weight_table_l0;
  295. bool luma_weight_l1_flag;
  296. bool chroma_weight_l1_flag;
  297. H264WeightingFactors pred_weight_table_l1;
  298. bool no_output_of_prior_pics_flag;
  299. bool long_term_reference_flag;
  300. bool adaptive_ref_pic_marking_mode_flag;
  301. H264DecRefPicMarking ref_pic_marking[kRefListSize];
  302. int cabac_init_idc;
  303. int slice_qp_delta;
  304. bool sp_for_switch_flag;
  305. int slice_qs_delta;
  306. int disable_deblocking_filter_idc;
  307. int slice_alpha_c0_offset_div2;
  308. int slice_beta_offset_div2;
  309. // Calculated.
  310. // Size in bits of dec_ref_pic_marking() syntax element.
  311. size_t dec_ref_pic_marking_bit_size;
  312. size_t pic_order_cnt_bit_size;
  313. // This is when we are using full sample encryption and only the portions
  314. // needed for DPB management are filled in, the rest will already be known
  315. // by the accelerator and we will not need to specify it.
  316. bool full_sample_encryption;
  317. // This is used by some accelerators to handle decoding after slice header
  318. // parsing.
  319. uint32_t full_sample_index;
  320. };
  321. struct H264SEIRecoveryPoint {
  322. int recovery_frame_cnt;
  323. bool exact_match_flag;
  324. bool broken_link_flag;
  325. int changing_slice_group_idc;
  326. };
  327. struct MEDIA_EXPORT H264SEIMessage {
  328. H264SEIMessage();
  329. enum Type {
  330. kSEIRecoveryPoint = 6,
  331. };
  332. int type;
  333. int payload_size;
  334. union {
  335. // Placeholder; in future more supported types will contribute to more
  336. // union members here.
  337. H264SEIRecoveryPoint recovery_point;
  338. };
  339. };
  340. // Class to parse an Annex-B H.264 stream,
  341. // as specified in chapters 7 and Annex B of the H.264 spec.
  342. class MEDIA_EXPORT H264Parser {
  343. public:
  344. enum Result {
  345. kOk,
  346. kInvalidStream, // error in stream
  347. kUnsupportedStream, // stream not supported by the parser
  348. kEOStream, // end of stream
  349. };
  350. // Find offset from start of data to next NALU start code
  351. // and size of found start code (3 or 4 bytes).
  352. // If no start code is found, offset is pointing to the first unprocessed byte
  353. // (i.e. the first byte that was not considered as a possible start of a start
  354. // code) and |*start_code_size| is set to 0.
  355. // Preconditions:
  356. // - |data_size| >= 0
  357. // Postconditions:
  358. // - |*offset| is between 0 and |data_size| included.
  359. // It is strictly less than |data_size| if |data_size| > 0.
  360. // - |*start_code_size| is either 0, 3 or 4.
  361. static bool FindStartCode(const uint8_t* data,
  362. off_t data_size,
  363. off_t* offset,
  364. off_t* start_code_size);
  365. // Wrapper for FindStartCode() that skips over start codes that
  366. // may appear inside of |encrypted_ranges_|.
  367. // Returns true if a start code was found. Otherwise returns false.
  368. static bool FindStartCodeInClearRanges(const uint8_t* data,
  369. off_t data_size,
  370. const Ranges<const uint8_t*>& ranges,
  371. off_t* offset,
  372. off_t* start_code_size);
  373. static VideoCodecProfile ProfileIDCToVideoCodecProfile(int profile_idc);
  374. // Parses the input stream and returns all the NALUs through |nalus|. Returns
  375. // false if the stream is invalid.
  376. static bool ParseNALUs(const uint8_t* stream,
  377. size_t stream_size,
  378. std::vector<H264NALU>* nalus);
  379. H264Parser();
  380. H264Parser(const H264Parser&) = delete;
  381. H264Parser& operator=(const H264Parser&) = delete;
  382. ~H264Parser();
  383. void Reset();
  384. // Set current stream pointer to |stream| of |stream_size| in bytes,
  385. // |stream| owned by caller.
  386. // |subsamples| contains information about what parts of |stream| are
  387. // encrypted.
  388. void SetStream(const uint8_t* stream, off_t stream_size);
  389. void SetEncryptedStream(const uint8_t* stream,
  390. off_t stream_size,
  391. const std::vector<SubsampleEntry>& subsamples);
  392. // Read the stream to find the next NALU, identify it and return
  393. // that information in |*nalu|. This advances the stream to the beginning
  394. // of this NALU, but not past it, so subsequent calls to NALU-specific
  395. // parsing functions (ParseSPS, etc.) will parse this NALU.
  396. // If the caller wishes to skip the current NALU, it can call this function
  397. // again, instead of any NALU-type specific parse functions below.
  398. Result AdvanceToNextNALU(H264NALU* nalu);
  399. // NALU-specific parsing functions.
  400. // These should be called after AdvanceToNextNALU().
  401. // SPSes and PPSes are owned by the parser class and the memory for their
  402. // structures is managed here, not by the caller, as they are reused
  403. // across NALUs.
  404. //
  405. // Parse an SPS/PPS NALU and save their data in the parser, returning id
  406. // of the parsed structure in |*pps_id|/|*sps_id|.
  407. // To get a pointer to a given SPS/PPS structure, use GetSPS()/GetPPS(),
  408. // passing the returned |*sps_id|/|*pps_id| as parameter.
  409. // TODO(posciak,fischman): consider replacing returning Result from Parse*()
  410. // methods with a scoped_ptr and adding an AtEOS() function to check for EOS
  411. // if Parse*() return NULL.
  412. Result ParseSPS(int* sps_id);
  413. Result ParsePPS(int* pps_id);
  414. // Parses the SPS ID from the SPSExt, but otherwise does nothing.
  415. Result ParseSPSExt(int* sps_id);
  416. // Return a pointer to SPS/PPS with given |sps_id|/|pps_id| or NULL if not
  417. // present.
  418. const H264SPS* GetSPS(int sps_id) const;
  419. const H264PPS* GetPPS(int pps_id) const;
  420. // Slice headers and SEI messages are not used across NALUs by the parser
  421. // and can be discarded after current NALU, so the parser does not store
  422. // them, nor does it manage their memory.
  423. // The caller has to provide and manage it instead.
  424. // Parse a slice header, returning it in |*shdr|. |*nalu| must be set to
  425. // the NALU returned from AdvanceToNextNALU() and corresponding to |*shdr|.
  426. Result ParseSliceHeader(const H264NALU& nalu, H264SliceHeader* shdr);
  427. // Parse a SEI message, returning it in |*sei_msg|, provided and managed
  428. // by the caller.
  429. Result ParseSEI(H264SEIMessage* sei_msg);
  430. // The return value of this method changes for every successful call to
  431. // AdvanceToNextNALU().
  432. // This returns the subsample information for the last NALU that was output
  433. // from AdvanceToNextNALU().
  434. std::vector<SubsampleEntry> GetCurrentSubsamples();
  435. private:
  436. // Move the stream pointer to the beginning of the next NALU,
  437. // i.e. pointing at the next start code.
  438. // Return true if a NALU has been found.
  439. // If a NALU is found:
  440. // - its size in bytes is returned in |*nalu_size| and includes
  441. // the start code as well as the trailing zero bits.
  442. // - the size in bytes of the start code is returned in |*start_code_size|.
  443. bool LocateNALU(off_t* nalu_size, off_t* start_code_size);
  444. // Exp-Golomb code parsing as specified in chapter 9.1 of the spec.
  445. // Read one unsigned exp-Golomb code from the stream and return in |*val|.
  446. Result ReadUE(int* val);
  447. // Read one signed exp-Golomb code from the stream and return in |*val|.
  448. Result ReadSE(int* val);
  449. // Parse scaling lists (see spec).
  450. Result ParseScalingList(int size, int* scaling_list, bool* use_default);
  451. Result ParseSPSScalingLists(H264SPS* sps);
  452. Result ParsePPSScalingLists(const H264SPS& sps, H264PPS* pps);
  453. // Parse optional VUI parameters in SPS (see spec).
  454. Result ParseVUIParameters(H264SPS* sps);
  455. // Set |hrd_parameters_present| to true only if they are present.
  456. Result ParseAndIgnoreHRDParameters(bool* hrd_parameters_present);
  457. // Parse reference picture lists' modifications (see spec).
  458. Result ParseRefPicListModifications(H264SliceHeader* shdr);
  459. Result ParseRefPicListModification(int num_ref_idx_active_minus1,
  460. H264ModificationOfPicNum* ref_list_mods);
  461. // Parse prediction weight table (see spec).
  462. Result ParsePredWeightTable(const H264SPS& sps, H264SliceHeader* shdr);
  463. // Parse weighting factors (see spec).
  464. Result ParseWeightingFactors(int num_ref_idx_active_minus1,
  465. int chroma_array_type,
  466. int luma_log2_weight_denom,
  467. int chroma_log2_weight_denom,
  468. H264WeightingFactors* w_facts);
  469. // Parse decoded reference picture marking information (see spec).
  470. Result ParseDecRefPicMarking(H264SliceHeader* shdr);
  471. // Pointer to the current NALU in the stream.
  472. const uint8_t* stream_;
  473. // Bytes left in the stream after the current NALU.
  474. off_t bytes_left_;
  475. H264BitReader br_;
  476. // PPSes and SPSes stored for future reference.
  477. std::map<int, std::unique_ptr<H264SPS>> active_SPSes_;
  478. std::map<int, std::unique_ptr<H264PPS>> active_PPSes_;
  479. // Ranges of encrypted bytes in the buffer passed to
  480. // SetEncryptedStream().
  481. Ranges<const uint8_t*> encrypted_ranges_;
  482. // This contains the range of the previous NALU found in
  483. // AdvanceToNextNalu(). Holds exactly one range.
  484. Ranges<const uint8_t*> previous_nalu_range_;
  485. };
  486. } // namespace media
  487. #endif // MEDIA_VIDEO_H264_PARSER_H_