h264_decoder.cc 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/h264_decoder.h"
  5. #include <algorithm>
  6. #include <limits>
  7. #include <memory>
  8. #include "base/bind.h"
  9. #include "base/callback_helpers.h"
  10. #include "base/feature_list.h"
  11. #include "base/logging.h"
  12. #include "base/metrics/histogram_functions.h"
  13. #include "base/numerics/safe_conversions.h"
  14. #include "media/base/media_switches.h"
  15. #include "media/video/h264_level_limits.h"
  16. #include "third_party/abseil-cpp/absl/types/optional.h"
  17. namespace media {
  18. namespace {
  19. bool ParseBitDepth(const H264SPS& sps, uint8_t& bit_depth) {
  20. // Spec 7.4.2.1.1
  21. if (sps.bit_depth_luma_minus8 != sps.bit_depth_chroma_minus8) {
  22. DVLOG(1) << "H264Decoder doesn't support different bit depths between luma"
  23. << "and chroma, bit_depth_luma_minus8="
  24. << sps.bit_depth_luma_minus8
  25. << ", bit_depth_chroma_minus8=" << sps.bit_depth_chroma_minus8;
  26. return false;
  27. }
  28. DCHECK_GE(sps.bit_depth_luma_minus8, 0);
  29. DCHECK_LE(sps.bit_depth_luma_minus8, 6);
  30. switch (sps.bit_depth_luma_minus8) {
  31. case 0:
  32. bit_depth = 8u;
  33. break;
  34. case 2:
  35. bit_depth = 10u;
  36. break;
  37. case 4:
  38. bit_depth = 12u;
  39. break;
  40. case 6:
  41. bit_depth = 14u;
  42. break;
  43. default:
  44. DVLOG(1) << "Invalid bit depth: "
  45. << base::checked_cast<int>(sps.bit_depth_luma_minus8 + 8);
  46. return false;
  47. }
  48. return true;
  49. }
  50. bool IsValidBitDepth(uint8_t bit_depth, VideoCodecProfile profile) {
  51. // Spec A.2.
  52. switch (profile) {
  53. case H264PROFILE_BASELINE:
  54. case H264PROFILE_MAIN:
  55. case H264PROFILE_EXTENDED:
  56. case H264PROFILE_HIGH:
  57. return bit_depth == 8u;
  58. case H264PROFILE_HIGH10PROFILE:
  59. case H264PROFILE_HIGH422PROFILE:
  60. return bit_depth == 8u || bit_depth == 10u;
  61. case H264PROFILE_HIGH444PREDICTIVEPROFILE:
  62. return bit_depth == 8u || bit_depth == 10u || bit_depth == 12u ||
  63. bit_depth == 14u;
  64. case H264PROFILE_SCALABLEBASELINE:
  65. case H264PROFILE_SCALABLEHIGH:
  66. // Spec G.10.1.
  67. return bit_depth == 8u;
  68. case H264PROFILE_STEREOHIGH:
  69. case H264PROFILE_MULTIVIEWHIGH:
  70. // Spec H.10.1.1 and H.10.1.2.
  71. return bit_depth == 8u;
  72. default:
  73. NOTREACHED();
  74. return false;
  75. }
  76. }
  77. } // namespace
  78. H264Decoder::H264Accelerator::H264Accelerator() = default;
  79. H264Decoder::H264Accelerator::~H264Accelerator() = default;
  80. void H264Decoder::H264Accelerator::ProcessSPS(
  81. const H264SPS* sps,
  82. base::span<const uint8_t> sps_nalu_data) {}
  83. void H264Decoder::H264Accelerator::ProcessPPS(
  84. const H264PPS* pps,
  85. base::span<const uint8_t> pps_nalu_data) {}
  86. H264Decoder::H264Accelerator::Status
  87. H264Decoder::H264Accelerator::ParseEncryptedSliceHeader(
  88. const std::vector<base::span<const uint8_t>>& data,
  89. const std::vector<SubsampleEntry>& subsamples,
  90. H264SliceHeader* slice_header_out) {
  91. return H264Decoder::H264Accelerator::Status::kNotSupported;
  92. }
  93. H264Decoder::H264Accelerator::Status H264Decoder::H264Accelerator::SetStream(
  94. base::span<const uint8_t> stream,
  95. const DecryptConfig* decrypt_config) {
  96. return H264Decoder::H264Accelerator::Status::kNotSupported;
  97. }
  98. H264Decoder::H264Decoder(std::unique_ptr<H264Accelerator> accelerator,
  99. VideoCodecProfile profile,
  100. const VideoColorSpace& container_color_space)
  101. : state_(State::kNeedStreamMetadata),
  102. container_color_space_(container_color_space),
  103. max_frame_num_(0),
  104. max_pic_num_(0),
  105. max_long_term_frame_idx_(0),
  106. max_num_reorder_frames_(0),
  107. // TODO(hiroh): Set profile to UNKNOWN.
  108. profile_(profile),
  109. accelerator_(std::move(accelerator)) {
  110. DCHECK(accelerator_);
  111. Reset();
  112. }
  113. H264Decoder::~H264Decoder() = default;
  114. void H264Decoder::Reset() {
  115. curr_pic_ = nullptr;
  116. curr_nalu_ = nullptr;
  117. curr_slice_hdr_ = nullptr;
  118. curr_sps_id_ = -1;
  119. curr_pps_id_ = -1;
  120. prev_frame_num_ = -1;
  121. prev_ref_frame_num_ = -1;
  122. prev_frame_num_offset_ = -1;
  123. prev_has_memmgmnt5_ = false;
  124. prev_ref_has_memmgmnt5_ = false;
  125. prev_ref_top_field_order_cnt_ = -1;
  126. prev_ref_pic_order_cnt_msb_ = -1;
  127. prev_ref_pic_order_cnt_lsb_ = -1;
  128. prev_ref_field_ = H264Picture::FIELD_NONE;
  129. ref_pic_list_p0_.clear();
  130. ref_pic_list_b0_.clear();
  131. ref_pic_list_b1_.clear();
  132. dpb_.Clear();
  133. parser_.Reset();
  134. accelerator_->Reset();
  135. last_output_poc_ = std::numeric_limits<int>::min();
  136. encrypted_sei_nalus_.clear();
  137. sei_subsamples_.clear();
  138. recovery_frame_num_.reset();
  139. recovery_frame_cnt_.reset();
  140. // If we are in kDecoding, we can resume without processing an SPS.
  141. // The state becomes kDecoding again, (1) at the first IDR slice or (2) at
  142. // the first slice after the recovery point SEI.
  143. if (state_ == State::kDecoding)
  144. state_ = State::kAfterReset;
  145. }
  146. void H264Decoder::PrepareRefPicLists() {
  147. ConstructReferencePicListsP();
  148. ConstructReferencePicListsB();
  149. }
  150. bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr,
  151. H264Picture::Vector* ref_pic_list0,
  152. H264Picture::Vector* ref_pic_list1) {
  153. ref_pic_list0->clear();
  154. ref_pic_list1->clear();
  155. // Fill reference picture lists for B and S/SP slices.
  156. if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) {
  157. *ref_pic_list0 = ref_pic_list_p0_;
  158. return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0);
  159. } else if (slice_hdr->IsBSlice()) {
  160. *ref_pic_list0 = ref_pic_list_b0_;
  161. *ref_pic_list1 = ref_pic_list_b1_;
  162. return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) &&
  163. ModifyReferencePicList(slice_hdr, 1, ref_pic_list1);
  164. }
  165. return true;
  166. }
  167. H264Decoder::H264Accelerator::Status H264Decoder::DecodePicture() {
  168. DCHECK(curr_pic_.get());
  169. return accelerator_->SubmitDecode(curr_pic_);
  170. }
  171. bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic,
  172. int frame_num) {
  173. pic->nonexisting = true;
  174. pic->nal_ref_idc = 1;
  175. pic->frame_num = pic->pic_num = frame_num;
  176. pic->adaptive_ref_pic_marking_mode_flag = false;
  177. pic->ref = true;
  178. pic->long_term_reference_flag = false;
  179. pic->field = H264Picture::FIELD_NONE;
  180. return CalculatePicOrderCounts(pic);
  181. }
  182. bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) {
  183. if (!FillH264PictureFromSliceHeader(parser_.GetSPS(curr_sps_id_), *slice_hdr,
  184. curr_pic_.get())) {
  185. return false;
  186. }
  187. if (!CalculatePicOrderCounts(curr_pic_))
  188. return false;
  189. curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag;
  190. curr_pic_->adaptive_ref_pic_marking_mode_flag =
  191. slice_hdr->adaptive_ref_pic_marking_mode_flag;
  192. // If the slice header indicates we will have to perform reference marking
  193. // process after this picture is decoded, store required data for that
  194. // purpose.
  195. if (slice_hdr->adaptive_ref_pic_marking_mode_flag) {
  196. static_assert(sizeof(curr_pic_->ref_pic_marking) ==
  197. sizeof(slice_hdr->ref_pic_marking),
  198. "Array sizes of ref pic marking do not match.");
  199. memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking,
  200. sizeof(curr_pic_->ref_pic_marking));
  201. }
  202. curr_pic_->set_visible_rect(visible_rect_);
  203. curr_pic_->set_bitstream_id(stream_id_);
  204. return true;
  205. }
  206. bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) {
  207. const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
  208. if (!sps)
  209. return false;
  210. switch (pic->pic_order_cnt_type) {
  211. case 0: {
  212. // See spec 8.2.1.1.
  213. int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb;
  214. if (pic->idr) {
  215. prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0;
  216. } else {
  217. if (prev_ref_has_memmgmnt5_) {
  218. if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) {
  219. prev_pic_order_cnt_msb = 0;
  220. prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_;
  221. } else {
  222. prev_pic_order_cnt_msb = 0;
  223. prev_pic_order_cnt_lsb = 0;
  224. }
  225. } else {
  226. prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_;
  227. prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_;
  228. }
  229. }
  230. int max_pic_order_cnt_lsb =
  231. 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
  232. DCHECK_NE(max_pic_order_cnt_lsb, 0);
  233. if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
  234. (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >=
  235. max_pic_order_cnt_lsb / 2)) {
  236. pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb;
  237. } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) &&
  238. (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb >
  239. max_pic_order_cnt_lsb / 2)) {
  240. pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb;
  241. } else {
  242. pic->pic_order_cnt_msb = prev_pic_order_cnt_msb;
  243. }
  244. if (pic->field != H264Picture::FIELD_BOTTOM) {
  245. pic->top_field_order_cnt =
  246. pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
  247. }
  248. if (pic->field != H264Picture::FIELD_TOP) {
  249. if (pic->field == H264Picture::FIELD_NONE) {
  250. pic->bottom_field_order_cnt =
  251. pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom;
  252. } else {
  253. pic->bottom_field_order_cnt =
  254. pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
  255. }
  256. }
  257. break;
  258. }
  259. case 1: {
  260. // See spec 8.2.1.2.
  261. if (prev_has_memmgmnt5_)
  262. prev_frame_num_offset_ = 0;
  263. if (pic->idr)
  264. pic->frame_num_offset = 0;
  265. else if (prev_frame_num_ > pic->frame_num)
  266. pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
  267. else
  268. pic->frame_num_offset = prev_frame_num_offset_;
  269. int abs_frame_num = 0;
  270. if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
  271. abs_frame_num = pic->frame_num_offset + pic->frame_num;
  272. else
  273. abs_frame_num = 0;
  274. if (pic->nal_ref_idc == 0 && abs_frame_num > 0)
  275. --abs_frame_num;
  276. int expected_pic_order_cnt = 0;
  277. if (abs_frame_num > 0) {
  278. if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) {
  279. DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle "
  280. << "in stream";
  281. return false;
  282. }
  283. int pic_order_cnt_cycle_cnt =
  284. (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle;
  285. int frame_num_in_pic_order_cnt_cycle =
  286. (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle;
  287. expected_pic_order_cnt = pic_order_cnt_cycle_cnt *
  288. sps->expected_delta_per_pic_order_cnt_cycle;
  289. // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser
  290. for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i)
  291. expected_pic_order_cnt += sps->offset_for_ref_frame[i];
  292. }
  293. if (!pic->nal_ref_idc)
  294. expected_pic_order_cnt += sps->offset_for_non_ref_pic;
  295. if (pic->field == H264Picture::FIELD_NONE) {
  296. pic->top_field_order_cnt =
  297. expected_pic_order_cnt + pic->delta_pic_order_cnt0;
  298. pic->bottom_field_order_cnt = pic->top_field_order_cnt +
  299. sps->offset_for_top_to_bottom_field +
  300. pic->delta_pic_order_cnt1;
  301. } else if (pic->field != H264Picture::FIELD_BOTTOM) {
  302. pic->top_field_order_cnt =
  303. expected_pic_order_cnt + pic->delta_pic_order_cnt0;
  304. } else {
  305. pic->bottom_field_order_cnt = expected_pic_order_cnt +
  306. sps->offset_for_top_to_bottom_field +
  307. pic->delta_pic_order_cnt0;
  308. }
  309. break;
  310. }
  311. case 2: {
  312. // See spec 8.2.1.3.
  313. if (prev_has_memmgmnt5_)
  314. prev_frame_num_offset_ = 0;
  315. if (pic->idr)
  316. pic->frame_num_offset = 0;
  317. else if (prev_frame_num_ > pic->frame_num)
  318. pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
  319. else
  320. pic->frame_num_offset = prev_frame_num_offset_;
  321. int temp_pic_order_cnt;
  322. if (pic->idr) {
  323. temp_pic_order_cnt = 0;
  324. } else if (!pic->nal_ref_idc) {
  325. temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1;
  326. } else {
  327. temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num);
  328. }
  329. if (pic->field == H264Picture::FIELD_NONE) {
  330. pic->top_field_order_cnt = temp_pic_order_cnt;
  331. pic->bottom_field_order_cnt = temp_pic_order_cnt;
  332. } else if (pic->field == H264Picture::FIELD_BOTTOM) {
  333. pic->bottom_field_order_cnt = temp_pic_order_cnt;
  334. } else {
  335. pic->top_field_order_cnt = temp_pic_order_cnt;
  336. }
  337. break;
  338. }
  339. default:
  340. DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type;
  341. return false;
  342. }
  343. switch (pic->field) {
  344. case H264Picture::FIELD_NONE:
  345. pic->pic_order_cnt =
  346. std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt);
  347. break;
  348. case H264Picture::FIELD_TOP:
  349. pic->pic_order_cnt = pic->top_field_order_cnt;
  350. break;
  351. case H264Picture::FIELD_BOTTOM:
  352. pic->pic_order_cnt = pic->bottom_field_order_cnt;
  353. break;
  354. }
  355. return true;
  356. }
  357. void H264Decoder::UpdatePicNums(int frame_num) {
  358. for (auto& pic : dpb_) {
  359. if (!pic->ref)
  360. continue;
  361. // 8.2.4.1. Assumes non-interlaced stream.
  362. DCHECK_EQ(pic->field, H264Picture::FIELD_NONE);
  363. if (pic->long_term) {
  364. pic->long_term_pic_num = pic->long_term_frame_idx;
  365. } else {
  366. if (pic->frame_num > frame_num)
  367. pic->frame_num_wrap = pic->frame_num - max_frame_num_;
  368. else
  369. pic->frame_num_wrap = pic->frame_num;
  370. pic->pic_num = pic->frame_num_wrap;
  371. }
  372. }
  373. }
  374. struct PicNumDescCompare {
  375. bool operator()(const scoped_refptr<H264Picture>& a,
  376. const scoped_refptr<H264Picture>& b) const {
  377. return a->pic_num > b->pic_num;
  378. }
  379. };
  380. struct LongTermPicNumAscCompare {
  381. bool operator()(const scoped_refptr<H264Picture>& a,
  382. const scoped_refptr<H264Picture>& b) const {
  383. return a->long_term_pic_num < b->long_term_pic_num;
  384. }
  385. };
  386. void H264Decoder::ConstructReferencePicListsP() {
  387. // RefPicList0 (8.2.4.2.1) [[1] [2]], where:
  388. // [1] shortterm ref pics sorted by descending pic_num,
  389. // [2] longterm ref pics by ascending long_term_pic_num.
  390. ref_pic_list_p0_.clear();
  391. // First get the short ref pics...
  392. dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_);
  393. size_t num_short_refs = ref_pic_list_p0_.size();
  394. // and sort them to get [1].
  395. std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(),
  396. PicNumDescCompare());
  397. // Now get long term pics and sort them by long_term_pic_num to get [2].
  398. dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_);
  399. std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(),
  400. LongTermPicNumAscCompare());
  401. }
  402. struct POCAscCompare {
  403. bool operator()(const scoped_refptr<H264Picture>& a,
  404. const scoped_refptr<H264Picture>& b) const {
  405. return a->pic_order_cnt < b->pic_order_cnt;
  406. }
  407. };
  408. struct POCDescCompare {
  409. bool operator()(const scoped_refptr<H264Picture>& a,
  410. const scoped_refptr<H264Picture>& b) const {
  411. return a->pic_order_cnt > b->pic_order_cnt;
  412. }
  413. };
  414. void H264Decoder::ConstructReferencePicListsB() {
  415. // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where:
  416. // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC,
  417. // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC,
  418. // [3] longterm ref pics by ascending long_term_pic_num.
  419. ref_pic_list_b0_.clear();
  420. ref_pic_list_b1_.clear();
  421. dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_);
  422. size_t num_short_refs = ref_pic_list_b0_.size();
  423. // First sort ascending, this will put [1] in right place and finish [2].
  424. std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare());
  425. // Find first with POC > curr_pic's POC to get first element in [2]...
  426. H264Picture::Vector::iterator iter;
  427. iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
  428. curr_pic_.get(), POCAscCompare());
  429. // and sort [1] descending, thus finishing sequence [1] [2].
  430. std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare());
  431. // Now add [3] and sort by ascending long_term_pic_num.
  432. dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_);
  433. std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(),
  434. LongTermPicNumAscCompare());
  435. // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where:
  436. // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
  437. // [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
  438. // [3] longterm ref pics by ascending long_term_pic_num.
  439. dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_);
  440. num_short_refs = ref_pic_list_b1_.size();
  441. // First sort by descending POC.
  442. std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare());
  443. // Find first with POC < curr_pic's POC to get first element in [2]...
  444. iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(),
  445. curr_pic_.get(), POCDescCompare());
  446. // and sort [1] ascending.
  447. std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare());
  448. // Now add [3] and sort by ascending long_term_pic_num
  449. dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b1_);
  450. std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(),
  451. LongTermPicNumAscCompare());
  452. // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3)
  453. if (ref_pic_list_b1_.size() > 1 &&
  454. std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
  455. ref_pic_list_b1_.begin()))
  456. std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]);
  457. }
  458. // See 8.2.4
  459. int H264Decoder::PicNumF(const H264Picture& pic) {
  460. if (!pic.long_term)
  461. return pic.pic_num;
  462. else
  463. return max_pic_num_;
  464. }
  465. // See 8.2.4
  466. int H264Decoder::LongTermPicNumF(const H264Picture& pic) {
  467. if (pic.ref && pic.long_term)
  468. return pic.long_term_pic_num;
  469. else
  470. return 2 * (max_long_term_frame_idx_ + 1);
  471. }
  472. // Shift elements on the |v| starting from |from| to |to|, inclusive,
  473. // one position to the right and insert pic at |from|.
  474. static void ShiftRightAndInsert(H264Picture::Vector* v,
  475. int from,
  476. int to,
  477. scoped_refptr<H264Picture> pic) {
  478. // Security checks, do not disable in Debug mode.
  479. CHECK(from <= to);
  480. CHECK(to <= std::numeric_limits<int>::max() - 2);
  481. // Additional checks. Debug mode ok.
  482. DCHECK(v);
  483. DCHECK(pic);
  484. DCHECK((to + 1 == static_cast<int>(v->size())) ||
  485. (to + 2 == static_cast<int>(v->size())));
  486. v->resize(to + 2);
  487. for (int i = to + 1; i > from; --i)
  488. (*v)[i] = (*v)[i - 1];
  489. (*v)[from] = std::move(pic);
  490. }
  491. bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr,
  492. int list,
  493. H264Picture::Vector* ref_pic_listx) {
  494. bool ref_pic_list_modification_flag_lX;
  495. int num_ref_idx_lX_active_minus1;
  496. const H264ModificationOfPicNum* list_mod;
  497. // This can process either ref_pic_list0 or ref_pic_list1, depending on
  498. // the list argument. Set up pointers to proper list to be processed here.
  499. if (list == 0) {
  500. ref_pic_list_modification_flag_lX =
  501. slice_hdr->ref_pic_list_modification_flag_l0;
  502. num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1;
  503. list_mod = slice_hdr->ref_list_l0_modifications;
  504. } else {
  505. ref_pic_list_modification_flag_lX =
  506. slice_hdr->ref_pic_list_modification_flag_l1;
  507. num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1;
  508. list_mod = slice_hdr->ref_list_l1_modifications;
  509. }
  510. // Resize the list to the size requested in the slice header.
  511. // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to
  512. // indicate there should be more ref pics on list than we constructed.
  513. // Those superfluous ones should be treated as non-reference and will be
  514. // initialized to nullptr, which must be handled by clients.
  515. DCHECK_GE(num_ref_idx_lX_active_minus1, 0);
  516. ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
  517. if (!ref_pic_list_modification_flag_lX)
  518. return true;
  519. // Spec 8.2.4.3:
  520. // Reorder pictures on the list in a way specified in the stream.
  521. int pic_num_lx_pred = curr_pic_->pic_num;
  522. int ref_idx_lx = 0;
  523. int pic_num_lx_no_wrap;
  524. int pic_num_lx;
  525. bool done = false;
  526. scoped_refptr<H264Picture> pic;
  527. for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) {
  528. switch (list_mod->modification_of_pic_nums_idc) {
  529. case 0:
  530. case 1:
  531. // Modify short reference picture position.
  532. if (list_mod->modification_of_pic_nums_idc == 0) {
  533. // Subtract given value from predicted PicNum.
  534. pic_num_lx_no_wrap =
  535. pic_num_lx_pred -
  536. (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
  537. // Wrap around max_pic_num_ if it becomes < 0 as result
  538. // of subtraction.
  539. if (pic_num_lx_no_wrap < 0)
  540. pic_num_lx_no_wrap += max_pic_num_;
  541. } else {
  542. // Add given value to predicted PicNum.
  543. pic_num_lx_no_wrap =
  544. pic_num_lx_pred +
  545. (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
  546. // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result
  547. // of the addition.
  548. if (pic_num_lx_no_wrap >= max_pic_num_)
  549. pic_num_lx_no_wrap -= max_pic_num_;
  550. }
  551. // For use in next iteration.
  552. pic_num_lx_pred = pic_num_lx_no_wrap;
  553. if (pic_num_lx_no_wrap > curr_pic_->pic_num)
  554. pic_num_lx = pic_num_lx_no_wrap - max_pic_num_;
  555. else
  556. pic_num_lx = pic_num_lx_no_wrap;
  557. DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
  558. H264SliceHeader::kRefListModSize);
  559. pic = dpb_.GetShortRefPicByPicNum(pic_num_lx);
  560. if (!pic) {
  561. DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx;
  562. return false;
  563. }
  564. if (ref_idx_lx > num_ref_idx_lX_active_minus1) {
  565. DVLOG(1) << "Bounds mismatch: expected " << ref_idx_lx
  566. << " <= " << num_ref_idx_lX_active_minus1;
  567. return false;
  568. }
  569. ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
  570. num_ref_idx_lX_active_minus1, pic);
  571. ref_idx_lx++;
  572. for (int src = ref_idx_lx, dst = ref_idx_lx;
  573. src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
  574. auto* src_pic = (*ref_pic_listx)[src].get();
  575. int src_pic_num_lx = src_pic ? PicNumF(*src_pic) : -1;
  576. if (src_pic_num_lx != pic_num_lx)
  577. (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
  578. }
  579. break;
  580. case 2:
  581. // Modify long term reference picture position.
  582. DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
  583. H264SliceHeader::kRefListModSize);
  584. pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num);
  585. if (!pic) {
  586. DVLOG(1) << "Malformed stream, no pic num "
  587. << list_mod->long_term_pic_num;
  588. return false;
  589. }
  590. ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
  591. num_ref_idx_lX_active_minus1, pic);
  592. ref_idx_lx++;
  593. for (int src = ref_idx_lx, dst = ref_idx_lx;
  594. src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
  595. if (LongTermPicNumF(*(*ref_pic_listx)[src]) !=
  596. static_cast<int>(list_mod->long_term_pic_num))
  597. (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
  598. }
  599. break;
  600. case 3:
  601. // End of modification list.
  602. done = true;
  603. break;
  604. default:
  605. // May be recoverable.
  606. DVLOG(1) << "Invalid modification_of_pic_nums_idc="
  607. << list_mod->modification_of_pic_nums_idc << " in position "
  608. << i;
  609. break;
  610. }
  611. ++list_mod;
  612. }
  613. // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is
  614. // temporarily made one element longer than the required final list.
  615. // Resize the list back to its required size.
  616. ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
  617. return true;
  618. }
  619. bool H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) {
  620. DCHECK(!pic->outputted);
  621. pic->outputted = true;
  622. VideoColorSpace colorspace_for_frame = container_color_space_;
  623. const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
  624. if (sps && sps->GetColorSpace().IsSpecified())
  625. colorspace_for_frame = sps->GetColorSpace();
  626. pic->set_colorspace(colorspace_for_frame);
  627. if (pic->nonexisting) {
  628. DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num;
  629. return true;
  630. }
  631. DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_)
  632. << "Outputting out of order, likely a broken stream: " << last_output_poc_
  633. << " -> " << pic->pic_order_cnt;
  634. last_output_poc_ = pic->pic_order_cnt;
  635. DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt;
  636. return accelerator_->OutputPicture(pic);
  637. }
  638. void H264Decoder::ClearDPB() {
  639. // Clear DPB contents, marking the pictures as unused first.
  640. dpb_.Clear();
  641. last_output_poc_ = std::numeric_limits<int>::min();
  642. }
  643. bool H264Decoder::OutputAllRemainingPics() {
  644. // Output all pictures that are waiting to be outputted.
  645. if (FinishPrevFrameIfPresent() != H264Accelerator::Status::kOk)
  646. return false;
  647. H264Picture::Vector to_output;
  648. dpb_.GetNotOutputtedPicsAppending(&to_output);
  649. // Sort them by ascending POC to output in order.
  650. std::sort(to_output.begin(), to_output.end(), POCAscCompare());
  651. for (auto& pic : to_output) {
  652. if (!OutputPic(pic))
  653. return false;
  654. }
  655. return true;
  656. }
  657. bool H264Decoder::Flush() {
  658. DVLOG(2) << "Decoder flush";
  659. if (!OutputAllRemainingPics())
  660. return false;
  661. ClearDPB();
  662. DVLOG(2) << "Decoder flush finished";
  663. return true;
  664. }
  665. H264Decoder::H264Accelerator::Status H264Decoder::StartNewFrame(
  666. const H264SliceHeader* slice_hdr) {
  667. // TODO posciak: add handling of max_num_ref_frames per spec.
  668. CHECK(curr_pic_.get());
  669. DCHECK(slice_hdr);
  670. curr_pps_id_ = slice_hdr->pic_parameter_set_id;
  671. const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
  672. if (!pps)
  673. return H264Accelerator::Status::kFail;
  674. curr_sps_id_ = pps->seq_parameter_set_id;
  675. const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
  676. if (!sps)
  677. return H264Accelerator::Status::kFail;
  678. max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4);
  679. int frame_num = slice_hdr->frame_num;
  680. if (slice_hdr->idr_pic_flag)
  681. prev_ref_frame_num_ = 0;
  682. // 7.4.3
  683. if (frame_num != prev_ref_frame_num_ &&
  684. frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) {
  685. if (!HandleFrameNumGap(frame_num))
  686. return H264Accelerator::Status::kFail;
  687. }
  688. if (!InitCurrPicture(slice_hdr))
  689. return H264Accelerator::Status::kFail;
  690. UpdatePicNums(frame_num);
  691. PrepareRefPicLists();
  692. return accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_,
  693. ref_pic_list_b0_, ref_pic_list_b1_,
  694. curr_pic_.get());
  695. }
  696. bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) {
  697. // 8.2.5.4
  698. for (size_t i = 0; i < std::size(pic->ref_pic_marking); ++i) {
  699. // Code below does not support interlaced stream (per-field pictures).
  700. H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i];
  701. scoped_refptr<H264Picture> to_mark;
  702. int pic_num_x;
  703. switch (ref_pic_marking->memory_mgmnt_control_operation) {
  704. case 0:
  705. // Normal end of operations' specification.
  706. return true;
  707. case 1:
  708. // Mark a short term reference picture as unused so it can be removed
  709. // if outputted.
  710. pic_num_x =
  711. pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
  712. to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
  713. if (to_mark) {
  714. to_mark->ref = false;
  715. } else {
  716. DVLOG(1) << "Invalid short ref pic num to unmark";
  717. return false;
  718. }
  719. break;
  720. case 2:
  721. // Mark a long term reference picture as unused so it can be removed
  722. // if outputted.
  723. to_mark = dpb_.GetLongRefPicByLongTermPicNum(
  724. ref_pic_marking->long_term_pic_num);
  725. if (to_mark) {
  726. to_mark->ref = false;
  727. } else {
  728. DVLOG(1) << "Invalid long term ref pic num to unmark";
  729. return false;
  730. }
  731. break;
  732. case 3:
  733. // Mark a short term reference picture as long term reference.
  734. pic_num_x =
  735. pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
  736. to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
  737. if (to_mark) {
  738. DCHECK(to_mark->ref && !to_mark->long_term);
  739. to_mark->long_term = true;
  740. to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
  741. } else {
  742. DVLOG(1) << "Invalid short term ref pic num to mark as long ref";
  743. return false;
  744. }
  745. break;
  746. case 4: {
  747. // Unmark all reference pictures with long_term_frame_idx over new max.
  748. max_long_term_frame_idx_ =
  749. ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
  750. H264Picture::Vector long_terms;
  751. dpb_.GetLongTermRefPicsAppending(&long_terms);
  752. for (size_t long_term = 0; long_term < long_terms.size(); ++long_term) {
  753. scoped_refptr<H264Picture>& long_term_pic = long_terms[long_term];
  754. DCHECK(long_term_pic->ref && long_term_pic->long_term);
  755. // Ok to cast, max_long_term_frame_idx is much smaller than 16bit.
  756. if (long_term_pic->long_term_frame_idx >
  757. static_cast<int>(max_long_term_frame_idx_))
  758. long_term_pic->ref = false;
  759. }
  760. break;
  761. }
  762. case 5:
  763. // Unmark all reference pictures.
  764. dpb_.MarkAllUnusedForRef();
  765. max_long_term_frame_idx_ = -1;
  766. pic->mem_mgmt_5 = true;
  767. break;
  768. case 6: {
  769. // Replace long term reference pictures with current picture.
  770. // First unmark if any existing with this long_term_frame_idx...
  771. H264Picture::Vector long_terms;
  772. dpb_.GetLongTermRefPicsAppending(&long_terms);
  773. for (size_t long_term = 0; long_term < long_terms.size(); ++long_term) {
  774. scoped_refptr<H264Picture>& long_term_pic = long_terms[long_term];
  775. DCHECK(long_term_pic->ref && long_term_pic->long_term);
  776. // Ok to cast, long_term_frame_idx is much smaller than 16bit.
  777. if (long_term_pic->long_term_frame_idx ==
  778. static_cast<int>(ref_pic_marking->long_term_frame_idx))
  779. long_term_pic->ref = false;
  780. }
  781. // and mark the current one instead.
  782. pic->ref = true;
  783. pic->long_term = true;
  784. pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
  785. break;
  786. }
  787. default:
  788. // Would indicate a bug in parser.
  789. NOTREACHED();
  790. }
  791. }
  792. return true;
  793. }
  794. // This method ensures that DPB does not overflow, either by removing
  795. // reference pictures as specified in the stream, or using a sliding window
  796. // procedure to remove the oldest one.
  797. // It also performs marking and unmarking pictures as reference.
  798. // See spac 8.2.5.1.
  799. bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) {
  800. // If the current picture is an IDR, all reference pictures are unmarked.
  801. if (pic->idr) {
  802. dpb_.MarkAllUnusedForRef();
  803. if (pic->long_term_reference_flag) {
  804. pic->long_term = true;
  805. pic->long_term_frame_idx = 0;
  806. max_long_term_frame_idx_ = 0;
  807. } else {
  808. pic->long_term = false;
  809. max_long_term_frame_idx_ = -1;
  810. }
  811. return true;
  812. }
  813. // Not an IDR. If the stream contains instructions on how to discard pictures
  814. // from DPB and how to mark/unmark existing reference pictures, do so.
  815. // Otherwise, fall back to default sliding window process.
  816. if (pic->adaptive_ref_pic_marking_mode_flag) {
  817. DCHECK(!pic->nonexisting);
  818. return HandleMemoryManagementOps(pic);
  819. } else {
  820. return SlidingWindowPictureMarking();
  821. }
  822. }
  823. bool H264Decoder::SlidingWindowPictureMarking() {
  824. const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
  825. if (!sps)
  826. return false;
  827. // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture.
  828. int num_ref_pics = dpb_.CountRefPics();
  829. DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1));
  830. if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) {
  831. // Max number of reference pics reached, need to remove one of the short
  832. // term ones. Find smallest frame_num_wrap short reference picture and mark
  833. // it as unused.
  834. scoped_refptr<H264Picture> to_unmark =
  835. dpb_.GetLowestFrameNumWrapShortRefPic();
  836. if (!to_unmark) {
  837. DVLOG(1) << "Couldn't find a short ref picture to unmark";
  838. return false;
  839. }
  840. to_unmark->ref = false;
  841. }
  842. return true;
  843. }
  844. bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) {
  845. // Finish processing the picture.
  846. // Start by storing previous picture data for later use.
  847. if (pic->ref) {
  848. ReferencePictureMarking(pic);
  849. prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5;
  850. prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt;
  851. prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb;
  852. prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb;
  853. prev_ref_field_ = pic->field;
  854. prev_ref_frame_num_ = pic->frame_num;
  855. }
  856. prev_frame_num_ = pic->frame_num;
  857. prev_has_memmgmnt5_ = pic->mem_mgmt_5;
  858. prev_frame_num_offset_ = pic->frame_num_offset;
  859. // Remove unused (for reference or later output) pictures from DPB, marking
  860. // them as such.
  861. dpb_.DeleteUnused();
  862. DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num
  863. << ", entries in DPB: " << dpb_.size();
  864. if (recovery_frame_cnt_) {
  865. // This is the first picture after the recovery point SEI message. Computes
  866. // the frame_num of the frame that should be output from (Spec D.2.8).
  867. recovery_frame_num_ =
  868. (*recovery_frame_cnt_ + pic->frame_num) % max_frame_num_;
  869. DVLOG(3) << "recovery_frame_num_" << *recovery_frame_num_;
  870. recovery_frame_cnt_.reset();
  871. }
  872. // The ownership of pic will either be transferred to DPB - if the picture is
  873. // still needed (for output and/or reference) - or we will release it
  874. // immediately if we manage to output it here and won't have to store it for
  875. // future reference.
  876. // Get all pictures that haven't been outputted yet.
  877. H264Picture::Vector not_outputted;
  878. dpb_.GetNotOutputtedPicsAppending(&not_outputted);
  879. // Include the one we've just decoded.
  880. not_outputted.push_back(pic);
  881. // Sort in output order.
  882. std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
  883. // Try to output as many pictures as we can. A picture can be output,
  884. // if the number of decoded and not yet outputted pictures that would remain
  885. // in DPB afterwards would at least be equal to max_num_reorder_frames.
  886. // If the outputted picture is not a reference picture, it doesn't have
  887. // to remain in the DPB and can be removed.
  888. auto output_candidate = not_outputted.begin();
  889. size_t num_remaining = not_outputted.size();
  890. while (num_remaining > max_num_reorder_frames_ ||
  891. // If the condition below is used, this is an invalid stream. We should
  892. // not be forced to output beyond max_num_reorder_frames in order to
  893. // make room in DPB to store the current picture (if we need to do so).
  894. // However, if this happens, ignore max_num_reorder_frames and try
  895. // to output more. This may cause out-of-order output, but is not
  896. // fatal, and better than failing instead.
  897. ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) {
  898. DVLOG_IF(1, num_remaining <= max_num_reorder_frames_)
  899. << "Invalid stream: max_num_reorder_frames not preserved";
  900. if (!recovery_frame_num_ ||
  901. // If we are decoding ahead to reach a SEI recovery point, skip
  902. // outputting all pictures before it, to avoid outputting corrupted
  903. // frames.
  904. (*output_candidate)->frame_num == *recovery_frame_num_) {
  905. recovery_frame_num_ = absl::nullopt;
  906. if (!OutputPic(*output_candidate))
  907. return false;
  908. }
  909. if (!(*output_candidate)->ref) {
  910. // Current picture hasn't been inserted into DPB yet, so don't remove it
  911. // if we managed to output it immediately.
  912. if (*output_candidate != pic)
  913. dpb_.Delete(*output_candidate);
  914. }
  915. ++output_candidate;
  916. --num_remaining;
  917. }
  918. // If we haven't managed to output the picture that we just decoded, or if
  919. // it's a reference picture, we have to store it in DPB.
  920. if (!pic->outputted || pic->ref) {
  921. if (dpb_.IsFull()) {
  922. // If we haven't managed to output anything to free up space in DPB
  923. // to store this picture, it's an error in the stream.
  924. DVLOG(1) << "Could not free up space in DPB!";
  925. return false;
  926. }
  927. dpb_.StorePic(std::move(pic));
  928. }
  929. return true;
  930. }
  931. bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) {
  932. if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
  933. max_num_reorder_frames_ =
  934. base::checked_cast<size_t>(sps->max_num_reorder_frames);
  935. if (max_num_reorder_frames_ > dpb_.max_num_pics()) {
  936. DVLOG(1)
  937. << "max_num_reorder_frames present, but larger than MaxDpbFrames ("
  938. << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")";
  939. max_num_reorder_frames_ = 0;
  940. return false;
  941. }
  942. return true;
  943. }
  944. // max_num_reorder_frames not present, infer from profile/constraints
  945. // (see VUI semantics in spec).
  946. if (sps->constraint_set3_flag) {
  947. switch (sps->profile_idc) {
  948. case 44:
  949. case 86:
  950. case 100:
  951. case 110:
  952. case 122:
  953. case 244:
  954. max_num_reorder_frames_ = 0;
  955. break;
  956. default:
  957. max_num_reorder_frames_ = dpb_.max_num_pics();
  958. break;
  959. }
  960. } else {
  961. max_num_reorder_frames_ = dpb_.max_num_pics();
  962. }
  963. return true;
  964. }
  965. bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
  966. DVLOG(4) << "Processing SPS id:" << sps_id;
  967. const H264SPS* sps = parser_.GetSPS(sps_id);
  968. if (!sps)
  969. return false;
  970. *need_new_buffers = false;
  971. if (sps->frame_mbs_only_flag == 0) {
  972. DVLOG(1) << "frame_mbs_only_flag != 1 not supported";
  973. return false;
  974. }
  975. gfx::Size new_pic_size = sps->GetCodedSize().value_or(gfx::Size());
  976. if (new_pic_size.IsEmpty()) {
  977. DVLOG(1) << "Invalid picture size";
  978. return false;
  979. }
  980. int width_mb = new_pic_size.width() / 16;
  981. int height_mb = new_pic_size.height() / 16;
  982. // Verify that the values are not too large before multiplying.
  983. if (std::numeric_limits<int>::max() / width_mb < height_mb) {
  984. DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString();
  985. return false;
  986. }
  987. // Spec A.3.1 and A.3.2
  988. // For Baseline, Constrained Baseline and Main profile, the indicated level is
  989. // Level 1b if level_idc is equal to 11 and constraint_set3_flag is equal to 1
  990. uint8_t level = base::checked_cast<uint8_t>(sps->level_idc);
  991. if ((sps->profile_idc == H264SPS::kProfileIDCBaseline ||
  992. sps->profile_idc == H264SPS::kProfileIDCConstrainedBaseline ||
  993. sps->profile_idc == H264SPS::kProfileIDCMain) &&
  994. level == 11 && sps->constraint_set3_flag) {
  995. level = 9; // Level 1b
  996. }
  997. int max_dpb_mbs = base::checked_cast<int>(H264LevelToMaxDpbMbs(level));
  998. if (max_dpb_mbs == 0)
  999. return false;
  1000. // MaxDpbFrames from level limits per spec.
  1001. size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb),
  1002. static_cast<int>(H264DPB::kDPBMaxSize));
  1003. DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames
  1004. << ", max_num_ref_frames: " << sps->max_num_ref_frames
  1005. << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering;
  1006. // Set DPB size to at least the level limit, or what the stream requires.
  1007. size_t max_dpb_size =
  1008. std::max(static_cast<int>(max_dpb_frames),
  1009. std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering));
  1010. // Some non-conforming streams specify more frames are needed than the current
  1011. // level limit. Allow this, but only up to the maximum number of reference
  1012. // frames allowed per spec.
  1013. DVLOG_IF(1, max_dpb_size > max_dpb_frames)
  1014. << "Invalid stream, DPB size > MaxDpbFrames";
  1015. if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) {
  1016. DVLOG(1) << "Invalid DPB size: " << max_dpb_size;
  1017. return false;
  1018. }
  1019. VideoChromaSampling new_chroma_sampling = sps->GetChromaSampling();
  1020. if (new_chroma_sampling != chroma_sampling_) {
  1021. chroma_sampling_ = new_chroma_sampling;
  1022. base::UmaHistogramEnumeration("Media.PlatformVideoDecoding.ChromaSampling",
  1023. chroma_sampling_);
  1024. }
  1025. if (chroma_sampling_ != VideoChromaSampling::k420) {
  1026. DVLOG(1) << "Only YUV 4:2:0 is supported";
  1027. return false;
  1028. }
  1029. VideoCodecProfile new_profile =
  1030. H264Parser::ProfileIDCToVideoCodecProfile(sps->profile_idc);
  1031. uint8_t new_bit_depth = 0;
  1032. if (!ParseBitDepth(*sps, new_bit_depth))
  1033. return false;
  1034. if (!IsValidBitDepth(new_bit_depth, new_profile)) {
  1035. DVLOG(1) << "Invalid bit depth=" << base::strict_cast<int>(new_bit_depth)
  1036. << ", profile=" << GetProfileName(new_profile);
  1037. return false;
  1038. }
  1039. if (pic_size_ != new_pic_size || dpb_.max_num_pics() != max_dpb_size ||
  1040. profile_ != new_profile || bit_depth_ != new_bit_depth) {
  1041. if (!Flush())
  1042. return false;
  1043. DVLOG(1) << "Codec profile: " << GetProfileName(new_profile)
  1044. << ", level: " << base::strict_cast<int>(level)
  1045. << ", DPB size: " << max_dpb_size
  1046. << ", Picture size: " << new_pic_size.ToString()
  1047. << ", bit depth: " << base::strict_cast<int>(new_bit_depth);
  1048. *need_new_buffers = true;
  1049. profile_ = new_profile;
  1050. bit_depth_ = new_bit_depth;
  1051. pic_size_ = new_pic_size;
  1052. dpb_.set_max_num_pics(max_dpb_size);
  1053. }
  1054. gfx::Rect new_visible_rect = sps->GetVisibleRect().value_or(gfx::Rect());
  1055. if (visible_rect_ != new_visible_rect) {
  1056. DVLOG(2) << "New visible rect: " << new_visible_rect.ToString();
  1057. visible_rect_ = new_visible_rect;
  1058. }
  1059. if (!UpdateMaxNumReorderFrames(sps))
  1060. return false;
  1061. DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_;
  1062. return true;
  1063. }
  1064. H264Decoder::H264Accelerator::Status H264Decoder::FinishPrevFrameIfPresent() {
  1065. // If we already have a frame waiting to be decoded, decode it and finish.
  1066. if (curr_pic_) {
  1067. H264Accelerator::Status result = DecodePicture();
  1068. if (result != H264Accelerator::Status::kOk)
  1069. return result;
  1070. scoped_refptr<H264Picture> pic = curr_pic_;
  1071. curr_pic_ = nullptr;
  1072. if (!FinishPicture(pic))
  1073. return H264Accelerator::Status::kFail;
  1074. }
  1075. return H264Accelerator::Status::kOk;
  1076. }
  1077. bool H264Decoder::HandleFrameNumGap(int frame_num) {
  1078. const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
  1079. if (!sps)
  1080. return false;
  1081. if (!sps->gaps_in_frame_num_value_allowed_flag) {
  1082. DVLOG(1) << "Invalid frame_num: " << frame_num;
  1083. // TODO(b:129119729, b:146914440): Youtube android app sometimes sends an
  1084. // invalid frame number after a seek. The sequence goes like:
  1085. // Seek, SPS, PPS, IDR-frame, non-IDR, ... non-IDR with invalid number.
  1086. // The only way to work around this reliably is to ignore this error.
  1087. // Video playback is not affected, no artefacts are visible.
  1088. // return false;
  1089. }
  1090. DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->"
  1091. << frame_num;
  1092. // 7.4.3/7-23
  1093. int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_;
  1094. while (unused_short_term_frame_num != frame_num) {
  1095. scoped_refptr<H264Picture> pic = new H264Picture();
  1096. if (!InitNonexistingPicture(pic, unused_short_term_frame_num))
  1097. return false;
  1098. UpdatePicNums(unused_short_term_frame_num);
  1099. if (!FinishPicture(pic))
  1100. return false;
  1101. unused_short_term_frame_num++;
  1102. unused_short_term_frame_num %= max_frame_num_;
  1103. }
  1104. return true;
  1105. }
  1106. H264Decoder::H264Accelerator::Status H264Decoder::ProcessEncryptedSliceHeader(
  1107. const std::vector<SubsampleEntry>& subsamples) {
  1108. DCHECK(curr_nalu_);
  1109. DCHECK(curr_slice_hdr_);
  1110. std::vector<base::span<const uint8_t>> spans(encrypted_sei_nalus_.size() + 1);
  1111. spans.assign(encrypted_sei_nalus_.begin(), encrypted_sei_nalus_.end());
  1112. spans.emplace_back(curr_nalu_->data, curr_nalu_->size);
  1113. std::vector<SubsampleEntry> all_subsamples(sei_subsamples_.size() + 1);
  1114. all_subsamples.assign(sei_subsamples_.begin(), sei_subsamples_.end());
  1115. all_subsamples.insert(all_subsamples.end(), subsamples.begin(),
  1116. subsamples.end());
  1117. return accelerator_->ParseEncryptedSliceHeader(spans, all_subsamples,
  1118. curr_slice_hdr_.get());
  1119. }
  1120. H264Decoder::H264Accelerator::Status H264Decoder::PreprocessCurrentSlice() {
  1121. const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
  1122. DCHECK(slice_hdr);
  1123. if (IsNewPrimaryCodedPicture(curr_pic_.get(), curr_pps_id_,
  1124. parser_.GetSPS(curr_sps_id_), *slice_hdr)) {
  1125. // New picture, so first finish the previous one before processing it.
  1126. H264Accelerator::Status result = FinishPrevFrameIfPresent();
  1127. if (result != H264Accelerator::Status::kOk)
  1128. return result;
  1129. DCHECK(!curr_pic_);
  1130. if (slice_hdr->first_mb_in_slice != 0) {
  1131. DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: "
  1132. << slice_hdr->first_mb_in_slice;
  1133. return H264Accelerator::Status::kFail;
  1134. }
  1135. // If the new picture is an IDR, flush DPB.
  1136. if (slice_hdr->idr_pic_flag) {
  1137. // Output all remaining pictures, unless we are explicitly instructed
  1138. // not to do so.
  1139. if (!slice_hdr->no_output_of_prior_pics_flag) {
  1140. if (!Flush())
  1141. return H264Accelerator::Status::kFail;
  1142. }
  1143. dpb_.Clear();
  1144. last_output_poc_ = std::numeric_limits<int>::min();
  1145. }
  1146. }
  1147. return H264Accelerator::Status::kOk;
  1148. }
  1149. H264Decoder::H264Accelerator::Status H264Decoder::ProcessCurrentSlice() {
  1150. DCHECK(curr_pic_);
  1151. const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
  1152. DCHECK(slice_hdr);
  1153. if (slice_hdr->field_pic_flag == 0)
  1154. max_pic_num_ = max_frame_num_;
  1155. else
  1156. max_pic_num_ = 2 * max_frame_num_;
  1157. H264Picture::Vector ref_pic_list0, ref_pic_list1;
  1158. // If we are using full sample encryption then we do not have the information
  1159. // we need to update the ref pic lists here, but that's OK because the
  1160. // accelerator doesn't actually need to submit them in this case.
  1161. if (!slice_hdr->full_sample_encryption &&
  1162. !ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1)) {
  1163. return H264Accelerator::Status::kFail;
  1164. }
  1165. const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
  1166. if (!pps)
  1167. return H264Accelerator::Status::kFail;
  1168. return accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1,
  1169. curr_pic_.get(), slice_hdr->nalu_data,
  1170. slice_hdr->nalu_size,
  1171. parser_.GetCurrentSubsamples());
  1172. }
  1173. #define SET_ERROR_AND_RETURN() \
  1174. do { \
  1175. DVLOG(1) << "Error during decode"; \
  1176. state_ = State::kError; \
  1177. return H264Decoder::kDecodeError; \
  1178. } while (0)
  1179. #define CHECK_ACCELERATOR_RESULT(func) \
  1180. do { \
  1181. H264Accelerator::Status result = (func); \
  1182. switch (result) { \
  1183. case H264Accelerator::Status::kOk: \
  1184. break; \
  1185. case H264Accelerator::Status::kTryAgain: \
  1186. DVLOG(1) << #func " needs to try again"; \
  1187. return H264Decoder::kTryAgain; \
  1188. case H264Accelerator::Status::kFail: \
  1189. case H264Accelerator::Status::kNotSupported: \
  1190. SET_ERROR_AND_RETURN(); \
  1191. } \
  1192. } while (0)
  1193. void H264Decoder::SetStream(int32_t id, const DecoderBuffer& decoder_buffer) {
  1194. const uint8_t* ptr = decoder_buffer.data();
  1195. const size_t size = decoder_buffer.data_size();
  1196. const DecryptConfig* decrypt_config = decoder_buffer.decrypt_config();
  1197. DCHECK(ptr);
  1198. DCHECK(size);
  1199. DVLOG(4) << "New input stream id: " << id << " at: " << (void*)ptr
  1200. << " size: " << size;
  1201. stream_id_ = id;
  1202. current_stream_ = ptr;
  1203. current_stream_size_ = size;
  1204. current_stream_has_been_changed_ = true;
  1205. encrypted_sei_nalus_.clear();
  1206. sei_subsamples_.clear();
  1207. if (decrypt_config) {
  1208. parser_.SetEncryptedStream(ptr, size, decrypt_config->subsamples());
  1209. current_decrypt_config_ = decrypt_config->Clone();
  1210. } else {
  1211. parser_.SetStream(ptr, size);
  1212. current_decrypt_config_ = nullptr;
  1213. }
  1214. }
  1215. H264Decoder::DecodeResult H264Decoder::Decode() {
  1216. if (state_ == State::kError) {
  1217. DVLOG(1) << "Decoder in error state";
  1218. return kDecodeError;
  1219. }
  1220. if (current_stream_has_been_changed_) {
  1221. // Calling H264Accelerator::SetStream() here instead of when the stream is
  1222. // originally set in case the accelerator needs to return kTryAgain.
  1223. H264Accelerator::Status result = accelerator_->SetStream(
  1224. base::span<const uint8_t>(current_stream_, current_stream_size_),
  1225. current_decrypt_config_.get());
  1226. switch (result) {
  1227. case H264Accelerator::Status::kOk:
  1228. case H264Accelerator::Status::kNotSupported:
  1229. // kNotSupported means the accelerator can't handle this stream,
  1230. // so everything will be done through the parser.
  1231. break;
  1232. case H264Accelerator::Status::kTryAgain:
  1233. DVLOG(1) << "SetStream() needs to try again";
  1234. return H264Decoder::kTryAgain;
  1235. case H264Accelerator::Status::kFail:
  1236. SET_ERROR_AND_RETURN();
  1237. }
  1238. // Reset the flag so that this is only called again next time SetStream()
  1239. // is called.
  1240. current_stream_has_been_changed_ = false;
  1241. }
  1242. while (true) {
  1243. H264Parser::Result par_res;
  1244. if (!curr_nalu_) {
  1245. curr_nalu_ = std::make_unique<H264NALU>();
  1246. par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
  1247. if (par_res == H264Parser::kEOStream) {
  1248. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  1249. return kRanOutOfStreamData;
  1250. } else if (par_res != H264Parser::kOk) {
  1251. SET_ERROR_AND_RETURN();
  1252. }
  1253. DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type);
  1254. }
  1255. switch (curr_nalu_->nal_unit_type) {
  1256. case H264NALU::kNonIDRSlice:
  1257. // We can't resume from a non-IDR slice unless recovery point SEI
  1258. // process is going.
  1259. if (state_ == State::kError ||
  1260. (state_ == State::kAfterReset && !recovery_frame_cnt_))
  1261. break;
  1262. [[fallthrough]];
  1263. case H264NALU::kIDRSlice: {
  1264. // TODO(posciak): the IDR may require an SPS that we don't have
  1265. // available. For now we'd fail if that happens, but ideally we'd like
  1266. // to keep going until the next SPS in the stream.
  1267. if (state_ == State::kNeedStreamMetadata) {
  1268. // We need an SPS, skip this IDR and keep looking.
  1269. break;
  1270. }
  1271. // If after reset or waiting for a key, we should be able to recover
  1272. // from an IDR. |state_|, |curr_slice_hdr_|, and |curr_pic_| are used
  1273. // to keep track of what has previously been attempted, so that after
  1274. // a retryable result is returned, subsequent calls to Decode() retry
  1275. // the call that failed previously. If it succeeds (it may not if no
  1276. // additional key has been provided, for example), then the remaining
  1277. // steps will be executed.
  1278. if (!curr_slice_hdr_) {
  1279. curr_slice_hdr_ = std::make_unique<H264SliceHeader>();
  1280. state_ = State::kParseSliceHeader;
  1281. }
  1282. if (state_ == State::kParseSliceHeader) {
  1283. // Check if the slice header is encrypted.
  1284. bool parsed_header = false;
  1285. if (current_decrypt_config_) {
  1286. const std::vector<SubsampleEntry>& subsamples =
  1287. parser_.GetCurrentSubsamples();
  1288. // There is only a single clear byte for the NALU information for
  1289. // full sample encryption, and the rest is encrypted.
  1290. if (!subsamples.empty() && subsamples[0].clear_bytes == 1) {
  1291. CHECK_ACCELERATOR_RESULT(ProcessEncryptedSliceHeader(subsamples));
  1292. parsed_header = true;
  1293. curr_slice_hdr_->pic_parameter_set_id = last_parsed_pps_id_;
  1294. encrypted_sei_nalus_.clear();
  1295. sei_subsamples_.clear();
  1296. }
  1297. }
  1298. if (!parsed_header) {
  1299. par_res =
  1300. parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get());
  1301. if (par_res != H264Parser::kOk)
  1302. SET_ERROR_AND_RETURN();
  1303. }
  1304. state_ = State::kTryPreprocessCurrentSlice;
  1305. }
  1306. if (state_ == State::kTryPreprocessCurrentSlice) {
  1307. CHECK_ACCELERATOR_RESULT(PreprocessCurrentSlice());
  1308. state_ = State::kEnsurePicture;
  1309. }
  1310. if (state_ == State::kEnsurePicture) {
  1311. if (curr_pic_) {
  1312. // |curr_pic_| already exists, so skip to ProcessCurrentSlice().
  1313. state_ = State::kTryCurrentSlice;
  1314. } else {
  1315. // New picture/finished previous one, try to start a new one
  1316. // or tell the client we need more surfaces.
  1317. curr_pic_ = accelerator_->CreateH264Picture();
  1318. if (!curr_pic_)
  1319. return kRanOutOfSurfaces;
  1320. if (current_decrypt_config_)
  1321. curr_pic_->set_decrypt_config(current_decrypt_config_->Clone());
  1322. state_ = State::kTryNewFrame;
  1323. }
  1324. }
  1325. if (state_ == State::kTryNewFrame) {
  1326. CHECK_ACCELERATOR_RESULT(StartNewFrame(curr_slice_hdr_.get()));
  1327. state_ = State::kTryCurrentSlice;
  1328. }
  1329. DCHECK_EQ(state_, State::kTryCurrentSlice);
  1330. CHECK_ACCELERATOR_RESULT(ProcessCurrentSlice());
  1331. curr_slice_hdr_.reset();
  1332. state_ = State::kDecoding;
  1333. break;
  1334. }
  1335. case H264NALU::kSPS: {
  1336. int sps_id;
  1337. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  1338. par_res = parser_.ParseSPS(&sps_id);
  1339. if (par_res != H264Parser::kOk)
  1340. SET_ERROR_AND_RETURN();
  1341. bool need_new_buffers = false;
  1342. if (!ProcessSPS(sps_id, &need_new_buffers))
  1343. SET_ERROR_AND_RETURN();
  1344. accelerator_->ProcessSPS(
  1345. parser_.GetSPS(sps_id),
  1346. base::span<const uint8_t>(curr_nalu_->data, curr_nalu_->size));
  1347. if (state_ == State::kNeedStreamMetadata)
  1348. state_ = State::kAfterReset;
  1349. if (need_new_buffers) {
  1350. curr_pic_ = nullptr;
  1351. curr_nalu_ = nullptr;
  1352. ref_pic_list_p0_.clear();
  1353. ref_pic_list_b0_.clear();
  1354. ref_pic_list_b1_.clear();
  1355. return kConfigChange;
  1356. }
  1357. break;
  1358. }
  1359. case H264NALU::kPPS: {
  1360. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  1361. par_res = parser_.ParsePPS(&last_parsed_pps_id_);
  1362. if (par_res != H264Parser::kOk)
  1363. SET_ERROR_AND_RETURN();
  1364. accelerator_->ProcessPPS(
  1365. parser_.GetPPS(last_parsed_pps_id_),
  1366. base::span<const uint8_t>(curr_nalu_->data, curr_nalu_->size));
  1367. break;
  1368. }
  1369. case H264NALU::kAUD:
  1370. case H264NALU::kEOSeq:
  1371. case H264NALU::kEOStream:
  1372. if (state_ != State::kDecoding)
  1373. break;
  1374. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  1375. break;
  1376. case H264NALU::kSEIMessage:
  1377. if (current_decrypt_config_) {
  1378. // If there are encrypted SEI NALUs as part of CENCv1, then we also
  1379. // need to save those so we can send them into the accelerator so it
  1380. // can decrypt the sample properly (otherwise it would be starting
  1381. // partway into a block).
  1382. const std::vector<SubsampleEntry>& subsamples =
  1383. parser_.GetCurrentSubsamples();
  1384. if (!subsamples.empty()) {
  1385. encrypted_sei_nalus_.emplace_back(curr_nalu_->data,
  1386. curr_nalu_->size);
  1387. DCHECK_EQ(1u, subsamples.size());
  1388. sei_subsamples_.push_back(subsamples[0]);
  1389. // Since the SEI is encrypted, do not try to parse it below as it
  1390. // may fail or yield incorrect results.
  1391. DVLOG(3) << "Skipping parsing of encrypted SEI NALU";
  1392. break;
  1393. }
  1394. }
  1395. if (state_ == State::kAfterReset && !recovery_frame_cnt_ &&
  1396. !recovery_frame_num_) {
  1397. // If we are after reset, we can also resume from a SEI recovery point
  1398. // (spec D.2.8) if one is present. However, if we are already in the
  1399. // process of handling one, skip any subsequent ones until we are done
  1400. // processing.
  1401. H264SEIMessage sei{};
  1402. if (parser_.ParseSEI(&sei) != H264Parser::kOk)
  1403. SET_ERROR_AND_RETURN();
  1404. if (sei.type == H264SEIMessage::kSEIRecoveryPoint) {
  1405. recovery_frame_cnt_ = sei.recovery_point.recovery_frame_cnt;
  1406. if (0 > *recovery_frame_cnt_ ||
  1407. *recovery_frame_cnt_ >= max_frame_num_) {
  1408. DVLOG(1) << "Invalid recovery_frame_cnt=" << *recovery_frame_cnt_
  1409. << " (it must be [0, max_frame_num_-1="
  1410. << max_frame_num_ - 1 << "])";
  1411. SET_ERROR_AND_RETURN();
  1412. }
  1413. DVLOG(3) << "Recovery point SEI is found, recovery_frame_cnt_="
  1414. << *recovery_frame_cnt_;
  1415. break;
  1416. }
  1417. }
  1418. [[fallthrough]];
  1419. default:
  1420. DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
  1421. break;
  1422. }
  1423. DVLOG(4) << "NALU done";
  1424. curr_nalu_.reset();
  1425. }
  1426. }
  1427. gfx::Size H264Decoder::GetPicSize() const {
  1428. return pic_size_;
  1429. }
  1430. gfx::Rect H264Decoder::GetVisibleRect() const {
  1431. return visible_rect_;
  1432. }
  1433. VideoCodecProfile H264Decoder::GetProfile() const {
  1434. return profile_;
  1435. }
  1436. uint8_t H264Decoder::GetBitDepth() const {
  1437. return bit_depth_;
  1438. }
  1439. VideoChromaSampling H264Decoder::GetChromaSampling() const {
  1440. return chroma_sampling_;
  1441. }
  1442. size_t H264Decoder::GetRequiredNumOfPictures() const {
  1443. constexpr size_t kPicsInPipeline = limits::kMaxVideoFrames + 1;
  1444. return GetNumReferenceFrames() + kPicsInPipeline;
  1445. }
  1446. size_t H264Decoder::GetNumReferenceFrames() const {
  1447. // Use the maximum number of pictures in the Decoded Picture Buffer.
  1448. return dpb_.max_num_pics();
  1449. }
  1450. // static
  1451. bool H264Decoder::FillH264PictureFromSliceHeader(
  1452. const H264SPS* sps,
  1453. const H264SliceHeader& slice_hdr,
  1454. H264Picture* pic) {
  1455. DCHECK(pic);
  1456. pic->idr = slice_hdr.idr_pic_flag;
  1457. if (pic->idr)
  1458. pic->idr_pic_id = slice_hdr.idr_pic_id;
  1459. if (slice_hdr.field_pic_flag) {
  1460. pic->field = slice_hdr.bottom_field_flag ? H264Picture::FIELD_BOTTOM
  1461. : H264Picture::FIELD_TOP;
  1462. } else {
  1463. pic->field = H264Picture::FIELD_NONE;
  1464. }
  1465. if (pic->field != H264Picture::FIELD_NONE) {
  1466. DVLOG(1) << "Interlaced video not supported.";
  1467. return false;
  1468. }
  1469. pic->nal_ref_idc = slice_hdr.nal_ref_idc;
  1470. pic->ref = slice_hdr.nal_ref_idc != 0;
  1471. // This assumes non-interlaced stream.
  1472. pic->frame_num = pic->pic_num = slice_hdr.frame_num;
  1473. if (!sps)
  1474. return false;
  1475. pic->pic_order_cnt_type = sps->pic_order_cnt_type;
  1476. switch (pic->pic_order_cnt_type) {
  1477. case 0:
  1478. pic->pic_order_cnt_lsb = slice_hdr.pic_order_cnt_lsb;
  1479. pic->delta_pic_order_cnt_bottom = slice_hdr.delta_pic_order_cnt_bottom;
  1480. break;
  1481. case 1:
  1482. pic->delta_pic_order_cnt0 = slice_hdr.delta_pic_order_cnt0;
  1483. pic->delta_pic_order_cnt1 = slice_hdr.delta_pic_order_cnt1;
  1484. break;
  1485. case 2:
  1486. break;
  1487. default:
  1488. NOTREACHED();
  1489. return false;
  1490. }
  1491. return true;
  1492. }
  1493. // static
  1494. bool H264Decoder::IsNewPrimaryCodedPicture(const H264Picture* curr_pic,
  1495. int curr_pps_id,
  1496. const H264SPS* sps,
  1497. const H264SliceHeader& slice_hdr) {
  1498. if (!curr_pic)
  1499. return true;
  1500. // 7.4.1.2.4, assumes non-interlaced.
  1501. if (slice_hdr.frame_num != curr_pic->frame_num ||
  1502. slice_hdr.pic_parameter_set_id != curr_pps_id ||
  1503. slice_hdr.nal_ref_idc != curr_pic->nal_ref_idc ||
  1504. slice_hdr.idr_pic_flag != curr_pic->idr ||
  1505. (slice_hdr.idr_pic_flag &&
  1506. (slice_hdr.idr_pic_id != curr_pic->idr_pic_id ||
  1507. // If we have two consecutive IDR slices, and the second one has
  1508. // first_mb_in_slice == 0, treat it as a new picture.
  1509. // Per spec, idr_pic_id should not be equal in this case (and we should
  1510. // have hit the condition above instead, see spec 7.4.3 on idr_pic_id),
  1511. // but some encoders neglect changing idr_pic_id for two consecutive
  1512. // IDRs. Work around this by checking if the next slice contains the
  1513. // zeroth macroblock, i.e. data that belongs to the next picture.
  1514. slice_hdr.first_mb_in_slice == 0)))
  1515. return true;
  1516. if (!sps)
  1517. return false;
  1518. if (sps->pic_order_cnt_type == curr_pic->pic_order_cnt_type) {
  1519. if (curr_pic->pic_order_cnt_type == 0) {
  1520. if (slice_hdr.pic_order_cnt_lsb != curr_pic->pic_order_cnt_lsb ||
  1521. slice_hdr.delta_pic_order_cnt_bottom !=
  1522. curr_pic->delta_pic_order_cnt_bottom)
  1523. return true;
  1524. } else if (curr_pic->pic_order_cnt_type == 1) {
  1525. if (slice_hdr.delta_pic_order_cnt0 != curr_pic->delta_pic_order_cnt0 ||
  1526. slice_hdr.delta_pic_order_cnt1 != curr_pic->delta_pic_order_cnt1)
  1527. return true;
  1528. }
  1529. }
  1530. return false;
  1531. }
  1532. } // namespace media