1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786 |
- // Copyright (c) 2012 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "media/gpu/h264_decoder.h"
- #include <algorithm>
- #include <limits>
- #include <memory>
- #include "base/bind.h"
- #include "base/callback_helpers.h"
- #include "base/feature_list.h"
- #include "base/logging.h"
- #include "base/metrics/histogram_functions.h"
- #include "base/numerics/safe_conversions.h"
- #include "media/base/media_switches.h"
- #include "media/video/h264_level_limits.h"
- #include "third_party/abseil-cpp/absl/types/optional.h"
- namespace media {
- namespace {
- bool ParseBitDepth(const H264SPS& sps, uint8_t& bit_depth) {
- // Spec 7.4.2.1.1
- if (sps.bit_depth_luma_minus8 != sps.bit_depth_chroma_minus8) {
- DVLOG(1) << "H264Decoder doesn't support different bit depths between luma"
- << "and chroma, bit_depth_luma_minus8="
- << sps.bit_depth_luma_minus8
- << ", bit_depth_chroma_minus8=" << sps.bit_depth_chroma_minus8;
- return false;
- }
- DCHECK_GE(sps.bit_depth_luma_minus8, 0);
- DCHECK_LE(sps.bit_depth_luma_minus8, 6);
- switch (sps.bit_depth_luma_minus8) {
- case 0:
- bit_depth = 8u;
- break;
- case 2:
- bit_depth = 10u;
- break;
- case 4:
- bit_depth = 12u;
- break;
- case 6:
- bit_depth = 14u;
- break;
- default:
- DVLOG(1) << "Invalid bit depth: "
- << base::checked_cast<int>(sps.bit_depth_luma_minus8 + 8);
- return false;
- }
- return true;
- }
- bool IsValidBitDepth(uint8_t bit_depth, VideoCodecProfile profile) {
- // Spec A.2.
- switch (profile) {
- case H264PROFILE_BASELINE:
- case H264PROFILE_MAIN:
- case H264PROFILE_EXTENDED:
- case H264PROFILE_HIGH:
- return bit_depth == 8u;
- case H264PROFILE_HIGH10PROFILE:
- case H264PROFILE_HIGH422PROFILE:
- return bit_depth == 8u || bit_depth == 10u;
- case H264PROFILE_HIGH444PREDICTIVEPROFILE:
- return bit_depth == 8u || bit_depth == 10u || bit_depth == 12u ||
- bit_depth == 14u;
- case H264PROFILE_SCALABLEBASELINE:
- case H264PROFILE_SCALABLEHIGH:
- // Spec G.10.1.
- return bit_depth == 8u;
- case H264PROFILE_STEREOHIGH:
- case H264PROFILE_MULTIVIEWHIGH:
- // Spec H.10.1.1 and H.10.1.2.
- return bit_depth == 8u;
- default:
- NOTREACHED();
- return false;
- }
- }
- } // namespace
- H264Decoder::H264Accelerator::H264Accelerator() = default;
- H264Decoder::H264Accelerator::~H264Accelerator() = default;
- void H264Decoder::H264Accelerator::ProcessSPS(
- const H264SPS* sps,
- base::span<const uint8_t> sps_nalu_data) {}
- void H264Decoder::H264Accelerator::ProcessPPS(
- const H264PPS* pps,
- base::span<const uint8_t> pps_nalu_data) {}
- H264Decoder::H264Accelerator::Status
- H264Decoder::H264Accelerator::ParseEncryptedSliceHeader(
- const std::vector<base::span<const uint8_t>>& data,
- const std::vector<SubsampleEntry>& subsamples,
- H264SliceHeader* slice_header_out) {
- return H264Decoder::H264Accelerator::Status::kNotSupported;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::H264Accelerator::SetStream(
- base::span<const uint8_t> stream,
- const DecryptConfig* decrypt_config) {
- return H264Decoder::H264Accelerator::Status::kNotSupported;
- }
- H264Decoder::H264Decoder(std::unique_ptr<H264Accelerator> accelerator,
- VideoCodecProfile profile,
- const VideoColorSpace& container_color_space)
- : state_(State::kNeedStreamMetadata),
- container_color_space_(container_color_space),
- max_frame_num_(0),
- max_pic_num_(0),
- max_long_term_frame_idx_(0),
- max_num_reorder_frames_(0),
- // TODO(hiroh): Set profile to UNKNOWN.
- profile_(profile),
- accelerator_(std::move(accelerator)) {
- DCHECK(accelerator_);
- Reset();
- }
- H264Decoder::~H264Decoder() = default;
- void H264Decoder::Reset() {
- curr_pic_ = nullptr;
- curr_nalu_ = nullptr;
- curr_slice_hdr_ = nullptr;
- curr_sps_id_ = -1;
- curr_pps_id_ = -1;
- prev_frame_num_ = -1;
- prev_ref_frame_num_ = -1;
- prev_frame_num_offset_ = -1;
- prev_has_memmgmnt5_ = false;
- prev_ref_has_memmgmnt5_ = false;
- prev_ref_top_field_order_cnt_ = -1;
- prev_ref_pic_order_cnt_msb_ = -1;
- prev_ref_pic_order_cnt_lsb_ = -1;
- prev_ref_field_ = H264Picture::FIELD_NONE;
- ref_pic_list_p0_.clear();
- ref_pic_list_b0_.clear();
- ref_pic_list_b1_.clear();
- dpb_.Clear();
- parser_.Reset();
- accelerator_->Reset();
- last_output_poc_ = std::numeric_limits<int>::min();
- encrypted_sei_nalus_.clear();
- sei_subsamples_.clear();
- recovery_frame_num_.reset();
- recovery_frame_cnt_.reset();
- // If we are in kDecoding, we can resume without processing an SPS.
- // The state becomes kDecoding again, (1) at the first IDR slice or (2) at
- // the first slice after the recovery point SEI.
- if (state_ == State::kDecoding)
- state_ = State::kAfterReset;
- }
- void H264Decoder::PrepareRefPicLists() {
- ConstructReferencePicListsP();
- ConstructReferencePicListsB();
- }
- bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr,
- H264Picture::Vector* ref_pic_list0,
- H264Picture::Vector* ref_pic_list1) {
- ref_pic_list0->clear();
- ref_pic_list1->clear();
- // Fill reference picture lists for B and S/SP slices.
- if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) {
- *ref_pic_list0 = ref_pic_list_p0_;
- return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0);
- } else if (slice_hdr->IsBSlice()) {
- *ref_pic_list0 = ref_pic_list_b0_;
- *ref_pic_list1 = ref_pic_list_b1_;
- return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) &&
- ModifyReferencePicList(slice_hdr, 1, ref_pic_list1);
- }
- return true;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::DecodePicture() {
- DCHECK(curr_pic_.get());
- return accelerator_->SubmitDecode(curr_pic_);
- }
- bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic,
- int frame_num) {
- pic->nonexisting = true;
- pic->nal_ref_idc = 1;
- pic->frame_num = pic->pic_num = frame_num;
- pic->adaptive_ref_pic_marking_mode_flag = false;
- pic->ref = true;
- pic->long_term_reference_flag = false;
- pic->field = H264Picture::FIELD_NONE;
- return CalculatePicOrderCounts(pic);
- }
- bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) {
- if (!FillH264PictureFromSliceHeader(parser_.GetSPS(curr_sps_id_), *slice_hdr,
- curr_pic_.get())) {
- return false;
- }
- if (!CalculatePicOrderCounts(curr_pic_))
- return false;
- curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag;
- curr_pic_->adaptive_ref_pic_marking_mode_flag =
- slice_hdr->adaptive_ref_pic_marking_mode_flag;
- // If the slice header indicates we will have to perform reference marking
- // process after this picture is decoded, store required data for that
- // purpose.
- if (slice_hdr->adaptive_ref_pic_marking_mode_flag) {
- static_assert(sizeof(curr_pic_->ref_pic_marking) ==
- sizeof(slice_hdr->ref_pic_marking),
- "Array sizes of ref pic marking do not match.");
- memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking,
- sizeof(curr_pic_->ref_pic_marking));
- }
- curr_pic_->set_visible_rect(visible_rect_);
- curr_pic_->set_bitstream_id(stream_id_);
- return true;
- }
- bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) {
- const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
- if (!sps)
- return false;
- switch (pic->pic_order_cnt_type) {
- case 0: {
- // See spec 8.2.1.1.
- int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb;
- if (pic->idr) {
- prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0;
- } else {
- if (prev_ref_has_memmgmnt5_) {
- if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) {
- prev_pic_order_cnt_msb = 0;
- prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_;
- } else {
- prev_pic_order_cnt_msb = 0;
- prev_pic_order_cnt_lsb = 0;
- }
- } else {
- prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_;
- prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_;
- }
- }
- int max_pic_order_cnt_lsb =
- 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
- DCHECK_NE(max_pic_order_cnt_lsb, 0);
- if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
- (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >=
- max_pic_order_cnt_lsb / 2)) {
- pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb;
- } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) &&
- (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb >
- max_pic_order_cnt_lsb / 2)) {
- pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb;
- } else {
- pic->pic_order_cnt_msb = prev_pic_order_cnt_msb;
- }
- if (pic->field != H264Picture::FIELD_BOTTOM) {
- pic->top_field_order_cnt =
- pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
- }
- if (pic->field != H264Picture::FIELD_TOP) {
- if (pic->field == H264Picture::FIELD_NONE) {
- pic->bottom_field_order_cnt =
- pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom;
- } else {
- pic->bottom_field_order_cnt =
- pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb;
- }
- }
- break;
- }
- case 1: {
- // See spec 8.2.1.2.
- if (prev_has_memmgmnt5_)
- prev_frame_num_offset_ = 0;
- if (pic->idr)
- pic->frame_num_offset = 0;
- else if (prev_frame_num_ > pic->frame_num)
- pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
- else
- pic->frame_num_offset = prev_frame_num_offset_;
- int abs_frame_num = 0;
- if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
- abs_frame_num = pic->frame_num_offset + pic->frame_num;
- else
- abs_frame_num = 0;
- if (pic->nal_ref_idc == 0 && abs_frame_num > 0)
- --abs_frame_num;
- int expected_pic_order_cnt = 0;
- if (abs_frame_num > 0) {
- if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) {
- DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle "
- << "in stream";
- return false;
- }
- int pic_order_cnt_cycle_cnt =
- (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle;
- int frame_num_in_pic_order_cnt_cycle =
- (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle;
- expected_pic_order_cnt = pic_order_cnt_cycle_cnt *
- sps->expected_delta_per_pic_order_cnt_cycle;
- // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser
- for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i)
- expected_pic_order_cnt += sps->offset_for_ref_frame[i];
- }
- if (!pic->nal_ref_idc)
- expected_pic_order_cnt += sps->offset_for_non_ref_pic;
- if (pic->field == H264Picture::FIELD_NONE) {
- pic->top_field_order_cnt =
- expected_pic_order_cnt + pic->delta_pic_order_cnt0;
- pic->bottom_field_order_cnt = pic->top_field_order_cnt +
- sps->offset_for_top_to_bottom_field +
- pic->delta_pic_order_cnt1;
- } else if (pic->field != H264Picture::FIELD_BOTTOM) {
- pic->top_field_order_cnt =
- expected_pic_order_cnt + pic->delta_pic_order_cnt0;
- } else {
- pic->bottom_field_order_cnt = expected_pic_order_cnt +
- sps->offset_for_top_to_bottom_field +
- pic->delta_pic_order_cnt0;
- }
- break;
- }
- case 2: {
- // See spec 8.2.1.3.
- if (prev_has_memmgmnt5_)
- prev_frame_num_offset_ = 0;
- if (pic->idr)
- pic->frame_num_offset = 0;
- else if (prev_frame_num_ > pic->frame_num)
- pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_;
- else
- pic->frame_num_offset = prev_frame_num_offset_;
- int temp_pic_order_cnt;
- if (pic->idr) {
- temp_pic_order_cnt = 0;
- } else if (!pic->nal_ref_idc) {
- temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1;
- } else {
- temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num);
- }
- if (pic->field == H264Picture::FIELD_NONE) {
- pic->top_field_order_cnt = temp_pic_order_cnt;
- pic->bottom_field_order_cnt = temp_pic_order_cnt;
- } else if (pic->field == H264Picture::FIELD_BOTTOM) {
- pic->bottom_field_order_cnt = temp_pic_order_cnt;
- } else {
- pic->top_field_order_cnt = temp_pic_order_cnt;
- }
- break;
- }
- default:
- DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type;
- return false;
- }
- switch (pic->field) {
- case H264Picture::FIELD_NONE:
- pic->pic_order_cnt =
- std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt);
- break;
- case H264Picture::FIELD_TOP:
- pic->pic_order_cnt = pic->top_field_order_cnt;
- break;
- case H264Picture::FIELD_BOTTOM:
- pic->pic_order_cnt = pic->bottom_field_order_cnt;
- break;
- }
- return true;
- }
- void H264Decoder::UpdatePicNums(int frame_num) {
- for (auto& pic : dpb_) {
- if (!pic->ref)
- continue;
- // 8.2.4.1. Assumes non-interlaced stream.
- DCHECK_EQ(pic->field, H264Picture::FIELD_NONE);
- if (pic->long_term) {
- pic->long_term_pic_num = pic->long_term_frame_idx;
- } else {
- if (pic->frame_num > frame_num)
- pic->frame_num_wrap = pic->frame_num - max_frame_num_;
- else
- pic->frame_num_wrap = pic->frame_num;
- pic->pic_num = pic->frame_num_wrap;
- }
- }
- }
- struct PicNumDescCompare {
- bool operator()(const scoped_refptr<H264Picture>& a,
- const scoped_refptr<H264Picture>& b) const {
- return a->pic_num > b->pic_num;
- }
- };
- struct LongTermPicNumAscCompare {
- bool operator()(const scoped_refptr<H264Picture>& a,
- const scoped_refptr<H264Picture>& b) const {
- return a->long_term_pic_num < b->long_term_pic_num;
- }
- };
- void H264Decoder::ConstructReferencePicListsP() {
- // RefPicList0 (8.2.4.2.1) [[1] [2]], where:
- // [1] shortterm ref pics sorted by descending pic_num,
- // [2] longterm ref pics by ascending long_term_pic_num.
- ref_pic_list_p0_.clear();
- // First get the short ref pics...
- dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_);
- size_t num_short_refs = ref_pic_list_p0_.size();
- // and sort them to get [1].
- std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(),
- PicNumDescCompare());
- // Now get long term pics and sort them by long_term_pic_num to get [2].
- dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_);
- std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(),
- LongTermPicNumAscCompare());
- }
- struct POCAscCompare {
- bool operator()(const scoped_refptr<H264Picture>& a,
- const scoped_refptr<H264Picture>& b) const {
- return a->pic_order_cnt < b->pic_order_cnt;
- }
- };
- struct POCDescCompare {
- bool operator()(const scoped_refptr<H264Picture>& a,
- const scoped_refptr<H264Picture>& b) const {
- return a->pic_order_cnt > b->pic_order_cnt;
- }
- };
- void H264Decoder::ConstructReferencePicListsB() {
- // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where:
- // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC,
- // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC,
- // [3] longterm ref pics by ascending long_term_pic_num.
- ref_pic_list_b0_.clear();
- ref_pic_list_b1_.clear();
- dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_);
- size_t num_short_refs = ref_pic_list_b0_.size();
- // First sort ascending, this will put [1] in right place and finish [2].
- std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare());
- // Find first with POC > curr_pic's POC to get first element in [2]...
- H264Picture::Vector::iterator iter;
- iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
- curr_pic_.get(), POCAscCompare());
- // and sort [1] descending, thus finishing sequence [1] [2].
- std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare());
- // Now add [3] and sort by ascending long_term_pic_num.
- dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_);
- std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(),
- LongTermPicNumAscCompare());
- // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where:
- // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
- // [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
- // [3] longterm ref pics by ascending long_term_pic_num.
- dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_);
- num_short_refs = ref_pic_list_b1_.size();
- // First sort by descending POC.
- std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare());
- // Find first with POC < curr_pic's POC to get first element in [2]...
- iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(),
- curr_pic_.get(), POCDescCompare());
- // and sort [1] ascending.
- std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare());
- // Now add [3] and sort by ascending long_term_pic_num
- dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b1_);
- std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(),
- LongTermPicNumAscCompare());
- // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3)
- if (ref_pic_list_b1_.size() > 1 &&
- std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(),
- ref_pic_list_b1_.begin()))
- std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]);
- }
- // See 8.2.4
- int H264Decoder::PicNumF(const H264Picture& pic) {
- if (!pic.long_term)
- return pic.pic_num;
- else
- return max_pic_num_;
- }
- // See 8.2.4
- int H264Decoder::LongTermPicNumF(const H264Picture& pic) {
- if (pic.ref && pic.long_term)
- return pic.long_term_pic_num;
- else
- return 2 * (max_long_term_frame_idx_ + 1);
- }
- // Shift elements on the |v| starting from |from| to |to|, inclusive,
- // one position to the right and insert pic at |from|.
- static void ShiftRightAndInsert(H264Picture::Vector* v,
- int from,
- int to,
- scoped_refptr<H264Picture> pic) {
- // Security checks, do not disable in Debug mode.
- CHECK(from <= to);
- CHECK(to <= std::numeric_limits<int>::max() - 2);
- // Additional checks. Debug mode ok.
- DCHECK(v);
- DCHECK(pic);
- DCHECK((to + 1 == static_cast<int>(v->size())) ||
- (to + 2 == static_cast<int>(v->size())));
- v->resize(to + 2);
- for (int i = to + 1; i > from; --i)
- (*v)[i] = (*v)[i - 1];
- (*v)[from] = std::move(pic);
- }
- bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr,
- int list,
- H264Picture::Vector* ref_pic_listx) {
- bool ref_pic_list_modification_flag_lX;
- int num_ref_idx_lX_active_minus1;
- const H264ModificationOfPicNum* list_mod;
- // This can process either ref_pic_list0 or ref_pic_list1, depending on
- // the list argument. Set up pointers to proper list to be processed here.
- if (list == 0) {
- ref_pic_list_modification_flag_lX =
- slice_hdr->ref_pic_list_modification_flag_l0;
- num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1;
- list_mod = slice_hdr->ref_list_l0_modifications;
- } else {
- ref_pic_list_modification_flag_lX =
- slice_hdr->ref_pic_list_modification_flag_l1;
- num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1;
- list_mod = slice_hdr->ref_list_l1_modifications;
- }
- // Resize the list to the size requested in the slice header.
- // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to
- // indicate there should be more ref pics on list than we constructed.
- // Those superfluous ones should be treated as non-reference and will be
- // initialized to nullptr, which must be handled by clients.
- DCHECK_GE(num_ref_idx_lX_active_minus1, 0);
- ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
- if (!ref_pic_list_modification_flag_lX)
- return true;
- // Spec 8.2.4.3:
- // Reorder pictures on the list in a way specified in the stream.
- int pic_num_lx_pred = curr_pic_->pic_num;
- int ref_idx_lx = 0;
- int pic_num_lx_no_wrap;
- int pic_num_lx;
- bool done = false;
- scoped_refptr<H264Picture> pic;
- for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) {
- switch (list_mod->modification_of_pic_nums_idc) {
- case 0:
- case 1:
- // Modify short reference picture position.
- if (list_mod->modification_of_pic_nums_idc == 0) {
- // Subtract given value from predicted PicNum.
- pic_num_lx_no_wrap =
- pic_num_lx_pred -
- (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
- // Wrap around max_pic_num_ if it becomes < 0 as result
- // of subtraction.
- if (pic_num_lx_no_wrap < 0)
- pic_num_lx_no_wrap += max_pic_num_;
- } else {
- // Add given value to predicted PicNum.
- pic_num_lx_no_wrap =
- pic_num_lx_pred +
- (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1);
- // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result
- // of the addition.
- if (pic_num_lx_no_wrap >= max_pic_num_)
- pic_num_lx_no_wrap -= max_pic_num_;
- }
- // For use in next iteration.
- pic_num_lx_pred = pic_num_lx_no_wrap;
- if (pic_num_lx_no_wrap > curr_pic_->pic_num)
- pic_num_lx = pic_num_lx_no_wrap - max_pic_num_;
- else
- pic_num_lx = pic_num_lx_no_wrap;
- DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
- H264SliceHeader::kRefListModSize);
- pic = dpb_.GetShortRefPicByPicNum(pic_num_lx);
- if (!pic) {
- DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx;
- return false;
- }
- if (ref_idx_lx > num_ref_idx_lX_active_minus1) {
- DVLOG(1) << "Bounds mismatch: expected " << ref_idx_lx
- << " <= " << num_ref_idx_lX_active_minus1;
- return false;
- }
- ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
- num_ref_idx_lX_active_minus1, pic);
- ref_idx_lx++;
- for (int src = ref_idx_lx, dst = ref_idx_lx;
- src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
- auto* src_pic = (*ref_pic_listx)[src].get();
- int src_pic_num_lx = src_pic ? PicNumF(*src_pic) : -1;
- if (src_pic_num_lx != pic_num_lx)
- (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
- }
- break;
- case 2:
- // Modify long term reference picture position.
- DCHECK_LT(num_ref_idx_lX_active_minus1 + 1,
- H264SliceHeader::kRefListModSize);
- pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num);
- if (!pic) {
- DVLOG(1) << "Malformed stream, no pic num "
- << list_mod->long_term_pic_num;
- return false;
- }
- ShiftRightAndInsert(ref_pic_listx, ref_idx_lx,
- num_ref_idx_lX_active_minus1, pic);
- ref_idx_lx++;
- for (int src = ref_idx_lx, dst = ref_idx_lx;
- src <= num_ref_idx_lX_active_minus1 + 1; ++src) {
- if (LongTermPicNumF(*(*ref_pic_listx)[src]) !=
- static_cast<int>(list_mod->long_term_pic_num))
- (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src];
- }
- break;
- case 3:
- // End of modification list.
- done = true;
- break;
- default:
- // May be recoverable.
- DVLOG(1) << "Invalid modification_of_pic_nums_idc="
- << list_mod->modification_of_pic_nums_idc << " in position "
- << i;
- break;
- }
- ++list_mod;
- }
- // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is
- // temporarily made one element longer than the required final list.
- // Resize the list back to its required size.
- ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1);
- return true;
- }
- bool H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) {
- DCHECK(!pic->outputted);
- pic->outputted = true;
- VideoColorSpace colorspace_for_frame = container_color_space_;
- const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
- if (sps && sps->GetColorSpace().IsSpecified())
- colorspace_for_frame = sps->GetColorSpace();
- pic->set_colorspace(colorspace_for_frame);
- if (pic->nonexisting) {
- DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num;
- return true;
- }
- DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_)
- << "Outputting out of order, likely a broken stream: " << last_output_poc_
- << " -> " << pic->pic_order_cnt;
- last_output_poc_ = pic->pic_order_cnt;
- DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt;
- return accelerator_->OutputPicture(pic);
- }
- void H264Decoder::ClearDPB() {
- // Clear DPB contents, marking the pictures as unused first.
- dpb_.Clear();
- last_output_poc_ = std::numeric_limits<int>::min();
- }
- bool H264Decoder::OutputAllRemainingPics() {
- // Output all pictures that are waiting to be outputted.
- if (FinishPrevFrameIfPresent() != H264Accelerator::Status::kOk)
- return false;
- H264Picture::Vector to_output;
- dpb_.GetNotOutputtedPicsAppending(&to_output);
- // Sort them by ascending POC to output in order.
- std::sort(to_output.begin(), to_output.end(), POCAscCompare());
- for (auto& pic : to_output) {
- if (!OutputPic(pic))
- return false;
- }
- return true;
- }
- bool H264Decoder::Flush() {
- DVLOG(2) << "Decoder flush";
- if (!OutputAllRemainingPics())
- return false;
- ClearDPB();
- DVLOG(2) << "Decoder flush finished";
- return true;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::StartNewFrame(
- const H264SliceHeader* slice_hdr) {
- // TODO posciak: add handling of max_num_ref_frames per spec.
- CHECK(curr_pic_.get());
- DCHECK(slice_hdr);
- curr_pps_id_ = slice_hdr->pic_parameter_set_id;
- const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
- if (!pps)
- return H264Accelerator::Status::kFail;
- curr_sps_id_ = pps->seq_parameter_set_id;
- const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
- if (!sps)
- return H264Accelerator::Status::kFail;
- max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4);
- int frame_num = slice_hdr->frame_num;
- if (slice_hdr->idr_pic_flag)
- prev_ref_frame_num_ = 0;
- // 7.4.3
- if (frame_num != prev_ref_frame_num_ &&
- frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) {
- if (!HandleFrameNumGap(frame_num))
- return H264Accelerator::Status::kFail;
- }
- if (!InitCurrPicture(slice_hdr))
- return H264Accelerator::Status::kFail;
- UpdatePicNums(frame_num);
- PrepareRefPicLists();
- return accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_,
- ref_pic_list_b0_, ref_pic_list_b1_,
- curr_pic_.get());
- }
- bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) {
- // 8.2.5.4
- for (size_t i = 0; i < std::size(pic->ref_pic_marking); ++i) {
- // Code below does not support interlaced stream (per-field pictures).
- H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i];
- scoped_refptr<H264Picture> to_mark;
- int pic_num_x;
- switch (ref_pic_marking->memory_mgmnt_control_operation) {
- case 0:
- // Normal end of operations' specification.
- return true;
- case 1:
- // Mark a short term reference picture as unused so it can be removed
- // if outputted.
- pic_num_x =
- pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
- to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
- if (to_mark) {
- to_mark->ref = false;
- } else {
- DVLOG(1) << "Invalid short ref pic num to unmark";
- return false;
- }
- break;
- case 2:
- // Mark a long term reference picture as unused so it can be removed
- // if outputted.
- to_mark = dpb_.GetLongRefPicByLongTermPicNum(
- ref_pic_marking->long_term_pic_num);
- if (to_mark) {
- to_mark->ref = false;
- } else {
- DVLOG(1) << "Invalid long term ref pic num to unmark";
- return false;
- }
- break;
- case 3:
- // Mark a short term reference picture as long term reference.
- pic_num_x =
- pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1);
- to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x);
- if (to_mark) {
- DCHECK(to_mark->ref && !to_mark->long_term);
- to_mark->long_term = true;
- to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
- } else {
- DVLOG(1) << "Invalid short term ref pic num to mark as long ref";
- return false;
- }
- break;
- case 4: {
- // Unmark all reference pictures with long_term_frame_idx over new max.
- max_long_term_frame_idx_ =
- ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
- H264Picture::Vector long_terms;
- dpb_.GetLongTermRefPicsAppending(&long_terms);
- for (size_t long_term = 0; long_term < long_terms.size(); ++long_term) {
- scoped_refptr<H264Picture>& long_term_pic = long_terms[long_term];
- DCHECK(long_term_pic->ref && long_term_pic->long_term);
- // Ok to cast, max_long_term_frame_idx is much smaller than 16bit.
- if (long_term_pic->long_term_frame_idx >
- static_cast<int>(max_long_term_frame_idx_))
- long_term_pic->ref = false;
- }
- break;
- }
- case 5:
- // Unmark all reference pictures.
- dpb_.MarkAllUnusedForRef();
- max_long_term_frame_idx_ = -1;
- pic->mem_mgmt_5 = true;
- break;
- case 6: {
- // Replace long term reference pictures with current picture.
- // First unmark if any existing with this long_term_frame_idx...
- H264Picture::Vector long_terms;
- dpb_.GetLongTermRefPicsAppending(&long_terms);
- for (size_t long_term = 0; long_term < long_terms.size(); ++long_term) {
- scoped_refptr<H264Picture>& long_term_pic = long_terms[long_term];
- DCHECK(long_term_pic->ref && long_term_pic->long_term);
- // Ok to cast, long_term_frame_idx is much smaller than 16bit.
- if (long_term_pic->long_term_frame_idx ==
- static_cast<int>(ref_pic_marking->long_term_frame_idx))
- long_term_pic->ref = false;
- }
- // and mark the current one instead.
- pic->ref = true;
- pic->long_term = true;
- pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
- break;
- }
- default:
- // Would indicate a bug in parser.
- NOTREACHED();
- }
- }
- return true;
- }
- // This method ensures that DPB does not overflow, either by removing
- // reference pictures as specified in the stream, or using a sliding window
- // procedure to remove the oldest one.
- // It also performs marking and unmarking pictures as reference.
- // See spac 8.2.5.1.
- bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) {
- // If the current picture is an IDR, all reference pictures are unmarked.
- if (pic->idr) {
- dpb_.MarkAllUnusedForRef();
- if (pic->long_term_reference_flag) {
- pic->long_term = true;
- pic->long_term_frame_idx = 0;
- max_long_term_frame_idx_ = 0;
- } else {
- pic->long_term = false;
- max_long_term_frame_idx_ = -1;
- }
- return true;
- }
- // Not an IDR. If the stream contains instructions on how to discard pictures
- // from DPB and how to mark/unmark existing reference pictures, do so.
- // Otherwise, fall back to default sliding window process.
- if (pic->adaptive_ref_pic_marking_mode_flag) {
- DCHECK(!pic->nonexisting);
- return HandleMemoryManagementOps(pic);
- } else {
- return SlidingWindowPictureMarking();
- }
- }
- bool H264Decoder::SlidingWindowPictureMarking() {
- const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
- if (!sps)
- return false;
- // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture.
- int num_ref_pics = dpb_.CountRefPics();
- DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1));
- if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) {
- // Max number of reference pics reached, need to remove one of the short
- // term ones. Find smallest frame_num_wrap short reference picture and mark
- // it as unused.
- scoped_refptr<H264Picture> to_unmark =
- dpb_.GetLowestFrameNumWrapShortRefPic();
- if (!to_unmark) {
- DVLOG(1) << "Couldn't find a short ref picture to unmark";
- return false;
- }
- to_unmark->ref = false;
- }
- return true;
- }
- bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) {
- // Finish processing the picture.
- // Start by storing previous picture data for later use.
- if (pic->ref) {
- ReferencePictureMarking(pic);
- prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5;
- prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt;
- prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb;
- prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb;
- prev_ref_field_ = pic->field;
- prev_ref_frame_num_ = pic->frame_num;
- }
- prev_frame_num_ = pic->frame_num;
- prev_has_memmgmnt5_ = pic->mem_mgmt_5;
- prev_frame_num_offset_ = pic->frame_num_offset;
- // Remove unused (for reference or later output) pictures from DPB, marking
- // them as such.
- dpb_.DeleteUnused();
- DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num
- << ", entries in DPB: " << dpb_.size();
- if (recovery_frame_cnt_) {
- // This is the first picture after the recovery point SEI message. Computes
- // the frame_num of the frame that should be output from (Spec D.2.8).
- recovery_frame_num_ =
- (*recovery_frame_cnt_ + pic->frame_num) % max_frame_num_;
- DVLOG(3) << "recovery_frame_num_" << *recovery_frame_num_;
- recovery_frame_cnt_.reset();
- }
- // The ownership of pic will either be transferred to DPB - if the picture is
- // still needed (for output and/or reference) - or we will release it
- // immediately if we manage to output it here and won't have to store it for
- // future reference.
- // Get all pictures that haven't been outputted yet.
- H264Picture::Vector not_outputted;
- dpb_.GetNotOutputtedPicsAppending(¬_outputted);
- // Include the one we've just decoded.
- not_outputted.push_back(pic);
- // Sort in output order.
- std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
- // Try to output as many pictures as we can. A picture can be output,
- // if the number of decoded and not yet outputted pictures that would remain
- // in DPB afterwards would at least be equal to max_num_reorder_frames.
- // If the outputted picture is not a reference picture, it doesn't have
- // to remain in the DPB and can be removed.
- auto output_candidate = not_outputted.begin();
- size_t num_remaining = not_outputted.size();
- while (num_remaining > max_num_reorder_frames_ ||
- // If the condition below is used, this is an invalid stream. We should
- // not be forced to output beyond max_num_reorder_frames in order to
- // make room in DPB to store the current picture (if we need to do so).
- // However, if this happens, ignore max_num_reorder_frames and try
- // to output more. This may cause out-of-order output, but is not
- // fatal, and better than failing instead.
- ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) {
- DVLOG_IF(1, num_remaining <= max_num_reorder_frames_)
- << "Invalid stream: max_num_reorder_frames not preserved";
- if (!recovery_frame_num_ ||
- // If we are decoding ahead to reach a SEI recovery point, skip
- // outputting all pictures before it, to avoid outputting corrupted
- // frames.
- (*output_candidate)->frame_num == *recovery_frame_num_) {
- recovery_frame_num_ = absl::nullopt;
- if (!OutputPic(*output_candidate))
- return false;
- }
- if (!(*output_candidate)->ref) {
- // Current picture hasn't been inserted into DPB yet, so don't remove it
- // if we managed to output it immediately.
- if (*output_candidate != pic)
- dpb_.Delete(*output_candidate);
- }
- ++output_candidate;
- --num_remaining;
- }
- // If we haven't managed to output the picture that we just decoded, or if
- // it's a reference picture, we have to store it in DPB.
- if (!pic->outputted || pic->ref) {
- if (dpb_.IsFull()) {
- // If we haven't managed to output anything to free up space in DPB
- // to store this picture, it's an error in the stream.
- DVLOG(1) << "Could not free up space in DPB!";
- return false;
- }
- dpb_.StorePic(std::move(pic));
- }
- return true;
- }
- bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) {
- if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) {
- max_num_reorder_frames_ =
- base::checked_cast<size_t>(sps->max_num_reorder_frames);
- if (max_num_reorder_frames_ > dpb_.max_num_pics()) {
- DVLOG(1)
- << "max_num_reorder_frames present, but larger than MaxDpbFrames ("
- << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")";
- max_num_reorder_frames_ = 0;
- return false;
- }
- return true;
- }
- // max_num_reorder_frames not present, infer from profile/constraints
- // (see VUI semantics in spec).
- if (sps->constraint_set3_flag) {
- switch (sps->profile_idc) {
- case 44:
- case 86:
- case 100:
- case 110:
- case 122:
- case 244:
- max_num_reorder_frames_ = 0;
- break;
- default:
- max_num_reorder_frames_ = dpb_.max_num_pics();
- break;
- }
- } else {
- max_num_reorder_frames_ = dpb_.max_num_pics();
- }
- return true;
- }
- bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) {
- DVLOG(4) << "Processing SPS id:" << sps_id;
- const H264SPS* sps = parser_.GetSPS(sps_id);
- if (!sps)
- return false;
- *need_new_buffers = false;
- if (sps->frame_mbs_only_flag == 0) {
- DVLOG(1) << "frame_mbs_only_flag != 1 not supported";
- return false;
- }
- gfx::Size new_pic_size = sps->GetCodedSize().value_or(gfx::Size());
- if (new_pic_size.IsEmpty()) {
- DVLOG(1) << "Invalid picture size";
- return false;
- }
- int width_mb = new_pic_size.width() / 16;
- int height_mb = new_pic_size.height() / 16;
- // Verify that the values are not too large before multiplying.
- if (std::numeric_limits<int>::max() / width_mb < height_mb) {
- DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString();
- return false;
- }
- // Spec A.3.1 and A.3.2
- // For Baseline, Constrained Baseline and Main profile, the indicated level is
- // Level 1b if level_idc is equal to 11 and constraint_set3_flag is equal to 1
- uint8_t level = base::checked_cast<uint8_t>(sps->level_idc);
- if ((sps->profile_idc == H264SPS::kProfileIDCBaseline ||
- sps->profile_idc == H264SPS::kProfileIDCConstrainedBaseline ||
- sps->profile_idc == H264SPS::kProfileIDCMain) &&
- level == 11 && sps->constraint_set3_flag) {
- level = 9; // Level 1b
- }
- int max_dpb_mbs = base::checked_cast<int>(H264LevelToMaxDpbMbs(level));
- if (max_dpb_mbs == 0)
- return false;
- // MaxDpbFrames from level limits per spec.
- size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb),
- static_cast<int>(H264DPB::kDPBMaxSize));
- DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames
- << ", max_num_ref_frames: " << sps->max_num_ref_frames
- << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering;
- // Set DPB size to at least the level limit, or what the stream requires.
- size_t max_dpb_size =
- std::max(static_cast<int>(max_dpb_frames),
- std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering));
- // Some non-conforming streams specify more frames are needed than the current
- // level limit. Allow this, but only up to the maximum number of reference
- // frames allowed per spec.
- DVLOG_IF(1, max_dpb_size > max_dpb_frames)
- << "Invalid stream, DPB size > MaxDpbFrames";
- if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) {
- DVLOG(1) << "Invalid DPB size: " << max_dpb_size;
- return false;
- }
- VideoChromaSampling new_chroma_sampling = sps->GetChromaSampling();
- if (new_chroma_sampling != chroma_sampling_) {
- chroma_sampling_ = new_chroma_sampling;
- base::UmaHistogramEnumeration("Media.PlatformVideoDecoding.ChromaSampling",
- chroma_sampling_);
- }
- if (chroma_sampling_ != VideoChromaSampling::k420) {
- DVLOG(1) << "Only YUV 4:2:0 is supported";
- return false;
- }
- VideoCodecProfile new_profile =
- H264Parser::ProfileIDCToVideoCodecProfile(sps->profile_idc);
- uint8_t new_bit_depth = 0;
- if (!ParseBitDepth(*sps, new_bit_depth))
- return false;
- if (!IsValidBitDepth(new_bit_depth, new_profile)) {
- DVLOG(1) << "Invalid bit depth=" << base::strict_cast<int>(new_bit_depth)
- << ", profile=" << GetProfileName(new_profile);
- return false;
- }
- if (pic_size_ != new_pic_size || dpb_.max_num_pics() != max_dpb_size ||
- profile_ != new_profile || bit_depth_ != new_bit_depth) {
- if (!Flush())
- return false;
- DVLOG(1) << "Codec profile: " << GetProfileName(new_profile)
- << ", level: " << base::strict_cast<int>(level)
- << ", DPB size: " << max_dpb_size
- << ", Picture size: " << new_pic_size.ToString()
- << ", bit depth: " << base::strict_cast<int>(new_bit_depth);
- *need_new_buffers = true;
- profile_ = new_profile;
- bit_depth_ = new_bit_depth;
- pic_size_ = new_pic_size;
- dpb_.set_max_num_pics(max_dpb_size);
- }
- gfx::Rect new_visible_rect = sps->GetVisibleRect().value_or(gfx::Rect());
- if (visible_rect_ != new_visible_rect) {
- DVLOG(2) << "New visible rect: " << new_visible_rect.ToString();
- visible_rect_ = new_visible_rect;
- }
- if (!UpdateMaxNumReorderFrames(sps))
- return false;
- DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_;
- return true;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::FinishPrevFrameIfPresent() {
- // If we already have a frame waiting to be decoded, decode it and finish.
- if (curr_pic_) {
- H264Accelerator::Status result = DecodePicture();
- if (result != H264Accelerator::Status::kOk)
- return result;
- scoped_refptr<H264Picture> pic = curr_pic_;
- curr_pic_ = nullptr;
- if (!FinishPicture(pic))
- return H264Accelerator::Status::kFail;
- }
- return H264Accelerator::Status::kOk;
- }
- bool H264Decoder::HandleFrameNumGap(int frame_num) {
- const H264SPS* sps = parser_.GetSPS(curr_sps_id_);
- if (!sps)
- return false;
- if (!sps->gaps_in_frame_num_value_allowed_flag) {
- DVLOG(1) << "Invalid frame_num: " << frame_num;
- // TODO(b:129119729, b:146914440): Youtube android app sometimes sends an
- // invalid frame number after a seek. The sequence goes like:
- // Seek, SPS, PPS, IDR-frame, non-IDR, ... non-IDR with invalid number.
- // The only way to work around this reliably is to ignore this error.
- // Video playback is not affected, no artefacts are visible.
- // return false;
- }
- DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->"
- << frame_num;
- // 7.4.3/7-23
- int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_;
- while (unused_short_term_frame_num != frame_num) {
- scoped_refptr<H264Picture> pic = new H264Picture();
- if (!InitNonexistingPicture(pic, unused_short_term_frame_num))
- return false;
- UpdatePicNums(unused_short_term_frame_num);
- if (!FinishPicture(pic))
- return false;
- unused_short_term_frame_num++;
- unused_short_term_frame_num %= max_frame_num_;
- }
- return true;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::ProcessEncryptedSliceHeader(
- const std::vector<SubsampleEntry>& subsamples) {
- DCHECK(curr_nalu_);
- DCHECK(curr_slice_hdr_);
- std::vector<base::span<const uint8_t>> spans(encrypted_sei_nalus_.size() + 1);
- spans.assign(encrypted_sei_nalus_.begin(), encrypted_sei_nalus_.end());
- spans.emplace_back(curr_nalu_->data, curr_nalu_->size);
- std::vector<SubsampleEntry> all_subsamples(sei_subsamples_.size() + 1);
- all_subsamples.assign(sei_subsamples_.begin(), sei_subsamples_.end());
- all_subsamples.insert(all_subsamples.end(), subsamples.begin(),
- subsamples.end());
- return accelerator_->ParseEncryptedSliceHeader(spans, all_subsamples,
- curr_slice_hdr_.get());
- }
- H264Decoder::H264Accelerator::Status H264Decoder::PreprocessCurrentSlice() {
- const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
- DCHECK(slice_hdr);
- if (IsNewPrimaryCodedPicture(curr_pic_.get(), curr_pps_id_,
- parser_.GetSPS(curr_sps_id_), *slice_hdr)) {
- // New picture, so first finish the previous one before processing it.
- H264Accelerator::Status result = FinishPrevFrameIfPresent();
- if (result != H264Accelerator::Status::kOk)
- return result;
- DCHECK(!curr_pic_);
- if (slice_hdr->first_mb_in_slice != 0) {
- DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: "
- << slice_hdr->first_mb_in_slice;
- return H264Accelerator::Status::kFail;
- }
- // If the new picture is an IDR, flush DPB.
- if (slice_hdr->idr_pic_flag) {
- // Output all remaining pictures, unless we are explicitly instructed
- // not to do so.
- if (!slice_hdr->no_output_of_prior_pics_flag) {
- if (!Flush())
- return H264Accelerator::Status::kFail;
- }
- dpb_.Clear();
- last_output_poc_ = std::numeric_limits<int>::min();
- }
- }
- return H264Accelerator::Status::kOk;
- }
- H264Decoder::H264Accelerator::Status H264Decoder::ProcessCurrentSlice() {
- DCHECK(curr_pic_);
- const H264SliceHeader* slice_hdr = curr_slice_hdr_.get();
- DCHECK(slice_hdr);
- if (slice_hdr->field_pic_flag == 0)
- max_pic_num_ = max_frame_num_;
- else
- max_pic_num_ = 2 * max_frame_num_;
- H264Picture::Vector ref_pic_list0, ref_pic_list1;
- // If we are using full sample encryption then we do not have the information
- // we need to update the ref pic lists here, but that's OK because the
- // accelerator doesn't actually need to submit them in this case.
- if (!slice_hdr->full_sample_encryption &&
- !ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1)) {
- return H264Accelerator::Status::kFail;
- }
- const H264PPS* pps = parser_.GetPPS(curr_pps_id_);
- if (!pps)
- return H264Accelerator::Status::kFail;
- return accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1,
- curr_pic_.get(), slice_hdr->nalu_data,
- slice_hdr->nalu_size,
- parser_.GetCurrentSubsamples());
- }
- #define SET_ERROR_AND_RETURN() \
- do { \
- DVLOG(1) << "Error during decode"; \
- state_ = State::kError; \
- return H264Decoder::kDecodeError; \
- } while (0)
- #define CHECK_ACCELERATOR_RESULT(func) \
- do { \
- H264Accelerator::Status result = (func); \
- switch (result) { \
- case H264Accelerator::Status::kOk: \
- break; \
- case H264Accelerator::Status::kTryAgain: \
- DVLOG(1) << #func " needs to try again"; \
- return H264Decoder::kTryAgain; \
- case H264Accelerator::Status::kFail: \
- case H264Accelerator::Status::kNotSupported: \
- SET_ERROR_AND_RETURN(); \
- } \
- } while (0)
- void H264Decoder::SetStream(int32_t id, const DecoderBuffer& decoder_buffer) {
- const uint8_t* ptr = decoder_buffer.data();
- const size_t size = decoder_buffer.data_size();
- const DecryptConfig* decrypt_config = decoder_buffer.decrypt_config();
- DCHECK(ptr);
- DCHECK(size);
- DVLOG(4) << "New input stream id: " << id << " at: " << (void*)ptr
- << " size: " << size;
- stream_id_ = id;
- current_stream_ = ptr;
- current_stream_size_ = size;
- current_stream_has_been_changed_ = true;
- encrypted_sei_nalus_.clear();
- sei_subsamples_.clear();
- if (decrypt_config) {
- parser_.SetEncryptedStream(ptr, size, decrypt_config->subsamples());
- current_decrypt_config_ = decrypt_config->Clone();
- } else {
- parser_.SetStream(ptr, size);
- current_decrypt_config_ = nullptr;
- }
- }
- H264Decoder::DecodeResult H264Decoder::Decode() {
- if (state_ == State::kError) {
- DVLOG(1) << "Decoder in error state";
- return kDecodeError;
- }
- if (current_stream_has_been_changed_) {
- // Calling H264Accelerator::SetStream() here instead of when the stream is
- // originally set in case the accelerator needs to return kTryAgain.
- H264Accelerator::Status result = accelerator_->SetStream(
- base::span<const uint8_t>(current_stream_, current_stream_size_),
- current_decrypt_config_.get());
- switch (result) {
- case H264Accelerator::Status::kOk:
- case H264Accelerator::Status::kNotSupported:
- // kNotSupported means the accelerator can't handle this stream,
- // so everything will be done through the parser.
- break;
- case H264Accelerator::Status::kTryAgain:
- DVLOG(1) << "SetStream() needs to try again";
- return H264Decoder::kTryAgain;
- case H264Accelerator::Status::kFail:
- SET_ERROR_AND_RETURN();
- }
- // Reset the flag so that this is only called again next time SetStream()
- // is called.
- current_stream_has_been_changed_ = false;
- }
- while (true) {
- H264Parser::Result par_res;
- if (!curr_nalu_) {
- curr_nalu_ = std::make_unique<H264NALU>();
- par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
- if (par_res == H264Parser::kEOStream) {
- CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
- return kRanOutOfStreamData;
- } else if (par_res != H264Parser::kOk) {
- SET_ERROR_AND_RETURN();
- }
- DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type);
- }
- switch (curr_nalu_->nal_unit_type) {
- case H264NALU::kNonIDRSlice:
- // We can't resume from a non-IDR slice unless recovery point SEI
- // process is going.
- if (state_ == State::kError ||
- (state_ == State::kAfterReset && !recovery_frame_cnt_))
- break;
- [[fallthrough]];
- case H264NALU::kIDRSlice: {
- // TODO(posciak): the IDR may require an SPS that we don't have
- // available. For now we'd fail if that happens, but ideally we'd like
- // to keep going until the next SPS in the stream.
- if (state_ == State::kNeedStreamMetadata) {
- // We need an SPS, skip this IDR and keep looking.
- break;
- }
- // If after reset or waiting for a key, we should be able to recover
- // from an IDR. |state_|, |curr_slice_hdr_|, and |curr_pic_| are used
- // to keep track of what has previously been attempted, so that after
- // a retryable result is returned, subsequent calls to Decode() retry
- // the call that failed previously. If it succeeds (it may not if no
- // additional key has been provided, for example), then the remaining
- // steps will be executed.
- if (!curr_slice_hdr_) {
- curr_slice_hdr_ = std::make_unique<H264SliceHeader>();
- state_ = State::kParseSliceHeader;
- }
- if (state_ == State::kParseSliceHeader) {
- // Check if the slice header is encrypted.
- bool parsed_header = false;
- if (current_decrypt_config_) {
- const std::vector<SubsampleEntry>& subsamples =
- parser_.GetCurrentSubsamples();
- // There is only a single clear byte for the NALU information for
- // full sample encryption, and the rest is encrypted.
- if (!subsamples.empty() && subsamples[0].clear_bytes == 1) {
- CHECK_ACCELERATOR_RESULT(ProcessEncryptedSliceHeader(subsamples));
- parsed_header = true;
- curr_slice_hdr_->pic_parameter_set_id = last_parsed_pps_id_;
- encrypted_sei_nalus_.clear();
- sei_subsamples_.clear();
- }
- }
- if (!parsed_header) {
- par_res =
- parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get());
- if (par_res != H264Parser::kOk)
- SET_ERROR_AND_RETURN();
- }
- state_ = State::kTryPreprocessCurrentSlice;
- }
- if (state_ == State::kTryPreprocessCurrentSlice) {
- CHECK_ACCELERATOR_RESULT(PreprocessCurrentSlice());
- state_ = State::kEnsurePicture;
- }
- if (state_ == State::kEnsurePicture) {
- if (curr_pic_) {
- // |curr_pic_| already exists, so skip to ProcessCurrentSlice().
- state_ = State::kTryCurrentSlice;
- } else {
- // New picture/finished previous one, try to start a new one
- // or tell the client we need more surfaces.
- curr_pic_ = accelerator_->CreateH264Picture();
- if (!curr_pic_)
- return kRanOutOfSurfaces;
- if (current_decrypt_config_)
- curr_pic_->set_decrypt_config(current_decrypt_config_->Clone());
- state_ = State::kTryNewFrame;
- }
- }
- if (state_ == State::kTryNewFrame) {
- CHECK_ACCELERATOR_RESULT(StartNewFrame(curr_slice_hdr_.get()));
- state_ = State::kTryCurrentSlice;
- }
- DCHECK_EQ(state_, State::kTryCurrentSlice);
- CHECK_ACCELERATOR_RESULT(ProcessCurrentSlice());
- curr_slice_hdr_.reset();
- state_ = State::kDecoding;
- break;
- }
- case H264NALU::kSPS: {
- int sps_id;
- CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
- par_res = parser_.ParseSPS(&sps_id);
- if (par_res != H264Parser::kOk)
- SET_ERROR_AND_RETURN();
- bool need_new_buffers = false;
- if (!ProcessSPS(sps_id, &need_new_buffers))
- SET_ERROR_AND_RETURN();
- accelerator_->ProcessSPS(
- parser_.GetSPS(sps_id),
- base::span<const uint8_t>(curr_nalu_->data, curr_nalu_->size));
- if (state_ == State::kNeedStreamMetadata)
- state_ = State::kAfterReset;
- if (need_new_buffers) {
- curr_pic_ = nullptr;
- curr_nalu_ = nullptr;
- ref_pic_list_p0_.clear();
- ref_pic_list_b0_.clear();
- ref_pic_list_b1_.clear();
- return kConfigChange;
- }
- break;
- }
- case H264NALU::kPPS: {
- CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
- par_res = parser_.ParsePPS(&last_parsed_pps_id_);
- if (par_res != H264Parser::kOk)
- SET_ERROR_AND_RETURN();
- accelerator_->ProcessPPS(
- parser_.GetPPS(last_parsed_pps_id_),
- base::span<const uint8_t>(curr_nalu_->data, curr_nalu_->size));
- break;
- }
- case H264NALU::kAUD:
- case H264NALU::kEOSeq:
- case H264NALU::kEOStream:
- if (state_ != State::kDecoding)
- break;
- CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
- break;
- case H264NALU::kSEIMessage:
- if (current_decrypt_config_) {
- // If there are encrypted SEI NALUs as part of CENCv1, then we also
- // need to save those so we can send them into the accelerator so it
- // can decrypt the sample properly (otherwise it would be starting
- // partway into a block).
- const std::vector<SubsampleEntry>& subsamples =
- parser_.GetCurrentSubsamples();
- if (!subsamples.empty()) {
- encrypted_sei_nalus_.emplace_back(curr_nalu_->data,
- curr_nalu_->size);
- DCHECK_EQ(1u, subsamples.size());
- sei_subsamples_.push_back(subsamples[0]);
- // Since the SEI is encrypted, do not try to parse it below as it
- // may fail or yield incorrect results.
- DVLOG(3) << "Skipping parsing of encrypted SEI NALU";
- break;
- }
- }
- if (state_ == State::kAfterReset && !recovery_frame_cnt_ &&
- !recovery_frame_num_) {
- // If we are after reset, we can also resume from a SEI recovery point
- // (spec D.2.8) if one is present. However, if we are already in the
- // process of handling one, skip any subsequent ones until we are done
- // processing.
- H264SEIMessage sei{};
- if (parser_.ParseSEI(&sei) != H264Parser::kOk)
- SET_ERROR_AND_RETURN();
- if (sei.type == H264SEIMessage::kSEIRecoveryPoint) {
- recovery_frame_cnt_ = sei.recovery_point.recovery_frame_cnt;
- if (0 > *recovery_frame_cnt_ ||
- *recovery_frame_cnt_ >= max_frame_num_) {
- DVLOG(1) << "Invalid recovery_frame_cnt=" << *recovery_frame_cnt_
- << " (it must be [0, max_frame_num_-1="
- << max_frame_num_ - 1 << "])";
- SET_ERROR_AND_RETURN();
- }
- DVLOG(3) << "Recovery point SEI is found, recovery_frame_cnt_="
- << *recovery_frame_cnt_;
- break;
- }
- }
- [[fallthrough]];
- default:
- DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
- break;
- }
- DVLOG(4) << "NALU done";
- curr_nalu_.reset();
- }
- }
- gfx::Size H264Decoder::GetPicSize() const {
- return pic_size_;
- }
- gfx::Rect H264Decoder::GetVisibleRect() const {
- return visible_rect_;
- }
- VideoCodecProfile H264Decoder::GetProfile() const {
- return profile_;
- }
- uint8_t H264Decoder::GetBitDepth() const {
- return bit_depth_;
- }
- VideoChromaSampling H264Decoder::GetChromaSampling() const {
- return chroma_sampling_;
- }
- size_t H264Decoder::GetRequiredNumOfPictures() const {
- constexpr size_t kPicsInPipeline = limits::kMaxVideoFrames + 1;
- return GetNumReferenceFrames() + kPicsInPipeline;
- }
- size_t H264Decoder::GetNumReferenceFrames() const {
- // Use the maximum number of pictures in the Decoded Picture Buffer.
- return dpb_.max_num_pics();
- }
- // static
- bool H264Decoder::FillH264PictureFromSliceHeader(
- const H264SPS* sps,
- const H264SliceHeader& slice_hdr,
- H264Picture* pic) {
- DCHECK(pic);
- pic->idr = slice_hdr.idr_pic_flag;
- if (pic->idr)
- pic->idr_pic_id = slice_hdr.idr_pic_id;
- if (slice_hdr.field_pic_flag) {
- pic->field = slice_hdr.bottom_field_flag ? H264Picture::FIELD_BOTTOM
- : H264Picture::FIELD_TOP;
- } else {
- pic->field = H264Picture::FIELD_NONE;
- }
- if (pic->field != H264Picture::FIELD_NONE) {
- DVLOG(1) << "Interlaced video not supported.";
- return false;
- }
- pic->nal_ref_idc = slice_hdr.nal_ref_idc;
- pic->ref = slice_hdr.nal_ref_idc != 0;
- // This assumes non-interlaced stream.
- pic->frame_num = pic->pic_num = slice_hdr.frame_num;
- if (!sps)
- return false;
- pic->pic_order_cnt_type = sps->pic_order_cnt_type;
- switch (pic->pic_order_cnt_type) {
- case 0:
- pic->pic_order_cnt_lsb = slice_hdr.pic_order_cnt_lsb;
- pic->delta_pic_order_cnt_bottom = slice_hdr.delta_pic_order_cnt_bottom;
- break;
- case 1:
- pic->delta_pic_order_cnt0 = slice_hdr.delta_pic_order_cnt0;
- pic->delta_pic_order_cnt1 = slice_hdr.delta_pic_order_cnt1;
- break;
- case 2:
- break;
- default:
- NOTREACHED();
- return false;
- }
- return true;
- }
- // static
- bool H264Decoder::IsNewPrimaryCodedPicture(const H264Picture* curr_pic,
- int curr_pps_id,
- const H264SPS* sps,
- const H264SliceHeader& slice_hdr) {
- if (!curr_pic)
- return true;
- // 7.4.1.2.4, assumes non-interlaced.
- if (slice_hdr.frame_num != curr_pic->frame_num ||
- slice_hdr.pic_parameter_set_id != curr_pps_id ||
- slice_hdr.nal_ref_idc != curr_pic->nal_ref_idc ||
- slice_hdr.idr_pic_flag != curr_pic->idr ||
- (slice_hdr.idr_pic_flag &&
- (slice_hdr.idr_pic_id != curr_pic->idr_pic_id ||
- // If we have two consecutive IDR slices, and the second one has
- // first_mb_in_slice == 0, treat it as a new picture.
- // Per spec, idr_pic_id should not be equal in this case (and we should
- // have hit the condition above instead, see spec 7.4.3 on idr_pic_id),
- // but some encoders neglect changing idr_pic_id for two consecutive
- // IDRs. Work around this by checking if the next slice contains the
- // zeroth macroblock, i.e. data that belongs to the next picture.
- slice_hdr.first_mb_in_slice == 0)))
- return true;
- if (!sps)
- return false;
- if (sps->pic_order_cnt_type == curr_pic->pic_order_cnt_type) {
- if (curr_pic->pic_order_cnt_type == 0) {
- if (slice_hdr.pic_order_cnt_lsb != curr_pic->pic_order_cnt_lsb ||
- slice_hdr.delta_pic_order_cnt_bottom !=
- curr_pic->delta_pic_order_cnt_bottom)
- return true;
- } else if (curr_pic->pic_order_cnt_type == 1) {
- if (slice_hdr.delta_pic_order_cnt0 != curr_pic->delta_pic_order_cnt0 ||
- slice_hdr.delta_pic_order_cnt1 != curr_pic->delta_pic_order_cnt1)
- return true;
- }
- }
- return false;
- }
- } // namespace media
|