h265_decoder.cc 34 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include <algorithm>
  5. #include "base/logging.h"
  6. #include "base/metrics/histogram_functions.h"
  7. #include "base/notreached.h"
  8. #include "media/base/limits.h"
  9. #include "media/gpu/h265_decoder.h"
  10. namespace media {
  11. namespace {
  12. struct POCAscCompare {
  13. bool operator()(const scoped_refptr<H265Picture>& a,
  14. const scoped_refptr<H265Picture>& b) const {
  15. return a->pic_order_cnt_val_ < b->pic_order_cnt_val_;
  16. }
  17. };
  18. bool ParseBitDepth(const H265SPS& sps, uint8_t& bit_depth) {
  19. // Spec 7.4.3.2.1
  20. if (sps.bit_depth_y != sps.bit_depth_c) {
  21. DVLOG(1) << "Different bit depths among planes is not supported";
  22. return false;
  23. }
  24. bit_depth = base::checked_cast<uint8_t>(sps.bit_depth_y);
  25. return true;
  26. }
  27. bool IsValidBitDepth(uint8_t bit_depth, VideoCodecProfile profile) {
  28. switch (profile) {
  29. // Spec A.3.2
  30. case HEVCPROFILE_MAIN:
  31. return bit_depth == 8u;
  32. // Spec A.3.3
  33. case HEVCPROFILE_MAIN10:
  34. return bit_depth == 8u || bit_depth == 10u;
  35. // Spec A.3.4
  36. case HEVCPROFILE_MAIN_STILL_PICTURE:
  37. return bit_depth == 8u;
  38. // Spec A.3.5
  39. case HEVCPROFILE_REXT:
  40. return bit_depth == 8u || bit_depth == 10u || bit_depth == 12u ||
  41. bit_depth == 14u || bit_depth == 16u;
  42. // Spec A.3.6
  43. case HEVCPROFILE_HIGH_THROUGHPUT:
  44. return bit_depth == 8u || bit_depth == 10u || bit_depth == 14u ||
  45. bit_depth == 16u;
  46. // Spec G.11.1.1
  47. case HEVCPROFILE_MULTIVIEW_MAIN:
  48. return bit_depth == 8u;
  49. // Spec H.11.1.1
  50. case HEVCPROFILE_SCALABLE_MAIN:
  51. return bit_depth == 8u || bit_depth == 10u;
  52. // Spec I.11.1.1
  53. case HEVCPROFILE_3D_MAIN:
  54. return bit_depth == 8u;
  55. // Spec A.3.7
  56. case HEVCPROFILE_SCREEN_EXTENDED:
  57. return bit_depth == 8u || bit_depth == 10u;
  58. // Spec H.11.1.2
  59. case HEVCPROFILE_SCALABLE_REXT:
  60. return bit_depth == 8u || bit_depth == 12u || bit_depth == 16u;
  61. // Spec A.3.8
  62. case HEVCPROFILE_HIGH_THROUGHPUT_SCREEN_EXTENDED:
  63. return bit_depth == 8u || bit_depth == 10u || bit_depth == 14u;
  64. default:
  65. DVLOG(1) << "Invalid profile specified for H265";
  66. return false;
  67. }
  68. }
  69. } // namespace
  70. H265Decoder::H265Accelerator::H265Accelerator() = default;
  71. H265Decoder::H265Accelerator::~H265Accelerator() = default;
  72. H265Decoder::H265Accelerator::Status H265Decoder::H265Accelerator::SetStream(
  73. base::span<const uint8_t> stream,
  74. const DecryptConfig* decrypt_config) {
  75. return H265Decoder::H265Accelerator::Status::kNotSupported;
  76. }
  77. H265Decoder::H265Decoder(std::unique_ptr<H265Accelerator> accelerator,
  78. VideoCodecProfile profile,
  79. const VideoColorSpace& container_color_space)
  80. : state_(kAfterReset),
  81. container_color_space_(container_color_space),
  82. profile_(profile),
  83. accelerator_(std::move(accelerator)) {
  84. DCHECK(accelerator_);
  85. Reset();
  86. }
  87. H265Decoder::~H265Decoder() = default;
  88. #define SET_ERROR_AND_RETURN() \
  89. do { \
  90. DVLOG(1) << "Error during decode"; \
  91. state_ = kError; \
  92. return H265Decoder::kDecodeError; \
  93. } while (0)
  94. #define CHECK_ACCELERATOR_RESULT(func) \
  95. do { \
  96. H265Accelerator::Status result = (func); \
  97. switch (result) { \
  98. case H265Accelerator::Status::kOk: \
  99. break; \
  100. case H265Accelerator::Status::kTryAgain: \
  101. DVLOG(1) << #func " needs to try again"; \
  102. return H265Decoder::kTryAgain; \
  103. case H265Accelerator::Status::kFail: /* fallthrough */ \
  104. case H265Accelerator::Status::kNotSupported: \
  105. SET_ERROR_AND_RETURN(); \
  106. } \
  107. } while (0)
  108. void H265Decoder::SetStream(int32_t id, const DecoderBuffer& decoder_buffer) {
  109. const uint8_t* ptr = decoder_buffer.data();
  110. const size_t size = decoder_buffer.data_size();
  111. const DecryptConfig* decrypt_config = decoder_buffer.decrypt_config();
  112. DCHECK(ptr);
  113. DCHECK(size);
  114. DVLOG(4) << "New input stream id: " << id << " at: " << (void*)ptr
  115. << " size: " << size;
  116. stream_id_ = id;
  117. current_stream_ = ptr;
  118. current_stream_size_ = size;
  119. current_stream_has_been_changed_ = true;
  120. if (decrypt_config) {
  121. parser_.SetEncryptedStream(ptr, size, decrypt_config->subsamples());
  122. current_decrypt_config_ = decrypt_config->Clone();
  123. } else {
  124. parser_.SetStream(ptr, size);
  125. current_decrypt_config_ = nullptr;
  126. }
  127. }
  128. void H265Decoder::Reset() {
  129. curr_pic_ = nullptr;
  130. curr_nalu_ = nullptr;
  131. curr_slice_hdr_ = nullptr;
  132. last_slice_hdr_ = nullptr;
  133. curr_sps_id_ = -1;
  134. curr_pps_id_ = -1;
  135. prev_tid0_pic_ = nullptr;
  136. ref_pic_list_.clear();
  137. ref_pic_list0_.clear();
  138. ref_pic_list1_.clear();
  139. ref_pic_set_lt_curr_.clear();
  140. ref_pic_set_st_curr_after_.clear();
  141. ref_pic_set_st_curr_before_.clear();
  142. dpb_.Clear();
  143. parser_.Reset();
  144. accelerator_->Reset();
  145. state_ = kAfterReset;
  146. }
  147. H265Decoder::DecodeResult H265Decoder::Decode() {
  148. if (state_ == kError) {
  149. DVLOG(1) << "Decoder in error state";
  150. return kDecodeError;
  151. }
  152. if (current_stream_has_been_changed_) {
  153. // Calling H265Accelerator::SetStream() here instead of when the stream is
  154. // originally set in case the accelerator needs to return kTryAgain.
  155. H265Accelerator::Status result = accelerator_->SetStream(
  156. base::span<const uint8_t>(current_stream_, current_stream_size_),
  157. current_decrypt_config_.get());
  158. switch (result) {
  159. case H265Accelerator::Status::kOk: // fallthrough
  160. case H265Accelerator::Status::kNotSupported:
  161. // kNotSupported means the accelerator can't handle this stream,
  162. // so everything will be done through the parser.
  163. break;
  164. case H265Accelerator::Status::kTryAgain:
  165. DVLOG(1) << "SetStream() needs to try again";
  166. return H265Decoder::kTryAgain;
  167. case H265Accelerator::Status::kFail:
  168. SET_ERROR_AND_RETURN();
  169. }
  170. // Reset the flag so that this is only called again next time SetStream()
  171. // is called.
  172. current_stream_has_been_changed_ = false;
  173. }
  174. while (true) {
  175. H265Parser::Result par_res;
  176. if (!curr_nalu_) {
  177. curr_nalu_ = std::make_unique<H265NALU>();
  178. par_res = parser_.AdvanceToNextNALU(curr_nalu_.get());
  179. if (par_res == H265Parser::kEOStream) {
  180. curr_nalu_.reset();
  181. // We receive one frame per buffer, so we can output the frame now.
  182. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  183. return kRanOutOfStreamData;
  184. }
  185. if (par_res != H265Parser::kOk) {
  186. curr_nalu_.reset();
  187. SET_ERROR_AND_RETURN();
  188. }
  189. DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type);
  190. }
  191. // 8.1.2 We only want nuh_layer_id of zero.
  192. if (curr_nalu_->nuh_layer_id) {
  193. DVLOG(4) << "Skipping NALU with nuh_layer_id="
  194. << curr_nalu_->nuh_layer_id;
  195. curr_nalu_.reset();
  196. continue;
  197. }
  198. switch (curr_nalu_->nal_unit_type) {
  199. case H265NALU::BLA_W_LP: // fallthrough
  200. case H265NALU::BLA_W_RADL:
  201. case H265NALU::BLA_N_LP:
  202. case H265NALU::IDR_W_RADL:
  203. case H265NALU::IDR_N_LP:
  204. case H265NALU::TRAIL_N:
  205. case H265NALU::TRAIL_R:
  206. case H265NALU::TSA_N:
  207. case H265NALU::TSA_R:
  208. case H265NALU::STSA_N:
  209. case H265NALU::STSA_R:
  210. case H265NALU::RADL_N:
  211. case H265NALU::RADL_R:
  212. case H265NALU::RASL_N:
  213. case H265NALU::RASL_R:
  214. case H265NALU::CRA_NUT:
  215. if (!curr_slice_hdr_) {
  216. curr_slice_hdr_.reset(new H265SliceHeader());
  217. par_res = parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get(),
  218. last_slice_hdr_.get());
  219. if (par_res == H265Parser::kMissingParameterSet) {
  220. // We may still be able to recover if we skip until we find the
  221. // SPS/PPS.
  222. curr_slice_hdr_.reset();
  223. last_slice_hdr_.reset();
  224. break;
  225. }
  226. if (par_res != H265Parser::kOk)
  227. SET_ERROR_AND_RETURN();
  228. if (!curr_slice_hdr_->irap_pic && state_ == kAfterReset) {
  229. // We can't resume from a non-IRAP picture.
  230. curr_slice_hdr_.reset();
  231. last_slice_hdr_.reset();
  232. break;
  233. }
  234. state_ = kTryPreprocessCurrentSlice;
  235. if (curr_slice_hdr_->irap_pic) {
  236. bool need_new_buffers = false;
  237. if (!ProcessPPS(curr_slice_hdr_->slice_pic_parameter_set_id,
  238. &need_new_buffers)) {
  239. SET_ERROR_AND_RETURN();
  240. }
  241. if (need_new_buffers) {
  242. curr_pic_ = nullptr;
  243. return kConfigChange;
  244. }
  245. }
  246. }
  247. if (state_ == kTryPreprocessCurrentSlice) {
  248. CHECK_ACCELERATOR_RESULT(PreprocessCurrentSlice());
  249. state_ = kEnsurePicture;
  250. }
  251. if (state_ == kEnsurePicture) {
  252. if (curr_pic_) {
  253. // |curr_pic_| already exists, so skip to ProcessCurrentSlice().
  254. state_ = kTryCurrentSlice;
  255. } else {
  256. // New picture, try to start a new one or tell client we need more
  257. // surfaces.
  258. curr_pic_ = accelerator_->CreateH265Picture();
  259. if (!curr_pic_)
  260. return kRanOutOfSurfaces;
  261. if (current_decrypt_config_)
  262. curr_pic_->set_decrypt_config(current_decrypt_config_->Clone());
  263. curr_pic_->first_picture_ = first_picture_;
  264. first_picture_ = false;
  265. state_ = kTryNewFrame;
  266. }
  267. }
  268. if (state_ == kTryNewFrame) {
  269. CHECK_ACCELERATOR_RESULT(StartNewFrame(curr_slice_hdr_.get()));
  270. state_ = kTryCurrentSlice;
  271. }
  272. DCHECK_EQ(state_, kTryCurrentSlice);
  273. CHECK_ACCELERATOR_RESULT(ProcessCurrentSlice());
  274. state_ = kDecoding;
  275. last_slice_hdr_.swap(curr_slice_hdr_);
  276. curr_slice_hdr_.reset();
  277. break;
  278. case H265NALU::SPS_NUT:
  279. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  280. int sps_id;
  281. par_res = parser_.ParseSPS(&sps_id);
  282. if (par_res != H265Parser::kOk)
  283. SET_ERROR_AND_RETURN();
  284. break;
  285. case H265NALU::PPS_NUT:
  286. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  287. int pps_id;
  288. par_res = parser_.ParsePPS(*curr_nalu_, &pps_id);
  289. if (par_res != H265Parser::kOk)
  290. SET_ERROR_AND_RETURN();
  291. // For ARC CTS tests they expect us to request the buffers after only
  292. // processing the SPS/PPS, we can't wait until we get the first IDR. To
  293. // resolve the problem that was created by originally doing that, only
  294. // do it if we don't have an active PPS set yet so it won't disturb an
  295. // active stream.
  296. if (curr_pps_id_ == -1) {
  297. bool need_new_buffers = false;
  298. if (!ProcessPPS(pps_id, &need_new_buffers)) {
  299. SET_ERROR_AND_RETURN();
  300. }
  301. if (need_new_buffers) {
  302. curr_nalu_.reset();
  303. return kConfigChange;
  304. }
  305. }
  306. break;
  307. case H265NALU::EOS_NUT:
  308. first_picture_ = true;
  309. [[fallthrough]];
  310. case H265NALU::EOB_NUT: // fallthrough
  311. case H265NALU::AUD_NUT:
  312. case H265NALU::RSV_NVCL41:
  313. case H265NALU::RSV_NVCL42:
  314. case H265NALU::RSV_NVCL43:
  315. case H265NALU::RSV_NVCL44:
  316. case H265NALU::UNSPEC48:
  317. case H265NALU::UNSPEC49:
  318. case H265NALU::UNSPEC50:
  319. case H265NALU::UNSPEC51:
  320. case H265NALU::UNSPEC52:
  321. case H265NALU::UNSPEC53:
  322. case H265NALU::UNSPEC54:
  323. case H265NALU::UNSPEC55:
  324. CHECK_ACCELERATOR_RESULT(FinishPrevFrameIfPresent());
  325. break;
  326. default:
  327. DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type;
  328. break;
  329. }
  330. DVLOG(4) << "NALU done";
  331. curr_nalu_.reset();
  332. }
  333. }
  334. gfx::Size H265Decoder::GetPicSize() const {
  335. return pic_size_;
  336. }
  337. gfx::Rect H265Decoder::GetVisibleRect() const {
  338. return visible_rect_;
  339. }
  340. VideoCodecProfile H265Decoder::GetProfile() const {
  341. return profile_;
  342. }
  343. uint8_t H265Decoder::GetBitDepth() const {
  344. return bit_depth_;
  345. }
  346. VideoChromaSampling H265Decoder::GetChromaSampling() const {
  347. return chroma_sampling_;
  348. }
  349. size_t H265Decoder::GetRequiredNumOfPictures() const {
  350. constexpr size_t kPicsInPipeline = limits::kMaxVideoFrames + 1;
  351. return GetNumReferenceFrames() + kPicsInPipeline;
  352. }
  353. size_t H265Decoder::GetNumReferenceFrames() const {
  354. // Use the maximum number of pictures in the Decoded Picture Buffer.
  355. return dpb_.max_num_pics();
  356. }
  357. bool H265Decoder::ProcessPPS(int pps_id, bool* need_new_buffers) {
  358. DVLOG(4) << "Processing PPS id:" << pps_id;
  359. const H265PPS* pps = parser_.GetPPS(pps_id);
  360. // Slice header parsing already verified this should exist.
  361. DCHECK(pps);
  362. const H265SPS* sps = parser_.GetSPS(pps->pps_seq_parameter_set_id);
  363. // PPS parsing already verified this should exist.
  364. DCHECK(sps);
  365. if (need_new_buffers)
  366. *need_new_buffers = false;
  367. gfx::Size new_pic_size = sps->GetCodedSize();
  368. gfx::Rect new_visible_rect = sps->GetVisibleRect();
  369. if (visible_rect_ != new_visible_rect) {
  370. DVLOG(2) << "New visible rect: " << new_visible_rect.ToString();
  371. visible_rect_ = new_visible_rect;
  372. }
  373. VideoChromaSampling new_chroma_sampling = sps->GetChromaSampling();
  374. if (new_chroma_sampling != chroma_sampling_) {
  375. base::UmaHistogramEnumeration("Media.PlatformVideoDecoding.ChromaSampling",
  376. new_chroma_sampling);
  377. }
  378. if (!accelerator_->IsChromaSamplingSupported(new_chroma_sampling)) {
  379. DVLOG(1) << "Only YUV 4:2:0 is supported";
  380. return false;
  381. }
  382. // Equation 7-8
  383. max_pic_order_cnt_lsb_ =
  384. std::pow(2, sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
  385. VideoCodecProfile new_profile = H265Parser::ProfileIDCToVideoCodecProfile(
  386. sps->profile_tier_level.general_profile_idc);
  387. uint8_t new_bit_depth = 0;
  388. if (!ParseBitDepth(*sps, new_bit_depth))
  389. return false;
  390. if (!IsValidBitDepth(new_bit_depth, new_profile)) {
  391. DVLOG(1) << "Invalid bit depth=" << base::strict_cast<int>(new_bit_depth)
  392. << ", profile=" << GetProfileName(new_profile);
  393. return false;
  394. }
  395. if (pic_size_ != new_pic_size || dpb_.max_num_pics() != sps->max_dpb_size ||
  396. profile_ != new_profile || bit_depth_ != new_bit_depth ||
  397. chroma_sampling_ != new_chroma_sampling) {
  398. if (!Flush())
  399. return false;
  400. DVLOG(1) << "Codec profile: " << GetProfileName(new_profile)
  401. << ", level(x30): " << sps->profile_tier_level.general_level_idc
  402. << ", DPB size: " << sps->max_dpb_size
  403. << ", Picture size: " << new_pic_size.ToString()
  404. << ", bit_depth: " << base::strict_cast<int>(new_bit_depth)
  405. << ", chroma_sampling_format: "
  406. << VideoChromaSamplingToString(new_chroma_sampling);
  407. profile_ = new_profile;
  408. bit_depth_ = new_bit_depth;
  409. pic_size_ = new_pic_size;
  410. chroma_sampling_ = new_chroma_sampling;
  411. dpb_.set_max_num_pics(sps->max_dpb_size);
  412. if (need_new_buffers)
  413. *need_new_buffers = true;
  414. }
  415. return true;
  416. }
  417. H265Decoder::H265Accelerator::Status H265Decoder::PreprocessCurrentSlice() {
  418. const H265SliceHeader* slice_hdr = curr_slice_hdr_.get();
  419. DCHECK(slice_hdr);
  420. if (slice_hdr->first_slice_segment_in_pic_flag) {
  421. // New picture, so first finish the previous one before processing it.
  422. H265Accelerator::Status result = FinishPrevFrameIfPresent();
  423. if (result != H265Accelerator::Status::kOk)
  424. return result;
  425. DCHECK(!curr_pic_);
  426. }
  427. return H265Accelerator::Status::kOk;
  428. }
  429. H265Decoder::H265Accelerator::Status H265Decoder::ProcessCurrentSlice() {
  430. DCHECK(curr_pic_);
  431. const H265SliceHeader* slice_hdr = curr_slice_hdr_.get();
  432. DCHECK(slice_hdr);
  433. const H265SPS* sps = parser_.GetSPS(curr_sps_id_);
  434. DCHECK(sps);
  435. const H265PPS* pps = parser_.GetPPS(curr_pps_id_);
  436. DCHECK(pps);
  437. return accelerator_->SubmitSlice(
  438. sps, pps, slice_hdr, ref_pic_list0_, ref_pic_list1_, ref_pic_set_lt_curr_,
  439. ref_pic_set_st_curr_after_, ref_pic_set_st_curr_before_, curr_pic_.get(),
  440. slice_hdr->nalu_data, slice_hdr->nalu_size,
  441. parser_.GetCurrentSubsamples());
  442. }
  443. void H265Decoder::CalcPicOutputFlags(const H265SliceHeader* slice_hdr) {
  444. if (slice_hdr->irap_pic) {
  445. // 8.1.3
  446. curr_pic_->no_rasl_output_flag_ =
  447. (curr_nalu_->nal_unit_type >= H265NALU::BLA_W_LP &&
  448. curr_nalu_->nal_unit_type <= H265NALU::IDR_N_LP) ||
  449. curr_pic_->first_picture_;
  450. } else {
  451. curr_pic_->no_rasl_output_flag_ = false;
  452. }
  453. // C.5.2.2
  454. if (slice_hdr->irap_pic && curr_pic_->no_rasl_output_flag_ &&
  455. !curr_pic_->first_picture_) {
  456. curr_pic_->no_output_of_prior_pics_flag_ =
  457. (slice_hdr->nal_unit_type == H265NALU::CRA_NUT) ||
  458. slice_hdr->no_output_of_prior_pics_flag;
  459. } else {
  460. curr_pic_->no_output_of_prior_pics_flag_ = false;
  461. }
  462. if ((slice_hdr->nal_unit_type == H265NALU::RASL_N ||
  463. slice_hdr->nal_unit_type == H265NALU::RASL_R) &&
  464. curr_pic_->no_rasl_output_flag_) {
  465. curr_pic_->pic_output_flag_ = false;
  466. } else {
  467. curr_pic_->pic_output_flag_ = slice_hdr->pic_output_flag;
  468. }
  469. }
  470. void H265Decoder::CalcPictureOrderCount(const H265PPS* pps,
  471. const H265SliceHeader* slice_hdr) {
  472. // 8.3.1 Decoding process for picture order count.
  473. curr_pic_->valid_for_prev_tid0_pic_ =
  474. !pps->temporal_id && (slice_hdr->nal_unit_type < H265NALU::RADL_N ||
  475. slice_hdr->nal_unit_type > H265NALU::RSV_VCL_N14);
  476. curr_pic_->slice_pic_order_cnt_lsb_ = slice_hdr->slice_pic_order_cnt_lsb;
  477. // Calculate POC for current picture.
  478. if ((!slice_hdr->irap_pic || !curr_pic_->no_rasl_output_flag_) &&
  479. prev_tid0_pic_) {
  480. const int prev_pic_order_cnt_lsb = prev_tid0_pic_->slice_pic_order_cnt_lsb_;
  481. const int prev_pic_order_cnt_msb = prev_tid0_pic_->pic_order_cnt_msb_;
  482. if ((slice_hdr->slice_pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
  483. ((prev_pic_order_cnt_lsb - slice_hdr->slice_pic_order_cnt_lsb) >=
  484. (max_pic_order_cnt_lsb_ / 2))) {
  485. curr_pic_->pic_order_cnt_msb_ =
  486. prev_pic_order_cnt_msb + max_pic_order_cnt_lsb_;
  487. } else if ((slice_hdr->slice_pic_order_cnt_lsb > prev_pic_order_cnt_lsb) &&
  488. ((slice_hdr->slice_pic_order_cnt_lsb - prev_pic_order_cnt_lsb) >
  489. (max_pic_order_cnt_lsb_ / 2))) {
  490. curr_pic_->pic_order_cnt_msb_ =
  491. prev_pic_order_cnt_msb - max_pic_order_cnt_lsb_;
  492. } else {
  493. curr_pic_->pic_order_cnt_msb_ = prev_pic_order_cnt_msb;
  494. }
  495. } else {
  496. curr_pic_->pic_order_cnt_msb_ = 0;
  497. }
  498. curr_pic_->pic_order_cnt_val_ =
  499. curr_pic_->pic_order_cnt_msb_ + slice_hdr->slice_pic_order_cnt_lsb;
  500. }
  501. bool H265Decoder::CalcRefPicPocs(const H265SPS* sps,
  502. const H265PPS* pps,
  503. const H265SliceHeader* slice_hdr) {
  504. if (slice_hdr->nal_unit_type == H265NALU::IDR_W_RADL ||
  505. slice_hdr->nal_unit_type == H265NALU::IDR_N_LP) {
  506. num_poc_st_curr_before_ = num_poc_st_curr_after_ = num_poc_st_foll_ =
  507. num_poc_lt_curr_ = num_poc_lt_foll_ = 0;
  508. return true;
  509. }
  510. // 8.3.2 - NOTE 2
  511. const H265StRefPicSet& curr_st_ref_pic_set = slice_hdr->GetStRefPicSet(sps);
  512. // Equation 8-5.
  513. int i, j, k;
  514. for (i = 0, j = 0, k = 0; i < curr_st_ref_pic_set.num_negative_pics; ++i) {
  515. base::CheckedNumeric<int> poc = curr_pic_->pic_order_cnt_val_;
  516. poc += curr_st_ref_pic_set.delta_poc_s0[i];
  517. if (!poc.IsValid()) {
  518. DVLOG(1) << "Invalid POC";
  519. return false;
  520. }
  521. if (curr_st_ref_pic_set.used_by_curr_pic_s0[i])
  522. poc_st_curr_before_[j++] = poc.ValueOrDefault(0);
  523. else
  524. poc_st_foll_[k++] = poc.ValueOrDefault(0);
  525. }
  526. num_poc_st_curr_before_ = j;
  527. for (i = 0, j = 0; i < curr_st_ref_pic_set.num_positive_pics; ++i) {
  528. base::CheckedNumeric<int> poc = curr_pic_->pic_order_cnt_val_;
  529. poc += curr_st_ref_pic_set.delta_poc_s1[i];
  530. if (!poc.IsValid()) {
  531. DVLOG(1) << "Invalid POC";
  532. return false;
  533. }
  534. if (curr_st_ref_pic_set.used_by_curr_pic_s1[i])
  535. poc_st_curr_after_[j++] = poc.ValueOrDefault(0);
  536. else
  537. poc_st_foll_[k++] = poc.ValueOrDefault(0);
  538. }
  539. num_poc_st_curr_after_ = j;
  540. num_poc_st_foll_ = k;
  541. for (i = 0, j = 0, k = 0;
  542. i < slice_hdr->num_long_term_sps + slice_hdr->num_long_term_pics; ++i) {
  543. base::CheckedNumeric<int> poc_lt = slice_hdr->poc_lsb_lt[i];
  544. if (slice_hdr->delta_poc_msb_present_flag[i]) {
  545. poc_lt += curr_pic_->pic_order_cnt_val_;
  546. base::CheckedNumeric<int> poc_delta =
  547. slice_hdr->delta_poc_msb_cycle_lt[i];
  548. poc_delta *= max_pic_order_cnt_lsb_;
  549. if (!poc_delta.IsValid()) {
  550. DVLOG(1) << "Invalid POC";
  551. return false;
  552. }
  553. poc_lt -= poc_delta.ValueOrDefault(0);
  554. poc_lt -= curr_pic_->pic_order_cnt_val_ & (max_pic_order_cnt_lsb_ - 1);
  555. }
  556. if (!poc_lt.IsValid()) {
  557. DVLOG(1) << "Invalid POC";
  558. return false;
  559. }
  560. if (slice_hdr->used_by_curr_pic_lt[i]) {
  561. poc_lt_curr_[j] = poc_lt.ValueOrDefault(0);
  562. curr_delta_poc_msb_present_flag_[j++] =
  563. slice_hdr->delta_poc_msb_present_flag[i];
  564. } else {
  565. poc_lt_foll_[k] = poc_lt.ValueOrDefault(0);
  566. foll_delta_poc_msb_present_flag_[k++] =
  567. slice_hdr->delta_poc_msb_present_flag[i];
  568. }
  569. }
  570. num_poc_lt_curr_ = j;
  571. num_poc_lt_foll_ = k;
  572. // Check conformance for |num_pic_total_curr|.
  573. if (slice_hdr->nal_unit_type == H265NALU::CRA_NUT ||
  574. (slice_hdr->nal_unit_type >= H265NALU::BLA_W_LP &&
  575. slice_hdr->nal_unit_type <= H265NALU::BLA_N_LP)) {
  576. if (slice_hdr->num_pic_total_curr) {
  577. DVLOG(1) << "Invalid value for num_pic_total_curr";
  578. return false;
  579. }
  580. } else if ((slice_hdr->IsBSlice() || slice_hdr->IsPSlice()) &&
  581. !slice_hdr->num_pic_total_curr) {
  582. DVLOG(1) << "Invalid value for num_pic_total_curr";
  583. return false;
  584. }
  585. return true;
  586. }
  587. bool H265Decoder::BuildRefPicLists(const H265SPS* sps,
  588. const H265PPS* pps,
  589. const H265SliceHeader* slice_hdr) {
  590. ref_pic_set_lt_curr_.clear();
  591. ref_pic_set_lt_curr_.resize(kMaxDpbSize);
  592. ref_pic_set_st_curr_after_.clear();
  593. ref_pic_set_st_curr_after_.resize(kMaxDpbSize);
  594. ref_pic_set_st_curr_before_.clear();
  595. ref_pic_set_st_curr_before_.resize(kMaxDpbSize);
  596. scoped_refptr<H265Picture> ref_pic_set_lt_foll[kMaxDpbSize];
  597. scoped_refptr<H265Picture> ref_pic_set_st_foll[kMaxDpbSize];
  598. // Mark everything in the DPB as unused for reference now. When we determine
  599. // the pics in the ref list, then we will mark them appropriately.
  600. dpb_.MarkAllUnusedForReference();
  601. // Equation 8-6.
  602. // We may be missing reference pictures, if so then we just don't specify
  603. // them and let the accelerator deal with the missing reference pictures
  604. // which is covered in the spec.
  605. int total_ref_pics = 0;
  606. for (int i = 0; i < num_poc_lt_curr_; ++i) {
  607. if (!curr_delta_poc_msb_present_flag_[i])
  608. ref_pic_set_lt_curr_[i] = dpb_.GetPicByPocMaskedAndMark(
  609. poc_lt_curr_[i], sps->max_pic_order_cnt_lsb - 1,
  610. H265Picture::kLongTermCurr);
  611. else
  612. ref_pic_set_lt_curr_[i] =
  613. dpb_.GetPicByPocAndMark(poc_lt_curr_[i], H265Picture::kLongTermCurr);
  614. if (ref_pic_set_lt_curr_[i])
  615. total_ref_pics++;
  616. }
  617. for (int i = 0; i < num_poc_lt_foll_; ++i) {
  618. if (!foll_delta_poc_msb_present_flag_[i])
  619. ref_pic_set_lt_foll[i] = dpb_.GetPicByPocMaskedAndMark(
  620. poc_lt_foll_[i], sps->max_pic_order_cnt_lsb - 1,
  621. H265Picture::kLongTermFoll);
  622. else
  623. ref_pic_set_lt_foll[i] =
  624. dpb_.GetPicByPocAndMark(poc_lt_foll_[i], H265Picture::kLongTermFoll);
  625. if (ref_pic_set_lt_foll[i])
  626. total_ref_pics++;
  627. }
  628. // Equation 8-7.
  629. for (int i = 0; i < num_poc_st_curr_before_; ++i) {
  630. ref_pic_set_st_curr_before_[i] = dpb_.GetPicByPocAndMark(
  631. poc_st_curr_before_[i], H265Picture::kShortTermCurrBefore);
  632. if (ref_pic_set_st_curr_before_[i])
  633. total_ref_pics++;
  634. }
  635. for (int i = 0; i < num_poc_st_curr_after_; ++i) {
  636. ref_pic_set_st_curr_after_[i] = dpb_.GetPicByPocAndMark(
  637. poc_st_curr_after_[i], H265Picture::kShortTermCurrAfter);
  638. if (ref_pic_set_st_curr_after_[i])
  639. total_ref_pics++;
  640. }
  641. for (int i = 0; i < num_poc_st_foll_; ++i) {
  642. ref_pic_set_st_foll[i] =
  643. dpb_.GetPicByPocAndMark(poc_st_foll_[i], H265Picture::kShortTermFoll);
  644. if (ref_pic_set_st_foll[i])
  645. total_ref_pics++;
  646. }
  647. // Verify that the total number of reference pictures in the DPB matches the
  648. // total count of reference pics. This ensures that a picture is not in more
  649. // than one list, per the spec.
  650. if (dpb_.GetReferencePicCount() != total_ref_pics) {
  651. DVLOG(1) << "Conformance problem, reference pic is in more than one list";
  652. return false;
  653. }
  654. ref_pic_list_.clear();
  655. dpb_.AppendReferencePics(&ref_pic_list_);
  656. ref_pic_list0_.clear();
  657. ref_pic_list1_.clear();
  658. // 8.3.3 Generation of unavailable reference pictures is something we do not
  659. // need to handle here. It's handled by the accelerator itself when we do not
  660. // specify a reference picture that it needs.
  661. if (slice_hdr->IsPSlice() || slice_hdr->IsBSlice()) {
  662. // 8.3.4 Decoding process for reference picture lists construction
  663. int num_rps_curr_temp_list0 =
  664. std::max(slice_hdr->num_ref_idx_l0_active_minus1 + 1,
  665. slice_hdr->num_pic_total_curr);
  666. scoped_refptr<H265Picture> ref_pic_list_temp0[kMaxDpbSize];
  667. // Equation 8-8.
  668. int r_idx = 0;
  669. while (r_idx < num_rps_curr_temp_list0) {
  670. for (int i = 0;
  671. i < num_poc_st_curr_before_ && r_idx < num_rps_curr_temp_list0;
  672. ++i, ++r_idx) {
  673. ref_pic_list_temp0[r_idx] = ref_pic_set_st_curr_before_[i];
  674. }
  675. for (int i = 0;
  676. i < num_poc_st_curr_after_ && r_idx < num_rps_curr_temp_list0;
  677. ++i, ++r_idx) {
  678. ref_pic_list_temp0[r_idx] = ref_pic_set_st_curr_after_[i];
  679. }
  680. for (int i = 0; i < num_poc_lt_curr_ && r_idx < num_rps_curr_temp_list0;
  681. ++i, ++r_idx) {
  682. ref_pic_list_temp0[r_idx] = ref_pic_set_lt_curr_[i];
  683. }
  684. }
  685. // Equation 8-9.
  686. for (r_idx = 0; r_idx <= slice_hdr->num_ref_idx_l0_active_minus1; ++r_idx) {
  687. ref_pic_list0_.push_back(
  688. slice_hdr->ref_pic_lists_modification
  689. .ref_pic_list_modification_flag_l0
  690. ? ref_pic_list_temp0[slice_hdr->ref_pic_lists_modification
  691. .list_entry_l0[r_idx]]
  692. : ref_pic_list_temp0[r_idx]);
  693. }
  694. if (slice_hdr->IsBSlice()) {
  695. int num_rps_curr_temp_list1 =
  696. std::max(slice_hdr->num_ref_idx_l1_active_minus1 + 1,
  697. slice_hdr->num_pic_total_curr);
  698. scoped_refptr<H265Picture> ref_pic_list_temp1[kMaxDpbSize];
  699. // Equation 8-10.
  700. r_idx = 0;
  701. while (r_idx < num_rps_curr_temp_list1) {
  702. for (int i = 0;
  703. i < num_poc_st_curr_after_ && r_idx < num_rps_curr_temp_list1;
  704. ++i, r_idx++) {
  705. ref_pic_list_temp1[r_idx] = ref_pic_set_st_curr_after_[i];
  706. }
  707. for (int i = 0;
  708. i < num_poc_st_curr_before_ && r_idx < num_rps_curr_temp_list1;
  709. ++i, r_idx++) {
  710. ref_pic_list_temp1[r_idx] = ref_pic_set_st_curr_before_[i];
  711. }
  712. for (int i = 0; i < num_poc_lt_curr_ && r_idx < num_rps_curr_temp_list1;
  713. ++i, r_idx++) {
  714. ref_pic_list_temp1[r_idx] = ref_pic_set_lt_curr_[i];
  715. }
  716. }
  717. // Equation 8-11.
  718. for (r_idx = 0; r_idx <= slice_hdr->num_ref_idx_l1_active_minus1;
  719. ++r_idx) {
  720. ref_pic_list1_.push_back(
  721. slice_hdr->ref_pic_lists_modification
  722. .ref_pic_list_modification_flag_l1
  723. ? ref_pic_list_temp1[slice_hdr->ref_pic_lists_modification
  724. .list_entry_l1[r_idx]]
  725. : ref_pic_list_temp1[r_idx]);
  726. }
  727. }
  728. }
  729. return true;
  730. }
  731. H265Decoder::H265Accelerator::Status H265Decoder::StartNewFrame(
  732. const H265SliceHeader* slice_hdr) {
  733. CHECK(curr_pic_.get());
  734. DCHECK(slice_hdr);
  735. curr_pps_id_ = slice_hdr->slice_pic_parameter_set_id;
  736. const H265PPS* pps = parser_.GetPPS(curr_pps_id_);
  737. // Slice header parsing already verified this should exist.
  738. DCHECK(pps);
  739. curr_sps_id_ = pps->pps_seq_parameter_set_id;
  740. const H265SPS* sps = parser_.GetSPS(curr_sps_id_);
  741. // Slice header parsing already verified this should exist.
  742. DCHECK(sps);
  743. // If this is from a retry on SubmitFrameMetadata, we should not redo all of
  744. // these calculations.
  745. if (!curr_pic_->processed_) {
  746. // Copy slice/pps variables we need to the picture.
  747. curr_pic_->nal_unit_type_ = curr_nalu_->nal_unit_type;
  748. curr_pic_->irap_pic_ = slice_hdr->irap_pic;
  749. curr_pic_->set_visible_rect(visible_rect_);
  750. curr_pic_->set_bitstream_id(stream_id_);
  751. if (sps->GetColorSpace().IsSpecified())
  752. curr_pic_->set_colorspace(sps->GetColorSpace());
  753. else
  754. curr_pic_->set_colorspace(container_color_space_);
  755. CalcPicOutputFlags(slice_hdr);
  756. CalcPictureOrderCount(pps, slice_hdr);
  757. if (!CalcRefPicPocs(sps, pps, slice_hdr)) {
  758. return H265Accelerator::Status::kFail;
  759. }
  760. if (!BuildRefPicLists(sps, pps, slice_hdr)) {
  761. return H265Accelerator::Status::kFail;
  762. }
  763. if (!PerformDpbOperations(sps)) {
  764. return H265Accelerator::Status::kFail;
  765. }
  766. curr_pic_->processed_ = true;
  767. }
  768. return accelerator_->SubmitFrameMetadata(sps, pps, slice_hdr, ref_pic_list_,
  769. curr_pic_);
  770. }
  771. H265Decoder::H265Accelerator::Status H265Decoder::FinishPrevFrameIfPresent() {
  772. // If we already have a frame waiting to be decoded, decode it and finish.
  773. if (curr_pic_) {
  774. H265Accelerator::Status result = DecodePicture();
  775. if (result != H265Accelerator::Status::kOk)
  776. return result;
  777. scoped_refptr<H265Picture> pic = curr_pic_;
  778. curr_pic_ = nullptr;
  779. FinishPicture(pic);
  780. }
  781. return H265Accelerator::Status::kOk;
  782. }
  783. bool H265Decoder::PerformDpbOperations(const H265SPS* sps) {
  784. // C.5.2.2
  785. if (curr_pic_->irap_pic_ && curr_pic_->no_rasl_output_flag_ &&
  786. !curr_pic_->first_picture_) {
  787. if (!curr_pic_->no_output_of_prior_pics_flag_) {
  788. OutputAllRemainingPics();
  789. }
  790. dpb_.Clear();
  791. } else {
  792. int num_to_output;
  793. do {
  794. dpb_.DeleteUnused();
  795. // Get all pictures that haven't been outputted yet.
  796. H265Picture::Vector not_outputted;
  797. dpb_.AppendPendingOutputPics(&not_outputted);
  798. // Sort in output order.
  799. std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare());
  800. // Calculate how many pictures we need to output.
  801. num_to_output = 0;
  802. int highest_tid = sps->sps_max_sub_layers_minus1;
  803. num_to_output = std::max(num_to_output,
  804. static_cast<int>(not_outputted.size()) -
  805. sps->sps_max_num_reorder_pics[highest_tid]);
  806. num_to_output =
  807. std::max(num_to_output,
  808. static_cast<int>(dpb_.size()) -
  809. sps->sps_max_dec_pic_buffering_minus1[highest_tid]);
  810. num_to_output =
  811. std::min(num_to_output, static_cast<int>(not_outputted.size()));
  812. if (!num_to_output && dpb_.IsFull()) {
  813. // This is wrong, we should try to output pictures until we can clear
  814. // one from the DPB. This is better than failing, but we then may end up
  815. // with something out of order.
  816. DVLOG(1) << "Forcibly outputting pictures to make room in DPB.";
  817. for (const auto& pic : not_outputted) {
  818. num_to_output++;
  819. if (pic->ref_ == H265Picture::kUnused)
  820. break;
  821. }
  822. }
  823. // TODO(jkardatzke): There's another output picture requirement regarding
  824. // the sps_max_latency_increase_plus1, but I have yet to understand how
  825. // that could be larger than the sps_max_num_reorder_pics since the actual
  826. // latency value used is the sum of both.
  827. not_outputted.resize(num_to_output);
  828. for (auto& pic : not_outputted) {
  829. if (!OutputPic(pic))
  830. return false;
  831. }
  832. dpb_.DeleteUnused();
  833. } while (dpb_.IsFull() && num_to_output);
  834. }
  835. if (dpb_.IsFull()) {
  836. DVLOG(1) << "Could not free up space in DPB for current picture";
  837. return false;
  838. }
  839. // Put the current pic in the DPB.
  840. dpb_.StorePicture(curr_pic_, H265Picture::kShortTermFoll);
  841. return true;
  842. }
  843. void H265Decoder::FinishPicture(scoped_refptr<H265Picture> pic) {
  844. // 8.3.1
  845. if (pic->valid_for_prev_tid0_pic_)
  846. prev_tid0_pic_ = pic;
  847. ref_pic_list_.clear();
  848. ref_pic_list0_.clear();
  849. ref_pic_list1_.clear();
  850. ref_pic_set_lt_curr_.clear();
  851. ref_pic_set_st_curr_after_.clear();
  852. ref_pic_set_st_curr_before_.clear();
  853. last_slice_hdr_.reset();
  854. }
  855. H265Decoder::H265Accelerator::Status H265Decoder::DecodePicture() {
  856. DCHECK(curr_pic_.get());
  857. return accelerator_->SubmitDecode(curr_pic_);
  858. }
  859. bool H265Decoder::OutputPic(scoped_refptr<H265Picture> pic) {
  860. DCHECK(!pic->outputted_);
  861. pic->outputted_ = true;
  862. DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt_val_;
  863. return accelerator_->OutputPicture(std::move(pic));
  864. }
  865. bool H265Decoder::OutputAllRemainingPics() {
  866. // Output all pictures that are waiting to be outputted.
  867. H265Picture::Vector to_output;
  868. dpb_.AppendPendingOutputPics(&to_output);
  869. // Sort them by ascending POC to output in order.
  870. std::sort(to_output.begin(), to_output.end(), POCAscCompare());
  871. for (auto& pic : to_output) {
  872. if (!OutputPic(std::move(pic)))
  873. return false;
  874. }
  875. return true;
  876. }
  877. bool H265Decoder::Flush() {
  878. DVLOG(2) << "Decoder flush";
  879. if (!OutputAllRemainingPics())
  880. return false;
  881. dpb_.Clear();
  882. prev_tid0_pic_ = nullptr;
  883. return true;
  884. }
  885. } // namespace media