mp4_stream_parser.cc 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/formats/mp4/mp4_stream_parser.h"
  5. #include <stddef.h>
  6. #include <limits>
  7. #include <memory>
  8. #include <utility>
  9. #include <vector>
  10. #include "base/callback_helpers.h"
  11. #include "base/logging.h"
  12. #include "base/numerics/math_constants.h"
  13. #include "base/strings/string_number_conversions.h"
  14. #include "base/time/time.h"
  15. #include "build/build_config.h"
  16. #include "media/base/audio_decoder_config.h"
  17. #include "media/base/encryption_pattern.h"
  18. #include "media/base/encryption_scheme.h"
  19. #include "media/base/media_tracks.h"
  20. #include "media/base/media_util.h"
  21. #include "media/base/stream_parser_buffer.h"
  22. #include "media/base/text_track_config.h"
  23. #include "media/base/timestamp_constants.h"
  24. #include "media/base/video_decoder_config.h"
  25. #include "media/base/video_util.h"
  26. #include "media/formats/mp4/box_definitions.h"
  27. #include "media/formats/mp4/box_reader.h"
  28. #include "media/formats/mp4/es_descriptor.h"
  29. #include "media/formats/mp4/rcheck.h"
  30. #include "media/formats/mpeg/adts_constants.h"
  31. namespace media {
  32. namespace mp4 {
  33. namespace {
  34. const int kMaxEmptySampleLogs = 20;
  35. const int kMaxInvalidConversionLogs = 20;
  36. const int kMaxVideoKeyframeMismatchLogs = 10;
  37. // Caller should be prepared to handle return of EncryptionScheme::kUnencrypted
  38. // in case of unsupported scheme.
  39. EncryptionScheme GetEncryptionScheme(const ProtectionSchemeInfo& sinf) {
  40. if (!sinf.HasSupportedScheme())
  41. return EncryptionScheme::kUnencrypted;
  42. FourCC fourcc = sinf.type.type;
  43. switch (fourcc) {
  44. case FOURCC_CENC:
  45. return EncryptionScheme::kCenc;
  46. case FOURCC_CBCS:
  47. return EncryptionScheme::kCbcs;
  48. default:
  49. NOTREACHED();
  50. break;
  51. }
  52. return EncryptionScheme::kUnencrypted;
  53. }
  54. gfx::ColorVolumeMetadata ConvertMdcvToColorVolumeMetadata(
  55. const MasteringDisplayColorVolume& mdcv) {
  56. gfx::ColorVolumeMetadata color_volume_metadata;
  57. color_volume_metadata.primary_r = gfx::ColorVolumeMetadata::Chromaticity(
  58. mdcv.display_primaries_rx, mdcv.display_primaries_ry);
  59. color_volume_metadata.primary_g = gfx::ColorVolumeMetadata::Chromaticity(
  60. mdcv.display_primaries_gx, mdcv.display_primaries_gy);
  61. color_volume_metadata.primary_b = gfx::ColorVolumeMetadata::Chromaticity(
  62. mdcv.display_primaries_bx, mdcv.display_primaries_by);
  63. color_volume_metadata.white_point = gfx::ColorVolumeMetadata::Chromaticity(
  64. mdcv.white_point_x, mdcv.white_point_y);
  65. color_volume_metadata.luminance_max = mdcv.max_display_mastering_luminance;
  66. color_volume_metadata.luminance_min = mdcv.min_display_mastering_luminance;
  67. return color_volume_metadata;
  68. }
  69. } // namespace
  70. MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
  71. bool has_sbr,
  72. bool has_flac)
  73. : state_(kWaitingForInit),
  74. moof_head_(0),
  75. mdat_tail_(0),
  76. highest_end_offset_(0),
  77. has_audio_(false),
  78. has_video_(false),
  79. audio_object_types_(audio_object_types),
  80. has_sbr_(has_sbr),
  81. has_flac_(has_flac),
  82. num_empty_samples_skipped_(0),
  83. num_invalid_conversions_(0),
  84. num_video_keyframe_mismatches_(0) {}
  85. MP4StreamParser::~MP4StreamParser() = default;
  86. void MP4StreamParser::Init(
  87. InitCB init_cb,
  88. const NewConfigCB& config_cb,
  89. const NewBuffersCB& new_buffers_cb,
  90. bool /* ignore_text_tracks */,
  91. const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
  92. const NewMediaSegmentCB& new_segment_cb,
  93. const EndMediaSegmentCB& end_of_segment_cb,
  94. MediaLog* media_log) {
  95. DCHECK_EQ(state_, kWaitingForInit);
  96. DCHECK(!init_cb_);
  97. DCHECK(init_cb);
  98. DCHECK(config_cb);
  99. DCHECK(new_buffers_cb);
  100. DCHECK(encrypted_media_init_data_cb);
  101. DCHECK(new_segment_cb);
  102. DCHECK(end_of_segment_cb);
  103. ChangeState(kParsingBoxes);
  104. init_cb_ = std::move(init_cb);
  105. config_cb_ = config_cb;
  106. new_buffers_cb_ = new_buffers_cb;
  107. encrypted_media_init_data_cb_ = encrypted_media_init_data_cb;
  108. new_segment_cb_ = new_segment_cb;
  109. end_of_segment_cb_ = end_of_segment_cb;
  110. media_log_ = media_log;
  111. }
  112. void MP4StreamParser::Reset() {
  113. queue_.Reset();
  114. runs_.reset();
  115. moof_head_ = 0;
  116. mdat_tail_ = 0;
  117. }
  118. void MP4StreamParser::Flush() {
  119. DCHECK_NE(state_, kWaitingForInit);
  120. Reset();
  121. ChangeState(kParsingBoxes);
  122. }
  123. bool MP4StreamParser::GetGenerateTimestampsFlag() const {
  124. return false;
  125. }
  126. bool MP4StreamParser::Parse(const uint8_t* buf, int size) {
  127. DCHECK_NE(state_, kWaitingForInit);
  128. if (state_ == kError)
  129. return false;
  130. queue_.Push(buf, size);
  131. BufferQueueMap buffers;
  132. // TODO(sandersd): Remove these bools. ParseResult replaced their purpose, but
  133. // this method needs to be refactored to complete that work.
  134. bool result = false;
  135. bool err = false;
  136. do {
  137. switch (state_) {
  138. case kWaitingForInit:
  139. case kError:
  140. NOTREACHED();
  141. return false;
  142. case kParsingBoxes: {
  143. ParseResult pr = ParseBox();
  144. result = pr == ParseResult::kOk;
  145. err = pr == ParseResult::kError;
  146. break;
  147. }
  148. case kWaitingForSampleData:
  149. result = HaveEnoughDataToEnqueueSamples();
  150. if (result)
  151. ChangeState(kEmittingSamples);
  152. break;
  153. case kEmittingSamples: {
  154. ParseResult pr = EnqueueSample(&buffers);
  155. result = pr == ParseResult::kOk;
  156. err = pr == ParseResult::kError;
  157. if (result) {
  158. int64_t max_clear = runs_->GetMaxClearOffset() + moof_head_;
  159. err = !ReadAndDiscardMDATsUntil(max_clear);
  160. }
  161. break;
  162. }
  163. }
  164. } while (result && !err);
  165. if (!err)
  166. err = !SendAndFlushSamples(&buffers);
  167. if (err) {
  168. DLOG(ERROR) << "Error while parsing MP4";
  169. moov_.reset();
  170. Reset();
  171. ChangeState(kError);
  172. return false;
  173. }
  174. return true;
  175. }
  176. ParseResult MP4StreamParser::ParseBox() {
  177. const uint8_t* buf;
  178. int size;
  179. queue_.Peek(&buf, &size);
  180. if (!size)
  181. return ParseResult::kNeedMoreData;
  182. std::unique_ptr<BoxReader> reader;
  183. ParseResult result =
  184. BoxReader::ReadTopLevelBox(buf, size, media_log_, &reader);
  185. if (result != ParseResult::kOk)
  186. return result;
  187. DCHECK(reader);
  188. if (reader->type() == FOURCC_MOOV) {
  189. if (!ParseMoov(reader.get()))
  190. return ParseResult::kError;
  191. } else if (reader->type() == FOURCC_MOOF) {
  192. moof_head_ = queue_.head();
  193. if (!ParseMoof(reader.get()))
  194. return ParseResult::kError;
  195. // Set up first mdat offset for ReadMDATsUntil().
  196. mdat_tail_ = queue_.head() + reader->box_size();
  197. // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
  198. // be located anywhere in the file, including inside the 'moof' itself.
  199. // (Since 'default-base-is-moof' is mandated, no data references can come
  200. // before the head of the 'moof', so keeping this box around is sufficient.)
  201. return ParseResult::kOk;
  202. } else {
  203. // TODO(wolenetz,chcunningham): Enforce more strict adherence to MSE byte
  204. // stream spec for ftyp and styp. See http://crbug.com/504514.
  205. DVLOG(2) << "Skipping top-level box: " << FourCCToString(reader->type());
  206. }
  207. queue_.Pop(reader->box_size());
  208. return ParseResult::kOk;
  209. }
  210. VideoTransformation MP4StreamParser::CalculateRotation(
  211. const TrackHeader& track,
  212. const MovieHeader& movie) {
  213. static_assert(kDisplayMatrixDimension == 9, "Display matrix must be 3x3");
  214. // 3x3 matrix: [ a b c ]
  215. // [ d e f ]
  216. // [ x y z ]
  217. int32_t rotation_matrix[kDisplayMatrixDimension] = {0};
  218. // Shift values for fixed point multiplications.
  219. const int32_t shifts[kDisplayMatrixHeight] = {16, 16, 30};
  220. // Matrix multiplication for
  221. // track.display_matrix * movie.display_matrix
  222. // with special consideration taken that entries a-f are 16.16 fixed point
  223. // decimals and x-z are 2.30 fixed point decimals.
  224. for (int i = 0; i < kDisplayMatrixWidth; i++) {
  225. for (int j = 0; j < kDisplayMatrixHeight; j++) {
  226. for (int e = 0; e < kDisplayMatrixHeight; e++) {
  227. rotation_matrix[i * kDisplayMatrixHeight + j] +=
  228. ((int64_t)track.display_matrix[i * kDisplayMatrixHeight + e] *
  229. movie.display_matrix[e * kDisplayMatrixHeight + j]) >>
  230. shifts[e];
  231. }
  232. }
  233. }
  234. int32_t rotation_only[4] = {rotation_matrix[0], rotation_matrix[1],
  235. rotation_matrix[3], rotation_matrix[4]};
  236. return VideoTransformation(rotation_only);
  237. }
  238. bool MP4StreamParser::ParseMoov(BoxReader* reader) {
  239. moov_ = std::make_unique<Movie>();
  240. RCHECK(moov_->Parse(reader));
  241. runs_.reset();
  242. audio_track_ids_.clear();
  243. video_track_ids_.clear();
  244. has_audio_ = false;
  245. has_video_ = false;
  246. std::unique_ptr<MediaTracks> media_tracks(new MediaTracks());
  247. AudioDecoderConfig audio_config;
  248. VideoDecoderConfig video_config;
  249. int detected_audio_track_count = 0;
  250. int detected_video_track_count = 0;
  251. int detected_text_track_count = 0;
  252. for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
  253. track != moov_->tracks.end(); ++track) {
  254. const SampleDescription& samp_descr =
  255. track->media.information.sample_table.description;
  256. // TODO(wolenetz): When codec reconfigurations are supported, detect and
  257. // send a codec reconfiguration for fragments using a sample description
  258. // index different from the previous one. See https://crbug.com/748250.
  259. size_t desc_idx = 0;
  260. for (size_t t = 0; t < moov_->extends.tracks.size(); t++) {
  261. const TrackExtends& trex = moov_->extends.tracks[t];
  262. if (trex.track_id == track->header.track_id) {
  263. desc_idx = trex.default_sample_description_index;
  264. break;
  265. }
  266. }
  267. RCHECK(desc_idx > 0);
  268. desc_idx -= 1; // BMFF descriptor index is one-based
  269. if (track->media.handler.type == kAudio) {
  270. detected_audio_track_count++;
  271. RCHECK(!samp_descr.audio_entries.empty());
  272. // It is not uncommon to find otherwise-valid files with incorrect sample
  273. // description indices, so we fail gracefully in that case.
  274. if (desc_idx >= samp_descr.audio_entries.size())
  275. desc_idx = 0;
  276. const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
  277. // For encrypted audio streams entry.format is FOURCC_ENCA and actual
  278. // format is in entry.sinf.format.format.
  279. FourCC audio_format = (entry.format == FOURCC_ENCA)
  280. ? entry.sinf.format.format
  281. : entry.format;
  282. if (audio_format != FOURCC_OPUS && audio_format != FOURCC_FLAC &&
  283. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  284. audio_format != FOURCC_AC3 && audio_format != FOURCC_EAC3 &&
  285. #endif
  286. #if BUILDFLAG(USE_PROPRIETARY_CODECS) && BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO)
  287. audio_format != FOURCC_DTSC && audio_format != FOURCC_DTSX &&
  288. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) &&
  289. // BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO)
  290. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  291. audio_format != FOURCC_MHM1 && audio_format != FOURCC_MHA1 &&
  292. #endif
  293. audio_format != FOURCC_MP4A) {
  294. MEDIA_LOG(ERROR, media_log_)
  295. << "Unsupported audio format 0x" << std::hex << entry.format
  296. << " in stsd box.";
  297. return false;
  298. }
  299. AudioCodec codec = AudioCodec::kUnknown;
  300. ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE;
  301. int sample_per_second = 0;
  302. int codec_delay_in_frames = 0;
  303. base::TimeDelta seek_preroll;
  304. std::vector<uint8_t> extra_data;
  305. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  306. AudioCodecProfile profile = AudioCodecProfile::kUnknown;
  307. std::vector<uint8_t> aac_extra_data;
  308. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  309. if (audio_format == FOURCC_OPUS) {
  310. codec = AudioCodec::kOpus;
  311. channel_layout = GuessChannelLayout(entry.dops.channel_count);
  312. sample_per_second = entry.dops.sample_rate;
  313. codec_delay_in_frames = entry.dops.codec_delay_in_frames;
  314. seek_preroll = entry.dops.seek_preroll;
  315. extra_data = entry.dops.extradata;
  316. } else if (audio_format == FOURCC_FLAC) {
  317. // FLAC-in-ISOBMFF does not use object type indication. |audio_format|
  318. // is sufficient for identifying FLAC codec.
  319. if (!has_flac_) {
  320. MEDIA_LOG(ERROR, media_log_) << "FLAC audio stream detected in MP4, "
  321. "mismatching what is specified in "
  322. "the mimetype.";
  323. return false;
  324. }
  325. codec = AudioCodec::kFLAC;
  326. channel_layout = GuessChannelLayout(entry.channelcount);
  327. sample_per_second = entry.samplerate;
  328. extra_data = entry.dfla.stream_info;
  329. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  330. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  331. } else if (audio_format == FOURCC_MHM1 || audio_format == FOURCC_MHA1) {
  332. codec = AudioCodec::kMpegHAudio;
  333. channel_layout = CHANNEL_LAYOUT_BITSTREAM;
  334. sample_per_second = entry.samplerate;
  335. extra_data = entry.dfla.stream_info;
  336. #endif
  337. } else {
  338. uint8_t audio_type = entry.esds.object_type;
  339. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  340. if (audio_type == kForbidden) {
  341. if (audio_format == FOURCC_AC3)
  342. audio_type = kAC3;
  343. if (audio_format == FOURCC_EAC3)
  344. audio_type = kEAC3;
  345. }
  346. #endif
  347. #if BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO)
  348. if (audio_type == kForbidden) {
  349. if (audio_format == FOURCC_DTSC)
  350. audio_type = kDTS;
  351. if (audio_format == FOURCC_DTSX)
  352. audio_type = kDTSX;
  353. }
  354. #endif
  355. DVLOG(1) << "audio_type 0x" << std::hex << static_cast<int>(audio_type);
  356. if (audio_object_types_.find(audio_type) == audio_object_types_.end()) {
  357. MEDIA_LOG(ERROR, media_log_)
  358. << "audio object type 0x" << std::hex
  359. << static_cast<int>(audio_type)
  360. << " does not match what is specified in the mimetype.";
  361. return false;
  362. }
  363. // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
  364. // supported MPEG2 AAC varients.
  365. if (ESDescriptor::IsAAC(audio_type)) {
  366. const AAC& aac = entry.esds.aac;
  367. codec = AudioCodec::kAAC;
  368. profile = aac.GetProfile();
  369. channel_layout = aac.GetChannelLayout(has_sbr_);
  370. sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_);
  371. // Set `aac_extra_data` on all platforms but only set `extra_data` on
  372. // Android. This is for backward compatibility until we have a better
  373. // solution. See crbug.com/1245123 for details.
  374. aac_extra_data = aac.codec_specific_data();
  375. #if BUILDFLAG(IS_ANDROID)
  376. extra_data = aac.codec_specific_data();
  377. #endif // BUILDFLAG(IS_ANDROID)
  378. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  379. } else if (audio_type == kAC3) {
  380. codec = AudioCodec::kAC3;
  381. channel_layout = GuessChannelLayout(entry.channelcount);
  382. sample_per_second = entry.samplerate;
  383. } else if (audio_type == kEAC3) {
  384. codec = AudioCodec::kEAC3;
  385. channel_layout = GuessChannelLayout(entry.channelcount);
  386. sample_per_second = entry.samplerate;
  387. #endif
  388. #if BUILDFLAG(ENABLE_PLATFORM_DTS_AUDIO)
  389. } else if (audio_type == kDTS) {
  390. codec = AudioCodec::kDTS;
  391. channel_layout = GuessChannelLayout(entry.channelcount);
  392. sample_per_second = entry.samplerate;
  393. } else if (audio_type == kDTSX) {
  394. // HDMI versions pre HDMI 2.0 can only transmit 8 raw PCM channels.
  395. // In the case of a 5_1_4 stream we downmix to 5_1.
  396. codec = AudioCodec::kDTSXP2;
  397. channel_layout = GuessChannelLayout(entry.channelcount);
  398. sample_per_second = entry.samplerate;
  399. #endif
  400. } else {
  401. MEDIA_LOG(ERROR, media_log_)
  402. << "Unsupported audio object type 0x" << std::hex
  403. << static_cast<int>(audio_type) << " in esds.";
  404. return false;
  405. }
  406. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  407. }
  408. SampleFormat sample_format;
  409. if (entry.samplesize == 8) {
  410. sample_format = kSampleFormatU8;
  411. } else if (entry.samplesize == 16) {
  412. sample_format = kSampleFormatS16;
  413. } else if (entry.samplesize == 24) {
  414. sample_format = kSampleFormatS24;
  415. } else if (entry.samplesize == 32) {
  416. sample_format = kSampleFormatS32;
  417. } else {
  418. LOG(ERROR) << "Unsupported sample size.";
  419. return false;
  420. }
  421. uint32_t audio_track_id = track->header.track_id;
  422. if (audio_track_ids_.find(audio_track_id) != audio_track_ids_.end()) {
  423. MEDIA_LOG(ERROR, media_log_)
  424. << "Audio track with track_id=" << audio_track_id
  425. << " already present.";
  426. return false;
  427. }
  428. bool is_track_encrypted = entry.sinf.info.track_encryption.is_encrypted;
  429. EncryptionScheme scheme = EncryptionScheme::kUnencrypted;
  430. if (is_track_encrypted) {
  431. scheme = GetEncryptionScheme(entry.sinf);
  432. if (scheme == EncryptionScheme::kUnencrypted)
  433. return false;
  434. }
  435. audio_config.Initialize(codec, sample_format, channel_layout,
  436. sample_per_second, extra_data, scheme,
  437. seek_preroll, codec_delay_in_frames);
  438. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  439. if (codec == AudioCodec::kAAC) {
  440. audio_config.disable_discard_decoder_delay();
  441. audio_config.set_profile(profile);
  442. audio_config.set_aac_extra_data(std::move(aac_extra_data));
  443. }
  444. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  445. DVLOG(1) << "audio_track_id=" << audio_track_id
  446. << " config=" << audio_config.AsHumanReadableString();
  447. if (!audio_config.IsValidConfig()) {
  448. MEDIA_LOG(ERROR, media_log_) << "Invalid audio decoder config: "
  449. << audio_config.AsHumanReadableString();
  450. return false;
  451. }
  452. has_audio_ = true;
  453. audio_track_ids_.insert(audio_track_id);
  454. const char* track_kind = (audio_track_ids_.size() == 1 ? "main" : "");
  455. media_tracks->AddAudioTrack(
  456. audio_config, audio_track_id, MediaTrack::Kind(track_kind),
  457. MediaTrack::Label(track->media.handler.name),
  458. MediaTrack::Language(track->media.header.language()));
  459. continue;
  460. }
  461. if (track->media.handler.type == kVideo) {
  462. detected_video_track_count++;
  463. RCHECK(!samp_descr.video_entries.empty());
  464. if (desc_idx >= samp_descr.video_entries.size())
  465. desc_idx = 0;
  466. const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
  467. if (!entry.IsFormatValid()) {
  468. MEDIA_LOG(ERROR, media_log_) << "Unsupported video format 0x"
  469. << std::hex << entry.format
  470. << " in stsd box.";
  471. return false;
  472. }
  473. // TODO(strobe): Recover correct crop box
  474. gfx::Size coded_size(entry.width, entry.height);
  475. gfx::Rect visible_rect(coded_size);
  476. // If PASP is available, use the coded size and PASP to calculate the
  477. // natural size. Otherwise, use the size in track header for natural size.
  478. VideoAspectRatio aspect_ratio;
  479. if (entry.pixel_aspect.h_spacing != 1 ||
  480. entry.pixel_aspect.v_spacing != 1) {
  481. aspect_ratio = VideoAspectRatio::PAR(entry.pixel_aspect.h_spacing,
  482. entry.pixel_aspect.v_spacing);
  483. } else if (track->header.width && track->header.height) {
  484. aspect_ratio =
  485. VideoAspectRatio::DAR(track->header.width, track->header.height);
  486. }
  487. gfx::Size natural_size = aspect_ratio.GetNaturalSize(visible_rect);
  488. uint32_t video_track_id = track->header.track_id;
  489. if (video_track_ids_.find(video_track_id) != video_track_ids_.end()) {
  490. MEDIA_LOG(ERROR, media_log_)
  491. << "Video track with track_id=" << video_track_id
  492. << " already present.";
  493. return false;
  494. }
  495. bool is_track_encrypted = entry.sinf.info.track_encryption.is_encrypted;
  496. EncryptionScheme scheme = EncryptionScheme::kUnencrypted;
  497. if (is_track_encrypted) {
  498. scheme = GetEncryptionScheme(entry.sinf);
  499. if (scheme == EncryptionScheme::kUnencrypted)
  500. return false;
  501. }
  502. video_config.Initialize(entry.video_codec, entry.video_codec_profile,
  503. VideoDecoderConfig::AlphaMode::kIsOpaque,
  504. VideoColorSpace::REC709(),
  505. CalculateRotation(track->header, moov_->header),
  506. coded_size, visible_rect, natural_size,
  507. // No decoder-specific buffer needed for AVC;
  508. // SPS/PPS are embedded in the video stream
  509. EmptyExtraData(), scheme);
  510. video_config.set_aspect_ratio(aspect_ratio);
  511. video_config.set_level(entry.video_codec_level);
  512. if (entry.video_color_space.IsSpecified())
  513. video_config.set_color_space_info(entry.video_color_space);
  514. if (entry.mastering_display_color_volume ||
  515. entry.content_light_level_information) {
  516. gfx::HDRMetadata hdr_metadata;
  517. if (entry.mastering_display_color_volume) {
  518. hdr_metadata.color_volume_metadata = ConvertMdcvToColorVolumeMetadata(
  519. *entry.mastering_display_color_volume);
  520. }
  521. if (entry.content_light_level_information) {
  522. hdr_metadata.max_content_light_level =
  523. entry.content_light_level_information->max_content_light_level;
  524. hdr_metadata.max_frame_average_light_level =
  525. entry.content_light_level_information
  526. ->max_pic_average_light_level;
  527. }
  528. video_config.set_hdr_metadata(hdr_metadata);
  529. }
  530. DVLOG(1) << "video_track_id=" << video_track_id
  531. << " config=" << video_config.AsHumanReadableString();
  532. if (!video_config.IsValidConfig()) {
  533. MEDIA_LOG(ERROR, media_log_) << "Invalid video decoder config: "
  534. << video_config.AsHumanReadableString();
  535. return false;
  536. }
  537. has_video_ = true;
  538. video_track_ids_.insert(video_track_id);
  539. auto track_kind =
  540. MediaTrack::Kind(video_track_ids_.size() == 1 ? "main" : "");
  541. media_tracks->AddVideoTrack(
  542. video_config, video_track_id, track_kind,
  543. MediaTrack::Label(track->media.handler.name),
  544. MediaTrack::Language(track->media.header.language()));
  545. continue;
  546. }
  547. // TODO(wolenetz): Investigate support in MSE and Chrome MSE for CEA 608/708
  548. // embedded caption data in video track. At time of init segment parsing, we
  549. // don't have this data (unless maybe by SourceBuffer's mimetype).
  550. // See https://crbug.com/597073
  551. if (track->media.handler.type == kText)
  552. detected_text_track_count++;
  553. }
  554. if (!moov_->pssh.empty())
  555. OnEncryptedMediaInitData(moov_->pssh);
  556. RCHECK(config_cb_.Run(std::move(media_tracks), TextTrackConfigMap()));
  557. StreamParser::InitParameters params(kInfiniteDuration);
  558. if (moov_->extends.header.fragment_duration > 0) {
  559. params.duration = TimeDeltaFromRational(
  560. moov_->extends.header.fragment_duration, moov_->header.timescale);
  561. if (params.duration == kNoTimestamp) {
  562. MEDIA_LOG(ERROR, media_log_) << "Fragment duration exceeds representable "
  563. << "limit";
  564. return false;
  565. }
  566. params.liveness = StreamLiveness::kRecorded;
  567. } else if (moov_->header.duration > 0 &&
  568. ((moov_->header.version == 0 &&
  569. moov_->header.duration !=
  570. std::numeric_limits<uint32_t>::max()) ||
  571. (moov_->header.version == 1 &&
  572. moov_->header.duration !=
  573. std::numeric_limits<uint64_t>::max()))) {
  574. // In ISO/IEC 14496-12:2012, 8.2.2.3: "If the duration cannot be determined
  575. // then duration is set to all 1s."
  576. // The duration field is either 32-bit or 64-bit depending on the version in
  577. // MovieHeaderBox. We interpret not 0 and not all 1's here as "known
  578. // duration".
  579. params.duration =
  580. TimeDeltaFromRational(moov_->header.duration, moov_->header.timescale);
  581. if (params.duration == kNoTimestamp) {
  582. MEDIA_LOG(ERROR, media_log_) << "Movie duration exceeds representable "
  583. << "limit";
  584. return false;
  585. }
  586. params.liveness = StreamLiveness::kRecorded;
  587. } else {
  588. // In ISO/IEC 14496-12:2005(E), 8.30.2: ".. If an MP4 file is created in
  589. // real-time, such as used in live streaming, it is not likely that the
  590. // fragment_duration is known in advance and this (mehd) box may be
  591. // omitted."
  592. // We have an unknown duration (neither any mvex fragment_duration nor moov
  593. // duration value indicated a known duration, above.)
  594. // TODO(wolenetz): Investigate gating liveness detection on timeline_offset
  595. // when it's populated. See http://crbug.com/312699
  596. params.liveness = StreamLiveness::kLive;
  597. }
  598. DVLOG(1) << "liveness: " << GetStreamLivenessName(params.liveness);
  599. if (init_cb_) {
  600. params.detected_audio_track_count = detected_audio_track_count;
  601. params.detected_video_track_count = detected_video_track_count;
  602. params.detected_text_track_count = detected_text_track_count;
  603. std::move(init_cb_).Run(params);
  604. }
  605. return true;
  606. }
  607. bool MP4StreamParser::ParseMoof(BoxReader* reader) {
  608. RCHECK(moov_.get()); // Must already have initialization segment
  609. MovieFragment moof;
  610. RCHECK(moof.Parse(reader));
  611. if (!runs_)
  612. runs_ = std::make_unique<TrackRunIterator>(moov_.get(), media_log_);
  613. RCHECK(runs_->Init(moof));
  614. RCHECK(ComputeHighestEndOffset(moof));
  615. if (!moof.pssh.empty())
  616. OnEncryptedMediaInitData(moof.pssh);
  617. new_segment_cb_.Run();
  618. ChangeState(kWaitingForSampleData);
  619. return true;
  620. }
  621. void MP4StreamParser::OnEncryptedMediaInitData(
  622. const std::vector<ProtectionSystemSpecificHeader>& headers) {
  623. // TODO(strobe): ensure that the value of init_data (all PSSH headers
  624. // concatenated in arbitrary order) matches the EME spec.
  625. // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
  626. size_t total_size = 0;
  627. for (size_t i = 0; i < headers.size(); i++)
  628. total_size += headers[i].raw_box.size();
  629. std::vector<uint8_t> init_data(total_size);
  630. size_t pos = 0;
  631. for (size_t i = 0; i < headers.size(); i++) {
  632. memcpy(&init_data[pos], &headers[i].raw_box[0],
  633. headers[i].raw_box.size());
  634. pos += headers[i].raw_box.size();
  635. }
  636. encrypted_media_init_data_cb_.Run(EmeInitDataType::CENC, init_data);
  637. }
  638. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  639. bool MP4StreamParser::PrepareAACBuffer(
  640. const AAC& aac_config,
  641. std::vector<uint8_t>* frame_buf,
  642. std::vector<SubsampleEntry>* subsamples) const {
  643. // Append an ADTS header to every audio sample.
  644. RCHECK(aac_config.ConvertEsdsToADTS(frame_buf));
  645. // As above, adjust subsample information to account for the headers. AAC is
  646. // not required to use subsample encryption, so we may need to add an entry.
  647. if (subsamples->empty()) {
  648. subsamples->push_back(SubsampleEntry(
  649. kADTSHeaderMinSize, frame_buf->size() - kADTSHeaderMinSize));
  650. } else {
  651. (*subsamples)[0].clear_bytes += kADTSHeaderMinSize;
  652. }
  653. return true;
  654. }
  655. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  656. ParseResult MP4StreamParser::EnqueueSample(BufferQueueMap* buffers) {
  657. DCHECK_EQ(state_, kEmittingSamples);
  658. if (!runs_->IsRunValid()) {
  659. // Flush any buffers we've gotten in this chunk so that buffers don't
  660. // cross |new_segment_cb_| calls
  661. if (!SendAndFlushSamples(buffers))
  662. return ParseResult::kError;
  663. // Remain in kEmittingSamples state, discarding data, until the end of
  664. // the current 'mdat' box has been appended to the queue.
  665. // TODO(sandersd): As I understand it, this Trim() will always succeed,
  666. // since |mdat_tail_| is never outside of the queue. It's also plausible
  667. // that this Trim() is always a no-op, but perhaps if all runs are empty
  668. // this still does something?
  669. if (!queue_.Trim(mdat_tail_))
  670. return ParseResult::kNeedMoreData;
  671. ChangeState(kParsingBoxes);
  672. end_of_segment_cb_.Run();
  673. return ParseResult::kOk;
  674. }
  675. if (!runs_->IsSampleValid()) {
  676. if (!runs_->AdvanceRun())
  677. return ParseResult::kError;
  678. return ParseResult::kOk;
  679. }
  680. const uint8_t* buf;
  681. int buf_size;
  682. queue_.Peek(&buf, &buf_size);
  683. if (!buf_size)
  684. return ParseResult::kNeedMoreData;
  685. bool audio =
  686. audio_track_ids_.find(runs_->track_id()) != audio_track_ids_.end();
  687. bool video =
  688. video_track_ids_.find(runs_->track_id()) != video_track_ids_.end();
  689. // Skip this entire track if it's not one we're interested in
  690. if (!audio && !video) {
  691. if (!runs_->AdvanceRun())
  692. return ParseResult::kError;
  693. return ParseResult::kOk;
  694. }
  695. // Attempt to cache the auxiliary information first. Aux info is usually
  696. // placed in a contiguous block before the sample data, rather than being
  697. // interleaved. If we didn't cache it, this would require that we retain the
  698. // start of the segment buffer while reading samples. Aux info is typically
  699. // quite small compared to sample data, so this pattern is useful on
  700. // memory-constrained devices where the source buffer consumes a substantial
  701. // portion of the total system memory.
  702. if (runs_->AuxInfoNeedsToBeCached()) {
  703. queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size);
  704. if (buf_size < runs_->aux_info_size())
  705. return ParseResult::kNeedMoreData;
  706. if (!runs_->CacheAuxInfo(buf, buf_size))
  707. return ParseResult::kError;
  708. return ParseResult::kOk;
  709. }
  710. queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size);
  711. if (runs_->sample_size() >
  712. static_cast<uint32_t>(std::numeric_limits<int>::max())) {
  713. MEDIA_LOG(ERROR, media_log_) << "Sample size is too large";
  714. return ParseResult::kError;
  715. }
  716. int sample_size = base::checked_cast<int>(runs_->sample_size());
  717. if (buf_size < sample_size)
  718. return ParseResult::kNeedMoreData;
  719. if (sample_size == 0) {
  720. // Generally not expected, but spec allows it. Code below this block assumes
  721. // the current sample is not empty.
  722. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_empty_samples_skipped_,
  723. kMaxEmptySampleLogs)
  724. << "Skipping 'trun' sample with size of 0.";
  725. if (!runs_->AdvanceSample())
  726. return ParseResult::kError;
  727. return ParseResult::kOk;
  728. }
  729. std::unique_ptr<DecryptConfig> decrypt_config;
  730. std::vector<SubsampleEntry> subsamples;
  731. if (runs_->is_encrypted()) {
  732. decrypt_config = runs_->GetDecryptConfig();
  733. if (!decrypt_config)
  734. return ParseResult::kError;
  735. subsamples = decrypt_config->subsamples();
  736. }
  737. // This may change if analysis results indicate runs_->is_keyframe() is
  738. // opposite of what the coded frame contains.
  739. bool is_keyframe = runs_->is_keyframe();
  740. std::vector<uint8_t> frame_buf(buf, buf + sample_size);
  741. if (video) {
  742. if (runs_->video_description().video_codec == VideoCodec::kH264 ||
  743. runs_->video_description().video_codec == VideoCodec::kHEVC ||
  744. runs_->video_description().video_codec == VideoCodec::kDolbyVision) {
  745. DCHECK(runs_->video_description().frame_bitstream_converter);
  746. BitstreamConverter::AnalysisResult analysis;
  747. if (!runs_->video_description()
  748. .frame_bitstream_converter->ConvertAndAnalyzeFrame(
  749. &frame_buf, is_keyframe, &subsamples, &analysis)) {
  750. MEDIA_LOG(ERROR, media_log_)
  751. << "Failed to prepare video sample for decode";
  752. return ParseResult::kError;
  753. }
  754. // If conformance analysis was not actually performed, assume the frame is
  755. // conformant. If it was performed and found to be non-conformant, log
  756. // it.
  757. if (!analysis.is_conformant.value_or(true)) {
  758. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_invalid_conversions_,
  759. kMaxInvalidConversionLogs)
  760. << "Prepared video sample is not conformant";
  761. }
  762. // Use |analysis.is_keyframe|, if it was actually determined, for logging
  763. // if the analysis mismatches the container's keyframe metadata for
  764. // |frame_buf|.
  765. if (analysis.is_keyframe.has_value() &&
  766. is_keyframe != analysis.is_keyframe.value()) {
  767. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_video_keyframe_mismatches_,
  768. kMaxVideoKeyframeMismatchLogs)
  769. << "ISO-BMFF container metadata for video frame indicates that the "
  770. "frame is "
  771. << (is_keyframe ? "" : "not ")
  772. << "a keyframe, but the video frame contents indicate the "
  773. "opposite.";
  774. // As of September 2018, it appears that all of Edge, Firefox, Safari
  775. // work with content that marks non-avc-keyframes as a keyframe in the
  776. // container. Encoders/muxers/old streams still exist that produce
  777. // all-keyframe mp4 video tracks, though many of the coded frames are
  778. // not keyframes (likely workaround due to the impact on low-latency
  779. // live streams until https://crbug.com/229412 was fixed). We'll trust
  780. // the AVC frame's keyframe-ness over the mp4 container's metadata if
  781. // they mismatch. If other out-of-order codecs in mp4 (e.g. HEVC, DV)
  782. // implement keyframe analysis in their frame_bitstream_converter, we'll
  783. // similarly trust that analysis instead of the mp4.
  784. is_keyframe = analysis.is_keyframe.value();
  785. }
  786. }
  787. }
  788. if (audio) {
  789. if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type)) {
  790. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  791. if (!PrepareAACBuffer(runs_->audio_description().esds.aac, &frame_buf,
  792. &subsamples)) {
  793. MEDIA_LOG(ERROR, media_log_)
  794. << "Failed to prepare AAC sample for decode";
  795. return ParseResult::kError;
  796. }
  797. #else
  798. return ParseResult::kError;
  799. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  800. }
  801. }
  802. if (decrypt_config) {
  803. if (!subsamples.empty()) {
  804. // Create a new config with the updated subsamples.
  805. decrypt_config = std::make_unique<DecryptConfig>(
  806. decrypt_config->encryption_scheme(), decrypt_config->key_id(),
  807. decrypt_config->iv(), subsamples,
  808. decrypt_config->encryption_pattern());
  809. }
  810. // else, use the existing config.
  811. }
  812. StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO :
  813. DemuxerStream::VIDEO;
  814. scoped_refptr<StreamParserBuffer> stream_buf =
  815. StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(), is_keyframe,
  816. buffer_type, runs_->track_id());
  817. if (decrypt_config)
  818. stream_buf->set_decrypt_config(std::move(decrypt_config));
  819. if (runs_->duration() != kNoTimestamp) {
  820. stream_buf->set_duration(runs_->duration());
  821. } else {
  822. MEDIA_LOG(ERROR, media_log_) << "Frame duration exceeds representable "
  823. << "limit";
  824. return ParseResult::kError;
  825. }
  826. if (runs_->cts() != kNoTimestamp) {
  827. stream_buf->set_timestamp(runs_->cts());
  828. } else {
  829. MEDIA_LOG(ERROR, media_log_) << "Frame PTS exceeds representable limit";
  830. return ParseResult::kError;
  831. }
  832. if (runs_->dts() != kNoDecodeTimestamp) {
  833. stream_buf->SetDecodeTimestamp(runs_->dts());
  834. } else {
  835. MEDIA_LOG(ERROR, media_log_) << "Frame DTS exceeds representable limit";
  836. return ParseResult::kError;
  837. }
  838. DVLOG(3) << "Emit " << (audio ? "audio" : "video") << " frame: "
  839. << " track_id=" << runs_->track_id() << ", key=" << is_keyframe
  840. << ", dur=" << runs_->duration().InMilliseconds()
  841. << ", dts=" << runs_->dts().InMilliseconds()
  842. << ", cts=" << runs_->cts().InMilliseconds()
  843. << ", size=" << sample_size;
  844. (*buffers)[runs_->track_id()].push_back(stream_buf);
  845. if (!runs_->AdvanceSample())
  846. return ParseResult::kError;
  847. return ParseResult::kOk;
  848. }
  849. bool MP4StreamParser::SendAndFlushSamples(BufferQueueMap* buffers) {
  850. if (buffers->empty())
  851. return true;
  852. bool success = new_buffers_cb_.Run(*buffers);
  853. buffers->clear();
  854. return success;
  855. }
  856. bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64_t max_clear_offset) {
  857. ParseResult result = ParseResult::kOk;
  858. int64_t upper_bound = std::min(max_clear_offset, queue_.tail());
  859. while (mdat_tail_ < upper_bound) {
  860. const uint8_t* buf = NULL;
  861. int size = 0;
  862. queue_.PeekAt(mdat_tail_, &buf, &size);
  863. FourCC type;
  864. size_t box_sz;
  865. result = BoxReader::StartTopLevelBox(buf, size, media_log_, &type, &box_sz);
  866. if (result != ParseResult::kOk)
  867. break;
  868. if (type != FOURCC_MDAT) {
  869. MEDIA_LOG(DEBUG, media_log_)
  870. << "Unexpected box type while parsing MDATs: "
  871. << FourCCToString(type);
  872. }
  873. // TODO(chcunningham): Fix mdat_tail_ and ByteQueue classes to use size_t.
  874. // TODO(sandersd): The whole |mdat_tail_| mechanism appears to be pointless
  875. // because StartTopLevelBox() only succeeds for complete boxes. Either
  876. // remove |mdat_tail_| throughout this class or implement the ability to
  877. // discard partial mdats.
  878. mdat_tail_ += base::checked_cast<int64_t>(box_sz);
  879. }
  880. queue_.Trim(std::min(mdat_tail_, upper_bound));
  881. return result != ParseResult::kError;
  882. }
  883. void MP4StreamParser::ChangeState(State new_state) {
  884. DVLOG(2) << "Changing state: " << new_state;
  885. state_ = new_state;
  886. }
  887. bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
  888. DCHECK_EQ(state_, kWaitingForSampleData);
  889. // For muxed content, make sure we have data up to |highest_end_offset_|
  890. // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
  891. // data and allow per sample offset checks to meter sample enqueuing.
  892. // TODO(acolwell): Fix trun box handling so we don't have to special case
  893. // muxed content.
  894. return !(has_audio_ && has_video_ &&
  895. queue_.tail() < highest_end_offset_ + moof_head_);
  896. }
  897. bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment& moof) {
  898. highest_end_offset_ = 0;
  899. TrackRunIterator runs(moov_.get(), media_log_);
  900. RCHECK(runs.Init(moof));
  901. while (runs.IsRunValid()) {
  902. int64_t aux_info_end_offset = runs.aux_info_offset() + runs.aux_info_size();
  903. if (aux_info_end_offset > highest_end_offset_)
  904. highest_end_offset_ = aux_info_end_offset;
  905. while (runs.IsSampleValid()) {
  906. int64_t sample_end_offset = runs.sample_offset() + runs.sample_size();
  907. if (sample_end_offset > highest_end_offset_)
  908. highest_end_offset_ = sample_end_offset;
  909. if (!runs.AdvanceSample())
  910. return false;
  911. }
  912. if (!runs.AdvanceRun())
  913. return false;
  914. }
  915. return true;
  916. }
  917. } // namespace mp4
  918. } // namespace media