ffmpeg_common.cc 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/ffmpeg/ffmpeg_common.h"
  5. #include "base/hash/sha1.h"
  6. #include "base/logging.h"
  7. #include "base/strings/string_number_conversions.h"
  8. #include "base/strings/string_split.h"
  9. #include "base/strings/string_util.h"
  10. #include "build/build_config.h"
  11. #include "media/base/audio_decoder_config.h"
  12. #include "media/base/decoder_buffer.h"
  13. #include "media/base/encryption_scheme.h"
  14. #include "media/base/media_util.h"
  15. #include "media/base/video_aspect_ratio.h"
  16. #include "media/base/video_decoder_config.h"
  17. #include "media/base/video_util.h"
  18. #include "media/formats/mp4/box_definitions.h"
  19. #include "media/media_buildflags.h"
  20. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  21. #include "media/formats/mp4/aac.h"
  22. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  23. #include "media/formats/mp4/hevc.h"
  24. #endif
  25. #endif
  26. namespace media {
  27. namespace {
  28. EncryptionScheme GetEncryptionScheme(const AVStream* stream) {
  29. AVDictionaryEntry* key =
  30. av_dict_get(stream->metadata, "enc_key_id", nullptr, 0);
  31. return key ? EncryptionScheme::kCenc : EncryptionScheme::kUnencrypted;
  32. }
  33. } // namespace
  34. // Why AV_INPUT_BUFFER_PADDING_SIZE? FFmpeg assumes all input buffers are
  35. // padded. Check here to ensure FFmpeg only receives data padded to its
  36. // specifications.
  37. static_assert(DecoderBuffer::kPaddingSize >= AV_INPUT_BUFFER_PADDING_SIZE,
  38. "DecoderBuffer padding size does not fit ffmpeg requirement");
  39. // Alignment requirement by FFmpeg for input and output buffers. This need to
  40. // be updated to match FFmpeg when it changes.
  41. #if defined(ARCH_CPU_ARM_FAMILY)
  42. static const int kFFmpegBufferAddressAlignment = 16;
  43. #else
  44. static const int kFFmpegBufferAddressAlignment = 32;
  45. #endif
  46. // Check here to ensure FFmpeg only receives data aligned to its specifications.
  47. static_assert(
  48. DecoderBuffer::kAlignmentSize >= kFFmpegBufferAddressAlignment &&
  49. DecoderBuffer::kAlignmentSize % kFFmpegBufferAddressAlignment == 0,
  50. "DecoderBuffer alignment size does not fit ffmpeg requirement");
  51. // Allows faster SIMD YUV convert. Also, FFmpeg overreads/-writes occasionally.
  52. // See video_get_buffer() in libavcodec/utils.c.
  53. static const int kFFmpegOutputBufferPaddingSize = 16;
  54. static_assert(VideoFrame::kFrameSizePadding >= kFFmpegOutputBufferPaddingSize,
  55. "VideoFrame padding size does not fit ffmpeg requirement");
  56. static_assert(
  57. VideoFrame::kFrameAddressAlignment >= kFFmpegBufferAddressAlignment &&
  58. VideoFrame::kFrameAddressAlignment % kFFmpegBufferAddressAlignment == 0,
  59. "VideoFrame frame address alignment does not fit ffmpeg requirement");
  60. static const AVRational kMicrosBase = { 1, base::Time::kMicrosecondsPerSecond };
  61. base::TimeDelta ConvertFromTimeBase(const AVRational& time_base,
  62. int64_t timestamp) {
  63. int64_t microseconds = av_rescale_q(timestamp, time_base, kMicrosBase);
  64. return base::Microseconds(microseconds);
  65. }
  66. int64_t ConvertToTimeBase(const AVRational& time_base,
  67. const base::TimeDelta& timestamp) {
  68. return av_rescale_q(timestamp.InMicroseconds(), kMicrosBase, time_base);
  69. }
  70. AudioCodec CodecIDToAudioCodec(AVCodecID codec_id) {
  71. switch (codec_id) {
  72. case AV_CODEC_ID_AAC:
  73. return AudioCodec::kAAC;
  74. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  75. case AV_CODEC_ID_AC3:
  76. return AudioCodec::kAC3;
  77. case AV_CODEC_ID_EAC3:
  78. return AudioCodec::kEAC3;
  79. #endif
  80. case AV_CODEC_ID_MP3:
  81. return AudioCodec::kMP3;
  82. case AV_CODEC_ID_VORBIS:
  83. return AudioCodec::kVorbis;
  84. case AV_CODEC_ID_PCM_U8:
  85. case AV_CODEC_ID_PCM_S16LE:
  86. case AV_CODEC_ID_PCM_S24LE:
  87. case AV_CODEC_ID_PCM_S32LE:
  88. case AV_CODEC_ID_PCM_F32LE:
  89. return AudioCodec::kPCM;
  90. case AV_CODEC_ID_PCM_S16BE:
  91. return AudioCodec::kPCM_S16BE;
  92. case AV_CODEC_ID_PCM_S24BE:
  93. return AudioCodec::kPCM_S24BE;
  94. case AV_CODEC_ID_FLAC:
  95. return AudioCodec::kFLAC;
  96. case AV_CODEC_ID_AMR_NB:
  97. return AudioCodec::kAMR_NB;
  98. case AV_CODEC_ID_AMR_WB:
  99. return AudioCodec::kAMR_WB;
  100. case AV_CODEC_ID_GSM_MS:
  101. return AudioCodec::kGSM_MS;
  102. case AV_CODEC_ID_PCM_ALAW:
  103. return AudioCodec::kPCM_ALAW;
  104. case AV_CODEC_ID_PCM_MULAW:
  105. return AudioCodec::kPCM_MULAW;
  106. case AV_CODEC_ID_OPUS:
  107. return AudioCodec::kOpus;
  108. case AV_CODEC_ID_ALAC:
  109. return AudioCodec::kALAC;
  110. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  111. case AV_CODEC_ID_MPEGH_3D_AUDIO:
  112. return AudioCodec::kMpegHAudio;
  113. #endif
  114. default:
  115. DVLOG(1) << "Unknown audio CodecID: " << codec_id;
  116. }
  117. return AudioCodec::kUnknown;
  118. }
  119. AVCodecID AudioCodecToCodecID(AudioCodec audio_codec,
  120. SampleFormat sample_format) {
  121. switch (audio_codec) {
  122. case AudioCodec::kAAC:
  123. return AV_CODEC_ID_AAC;
  124. case AudioCodec::kALAC:
  125. return AV_CODEC_ID_ALAC;
  126. case AudioCodec::kMP3:
  127. return AV_CODEC_ID_MP3;
  128. case AudioCodec::kPCM:
  129. switch (sample_format) {
  130. case kSampleFormatU8:
  131. return AV_CODEC_ID_PCM_U8;
  132. case kSampleFormatS16:
  133. return AV_CODEC_ID_PCM_S16LE;
  134. case kSampleFormatS24:
  135. return AV_CODEC_ID_PCM_S24LE;
  136. case kSampleFormatS32:
  137. return AV_CODEC_ID_PCM_S32LE;
  138. case kSampleFormatF32:
  139. return AV_CODEC_ID_PCM_F32LE;
  140. default:
  141. DVLOG(1) << "Unsupported sample format: " << sample_format;
  142. }
  143. break;
  144. case AudioCodec::kPCM_S16BE:
  145. return AV_CODEC_ID_PCM_S16BE;
  146. case AudioCodec::kPCM_S24BE:
  147. return AV_CODEC_ID_PCM_S24BE;
  148. case AudioCodec::kVorbis:
  149. return AV_CODEC_ID_VORBIS;
  150. case AudioCodec::kFLAC:
  151. return AV_CODEC_ID_FLAC;
  152. case AudioCodec::kAMR_NB:
  153. return AV_CODEC_ID_AMR_NB;
  154. case AudioCodec::kAMR_WB:
  155. return AV_CODEC_ID_AMR_WB;
  156. case AudioCodec::kGSM_MS:
  157. return AV_CODEC_ID_GSM_MS;
  158. case AudioCodec::kPCM_ALAW:
  159. return AV_CODEC_ID_PCM_ALAW;
  160. case AudioCodec::kPCM_MULAW:
  161. return AV_CODEC_ID_PCM_MULAW;
  162. case AudioCodec::kOpus:
  163. return AV_CODEC_ID_OPUS;
  164. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  165. case AudioCodec::kMpegHAudio:
  166. return AV_CODEC_ID_MPEGH_3D_AUDIO;
  167. #endif
  168. default:
  169. DVLOG(1) << "Unknown AudioCodec: " << audio_codec;
  170. }
  171. return AV_CODEC_ID_NONE;
  172. }
  173. // Converts an FFmpeg video codec ID into its corresponding supported codec id.
  174. static VideoCodec CodecIDToVideoCodec(AVCodecID codec_id) {
  175. switch (codec_id) {
  176. case AV_CODEC_ID_H264:
  177. return VideoCodec::kH264;
  178. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  179. case AV_CODEC_ID_HEVC:
  180. return VideoCodec::kHEVC;
  181. #endif
  182. case AV_CODEC_ID_THEORA:
  183. return VideoCodec::kTheora;
  184. case AV_CODEC_ID_MPEG4:
  185. return VideoCodec::kMPEG4;
  186. case AV_CODEC_ID_VP8:
  187. return VideoCodec::kVP8;
  188. case AV_CODEC_ID_VP9:
  189. return VideoCodec::kVP9;
  190. case AV_CODEC_ID_AV1:
  191. return VideoCodec::kAV1;
  192. default:
  193. DVLOG(1) << "Unknown video CodecID: " << codec_id;
  194. }
  195. return VideoCodec::kUnknown;
  196. }
  197. AVCodecID VideoCodecToCodecID(VideoCodec video_codec) {
  198. switch (video_codec) {
  199. case VideoCodec::kH264:
  200. return AV_CODEC_ID_H264;
  201. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  202. case VideoCodec::kHEVC:
  203. return AV_CODEC_ID_HEVC;
  204. #endif
  205. case VideoCodec::kTheora:
  206. return AV_CODEC_ID_THEORA;
  207. case VideoCodec::kMPEG4:
  208. return AV_CODEC_ID_MPEG4;
  209. case VideoCodec::kVP8:
  210. return AV_CODEC_ID_VP8;
  211. case VideoCodec::kVP9:
  212. return AV_CODEC_ID_VP9;
  213. case VideoCodec::kAV1:
  214. return AV_CODEC_ID_AV1;
  215. default:
  216. DVLOG(1) << "Unknown VideoCodec: " << video_codec;
  217. }
  218. return AV_CODEC_ID_NONE;
  219. }
  220. static VideoCodecProfile ProfileIDToVideoCodecProfile(int profile) {
  221. // Clear out the CONSTRAINED & INTRA flags which are strict subsets of the
  222. // corresponding profiles with which they're used.
  223. profile &= ~FF_PROFILE_H264_CONSTRAINED;
  224. profile &= ~FF_PROFILE_H264_INTRA;
  225. switch (profile) {
  226. case FF_PROFILE_H264_BASELINE:
  227. return H264PROFILE_BASELINE;
  228. case FF_PROFILE_H264_MAIN:
  229. return H264PROFILE_MAIN;
  230. case FF_PROFILE_H264_EXTENDED:
  231. return H264PROFILE_EXTENDED;
  232. case FF_PROFILE_H264_HIGH:
  233. return H264PROFILE_HIGH;
  234. case FF_PROFILE_H264_HIGH_10:
  235. return H264PROFILE_HIGH10PROFILE;
  236. case FF_PROFILE_H264_HIGH_422:
  237. return H264PROFILE_HIGH422PROFILE;
  238. case FF_PROFILE_H264_HIGH_444_PREDICTIVE:
  239. return H264PROFILE_HIGH444PREDICTIVEPROFILE;
  240. default:
  241. DVLOG(1) << "Unknown profile id: " << profile;
  242. }
  243. return VIDEO_CODEC_PROFILE_UNKNOWN;
  244. }
  245. static int VideoCodecProfileToProfileID(VideoCodecProfile profile) {
  246. switch (profile) {
  247. case H264PROFILE_BASELINE:
  248. return FF_PROFILE_H264_BASELINE;
  249. case H264PROFILE_MAIN:
  250. return FF_PROFILE_H264_MAIN;
  251. case H264PROFILE_EXTENDED:
  252. return FF_PROFILE_H264_EXTENDED;
  253. case H264PROFILE_HIGH:
  254. return FF_PROFILE_H264_HIGH;
  255. case H264PROFILE_HIGH10PROFILE:
  256. return FF_PROFILE_H264_HIGH_10;
  257. case H264PROFILE_HIGH422PROFILE:
  258. return FF_PROFILE_H264_HIGH_422;
  259. case H264PROFILE_HIGH444PREDICTIVEPROFILE:
  260. return FF_PROFILE_H264_HIGH_444_PREDICTIVE;
  261. default:
  262. DVLOG(1) << "Unknown VideoCodecProfile: " << profile;
  263. }
  264. return FF_PROFILE_UNKNOWN;
  265. }
  266. SampleFormat AVSampleFormatToSampleFormat(AVSampleFormat sample_format,
  267. AVCodecID codec_id) {
  268. switch (sample_format) {
  269. case AV_SAMPLE_FMT_U8:
  270. return kSampleFormatU8;
  271. case AV_SAMPLE_FMT_S16:
  272. return kSampleFormatS16;
  273. case AV_SAMPLE_FMT_S32:
  274. if (codec_id == AV_CODEC_ID_PCM_S24LE)
  275. return kSampleFormatS24;
  276. else
  277. return kSampleFormatS32;
  278. case AV_SAMPLE_FMT_FLT:
  279. return kSampleFormatF32;
  280. case AV_SAMPLE_FMT_S16P:
  281. return kSampleFormatPlanarS16;
  282. case AV_SAMPLE_FMT_S32P:
  283. return kSampleFormatPlanarS32;
  284. case AV_SAMPLE_FMT_FLTP:
  285. return kSampleFormatPlanarF32;
  286. default:
  287. DVLOG(1) << "Unknown AVSampleFormat: " << sample_format;
  288. }
  289. return kUnknownSampleFormat;
  290. }
  291. static AVSampleFormat SampleFormatToAVSampleFormat(SampleFormat sample_format) {
  292. switch (sample_format) {
  293. case kSampleFormatU8:
  294. return AV_SAMPLE_FMT_U8;
  295. case kSampleFormatS16:
  296. return AV_SAMPLE_FMT_S16;
  297. // pcm_s24le is treated as a codec with sample format s32 in ffmpeg
  298. case kSampleFormatS24:
  299. case kSampleFormatS32:
  300. return AV_SAMPLE_FMT_S32;
  301. case kSampleFormatF32:
  302. return AV_SAMPLE_FMT_FLT;
  303. case kSampleFormatPlanarS16:
  304. return AV_SAMPLE_FMT_S16P;
  305. case kSampleFormatPlanarF32:
  306. return AV_SAMPLE_FMT_FLTP;
  307. default:
  308. DVLOG(1) << "Unknown SampleFormat: " << sample_format;
  309. }
  310. return AV_SAMPLE_FMT_NONE;
  311. }
  312. bool AVCodecContextToAudioDecoderConfig(const AVCodecContext* codec_context,
  313. EncryptionScheme encryption_scheme,
  314. AudioDecoderConfig* config) {
  315. DCHECK_EQ(codec_context->codec_type, AVMEDIA_TYPE_AUDIO);
  316. AudioCodec codec = CodecIDToAudioCodec(codec_context->codec_id);
  317. SampleFormat sample_format = AVSampleFormatToSampleFormat(
  318. codec_context->sample_fmt, codec_context->codec_id);
  319. ChannelLayout channel_layout =
  320. codec_context->ch_layout.nb_channels > 8
  321. ? CHANNEL_LAYOUT_DISCRETE
  322. : ChannelLayoutToChromeChannelLayout(
  323. codec_context->ch_layout.u.mask,
  324. codec_context->ch_layout.nb_channels);
  325. switch (codec) {
  326. // For AC3/EAC3 we enable only demuxing, but not decoding, so FFmpeg does
  327. // not fill |sample_fmt|.
  328. case AudioCodec::kAC3:
  329. case AudioCodec::kEAC3:
  330. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  331. // The spec for AC3/EAC3 audio is ETSI TS 102 366. According to sections
  332. // F.3.1 and F.5.1 in that spec the sample_format for AC3/EAC3 must be 16.
  333. sample_format = kSampleFormatS16;
  334. #else
  335. NOTREACHED();
  336. #endif
  337. break;
  338. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  339. case AudioCodec::kMpegHAudio:
  340. channel_layout = CHANNEL_LAYOUT_BITSTREAM;
  341. sample_format = kSampleFormatMpegHAudio;
  342. break;
  343. #endif
  344. default:
  345. break;
  346. }
  347. base::TimeDelta seek_preroll;
  348. if (codec_context->seek_preroll > 0) {
  349. seek_preroll = base::Microseconds(codec_context->seek_preroll * 1000000.0 /
  350. codec_context->sample_rate);
  351. }
  352. // AVStream occasionally has invalid extra data. See http://crbug.com/517163
  353. if ((codec_context->extradata_size == 0) !=
  354. (codec_context->extradata == nullptr)) {
  355. LOG(ERROR) << __func__
  356. << (codec_context->extradata == nullptr ? " NULL" : " Non-NULL")
  357. << " extra data cannot have size of "
  358. << codec_context->extradata_size << ".";
  359. return false;
  360. }
  361. std::vector<uint8_t> extra_data;
  362. if (codec_context->extradata_size > 0) {
  363. extra_data.assign(codec_context->extradata,
  364. codec_context->extradata + codec_context->extradata_size);
  365. }
  366. config->Initialize(codec, sample_format, channel_layout, codec_context->sample_rate,
  367. extra_data, encryption_scheme, seek_preroll,
  368. codec_context->delay);
  369. if (channel_layout == CHANNEL_LAYOUT_DISCRETE)
  370. config->SetChannelsForDiscrete(codec_context->ch_layout.nb_channels);
  371. #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO)
  372. // These are bitstream formats unknown to ffmpeg, so they don't have
  373. // a known sample format size.
  374. if (codec == AudioCodec::kAC3 || codec == AudioCodec::kEAC3)
  375. return true;
  376. #endif
  377. #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO)
  378. if (codec == AudioCodec::kMpegHAudio)
  379. return true;
  380. #endif
  381. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  382. if (codec == AudioCodec::kAAC) {
  383. config->set_aac_extra_data(extra_data);
  384. // TODO(dalecurtis): Just use the profile from the codec context if ffmpeg
  385. // ever starts supporting xHE-AAC.
  386. if (codec_context->profile == FF_PROFILE_UNKNOWN) {
  387. // Errors aren't fatal here, so just drop any MediaLog messages.
  388. NullMediaLog media_log;
  389. mp4::AAC aac_parser;
  390. if (aac_parser.Parse(extra_data, &media_log))
  391. config->set_profile(aac_parser.GetProfile());
  392. }
  393. }
  394. #endif
  395. // Verify that AudioConfig.bits_per_channel was calculated correctly for
  396. // codecs that have |sample_fmt| set by FFmpeg.
  397. DCHECK_EQ(av_get_bytes_per_sample(codec_context->sample_fmt) * 8,
  398. config->bits_per_channel());
  399. return true;
  400. }
  401. std::unique_ptr<AVCodecContext, ScopedPtrAVFreeContext>
  402. AVStreamToAVCodecContext(const AVStream* stream) {
  403. std::unique_ptr<AVCodecContext, ScopedPtrAVFreeContext> codec_context(
  404. avcodec_alloc_context3(nullptr));
  405. if (avcodec_parameters_to_context(codec_context.get(), stream->codecpar) <
  406. 0) {
  407. return nullptr;
  408. }
  409. return codec_context;
  410. }
  411. bool AVStreamToAudioDecoderConfig(const AVStream* stream,
  412. AudioDecoderConfig* config) {
  413. std::unique_ptr<AVCodecContext, ScopedPtrAVFreeContext> codec_context(
  414. AVStreamToAVCodecContext(stream));
  415. if (!codec_context)
  416. return false;
  417. return AVCodecContextToAudioDecoderConfig(
  418. codec_context.get(), GetEncryptionScheme(stream), config);
  419. }
  420. void AudioDecoderConfigToAVCodecContext(const AudioDecoderConfig& config,
  421. AVCodecContext* codec_context) {
  422. codec_context->codec_type = AVMEDIA_TYPE_AUDIO;
  423. codec_context->codec_id = AudioCodecToCodecID(config.codec(),
  424. config.sample_format());
  425. codec_context->sample_fmt = SampleFormatToAVSampleFormat(
  426. config.sample_format());
  427. // TODO(scherkus): should we set |channel_layout|? I'm not sure if FFmpeg uses
  428. // said information to decode.
  429. codec_context->ch_layout.nb_channels = config.channels();
  430. codec_context->sample_rate = config.samples_per_second();
  431. if (config.extra_data().empty()) {
  432. codec_context->extradata = nullptr;
  433. codec_context->extradata_size = 0;
  434. } else {
  435. codec_context->extradata_size = config.extra_data().size();
  436. codec_context->extradata = reinterpret_cast<uint8_t*>(
  437. av_malloc(config.extra_data().size() + AV_INPUT_BUFFER_PADDING_SIZE));
  438. memcpy(codec_context->extradata, &config.extra_data()[0],
  439. config.extra_data().size());
  440. memset(codec_context->extradata + config.extra_data().size(), '\0',
  441. AV_INPUT_BUFFER_PADDING_SIZE);
  442. }
  443. }
  444. bool AVStreamToVideoDecoderConfig(const AVStream* stream,
  445. VideoDecoderConfig* config) {
  446. std::unique_ptr<AVCodecContext, ScopedPtrAVFreeContext> codec_context(
  447. AVStreamToAVCodecContext(stream));
  448. if (!codec_context)
  449. return false;
  450. // TODO(vrk): This assumes decoded frame data starts at (0, 0), which is true
  451. // for now, but may not always be true forever. Fix this in the future.
  452. gfx::Rect visible_rect(codec_context->width, codec_context->height);
  453. gfx::Size coded_size = visible_rect.size();
  454. // In some cases a container may have a DAR but no PAR, but FFmpeg translates
  455. // everything to PAR. It is possible to get the render width and height, but I
  456. // didn't find a way to determine whether that should be preferred to the PAR.
  457. VideoAspectRatio aspect_ratio;
  458. if (stream->sample_aspect_ratio.num) {
  459. aspect_ratio = VideoAspectRatio::PAR(stream->sample_aspect_ratio.num,
  460. stream->sample_aspect_ratio.den);
  461. } else if (codec_context->sample_aspect_ratio.num) {
  462. aspect_ratio =
  463. VideoAspectRatio::PAR(codec_context->sample_aspect_ratio.num,
  464. codec_context->sample_aspect_ratio.den);
  465. }
  466. // Used to guess color space and to create the config. The first use should
  467. // probably change to coded size, and the second should be removed as part of
  468. // crbug.com/1214061.
  469. gfx::Size natural_size = aspect_ratio.GetNaturalSize(visible_rect);
  470. VideoCodec codec = CodecIDToVideoCodec(codec_context->codec_id);
  471. // Without the ffmpeg decoder configured, libavformat is unable to get the
  472. // profile, format, or coded size. So choose sensible defaults and let
  473. // decoders fail later if the configuration is actually unsupported.
  474. //
  475. // TODO(chcunningham): We need real profiles for all of the codecs below to
  476. // actually handle capabilities requests correctly. http://crbug.com/784610
  477. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  478. // Prefer the color space found by libavcodec if available
  479. VideoColorSpace color_space =
  480. VideoColorSpace(codec_context->color_primaries, codec_context->color_trc,
  481. codec_context->colorspace,
  482. codec_context->color_range == AVCOL_RANGE_JPEG
  483. ? gfx::ColorSpace::RangeID::FULL
  484. : gfx::ColorSpace::RangeID::LIMITED);
  485. switch (codec) {
  486. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  487. case VideoCodec::kH264: {
  488. profile = ProfileIDToVideoCodecProfile(codec_context->profile);
  489. // if the profile is still unknown, try to extract it from
  490. // the extradata using the internal parser
  491. if (profile == VIDEO_CODEC_PROFILE_UNKNOWN && codec_context->extradata &&
  492. codec_context->extradata_size) {
  493. mp4::AVCDecoderConfigurationRecord avc_config;
  494. if (avc_config.Parse(codec_context->extradata,
  495. codec_context->extradata_size)) {
  496. profile = ProfileIDToVideoCodecProfile(avc_config.profile_indication);
  497. }
  498. }
  499. // All the heuristics failed, let's assign a default profile
  500. if (profile == VIDEO_CODEC_PROFILE_UNKNOWN)
  501. profile = H264PROFILE_BASELINE;
  502. break;
  503. }
  504. #if BUILDFLAG(ENABLE_PLATFORM_HEVC)
  505. case VideoCodec::kHEVC: {
  506. int hevc_profile = -1;
  507. // We need to parse extradata each time, because we wont add ffmpeg
  508. // hevc decoder & parser to chromium and codec_context->profile
  509. // should always be FF_PROFILE_UNKNOWN (-99) here
  510. if (codec_context->extradata && codec_context->extradata_size) {
  511. mp4::HEVCDecoderConfigurationRecord hevc_config;
  512. if (hevc_config.Parse(codec_context->extradata,
  513. codec_context->extradata_size)) {
  514. hevc_profile = hevc_config.general_profile_idc;
  515. #if BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  516. if (!color_space.IsSpecified()) {
  517. // We should try to parsed color space from SPS if the
  518. // result from libavcodec is not specified in case
  519. // that some encoder not write extra colorspace info to
  520. // the container
  521. color_space = hevc_config.GetColorSpace();
  522. }
  523. #endif // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER)
  524. }
  525. }
  526. // The values of general_profile_idc are taken from the HEVC standard, see
  527. // the latest https://www.itu.int/rec/T-REC-H.265/en
  528. switch (hevc_profile) {
  529. case 1:
  530. profile = HEVCPROFILE_MAIN;
  531. break;
  532. case 2:
  533. profile = HEVCPROFILE_MAIN10;
  534. break;
  535. case 3:
  536. profile = HEVCPROFILE_MAIN_STILL_PICTURE;
  537. break;
  538. case 4:
  539. profile = HEVCPROFILE_REXT;
  540. break;
  541. case 5:
  542. profile = HEVCPROFILE_HIGH_THROUGHPUT;
  543. break;
  544. case 6:
  545. profile = HEVCPROFILE_MULTIVIEW_MAIN;
  546. break;
  547. case 7:
  548. profile = HEVCPROFILE_SCALABLE_MAIN;
  549. break;
  550. case 8:
  551. profile = HEVCPROFILE_3D_MAIN;
  552. break;
  553. case 9:
  554. profile = HEVCPROFILE_SCREEN_EXTENDED;
  555. break;
  556. case 10:
  557. profile = HEVCPROFILE_SCALABLE_REXT;
  558. break;
  559. case 11:
  560. profile = HEVCPROFILE_HIGH_THROUGHPUT_SCREEN_EXTENDED;
  561. break;
  562. default:
  563. // Always assign a default if all heuristics fail.
  564. profile = HEVCPROFILE_MAIN;
  565. break;
  566. }
  567. break;
  568. }
  569. #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC)
  570. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  571. case VideoCodec::kVP8:
  572. profile = VP8PROFILE_ANY;
  573. break;
  574. case VideoCodec::kVP9:
  575. switch (codec_context->profile) {
  576. case FF_PROFILE_VP9_0:
  577. profile = VP9PROFILE_PROFILE0;
  578. break;
  579. case FF_PROFILE_VP9_1:
  580. profile = VP9PROFILE_PROFILE1;
  581. break;
  582. case FF_PROFILE_VP9_2:
  583. profile = VP9PROFILE_PROFILE2;
  584. break;
  585. case FF_PROFILE_VP9_3:
  586. profile = VP9PROFILE_PROFILE3;
  587. break;
  588. default:
  589. profile = VP9PROFILE_MIN;
  590. break;
  591. }
  592. break;
  593. case VideoCodec::kAV1:
  594. profile = AV1PROFILE_PROFILE_MAIN;
  595. break;
  596. case VideoCodec::kTheora:
  597. profile = THEORAPROFILE_ANY;
  598. break;
  599. default:
  600. profile = ProfileIDToVideoCodecProfile(codec_context->profile);
  601. }
  602. auto* alpha_mode = av_dict_get(stream->metadata, "alpha_mode", nullptr, 0);
  603. const bool has_alpha = alpha_mode && !strcmp(alpha_mode->value, "1");
  604. void* display_matrix =
  605. av_stream_get_side_data(stream, AV_PKT_DATA_DISPLAYMATRIX, nullptr);
  606. VideoTransformation video_transformation = VideoTransformation();
  607. if (display_matrix) {
  608. video_transformation = VideoTransformation::FromFFmpegDisplayMatrix(
  609. static_cast<int32_t*>(display_matrix));
  610. }
  611. if (!color_space.IsSpecified()) {
  612. // VP9 frames may have color information, but that information cannot
  613. // express new color spaces, like HDR. For that reason, color space
  614. // information from the container should take precedence over color space
  615. // information from the VP9 stream. However, if we infer the color space
  616. // based on resolution here, it looks as if it came from the container.
  617. // Since this inference causes color shifts and is slated to go away
  618. // we just skip it for VP9 and leave the color space undefined, which
  619. // will make the VP9 decoder behave correctly..
  620. // We also ignore the resolution for AV1, since it's new and it's easy
  621. // to make it behave correctly from the get-go.
  622. // TODO(hubbe): Skip this inference for all codecs.
  623. if (codec_context->codec_id != AV_CODEC_ID_VP9 &&
  624. codec_context->codec_id != AV_CODEC_ID_AV1) {
  625. // Otherwise, assume that SD video is usually Rec.601, and HD is usually
  626. // Rec.709.
  627. color_space = (natural_size.height() < 720) ? VideoColorSpace::REC601()
  628. : VideoColorSpace::REC709();
  629. }
  630. } else if (codec_context->codec_id == AV_CODEC_ID_H264 &&
  631. codec_context->colorspace == AVCOL_SPC_RGB &&
  632. AVPixelFormatToVideoPixelFormat(codec_context->pix_fmt) ==
  633. PIXEL_FORMAT_I420) {
  634. // Some H.264 videos contain a VUI that specifies a color matrix of GBR,
  635. // when they are actually ordinary YUV. Only 4:2:0 formats are checked,
  636. // because GBR is reasonable for 4:4:4 content. See crbug.com/1067377.
  637. color_space = VideoColorSpace::REC709();
  638. }
  639. // AVCodecContext occasionally has invalid extra data. See
  640. // http://crbug.com/517163
  641. if (codec_context->extradata != nullptr &&
  642. codec_context->extradata_size == 0) {
  643. DLOG(ERROR) << __func__ << " Non-Null extra data cannot have size of 0.";
  644. return false;
  645. }
  646. std::vector<uint8_t> extra_data;
  647. if (codec_context->extradata_size > 0) {
  648. extra_data.assign(codec_context->extradata,
  649. codec_context->extradata + codec_context->extradata_size);
  650. }
  651. // TODO(tmathmeyer) ffmpeg can't provide us with an actual video rotation yet.
  652. config->Initialize(codec, profile,
  653. has_alpha ? VideoDecoderConfig::AlphaMode::kHasAlpha
  654. : VideoDecoderConfig::AlphaMode::kIsOpaque,
  655. color_space, video_transformation, coded_size,
  656. visible_rect, natural_size, extra_data,
  657. GetEncryptionScheme(stream));
  658. // Set the aspect ratio explicitly since our version hasn't been rounded.
  659. config->set_aspect_ratio(aspect_ratio);
  660. if (stream->nb_side_data) {
  661. for (int i = 0; i < stream->nb_side_data; ++i) {
  662. AVPacketSideData side_data = stream->side_data[i];
  663. if (side_data.type != AV_PKT_DATA_MASTERING_DISPLAY_METADATA)
  664. continue;
  665. gfx::HDRMetadata hdr_metadata{};
  666. AVMasteringDisplayMetadata* metadata =
  667. reinterpret_cast<AVMasteringDisplayMetadata*>(side_data.data);
  668. if (metadata->has_primaries) {
  669. hdr_metadata.color_volume_metadata.primary_r =
  670. gfx::PointF(av_q2d(metadata->display_primaries[0][0]),
  671. av_q2d(metadata->display_primaries[0][1]));
  672. hdr_metadata.color_volume_metadata.primary_g =
  673. gfx::PointF(av_q2d(metadata->display_primaries[1][0]),
  674. av_q2d(metadata->display_primaries[1][1]));
  675. hdr_metadata.color_volume_metadata.primary_b =
  676. gfx::PointF(av_q2d(metadata->display_primaries[2][0]),
  677. av_q2d(metadata->display_primaries[2][1]));
  678. hdr_metadata.color_volume_metadata.white_point = gfx::PointF(
  679. av_q2d(metadata->white_point[0]), av_q2d(metadata->white_point[1]));
  680. }
  681. if (metadata->has_luminance) {
  682. hdr_metadata.color_volume_metadata.luminance_max =
  683. av_q2d(metadata->max_luminance);
  684. hdr_metadata.color_volume_metadata.luminance_min =
  685. av_q2d(metadata->min_luminance);
  686. }
  687. config->set_hdr_metadata(hdr_metadata);
  688. }
  689. }
  690. return true;
  691. }
  692. void VideoDecoderConfigToAVCodecContext(
  693. const VideoDecoderConfig& config,
  694. AVCodecContext* codec_context) {
  695. codec_context->codec_type = AVMEDIA_TYPE_VIDEO;
  696. codec_context->codec_id = VideoCodecToCodecID(config.codec());
  697. codec_context->profile = VideoCodecProfileToProfileID(config.profile());
  698. codec_context->coded_width = config.coded_size().width();
  699. codec_context->coded_height = config.coded_size().height();
  700. if (config.color_space_info().range == gfx::ColorSpace::RangeID::FULL)
  701. codec_context->color_range = AVCOL_RANGE_JPEG;
  702. if (config.extra_data().empty()) {
  703. codec_context->extradata = nullptr;
  704. codec_context->extradata_size = 0;
  705. } else {
  706. codec_context->extradata_size = config.extra_data().size();
  707. codec_context->extradata = reinterpret_cast<uint8_t*>(
  708. av_malloc(config.extra_data().size() + AV_INPUT_BUFFER_PADDING_SIZE));
  709. memcpy(codec_context->extradata, &config.extra_data()[0],
  710. config.extra_data().size());
  711. memset(codec_context->extradata + config.extra_data().size(), '\0',
  712. AV_INPUT_BUFFER_PADDING_SIZE);
  713. }
  714. }
  715. ChannelLayout ChannelLayoutToChromeChannelLayout(int64_t layout, int channels) {
  716. switch (layout) {
  717. case AV_CH_LAYOUT_MONO:
  718. return CHANNEL_LAYOUT_MONO;
  719. case AV_CH_LAYOUT_STEREO:
  720. return CHANNEL_LAYOUT_STEREO;
  721. case AV_CH_LAYOUT_2_1:
  722. return CHANNEL_LAYOUT_2_1;
  723. case AV_CH_LAYOUT_SURROUND:
  724. return CHANNEL_LAYOUT_SURROUND;
  725. case AV_CH_LAYOUT_4POINT0:
  726. return CHANNEL_LAYOUT_4_0;
  727. case AV_CH_LAYOUT_2_2:
  728. return CHANNEL_LAYOUT_2_2;
  729. case AV_CH_LAYOUT_QUAD:
  730. return CHANNEL_LAYOUT_QUAD;
  731. case AV_CH_LAYOUT_5POINT0:
  732. return CHANNEL_LAYOUT_5_0;
  733. case AV_CH_LAYOUT_5POINT1:
  734. return CHANNEL_LAYOUT_5_1;
  735. case AV_CH_LAYOUT_5POINT0_BACK:
  736. return CHANNEL_LAYOUT_5_0_BACK;
  737. case AV_CH_LAYOUT_5POINT1_BACK:
  738. return CHANNEL_LAYOUT_5_1_BACK;
  739. case AV_CH_LAYOUT_7POINT0:
  740. return CHANNEL_LAYOUT_7_0;
  741. case AV_CH_LAYOUT_7POINT1:
  742. return CHANNEL_LAYOUT_7_1;
  743. case AV_CH_LAYOUT_7POINT1_WIDE:
  744. return CHANNEL_LAYOUT_7_1_WIDE;
  745. case AV_CH_LAYOUT_STEREO_DOWNMIX:
  746. return CHANNEL_LAYOUT_STEREO_DOWNMIX;
  747. case AV_CH_LAYOUT_2POINT1:
  748. return CHANNEL_LAYOUT_2POINT1;
  749. case AV_CH_LAYOUT_3POINT1:
  750. return CHANNEL_LAYOUT_3_1;
  751. case AV_CH_LAYOUT_4POINT1:
  752. return CHANNEL_LAYOUT_4_1;
  753. case AV_CH_LAYOUT_6POINT0:
  754. return CHANNEL_LAYOUT_6_0;
  755. case AV_CH_LAYOUT_6POINT0_FRONT:
  756. return CHANNEL_LAYOUT_6_0_FRONT;
  757. case AV_CH_LAYOUT_HEXAGONAL:
  758. return CHANNEL_LAYOUT_HEXAGONAL;
  759. case AV_CH_LAYOUT_6POINT1:
  760. return CHANNEL_LAYOUT_6_1;
  761. case AV_CH_LAYOUT_6POINT1_BACK:
  762. return CHANNEL_LAYOUT_6_1_BACK;
  763. case AV_CH_LAYOUT_6POINT1_FRONT:
  764. return CHANNEL_LAYOUT_6_1_FRONT;
  765. case AV_CH_LAYOUT_7POINT0_FRONT:
  766. return CHANNEL_LAYOUT_7_0_FRONT;
  767. #ifdef AV_CH_LAYOUT_7POINT1_WIDE_BACK
  768. case AV_CH_LAYOUT_7POINT1_WIDE_BACK:
  769. return CHANNEL_LAYOUT_7_1_WIDE_BACK;
  770. #endif
  771. case AV_CH_LAYOUT_OCTAGONAL:
  772. return CHANNEL_LAYOUT_OCTAGONAL;
  773. default:
  774. // FFmpeg channel_layout is 0 for .wav and .mp3. Attempt to guess layout
  775. // based on the channel count.
  776. return GuessChannelLayout(channels);
  777. }
  778. }
  779. #if !defined(ARCH_CPU_LITTLE_ENDIAN)
  780. #error The code below assumes little-endianness.
  781. #endif
  782. VideoPixelFormat AVPixelFormatToVideoPixelFormat(AVPixelFormat pixel_format) {
  783. // The YUVJ alternatives are FFmpeg's (deprecated, but still in use) way to
  784. // specify a pixel format and full range color combination.
  785. switch (pixel_format) {
  786. case AV_PIX_FMT_YUV444P:
  787. case AV_PIX_FMT_YUVJ444P:
  788. return PIXEL_FORMAT_I444;
  789. case AV_PIX_FMT_YUV420P:
  790. case AV_PIX_FMT_YUVJ420P:
  791. return PIXEL_FORMAT_I420;
  792. case AV_PIX_FMT_YUV422P:
  793. case AV_PIX_FMT_YUVJ422P:
  794. return PIXEL_FORMAT_I422;
  795. case AV_PIX_FMT_YUVA420P:
  796. return PIXEL_FORMAT_I420A;
  797. case AV_PIX_FMT_YUV420P9LE:
  798. return PIXEL_FORMAT_YUV420P9;
  799. case AV_PIX_FMT_YUV420P10LE:
  800. return PIXEL_FORMAT_YUV420P10;
  801. case AV_PIX_FMT_YUV420P12LE:
  802. return PIXEL_FORMAT_YUV420P12;
  803. case AV_PIX_FMT_YUV422P9LE:
  804. return PIXEL_FORMAT_YUV422P9;
  805. case AV_PIX_FMT_YUV422P10LE:
  806. return PIXEL_FORMAT_YUV422P10;
  807. case AV_PIX_FMT_YUV422P12LE:
  808. return PIXEL_FORMAT_YUV422P12;
  809. case AV_PIX_FMT_YUV444P9LE:
  810. return PIXEL_FORMAT_YUV444P9;
  811. case AV_PIX_FMT_YUV444P10LE:
  812. return PIXEL_FORMAT_YUV444P10;
  813. case AV_PIX_FMT_YUV444P12LE:
  814. return PIXEL_FORMAT_YUV444P12;
  815. case AV_PIX_FMT_P016LE:
  816. return PIXEL_FORMAT_P016LE;
  817. default:
  818. DVLOG(1) << "Unsupported AVPixelFormat: " << pixel_format;
  819. }
  820. return PIXEL_FORMAT_UNKNOWN;
  821. }
  822. VideoColorSpace AVColorSpaceToColorSpace(AVColorSpace color_space,
  823. AVColorRange color_range) {
  824. // TODO(hubbe): make this better
  825. if (color_range == AVCOL_RANGE_JPEG)
  826. return VideoColorSpace::JPEG();
  827. switch (color_space) {
  828. case AVCOL_SPC_UNSPECIFIED:
  829. break;
  830. case AVCOL_SPC_BT709:
  831. return VideoColorSpace::REC709();
  832. case AVCOL_SPC_SMPTE170M:
  833. case AVCOL_SPC_BT470BG:
  834. return VideoColorSpace::REC601();
  835. default:
  836. DVLOG(1) << "Unknown AVColorSpace: " << color_space;
  837. }
  838. return VideoColorSpace();
  839. }
  840. std::string AVErrorToString(int errnum) {
  841. char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
  842. av_strerror(errnum, errbuf, AV_ERROR_MAX_STRING_SIZE);
  843. return std::string(errbuf);
  844. }
  845. int32_t HashCodecName(const char* codec_name) {
  846. // Use the first 32-bits from the SHA1 hash as the identifier.
  847. int32_t hash;
  848. memcpy(&hash, base::SHA1HashString(codec_name).substr(0, 4).c_str(), 4);
  849. return hash;
  850. }
  851. } // namespace media