ffmpeg_audio_decoder.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/ffmpeg_audio_decoder.h"
  5. #include <stdint.h>
  6. #include <functional>
  7. #include <memory>
  8. #include "base/bind.h"
  9. #include "base/callback_helpers.h"
  10. #include "base/task/single_thread_task_runner.h"
  11. #include "media/base/audio_buffer.h"
  12. #include "media/base/audio_bus.h"
  13. #include "media/base/audio_decoder_config.h"
  14. #include "media/base/audio_discard_helper.h"
  15. #include "media/base/bind_to_current_loop.h"
  16. #include "media/base/decoder_buffer.h"
  17. #include "media/base/limits.h"
  18. #include "media/base/timestamp_constants.h"
  19. #include "media/ffmpeg/ffmpeg_common.h"
  20. #include "media/ffmpeg/ffmpeg_decoding_loop.h"
  21. #include "media/filters/ffmpeg_glue.h"
  22. namespace media {
  23. // Return the number of channels from the data in |frame|.
  24. static inline int DetermineChannels(AVFrame* frame) {
  25. return frame->ch_layout.nb_channels;
  26. }
  27. // Called by FFmpeg's allocation routine to allocate a buffer. Uses
  28. // AVCodecContext.opaque to get the object reference in order to call
  29. // GetAudioBuffer() to do the actual allocation.
  30. static int GetAudioBufferImpl(struct AVCodecContext* s,
  31. AVFrame* frame,
  32. int flags) {
  33. FFmpegAudioDecoder* decoder = static_cast<FFmpegAudioDecoder*>(s->opaque);
  34. return decoder->GetAudioBuffer(s, frame, flags);
  35. }
  36. // Called by FFmpeg's allocation routine to free a buffer. |opaque| is the
  37. // AudioBuffer allocated, so unref it.
  38. static void ReleaseAudioBufferImpl(void* opaque, uint8_t* data) {
  39. if (opaque)
  40. static_cast<AudioBuffer*>(opaque)->Release();
  41. }
  42. FFmpegAudioDecoder::FFmpegAudioDecoder(
  43. const scoped_refptr<base::SequencedTaskRunner>& task_runner,
  44. MediaLog* media_log)
  45. : task_runner_(task_runner),
  46. state_(DecoderState::kUninitialized),
  47. av_sample_format_(0),
  48. media_log_(media_log),
  49. pool_(new AudioBufferMemoryPool()) {
  50. DETACH_FROM_SEQUENCE(sequence_checker_);
  51. }
  52. FFmpegAudioDecoder::~FFmpegAudioDecoder() {
  53. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  54. if (state_ != DecoderState::kUninitialized)
  55. ReleaseFFmpegResources();
  56. }
  57. AudioDecoderType FFmpegAudioDecoder::GetDecoderType() const {
  58. return AudioDecoderType::kFFmpeg;
  59. }
  60. void FFmpegAudioDecoder::Initialize(const AudioDecoderConfig& config,
  61. CdmContext* /* cdm_context */,
  62. InitCB init_cb,
  63. const OutputCB& output_cb,
  64. const WaitingCB& /* waiting_cb */) {
  65. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  66. DCHECK(config.IsValidConfig());
  67. InitCB bound_init_cb = BindToCurrentLoop(std::move(init_cb));
  68. if (config.is_encrypted()) {
  69. std::move(bound_init_cb)
  70. .Run(DecoderStatus(
  71. DecoderStatus::Codes::kUnsupportedEncryptionMode,
  72. "FFmpegAudioDecoder does not support encrypted content"));
  73. return;
  74. }
  75. // TODO(dalecurtis): Remove this if ffmpeg ever gets xHE-AAC support.
  76. if (config.profile() == AudioCodecProfile::kXHE_AAC) {
  77. std::move(bound_init_cb)
  78. .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedProfile)
  79. .WithData("decoder", "FFmpegAudioDecoder")
  80. .WithData("profile", config.profile()));
  81. return;
  82. }
  83. if (!ConfigureDecoder(config)) {
  84. av_sample_format_ = 0;
  85. std::move(bound_init_cb).Run(DecoderStatus::Codes::kUnsupportedConfig);
  86. return;
  87. }
  88. // Success!
  89. config_ = config;
  90. output_cb_ = BindToCurrentLoop(output_cb);
  91. state_ = DecoderState::kNormal;
  92. std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
  93. }
  94. void FFmpegAudioDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
  95. DecodeCB decode_cb) {
  96. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  97. DCHECK(decode_cb);
  98. CHECK_NE(state_, DecoderState::kUninitialized);
  99. DecodeCB decode_cb_bound = BindToCurrentLoop(std::move(decode_cb));
  100. if (state_ == DecoderState::kError) {
  101. std::move(decode_cb_bound).Run(DecoderStatus::Codes::kFailed);
  102. return;
  103. }
  104. // Do nothing if decoding has finished.
  105. if (state_ == DecoderState::kDecodeFinished) {
  106. std::move(decode_cb_bound).Run(DecoderStatus::Codes::kOk);
  107. return;
  108. }
  109. DecodeBuffer(*buffer, std::move(decode_cb_bound));
  110. }
  111. void FFmpegAudioDecoder::Reset(base::OnceClosure closure) {
  112. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  113. avcodec_flush_buffers(codec_context_.get());
  114. state_ = DecoderState::kNormal;
  115. ResetTimestampState(config_);
  116. task_runner_->PostTask(FROM_HERE, std::move(closure));
  117. }
  118. void FFmpegAudioDecoder::DecodeBuffer(const DecoderBuffer& buffer,
  119. DecodeCB decode_cb) {
  120. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  121. DCHECK_NE(state_, DecoderState::kUninitialized);
  122. DCHECK_NE(state_, DecoderState::kDecodeFinished);
  123. DCHECK_NE(state_, DecoderState::kError);
  124. // Make sure we are notified if http://crbug.com/49709 returns. Issue also
  125. // occurs with some damaged files.
  126. if (!buffer.end_of_stream() && buffer.timestamp() == kNoTimestamp) {
  127. DVLOG(1) << "Received a buffer without timestamps!";
  128. std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
  129. return;
  130. }
  131. if (!FFmpegDecode(buffer)) {
  132. state_ = DecoderState::kError;
  133. std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
  134. return;
  135. }
  136. if (buffer.end_of_stream())
  137. state_ = DecoderState::kDecodeFinished;
  138. std::move(decode_cb).Run(DecoderStatus::Codes::kOk);
  139. }
  140. bool FFmpegAudioDecoder::FFmpegDecode(const DecoderBuffer& buffer) {
  141. AVPacket* packet = av_packet_alloc();
  142. if (buffer.end_of_stream()) {
  143. packet->data = NULL;
  144. packet->size = 0;
  145. } else {
  146. packet->data = const_cast<uint8_t*>(buffer.data());
  147. packet->size = buffer.data_size();
  148. DCHECK(packet->data);
  149. DCHECK_GT(packet->size, 0);
  150. }
  151. bool decoded_frame_this_loop = false;
  152. // base::Unretained and std::cref are safe to use with the callback given
  153. // to DecodePacket() since that callback is only used the function call.
  154. FFmpegDecodingLoop::DecodeStatus decode_status = decoding_loop_->DecodePacket(
  155. packet, base::BindRepeating(&FFmpegAudioDecoder::OnNewFrame,
  156. base::Unretained(this), std::cref(buffer),
  157. &decoded_frame_this_loop));
  158. av_packet_free(&packet);
  159. switch (decode_status) {
  160. case FFmpegDecodingLoop::DecodeStatus::kSendPacketFailed:
  161. MEDIA_LOG(ERROR, media_log_)
  162. << "Failed to send audio packet for decoding: "
  163. << buffer.AsHumanReadableString();
  164. return false;
  165. case FFmpegDecodingLoop::DecodeStatus::kFrameProcessingFailed:
  166. // OnNewFrame() should have already issued a MEDIA_LOG for this.
  167. return false;
  168. case FFmpegDecodingLoop::DecodeStatus::kDecodeFrameFailed:
  169. DCHECK(!buffer.end_of_stream())
  170. << "End of stream buffer produced an error! "
  171. << "This is quite possibly a bug in the audio decoder not handling "
  172. << "end of stream AVPackets correctly.";
  173. MEDIA_LOG(DEBUG, media_log_)
  174. << GetDecoderType() << " failed to decode an audio buffer: "
  175. << AVErrorToString(decoding_loop_->last_averror_code()) << ", at "
  176. << buffer.AsHumanReadableString();
  177. break;
  178. case FFmpegDecodingLoop::DecodeStatus::kOkay:
  179. break;
  180. }
  181. // Even if we didn't decode a frame this loop, we should still send the packet
  182. // to the discard helper for caching.
  183. if (!decoded_frame_this_loop && !buffer.end_of_stream()) {
  184. const bool result =
  185. discard_helper_->ProcessBuffers(buffer.time_info(), nullptr);
  186. DCHECK(!result);
  187. }
  188. return true;
  189. }
  190. bool FFmpegAudioDecoder::OnNewFrame(const DecoderBuffer& buffer,
  191. bool* decoded_frame_this_loop,
  192. AVFrame* frame) {
  193. const int channels = DetermineChannels(frame);
  194. // Translate unsupported into discrete layouts for discrete configurations;
  195. // ffmpeg does not have a labeled discrete configuration internally.
  196. ChannelLayout channel_layout = ChannelLayoutToChromeChannelLayout(
  197. codec_context_->ch_layout.u.mask, codec_context_->ch_layout.nb_channels);
  198. if (channel_layout == CHANNEL_LAYOUT_UNSUPPORTED &&
  199. config_.channel_layout() == CHANNEL_LAYOUT_DISCRETE) {
  200. channel_layout = CHANNEL_LAYOUT_DISCRETE;
  201. }
  202. const bool is_sample_rate_change =
  203. frame->sample_rate != config_.samples_per_second();
  204. const bool is_config_change = is_sample_rate_change ||
  205. channels != config_.channels() ||
  206. channel_layout != config_.channel_layout();
  207. if (is_config_change) {
  208. // Sample format is never expected to change.
  209. if (frame->format != av_sample_format_) {
  210. MEDIA_LOG(ERROR, media_log_)
  211. << "Unsupported midstream configuration change!"
  212. << " Sample Rate: " << frame->sample_rate << " vs "
  213. << config_.samples_per_second()
  214. << " ChannelLayout: " << channel_layout << " vs "
  215. << config_.channel_layout() << " << Channels: " << channels << " vs "
  216. << config_.channels() << ", Sample Format: " << frame->format
  217. << " vs " << av_sample_format_;
  218. // This is an unrecoverable error, so bail out.
  219. return false;
  220. }
  221. MEDIA_LOG(DEBUG, media_log_)
  222. << " Detected midstream configuration change"
  223. << " PTS:" << buffer.timestamp().InMicroseconds()
  224. << " Sample Rate: " << frame->sample_rate << " vs "
  225. << config_.samples_per_second() << ", ChannelLayout: " << channel_layout
  226. << " vs " << config_.channel_layout() << ", Channels: " << channels
  227. << " vs " << config_.channels();
  228. config_.Initialize(config_.codec(), config_.sample_format(), channel_layout,
  229. frame->sample_rate, config_.extra_data(),
  230. config_.encryption_scheme(), config_.seek_preroll(),
  231. config_.codec_delay());
  232. if (is_sample_rate_change)
  233. ResetTimestampState(config_);
  234. }
  235. // Get the AudioBuffer that the data was decoded into. Adjust the number
  236. // of frames, in case fewer than requested were actually decoded.
  237. scoped_refptr<AudioBuffer> output =
  238. reinterpret_cast<AudioBuffer*>(av_buffer_get_opaque(frame->buf[0]));
  239. DCHECK_EQ(config_.channels(), output->channel_count());
  240. const int unread_frames = output->frame_count() - frame->nb_samples;
  241. DCHECK_GE(unread_frames, 0);
  242. if (unread_frames > 0)
  243. output->TrimEnd(unread_frames);
  244. *decoded_frame_this_loop = true;
  245. if (discard_helper_->ProcessBuffers(buffer.time_info(), output.get())) {
  246. if (is_config_change &&
  247. output->sample_rate() != config_.samples_per_second()) {
  248. // At the boundary of the config change, FFmpeg's AAC decoder gives the
  249. // previous sample rate when calling our GetAudioBuffer. Set the correct
  250. // sample rate before sending the buffer along.
  251. // TODO(chcunningham): Fix FFmpeg and upstream it.
  252. output->AdjustSampleRate(config_.samples_per_second());
  253. }
  254. output_cb_.Run(output);
  255. }
  256. return true;
  257. }
  258. void FFmpegAudioDecoder::ReleaseFFmpegResources() {
  259. decoding_loop_.reset();
  260. codec_context_.reset();
  261. }
  262. bool FFmpegAudioDecoder::ConfigureDecoder(const AudioDecoderConfig& config) {
  263. DCHECK(config.IsValidConfig());
  264. DCHECK(!config.is_encrypted());
  265. // Release existing decoder resources if necessary.
  266. ReleaseFFmpegResources();
  267. // Initialize AVCodecContext structure.
  268. codec_context_.reset(avcodec_alloc_context3(NULL));
  269. AudioDecoderConfigToAVCodecContext(config, codec_context_.get());
  270. codec_context_->opaque = this;
  271. codec_context_->get_buffer2 = GetAudioBufferImpl;
  272. if (!config.should_discard_decoder_delay())
  273. codec_context_->flags2 |= AV_CODEC_FLAG2_SKIP_MANUAL;
  274. AVDictionary* codec_options = NULL;
  275. if (config.codec() == AudioCodec::kOpus) {
  276. codec_context_->request_sample_fmt = AV_SAMPLE_FMT_FLT;
  277. // Disable phase inversion to avoid artifacts in mono downmix. See
  278. // http://crbug.com/806219
  279. if (config.target_output_channel_layout() == CHANNEL_LAYOUT_MONO) {
  280. int result = av_dict_set(&codec_options, "apply_phase_inv", "0", 0);
  281. DCHECK_GE(result, 0);
  282. }
  283. }
  284. const AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
  285. if (!codec ||
  286. avcodec_open2(codec_context_.get(), codec, &codec_options) < 0) {
  287. DLOG(ERROR) << "Could not initialize audio decoder: "
  288. << codec_context_->codec_id;
  289. ReleaseFFmpegResources();
  290. state_ = DecoderState::kUninitialized;
  291. return false;
  292. }
  293. // Verify avcodec_open2() used all given options.
  294. DCHECK_EQ(0, av_dict_count(codec_options));
  295. // Success!
  296. av_sample_format_ = codec_context_->sample_fmt;
  297. if (codec_context_->ch_layout.nb_channels != config.channels()) {
  298. MEDIA_LOG(ERROR, media_log_)
  299. << "Audio configuration specified " << config.channels()
  300. << " channels, but FFmpeg thinks the file contains "
  301. << codec_context_->ch_layout.nb_channels << " channels";
  302. ReleaseFFmpegResources();
  303. state_ = DecoderState::kUninitialized;
  304. return false;
  305. }
  306. decoding_loop_ =
  307. std::make_unique<FFmpegDecodingLoop>(codec_context_.get(), true);
  308. ResetTimestampState(config);
  309. return true;
  310. }
  311. void FFmpegAudioDecoder::ResetTimestampState(const AudioDecoderConfig& config) {
  312. // Opus codec delay is handled by ffmpeg.
  313. const int codec_delay =
  314. config.codec() == AudioCodec::kOpus ? 0 : config.codec_delay();
  315. discard_helper_ = std::make_unique<AudioDiscardHelper>(
  316. config.samples_per_second(), codec_delay,
  317. config.codec() == AudioCodec::kVorbis);
  318. discard_helper_->Reset(codec_delay);
  319. }
  320. int FFmpegAudioDecoder::GetAudioBuffer(struct AVCodecContext* s,
  321. AVFrame* frame,
  322. int flags) {
  323. DCHECK(s->codec->capabilities & AV_CODEC_CAP_DR1);
  324. DCHECK_EQ(s->codec_type, AVMEDIA_TYPE_AUDIO);
  325. // Since this routine is called by FFmpeg when a buffer is required for
  326. // audio data, use the values supplied by FFmpeg (ignoring the current
  327. // settings). FFmpegDecode() gets to determine if the buffer is useable or
  328. // not.
  329. AVSampleFormat format = static_cast<AVSampleFormat>(frame->format);
  330. SampleFormat sample_format =
  331. AVSampleFormatToSampleFormat(format, s->codec_id);
  332. if (sample_format == kUnknownSampleFormat) {
  333. DLOG(ERROR) << "Unknown sample format: " << format;
  334. return AVERROR(EINVAL);
  335. }
  336. int channels = DetermineChannels(frame);
  337. if (channels <= 0 || channels >= limits::kMaxChannels) {
  338. DLOG(ERROR) << "Requested number of channels (" << channels
  339. << ") exceeds limit.";
  340. return AVERROR(EINVAL);
  341. }
  342. int bytes_per_channel = SampleFormatToBytesPerChannel(sample_format);
  343. if (frame->nb_samples <= 0)
  344. return AVERROR(EINVAL);
  345. if (s->ch_layout.nb_channels != channels) {
  346. DLOG(ERROR) << "AVCodecContext and AVFrame disagree on channel count.";
  347. return AVERROR(EINVAL);
  348. }
  349. if (s->sample_rate != frame->sample_rate) {
  350. DLOG(ERROR) << "AVCodecContext and AVFrame disagree on sample rate."
  351. << s->sample_rate << " vs " << frame->sample_rate;
  352. return AVERROR(EINVAL);
  353. }
  354. if (s->sample_rate < limits::kMinSampleRate ||
  355. s->sample_rate > limits::kMaxSampleRate) {
  356. DLOG(ERROR) << "Requested sample rate (" << s->sample_rate
  357. << ") is outside supported range (" << limits::kMinSampleRate
  358. << " to " << limits::kMaxSampleRate << ").";
  359. return AVERROR(EINVAL);
  360. }
  361. // Determine how big the buffer should be and allocate it. FFmpeg may adjust
  362. // how big each channel data is in order to meet the alignment policy, so
  363. // we need to take this into consideration.
  364. int buffer_size_in_bytes = av_samples_get_buffer_size(
  365. &frame->linesize[0], channels, frame->nb_samples, format,
  366. 0 /* align, use ffmpeg default */);
  367. // Check for errors from av_samples_get_buffer_size().
  368. if (buffer_size_in_bytes < 0)
  369. return buffer_size_in_bytes;
  370. int frames_required = buffer_size_in_bytes / bytes_per_channel / channels;
  371. DCHECK_GE(frames_required, frame->nb_samples);
  372. ChannelLayout channel_layout =
  373. config_.channel_layout() == CHANNEL_LAYOUT_DISCRETE
  374. ? CHANNEL_LAYOUT_DISCRETE
  375. : ChannelLayoutToChromeChannelLayout(s->ch_layout.u.mask,
  376. s->ch_layout.nb_channels);
  377. if (channel_layout == CHANNEL_LAYOUT_UNSUPPORTED) {
  378. DLOG(ERROR) << "Unsupported channel layout.";
  379. return AVERROR(EINVAL);
  380. }
  381. scoped_refptr<AudioBuffer> buffer =
  382. AudioBuffer::CreateBuffer(sample_format, channel_layout, channels,
  383. s->sample_rate, frames_required, pool_);
  384. // Initialize the data[] and extended_data[] fields to point into the memory
  385. // allocated for AudioBuffer. |number_of_planes| will be 1 for interleaved
  386. // audio and equal to |channels| for planar audio.
  387. int number_of_planes = buffer->channel_data().size();
  388. if (number_of_planes <= AV_NUM_DATA_POINTERS) {
  389. DCHECK_EQ(frame->extended_data, frame->data);
  390. for (int i = 0; i < number_of_planes; ++i)
  391. frame->data[i] = buffer->channel_data()[i];
  392. } else {
  393. // There are more channels than can fit into data[], so allocate
  394. // extended_data[] and fill appropriately.
  395. frame->extended_data = static_cast<uint8_t**>(
  396. av_malloc(number_of_planes * sizeof(*frame->extended_data)));
  397. int i = 0;
  398. for (; i < AV_NUM_DATA_POINTERS; ++i)
  399. frame->extended_data[i] = frame->data[i] = buffer->channel_data()[i];
  400. for (; i < number_of_planes; ++i)
  401. frame->extended_data[i] = buffer->channel_data()[i];
  402. }
  403. // Now create an AVBufferRef for the data just allocated. It will own the
  404. // reference to the AudioBuffer object.
  405. AudioBuffer* opaque = buffer.get();
  406. opaque->AddRef();
  407. frame->buf[0] = av_buffer_create(frame->data[0], buffer_size_in_bytes,
  408. ReleaseAudioBufferImpl, opaque, 0);
  409. return 0;
  410. }
  411. } // namespace media