123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480 |
- // Copyright (c) 2012 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "media/filters/ffmpeg_audio_decoder.h"
- #include <stdint.h>
- #include <functional>
- #include <memory>
- #include "base/bind.h"
- #include "base/callback_helpers.h"
- #include "base/task/single_thread_task_runner.h"
- #include "media/base/audio_buffer.h"
- #include "media/base/audio_bus.h"
- #include "media/base/audio_decoder_config.h"
- #include "media/base/audio_discard_helper.h"
- #include "media/base/bind_to_current_loop.h"
- #include "media/base/decoder_buffer.h"
- #include "media/base/limits.h"
- #include "media/base/timestamp_constants.h"
- #include "media/ffmpeg/ffmpeg_common.h"
- #include "media/ffmpeg/ffmpeg_decoding_loop.h"
- #include "media/filters/ffmpeg_glue.h"
- namespace media {
- // Return the number of channels from the data in |frame|.
- static inline int DetermineChannels(AVFrame* frame) {
- return frame->ch_layout.nb_channels;
- }
- // Called by FFmpeg's allocation routine to allocate a buffer. Uses
- // AVCodecContext.opaque to get the object reference in order to call
- // GetAudioBuffer() to do the actual allocation.
- static int GetAudioBufferImpl(struct AVCodecContext* s,
- AVFrame* frame,
- int flags) {
- FFmpegAudioDecoder* decoder = static_cast<FFmpegAudioDecoder*>(s->opaque);
- return decoder->GetAudioBuffer(s, frame, flags);
- }
- // Called by FFmpeg's allocation routine to free a buffer. |opaque| is the
- // AudioBuffer allocated, so unref it.
- static void ReleaseAudioBufferImpl(void* opaque, uint8_t* data) {
- if (opaque)
- static_cast<AudioBuffer*>(opaque)->Release();
- }
- FFmpegAudioDecoder::FFmpegAudioDecoder(
- const scoped_refptr<base::SequencedTaskRunner>& task_runner,
- MediaLog* media_log)
- : task_runner_(task_runner),
- state_(DecoderState::kUninitialized),
- av_sample_format_(0),
- media_log_(media_log),
- pool_(new AudioBufferMemoryPool()) {
- DETACH_FROM_SEQUENCE(sequence_checker_);
- }
- FFmpegAudioDecoder::~FFmpegAudioDecoder() {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- if (state_ != DecoderState::kUninitialized)
- ReleaseFFmpegResources();
- }
- AudioDecoderType FFmpegAudioDecoder::GetDecoderType() const {
- return AudioDecoderType::kFFmpeg;
- }
- void FFmpegAudioDecoder::Initialize(const AudioDecoderConfig& config,
- CdmContext* /* cdm_context */,
- InitCB init_cb,
- const OutputCB& output_cb,
- const WaitingCB& /* waiting_cb */) {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- DCHECK(config.IsValidConfig());
- InitCB bound_init_cb = BindToCurrentLoop(std::move(init_cb));
- if (config.is_encrypted()) {
- std::move(bound_init_cb)
- .Run(DecoderStatus(
- DecoderStatus::Codes::kUnsupportedEncryptionMode,
- "FFmpegAudioDecoder does not support encrypted content"));
- return;
- }
- // TODO(dalecurtis): Remove this if ffmpeg ever gets xHE-AAC support.
- if (config.profile() == AudioCodecProfile::kXHE_AAC) {
- std::move(bound_init_cb)
- .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedProfile)
- .WithData("decoder", "FFmpegAudioDecoder")
- .WithData("profile", config.profile()));
- return;
- }
- if (!ConfigureDecoder(config)) {
- av_sample_format_ = 0;
- std::move(bound_init_cb).Run(DecoderStatus::Codes::kUnsupportedConfig);
- return;
- }
- // Success!
- config_ = config;
- output_cb_ = BindToCurrentLoop(output_cb);
- state_ = DecoderState::kNormal;
- std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
- }
- void FFmpegAudioDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
- DecodeCB decode_cb) {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- DCHECK(decode_cb);
- CHECK_NE(state_, DecoderState::kUninitialized);
- DecodeCB decode_cb_bound = BindToCurrentLoop(std::move(decode_cb));
- if (state_ == DecoderState::kError) {
- std::move(decode_cb_bound).Run(DecoderStatus::Codes::kFailed);
- return;
- }
- // Do nothing if decoding has finished.
- if (state_ == DecoderState::kDecodeFinished) {
- std::move(decode_cb_bound).Run(DecoderStatus::Codes::kOk);
- return;
- }
- DecodeBuffer(*buffer, std::move(decode_cb_bound));
- }
- void FFmpegAudioDecoder::Reset(base::OnceClosure closure) {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- avcodec_flush_buffers(codec_context_.get());
- state_ = DecoderState::kNormal;
- ResetTimestampState(config_);
- task_runner_->PostTask(FROM_HERE, std::move(closure));
- }
- void FFmpegAudioDecoder::DecodeBuffer(const DecoderBuffer& buffer,
- DecodeCB decode_cb) {
- DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
- DCHECK_NE(state_, DecoderState::kUninitialized);
- DCHECK_NE(state_, DecoderState::kDecodeFinished);
- DCHECK_NE(state_, DecoderState::kError);
- // Make sure we are notified if http://crbug.com/49709 returns. Issue also
- // occurs with some damaged files.
- if (!buffer.end_of_stream() && buffer.timestamp() == kNoTimestamp) {
- DVLOG(1) << "Received a buffer without timestamps!";
- std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
- return;
- }
- if (!FFmpegDecode(buffer)) {
- state_ = DecoderState::kError;
- std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
- return;
- }
- if (buffer.end_of_stream())
- state_ = DecoderState::kDecodeFinished;
- std::move(decode_cb).Run(DecoderStatus::Codes::kOk);
- }
- bool FFmpegAudioDecoder::FFmpegDecode(const DecoderBuffer& buffer) {
- AVPacket* packet = av_packet_alloc();
- if (buffer.end_of_stream()) {
- packet->data = NULL;
- packet->size = 0;
- } else {
- packet->data = const_cast<uint8_t*>(buffer.data());
- packet->size = buffer.data_size();
- DCHECK(packet->data);
- DCHECK_GT(packet->size, 0);
- }
- bool decoded_frame_this_loop = false;
- // base::Unretained and std::cref are safe to use with the callback given
- // to DecodePacket() since that callback is only used the function call.
- FFmpegDecodingLoop::DecodeStatus decode_status = decoding_loop_->DecodePacket(
- packet, base::BindRepeating(&FFmpegAudioDecoder::OnNewFrame,
- base::Unretained(this), std::cref(buffer),
- &decoded_frame_this_loop));
- av_packet_free(&packet);
- switch (decode_status) {
- case FFmpegDecodingLoop::DecodeStatus::kSendPacketFailed:
- MEDIA_LOG(ERROR, media_log_)
- << "Failed to send audio packet for decoding: "
- << buffer.AsHumanReadableString();
- return false;
- case FFmpegDecodingLoop::DecodeStatus::kFrameProcessingFailed:
- // OnNewFrame() should have already issued a MEDIA_LOG for this.
- return false;
- case FFmpegDecodingLoop::DecodeStatus::kDecodeFrameFailed:
- DCHECK(!buffer.end_of_stream())
- << "End of stream buffer produced an error! "
- << "This is quite possibly a bug in the audio decoder not handling "
- << "end of stream AVPackets correctly.";
- MEDIA_LOG(DEBUG, media_log_)
- << GetDecoderType() << " failed to decode an audio buffer: "
- << AVErrorToString(decoding_loop_->last_averror_code()) << ", at "
- << buffer.AsHumanReadableString();
- break;
- case FFmpegDecodingLoop::DecodeStatus::kOkay:
- break;
- }
- // Even if we didn't decode a frame this loop, we should still send the packet
- // to the discard helper for caching.
- if (!decoded_frame_this_loop && !buffer.end_of_stream()) {
- const bool result =
- discard_helper_->ProcessBuffers(buffer.time_info(), nullptr);
- DCHECK(!result);
- }
- return true;
- }
- bool FFmpegAudioDecoder::OnNewFrame(const DecoderBuffer& buffer,
- bool* decoded_frame_this_loop,
- AVFrame* frame) {
- const int channels = DetermineChannels(frame);
- // Translate unsupported into discrete layouts for discrete configurations;
- // ffmpeg does not have a labeled discrete configuration internally.
- ChannelLayout channel_layout = ChannelLayoutToChromeChannelLayout(
- codec_context_->ch_layout.u.mask, codec_context_->ch_layout.nb_channels);
- if (channel_layout == CHANNEL_LAYOUT_UNSUPPORTED &&
- config_.channel_layout() == CHANNEL_LAYOUT_DISCRETE) {
- channel_layout = CHANNEL_LAYOUT_DISCRETE;
- }
- const bool is_sample_rate_change =
- frame->sample_rate != config_.samples_per_second();
- const bool is_config_change = is_sample_rate_change ||
- channels != config_.channels() ||
- channel_layout != config_.channel_layout();
- if (is_config_change) {
- // Sample format is never expected to change.
- if (frame->format != av_sample_format_) {
- MEDIA_LOG(ERROR, media_log_)
- << "Unsupported midstream configuration change!"
- << " Sample Rate: " << frame->sample_rate << " vs "
- << config_.samples_per_second()
- << " ChannelLayout: " << channel_layout << " vs "
- << config_.channel_layout() << " << Channels: " << channels << " vs "
- << config_.channels() << ", Sample Format: " << frame->format
- << " vs " << av_sample_format_;
- // This is an unrecoverable error, so bail out.
- return false;
- }
- MEDIA_LOG(DEBUG, media_log_)
- << " Detected midstream configuration change"
- << " PTS:" << buffer.timestamp().InMicroseconds()
- << " Sample Rate: " << frame->sample_rate << " vs "
- << config_.samples_per_second() << ", ChannelLayout: " << channel_layout
- << " vs " << config_.channel_layout() << ", Channels: " << channels
- << " vs " << config_.channels();
- config_.Initialize(config_.codec(), config_.sample_format(), channel_layout,
- frame->sample_rate, config_.extra_data(),
- config_.encryption_scheme(), config_.seek_preroll(),
- config_.codec_delay());
- if (is_sample_rate_change)
- ResetTimestampState(config_);
- }
- // Get the AudioBuffer that the data was decoded into. Adjust the number
- // of frames, in case fewer than requested were actually decoded.
- scoped_refptr<AudioBuffer> output =
- reinterpret_cast<AudioBuffer*>(av_buffer_get_opaque(frame->buf[0]));
- DCHECK_EQ(config_.channels(), output->channel_count());
- const int unread_frames = output->frame_count() - frame->nb_samples;
- DCHECK_GE(unread_frames, 0);
- if (unread_frames > 0)
- output->TrimEnd(unread_frames);
- *decoded_frame_this_loop = true;
- if (discard_helper_->ProcessBuffers(buffer.time_info(), output.get())) {
- if (is_config_change &&
- output->sample_rate() != config_.samples_per_second()) {
- // At the boundary of the config change, FFmpeg's AAC decoder gives the
- // previous sample rate when calling our GetAudioBuffer. Set the correct
- // sample rate before sending the buffer along.
- // TODO(chcunningham): Fix FFmpeg and upstream it.
- output->AdjustSampleRate(config_.samples_per_second());
- }
- output_cb_.Run(output);
- }
- return true;
- }
- void FFmpegAudioDecoder::ReleaseFFmpegResources() {
- decoding_loop_.reset();
- codec_context_.reset();
- }
- bool FFmpegAudioDecoder::ConfigureDecoder(const AudioDecoderConfig& config) {
- DCHECK(config.IsValidConfig());
- DCHECK(!config.is_encrypted());
- // Release existing decoder resources if necessary.
- ReleaseFFmpegResources();
- // Initialize AVCodecContext structure.
- codec_context_.reset(avcodec_alloc_context3(NULL));
- AudioDecoderConfigToAVCodecContext(config, codec_context_.get());
- codec_context_->opaque = this;
- codec_context_->get_buffer2 = GetAudioBufferImpl;
- if (!config.should_discard_decoder_delay())
- codec_context_->flags2 |= AV_CODEC_FLAG2_SKIP_MANUAL;
- AVDictionary* codec_options = NULL;
- if (config.codec() == AudioCodec::kOpus) {
- codec_context_->request_sample_fmt = AV_SAMPLE_FMT_FLT;
- // Disable phase inversion to avoid artifacts in mono downmix. See
- // http://crbug.com/806219
- if (config.target_output_channel_layout() == CHANNEL_LAYOUT_MONO) {
- int result = av_dict_set(&codec_options, "apply_phase_inv", "0", 0);
- DCHECK_GE(result, 0);
- }
- }
- const AVCodec* codec = avcodec_find_decoder(codec_context_->codec_id);
- if (!codec ||
- avcodec_open2(codec_context_.get(), codec, &codec_options) < 0) {
- DLOG(ERROR) << "Could not initialize audio decoder: "
- << codec_context_->codec_id;
- ReleaseFFmpegResources();
- state_ = DecoderState::kUninitialized;
- return false;
- }
- // Verify avcodec_open2() used all given options.
- DCHECK_EQ(0, av_dict_count(codec_options));
- // Success!
- av_sample_format_ = codec_context_->sample_fmt;
- if (codec_context_->ch_layout.nb_channels != config.channels()) {
- MEDIA_LOG(ERROR, media_log_)
- << "Audio configuration specified " << config.channels()
- << " channels, but FFmpeg thinks the file contains "
- << codec_context_->ch_layout.nb_channels << " channels";
- ReleaseFFmpegResources();
- state_ = DecoderState::kUninitialized;
- return false;
- }
- decoding_loop_ =
- std::make_unique<FFmpegDecodingLoop>(codec_context_.get(), true);
- ResetTimestampState(config);
- return true;
- }
- void FFmpegAudioDecoder::ResetTimestampState(const AudioDecoderConfig& config) {
- // Opus codec delay is handled by ffmpeg.
- const int codec_delay =
- config.codec() == AudioCodec::kOpus ? 0 : config.codec_delay();
- discard_helper_ = std::make_unique<AudioDiscardHelper>(
- config.samples_per_second(), codec_delay,
- config.codec() == AudioCodec::kVorbis);
- discard_helper_->Reset(codec_delay);
- }
- int FFmpegAudioDecoder::GetAudioBuffer(struct AVCodecContext* s,
- AVFrame* frame,
- int flags) {
- DCHECK(s->codec->capabilities & AV_CODEC_CAP_DR1);
- DCHECK_EQ(s->codec_type, AVMEDIA_TYPE_AUDIO);
- // Since this routine is called by FFmpeg when a buffer is required for
- // audio data, use the values supplied by FFmpeg (ignoring the current
- // settings). FFmpegDecode() gets to determine if the buffer is useable or
- // not.
- AVSampleFormat format = static_cast<AVSampleFormat>(frame->format);
- SampleFormat sample_format =
- AVSampleFormatToSampleFormat(format, s->codec_id);
- if (sample_format == kUnknownSampleFormat) {
- DLOG(ERROR) << "Unknown sample format: " << format;
- return AVERROR(EINVAL);
- }
- int channels = DetermineChannels(frame);
- if (channels <= 0 || channels >= limits::kMaxChannels) {
- DLOG(ERROR) << "Requested number of channels (" << channels
- << ") exceeds limit.";
- return AVERROR(EINVAL);
- }
- int bytes_per_channel = SampleFormatToBytesPerChannel(sample_format);
- if (frame->nb_samples <= 0)
- return AVERROR(EINVAL);
- if (s->ch_layout.nb_channels != channels) {
- DLOG(ERROR) << "AVCodecContext and AVFrame disagree on channel count.";
- return AVERROR(EINVAL);
- }
- if (s->sample_rate != frame->sample_rate) {
- DLOG(ERROR) << "AVCodecContext and AVFrame disagree on sample rate."
- << s->sample_rate << " vs " << frame->sample_rate;
- return AVERROR(EINVAL);
- }
- if (s->sample_rate < limits::kMinSampleRate ||
- s->sample_rate > limits::kMaxSampleRate) {
- DLOG(ERROR) << "Requested sample rate (" << s->sample_rate
- << ") is outside supported range (" << limits::kMinSampleRate
- << " to " << limits::kMaxSampleRate << ").";
- return AVERROR(EINVAL);
- }
- // Determine how big the buffer should be and allocate it. FFmpeg may adjust
- // how big each channel data is in order to meet the alignment policy, so
- // we need to take this into consideration.
- int buffer_size_in_bytes = av_samples_get_buffer_size(
- &frame->linesize[0], channels, frame->nb_samples, format,
- 0 /* align, use ffmpeg default */);
- // Check for errors from av_samples_get_buffer_size().
- if (buffer_size_in_bytes < 0)
- return buffer_size_in_bytes;
- int frames_required = buffer_size_in_bytes / bytes_per_channel / channels;
- DCHECK_GE(frames_required, frame->nb_samples);
- ChannelLayout channel_layout =
- config_.channel_layout() == CHANNEL_LAYOUT_DISCRETE
- ? CHANNEL_LAYOUT_DISCRETE
- : ChannelLayoutToChromeChannelLayout(s->ch_layout.u.mask,
- s->ch_layout.nb_channels);
- if (channel_layout == CHANNEL_LAYOUT_UNSUPPORTED) {
- DLOG(ERROR) << "Unsupported channel layout.";
- return AVERROR(EINVAL);
- }
- scoped_refptr<AudioBuffer> buffer =
- AudioBuffer::CreateBuffer(sample_format, channel_layout, channels,
- s->sample_rate, frames_required, pool_);
- // Initialize the data[] and extended_data[] fields to point into the memory
- // allocated for AudioBuffer. |number_of_planes| will be 1 for interleaved
- // audio and equal to |channels| for planar audio.
- int number_of_planes = buffer->channel_data().size();
- if (number_of_planes <= AV_NUM_DATA_POINTERS) {
- DCHECK_EQ(frame->extended_data, frame->data);
- for (int i = 0; i < number_of_planes; ++i)
- frame->data[i] = buffer->channel_data()[i];
- } else {
- // There are more channels than can fit into data[], so allocate
- // extended_data[] and fill appropriately.
- frame->extended_data = static_cast<uint8_t**>(
- av_malloc(number_of_planes * sizeof(*frame->extended_data)));
- int i = 0;
- for (; i < AV_NUM_DATA_POINTERS; ++i)
- frame->extended_data[i] = frame->data[i] = buffer->channel_data()[i];
- for (; i < number_of_planes; ++i)
- frame->extended_data[i] = buffer->channel_data()[i];
- }
- // Now create an AVBufferRef for the data just allocated. It will own the
- // reference to the AudioBuffer object.
- AudioBuffer* opaque = buffer.get();
- opaque->AddRef();
- frame->buf[0] = av_buffer_create(frame->data[0], buffer_size_in_bytes,
- ReleaseAudioBufferImpl, opaque, 0);
- return 0;
- }
- } // namespace media
|