123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474 |
- // Copyright 2020 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "fuchsia_web/webengine/renderer/web_engine_audio_output_device.h"
- #include "base/fuchsia/fuchsia_logging.h"
- #include "base/logging.h"
- #include "base/memory/shared_memory_mapping.h"
- #include "base/memory/writable_shared_memory_region.h"
- #include "base/no_destructor.h"
- #include "base/threading/thread.h"
- #include "base/threading/thread_task_runner_handle.h"
- #include "media/base/audio_timestamp_helper.h"
- namespace {
- // Total number of buffers used for AudioConsumer.
- constexpr size_t kNumBuffers = 4;
- // Extra lead time added to min_lead_time reported by AudioConsumer when
- // scheduling PumpSamples() timer. This is necessary to make it more likely
- // that each packet is sent on time, even if the timer is delayed. Higher values
- // increase playback latency, but make underflow less likely. 20ms allows to
- // keep latency reasonably low, while making playback reliable under normal
- // conditions.
- //
- // TODO(crbug.com/1153909): It may be possible to reduce this value to reduce
- // total latency, but that requires that an elevated scheduling profile is
- // applied to this thread.
- constexpr base::TimeDelta kLeadTimeExtra = base::Milliseconds(20);
- class DefaultAudioThread {
- public:
- DefaultAudioThread() : thread_("WebEngineAudioOutputDevice") {
- base::Thread::Options options(base::MessagePumpType::IO, 0);
- options.thread_type = base::ThreadType::kRealtimeAudio;
- thread_.StartWithOptions(std::move(options));
- }
- ~DefaultAudioThread() = default;
- scoped_refptr<base::SingleThreadTaskRunner> task_runner() {
- return thread_.task_runner();
- }
- private:
- base::Thread thread_;
- };
- scoped_refptr<base::SingleThreadTaskRunner> GetDefaultAudioTaskRunner() {
- static base::NoDestructor<DefaultAudioThread> default_audio_thread;
- return default_audio_thread->task_runner();
- }
- } // namespace
- // static
- scoped_refptr<WebEngineAudioOutputDevice> WebEngineAudioOutputDevice::Create(
- fidl::InterfaceHandle<fuchsia::media::AudioConsumer> audio_consumer_handle,
- scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
- scoped_refptr<WebEngineAudioOutputDevice> result(
- new WebEngineAudioOutputDevice(task_runner));
- task_runner->PostTask(
- FROM_HERE,
- base::BindOnce(
- &WebEngineAudioOutputDevice::BindAudioConsumerOnAudioThread, result,
- std::move(audio_consumer_handle)));
- return result;
- }
- // static
- scoped_refptr<WebEngineAudioOutputDevice>
- WebEngineAudioOutputDevice::CreateOnDefaultThread(
- fidl::InterfaceHandle<fuchsia::media::AudioConsumer>
- audio_consumer_handle) {
- return Create(std::move(audio_consumer_handle), GetDefaultAudioTaskRunner());
- }
- WebEngineAudioOutputDevice::WebEngineAudioOutputDevice(
- scoped_refptr<base::SingleThreadTaskRunner> task_runner)
- : task_runner_(std::move(task_runner)) {}
- WebEngineAudioOutputDevice::~WebEngineAudioOutputDevice() = default;
- void WebEngineAudioOutputDevice::Initialize(
- const media::AudioParameters& params,
- RenderCallback* callback) {
- DCHECK(callback);
- // Save |callback| synchronously here to handle the case when Stop() is called
- // before the DoInitialize() task is processed.
- {
- base::AutoLock auto_lock(callback_lock_);
- DCHECK(!callback_);
- callback_ = callback;
- }
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::InitializeOnAudioThread, this,
- params));
- }
- void WebEngineAudioOutputDevice::Start() {
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::StartOnAudioThread, this));
- }
- void WebEngineAudioOutputDevice::Stop() {
- {
- base::AutoLock auto_lock(callback_lock_);
- callback_ = nullptr;
- }
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::StopOnAudioThread, this));
- }
- void WebEngineAudioOutputDevice::Pause() {
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::PauseOnAudioThread, this));
- }
- void WebEngineAudioOutputDevice::Play() {
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::PlayOnAudioThread, this));
- }
- void WebEngineAudioOutputDevice::Flush() {
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::FlushOnAudioThread, this));
- }
- bool WebEngineAudioOutputDevice::SetVolume(double volume) {
- task_runner_->PostTask(
- FROM_HERE,
- base::BindOnce(&WebEngineAudioOutputDevice::SetVolumeOnAudioThread, this,
- volume));
- return true;
- }
- media::OutputDeviceInfo WebEngineAudioOutputDevice::GetOutputDeviceInfo() {
- // AudioConsumer doesn't provider any information about the output device.
- //
- // TODO(crbug.com/852834): Update this method when that functionality is
- // implemented.
- return media::OutputDeviceInfo(
- std::string(), media::OUTPUT_DEVICE_STATUS_OK,
- media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
- media::CHANNEL_LAYOUT_STEREO, 48000, 480));
- }
- void WebEngineAudioOutputDevice::GetOutputDeviceInfoAsync(
- OutputDeviceInfoCB info_cb) {
- std::move(info_cb).Run(GetOutputDeviceInfo());
- }
- bool WebEngineAudioOutputDevice::IsOptimizedForHardwareParameters() {
- // AudioConsumer doesn't provide device parameters (since target device may
- // change).
- return false;
- }
- bool WebEngineAudioOutputDevice::CurrentThreadIsRenderingThread() {
- return task_runner_->BelongsToCurrentThread();
- }
- void WebEngineAudioOutputDevice::BindAudioConsumerOnAudioThread(
- fidl::InterfaceHandle<fuchsia::media::AudioConsumer>
- audio_consumer_handle) {
- DCHECK(CurrentThreadIsRenderingThread());
- DCHECK(!audio_consumer_);
- audio_consumer_.Bind(std::move(audio_consumer_handle));
- audio_consumer_.set_error_handler([this](zx_status_t status) {
- ZX_LOG(ERROR, status) << "AudioConsumer disconnected.";
- ReportError();
- });
- }
- void WebEngineAudioOutputDevice::InitializeOnAudioThread(
- const media::AudioParameters& params) {
- DCHECK(CurrentThreadIsRenderingThread());
- params_ = params;
- audio_bus_ = media::AudioBus::Create(params_);
- UpdateVolume();
- WatchAudioConsumerStatus();
- }
- void WebEngineAudioOutputDevice::StartOnAudioThread() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!audio_consumer_)
- return;
- CreateStreamSink();
- media_pos_frames_ = 0;
- audio_consumer_->Start(fuchsia::media::AudioConsumerStartFlags::LOW_LATENCY,
- fuchsia::media::NO_TIMESTAMP, 0);
- // When AudioConsumer handles the Start() message sent above, it will update
- // its state and sent WatchStatus() response. OnAudioConsumerStatusChanged()
- // will then call SchedulePumpSamples() to start sending audio packets.
- }
- void WebEngineAudioOutputDevice::StopOnAudioThread() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!audio_consumer_)
- return;
- audio_consumer_->Stop();
- pump_samples_timer_.Stop();
- audio_consumer_.Unbind();
- stream_sink_.Unbind();
- volume_control_.Unbind();
- }
- void WebEngineAudioOutputDevice::PauseOnAudioThread() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!audio_consumer_)
- return;
- paused_ = true;
- audio_consumer_->SetRate(0.0);
- pump_samples_timer_.Stop();
- }
- void WebEngineAudioOutputDevice::PlayOnAudioThread() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!audio_consumer_)
- return;
- paused_ = false;
- audio_consumer_->SetRate(1.0);
- }
- void WebEngineAudioOutputDevice::FlushOnAudioThread() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!stream_sink_)
- return;
- stream_sink_->DiscardAllPacketsNoReply();
- }
- void WebEngineAudioOutputDevice::SetVolumeOnAudioThread(double volume) {
- DCHECK(CurrentThreadIsRenderingThread());
- volume_ = volume;
- if (audio_consumer_)
- UpdateVolume();
- }
- void WebEngineAudioOutputDevice::CreateStreamSink() {
- DCHECK(CurrentThreadIsRenderingThread());
- DCHECK(audio_consumer_);
- // Allocate buffers for the StreamSink.
- size_t buffer_size = params_.GetBytesPerBuffer(media::kSampleFormatF32);
- stream_sink_buffers_.reserve(kNumBuffers);
- available_buffers_indices_.clear();
- std::vector<zx::vmo> vmos_for_stream_sink;
- vmos_for_stream_sink.reserve(kNumBuffers);
- for (size_t i = 0; i < kNumBuffers; ++i) {
- auto region = base::WritableSharedMemoryRegion::Create(buffer_size);
- auto mapping = region.Map();
- if (!mapping.IsValid()) {
- LOG(WARNING) << "Failed to allocate VMO of size " << buffer_size;
- ReportError();
- return;
- }
- stream_sink_buffers_.push_back(std::move(mapping));
- available_buffers_indices_.push_back(i);
- auto read_only_region =
- base::WritableSharedMemoryRegion::ConvertToReadOnly(std::move(region));
- vmos_for_stream_sink.push_back(
- base::ReadOnlySharedMemoryRegion::TakeHandleForSerialization(
- std::move(read_only_region))
- .PassPlatformHandle());
- }
- // Configure StreamSink.
- fuchsia::media::AudioStreamType stream_type;
- stream_type.channels = params_.channels();
- stream_type.frames_per_second = params_.sample_rate();
- stream_type.sample_format = fuchsia::media::AudioSampleFormat::FLOAT;
- audio_consumer_->CreateStreamSink(std::move(vmos_for_stream_sink),
- std::move(stream_type), nullptr,
- stream_sink_.NewRequest());
- stream_sink_.set_error_handler([this](zx_status_t status) {
- ZX_LOG(ERROR, status) << "StreamSink disconnected.";
- ReportError();
- });
- }
- void WebEngineAudioOutputDevice::UpdateVolume() {
- DCHECK(CurrentThreadIsRenderingThread());
- DCHECK(audio_consumer_);
- if (!volume_control_) {
- audio_consumer_->BindVolumeControl(volume_control_.NewRequest());
- volume_control_.set_error_handler([](zx_status_t status) {
- ZX_LOG(ERROR, status) << "VolumeControl disconnected.";
- });
- }
- volume_control_->SetVolume(volume_);
- }
- void WebEngineAudioOutputDevice::WatchAudioConsumerStatus() {
- DCHECK(CurrentThreadIsRenderingThread());
- audio_consumer_->WatchStatus(fit::bind_member(
- this, &WebEngineAudioOutputDevice::OnAudioConsumerStatusChanged));
- }
- void WebEngineAudioOutputDevice::OnAudioConsumerStatusChanged(
- fuchsia::media::AudioConsumerStatus status) {
- DCHECK(CurrentThreadIsRenderingThread());
- if (!status.has_min_lead_time()) {
- DLOG(ERROR) << "AudioConsumerStatus.min_lead_time isn't set.";
- ReportError();
- return;
- }
- min_lead_time_ = base::Nanoseconds(status.min_lead_time());
- if (status.has_presentation_timeline()) {
- timeline_reference_time_ = base::TimeTicks::FromZxTime(
- status.presentation_timeline().reference_time);
- timeline_subject_time_ =
- base::Nanoseconds(status.presentation_timeline().subject_time);
- timeline_reference_delta_ = status.presentation_timeline().reference_delta;
- timeline_subject_delta_ = status.presentation_timeline().subject_delta;
- } else {
- // Reset |timeline_reference_time_| to null value, which is used to indicate
- // that there is no presentation timeline.
- timeline_reference_time_ = base::TimeTicks();
- }
- // Reschedule the timer for the new timeline.
- pump_samples_timer_.Stop();
- SchedulePumpSamples();
- WatchAudioConsumerStatus();
- }
- void WebEngineAudioOutputDevice::SchedulePumpSamples() {
- DCHECK(CurrentThreadIsRenderingThread());
- if (paused_ || timeline_reference_time_.is_null() ||
- pump_samples_timer_.IsRunning() || available_buffers_indices_.empty()) {
- return;
- }
- // Current position in the stream.
- auto media_pos = media::AudioTimestampHelper::FramesToTime(
- media_pos_frames_, params_.sample_rate());
- // Calculate expected playback time for the next sample based on the
- // presentation timeline provided by the AudioConsumer.
- // See https://fuchsia.dev/reference/fidl/fuchsia.media#formulas .
- // AudioConsumer uses monotonic clock (aka base::TimeTicks) as a reference
- // timeline. Subject timeline corresponds to position within the stream, which
- // is stored as |media_pos_frames_| and then passed in the |pts| field in each
- // packet produced in PumpSamples().
- auto playback_time = timeline_reference_time_ +
- (media_pos - timeline_subject_time_) *
- timeline_reference_delta_ / timeline_subject_delta_;
- base::TimeTicks now = base::TimeTicks::Now();
- // Target time for when PumpSamples() should run.
- base::TimeTicks target_time = playback_time - min_lead_time_ - kLeadTimeExtra;
- base::TimeDelta delay = target_time - now;
- pump_samples_timer_.Start(
- FROM_HERE, delay,
- base::BindOnce(&WebEngineAudioOutputDevice::PumpSamples, this,
- playback_time));
- }
- void WebEngineAudioOutputDevice::PumpSamples(base::TimeTicks playback_time) {
- DCHECK(CurrentThreadIsRenderingThread());
- auto now = base::TimeTicks::Now();
- int skipped_frames = 0;
- // Check if it's too late to send the next packet. If it is, then advance
- // current stream position.
- auto lead_time = playback_time - now;
- if (lead_time < min_lead_time_) {
- auto new_playback_time = now + min_lead_time_;
- auto skipped_time = new_playback_time - playback_time;
- skipped_frames = media::AudioTimestampHelper::TimeToFrames(
- skipped_time, params_.sample_rate());
- media_pos_frames_ += skipped_frames;
- playback_time += skipped_time;
- }
- int frames_filled;
- {
- base::AutoLock auto_lock(callback_lock_);
- // |callback_| may be reset in Stop(). No need to keep rendering the stream
- // in that case.
- if (!callback_)
- return;
- frames_filled = callback_->Render(playback_time - now, now, skipped_frames,
- audio_bus_.get());
- }
- if (frames_filled) {
- DCHECK(!available_buffers_indices_.empty());
- int buffer_index = available_buffers_indices_.back();
- available_buffers_indices_.pop_back();
- audio_bus_->ToInterleaved<media::Float32SampleTypeTraitsNoClip>(
- frames_filled,
- static_cast<float*>(stream_sink_buffers_[buffer_index].memory()));
- fuchsia::media::StreamPacket packet;
- packet.payload_buffer_id = buffer_index;
- packet.pts = media::AudioTimestampHelper::FramesToTime(
- media_pos_frames_, params_.sample_rate())
- .InNanoseconds();
- packet.payload_offset = 0;
- packet.payload_size = frames_filled * sizeof(float) * params_.channels();
- stream_sink_->SendPacket(std::move(packet), [this, buffer_index]() {
- OnStreamSendDone(buffer_index);
- });
- media_pos_frames_ += frames_filled;
- }
- SchedulePumpSamples();
- }
- void WebEngineAudioOutputDevice::OnStreamSendDone(size_t buffer_index) {
- DCHECK(CurrentThreadIsRenderingThread());
- available_buffers_indices_.push_back(buffer_index);
- SchedulePumpSamples();
- }
- void WebEngineAudioOutputDevice::ReportError() {
- DCHECK(CurrentThreadIsRenderingThread());
- audio_consumer_.Unbind();
- stream_sink_.Unbind();
- volume_control_.Unbind();
- pump_samples_timer_.Stop();
- {
- base::AutoLock auto_lock(callback_lock_);
- if (callback_)
- callback_->OnRenderError();
- }
- }
|