web_engine_audio_renderer.cc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "fuchsia_web/webengine/renderer/web_engine_audio_renderer.h"
  5. #include <lib/sys/cpp/component_context.h>
  6. #include "base/bind.h"
  7. #include "base/fuchsia/fuchsia_logging.h"
  8. #include "base/logging.h"
  9. #include "base/task/sequenced_task_runner.h"
  10. #include "base/threading/sequenced_task_runner_handle.h"
  11. #include "base/threading/thread_task_runner_handle.h"
  12. #include "base/time/time.h"
  13. #include "media/base/cdm_context.h"
  14. #include "media/base/decoder_buffer.h"
  15. #include "media/base/renderer_client.h"
  16. #include "media/fuchsia/cdm/fuchsia_cdm_context.h"
  17. #include "media/fuchsia/common/decrypting_sysmem_buffer_stream.h"
  18. #include "media/fuchsia/common/passthrough_sysmem_buffer_stream.h"
  19. namespace {
  20. // nullopt is returned in case the codec is not supported. nullptr is returned
  21. // for uncompressed PCM streams.
  22. absl::optional<std::unique_ptr<fuchsia::media::Compression>>
  23. GetFuchsiaCompressionFromDecoderConfig(media::AudioDecoderConfig config) {
  24. auto compression = std::make_unique<fuchsia::media::Compression>();
  25. switch (config.codec()) {
  26. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  27. case media::AudioCodec::kAAC:
  28. compression->type = fuchsia::media::AUDIO_ENCODING_AAC;
  29. break;
  30. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  31. case media::AudioCodec::kMP3:
  32. compression->type = fuchsia::media::AUDIO_ENCODING_MP3;
  33. break;
  34. case media::AudioCodec::kVorbis:
  35. compression->type = fuchsia::media::AUDIO_ENCODING_VORBIS;
  36. break;
  37. case media::AudioCodec::kOpus:
  38. compression->type = fuchsia::media::AUDIO_ENCODING_OPUS;
  39. break;
  40. case media::AudioCodec::kFLAC:
  41. compression->type = fuchsia::media::AUDIO_ENCODING_FLAC;
  42. break;
  43. case media::AudioCodec::kPCM:
  44. compression.reset();
  45. break;
  46. default:
  47. return absl::nullopt;
  48. }
  49. if (!config.extra_data().empty()) {
  50. compression->parameters = config.extra_data();
  51. }
  52. return std::move(compression);
  53. }
  54. absl::optional<fuchsia::media::AudioSampleFormat>
  55. GetFuchsiaSampleFormatFromSampleFormat(media::SampleFormat sample_format) {
  56. switch (sample_format) {
  57. case media::kSampleFormatU8:
  58. return fuchsia::media::AudioSampleFormat::UNSIGNED_8;
  59. case media::kSampleFormatS16:
  60. return fuchsia::media::AudioSampleFormat::SIGNED_16;
  61. case media::kSampleFormatS24:
  62. return fuchsia::media::AudioSampleFormat::SIGNED_24_IN_32;
  63. case media::kSampleFormatF32:
  64. return fuchsia::media::AudioSampleFormat::FLOAT;
  65. default:
  66. return absl::nullopt;
  67. }
  68. }
  69. // Helper that converts a PCM stream in kStreamFormatS24 to the layout
  70. // expected by AudioConsumer (i.e. SIGNED_24_IN_32).
  71. scoped_refptr<media::DecoderBuffer> PreparePcm24Buffer(
  72. scoped_refptr<media::DecoderBuffer> buffer) {
  73. static_assert(ARCH_CPU_LITTLE_ENDIAN,
  74. "Only little-endian CPUs are supported.");
  75. size_t samples = buffer->data_size() / 3;
  76. scoped_refptr<media::DecoderBuffer> result =
  77. base::MakeRefCounted<media::DecoderBuffer>(samples * 4);
  78. for (size_t i = 0; i < samples - 1; ++i) {
  79. reinterpret_cast<uint32_t*>(result->writable_data())[i] =
  80. *reinterpret_cast<const uint32_t*>(buffer->data() + i * 3) & 0x00ffffff;
  81. }
  82. size_t last_sample = samples - 1;
  83. reinterpret_cast<uint32_t*>(result->writable_data())[last_sample] =
  84. buffer->data()[last_sample * 3] |
  85. (buffer->data()[last_sample * 3 + 1] << 8) |
  86. (buffer->data()[last_sample * 3 + 2] << 16);
  87. result->set_timestamp(buffer->timestamp());
  88. result->set_duration(buffer->duration());
  89. if (buffer->decrypt_config())
  90. result->set_decrypt_config(buffer->decrypt_config()->Clone());
  91. return result;
  92. }
  93. } // namespace
  94. // Size of a single audio buffer: 100kB. It's enough to cover 100ms of PCM at
  95. // 48kHz, 2 channels, 16 bps.
  96. constexpr size_t kBufferSize = 100 * 1024;
  97. // Total number of buffers. 16 is the maximum allowed by AudioConsumer.
  98. constexpr size_t kNumBuffers = 16;
  99. WebEngineAudioRenderer::WebEngineAudioRenderer(
  100. media::MediaLog* media_log,
  101. fidl::InterfaceHandle<fuchsia::media::AudioConsumer> audio_consumer_handle)
  102. : audio_consumer_handle_(std::move(audio_consumer_handle)) {
  103. DETACH_FROM_THREAD(thread_checker_);
  104. }
  105. WebEngineAudioRenderer::~WebEngineAudioRenderer() {
  106. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  107. }
  108. void WebEngineAudioRenderer::Initialize(media::DemuxerStream* stream,
  109. media::CdmContext* cdm_context,
  110. media::RendererClient* client,
  111. media::PipelineStatusCallback init_cb) {
  112. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  113. DCHECK(!demuxer_stream_);
  114. DCHECK(!init_cb_);
  115. init_cb_ = std::move(init_cb);
  116. cdm_context_ = cdm_context;
  117. demuxer_stream_ = stream;
  118. client_ = client;
  119. audio_consumer_.Bind(std::move(audio_consumer_handle_));
  120. audio_consumer_.set_error_handler([this](zx_status_t status) {
  121. ZX_LOG(ERROR, status) << "AudioConsumer disconnected.";
  122. OnError(media::AUDIO_RENDERER_ERROR);
  123. });
  124. UpdateVolume();
  125. audio_consumer_.events().OnEndOfStream = [this]() { OnEndOfStream(); };
  126. RequestAudioConsumerStatus();
  127. InitializeStream();
  128. // Call `init_cb_`, unless it's been called by OnError().
  129. if (init_cb_) {
  130. std::move(init_cb_).Run(media::PIPELINE_OK);
  131. }
  132. }
  133. void WebEngineAudioRenderer::InitializeStream() {
  134. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  135. // AAC streams require bitstream conversion. Without it the demuxer may
  136. // produce decoded stream without ADTS headers which are required for AAC
  137. // streams in AudioConsumer.
  138. // TODO(crbug.com/1120095): Reconsider this logic.
  139. if (demuxer_stream_->audio_decoder_config().codec() ==
  140. media::AudioCodec::kAAC) {
  141. demuxer_stream_->EnableBitstreamConverter();
  142. }
  143. #endif // BUILDFLAG(USE_PROPRIETARY_CODECS)
  144. if (demuxer_stream_->audio_decoder_config().is_encrypted()) {
  145. if (!cdm_context_) {
  146. DLOG(ERROR) << "No cdm context for encrypted stream.";
  147. OnError(media::AUDIO_RENDERER_ERROR);
  148. return;
  149. }
  150. media::FuchsiaCdmContext* fuchsia_cdm =
  151. cdm_context_->GetFuchsiaCdmContext();
  152. if (fuchsia_cdm) {
  153. sysmem_buffer_stream_ = fuchsia_cdm->CreateStreamDecryptor(false);
  154. } else {
  155. sysmem_buffer_stream_ =
  156. std::make_unique<media::DecryptingSysmemBufferStream>(
  157. &sysmem_allocator_, cdm_context_, media::Decryptor::kAudio);
  158. }
  159. } else {
  160. sysmem_buffer_stream_ =
  161. std::make_unique<media::PassthroughSysmemBufferStream>(
  162. &sysmem_allocator_);
  163. }
  164. sysmem_buffer_stream_->Initialize(this, kBufferSize, kNumBuffers);
  165. }
  166. void WebEngineAudioRenderer::UpdateVolume() {
  167. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  168. DCHECK(audio_consumer_);
  169. if (!volume_control_) {
  170. audio_consumer_->BindVolumeControl(volume_control_.NewRequest());
  171. volume_control_.set_error_handler([](zx_status_t status) {
  172. ZX_LOG(ERROR, status) << "VolumeControl disconnected.";
  173. });
  174. }
  175. volume_control_->SetVolume(volume_);
  176. }
  177. void WebEngineAudioRenderer::OnBuffersAcquired(
  178. std::vector<media::VmoBuffer> buffers,
  179. const fuchsia::sysmem::SingleBufferSettings&) {
  180. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  181. input_buffers_ = std::move(buffers);
  182. InitializeStreamSink();
  183. while (!delayed_packets_.empty()) {
  184. auto packet = std::move(delayed_packets_.front());
  185. delayed_packets_.pop_front();
  186. SendInputPacket(std::move(packet));
  187. }
  188. if (has_delayed_end_of_stream_) {
  189. has_delayed_end_of_stream_ = false;
  190. OnSysmemBufferStreamEndOfStream();
  191. }
  192. }
  193. void WebEngineAudioRenderer::InitializeStreamSink() {
  194. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  195. DCHECK(!stream_sink_);
  196. // Clone |buffers| to pass to StreamSink.
  197. std::vector<zx::vmo> vmos_for_stream_sink;
  198. vmos_for_stream_sink.reserve(input_buffers_.size());
  199. for (media::VmoBuffer& buffer : input_buffers_) {
  200. vmos_for_stream_sink.push_back(buffer.Duplicate(/*writable=*/false));
  201. }
  202. auto config = demuxer_stream_->audio_decoder_config();
  203. auto compression = GetFuchsiaCompressionFromDecoderConfig(config);
  204. if (!compression) {
  205. LOG(ERROR) << "Unsupported audio codec: " << GetCodecName(config.codec());
  206. OnError(media::AUDIO_RENDERER_ERROR);
  207. return;
  208. }
  209. fuchsia::media::AudioStreamType stream_type;
  210. stream_type.channels = config.channels();
  211. stream_type.frames_per_second = config.samples_per_second();
  212. // Set sample_format for uncompressed streams.
  213. if (!compression.value()) {
  214. absl::optional<fuchsia::media::AudioSampleFormat> sample_format =
  215. GetFuchsiaSampleFormatFromSampleFormat(config.sample_format());
  216. if (!sample_format) {
  217. LOG(ERROR) << "Unsupported sample format: "
  218. << SampleFormatToString(config.sample_format());
  219. OnError(media::AUDIO_RENDERER_ERROR);
  220. return;
  221. }
  222. stream_type.sample_format = sample_format.value();
  223. } else {
  224. // For compressed formats sample format is determined by the decoder, but
  225. // this field is still required in AudioStreamType.
  226. stream_type.sample_format = fuchsia::media::AudioSampleFormat::SIGNED_16;
  227. }
  228. audio_consumer_->CreateStreamSink(
  229. std::move(vmos_for_stream_sink), std::move(stream_type),
  230. std::move(compression).value(), stream_sink_.NewRequest());
  231. if (GetPlaybackState() == PlaybackState::kStartPending)
  232. StartAudioConsumer();
  233. ScheduleReadDemuxerStream();
  234. }
  235. media::TimeSource* WebEngineAudioRenderer::GetTimeSource() {
  236. return this;
  237. }
  238. void WebEngineAudioRenderer::Flush(base::OnceClosure callback) {
  239. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  240. FlushInternal();
  241. renderer_started_ = false;
  242. std::move(callback).Run();
  243. }
  244. void WebEngineAudioRenderer::StartPlaying() {
  245. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  246. renderer_started_ = true;
  247. ScheduleReadDemuxerStream();
  248. }
  249. void WebEngineAudioRenderer::SetVolume(float volume) {
  250. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  251. volume_ = volume;
  252. if (audio_consumer_)
  253. UpdateVolume();
  254. }
  255. void WebEngineAudioRenderer::SetLatencyHint(
  256. absl::optional<base::TimeDelta> latency_hint) {
  257. // TODO(crbug.com/1131116): Implement at some later date after we've vetted
  258. // the API shape and usefulness outside of fuchsia.
  259. NOTIMPLEMENTED();
  260. }
  261. void WebEngineAudioRenderer::SetPreservesPitch(bool preserves_pitch) {
  262. NOTIMPLEMENTED();
  263. }
  264. void WebEngineAudioRenderer::SetWasPlayedWithUserActivation(
  265. bool was_played_with_user_activation) {
  266. NOTIMPLEMENTED();
  267. }
  268. void WebEngineAudioRenderer::StartTicking() {
  269. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  270. // If StreamSink hasn't been created yet, then delay starting AudioConsumer
  271. // until StreamSink is created.
  272. if (!stream_sink_) {
  273. base::AutoLock lock(timeline_lock_);
  274. SetPlaybackState(PlaybackState::kStartPending);
  275. return;
  276. }
  277. StartAudioConsumer();
  278. }
  279. void WebEngineAudioRenderer::StartAudioConsumer() {
  280. DCHECK(stream_sink_);
  281. fuchsia::media::AudioConsumerStartFlags flags{};
  282. if (demuxer_stream_->liveness() == media::StreamLiveness::kLive) {
  283. flags = fuchsia::media::AudioConsumerStartFlags::LOW_LATENCY;
  284. }
  285. // Stop the AudioConsumer if it's been started.
  286. switch (GetPlaybackState()) {
  287. case PlaybackState::kStopped:
  288. case PlaybackState::kStartPending:
  289. break;
  290. case PlaybackState::kStarting:
  291. case PlaybackState::kPlaying:
  292. audio_consumer_->Stop();
  293. break;
  294. }
  295. base::TimeDelta media_pos;
  296. {
  297. base::AutoLock lock(timeline_lock_);
  298. media_pos = media_pos_;
  299. SetPlaybackState(PlaybackState::kStarting);
  300. }
  301. audio_consumer_->Start(flags, fuchsia::media::NO_TIMESTAMP,
  302. media_pos.ToZxDuration());
  303. }
  304. void WebEngineAudioRenderer::StopTicking() {
  305. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  306. DCHECK(GetPlaybackState() != PlaybackState::kStopped);
  307. audio_consumer_->Stop();
  308. base::AutoLock lock(timeline_lock_);
  309. UpdateTimelineOnStop();
  310. SetPlaybackState(PlaybackState::kStopped);
  311. }
  312. void WebEngineAudioRenderer::SetPlaybackRate(double playback_rate) {
  313. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  314. audio_consumer_->SetRate(playback_rate);
  315. // AudioConsumer will update media timeline asynchronously. That update is
  316. // processed in OnAudioConsumerStatusChanged(). This might cause the clock to
  317. // go back. It's not desirable, e.g. because VideoRenderer could drop some
  318. // video frames that should be shown when the stream is resumed. To avoid this
  319. // issue update the timeline synchronously. OnAudioConsumerStatusChanged()
  320. // will still process the update from AudioConsumer to save the position when
  321. // the stream was actually paused, but that update would not move the clock
  322. // backward.
  323. if (playback_rate == 0.0) {
  324. base::AutoLock lock(timeline_lock_);
  325. UpdateTimelineOnStop();
  326. }
  327. }
  328. void WebEngineAudioRenderer::SetMediaTime(base::TimeDelta time) {
  329. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  330. DCHECK(GetPlaybackState() == PlaybackState::kStopped);
  331. {
  332. base::AutoLock lock(timeline_lock_);
  333. media_pos_ = time;
  334. // Reset reference timestamp. This is necessary to ensure that the correct
  335. // value is returned from GetWallClockTimes() until playback is resumed:
  336. // GetWallClockTimes() is required to return 0 wall clock between
  337. // SetMediaTime() and StartTicking().
  338. reference_time_ = base::TimeTicks();
  339. }
  340. FlushInternal();
  341. ScheduleReadDemuxerStream();
  342. }
  343. base::TimeDelta WebEngineAudioRenderer::CurrentMediaTime() {
  344. base::AutoLock lock(timeline_lock_);
  345. if (!IsTimeMoving())
  346. return media_pos_;
  347. return CurrentMediaTimeLocked();
  348. }
  349. bool WebEngineAudioRenderer::GetWallClockTimes(
  350. const std::vector<base::TimeDelta>& media_timestamps,
  351. std::vector<base::TimeTicks>* wall_clock_times) {
  352. wall_clock_times->reserve(media_timestamps.size());
  353. auto now = base::TimeTicks::Now();
  354. base::AutoLock lock(timeline_lock_);
  355. const bool is_time_moving = IsTimeMoving();
  356. if (media_timestamps.empty()) {
  357. wall_clock_times->push_back(is_time_moving ? now : reference_time_);
  358. return is_time_moving;
  359. }
  360. base::TimeTicks wall_clock_base = is_time_moving ? reference_time_ : now;
  361. for (base::TimeDelta timestamp : media_timestamps) {
  362. base::TimeTicks wall_clock_time;
  363. auto relative_pos = timestamp - media_pos_;
  364. if (is_time_moving) {
  365. // See https://fuchsia.dev/reference/fidl/fuchsia.media#formulas .
  366. relative_pos = relative_pos * reference_delta_ / media_delta_;
  367. }
  368. wall_clock_time = wall_clock_base + relative_pos;
  369. wall_clock_times->push_back(wall_clock_time);
  370. }
  371. return is_time_moving;
  372. }
  373. WebEngineAudioRenderer::PlaybackState
  374. WebEngineAudioRenderer::GetPlaybackState() {
  375. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  376. return state_;
  377. }
  378. void WebEngineAudioRenderer::SetPlaybackState(PlaybackState state) {
  379. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  380. state_ = state;
  381. }
  382. void WebEngineAudioRenderer::OnError(media::PipelineStatus status) {
  383. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  384. audio_consumer_.Unbind();
  385. stream_sink_.Unbind();
  386. sysmem_buffer_stream_.reset();
  387. if (is_demuxer_read_pending_) {
  388. drop_next_demuxer_read_result_ = true;
  389. }
  390. if (init_cb_) {
  391. std::move(init_cb_).Run(status);
  392. } else if (client_) {
  393. client_->OnError(status);
  394. }
  395. }
  396. void WebEngineAudioRenderer::RequestAudioConsumerStatus() {
  397. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  398. audio_consumer_->WatchStatus(fit::bind_member(
  399. this, &WebEngineAudioRenderer::OnAudioConsumerStatusChanged));
  400. }
  401. void WebEngineAudioRenderer::OnAudioConsumerStatusChanged(
  402. fuchsia::media::AudioConsumerStatus status) {
  403. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  404. if (status.has_error()) {
  405. LOG(ERROR) << "fuchsia::media::AudioConsumer reported an error";
  406. OnError(media::AUDIO_RENDERER_ERROR);
  407. return;
  408. }
  409. bool reschedule_read_timer = false;
  410. if (status.has_presentation_timeline()) {
  411. if (GetPlaybackState() != PlaybackState::kStopped) {
  412. base::AutoLock lock(timeline_lock_);
  413. if (GetPlaybackState() == PlaybackState::kStarting) {
  414. SetPlaybackState(PlaybackState::kPlaying);
  415. }
  416. reference_time_ = base::TimeTicks::FromZxTime(
  417. status.presentation_timeline().reference_time);
  418. media_pos_ = base::TimeDelta::FromZxDuration(
  419. status.presentation_timeline().subject_time);
  420. reference_delta_ = status.presentation_timeline().reference_delta;
  421. media_delta_ = status.presentation_timeline().subject_delta;
  422. reschedule_read_timer = true;
  423. }
  424. }
  425. if (status.has_min_lead_time()) {
  426. auto new_min_lead_time =
  427. base::TimeDelta::FromZxDuration(status.min_lead_time());
  428. DCHECK(!new_min_lead_time.is_zero());
  429. if (new_min_lead_time != min_lead_time_) {
  430. min_lead_time_ = new_min_lead_time;
  431. reschedule_read_timer = true;
  432. }
  433. }
  434. if (status.has_max_lead_time()) {
  435. auto new_max_lead_time =
  436. base::TimeDelta::FromZxDuration(status.max_lead_time());
  437. DCHECK(!new_max_lead_time.is_zero());
  438. if (new_max_lead_time != max_lead_time_) {
  439. max_lead_time_ = new_max_lead_time;
  440. reschedule_read_timer = true;
  441. }
  442. }
  443. if (reschedule_read_timer) {
  444. read_timer_.Stop();
  445. ScheduleReadDemuxerStream();
  446. }
  447. RequestAudioConsumerStatus();
  448. }
  449. void WebEngineAudioRenderer::ScheduleReadDemuxerStream() {
  450. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  451. if (!renderer_started_ || !demuxer_stream_ || read_timer_.IsRunning() ||
  452. is_demuxer_read_pending_ || is_at_end_of_stream_) {
  453. return;
  454. }
  455. base::TimeDelta next_read_delay;
  456. if (!last_packet_timestamp_.is_min()) {
  457. std::vector<base::TimeTicks> wall_clock_times;
  458. bool is_time_moving =
  459. GetWallClockTimes({last_packet_timestamp_}, &wall_clock_times);
  460. base::TimeDelta relative_buffer_pos =
  461. wall_clock_times[0] - base::TimeTicks::Now();
  462. // Check if we have buffered more than |max_lead_time_|.
  463. if (relative_buffer_pos >= max_lead_time_) {
  464. // If playback is not active then there is no need to buffer more.
  465. if (!is_time_moving)
  466. return;
  467. // If the buffer is larger than |max_lead_time_|, then the next read
  468. // should be delayed.
  469. next_read_delay = relative_buffer_pos - max_lead_time_;
  470. }
  471. }
  472. read_timer_.Start(FROM_HERE, next_read_delay,
  473. base::BindOnce(&WebEngineAudioRenderer::ReadDemuxerStream,
  474. base::Unretained(this)));
  475. }
  476. void WebEngineAudioRenderer::ReadDemuxerStream() {
  477. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  478. DCHECK(demuxer_stream_);
  479. DCHECK(!is_demuxer_read_pending_);
  480. is_demuxer_read_pending_ = true;
  481. demuxer_stream_->Read(
  482. base::BindOnce(&WebEngineAudioRenderer::OnDemuxerStreamReadDone,
  483. weak_factory_.GetWeakPtr()));
  484. }
  485. void WebEngineAudioRenderer::OnDemuxerStreamReadDone(
  486. media::DemuxerStream::Status read_status,
  487. scoped_refptr<media::DecoderBuffer> buffer) {
  488. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  489. DCHECK(is_demuxer_read_pending_);
  490. is_demuxer_read_pending_ = false;
  491. if (drop_next_demuxer_read_result_) {
  492. drop_next_demuxer_read_result_ = false;
  493. ScheduleReadDemuxerStream();
  494. return;
  495. }
  496. if (read_status != media::DemuxerStream::kOk) {
  497. if (read_status == media::DemuxerStream::kError) {
  498. OnError(media::PIPELINE_ERROR_READ);
  499. } else if (read_status == media::DemuxerStream::kConfigChanged) {
  500. stream_sink_.Unbind();
  501. // Re-initialize the stream for the new config.
  502. InitializeStream();
  503. // Continue reading the stream. Decryptor won't finish output buffer
  504. // initialization until it starts receiving data on the input.
  505. ScheduleReadDemuxerStream();
  506. client_->OnAudioConfigChange(demuxer_stream_->audio_decoder_config());
  507. } else {
  508. DCHECK_EQ(read_status, media::DemuxerStream::kAborted);
  509. }
  510. return;
  511. }
  512. if (buffer->end_of_stream()) {
  513. is_at_end_of_stream_ = true;
  514. } else {
  515. if (buffer->data_size() > kBufferSize) {
  516. DLOG(ERROR) << "Demuxer returned buffer that is too big: "
  517. << buffer->data_size();
  518. OnError(media::AUDIO_RENDERER_ERROR);
  519. return;
  520. }
  521. last_packet_timestamp_ = buffer->timestamp();
  522. if (buffer->duration() != media::kNoTimestamp)
  523. last_packet_timestamp_ += buffer->duration();
  524. }
  525. // Update layout for 24-bit PCM streams.
  526. if (!buffer->end_of_stream() &&
  527. demuxer_stream_->audio_decoder_config().codec() ==
  528. media::AudioCodec::kPCM &&
  529. demuxer_stream_->audio_decoder_config().sample_format() ==
  530. media::kSampleFormatS24) {
  531. buffer = PreparePcm24Buffer(std::move(buffer));
  532. }
  533. sysmem_buffer_stream_->EnqueueBuffer(std::move(buffer));
  534. ScheduleReadDemuxerStream();
  535. }
  536. void WebEngineAudioRenderer::SendInputPacket(
  537. media::StreamProcessorHelper::IoPacket packet) {
  538. const auto packet_size = packet.size();
  539. fuchsia::media::StreamPacket stream_packet;
  540. stream_packet.payload_buffer_id = packet.buffer_index();
  541. stream_packet.pts = packet.timestamp().ToZxDuration();
  542. stream_packet.payload_offset = packet.offset();
  543. stream_packet.payload_size = packet_size;
  544. stream_sink_->SendPacket(
  545. std::move(stream_packet),
  546. [this, packet = std::make_unique<media::StreamProcessorHelper::IoPacket>(
  547. std::move(packet))]() mutable {
  548. OnStreamSendDone(std::move(packet));
  549. });
  550. // AudioConsumer doesn't report exact time when the data is decoded, but it's
  551. // safe to report it as decoded right away since the packet is expected to be
  552. // decoded soon after AudioConsumer receives it.
  553. media::PipelineStatistics stats;
  554. stats.audio_bytes_decoded = packet_size;
  555. client_->OnStatisticsUpdate(stats);
  556. }
  557. void WebEngineAudioRenderer::OnStreamSendDone(
  558. std::unique_ptr<media::StreamProcessorHelper::IoPacket> packet) {
  559. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  560. // Check if we need to update buffering state after sending more than
  561. // |min_lead_time_| to the AudioConsumer.
  562. if (buffer_state_ == media::BUFFERING_HAVE_NOTHING) {
  563. std::vector<base::TimeTicks> wall_clock_times;
  564. GetWallClockTimes({packet->timestamp()}, &wall_clock_times);
  565. base::TimeDelta relative_buffer_pos =
  566. wall_clock_times[0] - base::TimeTicks::Now();
  567. if (relative_buffer_pos >= min_lead_time_)
  568. SetBufferState(media::BUFFERING_HAVE_ENOUGH);
  569. }
  570. ScheduleReadDemuxerStream();
  571. }
  572. void WebEngineAudioRenderer::SetBufferState(
  573. media::BufferingState buffer_state) {
  574. if (buffer_state != buffer_state_) {
  575. buffer_state_ = buffer_state;
  576. client_->OnBufferingStateChange(buffer_state_,
  577. media::BUFFERING_CHANGE_REASON_UNKNOWN);
  578. }
  579. }
  580. void WebEngineAudioRenderer::FlushInternal() {
  581. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  582. DCHECK(GetPlaybackState() == PlaybackState::kStopped || is_at_end_of_stream_);
  583. if (stream_sink_)
  584. stream_sink_->DiscardAllPacketsNoReply();
  585. SetBufferState(media::BUFFERING_HAVE_NOTHING);
  586. last_packet_timestamp_ = base::TimeDelta::Min();
  587. read_timer_.Stop();
  588. is_at_end_of_stream_ = false;
  589. if (is_demuxer_read_pending_) {
  590. drop_next_demuxer_read_result_ = true;
  591. }
  592. }
  593. void WebEngineAudioRenderer::OnEndOfStream() {
  594. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  595. client_->OnEnded();
  596. }
  597. bool WebEngineAudioRenderer::IsTimeMoving() {
  598. return state_ == PlaybackState::kPlaying && media_delta_ > 0;
  599. }
  600. void WebEngineAudioRenderer::UpdateTimelineOnStop() {
  601. if (!IsTimeMoving())
  602. return;
  603. media_pos_ = CurrentMediaTimeLocked();
  604. reference_time_ = base::TimeTicks::Now();
  605. media_delta_ = 0;
  606. }
  607. base::TimeDelta WebEngineAudioRenderer::CurrentMediaTimeLocked() {
  608. DCHECK(IsTimeMoving());
  609. // Calculate media position using formula specified by the TimelineFunction.
  610. // See https://fuchsia.dev/reference/fidl/fuchsia.media#formulas .
  611. return media_pos_ + (base::TimeTicks::Now() - reference_time_) *
  612. media_delta_ / reference_delta_;
  613. }
  614. void WebEngineAudioRenderer::OnSysmemBufferStreamBufferCollectionToken(
  615. fuchsia::sysmem::BufferCollectionTokenPtr token) {
  616. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  617. // Drop old buffers.
  618. input_buffers_.clear();
  619. stream_sink_.Unbind();
  620. // Acquire buffers for the new buffer collection.
  621. input_buffer_collection_ =
  622. sysmem_allocator_.BindSharedCollection(std::move(token));
  623. fuchsia::sysmem::BufferCollectionConstraints buffer_constraints =
  624. media::VmoBuffer::GetRecommendedConstraints(kNumBuffers, kBufferSize,
  625. /*writable=*/false);
  626. input_buffer_collection_->Initialize(std::move(buffer_constraints),
  627. "CrAudioRenderer");
  628. input_buffer_collection_->AcquireBuffers(base::BindOnce(
  629. &WebEngineAudioRenderer::OnBuffersAcquired, base::Unretained(this)));
  630. }
  631. void WebEngineAudioRenderer::OnSysmemBufferStreamOutputPacket(
  632. media::StreamProcessorHelper::IoPacket packet) {
  633. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  634. if (stream_sink_) {
  635. SendInputPacket(std::move(packet));
  636. } else {
  637. // The packet will be sent after StreamSink is connected.
  638. delayed_packets_.push_back(std::move(packet));
  639. }
  640. ScheduleReadDemuxerStream();
  641. }
  642. void WebEngineAudioRenderer::OnSysmemBufferStreamEndOfStream() {
  643. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  644. DCHECK(is_at_end_of_stream_);
  645. // Stream sink is not bound yet, queue EOS request until then.
  646. if (!stream_sink_) {
  647. has_delayed_end_of_stream_ = true;
  648. return;
  649. }
  650. stream_sink_->EndOfStream();
  651. // No more data is going to be buffered. Update buffering state to ensure
  652. // RendererImpl starts playback in case it was waiting for buffering to
  653. // finish.
  654. SetBufferState(media::BUFFERING_HAVE_ENOUGH);
  655. }
  656. void WebEngineAudioRenderer::OnSysmemBufferStreamError() {
  657. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  658. OnError(media::AUDIO_RENDERER_ERROR);
  659. }
  660. void WebEngineAudioRenderer::OnSysmemBufferStreamNoKey() {
  661. DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
  662. client_->OnWaiting(media::WaitingReason::kNoDecryptionKey);
  663. }