decoder_selector.cc 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/decoder_selector.h"
  5. #include <algorithm>
  6. #include <utility>
  7. #include "base/bind.h"
  8. #include "base/callback_helpers.h"
  9. #include "base/feature_list.h"
  10. #include "base/location.h"
  11. #include "base/logging.h"
  12. #include "base/metrics/histogram_functions.h"
  13. #include "base/strings/stringprintf.h"
  14. #include "base/task/sequenced_task_runner.h"
  15. #include "base/trace_event/trace_event.h"
  16. #include "build/build_config.h"
  17. #include "build/chromeos_buildflags.h"
  18. #include "media/base/audio_decoder.h"
  19. #include "media/base/cdm_context.h"
  20. #include "media/base/demuxer_stream.h"
  21. #include "media/base/media_log.h"
  22. #include "media/base/media_switches.h"
  23. #include "media/base/video_decoder.h"
  24. #include "media/filters/decoder_stream_traits.h"
  25. #include "media/filters/decrypting_demuxer_stream.h"
  26. namespace media {
  27. namespace {
  28. const char kSelectDecoderTrace[] = "DecoderSelector::SelectDecoder";
  29. bool SkipDecoderForRTC(const AudioDecoderConfig& /*config*/,
  30. const AudioDecoder& /*decoder*/) {
  31. return false;
  32. }
  33. bool SkipDecoderForRTC(const VideoDecoderConfig& config,
  34. const VideoDecoder& decoder) {
  35. // Skip non-platform decoders for rtc based on the feature flag.
  36. return config.is_rtc() && !decoder.IsPlatformDecoder() &&
  37. !base::FeatureList::IsEnabled(kExposeSwDecodersToWebRTC);
  38. }
  39. template <typename ConfigT, typename DecoderT>
  40. DecoderPriority NormalDecoderPriority(const ConfigT& config,
  41. const DecoderT& decoder) {
  42. if (SkipDecoderForRTC(config, decoder))
  43. return DecoderPriority::kSkipped;
  44. return DecoderPriority::kNormal;
  45. }
  46. DecoderPriority ResolutionBasedDecoderPriority(const VideoDecoderConfig& config,
  47. const VideoDecoder& decoder) {
  48. #if BUILDFLAG(IS_ANDROID)
  49. constexpr auto kSoftwareDecoderHeightCutoff = 360;
  50. #elif BUILDFLAG(IS_CHROMEOS_ASH)
  51. constexpr auto kSoftwareDecoderHeightCutoff = 360;
  52. #else
  53. constexpr auto kSoftwareDecoderHeightCutoff = 720;
  54. #endif
  55. if (SkipDecoderForRTC(config, decoder))
  56. return DecoderPriority::kSkipped;
  57. // We only do a height check to err on the side of prioritizing platform
  58. // decoders.
  59. const auto at_or_above_software_cutoff =
  60. config.visible_rect().height() >= kSoftwareDecoderHeightCutoff;
  61. // Platform decoders are deprioritized below the cutoff, and non-platform
  62. // decoders are deprioritized above it.
  63. return at_or_above_software_cutoff == decoder.IsPlatformDecoder()
  64. ? DecoderPriority::kNormal
  65. : DecoderPriority::kDeprioritized;
  66. }
  67. DecoderPriority PreferNonPlatformDecoders(const VideoDecoderConfig& config,
  68. const VideoDecoder& decoder) {
  69. // Prefer software decoders over hardware decoders. This is useful to force
  70. // software fallback for WebRTC, but still use hardware if there's no software
  71. // implementation to choose.
  72. return decoder.IsPlatformDecoder() ? DecoderPriority::kDeprioritized
  73. : DecoderPriority::kNormal;
  74. }
  75. DecoderPriority UnifiedDecoderPriority(const VideoDecoderConfig& config,
  76. const VideoDecoder& decoder) {
  77. if (config.is_rtc() ||
  78. base::FeatureList::IsEnabled(kResolutionBasedDecoderPriority)) {
  79. return ResolutionBasedDecoderPriority(config, decoder);
  80. } else {
  81. return NormalDecoderPriority(config, decoder);
  82. }
  83. }
  84. template <typename ConfigT, typename DecoderT>
  85. DecoderPriority SkipNonPlatformDecoders(const ConfigT& config,
  86. const DecoderT& decoder) {
  87. if (SkipDecoderForRTC(config, decoder))
  88. return DecoderPriority::kSkipped;
  89. return decoder.IsPlatformDecoder() ? DecoderPriority::kNormal
  90. : DecoderPriority::kSkipped;
  91. }
  92. void SetDefaultDecoderPriorityCB(
  93. VideoDecoderSelector::DecoderPriorityCB* out,
  94. const DecoderStreamTraits<DemuxerStream::VIDEO>* traits) {
  95. if (base::FeatureList::IsEnabled(kForceHardwareVideoDecoders)) {
  96. *out = base::BindRepeating(
  97. SkipNonPlatformDecoders<VideoDecoderConfig, VideoDecoder>);
  98. } else if (traits->GetPreferNonPlatformDecoders()) {
  99. *out = base::BindRepeating(PreferNonPlatformDecoders);
  100. } else {
  101. *out = base::BindRepeating(UnifiedDecoderPriority);
  102. }
  103. }
  104. void SetDefaultDecoderPriorityCB(
  105. AudioDecoderSelector::DecoderPriorityCB* out,
  106. const DecoderStreamTraits<DemuxerStream::AUDIO>*) {
  107. if (base::FeatureList::IsEnabled(kForceHardwareAudioDecoders)) {
  108. *out = base::BindRepeating(
  109. SkipNonPlatformDecoders<AudioDecoderConfig, AudioDecoder>);
  110. } else {
  111. // Platform audio decoders are not currently prioritized or deprioritized
  112. *out = base::BindRepeating(
  113. NormalDecoderPriority<AudioDecoderConfig, AudioDecoder>);
  114. }
  115. }
  116. } // namespace
  117. template <DemuxerStream::Type StreamType>
  118. DecoderSelector<StreamType>::DecoderSelector(
  119. scoped_refptr<base::SequencedTaskRunner> task_runner,
  120. CreateDecodersCB create_decoders_cb,
  121. MediaLog* media_log)
  122. : task_runner_(std::move(task_runner)),
  123. create_decoders_cb_(std::move(create_decoders_cb)),
  124. media_log_(media_log) {
  125. DETACH_FROM_SEQUENCE(sequence_checker_);
  126. }
  127. template <DemuxerStream::Type StreamType>
  128. DecoderSelector<StreamType>::~DecoderSelector() {
  129. DVLOG(2) << __func__;
  130. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  131. if (select_decoder_cb_)
  132. ReturnSelectionError(DecoderStatus::Codes::kFailed);
  133. }
  134. template <DemuxerStream::Type StreamType>
  135. void DecoderSelector<StreamType>::Initialize(StreamTraits* traits,
  136. DemuxerStream* stream,
  137. CdmContext* cdm_context,
  138. WaitingCB waiting_cb) {
  139. DVLOG(2) << __func__;
  140. DCHECK(traits);
  141. DCHECK(stream);
  142. traits_ = traits;
  143. stream_ = stream;
  144. cdm_context_ = cdm_context;
  145. waiting_cb_ = std::move(waiting_cb);
  146. // Only set this here if nobody has overridden it for tests.
  147. if (!decoder_priority_cb_)
  148. SetDefaultDecoderPriorityCB(&decoder_priority_cb_, traits_);
  149. }
  150. template <DemuxerStream::Type StreamType>
  151. void DecoderSelector<StreamType>::SelectDecoderInternal(
  152. SelectDecoderCB select_decoder_cb,
  153. typename Decoder::OutputCB output_cb,
  154. bool needs_new_decoders) {
  155. DVLOG(2) << __func__;
  156. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  157. DCHECK(select_decoder_cb);
  158. DCHECK(!select_decoder_cb_);
  159. select_decoder_cb_ = std::move(select_decoder_cb);
  160. output_cb_ = std::move(output_cb);
  161. config_ = traits_->GetDecoderConfig(stream_);
  162. TRACE_EVENT_ASYNC_BEGIN2("media", kSelectDecoderTrace, this, "type",
  163. DemuxerStream::GetTypeName(StreamType), "config",
  164. config_.AsHumanReadableString());
  165. if (!config_.IsValidConfig()) {
  166. DLOG(ERROR) << "Invalid stream config";
  167. ReturnSelectionError(DecoderStatus::Codes::kUnsupportedConfig);
  168. return;
  169. }
  170. if (needs_new_decoders) {
  171. decoder_selection_start_ = base::TimeTicks::Now();
  172. decode_failure_reinit_cause_ = absl::nullopt;
  173. CreateDecoders();
  174. }
  175. GetAndInitializeNextDecoder();
  176. }
  177. template <DemuxerStream::Type StreamType>
  178. void DecoderSelector<StreamType>::BeginDecoderSelection(
  179. SelectDecoderCB select_decoder_cb,
  180. typename Decoder::OutputCB output_cb) {
  181. SelectDecoderInternal(std::move(select_decoder_cb), std::move(output_cb),
  182. /*needs_new_decoders = */ true);
  183. }
  184. template <DemuxerStream::Type StreamType>
  185. void DecoderSelector<StreamType>::ResumeDecoderSelection(
  186. SelectDecoderCB select_decoder_cb,
  187. typename Decoder::OutputCB output_cb,
  188. DecoderStatus&& reinit_cause) {
  189. DVLOG(2) << __func__;
  190. if (!decode_failure_reinit_cause_.has_value())
  191. decode_failure_reinit_cause_ = std::move(reinit_cause);
  192. SelectDecoderInternal(std::move(select_decoder_cb), std::move(output_cb),
  193. /*needs_new_decoders = */ false);
  194. }
  195. template <DemuxerStream::Type StreamType>
  196. void DecoderSelector<StreamType>::FinalizeDecoderSelection() {
  197. DVLOG(2) << __func__;
  198. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  199. DCHECK(!select_decoder_cb_);
  200. const std::string decoder_type = is_platform_decoder_ ? "HW" : "SW";
  201. const std::string stream_type =
  202. StreamType == DemuxerStream::AUDIO ? "Audio" : "Video";
  203. if (is_selecting_for_config_change_) {
  204. is_selecting_for_config_change_ = false;
  205. base::UmaHistogramTimes("Media.ConfigChangeDecoderSelectionTime." +
  206. stream_type + "." + decoder_type,
  207. base::TimeTicks::Now() - decoder_selection_start_);
  208. } else {
  209. // Initial selection
  210. base::UmaHistogramTimes(
  211. "Media.InitialDecoderSelectionTime." + stream_type + "." + decoder_type,
  212. base::TimeTicks::Now() - decoder_selection_start_);
  213. }
  214. if (is_codec_changing_) {
  215. is_codec_changing_ = false;
  216. base::UmaHistogramTimes(
  217. "Media.MSE.CodecChangeTime." + stream_type + "." + decoder_type,
  218. base::TimeTicks::Now() - codec_change_start_);
  219. }
  220. // Discard any remaining decoder instances, they won't be used.
  221. decoders_.clear();
  222. }
  223. template <DemuxerStream::Type StreamType>
  224. void DecoderSelector<StreamType>::NotifyConfigChanged() {
  225. DVLOG(2) << __func__;
  226. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  227. is_selecting_for_config_change_ = true;
  228. DecoderConfig config = traits_->GetDecoderConfig(stream_);
  229. if (config.codec() != config_.codec()) {
  230. is_codec_changing_ = true;
  231. codec_change_start_ = base::TimeTicks::Now();
  232. }
  233. }
  234. template <DemuxerStream::Type StreamType>
  235. void DecoderSelector<StreamType>::PrependDecoder(
  236. std::unique_ptr<Decoder> decoder) {
  237. DVLOG(2) << __func__;
  238. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  239. // Decoders inserted directly should be given priority over those returned by
  240. // |create_decoders_cb_|.
  241. decoders_.insert(decoders_.begin(), std::move(decoder));
  242. FilterAndSortAvailableDecoders();
  243. }
  244. template <DemuxerStream::Type StreamType>
  245. void DecoderSelector<StreamType>::OverrideDecoderPriorityCBForTesting(
  246. DecoderPriorityCB decoder_priority_cb) {
  247. decoder_priority_cb_ = std::move(decoder_priority_cb);
  248. }
  249. template <DemuxerStream::Type StreamType>
  250. void DecoderSelector<StreamType>::CreateDecoders() {
  251. // Post-insert decoders returned by `create_decoders_cb_`, so that
  252. // any decoders added via `PrependDecoder()` are not overwritten and retain
  253. // priority (even if they are ultimately de-ranked by
  254. // `FilterAndSortAvailableDecoders()`)
  255. auto new_decoders = create_decoders_cb_.Run();
  256. std::move(new_decoders.begin(), new_decoders.end(),
  257. std::inserter(decoders_, decoders_.end()));
  258. FilterAndSortAvailableDecoders();
  259. }
  260. template <DemuxerStream::Type StreamType>
  261. void DecoderSelector<StreamType>::GetAndInitializeNextDecoder() {
  262. DVLOG(2) << __func__;
  263. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  264. DCHECK(!decoder_);
  265. if (decoders_.empty()) {
  266. // Decoder selection failed. If the stream is encrypted, try again using
  267. // DecryptingDemuxerStream.
  268. if (config_.is_encrypted() && cdm_context_) {
  269. InitializeDecryptingDemuxerStream();
  270. return;
  271. }
  272. if (decode_failure_reinit_cause_.has_value()) {
  273. ReturnSelectionError(std::move(*decode_failure_reinit_cause_));
  274. } else {
  275. ReturnSelectionError(DecoderStatus::Codes::kUnsupportedConfig);
  276. }
  277. return;
  278. }
  279. // Initialize the first decoder on the list.
  280. decoder_ = std::move(decoders_.front());
  281. decoders_.erase(decoders_.begin());
  282. is_platform_decoder_ = decoder_->IsPlatformDecoder();
  283. TRACE_EVENT_ASYNC_STEP_INTO0("media", kSelectDecoderTrace, this,
  284. GetDecoderName(decoder_->GetDecoderType()));
  285. DVLOG(2) << __func__ << ": initializing " << decoder_->GetDecoderType();
  286. const bool is_live = stream_->liveness() == StreamLiveness::kLive;
  287. traits_->InitializeDecoder(
  288. decoder_.get(), config_, is_live, cdm_context_,
  289. base::BindOnce(&DecoderSelector<StreamType>::OnDecoderInitializeDone,
  290. weak_this_factory_.GetWeakPtr()),
  291. output_cb_, waiting_cb_);
  292. }
  293. template <DemuxerStream::Type StreamType>
  294. void DecoderSelector<StreamType>::OnDecoderInitializeDone(
  295. DecoderStatus status) {
  296. DCHECK(decoder_);
  297. DVLOG(2) << __func__ << ": " << decoder_->GetDecoderType()
  298. << " success=" << static_cast<int>(status.code());
  299. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  300. if (!status.is_ok()) {
  301. // Note: Don't track this decode status, as it is the result of
  302. // initialization failure.
  303. MEDIA_LOG(INFO, media_log_)
  304. << "Failed to initialize " << decoder_->GetDecoderType();
  305. // Try the next decoder on the list.
  306. decoder_ = nullptr;
  307. GetAndInitializeNextDecoder();
  308. return;
  309. }
  310. RunSelectDecoderCB(std::move(decoder_));
  311. }
  312. template <DemuxerStream::Type StreamType>
  313. void DecoderSelector<StreamType>::ReturnSelectionError(DecoderStatus error) {
  314. DVLOG(1) << __func__ << ": No decoder selected";
  315. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  316. DCHECK(!error.is_ok());
  317. decrypting_demuxer_stream_.reset();
  318. decoders_.clear();
  319. RunSelectDecoderCB(std::move(error));
  320. }
  321. template <DemuxerStream::Type StreamType>
  322. void DecoderSelector<StreamType>::InitializeDecryptingDemuxerStream() {
  323. DCHECK(decoders_.empty());
  324. DCHECK(config_.is_encrypted());
  325. DCHECK(cdm_context_);
  326. TRACE_EVENT_ASYNC_STEP_INTO0("media", kSelectDecoderTrace, this,
  327. "DecryptingDemuxerStream");
  328. decrypting_demuxer_stream_ = std::make_unique<DecryptingDemuxerStream>(
  329. task_runner_, media_log_, waiting_cb_);
  330. decrypting_demuxer_stream_->Initialize(
  331. stream_, cdm_context_,
  332. base::BindOnce(
  333. &DecoderSelector<StreamType>::OnDecryptingDemuxerStreamInitializeDone,
  334. weak_this_factory_.GetWeakPtr()));
  335. }
  336. template <DemuxerStream::Type StreamType>
  337. void DecoderSelector<StreamType>::OnDecryptingDemuxerStreamInitializeDone(
  338. PipelineStatus status) {
  339. DVLOG(2) << __func__ << ": status=" << status;
  340. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  341. if (status != PIPELINE_OK) {
  342. // Since we already tried every potential decoder without DDS, give up.
  343. ReturnSelectionError(
  344. {DecoderStatus::Codes::kUnsupportedEncryptionMode, std::move(status)});
  345. return;
  346. }
  347. // Once DDS is enabled, there is no going back.
  348. // TODO(sandersd): Support transitions from encrypted to unencrypted.
  349. stream_ = decrypting_demuxer_stream_.get();
  350. cdm_context_ = nullptr;
  351. // We'll use the decrypted config from now on.
  352. config_ = traits_->GetDecoderConfig(stream_);
  353. DCHECK(!config_.is_encrypted());
  354. // Try decoder selection again now that DDS is being used.
  355. CreateDecoders();
  356. GetAndInitializeNextDecoder();
  357. }
  358. template <DemuxerStream::Type StreamType>
  359. void DecoderSelector<StreamType>::RunSelectDecoderCB(
  360. DecoderOrError decoder_or_error) {
  361. DCHECK(select_decoder_cb_);
  362. TRACE_EVENT_ASYNC_END2(
  363. "media", kSelectDecoderTrace, this, "type",
  364. DemuxerStream::GetTypeName(StreamType), "decoder",
  365. base::StringPrintf(
  366. "%s (%s)",
  367. decoder_or_error.has_value()
  368. ? GetDecoderName(decoder_or_error->GetDecoderType()).c_str()
  369. : "null",
  370. decrypting_demuxer_stream_ ? "encrypted" : "unencrypted"));
  371. task_runner_->PostTask(
  372. FROM_HERE,
  373. base::BindOnce(std::move(select_decoder_cb_), std::move(decoder_or_error),
  374. std::move(decrypting_demuxer_stream_)));
  375. }
  376. template <DemuxerStream::Type StreamType>
  377. void DecoderSelector<StreamType>::FilterAndSortAvailableDecoders() {
  378. std::vector<std::unique_ptr<Decoder>> decoders = std::move(decoders_);
  379. std::vector<std::unique_ptr<Decoder>> deprioritized_decoders;
  380. DCHECK(decoder_priority_cb_);
  381. for (auto& decoder : decoders) {
  382. // Skip the decoder if this decoder doesn't support encryption for a
  383. // decrypting config
  384. if (config_.is_encrypted() && !decoder->SupportsDecryption())
  385. continue;
  386. // Run the predicate on this decoder.
  387. switch (decoder_priority_cb_.Run(config_, *decoder)) {
  388. case DecoderPriority::kSkipped:
  389. continue;
  390. case DecoderPriority::kNormal:
  391. decoders_.push_back(std::move(decoder));
  392. break;
  393. case DecoderPriority::kDeprioritized:
  394. deprioritized_decoders.push_back(std::move(decoder));
  395. break;
  396. }
  397. }
  398. // Post-insert deprioritized decoders
  399. std::move(deprioritized_decoders.begin(), deprioritized_decoders.end(),
  400. std::inserter(decoders_, decoders_.end()));
  401. }
  402. // These forward declarations tell the compiler that we will use
  403. // DecoderSelector with these arguments, allowing us to keep these definitions
  404. // in our .cc without causing linker errors. This also means if anyone tries to
  405. // instantiate a DecoderSelector with anything but these two specializations
  406. // they'll most likely get linker errors.
  407. template class DecoderSelector<DemuxerStream::AUDIO>;
  408. template class DecoderSelector<DemuxerStream::VIDEO>;
  409. } // namespace media