media_codec_video_decoder.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_GPU_ANDROID_MEDIA_CODEC_VIDEO_DECODER_H_
  5. #define MEDIA_GPU_ANDROID_MEDIA_CODEC_VIDEO_DECODER_H_
  6. #include <vector>
  7. #include "base/containers/circular_deque.h"
  8. #include "base/memory/raw_ptr.h"
  9. #include "base/threading/thread_checker.h"
  10. #include "base/timer/elapsed_timer.h"
  11. #include "base/timer/timer.h"
  12. #include "gpu/command_buffer/service/ref_counted_lock.h"
  13. #include "gpu/config/gpu_feature_info.h"
  14. #include "gpu/config/gpu_preferences.h"
  15. #include "media/base/android/media_crypto_context.h"
  16. #include "media/base/android_overlay_mojo_factory.h"
  17. #include "media/base/callback_registry.h"
  18. #include "media/base/cdm_context.h"
  19. #include "media/base/overlay_info.h"
  20. #include "media/base/scoped_async_trace.h"
  21. #include "media/base/video_decoder.h"
  22. #include "media/base/video_decoder_config.h"
  23. #include "media/gpu/android/android_video_surface_chooser.h"
  24. #include "media/gpu/android/codec_allocator.h"
  25. #include "media/gpu/android/codec_wrapper.h"
  26. #include "media/gpu/android/device_info.h"
  27. #include "media/gpu/android/surface_chooser_helper.h"
  28. #include "media/gpu/android/video_frame_factory.h"
  29. #include "media/gpu/media_gpu_export.h"
  30. #include "third_party/abseil-cpp/absl/types/optional.h"
  31. namespace media {
  32. class MediaLog;
  33. struct SupportedVideoDecoderConfig;
  34. struct PendingDecode {
  35. static PendingDecode CreateEos();
  36. PendingDecode(scoped_refptr<DecoderBuffer> buffer,
  37. VideoDecoder::DecodeCB decode_cb);
  38. PendingDecode(const PendingDecode&) = delete;
  39. PendingDecode& operator=(const PendingDecode&) = delete;
  40. PendingDecode(PendingDecode&& other);
  41. ~PendingDecode();
  42. scoped_refptr<DecoderBuffer> buffer;
  43. VideoDecoder::DecodeCB decode_cb;
  44. };
  45. // An Android VideoDecoder that delegates to MediaCodec.
  46. //
  47. // This decoder initializes in two stages. Low overhead initialization is done
  48. // eagerly in Initialize(), but the rest is done lazily and is kicked off by the
  49. // first Decode() (see StartLazyInit()). We do this because there are cases in
  50. // our media pipeline where we'll initialize a decoder but never use it
  51. // (e.g., MSE with no media data appended), and if we eagerly allocator decoder
  52. // resources, like MediaCodecs and TextureOwners, we will block other
  53. // playbacks that need them.
  54. // TODO: Lazy initialization should be handled at a higher layer of the media
  55. // stack for both simplicity and cross platform support.
  56. class MEDIA_GPU_EXPORT MediaCodecVideoDecoder final
  57. : public VideoDecoder,
  58. public gpu::RefCountedLockHelperDrDc {
  59. public:
  60. static std::vector<SupportedVideoDecoderConfig> GetSupportedConfigs();
  61. MediaCodecVideoDecoder(const MediaCodecVideoDecoder&) = delete;
  62. MediaCodecVideoDecoder& operator=(const MediaCodecVideoDecoder&) = delete;
  63. ~MediaCodecVideoDecoder() override;
  64. static void DestroyAsync(std::unique_ptr<MediaCodecVideoDecoder>);
  65. static std::unique_ptr<VideoDecoder> Create(
  66. const gpu::GpuPreferences& gpu_preferences,
  67. const gpu::GpuFeatureInfo& gpu_feature_info,
  68. std::unique_ptr<MediaLog> media_log,
  69. DeviceInfo* device_info,
  70. CodecAllocator* codec_allocator,
  71. std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
  72. AndroidOverlayMojoFactoryCB overlay_factory_cb,
  73. RequestOverlayInfoCB request_overlay_info_cb,
  74. std::unique_ptr<VideoFrameFactory> video_frame_factory,
  75. scoped_refptr<gpu::RefCountedLock> drdc_lock);
  76. // VideoDecoder implementation:
  77. VideoDecoderType GetDecoderType() const override;
  78. void Initialize(const VideoDecoderConfig& config,
  79. bool low_delay,
  80. CdmContext* cdm_context,
  81. InitCB init_cb,
  82. const OutputCB& output_cb,
  83. const WaitingCB& waiting_cb) override;
  84. void Decode(scoped_refptr<DecoderBuffer> buffer, DecodeCB decode_cb) override;
  85. void Reset(base::OnceClosure closure) override;
  86. bool NeedsBitstreamConversion() const override;
  87. bool CanReadWithoutStalling() const override;
  88. int GetMaxDecodeRequests() const override;
  89. private:
  90. // The test has access for PumpCodec() and the constructor.
  91. friend class MediaCodecVideoDecoderTest;
  92. MediaCodecVideoDecoder(
  93. const gpu::GpuPreferences& gpu_preferences,
  94. const gpu::GpuFeatureInfo& gpu_feature_info,
  95. std::unique_ptr<MediaLog> media_log,
  96. DeviceInfo* device_info,
  97. CodecAllocator* codec_allocator,
  98. std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
  99. AndroidOverlayMojoFactoryCB overlay_factory_cb,
  100. RequestOverlayInfoCB request_overlay_info_cb,
  101. std::unique_ptr<VideoFrameFactory> video_frame_factory,
  102. scoped_refptr<gpu::RefCountedLock> drdc_lock);
  103. // Set up |cdm_context| as part of initialization. Guarantees that |init_cb|
  104. // will be called depending on the outcome, though not necessarily before this
  105. // function returns.
  106. void SetCdm(CdmContext* cdm_context, InitCB init_cb);
  107. // Called when the Cdm provides |media_crypto|. Will signal |init_cb| based
  108. // on the result, and set the codec config properly.
  109. void OnMediaCryptoReady(InitCB init_cb,
  110. JavaObjectPtr media_crypto,
  111. bool requires_secure_video_codec);
  112. enum class State {
  113. // Initializing resources required to create a codec.
  114. kInitializing,
  115. // Initialization has completed and we're running. This is the only state
  116. // in which |codec_| might be non-null. If |codec_| is null, a codec
  117. // creation is pending.
  118. kRunning,
  119. // A fatal error occurred. A terminal state.
  120. kError,
  121. // The output surface was destroyed, but SetOutputSurface() is not supported
  122. // by the device. In this case the consumer is responsible for destroying us
  123. // soon, so this is terminal state but not a decode error.
  124. kSurfaceDestroyed
  125. };
  126. enum class DrainType { kForReset, kForDestroy };
  127. // Finishes initialization.
  128. void StartLazyInit();
  129. void OnVideoFrameFactoryInitialized(
  130. scoped_refptr<gpu::TextureOwner> texture_owner);
  131. // Callback for the CDM to notify |this|. Resets |waiting_for_key_| to false,
  132. // indicating that MediaCodec might now accept buffers.
  133. void OnCdmContextEvent(CdmContext::Event event);
  134. // Updates |surface_chooser_| with the new overlay info.
  135. void OnOverlayInfoChanged(const OverlayInfo& overlay_info);
  136. void OnSurfaceChosen(std::unique_ptr<AndroidOverlay> overlay);
  137. void OnSurfaceDestroyed(AndroidOverlay* overlay);
  138. // Whether we have a codec and its surface is not equal to
  139. // |target_surface_bundle_|.
  140. bool SurfaceTransitionPending();
  141. // Sets |codecs_|'s output surface to |target_surface_bundle_|.
  142. void TransitionToTargetSurface();
  143. // Creates a codec asynchronously.
  144. void CreateCodec();
  145. // Trampoline helper which ensures correct release of MediaCodecBridge and
  146. // CodecSurfaceBundle even if this class goes away.
  147. static void OnCodecConfiguredInternal(
  148. base::WeakPtr<MediaCodecVideoDecoder> weak_this,
  149. CodecAllocator* codec_allocator,
  150. scoped_refptr<CodecSurfaceBundle> surface_bundle,
  151. std::unique_ptr<MediaCodecBridge> codec);
  152. void OnCodecConfigured(scoped_refptr<CodecSurfaceBundle> surface_bundle,
  153. std::unique_ptr<MediaCodecBridge> media_codec);
  154. // Flushes the codec, or if flush() is not supported, releases it and creates
  155. // a new one.
  156. void FlushCodec();
  157. // Attempts to queue input and dequeue output from the codec. Calls
  158. // StartTimerOrPumpCodec() even if the codec is idle when |force_start_timer|.
  159. void PumpCodec(bool force_start_timer);
  160. bool QueueInput();
  161. bool DequeueOutput();
  162. // Starts |pump_codec_timer_| if it's not started and resets the idle timeout.
  163. void StartTimerOrPumpCodec();
  164. void StopTimerIfIdle();
  165. // Runs |eos_decode_cb_| if it's valid and |reset_generation| matches
  166. // |reset_generation_|.
  167. void RunEosDecodeCb(int reset_generation);
  168. // Forwards |frame| via |output_cb_| if |reset_generation| matches
  169. // |reset_generation_|. |async_trace| is the (optional) scoped trace that
  170. // started when we dequeued the corresponding output buffer. |started_at| is
  171. // the wall clock time at which we dequeued the output buffer.
  172. void ForwardVideoFrame(int reset_generation,
  173. std::unique_ptr<ScopedAsyncTrace> async_trace,
  174. base::TimeTicks started_at,
  175. scoped_refptr<VideoFrame> frame);
  176. // Starts draining the codec by queuing an EOS if required. It skips the drain
  177. // if possible.
  178. void StartDrainingCodec(DrainType drain_type);
  179. void OnCodecDrained();
  180. void CancelPendingDecodes(DecoderStatus status);
  181. // Sets |state_| and does common teardown for the terminal states. |state_|
  182. // must be either kSurfaceDestroyed or kError. |reason| will be logged to
  183. // |media_log_| as an info event ("error" indicates that playback will stop,
  184. // but we don't know that the renderer will do that).
  185. void EnterTerminalState(State state, const char* reason);
  186. bool InTerminalState();
  187. // Releases |codec_| if it's not null.
  188. void ReleaseCodec();
  189. // Return true if we have a codec that's outputting to an overlay.
  190. bool IsUsingOverlay() const;
  191. // Notify us about a promotion hint.
  192. void NotifyPromotionHint(PromotionHintAggregator::Hint hint);
  193. // Update |cached_frame_information_|.
  194. void CacheFrameInformation();
  195. // Creates an overlay factory cb based on the value of overlay_info_.
  196. AndroidOverlayFactoryCB CreateOverlayFactoryCb();
  197. // Create a callback that will handle promotion hints, and set the overlay
  198. // position if required.
  199. PromotionHintAggregator::NotifyPromotionHintCB CreatePromotionHintCB();
  200. std::unique_ptr<MediaLog> media_log_;
  201. State state_ = State::kInitializing;
  202. // Whether initialization still needs to be done on the first decode call.
  203. bool lazy_init_pending_ = true;
  204. base::circular_deque<PendingDecode> pending_decodes_;
  205. // Whether we've seen MediaCodec return MEDIA_CODEC_NO_KEY indicating that
  206. // the corresponding key was not set yet, and MediaCodec will not accept
  207. // buffers until OnCdmContextEvent() is called with kHasAdditionalUsableKey.
  208. bool waiting_for_key_ = false;
  209. // The reason for the current drain operation if any.
  210. absl::optional<DrainType> drain_type_;
  211. // The current reset cb if a Reset() is in progress.
  212. base::OnceClosure reset_cb_;
  213. // A generation counter that's incremented every time Reset() is called.
  214. int reset_generation_ = 0;
  215. // The EOS decode cb for an EOS currently being processed by the codec. Called
  216. // when the EOS is output.
  217. DecodeCB eos_decode_cb_;
  218. OutputCB output_cb_;
  219. WaitingCB waiting_cb_;
  220. VideoDecoderConfig decoder_config_;
  221. // Codec specific data (SPS and PPS for H264). Some MediaCodecs initialize
  222. // more reliably if we explicitly pass these (http://crbug.com/649185).
  223. std::vector<uint8_t> csd0_;
  224. std::vector<uint8_t> csd1_;
  225. std::unique_ptr<CodecWrapper> codec_;
  226. base::ElapsedTimer idle_timer_;
  227. base::RepeatingTimer pump_codec_timer_;
  228. raw_ptr<CodecAllocator> codec_allocator_;
  229. // The current target surface that |codec_| should be rendering to. It
  230. // reflects the latest surface choice by |surface_chooser_|. If the codec is
  231. // configured with some other surface, then a transition is pending. It's
  232. // non-null from the first surface choice.
  233. scoped_refptr<CodecSurfaceBundle> target_surface_bundle_;
  234. // A TextureOwner bundle that is kept for the lifetime of MCVD so that if we
  235. // have to synchronously switch surfaces we always have one available.
  236. scoped_refptr<CodecSurfaceBundle> texture_owner_bundle_;
  237. // A callback for requesting overlay info updates.
  238. RequestOverlayInfoCB request_overlay_info_cb_;
  239. // The current overlay info, which possibly specifies an overlay to render to.
  240. OverlayInfo overlay_info_;
  241. // Set to true if the display compositor swap is done using SurfaceControl.
  242. const bool is_surface_control_enabled_;
  243. // The helper which manages our surface chooser for us.
  244. SurfaceChooserHelper surface_chooser_helper_;
  245. // The factory for creating VideoFrames from CodecOutputBuffers.
  246. std::unique_ptr<VideoFrameFactory> video_frame_factory_;
  247. // An optional factory callback for creating mojo AndroidOverlays.
  248. AndroidOverlayMojoFactoryCB overlay_factory_cb_;
  249. raw_ptr<DeviceInfo> device_info_;
  250. bool enable_threaded_texture_mailboxes_;
  251. // Most recently cached frame information, so that we can dispatch it without
  252. // recomputing it on every frame. It changes very rarely.
  253. SurfaceChooserHelper::FrameInformation cached_frame_information_ =
  254. SurfaceChooserHelper::FrameInformation::NON_OVERLAY_INSECURE;
  255. // CDM related stuff.
  256. // Owned by CDM which is external to this decoder.
  257. raw_ptr<MediaCryptoContext> media_crypto_context_ = nullptr;
  258. // To keep the CdmContext event callback registered.
  259. std::unique_ptr<CallbackRegistration> event_cb_registration_;
  260. // Do we need a hw-secure codec?
  261. bool requires_secure_codec_ = false;
  262. bool using_async_api_ = false;
  263. // Should we flush the codec on the next decode, and pretend that it is
  264. // drained currently? Note that we'll automatically flush if the codec is
  265. // drained; this flag indicates that we also elided the drain, so the codec is
  266. // in some random state, possibly with output buffers pending.
  267. bool deferred_flush_pending_ = false;
  268. // Should we upgrade the next flush to a full release / reallocation of the
  269. // codec? This lets us update our hints to the decoder about the size of the
  270. // expected video.
  271. bool deferred_reallocation_pending_ = false;
  272. // Width, in pixels, of the resolution that we last told the codec about.
  273. int last_width_ = 0;
  274. // KEY_MAX_INPUT_SIZE configured for the current codec.
  275. size_t max_input_size_ = 0;
  276. // Optional crypto object from the Cdm.
  277. base::android::ScopedJavaGlobalRef<jobject> media_crypto_;
  278. // For A/B power testing, this causes all non-L1 content to avoid overlays.
  279. // This is only for A/B power testing, and can be removed after that.
  280. // See https://crbug.com/1081346 .
  281. bool allow_nonsecure_overlays_ = true;
  282. // If set, then the next call to `CodecConfig()` will be allowed to retry if
  283. // it fails to get a codec. This is to work around b/191966399.
  284. bool should_retry_codec_allocation_ = false;
  285. base::WeakPtrFactory<MediaCodecVideoDecoder> weak_factory_{this};
  286. base::WeakPtrFactory<MediaCodecVideoDecoder> codec_allocator_weak_factory_{
  287. this};
  288. };
  289. } // namespace media
  290. namespace std {
  291. // Specialize std::default_delete to call Destroy().
  292. template <>
  293. struct MEDIA_GPU_EXPORT default_delete<media::MediaCodecVideoDecoder>
  294. : public default_delete<media::VideoDecoder> {};
  295. } // namespace std
  296. #endif // MEDIA_GPU_ANDROID_MEDIA_CODEC_VIDEO_DECODER_H_