vaapi_video_decode_accelerator.h 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. //
  5. // This file contains an implementation of VideoDecoderAccelerator
  6. // that utilizes hardware video decoder present on Intel CPUs.
  7. #ifndef MEDIA_GPU_VAAPI_VAAPI_VIDEO_DECODE_ACCELERATOR_H_
  8. #define MEDIA_GPU_VAAPI_VAAPI_VIDEO_DECODE_ACCELERATOR_H_
  9. #include <stddef.h>
  10. #include <stdint.h>
  11. #include <list>
  12. #include <map>
  13. #include <memory>
  14. #include <utility>
  15. #include <vector>
  16. #include "base/containers/queue.h"
  17. #include "base/containers/small_map.h"
  18. #include "base/memory/raw_ptr.h"
  19. #include "base/memory/weak_ptr.h"
  20. #include "base/synchronization/condition_variable.h"
  21. #include "base/synchronization/lock.h"
  22. #include "base/task/single_thread_task_runner.h"
  23. #include "base/thread_annotations.h"
  24. #include "base/threading/thread.h"
  25. #include "base/trace_event/memory_dump_provider.h"
  26. #include "media/base/bitstream_buffer.h"
  27. #include "media/gpu/decode_surface_handler.h"
  28. #include "media/gpu/gpu_video_decode_accelerator_helpers.h"
  29. #include "media/gpu/media_gpu_export.h"
  30. #include "media/gpu/vaapi/vaapi_picture_factory.h"
  31. #include "media/gpu/vaapi/vaapi_wrapper.h"
  32. #include "media/video/picture.h"
  33. #include "media/video/video_decode_accelerator.h"
  34. namespace gl {
  35. class GLImage;
  36. }
  37. namespace media {
  38. class AcceleratedVideoDecoder;
  39. template <typename T>
  40. class ScopedID;
  41. class VaapiVideoDecoderDelegate;
  42. class VaapiPicture;
  43. // Class to provide video decode acceleration for Intel systems with hardware
  44. // support for it, and on which libva is available.
  45. // Decoding tasks are performed in a separate decoding thread.
  46. //
  47. // Threading/life-cycle: this object is created & destroyed on the GPU
  48. // ChildThread. A few methods on it are called on the decoder thread which is
  49. // stopped during |this->Destroy()|, so any tasks posted to the decoder thread
  50. // can assume |*this| is still alive. See |weak_this_| below for more details.
  51. class MEDIA_GPU_EXPORT VaapiVideoDecodeAccelerator
  52. : public VideoDecodeAccelerator,
  53. public DecodeSurfaceHandler<VASurface>,
  54. public base::trace_event::MemoryDumpProvider {
  55. public:
  56. VaapiVideoDecodeAccelerator(
  57. const MakeGLContextCurrentCallback& make_context_current_cb,
  58. const BindGLImageCallback& bind_image_cb);
  59. VaapiVideoDecodeAccelerator(const VaapiVideoDecodeAccelerator&) = delete;
  60. VaapiVideoDecodeAccelerator& operator=(const VaapiVideoDecodeAccelerator&) =
  61. delete;
  62. ~VaapiVideoDecodeAccelerator() override;
  63. // VideoDecodeAccelerator implementation.
  64. bool Initialize(const Config& config, Client* client) override;
  65. void Decode(BitstreamBuffer bitstream_buffer) override;
  66. void Decode(scoped_refptr<DecoderBuffer> buffer,
  67. int32_t bitstream_id) override;
  68. void AssignPictureBuffers(const std::vector<PictureBuffer>& buffers) override;
  69. #if defined(USE_OZONE)
  70. void ImportBufferForPicture(
  71. int32_t picture_buffer_id,
  72. VideoPixelFormat pixel_format,
  73. gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle) override;
  74. #endif
  75. void ReusePictureBuffer(int32_t picture_buffer_id) override;
  76. void Flush() override;
  77. void Reset() override;
  78. void Destroy() override;
  79. bool TryToSetupDecodeOnSeparateThread(
  80. const base::WeakPtr<Client>& decode_client,
  81. const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner)
  82. override;
  83. static VideoDecodeAccelerator::SupportedProfiles GetSupportedProfiles();
  84. // DecodeSurfaceHandler implementation.
  85. scoped_refptr<VASurface> CreateSurface() override;
  86. void SurfaceReady(scoped_refptr<VASurface> va_surface,
  87. int32_t bitstream_id,
  88. const gfx::Rect& visible_rect,
  89. const VideoColorSpace& color_space) override;
  90. // base::trace_event::MemoryDumpProvider implementation.
  91. bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
  92. base::trace_event::ProcessMemoryDump* pmd) override;
  93. private:
  94. friend class VaapiVideoDecodeAcceleratorTest;
  95. // An input buffer with id provided by the client and awaiting consumption.
  96. class InputBuffer;
  97. // A self-cleaning VASurfaceID.
  98. using ScopedVASurfaceID = ScopedID<VASurfaceID>;
  99. // Notify the client that an error has occurred and decoding cannot continue.
  100. void NotifyError(Error error);
  101. void NotifyStatus(VaapiStatus status);
  102. // Queue a input buffer for decode.
  103. void QueueInputBuffer(scoped_refptr<DecoderBuffer> buffer,
  104. int32_t bitstream_id);
  105. // Gets a new |current_input_buffer_| from |input_buffers_| and sets it up in
  106. // |decoder_|. This method will sleep if no |input_buffers_| are available.
  107. // Returns true if a new buffer has been set up, false if an early exit has
  108. // been requested (due to initiated reset/flush/destroy).
  109. bool GetCurrInputBuffer_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_);
  110. // Signals the client that |curr_input_buffer_| has been read and can be
  111. // returned. Will also release the mapping.
  112. void ReturnCurrInputBuffer_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_);
  113. // Waits for more surfaces to become available. Returns true once they do or
  114. // false if an early exit has been requested (due to an initiated
  115. // reset/flush/destroy).
  116. bool WaitForSurfaces_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_);
  117. // Continue decoding given input buffers and sleep waiting for input/output
  118. // as needed. Will exit if a new set of surfaces or reset/flush/destroy
  119. // is requested.
  120. void DecodeTask();
  121. // Scheduled after receiving a flush request and executed after the current
  122. // decoding task finishes decoding pending inputs. Makes the decoder return
  123. // all remaining output pictures and puts it in an idle state, ready
  124. // to resume if needed and schedules a FinishFlush.
  125. void FlushTask();
  126. // Scheduled by the FlushTask after decoder is flushed to put VAVDA into idle
  127. // state and notify the client that flushing has been finished.
  128. void FinishFlush();
  129. // Scheduled after receiving a reset request and executed after the current
  130. // decoding task finishes decoding the current frame. Puts the decoder into
  131. // an idle state, ready to resume if needed, discarding decoded but not yet
  132. // outputted pictures (decoder keeps ownership of their associated picture
  133. // buffers). Schedules a FinishReset afterwards.
  134. void ResetTask();
  135. // Scheduled by ResetTask after it's done putting VAVDA into an idle state.
  136. // Drops remaining input buffers and notifies the client that reset has been
  137. // finished.
  138. void FinishReset();
  139. // Helper for Destroy(), doing all the actual work except for deleting self.
  140. void Cleanup();
  141. // Get a usable framebuffer configuration for use in binding textures
  142. // or return false on failure.
  143. bool InitializeFBConfig();
  144. // Callback to be executed once we have a |va_surface| to be output and an
  145. // available VaapiPicture in |available_picture_buffers_| for output. Puts
  146. // contents of |va_surface| into the latter, releases the surface and passes
  147. // the resulting picture to |client_| along with |visible_rect|.
  148. void OutputPicture(scoped_refptr<VASurface> va_surface,
  149. int32_t input_id,
  150. gfx::Rect visible_rect,
  151. const VideoColorSpace& picture_color_space);
  152. // Try to OutputPicture() if we have both a ready surface and picture.
  153. void TryOutputPicture();
  154. // Called when a VASurface is no longer in use by |decoder_| nor |client_|.
  155. // Returns it to |available_va_surfaces_|. |va_surface_id| is not used but it
  156. // must be here to bind this method as VASurface::ReleaseCB.
  157. void RecycleVASurface(std::unique_ptr<ScopedVASurfaceID> va_surface,
  158. VASurfaceID va_surface_id);
  159. // Request a new set of |num_pics| PictureBuffers to be allocated by
  160. // |client_|. Up to |num_reference_frames| out of |num_pics_| might be needed
  161. // by |decoder_|.
  162. void InitiateSurfaceSetChange(size_t num_pics,
  163. gfx::Size size,
  164. size_t num_reference_frames,
  165. const gfx::Rect& visible_rect);
  166. // Check if the surfaces have been released or post ourselves for later.
  167. void TryFinishSurfaceSetChange();
  168. // Different modes of internal buffer allocations.
  169. enum class BufferAllocationMode {
  170. // Only using |client_|s provided PictureBuffers, none internal.
  171. kNone,
  172. // Using a reduced amount of |client_|s provided PictureBuffers and
  173. // |decoder_|s GetNumReferenceFrames() internallly.
  174. kSuperReduced,
  175. // Similar to kSuperReduced, but we have to increase slightly the amount of
  176. // PictureBuffers allocated for the |client_|.
  177. kReduced,
  178. // VaapiVideoDecodeAccelerator can work with this mode on all platforms.
  179. // Using |client_|s provided PictureBuffers and as many internally
  180. // allocated.
  181. kNormal,
  182. };
  183. // Decides the concrete buffer allocation mode, depending on the hardware
  184. // platform and other parameters.
  185. BufferAllocationMode DecideBufferAllocationMode();
  186. bool IsBufferAllocationModeReducedOrSuperReduced() const;
  187. // VAVDA state.
  188. enum State {
  189. // Initialize() not called yet or failed.
  190. kUninitialized,
  191. // DecodeTask running.
  192. kDecoding,
  193. // Resetting, waiting for decoder to finish current task and cleanup.
  194. kResetting,
  195. // Idle, decoder in state ready to start/resume decoding.
  196. kIdle,
  197. // Destroying, waiting for the decoder to finish current task.
  198. kDestroying,
  199. };
  200. base::Lock lock_;
  201. State state_ GUARDED_BY(lock_);
  202. // Only used on |task_runner_|.
  203. Config::OutputMode output_mode_;
  204. // Queue of available InputBuffers.
  205. base::queue<std::unique_ptr<InputBuffer>> input_buffers_ GUARDED_BY(lock_);
  206. // Signalled when input buffers are queued onto |input_buffers_| queue.
  207. base::ConditionVariable input_ready_;
  208. // Current input buffer at decoder. Only used on |decoder_thread_task_runner_|
  209. std::unique_ptr<InputBuffer> curr_input_buffer_;
  210. // Only used on |task_runner_|.
  211. std::unique_ptr<VaapiPictureFactory> vaapi_picture_factory_;
  212. // The following variables are constructed/initialized in Initialize() when
  213. // the codec information is received. |vaapi_wrapper_| is thread safe.
  214. scoped_refptr<VaapiWrapper> vaapi_wrapper_;
  215. // Only used on |decoder_thread_task_runner_|.
  216. std::unique_ptr<AcceleratedVideoDecoder> decoder_;
  217. // TODO(crbug.com/1022246): Instead of having the raw pointer here, getting
  218. // the pointer from AcceleratedVideoDecoder.
  219. raw_ptr<VaapiVideoDecoderDelegate> decoder_delegate_ = nullptr;
  220. // Filled in during Initialize().
  221. BufferAllocationMode buffer_allocation_mode_;
  222. // VaapiWrapper for VPP (Video Post Processing). This is used for copying
  223. // from a decoded surface to a surface bound to client's PictureBuffer.
  224. scoped_refptr<VaapiWrapper> vpp_vaapi_wrapper_;
  225. // All allocated VaapiPictures, regardless of their current state. Pictures
  226. // are allocated at AssignPictureBuffers() and are kept until dtor or
  227. // TryFinishSurfaceSetChange(). Comes after |vaapi_wrapper_| to ensure all
  228. // pictures are destroyed before this is destroyed.
  229. base::small_map<std::map<int32_t, std::unique_ptr<VaapiPicture>>> pictures_
  230. GUARDED_BY(lock_);
  231. // List of PictureBuffer ids available to be sent to |client_| via
  232. // OutputPicture() (|client_| returns them via ReusePictureBuffer()).
  233. std::list<int32_t> available_picture_buffers_ GUARDED_BY(lock_);
  234. // VASurfaces available and that can be passed to |decoder_| for its use upon
  235. // CreateSurface() request (and then returned via RecycleVASurface()).
  236. std::list<std::unique_ptr<ScopedVASurfaceID>> available_va_surfaces_
  237. GUARDED_BY(lock_);
  238. // Signalled when output surfaces are queued into |available_va_surfaces_|.
  239. base::ConditionVariable surfaces_available_;
  240. // VASurfaceIDs format, filled in when created.
  241. unsigned int va_surface_format_;
  242. // Pending output requests from the decoder. When it indicates that we should
  243. // output a surface and we have an available Picture (i.e. texture) ready
  244. // to use, we'll execute the callback passing the Picture. The callback
  245. // will put the contents of the surface into the picture and return it to
  246. // the client, releasing the surface as well.
  247. // If we don't have any available |pictures_| at the time when the decoder
  248. // requests output, we'll store the request in this queue for later and run it
  249. // once the client gives us more textures via ReusePictureBuffer().
  250. // Only used on |task_runner_|.
  251. base::queue<base::OnceClosure> pending_output_cbs_;
  252. // WeakPtr<> pointing to |this| for use in posting tasks from the decoder
  253. // thread back to the ChildThread. Because the decoder thread is a member of
  254. // this class, any task running on the decoder thread is guaranteed that this
  255. // object is still alive. As a result, tasks posted from ChildThread to
  256. // decoder thread should use base::Unretained(this), and tasks posted from the
  257. // decoder thread to the ChildThread should use |weak_this_|.
  258. base::WeakPtr<VaapiVideoDecodeAccelerator> weak_this_;
  259. // Callback used to recycle VASurfaces. Only used on |task_runner_|.
  260. base::RepeatingCallback<void(std::unique_ptr<ScopedVASurfaceID>, VASurfaceID)>
  261. va_surface_recycle_cb_;
  262. // To expose client callbacks from VideoDecodeAccelerator. Used only on
  263. // |task_runner_|.
  264. std::unique_ptr<base::WeakPtrFactory<Client>> client_ptr_factory_;
  265. base::WeakPtr<Client> client_;
  266. // ChildThread's task runner.
  267. const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  268. base::Thread decoder_thread_;
  269. // Use this to post tasks to |decoder_thread_| instead of
  270. // |decoder_thread_.task_runner()| because the latter will be NULL once
  271. // |decoder_thread_.Stop()| returns.
  272. scoped_refptr<base::SingleThreadTaskRunner> decoder_thread_task_runner_;
  273. // Whether we are waiting for any |pending_output_cbs_| to be run before
  274. // NotifyingFlushDone. Only used on |task_runner_|.
  275. bool finish_flush_pending_;
  276. // Decoder requested a new surface set and we are waiting for all the surfaces
  277. // to be returned before we can free them. Only used on |task_runner_|.
  278. bool awaiting_va_surfaces_recycle_;
  279. // Last requested number/resolution/visible rectangle of output
  280. // PictureBuffers.
  281. size_t requested_num_pics_;
  282. gfx::Size requested_pic_size_;
  283. gfx::Rect requested_visible_rect_;
  284. // Potential extra PictureBuffers to request, used only on
  285. // BufferAllocationMode::kNone, see DecideBufferAllocationMode().
  286. size_t num_extra_pics_ = 0;
  287. // Max number of reference frames needed by |decoder_|. Only used on
  288. // |task_runner_| and when in BufferAllocationMode::kNone.
  289. size_t requested_num_reference_frames_;
  290. size_t previously_requested_num_reference_frames_;
  291. // The video stream's profile.
  292. VideoCodecProfile profile_;
  293. // Callback to make GL context current.
  294. MakeGLContextCurrentCallback make_context_current_cb_;
  295. // Callback to bind a GLImage to a given texture.
  296. BindGLImageCallback bind_image_cb_;
  297. // The WeakPtrFactory for |weak_this_|.
  298. base::WeakPtrFactory<VaapiVideoDecodeAccelerator> weak_this_factory_;
  299. };
  300. } // namespace media
  301. #endif // MEDIA_GPU_VAAPI_VAAPI_VIDEO_DECODE_ACCELERATOR_H_