vaapi_video_encode_accelerator.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_GPU_VAAPI_VAAPI_VIDEO_ENCODE_ACCELERATOR_H_
  5. #define MEDIA_GPU_VAAPI_VAAPI_VIDEO_ENCODE_ACCELERATOR_H_
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <map>
  9. #include <memory>
  10. #include "base/containers/queue.h"
  11. #include "base/containers/small_map.h"
  12. #include "base/memory/ref_counted_memory.h"
  13. #include "base/sequence_checker.h"
  14. #include "base/task/single_thread_task_runner.h"
  15. #include "base/trace_event/memory_dump_provider.h"
  16. #include "media/base/bitrate.h"
  17. #include "media/gpu/media_gpu_export.h"
  18. #include "media/gpu/vaapi/vaapi_utils.h"
  19. #include "media/gpu/vaapi/vaapi_video_encoder_delegate.h"
  20. #include "media/gpu/vaapi/vaapi_wrapper.h"
  21. #include "media/video/video_encode_accelerator.h"
  22. namespace media {
  23. // A VideoEncodeAccelerator implementation that uses VA-API
  24. // (https://01.org/vaapi) for HW-accelerated video encode.
  25. class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
  26. : public VideoEncodeAccelerator,
  27. public base::trace_event::MemoryDumpProvider {
  28. public:
  29. VaapiVideoEncodeAccelerator();
  30. VaapiVideoEncodeAccelerator(const VaapiVideoEncodeAccelerator&) = delete;
  31. VaapiVideoEncodeAccelerator& operator=(const VaapiVideoEncodeAccelerator&) =
  32. delete;
  33. ~VaapiVideoEncodeAccelerator() override;
  34. // VideoEncodeAccelerator implementation.
  35. SupportedProfiles GetSupportedProfiles() override;
  36. bool Initialize(const Config& config,
  37. Client* client,
  38. std::unique_ptr<MediaLog> media_log) override;
  39. void Encode(scoped_refptr<VideoFrame> frame, bool force_keyframe) override;
  40. void UseOutputBitstreamBuffer(BitstreamBuffer buffer) override;
  41. void RequestEncodingParametersChange(const Bitrate& bitrate,
  42. uint32_t framerate) override;
  43. void RequestEncodingParametersChange(
  44. const VideoBitrateAllocation& bitrate_allocation,
  45. uint32_t framerate) override;
  46. void Destroy() override;
  47. void Flush(FlushCallback flush_callback) override;
  48. bool IsFlushSupported() override;
  49. // base::trace_event::MemoryDumpProvider implementation.
  50. bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
  51. base::trace_event::ProcessMemoryDump* pmd) override;
  52. private:
  53. friend class VaapiVideoEncodeAcceleratorTest;
  54. using EncodeJob = VaapiVideoEncoderDelegate::EncodeJob;
  55. using EncodeResult = VaapiVideoEncoderDelegate::EncodeResult;
  56. // Encoder state.
  57. enum State {
  58. kUninitialized,
  59. kEncoding,
  60. kError,
  61. };
  62. struct SizeComparator {
  63. constexpr bool operator()(const gfx::Size& lhs,
  64. const gfx::Size& rhs) const {
  65. return std::forward_as_tuple(lhs.width(), lhs.height()) <
  66. std::forward_as_tuple(rhs.width(), rhs.height());
  67. }
  68. };
  69. // Maximum size is four to support the worst case of a given input of a
  70. // different resolution than the maximum number of spatial layers (3).
  71. static constexpr size_t kMaxNumSpatialLayersPlusOne = 3 + 1;
  72. using InputSurfaceMap = base::small_map<
  73. std::map<gfx::Size, std::unique_ptr<ScopedVASurface>, SizeComparator>,
  74. kMaxNumSpatialLayersPlusOne>;
  75. using EncodeSurfacesMap =
  76. base::small_map<std::map<gfx::Size,
  77. std::vector<std::unique_ptr<ScopedVASurface>>,
  78. SizeComparator>,
  79. kMaxNumSpatialLayersPlusOne>;
  80. using EncodeSurfacesCountMap =
  81. base::small_map<std::map<gfx::Size, size_t, SizeComparator>,
  82. kMaxNumSpatialLayersPlusOne>;
  83. // Holds input frames coming from the client ready to be encoded.
  84. struct InputFrameRef;
  85. // Holds output buffers coming from the client ready to be filled.
  86. struct BitstreamBufferRef;
  87. //
  88. // Tasks for each of the VEA interface calls to be executed on
  89. // |encoder_task_runner_|.
  90. //
  91. void InitializeTask(const Config& config);
  92. bool AttemptedInitialization() const { return !!client_ptr_factory_; }
  93. // Enqueues |frame| onto the queue of pending inputs and attempts to continue
  94. // encoding.
  95. void EncodeTask(scoped_refptr<VideoFrame> frame, bool force_keyframe);
  96. // Maps |buffer_ref|, push it onto the available_bitstream_buffers_, and
  97. // attempts to return any pending encoded data in it, if any.
  98. void UseOutputBitstreamBufferTask(
  99. std::unique_ptr<BitstreamBufferRef> buffer_ref);
  100. void RequestEncodingParametersChangeTask(
  101. VideoBitrateAllocation bitrate_allocation,
  102. uint32_t framerate);
  103. void DestroyTask();
  104. void FlushTask(FlushCallback flush_callback);
  105. // Create input and reconstructed surfaces used in encoding whose sizes are
  106. // |spatial_layer_resolutions| from GpuMemoryBuffer-based VideoFrame |frame|.
  107. // The created surfaces for input to an encoder driver are filled into
  108. // |input_surfaces| and, ones used as reconstructed surfaces by the driver are
  109. // filled to |reconstructed_surfaces|. This must be called only in native
  110. // input mode.
  111. bool CreateSurfacesForGpuMemoryBufferEncoding(
  112. const VideoFrame& frame,
  113. const std::vector<gfx::Size>& spatial_layer_resolutions,
  114. std::vector<scoped_refptr<VASurface>>* input_surfaces,
  115. std::vector<scoped_refptr<VASurface>>* reconstructed_surfaces);
  116. // Create input and reconstructed surfaces used in encoding from SharedMemory
  117. // VideoFrame |frame|. This must be called only in non native input mode.
  118. bool CreateSurfacesForShmemEncoding(
  119. const VideoFrame& frame,
  120. scoped_refptr<VASurface>* input_surface,
  121. scoped_refptr<VASurface>* reconstructed_surface);
  122. // Creates one |encode_size| VASurface using |vaapi_wrapper_|.
  123. // It returns a reference of an exiting available surface. If there is no
  124. // available surface and the number of previously allocated surfaces is less
  125. // than threshold, then it returns a reference to the newly created
  126. // surface, that is also added to |available_encode_surfaces_[encode_size]|.
  127. // Returns nullptr if too many surfaces have already been allocated, or if
  128. // creation fails.
  129. scoped_refptr<VASurface> CreateEncodeSurface(const gfx::Size& encode_size);
  130. // Creates VASurface using |vaapi_wrapper| whose sizes are |encode_size|
  131. // with |surface_usage_hints|. Returns nullptr if the surfaces fail to be
  132. // created successfully. The created surfaces are filled into
  133. // |input_surfaces_[encode_size]|.
  134. scoped_refptr<VASurface> CreateInputSurface(
  135. VaapiWrapper& vaapi_wrapper,
  136. const gfx::Size& encode_size,
  137. const std::vector<VaapiWrapper::SurfaceUsageHint>& surface_usage_hints);
  138. // Creates |vpp_vaapi_wrapper_| if it hasn't been created.
  139. scoped_refptr<VaapiWrapper> CreateVppVaapiWrapper();
  140. // Executes BlitSurface() using |vpp_vaapi_wrapper_| with |source_surface|,
  141. // |source_visible_rect|. Returns the destination VASurface in BlitSurface()
  142. // whose size is |encode_size| on success, otherwise nullptr.
  143. scoped_refptr<VASurface> ExecuteBlitSurface(
  144. const VASurface& source_surface,
  145. const gfx::Rect source_visible_rect,
  146. const gfx::Size& encode_size);
  147. // Checks if sufficient resources for a new encode job with |frame| as input
  148. // are available, and if so, claims them by associating them with
  149. // a EncodeJob, and returns the newly-created job, nullptr otherwise.
  150. std::unique_ptr<EncodeJob> CreateEncodeJob(
  151. bool force_keyframe,
  152. base::TimeDelta frame_timestamp,
  153. const VASurface& input_surface,
  154. scoped_refptr<VASurface> reconstructed_surface);
  155. // Continues encoding frames as long as input_queue_ is not empty, and we are
  156. // able to create new EncodeJobs.
  157. void EncodePendingInputs();
  158. // Callback that returns a no longer used ScopedVASurface to
  159. // |va_surfaces| for reuse and kicks EncodePendingInputs() again.
  160. void RecycleVASurface(
  161. std::vector<std::unique_ptr<ScopedVASurface>>* va_surfaces,
  162. std::unique_ptr<ScopedVASurface> va_surface,
  163. VASurfaceID va_surface_id);
  164. // Returns pending bitstream buffers to the client if we have both pending
  165. // encoded data to be completed and bitstream buffers available to download
  166. // the encoded data into.
  167. void TryToReturnBitstreamBuffers();
  168. // Downloads encoded data produced as a result of running |encode_result| into
  169. // |buffer|, and returns it to the client.
  170. void ReturnBitstreamBuffer(std::unique_ptr<EncodeResult> encode_result,
  171. std::unique_ptr<BitstreamBufferRef> buffer);
  172. // Puts the encoder into en error state and notifies the client
  173. // about the error.
  174. void NotifyError(Error error);
  175. // Sets the encoder state to |state| on the correct thread.
  176. void SetState(State state);
  177. bool IsConfiguredForTesting() const {
  178. return !supported_profiles_for_testing_.empty();
  179. }
  180. // Having too many encoder instances at once may cause us to run out of FDs
  181. // and subsequently crash (crbug.com/1289465). To avoid that, we limit the
  182. // maximum number of encoder instances that can exist at once.
  183. // |num_instances_| tracks that number.
  184. static constexpr int kMaxNumOfInstances = 10;
  185. static base::AtomicRefCount num_instances_;
  186. const bool can_use_encoder_;
  187. // The unchanged values are filled upon the construction. The varied values
  188. // are filled properly during encoding.
  189. VideoEncoderInfo encoder_info_;
  190. // VaapiWrapper is the owner of all HW resources (surfaces and buffers)
  191. // and will free them on destruction.
  192. scoped_refptr<VaapiWrapper> vaapi_wrapper_
  193. GUARDED_BY_CONTEXT(encoder_sequence_checker_);
  194. // The expected coded size of incoming video frames when |native_input_mode_|
  195. // is false.
  196. gfx::Size expected_input_coded_size_;
  197. // The codec of the stream to be produced. Set during initialization.
  198. VideoCodec output_codec_ = VideoCodec::kUnknown;
  199. // The visible rect to be encoded.
  200. gfx::Rect visible_rect_;
  201. // Size in bytes required for output bitstream buffers.
  202. size_t output_buffer_byte_size_;
  203. // This flag signals when the client is sending NV12 + DmaBuf-backed
  204. // VideoFrames to encode, which allows for skipping a copy-adaptation on
  205. // input.
  206. bool native_input_mode_ = false;
  207. // The number of frames that needs to be held on encoding.
  208. size_t num_frames_in_flight_;
  209. // All of the members below must be accessed on the encoder_task_runner_,
  210. // while it is running.
  211. // Encoder state. Encode tasks will only run in kEncoding state.
  212. State state_;
  213. // Encoder instance managing video codec state and preparing encode jobs.
  214. // Should only be used on |encoder_task_runner_|.
  215. std::unique_ptr<VaapiVideoEncoderDelegate> encoder_;
  216. // Map of input surfaces. In non |native_input_mode_|, this is always created
  217. // and memory-based encode input VideoFrame is written into this.
  218. // In |native_input_mode_|, this is created only if scaling or cropping is
  219. // required and used as a VPP destination.
  220. InputSurfaceMap input_surfaces_;
  221. // Map of available reconstructed surfaces for encoding index by a layer
  222. // resolution. These are stored as reference frames in
  223. // VaapiVideoEncoderDelegate if necessary.
  224. EncodeSurfacesMap available_encode_surfaces_;
  225. // Map of the number of allocated reconstructed surfaces for encoding
  226. // indexed by a layer resolution.
  227. EncodeSurfacesCountMap encode_surfaces_count_;
  228. // Queue of input frames to be encoded.
  229. base::queue<std::unique_ptr<InputFrameRef>> input_queue_;
  230. // BitstreamBuffers mapped, ready to be filled with encoded stream data.
  231. base::queue<std::unique_ptr<BitstreamBufferRef>> available_bitstream_buffers_;
  232. // VASurfaces already encoded and waiting for the bitstream buffer to
  233. // be downloaded.
  234. base::queue<std::unique_ptr<EncodeResult>> pending_encode_results_;
  235. // Task runner for interacting with the client, and its checker.
  236. const scoped_refptr<base::SingleThreadTaskRunner> child_task_runner_;
  237. SEQUENCE_CHECKER(child_sequence_checker_);
  238. // Encoder sequence and its checker. All tasks are executed on it.
  239. const scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
  240. SEQUENCE_CHECKER(encoder_sequence_checker_);
  241. // To expose client callbacks from VideoEncodeAccelerator.
  242. // NOTE: all calls to these objects *MUST* be executed on
  243. // child_task_runner_.
  244. std::unique_ptr<base::WeakPtrFactory<Client>> client_ptr_factory_;
  245. base::WeakPtr<Client> client_;
  246. // VaapiWrapper for VPP (Video Pre Processing). This is used for scale down
  247. // for the picture send to vaapi encoder.
  248. scoped_refptr<VaapiWrapper> vpp_vaapi_wrapper_
  249. GUARDED_BY_CONTEXT(encoder_sequence_checker_);
  250. // The completion callback of the Flush() function.
  251. FlushCallback flush_callback_;
  252. // Supported profiles that are filled if and only if in a unit test.
  253. SupportedProfiles supported_profiles_for_testing_;
  254. // WeakPtr of this, bound to |child_task_runner_|.
  255. base::WeakPtr<VaapiVideoEncodeAccelerator> child_weak_this_;
  256. // WeakPtr of this, bound to |encoder_task_runner_|.
  257. base::WeakPtr<VaapiVideoEncodeAccelerator> encoder_weak_this_;
  258. base::WeakPtrFactory<VaapiVideoEncodeAccelerator> child_weak_this_factory_{
  259. this};
  260. base::WeakPtrFactory<VaapiVideoEncodeAccelerator> encoder_weak_this_factory_{
  261. this};
  262. };
  263. } // namespace media
  264. #endif // MEDIA_GPU_VAAPI_VAAPI_VIDEO_ENCODE_ACCELERATOR_H_