vaapi_image_processor_backend.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/vaapi/vaapi_image_processor_backend.h"
  5. #include <stdint.h>
  6. #include <va/va.h>
  7. #include "base/bind.h"
  8. #include "base/callback.h"
  9. #include "base/containers/contains.h"
  10. #include "base/memory/ptr_util.h"
  11. #include "base/memory/scoped_refptr.h"
  12. #include "base/metrics/histogram_functions.h"
  13. #include "base/stl_util.h"
  14. #include "build/build_config.h"
  15. #include "build/chromeos_buildflags.h"
  16. #include "media/gpu/chromeos/fourcc.h"
  17. #include "media/gpu/chromeos/platform_video_frame_utils.h"
  18. #include "media/gpu/macros.h"
  19. #include "media/gpu/vaapi/va_surface.h"
  20. #include "media/gpu/vaapi/vaapi_utils.h"
  21. #include "media/gpu/vaapi/vaapi_wrapper.h"
  22. #include "ui/gfx/native_pixmap.h"
  23. namespace media {
  24. #if BUILDFLAG(IS_CHROMEOS)
  25. namespace {
  26. bool IsSupported(const ImageProcessorBackend::PortConfig& config) {
  27. if (!config.fourcc.ToVAFourCC())
  28. return false;
  29. const uint32_t va_fourcc = *config.fourcc.ToVAFourCC();
  30. if (!VaapiWrapper::IsVppFormatSupported(va_fourcc)) {
  31. VLOGF(2) << "Unsupported format: VA_FOURCC_" << FourccToString(va_fourcc);
  32. return false;
  33. }
  34. if (!VaapiWrapper::IsVppResolutionAllowed(config.size)) {
  35. VLOGF(2) << "Unsupported size: " << config.size.ToString();
  36. return false;
  37. }
  38. const gfx::Size& visible_size = config.visible_rect.size();
  39. if (!VaapiWrapper::IsVppResolutionAllowed(visible_size)) {
  40. VLOGF(2) << "Unsupported visible size: " << visible_size.ToString();
  41. return false;
  42. }
  43. if (!gfx::Rect(config.size).Contains(config.visible_rect)) {
  44. VLOGF(2) << "The frame size (" << config.size.ToString()
  45. << ") does not contain the visible rect ("
  46. << config.visible_rect.ToString() << ")";
  47. return false;
  48. }
  49. return true;
  50. }
  51. } // namespace
  52. #endif
  53. // static
  54. std::unique_ptr<ImageProcessorBackend> VaapiImageProcessorBackend::Create(
  55. const PortConfig& input_config,
  56. const PortConfig& output_config,
  57. OutputMode output_mode,
  58. VideoRotation relative_rotation,
  59. ErrorCB error_cb,
  60. scoped_refptr<base::SequencedTaskRunner> backend_task_runner) {
  61. DCHECK_EQ(output_mode, OutputMode::IMPORT)
  62. << "Only OutputMode::IMPORT supported";
  63. // VaapiImageProcessorBackend supports ChromeOS only.
  64. #if !BUILDFLAG(IS_CHROMEOS)
  65. return nullptr;
  66. #else
  67. if (!IsSupported(input_config) || !IsSupported(output_config))
  68. return nullptr;
  69. if (!base::Contains(input_config.preferred_storage_types,
  70. VideoFrame::STORAGE_GPU_MEMORY_BUFFER)) {
  71. VLOGF(2) << "VaapiImageProcessorBackend supports GpuMemoryBuffer based"
  72. "VideoFrame only for input";
  73. return nullptr;
  74. }
  75. if (!base::Contains(output_config.preferred_storage_types,
  76. VideoFrame::STORAGE_GPU_MEMORY_BUFFER)) {
  77. VLOGF(2) << "VaapiImageProcessorBackend supports GpuMemoryBuffer based"
  78. "VideoFrame only for output";
  79. return nullptr;
  80. }
  81. if (relative_rotation != VIDEO_ROTATION_0) {
  82. // Tests to see if the platform supports rotation.
  83. auto vaapi_wrapper =
  84. VaapiWrapper::Create(VaapiWrapper::kVideoProcess, VAProfileNone,
  85. EncryptionScheme::kUnencrypted, base::DoNothing());
  86. if (!vaapi_wrapper) {
  87. VLOGF(2) << "Failed to create VaapiWrapper";
  88. return nullptr;
  89. }
  90. // Size is irrelevant for a VPP context.
  91. if (!vaapi_wrapper->CreateContext(gfx::Size())) {
  92. VLOGF(2) << "Failed to create context for VPP";
  93. return nullptr;
  94. }
  95. if (!vaapi_wrapper->IsRotationSupported()) {
  96. VLOGF(2) << "VaapiImageProcessorBackend does not support rotation on this"
  97. "platform";
  98. return nullptr;
  99. }
  100. }
  101. // We should restrict the acceptable PortConfig for input and output both to
  102. // the one returned by GetPlatformVideoFrameLayout(). However,
  103. // ImageProcessorFactory interface doesn't provide information about what
  104. // ImageProcessor will be used for. (e.g. format conversion after decoding and
  105. // scaling before encoding). Thus we cannot execute
  106. // GetPlatformVideoFrameLayout() with a proper gfx::BufferUsage.
  107. // TODO(crbug.com/898423): Adjust layout once ImageProcessor provide the use
  108. // scenario.
  109. return base::WrapUnique<ImageProcessorBackend>(new VaapiImageProcessorBackend(
  110. input_config, output_config, OutputMode::IMPORT, relative_rotation,
  111. std::move(error_cb), std::move(backend_task_runner)));
  112. #endif
  113. }
  114. VaapiImageProcessorBackend::VaapiImageProcessorBackend(
  115. const PortConfig& input_config,
  116. const PortConfig& output_config,
  117. OutputMode output_mode,
  118. VideoRotation relative_rotation,
  119. ErrorCB error_cb,
  120. scoped_refptr<base::SequencedTaskRunner> backend_task_runner)
  121. : ImageProcessorBackend(input_config,
  122. output_config,
  123. output_mode,
  124. relative_rotation,
  125. std::move(error_cb),
  126. std::move(backend_task_runner)) {}
  127. VaapiImageProcessorBackend::~VaapiImageProcessorBackend() {
  128. DCHECK_CALLED_ON_VALID_SEQUENCE(backend_sequence_checker_);
  129. DCHECK(vaapi_wrapper_ || allocated_va_surfaces_.empty());
  130. if (vaapi_wrapper_) {
  131. // To clear |allocated_va_surfaces_|, we have to first DestroyContext().
  132. vaapi_wrapper_->DestroyContext();
  133. allocated_va_surfaces_.clear();
  134. }
  135. }
  136. const VASurface* VaapiImageProcessorBackend::GetSurfaceForVideoFrame(
  137. scoped_refptr<VideoFrame> frame,
  138. bool use_protected) {
  139. if (!frame->HasGpuMemoryBuffer())
  140. return nullptr;
  141. DCHECK_EQ(frame->storage_type(), VideoFrame::STORAGE_GPU_MEMORY_BUFFER);
  142. const gfx::GpuMemoryBufferId gmb_id = frame->GetGpuMemoryBuffer()->GetId();
  143. if (base::Contains(allocated_va_surfaces_, gmb_id)) {
  144. const VASurface* surface = allocated_va_surfaces_[gmb_id].get();
  145. CHECK_EQ(frame->GetGpuMemoryBuffer()->GetSize(), surface->size());
  146. const unsigned int format = VaapiWrapper::BufferFormatToVARTFormat(
  147. frame->GetGpuMemoryBuffer()->GetFormat());
  148. CHECK_NE(format, 0u);
  149. CHECK_EQ(format, surface->format());
  150. return surface;
  151. }
  152. scoped_refptr<gfx::NativePixmap> pixmap =
  153. CreateNativePixmapDmaBuf(frame.get());
  154. if (!pixmap) {
  155. VLOGF(1) << "Failed to create NativePixmap from VideoFrame";
  156. return nullptr;
  157. }
  158. DCHECK(vaapi_wrapper_);
  159. auto va_surface = vaapi_wrapper_->CreateVASurfaceForPixmap(std::move(pixmap),
  160. use_protected);
  161. if (!va_surface) {
  162. VLOGF(1) << "Failed to create VASurface from NativePixmap";
  163. return nullptr;
  164. }
  165. allocated_va_surfaces_[gmb_id] = std::move(va_surface);
  166. return allocated_va_surfaces_[gmb_id].get();
  167. }
  168. void VaapiImageProcessorBackend::Process(scoped_refptr<VideoFrame> input_frame,
  169. scoped_refptr<VideoFrame> output_frame,
  170. FrameReadyCB cb) {
  171. DVLOGF(4);
  172. DCHECK_CALLED_ON_VALID_SEQUENCE(backend_sequence_checker_);
  173. if (!vaapi_wrapper_) {
  174. // Note that EncryptionScheme::kUnencrypted is fine even for the use case
  175. // where the VPP is needed for processing protected content after decoding.
  176. // That's because when calling VaapiWrapper::BlitSurface(), we re-use the
  177. // protected session ID created at decoding time.
  178. auto vaapi_wrapper = VaapiWrapper::Create(
  179. VaapiWrapper::kVideoProcess, VAProfileNone,
  180. EncryptionScheme::kUnencrypted,
  181. base::BindRepeating(&ReportVaapiErrorToUMA,
  182. "Media.VaapiImageProcessorBackend.VAAPIError"));
  183. if (!vaapi_wrapper) {
  184. VLOGF(1) << "Failed to create VaapiWrapper";
  185. error_cb_.Run();
  186. return;
  187. }
  188. // Size is irrelevant for a VPP context.
  189. if (!vaapi_wrapper->CreateContext(gfx::Size())) {
  190. VLOGF(1) << "Failed to create context for VPP";
  191. error_cb_.Run();
  192. return;
  193. }
  194. CHECK(relative_rotation_ == VIDEO_ROTATION_0 ||
  195. vaapi_wrapper->IsRotationSupported());
  196. vaapi_wrapper_ = std::move(vaapi_wrapper);
  197. }
  198. bool use_protected = false;
  199. #if BUILDFLAG(IS_CHROMEOS_ASH)
  200. VAProtectedSessionID va_protected_session_id = VA_INVALID_ID;
  201. if (input_frame->metadata().hw_va_protected_session_id.has_value()) {
  202. static_assert(
  203. std::is_same<decltype(input_frame->metadata()
  204. .hw_va_protected_session_id)::value_type,
  205. VAProtectedSessionID>::value,
  206. "The optional type of VideoFrameMetadata::hw_va_protected_session_id "
  207. "is "
  208. "not VAProtectedSessionID");
  209. va_protected_session_id =
  210. input_frame->metadata().hw_va_protected_session_id.value();
  211. use_protected = va_protected_session_id != VA_INVALID_ID;
  212. }
  213. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  214. if (needs_context_ && !vaapi_wrapper_->CreateContext(gfx::Size())) {
  215. VLOGF(1) << "Failed to create context for VPP";
  216. error_cb_.Run();
  217. return;
  218. }
  219. needs_context_ = false;
  220. DCHECK(input_frame);
  221. DCHECK(output_frame);
  222. const VASurface* src_va_surface =
  223. GetSurfaceForVideoFrame(input_frame, use_protected);
  224. if (!src_va_surface) {
  225. error_cb_.Run();
  226. return;
  227. }
  228. const VASurface* dst_va_surface =
  229. GetSurfaceForVideoFrame(output_frame, use_protected);
  230. if (!dst_va_surface) {
  231. error_cb_.Run();
  232. return;
  233. }
  234. // VA-API performs pixel format conversion and scaling without any filters.
  235. if (!vaapi_wrapper_->BlitSurface(
  236. *src_va_surface, *dst_va_surface, input_frame->visible_rect(),
  237. output_frame->visible_rect(), relative_rotation_
  238. #if BUILDFLAG(IS_CHROMEOS_ASH)
  239. ,
  240. va_protected_session_id
  241. #endif
  242. )) {
  243. #if BUILDFLAG(IS_CHROMEOS_ASH)
  244. if (use_protected &&
  245. vaapi_wrapper_->IsProtectedSessionDead(va_protected_session_id)) {
  246. DCHECK_NE(va_protected_session_id, VA_INVALID_ID);
  247. // If the VPP failed because the protected session is dead, we should
  248. // still output the frame. That's because we don't want to put the
  249. // VideoDecoderPipeline into an unusable error state: the
  250. // VaapiVideoDecoder can recover from a dead protected session later and
  251. // the compositor should not try to render the frame we output here
  252. // anyway.
  253. output_frame->set_timestamp(input_frame->timestamp());
  254. output_frame->set_color_space(input_frame->ColorSpace());
  255. std::move(cb).Run(std::move(output_frame));
  256. return;
  257. }
  258. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  259. error_cb_.Run();
  260. return;
  261. }
  262. output_frame->set_timestamp(input_frame->timestamp());
  263. output_frame->set_color_space(input_frame->ColorSpace());
  264. std::move(cb).Run(std::move(output_frame));
  265. }
  266. void VaapiImageProcessorBackend::Reset() {
  267. DVLOGF(4);
  268. DCHECK_CALLED_ON_VALID_SEQUENCE(backend_sequence_checker_);
  269. DCHECK(vaapi_wrapper_ || allocated_va_surfaces_.empty());
  270. if (vaapi_wrapper_) {
  271. // To clear |allocated_va_surfaces_|, we have to first DestroyContext().
  272. vaapi_wrapper_->DestroyContext();
  273. allocated_va_surfaces_.clear();
  274. }
  275. needs_context_ = true;
  276. }
  277. } // namespace media