image_processor_factory.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/chromeos/image_processor_factory.h"
  5. #include <stddef.h>
  6. #include "base/bind.h"
  7. #include "base/callback.h"
  8. #include "base/containers/contains.h"
  9. #include "base/feature_list.h"
  10. #include "base/memory/scoped_refptr.h"
  11. #include "media/base/media_switches.h"
  12. #include "media/base/video_types.h"
  13. #include "media/gpu/buildflags.h"
  14. #include "media/gpu/chromeos/image_processor.h"
  15. #include "media/gpu/chromeos/libyuv_image_processor_backend.h"
  16. #include "media/gpu/macros.h"
  17. #if BUILDFLAG(USE_VAAPI)
  18. #include "media/gpu/vaapi/vaapi_image_processor_backend.h"
  19. #include "media/gpu/vaapi/vaapi_wrapper.h"
  20. #endif // BUILDFLAG(USE_VAAPI)
  21. #if BUILDFLAG(USE_V4L2_CODEC)
  22. #include "media/gpu/v4l2/v4l2_device.h"
  23. #include "media/gpu/v4l2/v4l2_image_processor_backend.h"
  24. #include "media/gpu/v4l2/v4l2_vda_helpers.h"
  25. #endif // BUILDFLAG(USE_V4L2_CODEC)
  26. namespace media {
  27. namespace {
  28. using PixelLayoutCandidate = ImageProcessor::PixelLayoutCandidate;
  29. #if BUILDFLAG(USE_VAAPI)
  30. std::unique_ptr<ImageProcessor> CreateVaapiImageProcessorWithInputCandidates(
  31. const std::vector<PixelLayoutCandidate>& input_candidates,
  32. const gfx::Rect& input_visible_rect,
  33. const gfx::Size& output_size,
  34. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  35. ImageProcessorFactory::PickFormatCB out_format_picker,
  36. ImageProcessor::ErrorCB error_cb) {
  37. std::vector<Fourcc> vpp_supported_formats =
  38. VaapiWrapper::GetVppSupportedFormats();
  39. absl::optional<PixelLayoutCandidate> chosen_input_candidate;
  40. for (const auto& input_candidate : input_candidates) {
  41. if (base::Contains(vpp_supported_formats, input_candidate.fourcc) &&
  42. VaapiWrapper::IsVppResolutionAllowed(input_candidate.size)) {
  43. chosen_input_candidate = input_candidate;
  44. break;
  45. }
  46. }
  47. if (!chosen_input_candidate)
  48. return nullptr;
  49. // Note that we pick the first input candidate as the preferred output format.
  50. // The reason is that in practice, the VaapiVideoDecoder will make
  51. // |input_candidates| either {NV12} or {P010} depending on the bitdepth. So
  52. // choosing the first (and only) element will keep the bitdepth of the frame
  53. // which is needed to display HDR content.
  54. auto chosen_output_format = out_format_picker.Run(
  55. /*candidates=*/vpp_supported_formats, input_candidates[0].fourcc);
  56. if (!chosen_output_format)
  57. return nullptr;
  58. // Note: the VaapiImageProcessorBackend doesn't use the ColorPlaneLayouts in
  59. // the PortConfigs, so we just pass an empty list of plane layouts.
  60. ImageProcessor::PortConfig input_config(
  61. chosen_input_candidate->fourcc, chosen_input_candidate->size,
  62. /*planes=*/{}, input_visible_rect,
  63. {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  64. ImageProcessor::PortConfig output_config(
  65. /*fourcc=*/*chosen_output_format, /*size=*/output_size, /*planes=*/{},
  66. /*visible_rect=*/gfx::Rect(output_size),
  67. {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  68. return ImageProcessor::Create(
  69. base::BindRepeating(&VaapiImageProcessorBackend::Create), input_config,
  70. output_config, ImageProcessor::OutputMode::IMPORT, VIDEO_ROTATION_0,
  71. std::move(error_cb), std::move(client_task_runner));
  72. }
  73. #endif // BUILDFLAG(USE_VAAPI)
  74. #if BUILDFLAG(USE_V4L2_CODEC) && !BUILDFLAG(USE_VAAPI)
  75. std::unique_ptr<ImageProcessor> CreateV4L2ImageProcessorWithInputCandidates(
  76. const std::vector<PixelLayoutCandidate>& input_candidates,
  77. const gfx::Rect& visible_rect,
  78. size_t num_buffers,
  79. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  80. ImageProcessorFactory::PickFormatCB out_format_picker,
  81. ImageProcessor::ErrorCB error_cb) {
  82. // Pick a renderable output format, and try each available input format.
  83. // TODO(akahuang): let |out_format_picker| return a list of supported output
  84. // formats, and try all combination of input/output format, if any platform
  85. // fails to create ImageProcessor via current approach.
  86. const std::vector<uint32_t> supported_output_formats =
  87. V4L2ImageProcessorBackend::GetSupportedOutputFormats();
  88. std::vector<Fourcc> supported_fourccs;
  89. for (const auto& format : supported_output_formats) {
  90. const auto fourcc = Fourcc::FromV4L2PixFmt(format);
  91. if (fourcc.has_value())
  92. supported_fourccs.push_back(*fourcc);
  93. }
  94. const auto output_fourcc = out_format_picker.Run(
  95. /*candidates=*/supported_fourccs, /*preferred_fourcc=*/absl::nullopt);
  96. if (!output_fourcc) {
  97. #if DCHECK_IS_ON()
  98. std::string output_fourccs_string;
  99. for (const auto fourcc : supported_fourccs) {
  100. output_fourccs_string += fourcc.ToString();
  101. output_fourccs_string += " ";
  102. }
  103. DVLOGF(1) << "None of " << output_fourccs_string << "formats is supported.";
  104. #endif
  105. return nullptr;
  106. }
  107. const auto supported_input_pixfmts =
  108. V4L2ImageProcessorBackend::GetSupportedInputFormats();
  109. for (const auto& input_candidate : input_candidates) {
  110. const Fourcc input_fourcc = input_candidate.fourcc;
  111. const gfx::Size& input_size = input_candidate.size;
  112. if (!base::Contains(supported_input_pixfmts, input_fourcc.ToV4L2PixFmt()))
  113. continue;
  114. // Ideally the ImageProcessor would be able to scale and crop |input_size|
  115. // to the |visible_rect| area of |output_size|. TryOutputFormat() below is
  116. // called to verify that a given combination of fourcc values and input/
  117. // output sizes are indeed supported (the driver can potentially return a
  118. // different supported |output_size|). Some Image Processors(e.g. MTK8183)
  119. // are not able to crop/scale correctly -- but TryOutputFormat() doesn't
  120. // return a "corrected" |output_size|. To avoid troubles (and, in general,
  121. // low performance), we set |output_size| to be equal to |input_size|; the
  122. // |visible_rect| will carry the information of exactly what part of the
  123. // video frame contains valid pixels, and the media/compositor pipeline
  124. // will take care of it.
  125. gfx::Size output_size = input_size;
  126. size_t num_planes = 0;
  127. if (!V4L2ImageProcessorBackend::TryOutputFormat(
  128. input_fourcc.ToV4L2PixFmt(), output_fourcc->ToV4L2PixFmt(),
  129. input_size, &output_size, &num_planes)) {
  130. VLOGF(2) << "Failed to get output size and plane count of IP";
  131. continue;
  132. }
  133. // This is very restrictive because it assumes the IP has the same alignment
  134. // criteria as the video decoder that will produce the input video frames.
  135. // In practice, this applies to all Image Processors, i.e. Mediatek devices.
  136. DCHECK_EQ(input_size, output_size);
  137. // |visible_rect| applies equally to both |input_size| and |output_size|.
  138. DCHECK(gfx::Rect(output_size).Contains(visible_rect));
  139. return v4l2_vda_helpers::CreateImageProcessor(
  140. input_fourcc, *output_fourcc, input_size, output_size, visible_rect,
  141. VideoFrame::StorageType::STORAGE_GPU_MEMORY_BUFFER, num_buffers,
  142. V4L2Device::Create(), ImageProcessor::OutputMode::IMPORT,
  143. std::move(client_task_runner), std::move(error_cb));
  144. }
  145. return nullptr;
  146. }
  147. std::unique_ptr<ImageProcessor> CreateLibYUVImageProcessorWithInputCandidates(
  148. const std::vector<PixelLayoutCandidate>& input_candidates,
  149. const gfx::Rect& input_visible_rect,
  150. const gfx::Size& output_size,
  151. size_t num_buffers,
  152. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  153. ImageProcessorFactory::PickFormatCB out_format_picker,
  154. ImageProcessor::ErrorCB error_cb) {
  155. if (input_candidates.size() != 1)
  156. return nullptr;
  157. if (input_candidates[0].fourcc != Fourcc(Fourcc::MM21))
  158. return nullptr;
  159. std::vector<Fourcc> supported_output_formats =
  160. LibYUVImageProcessorBackend::GetSupportedOutputFormats(
  161. Fourcc(Fourcc::MM21));
  162. auto output_format =
  163. out_format_picker.Run(supported_output_formats, Fourcc(Fourcc::NV12));
  164. if (!output_format)
  165. return nullptr;
  166. ImageProcessor::PortConfig input_config(
  167. Fourcc(Fourcc::MM21), input_candidates[0].size, /*planes=*/{},
  168. input_visible_rect, {VideoFrame::STORAGE_DMABUFS});
  169. ImageProcessor::PortConfig output_config(
  170. *output_format, output_size, /*planes=*/{}, gfx::Rect(output_size),
  171. {VideoFrame::STORAGE_GPU_MEMORY_BUFFER});
  172. return ImageProcessor::Create(
  173. base::BindRepeating(&LibYUVImageProcessorBackend::Create), input_config,
  174. output_config, ImageProcessor::OutputMode::IMPORT, VIDEO_ROTATION_0,
  175. std::move(error_cb), std::move(client_task_runner));
  176. }
  177. #endif // BUILDFLAG(USE_V4L2_CODEC) && !BUILDFLAG(USE_VAAPI)
  178. } // namespace
  179. // static
  180. std::unique_ptr<ImageProcessor> ImageProcessorFactory::Create(
  181. const ImageProcessor::PortConfig& input_config,
  182. const ImageProcessor::PortConfig& output_config,
  183. ImageProcessor::OutputMode output_mode,
  184. size_t num_buffers,
  185. VideoRotation relative_rotation,
  186. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  187. ImageProcessor::ErrorCB error_cb) {
  188. std::vector<ImageProcessor::CreateBackendCB> create_funcs;
  189. #if BUILDFLAG(USE_VAAPI)
  190. create_funcs.push_back(
  191. base::BindRepeating(&VaapiImageProcessorBackend::Create));
  192. #elif BUILDFLAG(USE_V4L2_CODEC)
  193. create_funcs.push_back(base::BindRepeating(
  194. &V4L2ImageProcessorBackend::Create, V4L2Device::Create(), num_buffers));
  195. #endif // BUILDFLAG(USE_V4L2_CODEC)
  196. create_funcs.push_back(
  197. base::BindRepeating(&LibYUVImageProcessorBackend::Create));
  198. std::unique_ptr<ImageProcessor> image_processor;
  199. for (auto& create_func : create_funcs) {
  200. image_processor = ImageProcessor::Create(
  201. std::move(create_func), input_config, output_config, output_mode,
  202. relative_rotation, error_cb, client_task_runner);
  203. if (image_processor)
  204. return image_processor;
  205. }
  206. return nullptr;
  207. }
  208. // static
  209. std::unique_ptr<ImageProcessor>
  210. ImageProcessorFactory::CreateWithInputCandidates(
  211. const std::vector<PixelLayoutCandidate>& input_candidates,
  212. const gfx::Rect& input_visible_rect,
  213. const gfx::Size& output_size,
  214. size_t num_buffers,
  215. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  216. PickFormatCB out_format_picker,
  217. ImageProcessor::ErrorCB error_cb) {
  218. #if BUILDFLAG(USE_VAAPI)
  219. auto processor = CreateVaapiImageProcessorWithInputCandidates(
  220. input_candidates, input_visible_rect, output_size, client_task_runner,
  221. out_format_picker, error_cb);
  222. if (processor)
  223. return processor;
  224. #elif BUILDFLAG(USE_V4L2_CODEC)
  225. if (base::FeatureList::IsEnabled(media::kPreferLibYuvImageProcessor)) {
  226. auto processor = CreateLibYUVImageProcessorWithInputCandidates(
  227. input_candidates, input_visible_rect, output_size, num_buffers,
  228. client_task_runner, out_format_picker, error_cb);
  229. if (processor)
  230. return processor;
  231. }
  232. auto processor = CreateV4L2ImageProcessorWithInputCandidates(
  233. input_candidates, input_visible_rect, num_buffers, client_task_runner,
  234. out_format_picker, error_cb);
  235. if (processor)
  236. return processor;
  237. #endif // BUILDFLAG(USE_V4L2_CODEC)
  238. // TODO(crbug.com/1004727): Implement LibYUVImageProcessorBackend. When doing
  239. // so, we must keep in mind that it might not be desirable to fallback to
  240. // libyuv if the hardware image processor fails (e.g., in the case of
  241. // protected content).
  242. return nullptr;
  243. }
  244. } // namespace media