image_processor.h 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_GPU_CHROMEOS_IMAGE_PROCESSOR_H_
  5. #define MEDIA_GPU_CHROMEOS_IMAGE_PROCESSOR_H_
  6. #include <stdint.h>
  7. #include <map>
  8. #include <utility>
  9. #include <vector>
  10. #include "base/callback_forward.h"
  11. #include "base/memory/scoped_refptr.h"
  12. #include "base/memory/weak_ptr.h"
  13. #include "base/sequence_checker.h"
  14. #include "base/task/sequenced_task_runner.h"
  15. #include "media/base/video_frame.h"
  16. #include "media/gpu/chromeos/fourcc.h"
  17. #include "media/gpu/chromeos/image_processor_backend.h"
  18. #include "media/gpu/media_gpu_export.h"
  19. #include "third_party/abseil-cpp/absl/types/optional.h"
  20. #include "ui/gfx/native_pixmap_handle.h"
  21. namespace media {
  22. // An image processor is used to convert from one image format to another (e.g.
  23. // I420 to NV12) while optionally scaling. It is useful in situations where
  24. // a given video hardware (e.g. decoder or encoder) accepts or produces data
  25. // in a format different from what the rest of the pipeline expects.
  26. class MEDIA_GPU_EXPORT ImageProcessor {
  27. public:
  28. struct PixelLayoutCandidate {
  29. Fourcc fourcc;
  30. gfx::Size size;
  31. uint64_t modifier = gfx::NativePixmapHandle::kNoModifier;
  32. // For testing.
  33. bool operator==(const PixelLayoutCandidate& candidate) const {
  34. return this->fourcc == candidate.fourcc && this->size == candidate.size &&
  35. this->modifier == candidate.modifier;
  36. }
  37. };
  38. using PortConfig = ImageProcessorBackend::PortConfig;
  39. using OutputMode = ImageProcessorBackend::OutputMode;
  40. using ErrorCB = ImageProcessorBackend::ErrorCB;
  41. using FrameReadyCB = ImageProcessorBackend::FrameReadyCB;
  42. using LegacyFrameReadyCB = ImageProcessorBackend::LegacyFrameReadyCB;
  43. // Callback type for creating a ImageProcessorBackend instance. This allows us
  44. // to create ImageProcessorBackend instance inside ImageProcessor::Create().
  45. using CreateBackendCB =
  46. base::RepeatingCallback<std::unique_ptr<ImageProcessorBackend>(
  47. const PortConfig& input_config,
  48. const PortConfig& output_config,
  49. OutputMode output_mode,
  50. VideoRotation relative_rotation,
  51. ErrorCB error_cb,
  52. scoped_refptr<base::SequencedTaskRunner> backend_task_runner)>;
  53. static std::unique_ptr<ImageProcessor> Create(
  54. CreateBackendCB create_backend_cb,
  55. const PortConfig& input_config,
  56. const PortConfig& output_config,
  57. OutputMode output_mode,
  58. VideoRotation relative_rotation,
  59. ErrorCB error_cb,
  60. scoped_refptr<base::SequencedTaskRunner> client_task_runner);
  61. ImageProcessor() = delete;
  62. ImageProcessor(const ImageProcessor&) = delete;
  63. ImageProcessor& operator=(const ImageProcessor&) = delete;
  64. virtual ~ImageProcessor();
  65. virtual const PortConfig& input_config() const;
  66. virtual const PortConfig& output_config() const;
  67. OutputMode output_mode() const { return backend_->output_mode(); }
  68. // Called by client to process |frame|. The resulting processed frame will be
  69. // stored in a ImageProcessor-owned output buffer and notified via |cb|. The
  70. // processor will drop all its references to |frame| after it finishes
  71. // accessing it.
  72. // Process() must be called on |client_task_runner_|. This should not be
  73. // blocking function.
  74. // TODO(crbug.com/907767): Remove this once ImageProcessor always works as
  75. // IMPORT mode for output.
  76. bool Process(scoped_refptr<VideoFrame> frame, LegacyFrameReadyCB cb);
  77. // Called by client to process |input_frame| and store in |output_frame|. This
  78. // can only be used when output mode is IMPORT. The processor will drop all
  79. // its references to |input_frame| and |output_frame| after it finishes
  80. // accessing it.
  81. // Process() must be called on |client_task_runner_|. This should not be
  82. // blocking function.
  83. bool Process(scoped_refptr<VideoFrame> input_frame,
  84. scoped_refptr<VideoFrame> output_frame,
  85. FrameReadyCB cb);
  86. // Reset all processing frames. After this method returns, no more callbacks
  87. // will be invoked. ImageProcessor is ready to process more frames.
  88. // Reset() must be called on |client_task_runner_|.
  89. bool Reset();
  90. // Returns true if and only if, in IMPORT mode, the image processor requires
  91. // the output video frames to be CPU-readable with a linear view of the data.
  92. bool needs_linear_output_buffers() const {
  93. return needs_linear_output_buffers_;
  94. }
  95. // Returns true if the image processor supports buffers allocated
  96. // incoherently. The MTK MDP3 image processor has coherency issues, but the
  97. // Libyuv image processor benefits greatly from incoherent allocations.
  98. // Defaults to false, since only Libyuv has been shown to support this feature
  99. // so far.
  100. bool SupportsIncoherentBufs() const {
  101. return backend_ && backend_->supports_incoherent_buffers();
  102. }
  103. protected:
  104. // Container for both FrameReadyCB and LegacyFrameReadyCB. With this class,
  105. // we could store both kind of callback in the same container.
  106. // TODO(crbug.com/907767): Remove this once ImageProcessor always works as
  107. // IMPORT mode for output.
  108. struct ClientCallback {
  109. ClientCallback(FrameReadyCB ready_cb);
  110. ClientCallback(LegacyFrameReadyCB legacy_ready_cb);
  111. ClientCallback(ClientCallback&&);
  112. ~ClientCallback();
  113. FrameReadyCB ready_cb;
  114. LegacyFrameReadyCB legacy_ready_cb;
  115. };
  116. ImageProcessor(std::unique_ptr<ImageProcessorBackend> backend,
  117. scoped_refptr<base::SequencedTaskRunner> client_task_runner,
  118. scoped_refptr<base::SequencedTaskRunner> backend_task_runner);
  119. // Callbacks of processing frames.
  120. static void OnProcessDoneThunk(
  121. scoped_refptr<base::SequencedTaskRunner> task_runner,
  122. absl::optional<base::WeakPtr<ImageProcessor>> weak_this,
  123. int cb_index,
  124. scoped_refptr<VideoFrame> frame);
  125. static void OnProcessLegacyDoneThunk(
  126. scoped_refptr<base::SequencedTaskRunner> task_runner,
  127. absl::optional<base::WeakPtr<ImageProcessor>> weak_this,
  128. int cb_index,
  129. size_t buffer_id,
  130. scoped_refptr<VideoFrame> frame);
  131. void OnProcessDone(int cb_index, scoped_refptr<VideoFrame> frame);
  132. void OnProcessLegacyDone(int cb_index,
  133. size_t buffer_id,
  134. scoped_refptr<VideoFrame> frame);
  135. // Store |cb| at |pending_cbs_| and return a index for the callback.
  136. int StoreCallback(ClientCallback cb);
  137. // The backend of image processor. All of its methods are called on
  138. // |backend_task_runner_|.
  139. std::unique_ptr<ImageProcessorBackend> backend_;
  140. // Store the callbacks from the client, accessed on |client_task_runner_|.
  141. // This storage is to guarantee the callbacks are called or destroyed on the
  142. // correct sequence.
  143. std::map<int /* cb_index */, ClientCallback /* cb */> pending_cbs_;
  144. // The index of next callback.
  145. int next_cb_index_ = 0;
  146. // The sequence and its checker for interacting with the client.
  147. scoped_refptr<base::SequencedTaskRunner> client_task_runner_;
  148. SEQUENCE_CHECKER(client_sequence_checker_);
  149. // The sequence for interacting with |backend_|.
  150. scoped_refptr<base::SequencedTaskRunner> backend_task_runner_;
  151. const bool needs_linear_output_buffers_;
  152. // The weak pointer of this, bound to |client_task_runner_|.
  153. base::WeakPtr<ImageProcessor> weak_this_;
  154. base::WeakPtrFactory<ImageProcessor> weak_this_factory_{this};
  155. };
  156. } // namespace media
  157. #endif // MEDIA_GPU_CHROMEOS_IMAGE_PROCESSOR_H_