paint_image.h 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef CC_PAINT_PAINT_IMAGE_H_
  5. #define CC_PAINT_PAINT_IMAGE_H_
  6. #include <string>
  7. #include <vector>
  8. #include "base/gtest_prod_util.h"
  9. #include "base/memory/scoped_refptr.h"
  10. #include "cc/paint/frame_metadata.h"
  11. #include "cc/paint/image_animation_count.h"
  12. #include "cc/paint/paint_export.h"
  13. #include "gpu/command_buffer/common/mailbox.h"
  14. #include "third_party/abseil-cpp/absl/types/optional.h"
  15. #include "third_party/skia/include/core/SkImage.h"
  16. #include "third_party/skia/include/core/SkImageInfo.h"
  17. #include "third_party/skia/include/core/SkRefCnt.h"
  18. #include "third_party/skia/include/core/SkYUVAPixmaps.h"
  19. #include "ui/gfx/display_color_spaces.h"
  20. #include "ui/gfx/geometry/rect.h"
  21. #include "ui/gfx/geometry/size.h"
  22. class SkBitmap;
  23. class SkColorSpace;
  24. struct SkISize;
  25. namespace blink {
  26. class VideoFrame;
  27. }
  28. namespace cc {
  29. class PaintImageGenerator;
  30. class PaintOpBuffer;
  31. class PaintWorkletInput;
  32. class TextureBacking;
  33. using PaintRecord = PaintOpBuffer;
  34. enum class ImageType {
  35. kPNG,
  36. kJPEG,
  37. kWEBP,
  38. kGIF,
  39. kICO,
  40. kBMP,
  41. kAVIF,
  42. kJXL,
  43. kInvalid
  44. };
  45. enum class YUVSubsampling { k410, k411, k420, k422, k440, k444, kUnknown };
  46. enum class YUVIndex { kY, kU, kV };
  47. // Should match the number of YUVIndex values.
  48. static constexpr int kNumYUVPlanes = 3;
  49. struct CC_PAINT_EXPORT ImageHeaderMetadata {
  50. public:
  51. ImageHeaderMetadata();
  52. ImageHeaderMetadata(const ImageHeaderMetadata& other);
  53. ImageHeaderMetadata& operator=(const ImageHeaderMetadata& other);
  54. ~ImageHeaderMetadata();
  55. // The image type, e.g., JPEG or WebP.
  56. ImageType image_type = ImageType::kInvalid;
  57. // The subsampling format used for the chroma planes, e.g., YUV 4:2:0.
  58. YUVSubsampling yuv_subsampling = YUVSubsampling::kUnknown;
  59. // The visible size of the image (i.e., the area that contains meaningful
  60. // pixels).
  61. gfx::Size image_size;
  62. // The size of the area containing coded data, if known. For example, if the
  63. // |image_size| for a 4:2:0 JPEG is 12x31, its coded size should be 16x32
  64. // because the size of a minimum-coded unit for 4:2:0 is 16x16.
  65. // A zero-initialized |coded_size| indicates an invalid image.
  66. absl::optional<gfx::Size> coded_size;
  67. // Whether the image embeds an ICC color profile.
  68. bool has_embedded_color_profile = false;
  69. // Whether all the data was received prior to starting decoding work.
  70. bool all_data_received_prior_to_decode = false;
  71. // For JPEGs only: whether the image is progressive (as opposed to baseline).
  72. absl::optional<bool> jpeg_is_progressive;
  73. // For WebPs only: whether this is a simple-format lossy image. See
  74. // https://developers.google.com/speed/webp/docs/riff_container#simple_file_format_lossy.
  75. absl::optional<bool> webp_is_non_extended_lossy;
  76. };
  77. // A representation of an image for the compositor. This is the most abstract
  78. // form of images, and represents what is known at paint time. Note that aside
  79. // from default construction, it can only be constructed using a
  80. // PaintImageBuilder, or copied/moved into using operator=. PaintImage can
  81. // be backed by different kinds of content, such as a lazy generator, a paint
  82. // record, a bitmap, or a texture.
  83. //
  84. // If backed by a generator, this image may not be decoded and information like
  85. // the animation frame, the target colorspace, or the scale at which it will be
  86. // used are not known yet. A DrawImage is a PaintImage with those decisions
  87. // known but that might not have been decoded yet. A DecodedDrawImage is a
  88. // DrawImage that has been decoded/scaled/uploaded with all of those parameters
  89. // applied.
  90. //
  91. // The PaintImage -> DrawImage -> DecodedDrawImage -> PaintImage (via SkImage)
  92. // path can be used to create a PaintImage that is snapshotted at a particular
  93. // scale or animation frame.
  94. class CC_PAINT_EXPORT PaintImage {
  95. public:
  96. using Id = int;
  97. using AnimationSequenceId = uint32_t;
  98. // A ContentId is used to identify the content for which images which can be
  99. // lazily generated (generator/record backed images). As opposed to Id, which
  100. // stays constant for the same image, the content id can be updated when the
  101. // backing encoded data for this image changes. For instance, in the case of
  102. // images which can be progressively updated as more encoded data is received.
  103. using ContentId = int;
  104. // A GeneratorClientId can be used to namespace different clients that are
  105. // using the output of a PaintImageGenerator.
  106. //
  107. // This is used to allow multiple compositors to simultaneously decode the
  108. // same image. Each compositor is assigned a unique GeneratorClientId which is
  109. // passed through to the decoder from PaintImage::Decode. Internally the
  110. // decoder ensures that requestes from different clients are executed in
  111. // parallel. This is particularly important for animated images, where
  112. // compositors displaying the same image can request decodes for different
  113. // frames from this image.
  114. using GeneratorClientId = int;
  115. static const GeneratorClientId kDefaultGeneratorClientId;
  116. // The default frame index to use if no index is provided. For multi-frame
  117. // images, this would imply the first frame of the animation.
  118. static const size_t kDefaultFrameIndex;
  119. static const Id kInvalidId;
  120. static const ContentId kInvalidContentId;
  121. class CC_PAINT_EXPORT FrameKey {
  122. public:
  123. FrameKey(ContentId content_id, size_t frame_index);
  124. bool operator==(const FrameKey& other) const;
  125. bool operator!=(const FrameKey& other) const;
  126. size_t hash() const { return hash_; }
  127. std::string ToString() const;
  128. size_t frame_index() const { return frame_index_; }
  129. ContentId content_id() const { return content_id_; }
  130. private:
  131. ContentId content_id_;
  132. size_t frame_index_;
  133. size_t hash_;
  134. };
  135. struct CC_PAINT_EXPORT FrameKeyHash {
  136. size_t operator()(const FrameKey& frame_key) const {
  137. return frame_key.hash();
  138. }
  139. };
  140. enum class AnimationType { ANIMATED, VIDEO, STATIC };
  141. enum class CompletionState { DONE, PARTIALLY_DONE };
  142. enum class DecodingMode {
  143. // No preference has been specified. The compositor may choose to use sync
  144. // or async decoding. See CheckerImageTracker for the default behaviour.
  145. kUnspecified,
  146. // It's preferred to display this image synchronously with the rest of the
  147. // content updates, skipping any heuristics.
  148. kSync,
  149. // Async is preferred. The compositor may decode async if it meets the
  150. // heuristics used to avoid flickering (for instance vetoing of multipart
  151. // response, animated, partially loaded images) and would be performant. See
  152. // CheckerImageTracker for all heuristics used.
  153. kAsync
  154. };
  155. // Returns the more conservative mode out of the two given ones.
  156. static DecodingMode GetConservative(DecodingMode one, DecodingMode two);
  157. static Id GetNextId();
  158. static ContentId GetNextContentId();
  159. static GeneratorClientId GetNextGeneratorClientId();
  160. // Creates a PaintImage wrapping |bitmap|. Note that the pixels will be copied
  161. // unless the bitmap is marked immutable.
  162. static PaintImage CreateFromBitmap(SkBitmap bitmap);
  163. PaintImage();
  164. PaintImage(const PaintImage& other);
  165. PaintImage(PaintImage&& other);
  166. ~PaintImage();
  167. PaintImage& operator=(const PaintImage& other);
  168. PaintImage& operator=(PaintImage&& other);
  169. bool operator==(const PaintImage& other) const;
  170. bool operator!=(const PaintImage& other) const { return !(*this == other); }
  171. // Returns the smallest size that is at least as big as the requested_size
  172. // such that we can decode to exactly that scale. If the requested size is
  173. // larger than the image, this returns the image size. Any returned value is
  174. // guaranteed to be stable. That is,
  175. // GetSupportedDecodeSize(GetSupportedDecodeSize(size)) is guaranteed to be
  176. // GetSupportedDecodeSize(size).
  177. SkISize GetSupportedDecodeSize(const SkISize& requested_size) const;
  178. // Decode the image into RGBX into the given memory for the given SkImageInfo.
  179. // - Size in |info| must be supported.
  180. // - The amount of memory allocated must be at least
  181. // |info|.minRowBytes() * |info|.height()
  182. // Returns true on success and false on failure. Updates |info| to match the
  183. // requested color space, if provided.
  184. // Note that for non-lazy images this will do a copy or readback if the image
  185. // is texture backed.
  186. bool Decode(void* memory,
  187. SkImageInfo* info,
  188. sk_sp<SkColorSpace> color_space,
  189. size_t frame_index,
  190. GeneratorClientId client_id) const;
  191. // Decode the image into YUV into |pixmaps|.
  192. // - SkPixmaps owned by |pixmaps| are preallocated to store the
  193. // planar data. They must have have color types, row bytes,
  194. // and sizes as indicated by PaintImage::IsYuv().
  195. // - The |frame_index| parameter will be passed along to
  196. // ImageDecoder::DecodeToYUV but for multi-frame YUV support, ImageDecoder
  197. // needs a separate YUV frame buffer cache.
  198. bool DecodeYuv(const SkYUVAPixmaps& pixmaps,
  199. size_t frame_index,
  200. GeneratorClientId client_id) const;
  201. // Returns the SkImage associated with this PaintImage. If PaintImage is
  202. // texture backed, this API will always do a readback from GPU to CPU memory,
  203. // so avoid using it unless actual pixels are needed. For other cases, prefer
  204. // using PaintImage APIs directly or use GetSkImageInfo() for metadata about
  205. // the SkImage.
  206. sk_sp<SkImage> GetSwSkImage() const;
  207. // Reads this image's pixels into caller-owned |dst_pixels|
  208. bool readPixels(const SkImageInfo& dst_info,
  209. void* dst_pixels,
  210. size_t dst_row_bytes,
  211. int src_x,
  212. int src_y) const;
  213. // Returned mailbox must not outlive this PaintImage.
  214. gpu::Mailbox GetMailbox() const;
  215. Id stable_id() const { return id_; }
  216. SkImageInfo GetSkImageInfo() const;
  217. AnimationType animation_type() const { return animation_type_; }
  218. CompletionState completion_state() const { return completion_state_; }
  219. bool is_multipart() const { return is_multipart_; }
  220. bool is_high_bit_depth() const { return is_high_bit_depth_; }
  221. bool may_be_lcp_candidate() const { return may_be_lcp_candidate_; }
  222. int repetition_count() const { return repetition_count_; }
  223. bool ShouldAnimate() const;
  224. AnimationSequenceId reset_animation_sequence_id() const {
  225. return reset_animation_sequence_id_;
  226. }
  227. DecodingMode decoding_mode() const { return decoding_mode_; }
  228. explicit operator bool() const {
  229. return paint_worklet_input_ || cached_sk_image_ || texture_backing_;
  230. }
  231. bool IsLazyGenerated() const {
  232. return paint_record_ || paint_image_generator_;
  233. }
  234. bool IsPaintWorklet() const { return !!paint_worklet_input_; }
  235. bool IsTextureBacked() const;
  236. // Skia internally buffers commands and flushes them as necessary but there
  237. // are some cases where we need to force a flush.
  238. void FlushPendingSkiaOps();
  239. int width() const { return GetSkImageInfo().width(); }
  240. int height() const { return GetSkImageInfo().height(); }
  241. SkColorSpace* color_space() const {
  242. return paint_worklet_input_ ? nullptr : GetSkImageInfo().colorSpace();
  243. }
  244. gfx::ContentColorUsage GetContentColorUsage(bool* is_hlg = nullptr) const;
  245. // Returns whether this image will be decoded and rendered from YUV data
  246. // and fills out |info|. |supported_data_types| indicates the bit depths and
  247. // data types allowed. If successful, the caller can use |info| to allocate
  248. // SkPixmaps to pass DecodeYuv() and render with the correct YUV->RGB
  249. // transformation.
  250. bool IsYuv(const SkYUVAPixmapInfo::SupportedDataTypes& supported_data_types,
  251. SkYUVAPixmapInfo* info = nullptr) const;
  252. // Get metadata associated with this image.
  253. SkColorType GetColorType() const { return GetSkImageInfo().colorType(); }
  254. SkAlphaType GetAlphaType() const { return GetSkImageInfo().alphaType(); }
  255. // Returns general information about the underlying image. Returns nullptr if
  256. // there is no available |paint_image_generator_|.
  257. const ImageHeaderMetadata* GetImageHeaderMetadata() const;
  258. // Returns a unique id for the pixel data for the frame at |frame_index|.
  259. FrameKey GetKeyForFrame(size_t frame_index) const;
  260. PaintImage::ContentId GetContentIdForFrame(size_t frame_index) const;
  261. // Returns the metadata for each frame of a multi-frame image. Should only be
  262. // used with animated images.
  263. const std::vector<FrameMetadata>& GetFrameMetadata() const;
  264. // Returns the total number of frames known to exist in this image.
  265. size_t FrameCount() const;
  266. // Returns an SkImage for the frame at |index|.
  267. sk_sp<SkImage> GetSkImageForFrame(size_t index,
  268. GeneratorClientId client_id) const;
  269. const scoped_refptr<PaintWorkletInput>& paint_worklet_input() const {
  270. return paint_worklet_input_;
  271. }
  272. bool IsOpaque() const;
  273. std::string ToString() const;
  274. private:
  275. friend class PaintImageBuilder;
  276. FRIEND_TEST_ALL_PREFIXES(PaintImageTest, Subsetting);
  277. // Used internally for PaintImages created at raster.
  278. static const Id kNonLazyStableId;
  279. friend class ScopedRasterFlags;
  280. friend class PaintOpReader;
  281. friend class PlaybackImageProvider;
  282. friend class DrawImageRectOp;
  283. friend class DrawImageOp;
  284. friend class DrawSkottieOp;
  285. // TODO(crbug.com/1031051): Remove these once GetSkImage()
  286. // is fully removed.
  287. friend class ImagePaintFilter;
  288. friend class PaintShader;
  289. friend class blink::VideoFrame;
  290. bool DecodeFromGenerator(void* memory,
  291. SkImageInfo* info,
  292. sk_sp<SkColorSpace> color_space,
  293. size_t frame_index,
  294. GeneratorClientId client_id) const;
  295. bool DecodeFromSkImage(void* memory,
  296. SkImageInfo* info,
  297. sk_sp<SkColorSpace> color_space,
  298. size_t frame_index,
  299. GeneratorClientId client_id) const;
  300. void CreateSkImage();
  301. // Only supported in non-OOPR contexts by friend callers.
  302. sk_sp<SkImage> GetAcceleratedSkImage() const;
  303. // GetSkImage() is being deprecated, see crbug.com/1031051.
  304. // Prefer using GetSwSkImage() or GetSkImageInfo().
  305. const sk_sp<SkImage>& GetSkImage() const;
  306. sk_sp<SkImage> sk_image_;
  307. sk_sp<PaintRecord> paint_record_;
  308. gfx::Rect paint_record_rect_;
  309. ContentId content_id_ = kInvalidContentId;
  310. sk_sp<PaintImageGenerator> paint_image_generator_;
  311. sk_sp<TextureBacking> texture_backing_;
  312. Id id_ = 0;
  313. AnimationType animation_type_ = AnimationType::STATIC;
  314. CompletionState completion_state_ = CompletionState::DONE;
  315. int repetition_count_ = kAnimationNone;
  316. // Whether the data fetched for this image is a part of a multpart response.
  317. bool is_multipart_ = false;
  318. // Whether this image has more than 8 bits per color channel.
  319. bool is_high_bit_depth_ = false;
  320. // Whether this image may untimately be a candidate for Largest Contentful
  321. // Paint. The final LCP contribution of an image is unknown until we present
  322. // it, but this flag is intended for metrics on when we do not present the
  323. // image when the system claims.
  324. bool may_be_lcp_candidate_ = false;
  325. // An incrementing sequence number maintained by the painter to indicate if
  326. // this animation should be reset in the compositor. Incrementing this number
  327. // will reset this animation in the compositor for the first frame which has a
  328. // recording with a PaintImage storing the updated sequence id.
  329. AnimationSequenceId reset_animation_sequence_id_ = 0u;
  330. DecodingMode decoding_mode_ = DecodingMode::kSync;
  331. // The |cached_sk_image_| can be derived/created from other inputs present in
  332. // the PaintImage but we always construct it at creation time for 2 reasons:
  333. // 1) This ensures that the underlying SkImage is shared across PaintImage
  334. // copies, which is necessary to allow reuse of decodes from this image in
  335. // skia's cache.
  336. // 2) Ensures that accesses to it are thread-safe.
  337. sk_sp<SkImage> cached_sk_image_;
  338. // The input parameters that are needed to execute the JS paint callback.
  339. scoped_refptr<PaintWorkletInput> paint_worklet_input_;
  340. };
  341. } // namespace cc
  342. #endif // CC_PAINT_PAINT_IMAGE_H_