delegated_ink_point_renderer_gpu.h 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef UI_GL_DELEGATED_INK_POINT_RENDERER_GPU_H_
  5. #define UI_GL_DELEGATED_INK_POINT_RENDERER_GPU_H_
  6. #include <dcomp.h>
  7. #include <wrl/client.h>
  8. #include <algorithm>
  9. #include <iterator>
  10. #include <memory>
  11. #include <utility>
  12. #include <vector>
  13. #include "base/containers/flat_map.h"
  14. #include "base/memory/raw_ptr.h"
  15. #include "base/time/time.h"
  16. #include "base/trace_event/trace_event.h"
  17. #include "mojo/public/cpp/bindings/receiver.h"
  18. #include "third_party/abseil-cpp/absl/types/optional.h"
  19. #include "ui/gfx/delegated_ink_metadata.h"
  20. #include "ui/gfx/mojom/delegated_ink_point_renderer.mojom.h"
  21. // Required for SFINAE to check if these Win10 types exist.
  22. // TODO(1171374): Remove this when the types are available in the Win10 SDK.
  23. struct IDCompositionInkTrailDevice;
  24. struct IDCompositionDelegatedInkTrail;
  25. struct DCompositionInkTrailPoint;
  26. namespace gl {
  27. namespace {
  28. // Maximum number of points that can be drawn. This is used to limit the total
  29. // number of ink trail tokens that we will store, and the total number of points
  30. // that we will store to provide to the Direct Composition APIs. It should match
  31. // the exact number of points that the OS Compositor will store to draw as part
  32. // of a trail.
  33. constexpr int kMaximumNumberOfPoints = 128;
  34. // Maximum number of pointer ids that will be considered for drawing. Number is
  35. // chosen arbitrarily, as it seems unlikely that the total number of input
  36. // sources using delegated ink trails would exceed 10, but it can be raised if
  37. // the need arises.
  38. constexpr uint64_t kMaximumNumberOfPointerIds = 10;
  39. struct DelegatedInkPointCompare {
  40. bool operator()(const gfx::DelegatedInkPoint& lhs,
  41. const gfx::DelegatedInkPoint& rhs) const {
  42. return lhs.timestamp() < rhs.timestamp();
  43. }
  44. };
  45. using DelegatedInkPointTokenMap = base::flat_map<gfx::DelegatedInkPoint,
  46. absl::optional<unsigned int>,
  47. DelegatedInkPointCompare>;
  48. } // namespace
  49. // TODO(1171374): Remove this class and remove templates when the types are
  50. // available in the Win10 SDK.
  51. template <typename InkTrailDevice,
  52. typename DelegatedInkTrail,
  53. typename InkTrailPoint,
  54. typename = void,
  55. typename = void,
  56. typename = void>
  57. class DelegatedInkPointRendererGpu {
  58. public:
  59. DelegatedInkPointRendererGpu() = default;
  60. bool Initialize(
  61. const Microsoft::WRL::ComPtr<IDCompositionDevice2>& dcomp_device,
  62. const Microsoft::WRL::ComPtr<IDXGISwapChain1>& root_swap_chain) {
  63. NOTREACHED();
  64. return false;
  65. }
  66. bool HasBeenInitialized() const { return false; }
  67. IDCompositionVisual* GetInkVisual() const {
  68. NOTREACHED();
  69. return nullptr;
  70. }
  71. bool DelegatedInkIsSupported(
  72. const Microsoft::WRL::ComPtr<IDCompositionDevice2>& dcomp_device) const {
  73. return false;
  74. }
  75. void SetDelegatedInkTrailStartPoint(
  76. std::unique_ptr<gfx::DelegatedInkMetadata> metadata) {
  77. NOTREACHED();
  78. }
  79. void InitMessagePipeline(
  80. mojo::PendingReceiver<gfx::mojom::DelegatedInkPointRenderer> receiver) {
  81. NOTREACHED();
  82. }
  83. void StoreDelegatedInkPoint(const gfx::DelegatedInkPoint& point) {
  84. NOTREACHED();
  85. }
  86. gfx::DelegatedInkMetadata* MetadataForTesting() const {
  87. NOTREACHED();
  88. return nullptr;
  89. }
  90. uint64_t DelegatedInkPointPointerIdCountForTesting() const {
  91. NOTREACHED();
  92. return 0u;
  93. }
  94. DelegatedInkPointTokenMap DelegatedInkPointsForTesting(
  95. int32_t pointer_id) const {
  96. NOTREACHED();
  97. return DelegatedInkPointTokenMap();
  98. }
  99. uint64_t InkTrailTokenCountForTesting() const {
  100. NOTREACHED();
  101. return 0u;
  102. }
  103. bool WaitForNewTrailToDrawForTesting() const {
  104. NOTREACHED();
  105. return false;
  106. }
  107. bool CheckForPointerIdForTesting(int32_t pointer_id) const {
  108. NOTREACHED();
  109. return false;
  110. }
  111. uint64_t GetMaximumNumberOfPointerIdsForTesting() const {
  112. NOTREACHED();
  113. return kMaximumNumberOfPointerIds;
  114. }
  115. void SetNeedsDcompPropertiesUpdate() const { NOTREACHED(); }
  116. };
  117. // This class will call the OS delegated ink trail APIs when they become
  118. // available. Currently using SFINAE to land changes ahead of the SDK, so that
  119. // the OS APIs can be used immediately.
  120. // TODO(1171374): Remove the template SFINAE.
  121. //
  122. // On construction, this class will create a new visual for the visual tree with
  123. // an IDCompositionDelegatedInk object as the contents. This will be added as
  124. // a child of the root surface visual in the tree, and the trail will be drawn
  125. // to it. It is a child of the root surface visual because this visual contains
  126. // the swapchain, and there will be no transforms applied to the delegated ink
  127. // visual this way.
  128. // For more information about the design of this class and using the OS APIs,
  129. // view the design doc here: https://aka.ms/GPUBackedDesignDoc
  130. template <typename InkTrailDevice,
  131. typename DelegatedInkTrail,
  132. typename InkTrailPoint>
  133. class DelegatedInkPointRendererGpu<InkTrailDevice,
  134. DelegatedInkTrail,
  135. InkTrailPoint,
  136. decltype(typeid(InkTrailDevice), void()),
  137. decltype(typeid(DelegatedInkTrail), void()),
  138. decltype(typeid(InkTrailPoint), void())>
  139. : public gfx::mojom::DelegatedInkPointRenderer {
  140. public:
  141. DelegatedInkPointRendererGpu() = default;
  142. void InitMessagePipeline(
  143. mojo::PendingReceiver<gfx::mojom::DelegatedInkPointRenderer>
  144. pending_receiver) {
  145. // The remote end of this pipeline exists on a per-tab basis, so if tab A
  146. // is using the feature and then tab B starts trying to use it, a new
  147. // PendingReceiver will arrive here while |receiver_| is still bound to the
  148. // remote in tab A. Because of this possibility, we unconditionally reset
  149. // |receiver_| before trying to bind anything here. This is fine to do
  150. // because we only ever need to actively receive points from one tab as only
  151. // one tab can be in the foreground and actively inking at a time.
  152. receiver_.reset();
  153. receiver_.Bind(std::move(pending_receiver));
  154. }
  155. bool Initialize(
  156. const Microsoft::WRL::ComPtr<IDCompositionDevice2>& dcomp_device2,
  157. const Microsoft::WRL::ComPtr<IDXGISwapChain1>& root_swap_chain) {
  158. if (dcomp_device_ == dcomp_device2.Get() &&
  159. swap_chain_ == root_swap_chain.Get() && HasBeenInitialized()) {
  160. return true;
  161. }
  162. dcomp_device_ = dcomp_device2.Get();
  163. swap_chain_ = root_swap_chain.Get();
  164. Microsoft::WRL::ComPtr<InkTrailDevice> ink_trail_device;
  165. TraceEventOnFailure(dcomp_device2.As(&ink_trail_device),
  166. "DelegatedInkPointRendererGpu::Initialize - "
  167. "DCompDevice2 as InkTrailDevice failed");
  168. if (!ink_trail_device)
  169. return false;
  170. TraceEventOnFailure(
  171. ink_trail_device->CreateDelegatedInkTrailForSwapChain(
  172. root_swap_chain.Get(), &delegated_ink_trail_),
  173. "DelegatedInkPointRendererGpu::Initialize - Failed to create "
  174. "delegated ink trail.");
  175. if (!delegated_ink_trail_)
  176. return false;
  177. Microsoft::WRL::ComPtr<IDCompositionDevice> dcomp_device;
  178. TraceEventOnFailure(dcomp_device2.As(&dcomp_device),
  179. "DelegatedInkPointRendererGpu::Initialize - "
  180. "DCompDevice2 as DCompDevice failed");
  181. if (!dcomp_device)
  182. return false;
  183. TraceEventOnFailure(dcomp_device->CreateVisual(&ink_visual_),
  184. "DelegatedInkPointRendererGpu::Initialize - "
  185. "Failed to create ink visual.");
  186. if (!ink_visual_)
  187. return false;
  188. if (TraceEventOnFailure(
  189. ink_visual_->SetContent(delegated_ink_trail_.Get()),
  190. "DelegatedInkPointRendererGpu::Initialize - SetContent failed")) {
  191. // Initialization has failed because SetContent failed. However, we must
  192. // reset the members so that HasBeenInitialized() does not return true
  193. // when queried.
  194. ink_visual_.Reset();
  195. delegated_ink_trail_.Reset();
  196. return false;
  197. }
  198. return true;
  199. }
  200. bool HasBeenInitialized() const {
  201. return ink_visual_ && delegated_ink_trail_;
  202. }
  203. IDCompositionVisual* GetInkVisual() const { return ink_visual_.Get(); }
  204. bool DelegatedInkIsSupported(
  205. const Microsoft::WRL::ComPtr<IDCompositionDevice2>& dcomp_device) const {
  206. Microsoft::WRL::ComPtr<InkTrailDevice> ink_trail_device;
  207. HRESULT hr = dcomp_device.As(&ink_trail_device);
  208. return hr == S_OK;
  209. }
  210. void SetDelegatedInkTrailStartPoint(
  211. std::unique_ptr<gfx::DelegatedInkMetadata> metadata) {
  212. TRACE_EVENT_WITH_FLOW1(
  213. "delegated_ink_trails",
  214. "DelegatedInkPointRendererGpu::SetDelegatedInkTrailStartPoint",
  215. TRACE_ID_GLOBAL(metadata_->trace_id()), TRACE_EVENT_FLAG_FLOW_IN,
  216. "metadata", metadata_->ToString());
  217. DCHECK(ink_visual_);
  218. DCHECK(delegated_ink_trail_);
  219. // If certain conditions are met, we can simply erase the trail up to the
  220. // point matching |metadata|, instead of starting a new trail. These
  221. // conditions include:
  222. // 1. A trail has already been started, meaning that |metadata_| and
  223. // |pointer_id_| exist.
  224. // 2. The current |metadata_| and |metadata| both have the same color, so
  225. // the color of the trail does not need to change.
  226. // 3. |needs_dcomp_properties_update_| isn't forcing an update of the DCOMP
  227. // resources: |ink_visual_|, |delegated_ink_trail_|. This happens after
  228. // a swap chain recreation.
  229. // (continued below)
  230. if (!needs_dcomp_properties_update_ && metadata_ &&
  231. metadata->color() == metadata_->color() && pointer_id_) {
  232. DelegatedInkPointTokenMap& token_map =
  233. delegated_ink_points_[pointer_id_.value()];
  234. auto point_matching_metadata_it = token_map.find(
  235. gfx::DelegatedInkPoint(metadata->point(), metadata->timestamp()));
  236. // (continued from above)
  237. // 4. We have a DelegatedInkPoint with a timestamp equal to or greater
  238. // than |metadata|'s timestamp in the token map associated with
  239. // |pointer_id_|.
  240. // (continued below)
  241. if (point_matching_metadata_it != token_map.end()) {
  242. gfx::DelegatedInkPoint point_matching_metadata =
  243. point_matching_metadata_it->first;
  244. absl::optional<unsigned int> token = point_matching_metadata_it->second;
  245. // (continued from above)
  246. // 5. The DelegatedInkPoint retrieved above matches |metadata| - this
  247. // means that the timestamp is an exact match and the point is
  248. // acceptably close.
  249. // 6. The token associated with this DelegatedInkPoint is valid,
  250. // meaning that the point has previously been drawn as part of a
  251. // trail.
  252. //
  253. // If all of these conditions are met, then we can attempt to just
  254. // remove points from the trail instead of starting a new trail
  255. // altogether.
  256. if (point_matching_metadata.MatchesDelegatedInkMetadata(
  257. metadata.get()) &&
  258. token) {
  259. bool remove_trail_points_failed = TraceEventOnFailure(
  260. delegated_ink_trail_->RemoveTrailPoints(token.value()),
  261. "DelegatedInkPointRendererGpu::SetDelegatedInkTrailStartPoint - "
  262. "Failed to remove trail points.");
  263. if (!remove_trail_points_failed &&
  264. UpdateVisualClip(metadata->presentation_area(), false)) {
  265. // Remove all points up to and including the point that matches
  266. // |metadata|. No need to hold on to the point that matches metadata
  267. // because we've already added it to AddTrailPoints previously, and
  268. // the next valid |metadata| is guaranteed to be after it.
  269. token_map.erase(token_map.begin(),
  270. std::next(point_matching_metadata_it));
  271. metadata_ = std::move(metadata);
  272. return;
  273. }
  274. }
  275. }
  276. }
  277. if (!UpdateVisualClip(metadata->presentation_area(),
  278. needs_dcomp_properties_update_))
  279. return;
  280. D2D1_COLOR_F d2d1_color;
  281. const float kMaxRGBAValue = 255.f;
  282. d2d1_color.a = SkColorGetA(metadata->color()) / kMaxRGBAValue;
  283. d2d1_color.r = SkColorGetR(metadata->color()) / kMaxRGBAValue;
  284. d2d1_color.g = SkColorGetG(metadata->color()) / kMaxRGBAValue;
  285. d2d1_color.b = SkColorGetB(metadata->color()) / kMaxRGBAValue;
  286. if (TraceEventOnFailure(
  287. delegated_ink_trail_->StartNewTrail(d2d1_color),
  288. "DelegatedInkPointRendererGpu::SetDelegatedInkTrailStartPoint - "
  289. "Failed to start a new trail.")) {
  290. return;
  291. }
  292. wait_for_new_trail_to_draw_ = false;
  293. metadata_ = std::move(metadata);
  294. DrawSavedTrailPoints();
  295. needs_dcomp_properties_update_ = false;
  296. }
  297. void StoreDelegatedInkPoint(const gfx::DelegatedInkPoint& point) override {
  298. TRACE_EVENT_WITH_FLOW1(
  299. "delegated_ink_trails",
  300. "DelegatedInkPointRendererGpu::StoreDelegatedInkPoint",
  301. TRACE_ID_GLOBAL(point.trace_id()),
  302. TRACE_EVENT_FLAG_FLOW_IN | TRACE_EVENT_FLAG_FLOW_OUT, "point",
  303. point.ToString());
  304. const int32_t pointer_id = point.pointer_id();
  305. // TODO(1238497): Understand why we are being sent points from browser
  306. // process that break this assertion so frequently and prevent it from
  307. // happening.
  308. // DCHECK(delegated_ink_points_.find(pointer_id) ==
  309. // delegated_ink_points_.end() ||
  310. // point.timestamp() >
  311. // delegated_ink_points_[pointer_id].rbegin()->
  312. // first.timestamp());
  313. if (metadata_ && point.timestamp() < metadata_->timestamp())
  314. return;
  315. DelegatedInkPointTokenMap& token_map = delegated_ink_points_[pointer_id];
  316. // Always save the point so that it can be drawn later. This allows points
  317. // that arrive before the first metadata makes it here to be drawn after
  318. // StartNewTrail is called. If we get to the maximum number of points that
  319. // we will store, start erasing the oldest ones first. This matches what the
  320. // OS compositor does internally when it hits the max number of points.
  321. if (token_map.size() == kMaximumNumberOfPoints)
  322. token_map.erase(token_map.begin());
  323. token_map.insert({point, absl::nullopt});
  324. EraseExcessPointerIds();
  325. if (pointer_id_ && pointer_id_.value() == pointer_id) {
  326. DrawDelegatedInkPoint(point);
  327. } else if (!pointer_id_ && metadata_) {
  328. DrawSavedTrailPoints();
  329. }
  330. }
  331. void ResetPrediction() override {
  332. // Don't reset |metadata_| here so that RemoveTrailPoints() can continue
  333. // to be called as the final metadata(s) arrive.
  334. // TODO(1052145): Start predicting points and reset it here.
  335. wait_for_new_trail_to_draw_ = true;
  336. }
  337. gfx::DelegatedInkMetadata* MetadataForTesting() const {
  338. return metadata_.get();
  339. }
  340. uint64_t InkTrailTokenCountForTesting() const {
  341. DCHECK_EQ(delegated_ink_points_.size(), 1u);
  342. uint64_t valid_tokens = 0u;
  343. for (const auto& it : delegated_ink_points_.begin()->second) {
  344. if (it.second)
  345. valid_tokens++;
  346. }
  347. return valid_tokens;
  348. }
  349. uint64_t DelegatedInkPointPointerIdCountForTesting() const {
  350. return delegated_ink_points_.size();
  351. }
  352. bool CheckForPointerIdForTesting(int32_t pointer_id) const {
  353. return delegated_ink_points_.find(pointer_id) !=
  354. delegated_ink_points_.end();
  355. }
  356. const DelegatedInkPointTokenMap& DelegatedInkPointsForTesting(
  357. int32_t pointer_id) {
  358. DCHECK(delegated_ink_points_.find(pointer_id) !=
  359. delegated_ink_points_.end());
  360. return delegated_ink_points_[pointer_id];
  361. }
  362. bool WaitForNewTrailToDrawForTesting() const {
  363. return wait_for_new_trail_to_draw_;
  364. }
  365. uint64_t GetMaximumNumberOfPointerIdsForTesting() const {
  366. return kMaximumNumberOfPointerIds;
  367. }
  368. void SetNeedsDcompPropertiesUpdate() {
  369. // This should be set from an external event that invalidates our DCOMP
  370. // resources: |ink_visual_|, |delegated_ink_trail_|. This will be checked in
  371. // the next call to |SetDelegatedInkTrailStartPoint| - the entry point for
  372. // using these resources to render a new trail. That code optimizes based on
  373. // the consideration that properties persist after being set.
  374. needs_dcomp_properties_update_ = true;
  375. }
  376. private:
  377. // Note that this returns true if the HRESULT is anything other than S_OK,
  378. // meaning that it returns true when an event is traced (because of a
  379. // failure).
  380. static bool TraceEventOnFailure(HRESULT hr, const char* name) {
  381. if (SUCCEEDED(hr))
  382. return false;
  383. TRACE_EVENT_INSTANT1("delegated_ink_trails", name, TRACE_EVENT_SCOPE_THREAD,
  384. "hr", hr);
  385. return true;
  386. }
  387. bool UpdateVisualClip(const gfx::RectF& new_presentation_area,
  388. bool force_update) {
  389. if (!force_update && metadata_ &&
  390. metadata_->presentation_area() == new_presentation_area)
  391. return true;
  392. // If (0,0) of a visual is clipped out, it can result in delegated ink not
  393. // being drawn at all. This is more common when DComp Surfaces are enabled,
  394. // but doesn't negatively impact things when the swapchain is used, so just
  395. // offset the visual instead of clipping the top left corner in all cases.
  396. ink_visual_->SetOffsetX(new_presentation_area.x());
  397. ink_visual_->SetOffsetY(new_presentation_area.y());
  398. D2D_RECT_F clip_rect;
  399. clip_rect.bottom = new_presentation_area.bottom();
  400. clip_rect.right = new_presentation_area.right();
  401. // If setting the clip or committing failed, we want to bail early so that a
  402. // trail can't incorrectly appear over things on the user's screen.
  403. return SUCCEEDED(ink_visual_->SetClip(clip_rect)) &&
  404. SUCCEEDED(dcomp_device_->Commit());
  405. }
  406. void EraseExcessPointerIds() {
  407. auto token_map_it = delegated_ink_points_.begin();
  408. while (token_map_it != delegated_ink_points_.end()) {
  409. const DelegatedInkPointTokenMap& token_map = token_map_it->second;
  410. if (token_map.empty())
  411. token_map_it = delegated_ink_points_.erase(token_map_it);
  412. else
  413. token_map_it++;
  414. }
  415. // In order to get to the maximum pointer id limit, sometimes one token map
  416. // will need to be removed even if it has a valid DelegatedInkPoint. In this
  417. // case, we'll remove the token map that has gone the longest without
  418. // receiving a new point - similar to a least recently used eviction policy.
  419. // This would also mean that the token map that would result in the smallest
  420. // latency improvement if it were drawn is being removed. The only exception
  421. // to this is if the above heuristic would result in the token map that has
  422. // a pointer id matching |pointer_id_| being removed. Since |pointer_id_|
  423. // matches the trail that is currently being drawn to the screen, we prefer
  424. // to save it so we can try to continue that trail. In this case, just
  425. // remove the token map with the oldest new point that is not currently
  426. // being drawn.
  427. if (delegated_ink_points_.size() > kMaximumNumberOfPointerIds) {
  428. std::vector<std::pair<base::TimeTicks, int32_t>>
  429. earliest_time_and_pointer_id;
  430. for (const auto& it : delegated_ink_points_) {
  431. earliest_time_and_pointer_id.emplace_back(
  432. it.second.rbegin()->first.timestamp(), it.first);
  433. }
  434. std::sort(earliest_time_and_pointer_id.begin(),
  435. earliest_time_and_pointer_id.end());
  436. uint64_t pointer_id_to_remove = 0;
  437. while (pointer_id_to_remove < earliest_time_and_pointer_id.size() &&
  438. pointer_id_ &&
  439. earliest_time_and_pointer_id[pointer_id_to_remove].second ==
  440. pointer_id_.value()) {
  441. pointer_id_to_remove++;
  442. }
  443. CHECK_LT(pointer_id_to_remove, earliest_time_and_pointer_id.size());
  444. delegated_ink_points_.erase(
  445. earliest_time_and_pointer_id[pointer_id_to_remove].second);
  446. }
  447. DCHECK_LE(delegated_ink_points_.size(), kMaximumNumberOfPointerIds);
  448. }
  449. absl::optional<int32_t> GetPointerIdForMetadata() {
  450. if (pointer_id_ && delegated_ink_points_.find(pointer_id_.value()) !=
  451. delegated_ink_points_.end()) {
  452. // Since we remove all DelegatedInkPoints with timestamp before
  453. // |metadata_|'s before calling GetPointerIdForMetadata(), we know we can
  454. // just grab the first DelegatedInkPoint matching |pointer_id_|.
  455. const gfx::DelegatedInkPoint& point_matching_metadata =
  456. delegated_ink_points_[pointer_id_.value()].begin()->first;
  457. if (point_matching_metadata.MatchesDelegatedInkMetadata(
  458. metadata_.get())) {
  459. return pointer_id_;
  460. }
  461. }
  462. pointer_id_ = absl::nullopt;
  463. for (auto token_map_it = delegated_ink_points_.begin();
  464. token_map_it != delegated_ink_points_.end() && !pointer_id_;
  465. ++token_map_it) {
  466. int32_t potential_pointer_id = token_map_it->first;
  467. const DelegatedInkPointTokenMap& token_map = token_map_it->second;
  468. DCHECK(!token_map.empty());
  469. if (token_map.begin()->first.MatchesDelegatedInkMetadata(
  470. metadata_.get())) {
  471. DCHECK(!pointer_id_);
  472. pointer_id_ = potential_pointer_id;
  473. }
  474. }
  475. return pointer_id_;
  476. }
  477. void DrawSavedTrailPoints() {
  478. DCHECK(metadata_);
  479. TRACE_EVENT0("delegated_ink_trails", "DrawSavedTrailPoints");
  480. // Remove all points that have a timestamp earlier than |metadata_|'s, since
  481. // we know that we won't need to draw them. This is subtly different than
  482. // the erasing done in SetDelegatedInkTrailStartPoint() - there the point
  483. // matching the metadata is removed, here it is not. The reason for this
  484. // difference is because there we know that it matches |metadata_| and
  485. // therefore it cannot possibly be drawn again, so it is safe to remove.
  486. // Here however, we are erasing all points up to, but not including, the
  487. // first point with a timestamp equal to or greater than |metadata_|'s so
  488. // that we can then check that point to confirm that it matches |metadata_|
  489. // before deciding to draw an ink trail. The point could then be erased
  490. // after it is successfully checked against |metadata_| and drawn, it just
  491. // isn't for simplicity's sake.
  492. for (auto& it : delegated_ink_points_) {
  493. DelegatedInkPointTokenMap& token_map = it.second;
  494. token_map.erase(token_map.begin(),
  495. token_map.lower_bound(gfx::DelegatedInkPoint(
  496. metadata_->point(), metadata_->timestamp())));
  497. }
  498. EraseExcessPointerIds();
  499. absl::optional<unsigned int> pointer_id = GetPointerIdForMetadata();
  500. // Now, the very first point must match |metadata_|, and as long as it does
  501. // we can continue to draw everything else. If at any point something can't
  502. // or fails to draw though, don't attempt to draw anything after it so that
  503. // the trail can match the user's actual stroke.
  504. if (pointer_id && delegated_ink_points_.find(pointer_id.value()) !=
  505. delegated_ink_points_.end()) {
  506. DelegatedInkPointTokenMap& token_map =
  507. delegated_ink_points_[pointer_id.value()];
  508. if (!token_map.empty() &&
  509. token_map.begin()->first.MatchesDelegatedInkMetadata(
  510. metadata_.get())) {
  511. for (const auto& it : token_map) {
  512. if (!DrawDelegatedInkPoint(it.first))
  513. break;
  514. }
  515. }
  516. } else {
  517. TRACE_EVENT_INSTANT0("delegated_ink_trails",
  518. "DrawSavedTrailPoints failed - no pointer id",
  519. TRACE_EVENT_SCOPE_THREAD);
  520. }
  521. }
  522. bool DrawDelegatedInkPoint(const gfx::DelegatedInkPoint& point) {
  523. // Always wait for a new trail to be started before attempting to draw
  524. // anything, even if |metadata_| exists.
  525. if (wait_for_new_trail_to_draw_)
  526. return false;
  527. DCHECK(metadata_);
  528. if (!metadata_->presentation_area().Contains(point.point()))
  529. return false;
  530. DCHECK(delegated_ink_trail_);
  531. InkTrailPoint ink_point;
  532. ink_point.radius = metadata_->diameter() / 2.f;
  533. // In order to account for the visual offset, the point must be offset in
  534. // the opposite direction.
  535. ink_point.x = point.point().x() - metadata_->presentation_area().x();
  536. ink_point.y = point.point().y() - metadata_->presentation_area().y();
  537. unsigned int token;
  538. // AddTrailPoints() can accept and draw more than one InkTrailPoint per
  539. // call. However, all the points get lumped together in the one token then,
  540. // which means that they can only be removed all together. This may be fine
  541. // in some scenarios, but in the vast majority of cases we will need to
  542. // remove one point at a time, so we choose to only add one InkTrailPoint at
  543. // a time.
  544. if (TraceEventOnFailure(
  545. delegated_ink_trail_->AddTrailPoints(&ink_point,
  546. /*inkPointsCount*/ 1, &token),
  547. "DelegatedInkPointRendererGpu::DrawDelegatedInkPoint - Failed to "
  548. "add trail point")) {
  549. // TODO(1052145): Start predicting points.
  550. return false;
  551. }
  552. TRACE_EVENT_WITH_FLOW1(
  553. "delegated_ink_trails",
  554. "DelegatedInkPointRendererGpu::DrawDelegatedInkPoint "
  555. "- Point added to trail",
  556. TRACE_ID_GLOBAL(point.trace_id()), TRACE_EVENT_FLAG_FLOW_IN, "point",
  557. point.ToString());
  558. delegated_ink_points_[point.pointer_id()][point] = token;
  559. return true;
  560. }
  561. // The visual within the tree that will contain the delegated ink trail. It
  562. // should be a child of the root surface visual.
  563. Microsoft::WRL::ComPtr<IDCompositionVisual> ink_visual_;
  564. // The delegated ink trail object that the ink trail is drawn on. This is the
  565. // content of the ink visual.
  566. Microsoft::WRL::ComPtr<DelegatedInkTrail> delegated_ink_trail_;
  567. // Remember the dcomp device and swap chain used to create
  568. // |delegated_ink_trail_| and |ink_visual_| so that we can avoid recreating
  569. // them when it isn't necessary.
  570. raw_ptr<IDCompositionDevice2> dcomp_device_ = nullptr;
  571. raw_ptr<IDXGISwapChain1> swap_chain_ = nullptr;
  572. // The most recent metadata received. The metadata marks the last point of
  573. // the app rendered stroke, which corresponds to the first point of the
  574. // delegated ink trail that will be drawn.
  575. std::unique_ptr<gfx::DelegatedInkMetadata> metadata_;
  576. // A base::flat_map of all the points that have arrived in
  577. // StoreDelegatedInkPoint() with a timestamp greater than or equal to that of
  578. // |metadata_|, where the key is the DelegatedInkPoint that was received, and
  579. // the value is an optional token. The value will be null until the
  580. // DelegatedInkPoint is added to the trail, at which point the value is the
  581. // token that was returned by AddTrailPoints. The elements of the flat_map are
  582. // sorted by the timestamp of the DelegatedInkPoint. All points are stored in
  583. // here until we receive a |metadata_|, then any DelegatedInkPoints that have
  584. // a timestamp earlier than |metadata_|'s are removed, and any new
  585. // DelegatedInkPoints that may arrive with an earlier timestamp are ignored.
  586. // Then as each new |metadata| arrives in SetDelegatedInkTrailStartPoint(),
  587. // we remove any old elements with earlier timestamps, up to and including the
  588. // element that matches the DelegatedInkMetadata.
  589. base::flat_map<int32_t, DelegatedInkPointTokenMap> delegated_ink_points_;
  590. // Cached pointer id of the most recently drawn trail.
  591. absl::optional<int32_t> pointer_id_;
  592. // Flag to know if new DelegatedInkPoints that arrive should be drawn
  593. // immediately or if they should wait for a new trail to be started. Set to
  594. // true when ResetPrediction is called, as this tells us that something about
  595. // the pointerevents in the browser process changed. When that happens, we
  596. // should continue to remove points for any metadata that arrive with
  597. // timestamps that match DelegatedInkPoints that arrived before the
  598. // ResetPrediction call. However, once a metadata arrives with a timestamp
  599. // after the ResetPrediction, then we know that inking should continue, so a
  600. // new trail is started and we try to draw everything in
  601. // |delegated_ink_points_|.
  602. bool wait_for_new_trail_to_draw_ = true;
  603. // When the visual tree was updated, all properties we've set on DCOMP are
  604. // outdated, and need to be re-set (i.e. from |ink_visual_| and
  605. // |delegated_ink_trail_|). This is done at |SetDelegatedInkTrailStartPoint|.
  606. bool needs_dcomp_properties_update_ = false;
  607. mojo::Receiver<gfx::mojom::DelegatedInkPointRenderer> receiver_{this};
  608. };
  609. } // namespace gl
  610. #endif // UI_GL_DELEGATED_INK_POINT_RENDERER_GPU_H_