assistant_screen_context_controller_impl.cc 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ash/assistant/assistant_screen_context_controller_impl.h"
  5. #include <utility>
  6. #include <vector>
  7. #include "ash/public/cpp/assistant/controller/assistant_screen_context_controller.h"
  8. #include "ash/public/cpp/shell_window_ids.h"
  9. #include "ash/shell.h"
  10. #include "ash/wm/mru_window_tracker.h"
  11. #include "base/bind.h"
  12. #include "base/containers/contains.h"
  13. #include "base/task/thread_pool.h"
  14. #include "chromeos/ui/base/window_properties.h"
  15. #include "ui/aura/client/aura_constants.h"
  16. #include "ui/compositor/layer.h"
  17. #include "ui/compositor/layer_tree_owner.h"
  18. #include "ui/gfx/codec/jpeg_codec.h"
  19. #include "ui/gfx/image/image.h"
  20. #include "ui/gfx/skbitmap_operations.h"
  21. #include "ui/snapshot/snapshot.h"
  22. #include "ui/snapshot/snapshot_aura.h"
  23. #include "ui/wm/core/focus_controller.h"
  24. #include "ui/wm/core/window_util.h"
  25. namespace ash {
  26. namespace {
  27. // When the screenshot's dimensions are smaller than this size, we will stop
  28. // downsampling.
  29. constexpr int kScreenshotMaxWidth = 1366;
  30. constexpr int kScreenshotMaxHeight = 768;
  31. std::vector<uint8_t> DownsampleAndEncodeImage(gfx::Image image) {
  32. // We'll downsample the screenshot to avoid exceeding max allowed size on
  33. // Assistant server side if we are taking screenshot from high-res screen.
  34. std::vector<uint8_t> res;
  35. gfx::JPEGCodec::Encode(
  36. SkBitmapOperations::DownsampleByTwoUntilSize(
  37. image.AsBitmap(), kScreenshotMaxWidth, kScreenshotMaxHeight),
  38. /*quality=*/100, &res);
  39. return res;
  40. }
  41. void EncodeScreenshotAndRunCallback(
  42. AssistantScreenContextController::RequestScreenshotCallback callback,
  43. std::unique_ptr<ui::LayerTreeOwner> layer_owner,
  44. gfx::Image image) {
  45. base::ThreadPool::PostTaskAndReplyWithResult(
  46. FROM_HERE, {base::MayBlock(), base::TaskPriority::USER_BLOCKING},
  47. base::BindOnce(&DownsampleAndEncodeImage, std::move(image)),
  48. std::move(callback));
  49. }
  50. void MirrorChildren(ui::Layer* to_mirror,
  51. ui::Layer* mirror,
  52. const ::wm::MapLayerFunc& map_func) {
  53. for (auto* child : to_mirror->children()) {
  54. ui::LayerOwner* owner = child->owner();
  55. ui::Layer* child_mirror = owner ? map_func.Run(owner).release() : nullptr;
  56. if (child_mirror) {
  57. mirror->Add(child_mirror);
  58. MirrorChildren(child, child_mirror, map_func);
  59. }
  60. }
  61. }
  62. std::unique_ptr<ui::LayerTreeOwner> MirrorLayersWithClosure(
  63. ui::LayerOwner* root,
  64. const ::wm::MapLayerFunc& map_func) {
  65. DCHECK(root->OwnsLayer());
  66. auto layer = map_func.Run(root);
  67. if (!layer)
  68. return nullptr;
  69. auto mirror = std::make_unique<ui::LayerTreeOwner>(std::move(layer));
  70. MirrorChildren(root->layer(), mirror->root(), map_func);
  71. return mirror;
  72. }
  73. std::unique_ptr<ui::LayerTreeOwner> CreateLayerForAssistantSnapshot(
  74. aura::Window* root_window) {
  75. using LayerSet = base::flat_set<const ui::Layer*>;
  76. LayerSet excluded_layers;
  77. LayerSet blocked_layers;
  78. aura::Window* overlay_container =
  79. ash::Shell::GetContainer(root_window, kShellWindowId_OverlayContainer);
  80. if (overlay_container)
  81. excluded_layers.insert(overlay_container->layer());
  82. aura::Window* always_on_top_container = ash::Shell::GetContainer(
  83. root_window, kShellWindowId_AlwaysOnTopContainer);
  84. // Ignore windows in always on top container. This will prevent assistant
  85. // window from being snapshot.
  86. // TODO(muyuanli): We can add Ash property to indicate specific windows to
  87. // be excluded from snapshot (e.g. assistant window itself).
  88. if (always_on_top_container)
  89. excluded_layers.insert(always_on_top_container->layer());
  90. aura::Window* app_list_container =
  91. ash::Shell::GetContainer(root_window, kShellWindowId_AppListContainer);
  92. aura::Window* app_list_tablet_mode_container =
  93. ash::Shell::GetContainer(root_window, kShellWindowId_HomeScreenContainer);
  94. // Prevent app list from being snapshot on top of other contents.
  95. if (app_list_container)
  96. excluded_layers.insert(app_list_container->layer());
  97. if (app_list_tablet_mode_container)
  98. excluded_layers.insert(app_list_tablet_mode_container->layer());
  99. MruWindowTracker::WindowList windows =
  100. Shell::Get()->mru_window_tracker()->BuildMruWindowList(kActiveDesk);
  101. for (aura::Window* window : windows) {
  102. if (window->GetProperty(chromeos::kBlockedForAssistantSnapshotKey))
  103. blocked_layers.insert(window->layer());
  104. }
  105. return MirrorLayersWithClosure(
  106. root_window,
  107. base::BindRepeating(
  108. [](LayerSet blocked_layers, LayerSet excluded_layers,
  109. ui::LayerOwner* owner) -> std::unique_ptr<ui::Layer> {
  110. // Parent layer is excluded meaning that it's pointless to clone
  111. // current child and all its descendants. This reduces the number
  112. // of layers to create.
  113. if (base::Contains(blocked_layers, owner->layer()->parent()))
  114. return nullptr;
  115. if (base::Contains(blocked_layers, owner->layer())) {
  116. // Blocked layers are replaced with solid black layers so that
  117. // they won't be included in the screenshot but still preserve
  118. // the window stacking.
  119. auto layer =
  120. std::make_unique<ui::Layer>(ui::LayerType::LAYER_SOLID_COLOR);
  121. layer->SetBounds(owner->layer()->bounds());
  122. layer->SetColor(SK_ColorBLACK);
  123. return layer;
  124. }
  125. if (excluded_layers.count(owner->layer()))
  126. return nullptr;
  127. return owner->layer()->Mirror();
  128. },
  129. std::move(blocked_layers), std::move(excluded_layers)));
  130. }
  131. } // namespace
  132. AssistantScreenContextControllerImpl::AssistantScreenContextControllerImpl() =
  133. default;
  134. AssistantScreenContextControllerImpl::~AssistantScreenContextControllerImpl() =
  135. default;
  136. void AssistantScreenContextControllerImpl::RequestScreenshot(
  137. const gfx::Rect& rect,
  138. RequestScreenshotCallback callback) {
  139. aura::Window* root_window = Shell::Get()->GetRootWindowForNewWindows();
  140. std::unique_ptr<ui::LayerTreeOwner> layer_owner =
  141. CreateLayerForAssistantSnapshot(root_window);
  142. ui::Layer* root_layer = layer_owner->root();
  143. gfx::Rect source_rect =
  144. rect.IsEmpty() ? gfx::Rect(root_window->bounds().size()) : rect;
  145. // The root layer might have a scaling transform applied (if the user has
  146. // changed the UI scale via Ctrl-Shift-Plus/Minus). Clear the transform so
  147. // that the snapshot is taken at 1:1 scale relative to screen pixels.
  148. root_layer->SetTransform(gfx::Transform());
  149. root_window->layer()->Add(root_layer);
  150. root_window->layer()->StackAtBottom(root_layer);
  151. ui::GrabLayerSnapshotAsync(
  152. root_layer, source_rect,
  153. base::BindOnce(&EncodeScreenshotAndRunCallback, std::move(callback),
  154. std::move(layer_owner)));
  155. }
  156. std::unique_ptr<ui::LayerTreeOwner>
  157. AssistantScreenContextControllerImpl::CreateLayerForAssistantSnapshotForTest() {
  158. aura::Window* root_window = Shell::GetPrimaryRootWindow();
  159. return CreateLayerForAssistantSnapshot(root_window);
  160. }
  161. } // namespace ash