keyboard.cc 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/exo/keyboard.h"
  5. #include "ash/accelerators/accelerator_table.h"
  6. #include "ash/constants/app_types.h"
  7. #include "ash/constants/ash_features.h"
  8. #include "ash/keyboard/ui/keyboard_ui_controller.h"
  9. #include "ash/keyboard/ui/keyboard_util.h"
  10. #include "ash/public/cpp/accelerators.h"
  11. #include "ash/public/cpp/keyboard/keyboard_controller.h"
  12. #include "ash/shell.h"
  13. #include "base/bind.h"
  14. #include "base/containers/contains.h"
  15. #include "base/containers/span.h"
  16. #include "base/no_destructor.h"
  17. #include "base/threading/thread_task_runner_handle.h"
  18. #include "base/trace_event/trace_event.h"
  19. #include "components/exo/input_trace.h"
  20. #include "components/exo/keyboard_delegate.h"
  21. #include "components/exo/keyboard_device_configuration_delegate.h"
  22. #include "components/exo/keyboard_modifiers.h"
  23. #include "components/exo/seat.h"
  24. #include "components/exo/shell_surface.h"
  25. #include "components/exo/shell_surface_util.h"
  26. #include "components/exo/surface.h"
  27. #include "components/exo/xkb_tracker.h"
  28. #include "ui/aura/client/aura_constants.h"
  29. #include "ui/aura/client/focus_client.h"
  30. #include "ui/aura/window.h"
  31. #include "ui/base/ime/input_method.h"
  32. #include "ui/events/base_event_utils.h"
  33. #include "ui/events/event.h"
  34. #include "ui/views/widget/widget.h"
  35. #include "ui/wm/core/window_util.h"
  36. namespace exo {
  37. namespace {
  38. // This value must be bigger than the priority for DataDevice.
  39. constexpr int kKeyboardSeatObserverPriority = 1;
  40. static_assert(Seat::IsValidObserverPriority(kKeyboardSeatObserverPriority),
  41. "kKeyboardSeatObserverPriority is not in the valid range.");
  42. // Delay until a key state change expected to be acknowledged is expired.
  43. constexpr int kExpirationDelayForPendingKeyAcksMs = 1000;
  44. // The accelerator keys reserved to be processed by chrome.
  45. constexpr struct {
  46. ui::KeyboardCode keycode;
  47. int modifiers;
  48. } kReservedAccelerators[] = {
  49. {ui::VKEY_F13, ui::EF_NONE},
  50. {ui::VKEY_I, ui::EF_SHIFT_DOWN | ui::EF_ALT_DOWN},
  51. {ui::VKEY_Z, ui::EF_CONTROL_DOWN | ui::EF_ALT_DOWN}};
  52. bool ProcessAccelerator(Surface* surface, const ui::KeyEvent* event) {
  53. views::Widget* widget =
  54. views::Widget::GetTopLevelWidgetForNativeView(surface->window());
  55. if (widget) {
  56. views::FocusManager* focus_manager = widget->GetFocusManager();
  57. return focus_manager->ProcessAccelerator(ui::Accelerator(*event));
  58. }
  59. return false;
  60. }
  61. bool IsVirtualKeyboardEnabled() {
  62. return keyboard::GetAccessibilityKeyboardEnabled() ||
  63. keyboard::GetTouchKeyboardEnabled() ||
  64. (keyboard::KeyboardUIController::HasInstance() &&
  65. keyboard::KeyboardUIController::Get()->IsEnableFlagSet(
  66. keyboard::KeyboardEnableFlag::kCommandLineEnabled));
  67. }
  68. bool IsReservedAccelerator(const ui::KeyEvent* event) {
  69. for (const auto& accelerator : kReservedAccelerators) {
  70. if (event->flags() == accelerator.modifiers &&
  71. event->key_code() == accelerator.keycode) {
  72. return true;
  73. }
  74. }
  75. return false;
  76. }
  77. // Returns false if an accelerator is not reserved or it's not enabled.
  78. bool ProcessAcceleratorIfReserved(Surface* surface, ui::KeyEvent* event) {
  79. return IsReservedAccelerator(event) && ProcessAccelerator(surface, event);
  80. }
  81. // Returns true if the surface needs to support IME.
  82. // TODO(yhanada, https://crbug.com/847500): Remove this when we find a way
  83. // to fix https://crbug.com/847500 without breaking ARC apps/Lacros browser.
  84. bool IsImeSupportedSurface(Surface* surface) {
  85. aura::Window* window = surface->window();
  86. while (window) {
  87. const auto app_type =
  88. static_cast<ash::AppType>(window->GetProperty(aura::client::kAppType));
  89. switch (app_type) {
  90. case ash::AppType::ARC_APP:
  91. case ash::AppType::LACROS:
  92. return true;
  93. case ash::AppType::CROSTINI_APP:
  94. return base::FeatureList::IsEnabled(ash::features::kCrostiniImeSupport);
  95. default:
  96. // Do nothing.
  97. break;
  98. }
  99. // For notifications, billing surfaces, etc. AppType::ARC_APP is not set
  100. // despite them being from ARC. Ideally AppType should be added to them, but
  101. // there is a risk that breaks other features e.g. full restore.
  102. // TODO(tetsui): find a way to remove this.
  103. if (window->GetProperty(aura::client::kSkipImeProcessing))
  104. return true;
  105. if (aura::Window* transient_parent = wm::GetTransientParent(window)) {
  106. window = transient_parent;
  107. } else {
  108. window = window->parent();
  109. }
  110. }
  111. return false;
  112. }
  113. // Returns true if the surface can consume ash accelerators.
  114. bool CanConsumeAshAccelerators(Surface* surface) {
  115. aura::Window* window = surface->window();
  116. for (; window; window = window->parent()) {
  117. const auto app_type =
  118. static_cast<ash::AppType>(window->GetProperty(aura::client::kAppType));
  119. // TOOD(hidehiko): get rid of this if check, after introducing capability,
  120. // followed by ARC/Crostini migration.
  121. if (app_type == ash::AppType::LACROS)
  122. return surface->is_keyboard_shortcuts_inhibited();
  123. }
  124. return true;
  125. }
  126. // Returns true if an accelerator is an ash accelerator which can be handled
  127. // before sending it to client and it is actually processed by ash-chrome.
  128. bool ProcessAshAcceleratorIfPossible(Surface* surface, ui::KeyEvent* event) {
  129. // Process ash accelerators before sending it to client only when the client
  130. // should not consume ash accelerators. (e.g. Lacros-chrome)
  131. if (CanConsumeAshAccelerators(surface))
  132. return false;
  133. // If accelerators can be processed by browser, send it to the app.
  134. static const base::NoDestructor<std::vector<ui::Accelerator>>
  135. kAppHandlingAccelerators([] {
  136. std::vector<ui::Accelerator> result;
  137. for (size_t i = 0; i < ash::kAcceleratorDataLength; ++i) {
  138. const auto& ash_entry = ash::kAcceleratorData[i];
  139. if (base::Contains(base::span<const ash::AcceleratorAction>(
  140. ash::kActionsInterceptableByBrowser,
  141. ash::kActionsInterceptableByBrowserLength),
  142. ash_entry.action) ||
  143. base::Contains(base::span<const ash::AcceleratorAction>(
  144. ash::kActionsDuplicatedWithBrowser,
  145. ash::kActionsDuplicatedWithBrowserLength),
  146. ash_entry.action)) {
  147. result.emplace_back(ash_entry.keycode, ash_entry.modifiers);
  148. }
  149. }
  150. return result;
  151. }());
  152. ui::Accelerator accelerator(*event);
  153. if (base::Contains(*kAppHandlingAccelerators, accelerator))
  154. return false;
  155. return ash::AcceleratorController::Get()->Process(accelerator);
  156. }
  157. } // namespace
  158. ////////////////////////////////////////////////////////////////////////////////
  159. // Keyboard, public:
  160. Keyboard::Keyboard(std::unique_ptr<KeyboardDelegate> delegate, Seat* seat)
  161. : delegate_(std::move(delegate)),
  162. seat_(seat),
  163. expiration_delay_for_pending_key_acks_(
  164. base::Milliseconds(kExpirationDelayForPendingKeyAcksMs)) {
  165. seat_->AddObserver(this, kKeyboardSeatObserverPriority);
  166. ash::KeyboardController::Get()->AddObserver(this);
  167. ash::ImeControllerImpl* ime_controller = ash::Shell::Get()->ime_controller();
  168. ime_controller->AddObserver(this);
  169. delegate_->OnKeyboardLayoutUpdated(seat_->xkb_tracker()->GetKeymap().get());
  170. OnSurfaceFocused(seat_->GetFocusedSurface(), nullptr,
  171. !!seat_->GetFocusedSurface());
  172. OnKeyRepeatSettingsChanged(
  173. ash::KeyboardController::Get()->GetKeyRepeatSettings());
  174. }
  175. Keyboard::~Keyboard() {
  176. RemoveEventHandler();
  177. for (KeyboardObserver& observer : observer_list_)
  178. observer.OnKeyboardDestroying(this);
  179. if (focus_)
  180. focus_->RemoveSurfaceObserver(this);
  181. ash::Shell::Get()->ime_controller()->RemoveObserver(this);
  182. ash::KeyboardController::Get()->RemoveObserver(this);
  183. seat_->RemoveObserver(this);
  184. }
  185. bool Keyboard::HasDeviceConfigurationDelegate() const {
  186. return !!device_configuration_delegate_;
  187. }
  188. void Keyboard::SetDeviceConfigurationDelegate(
  189. KeyboardDeviceConfigurationDelegate* delegate) {
  190. device_configuration_delegate_ = delegate;
  191. UpdateKeyboardType();
  192. }
  193. void Keyboard::AddObserver(KeyboardObserver* observer) {
  194. observer_list_.AddObserver(observer);
  195. }
  196. bool Keyboard::HasObserver(KeyboardObserver* observer) const {
  197. return observer_list_.HasObserver(observer);
  198. }
  199. void Keyboard::RemoveObserver(KeyboardObserver* observer) {
  200. observer_list_.RemoveObserver(observer);
  201. }
  202. void Keyboard::SetNeedKeyboardKeyAcks(bool need_acks) {
  203. RemoveEventHandler();
  204. are_keyboard_key_acks_needed_ = need_acks;
  205. AddEventHandler();
  206. }
  207. bool Keyboard::AreKeyboardKeyAcksNeeded() const {
  208. // Keyboard class doesn't need key acks while the spoken feedback is enabled.
  209. // While the spoken feedback is enabled, a key event is sent to both of a
  210. // wayland client and Chrome to give a chance to work to Chrome OS's
  211. // shortcuts.
  212. return are_keyboard_key_acks_needed_;
  213. }
  214. void Keyboard::AckKeyboardKey(uint32_t serial, bool handled) {
  215. auto it = pending_key_acks_.find(serial);
  216. if (it == pending_key_acks_.end())
  217. return;
  218. if (!handled && focus_)
  219. ProcessAccelerator(focus_, &it->second.first);
  220. pending_key_acks_.erase(serial);
  221. }
  222. ////////////////////////////////////////////////////////////////////////////////
  223. // ui::EventHandler overrides:
  224. void Keyboard::OnKeyEvent(ui::KeyEvent* event) {
  225. if (!focus_ || seat_->was_shutdown())
  226. return;
  227. DCHECK(GetShellRootSurface(static_cast<aura::Window*>(event->target())) ||
  228. Surface::AsSurface(static_cast<aura::Window*>(event->target())));
  229. // Ignore synthetic key repeat events.
  230. if (event->is_repeat()) {
  231. // Clients should not see key repeat events and instead handle them on the
  232. // client side.
  233. // Mark the key repeat events as handled to avoid them from invoking
  234. // accelerators.
  235. event->SetHandled();
  236. return;
  237. }
  238. TRACE_EXO_INPUT_EVENT(event);
  239. // Process reserved accelerators or ash accelerators which need to be handled
  240. // before sending it to client.
  241. if (ProcessAcceleratorIfReserved(focus_, event) ||
  242. ProcessAshAcceleratorIfPossible(focus_, event)) {
  243. // Discard a key press event if the corresponding accelerator is handled.
  244. event->SetHandled();
  245. // The current focus might have been reset while processing accelerators.
  246. if (!focus_)
  247. return;
  248. }
  249. // When IME ate a key event, we use the event only for tracking key states and
  250. // ignore for further processing. Otherwise it is handled in two places (IME
  251. // and client) and causes undesired behavior.
  252. // If the window should receive a key event before IME, Exo should send any
  253. // key events to a client. The client will send back the events to IME if
  254. // needed.
  255. const bool consumed_by_ime =
  256. !focus_->window()->GetProperty(aura::client::kSkipImeProcessing) &&
  257. ConsumedByIme(focus_->window(), *event);
  258. // Currently, physical keycode is tracked in Seat, assuming that the
  259. // Keyboard::OnKeyEvent is called between Seat::WillProcessEvent and
  260. // Seat::DidProcessEvent. However, if IME is enabled, it is no longer true,
  261. // because IME work in async approach, and on its dispatching, call stack
  262. // is split so actually Keyboard::OnKeyEvent is called after
  263. // Seat::DidProcessEvent.
  264. // TODO(yhanada): This is a quick fix for https://crbug.com/859071. Remove
  265. // ARC-/Lacros-specific code path once we can find a way to manage
  266. // press/release events pair for synthetic events.
  267. ui::DomCode physical_code =
  268. seat_->physical_code_for_currently_processing_event();
  269. if (physical_code == ui::DomCode::NONE && focused_on_ime_supported_surface_) {
  270. // This key event is a synthetic event.
  271. // Consider DomCode field of the event as a physical code
  272. // for synthetic events when focus surface belongs to an ARC application.
  273. physical_code = event->code();
  274. }
  275. switch (event->type()) {
  276. case ui::ET_KEY_PRESSED: {
  277. auto it = pressed_keys_.find(physical_code);
  278. if (it == pressed_keys_.end() && !event->handled() &&
  279. physical_code != ui::DomCode::NONE) {
  280. for (auto& observer : observer_list_)
  281. observer.OnKeyboardKey(event->time_stamp(), event->code(), true);
  282. if (!consumed_by_ime) {
  283. // Process key press event if not already handled and not already
  284. // pressed.
  285. uint32_t serial = delegate_->OnKeyboardKey(event->time_stamp(),
  286. event->code(), true);
  287. if (AreKeyboardKeyAcksNeeded()) {
  288. pending_key_acks_.insert(
  289. {serial,
  290. {*event, base::TimeTicks::Now() +
  291. expiration_delay_for_pending_key_acks_}});
  292. event->SetHandled();
  293. }
  294. }
  295. // Keep track of both the physical code and potentially re-written
  296. // code that this event generated.
  297. pressed_keys_.emplace(physical_code,
  298. KeyState{event->code(), consumed_by_ime});
  299. } else if (it != pressed_keys_.end() && !event->handled()) {
  300. // Non-repeate key events for already pressed key can be sent in some
  301. // cases (e.g. Holding 'A' key then holding 'B' key then releasing 'A'
  302. // key sends a non-repeat 'B' key press event).
  303. // When it happens, we don't want to send the press event to a client
  304. // and also want to avoid it from invoking any accelerator.
  305. if (AreKeyboardKeyAcksNeeded())
  306. event->SetHandled();
  307. }
  308. } break;
  309. case ui::ET_KEY_RELEASED: {
  310. // Process key release event if currently pressed.
  311. auto it = pressed_keys_.find(physical_code);
  312. if (it != pressed_keys_.end()) {
  313. for (auto& observer : observer_list_)
  314. observer.OnKeyboardKey(event->time_stamp(), it->second.code, false);
  315. if (!it->second.consumed_by_ime) {
  316. // We use the code that was generated when the physical key was
  317. // pressed rather than the current event code. This allows events
  318. // to be re-written before dispatch, while still allowing the
  319. // client to track the state of the physical keyboard.
  320. uint32_t serial = delegate_->OnKeyboardKey(event->time_stamp(),
  321. it->second.code, false);
  322. if (AreKeyboardKeyAcksNeeded()) {
  323. pending_key_acks_.insert(
  324. {serial,
  325. {*event, base::TimeTicks::Now() +
  326. expiration_delay_for_pending_key_acks_}});
  327. event->SetHandled();
  328. }
  329. }
  330. pressed_keys_.erase(it);
  331. }
  332. } break;
  333. default:
  334. NOTREACHED();
  335. break;
  336. }
  337. if (pending_key_acks_.empty())
  338. return;
  339. if (process_expired_pending_key_acks_pending_)
  340. return;
  341. ScheduleProcessExpiredPendingKeyAcks(expiration_delay_for_pending_key_acks_);
  342. }
  343. ////////////////////////////////////////////////////////////////////////////////
  344. // SurfaceObserver overrides:
  345. void Keyboard::OnSurfaceDestroying(Surface* surface) {
  346. DCHECK(surface == focus_);
  347. SetFocus(nullptr);
  348. }
  349. ////////////////////////////////////////////////////////////////////////////////
  350. // SeatObserver overrides:
  351. void Keyboard::OnSurfaceFocused(Surface* gained_focus,
  352. Surface* lost_focused,
  353. bool has_focused_surface) {
  354. Surface* gained_focus_surface =
  355. gained_focus && delegate_->CanAcceptKeyboardEventsForSurface(gained_focus)
  356. ? gained_focus
  357. : nullptr;
  358. if (gained_focus_surface != focus_)
  359. SetFocus(gained_focus_surface);
  360. }
  361. void Keyboard::OnKeyboardModifierUpdated() {
  362. // XkbTracker must be updated in the Seat, before calling this method.
  363. if (focus_)
  364. delegate_->OnKeyboardModifiers(seat_->xkb_tracker()->GetModifiers());
  365. }
  366. ////////////////////////////////////////////////////////////////////////////////
  367. // ash::KeyboardControllerObserver overrides:
  368. void Keyboard::OnKeyboardEnableFlagsChanged(
  369. const std::set<keyboard::KeyboardEnableFlag>& flags) {
  370. UpdateKeyboardType();
  371. }
  372. void Keyboard::OnKeyRepeatSettingsChanged(
  373. const ash::KeyRepeatSettings& settings) {
  374. delegate_->OnKeyRepeatSettingsChanged(settings.enabled, settings.delay,
  375. settings.interval);
  376. }
  377. ////////////////////////////////////////////////////////////////////////////////
  378. // ash::ImeControllerImpl::Observer overrides:
  379. void Keyboard::OnCapsLockChanged(bool enabled) {}
  380. void Keyboard::OnKeyboardLayoutNameChanged(const std::string& layout_name) {
  381. // XkbTracker must be updated in the Seat, before calling this method.
  382. // Ensured by the observer registration order.
  383. delegate_->OnKeyboardLayoutUpdated(seat_->xkb_tracker()->GetKeymap().get());
  384. }
  385. ////////////////////////////////////////////////////////////////////////////////
  386. // Keyboard, private:
  387. void Keyboard::SetFocus(Surface* surface) {
  388. if (focus_) {
  389. RemoveEventHandler();
  390. delegate_->OnKeyboardLeave(focus_);
  391. focus_->RemoveSurfaceObserver(this);
  392. focus_ = nullptr;
  393. pending_key_acks_.clear();
  394. }
  395. if (surface) {
  396. pressed_keys_ = seat_->pressed_keys();
  397. delegate_->OnKeyboardModifiers(seat_->xkb_tracker()->GetModifiers());
  398. delegate_->OnKeyboardEnter(surface, pressed_keys_);
  399. focus_ = surface;
  400. focus_->AddSurfaceObserver(this);
  401. focused_on_ime_supported_surface_ = IsImeSupportedSurface(surface);
  402. AddEventHandler();
  403. }
  404. }
  405. void Keyboard::ProcessExpiredPendingKeyAcks() {
  406. DCHECK(process_expired_pending_key_acks_pending_);
  407. process_expired_pending_key_acks_pending_ = false;
  408. // Check pending acks and process them as if it is handled if
  409. // expiration time passed.
  410. base::TimeTicks current_time = base::TimeTicks::Now();
  411. while (!pending_key_acks_.empty()) {
  412. auto it = pending_key_acks_.begin();
  413. const ui::KeyEvent event = it->second.first;
  414. if (it->second.second > current_time)
  415. break;
  416. // Expiration time has passed, assume the event was handled.
  417. pending_key_acks_.erase(it);
  418. }
  419. if (pending_key_acks_.empty())
  420. return;
  421. base::TimeDelta delay_until_next_process_expired_pending_key_acks =
  422. pending_key_acks_.begin()->second.second - current_time;
  423. ScheduleProcessExpiredPendingKeyAcks(
  424. delay_until_next_process_expired_pending_key_acks);
  425. }
  426. void Keyboard::ScheduleProcessExpiredPendingKeyAcks(base::TimeDelta delay) {
  427. DCHECK(!process_expired_pending_key_acks_pending_);
  428. process_expired_pending_key_acks_pending_ = true;
  429. base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
  430. FROM_HERE,
  431. base::BindOnce(&Keyboard::ProcessExpiredPendingKeyAcks,
  432. weak_ptr_factory_.GetWeakPtr()),
  433. delay);
  434. }
  435. void Keyboard::AddEventHandler() {
  436. if (!focus_)
  437. return;
  438. // Toplevel window can be not ShellSurface, for example for a notification
  439. // surface.
  440. aura::Window* toplevel_window = focus_->window();
  441. if (toplevel_window->GetToplevelWindow())
  442. toplevel_window = toplevel_window->GetToplevelWindow();
  443. if (are_keyboard_key_acks_needed_)
  444. toplevel_window->AddPreTargetHandler(this);
  445. else
  446. toplevel_window->AddPostTargetHandler(this);
  447. }
  448. void Keyboard::RemoveEventHandler() {
  449. if (!focus_)
  450. return;
  451. // Toplevel window can be not ShellSurface, for example for a notification
  452. // surface.
  453. aura::Window* toplevel_window = focus_->window();
  454. if (toplevel_window->GetToplevelWindow())
  455. toplevel_window = toplevel_window->GetToplevelWindow();
  456. if (are_keyboard_key_acks_needed_)
  457. toplevel_window->RemovePreTargetHandler(this);
  458. else
  459. toplevel_window->RemovePostTargetHandler(this);
  460. }
  461. void Keyboard::UpdateKeyboardType() {
  462. if (!device_configuration_delegate_)
  463. return;
  464. // Ignore kAndroidDisabled which affects |enabled| and just test for a11y
  465. // and touch enabled keyboards. TODO(yhanada): Fix this using an Android
  466. // specific KeyboardUI implementation. https://crbug.com/897655.
  467. const bool is_physical = !IsVirtualKeyboardEnabled();
  468. device_configuration_delegate_->OnKeyboardTypeChanged(is_physical);
  469. }
  470. } // namespace exo