input_injector_mac.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/host/input_injector.h"
  5. #include <ApplicationServices/ApplicationServices.h>
  6. #include <Carbon/Carbon.h>
  7. #include <IOKit/pwr_mgt/IOPMLib.h>
  8. #include <stdint.h>
  9. #include <algorithm>
  10. #include <utility>
  11. #include "base/bind.h"
  12. #include "base/compiler_specific.h"
  13. #include "base/i18n/break_iterator.h"
  14. #include "base/location.h"
  15. #include "base/logging.h"
  16. #include "base/mac/scoped_cftyperef.h"
  17. #include "base/memory/ptr_util.h"
  18. #include "base/memory/ref_counted.h"
  19. #include "base/strings/string_piece.h"
  20. #include "base/strings/utf_string_conversions.h"
  21. #include "base/task/single_thread_task_runner.h"
  22. #include "base/time/time.h"
  23. #include "remoting/host/clipboard.h"
  24. #include "remoting/proto/internal.pb.h"
  25. #include "remoting/protocol/message_decoder.h"
  26. #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  27. #include "third_party/webrtc/modules/desktop_capture/mac/desktop_configuration.h"
  28. #include "ui/events/keycodes/dom/keycode_converter.h"
  29. namespace remoting {
  30. namespace {
  31. void SetOrClearBit(uint64_t &value, uint64_t bit, bool set_bit) {
  32. value = set_bit ? (value | bit) : (value & ~bit);
  33. }
  34. // Must be called on UI thread.
  35. void CreateAndPostKeyEvent(int keycode,
  36. bool pressed,
  37. uint64_t flags,
  38. const std::u16string& unicode) {
  39. base::ScopedCFTypeRef<CGEventRef> eventRef(
  40. CGEventCreateKeyboardEvent(nullptr, keycode, pressed));
  41. if (eventRef) {
  42. CGEventSetFlags(eventRef, static_cast<CGEventFlags>(flags));
  43. if (!unicode.empty())
  44. CGEventKeyboardSetUnicodeString(
  45. eventRef, unicode.size(),
  46. reinterpret_cast<const UniChar*>(unicode.data()));
  47. CGEventPost(kCGSessionEventTap, eventRef);
  48. }
  49. }
  50. // Must be called on UI thread.
  51. void PostMouseEvent(int32_t x,
  52. int32_t y,
  53. bool left_down,
  54. bool right_down,
  55. bool middle_down) {
  56. // We use the deprecated CGPostMouseEvent API because we receive low-level
  57. // mouse events, whereas CGEventCreateMouseEvent is for injecting higher-level
  58. // events. For example, the deprecated APIs will detect double-clicks or drags
  59. // in a way that is consistent with how they would be generated using a local
  60. // mouse, whereas the new APIs expect us to inject these higher-level events
  61. // directly.
  62. //
  63. // See crbug.com/677857 for more details.
  64. CGPoint position = CGPointMake(x, y);
  65. #pragma clang diagnostic push
  66. #pragma clang diagnostic ignored "-Wdeprecated-declarations"
  67. CGError error =
  68. CGPostMouseEvent(position, true, 3, left_down, right_down, middle_down);
  69. #pragma clang diagnostic pop
  70. if (error != kCGErrorSuccess)
  71. LOG(WARNING) << "CGPostMouseEvent error " << error;
  72. }
  73. // Must be called on UI thread.
  74. void CreateAndPostScrollWheelEvent(int32_t delta_x, int32_t delta_y) {
  75. base::ScopedCFTypeRef<CGEventRef> eventRef(CGEventCreateScrollWheelEvent(
  76. nullptr, kCGScrollEventUnitPixel, 2, delta_y, delta_x));
  77. if (eventRef) {
  78. CGEventPost(kCGSessionEventTap, eventRef);
  79. }
  80. }
  81. // This value is not defined. Give it the obvious name so that if it is ever
  82. // added there will be a handy compilation error to remind us to remove this
  83. // definition.
  84. const int kVK_RightCommand = 0x36;
  85. // Determines the minimum amount of time between attempts to waken the display
  86. // in response to an input event.
  87. const int kWakeUpDisplayIntervalMs = 1000;
  88. using protocol::ClipboardEvent;
  89. using protocol::KeyEvent;
  90. using protocol::TextEvent;
  91. using protocol::MouseEvent;
  92. using protocol::TouchEvent;
  93. // A class to generate events on Mac.
  94. class InputInjectorMac : public InputInjector {
  95. public:
  96. explicit InputInjectorMac(
  97. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner,
  98. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner);
  99. InputInjectorMac(const InputInjectorMac&) = delete;
  100. InputInjectorMac& operator=(const InputInjectorMac&) = delete;
  101. ~InputInjectorMac() override;
  102. // ClipboardStub interface.
  103. void InjectClipboardEvent(const ClipboardEvent& event) override;
  104. // InputStub interface.
  105. void InjectKeyEvent(const KeyEvent& event) override;
  106. void InjectTextEvent(const TextEvent& event) override;
  107. void InjectMouseEvent(const MouseEvent& event) override;
  108. void InjectTouchEvent(const TouchEvent& event) override;
  109. // InputInjector interface.
  110. void Start(
  111. std::unique_ptr<protocol::ClipboardStub> client_clipboard) override;
  112. private:
  113. // The actual implementation resides in InputInjectorMac::Core class.
  114. class Core : public base::RefCountedThreadSafe<Core> {
  115. public:
  116. explicit Core(
  117. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner,
  118. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner);
  119. Core(const Core&) = delete;
  120. Core& operator=(const Core&) = delete;
  121. // Mirrors the ClipboardStub interface.
  122. void InjectClipboardEvent(const ClipboardEvent& event);
  123. // Mirrors the InputStub interface.
  124. void InjectKeyEvent(const KeyEvent& event);
  125. void InjectTextEvent(const TextEvent& event);
  126. void InjectMouseEvent(const MouseEvent& event);
  127. // Mirrors the InputInjector interface.
  128. void Start(std::unique_ptr<protocol::ClipboardStub> client_clipboard);
  129. void Stop();
  130. private:
  131. friend class base::RefCountedThreadSafe<Core>;
  132. virtual ~Core();
  133. void WakeUpDisplay();
  134. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner_;
  135. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner_;
  136. webrtc::DesktopVector mouse_pos_;
  137. uint32_t mouse_button_state_;
  138. std::unique_ptr<Clipboard> clipboard_;
  139. uint64_t left_modifiers_;
  140. uint64_t right_modifiers_;
  141. base::TimeTicks last_time_display_woken_;
  142. };
  143. scoped_refptr<Core> core_;
  144. };
  145. InputInjectorMac::InputInjectorMac(
  146. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner,
  147. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner) {
  148. core_ = new Core(input_thread_task_runner, ui_thread_task_runner);
  149. }
  150. InputInjectorMac::~InputInjectorMac() {
  151. core_->Stop();
  152. }
  153. void InputInjectorMac::InjectClipboardEvent(const ClipboardEvent& event) {
  154. core_->InjectClipboardEvent(event);
  155. }
  156. void InputInjectorMac::InjectKeyEvent(const KeyEvent& event) {
  157. core_->InjectKeyEvent(event);
  158. }
  159. void InputInjectorMac::InjectTextEvent(const TextEvent& event) {
  160. core_->InjectTextEvent(event);
  161. }
  162. void InputInjectorMac::InjectMouseEvent(const MouseEvent& event) {
  163. core_->InjectMouseEvent(event);
  164. }
  165. void InputInjectorMac::InjectTouchEvent(const TouchEvent& event) {
  166. NOTIMPLEMENTED() << "Raw touch event injection not implemented for Mac.";
  167. }
  168. void InputInjectorMac::Start(
  169. std::unique_ptr<protocol::ClipboardStub> client_clipboard) {
  170. core_->Start(std::move(client_clipboard));
  171. }
  172. InputInjectorMac::Core::Core(
  173. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner,
  174. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner)
  175. : input_thread_task_runner_(input_thread_task_runner),
  176. ui_thread_task_runner_(ui_thread_task_runner),
  177. mouse_button_state_(0),
  178. clipboard_(Clipboard::Create()),
  179. left_modifiers_(0),
  180. right_modifiers_(0) {
  181. // Ensure that local hardware events are not suppressed after injecting
  182. // input events. This allows LocalInputMonitor to detect if the local mouse
  183. // is being moved whilst a remote user is connected.
  184. // This API is deprecated, but it is needed when using the deprecated
  185. // injection APIs.
  186. // If the non-deprecated injection APIs were used instead, the equivalent of
  187. // this line would not be needed, as OS X defaults to _not_ suppressing local
  188. // inputs in that case.
  189. #pragma clang diagnostic push
  190. #pragma clang diagnostic ignored "-Wdeprecated-declarations"
  191. CGSetLocalEventsSuppressionInterval(0.0);
  192. #pragma clang diagnostic pop
  193. }
  194. void InputInjectorMac::Core::InjectClipboardEvent(const ClipboardEvent& event) {
  195. if (!input_thread_task_runner_->BelongsToCurrentThread()) {
  196. input_thread_task_runner_->PostTask(
  197. FROM_HERE, base::BindOnce(&Core::InjectClipboardEvent, this, event));
  198. return;
  199. }
  200. // |clipboard_| will ignore unknown MIME-types, and verify the data's format.
  201. clipboard_->InjectClipboardEvent(event);
  202. }
  203. void InputInjectorMac::Core::InjectKeyEvent(const KeyEvent& event) {
  204. // HostEventDispatcher should filter events missing the pressed field.
  205. if (!event.has_pressed() || !event.has_usb_keycode())
  206. return;
  207. WakeUpDisplay();
  208. int keycode =
  209. ui::KeycodeConverter::UsbKeycodeToNativeKeycode(event.usb_keycode());
  210. VLOG(3) << "Converting USB keycode: " << std::hex << event.usb_keycode()
  211. << " to keycode: " << keycode << std::dec;
  212. // If we couldn't determine the Mac virtual key code then ignore the event.
  213. if (keycode == ui::KeycodeConverter::InvalidNativeKeycode())
  214. return;
  215. // If this is a modifier key, remember its new state so that it can be
  216. // correctly applied to subsequent events.
  217. if (keycode == kVK_Command) {
  218. SetOrClearBit(left_modifiers_, kCGEventFlagMaskCommand, event.pressed());
  219. } else if (keycode == kVK_Shift) {
  220. SetOrClearBit(left_modifiers_, kCGEventFlagMaskShift, event.pressed());
  221. } else if (keycode == kVK_Control) {
  222. SetOrClearBit(left_modifiers_, kCGEventFlagMaskControl, event.pressed());
  223. } else if (keycode == kVK_Option) {
  224. SetOrClearBit(left_modifiers_, kCGEventFlagMaskAlternate, event.pressed());
  225. } else if (keycode == kVK_RightCommand) {
  226. SetOrClearBit(right_modifiers_, kCGEventFlagMaskCommand, event.pressed());
  227. } else if (keycode == kVK_RightShift) {
  228. SetOrClearBit(right_modifiers_, kCGEventFlagMaskShift, event.pressed());
  229. } else if (keycode == kVK_RightControl) {
  230. SetOrClearBit(right_modifiers_, kCGEventFlagMaskControl, event.pressed());
  231. } else if (keycode == kVK_RightOption) {
  232. SetOrClearBit(right_modifiers_, kCGEventFlagMaskAlternate, event.pressed());
  233. }
  234. // In addition to the modifier keys pressed right now, we also need to set
  235. // AlphaShift if caps lock was active at the client (Mac ignores NumLock).
  236. uint64_t flags = left_modifiers_ | right_modifiers_;
  237. if ((event.has_caps_lock_state() && event.caps_lock_state()) ||
  238. (event.has_lock_states() &&
  239. (event.lock_states() & protocol::KeyEvent::LOCK_STATES_CAPSLOCK) != 0)) {
  240. flags |= kCGEventFlagMaskAlphaShift;
  241. }
  242. ui_thread_task_runner_->PostTask(
  243. FROM_HERE, base::BindOnce(CreateAndPostKeyEvent, keycode, event.pressed(),
  244. flags, std::u16string()));
  245. }
  246. void InputInjectorMac::Core::InjectTextEvent(const TextEvent& event) {
  247. DCHECK(event.has_text());
  248. WakeUpDisplay();
  249. std::u16string text = base::UTF8ToUTF16(event.text());
  250. // CGEventKeyboardSetUnicodeString appears to only process up to 20 code
  251. // units (and key presses are generally expected to generate a single
  252. // character), so split the input text into graphemes.
  253. base::i18n::BreakIterator grapheme_iterator(
  254. text, base::i18n::BreakIterator::BREAK_CHARACTER);
  255. if (!grapheme_iterator.Init()) {
  256. LOG(ERROR) << "Failed to init grapheme iterator.";
  257. return;
  258. }
  259. while (grapheme_iterator.Advance()) {
  260. base::StringPiece16 grapheme = grapheme_iterator.GetStringPiece();
  261. if (grapheme.length() == 1 && grapheme[0] == '\n') {
  262. // On Mac, the return key sends "\r" rather than "\n", so handle it
  263. // specially.
  264. ui_thread_task_runner_->PostTask(
  265. FROM_HERE, base::BindOnce(CreateAndPostKeyEvent, kVK_Return,
  266. /*pressed=*/true, 0, std::u16string()));
  267. ui_thread_task_runner_->PostTask(
  268. FROM_HERE, base::BindOnce(CreateAndPostKeyEvent, kVK_Return,
  269. /*pressed=*/false, 0, std::u16string()));
  270. } else {
  271. // Applications that ignore UnicodeString field will see the text event as
  272. // Space key.
  273. ui_thread_task_runner_->PostTask(
  274. FROM_HERE,
  275. base::BindOnce(CreateAndPostKeyEvent, kVK_Space,
  276. /*pressed=*/true, 0, std::u16string(grapheme)));
  277. ui_thread_task_runner_->PostTask(
  278. FROM_HERE,
  279. base::BindOnce(CreateAndPostKeyEvent, kVK_Space,
  280. /*pressed=*/false, 0, std::u16string(grapheme)));
  281. }
  282. }
  283. }
  284. void InputInjectorMac::Core::InjectMouseEvent(const MouseEvent& event) {
  285. WakeUpDisplay();
  286. if (event.has_x() && event.has_y()) {
  287. mouse_pos_.set(event.x(), event.y());
  288. VLOG(3) << "Moving mouse to " << mouse_pos_.x() << "," << mouse_pos_.y();
  289. }
  290. if (event.has_button() && event.has_button_down()) {
  291. if (event.button() >= 1 && event.button() <= 3) {
  292. VLOG(2) << "Button " << event.button()
  293. << (event.button_down() ? " down" : " up");
  294. int button_change = 1 << (event.button() - 1);
  295. if (event.button_down())
  296. mouse_button_state_ |= button_change;
  297. else
  298. mouse_button_state_ &= ~button_change;
  299. } else {
  300. VLOG(1) << "Unknown mouse button: " << event.button();
  301. }
  302. }
  303. enum {
  304. LeftBit = 1 << (MouseEvent::BUTTON_LEFT - 1),
  305. MiddleBit = 1 << (MouseEvent::BUTTON_MIDDLE - 1),
  306. RightBit = 1 << (MouseEvent::BUTTON_RIGHT - 1)
  307. };
  308. ui_thread_task_runner_->PostTask(
  309. FROM_HERE, base::BindOnce(PostMouseEvent, mouse_pos_.x(), mouse_pos_.y(),
  310. (mouse_button_state_ & LeftBit) != 0,
  311. (mouse_button_state_ & RightBit) != 0,
  312. (mouse_button_state_ & MiddleBit) != 0));
  313. if (event.has_wheel_delta_x() && event.has_wheel_delta_y()) {
  314. ui_thread_task_runner_->PostTask(
  315. FROM_HERE,
  316. base::BindOnce(CreateAndPostScrollWheelEvent, event.wheel_delta_x(),
  317. event.wheel_delta_y()));
  318. }
  319. }
  320. void InputInjectorMac::Core::Start(
  321. std::unique_ptr<protocol::ClipboardStub> client_clipboard) {
  322. if (!input_thread_task_runner_->BelongsToCurrentThread()) {
  323. input_thread_task_runner_->PostTask(
  324. FROM_HERE,
  325. base::BindOnce(&Core::Start, this, std::move(client_clipboard)));
  326. return;
  327. }
  328. clipboard_->Start(std::move(client_clipboard));
  329. }
  330. void InputInjectorMac::Core::Stop() {
  331. if (!input_thread_task_runner_->BelongsToCurrentThread()) {
  332. input_thread_task_runner_->PostTask(FROM_HERE,
  333. base::BindOnce(&Core::Stop, this));
  334. return;
  335. }
  336. clipboard_.reset();
  337. }
  338. void InputInjectorMac::Core::WakeUpDisplay() {
  339. base::TimeTicks now = base::TimeTicks::Now();
  340. if (now - last_time_display_woken_ <
  341. base::Milliseconds(kWakeUpDisplayIntervalMs)) {
  342. return;
  343. }
  344. last_time_display_woken_ = now;
  345. // TODO(dcaiafa): Consolidate power management with webrtc::ScreenCapturer
  346. // (crbug.com/535769)
  347. // Normally one would want to create a power assertion and hold it for the
  348. // duration of the session. An active power assertion prevents the display
  349. // from going to sleep automatically, but it doesn't prevent the user from
  350. // forcing it to sleep (e.g. by going to a hot corner). The display is only
  351. // re-awaken at the moment the assertion is created.
  352. IOPMAssertionID power_assertion_id = kIOPMNullAssertionID;
  353. IOReturn result = IOPMAssertionCreateWithName(
  354. CFSTR("UserIsActive"),
  355. kIOPMAssertionLevelOn,
  356. CFSTR("Chrome Remote Desktop connection active"),
  357. &power_assertion_id);
  358. if (result == kIOReturnSuccess) {
  359. IOPMAssertionRelease(power_assertion_id);
  360. }
  361. }
  362. InputInjectorMac::Core::~Core() {}
  363. } // namespace
  364. // static
  365. std::unique_ptr<InputInjector> InputInjector::Create(
  366. scoped_refptr<base::SingleThreadTaskRunner> input_thread_task_runner,
  367. scoped_refptr<base::SingleThreadTaskRunner> ui_thread_task_runner) {
  368. return base::WrapUnique(
  369. new InputInjectorMac(input_thread_task_runner, ui_thread_task_runner));
  370. }
  371. // static
  372. bool InputInjector::SupportsTouchEvents() {
  373. return false;
  374. }
  375. } // namespace remoting