drag_drop_operation.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/exo/drag_drop_operation.h"
  5. #include "base/barrier_closure.h"
  6. #include "base/check.h"
  7. #include "base/pickle.h"
  8. #include "base/strings/string_split.h"
  9. #include "base/threading/sequenced_task_runner_handle.h"
  10. #include "build/chromeos_buildflags.h"
  11. #include "components/exo/data_exchange_delegate.h"
  12. #include "components/exo/data_offer.h"
  13. #include "components/exo/data_source.h"
  14. #include "components/exo/seat.h"
  15. #include "components/exo/shell_surface_base.h"
  16. #include "components/exo/shell_surface_util.h"
  17. #include "components/exo/surface.h"
  18. #include "components/exo/surface_tree_host.h"
  19. #include "components/viz/common/frame_sinks/copy_output_request.h"
  20. #include "components/viz/common/frame_sinks/copy_output_result.h"
  21. #include "ui/aura/client/drag_drop_client.h"
  22. #include "ui/aura/window_tracker.h"
  23. #include "ui/base/clipboard/file_info.h"
  24. #include "ui/base/data_transfer_policy/data_transfer_endpoint.h"
  25. #include "ui/base/dragdrop/drag_drop_types.h"
  26. #include "ui/base/dragdrop/mojom/drag_drop_types.mojom.h"
  27. #include "ui/base/dragdrop/os_exchange_data.h"
  28. #include "ui/compositor/layer.h"
  29. #include "ui/display/display.h"
  30. #include "ui/display/screen.h"
  31. #include "ui/gfx/geometry/point.h"
  32. #include "ui/gfx/geometry/point_conversions.h"
  33. #include "ui/gfx/geometry/point_f.h"
  34. #include "ui/gfx/geometry/transform_util.h"
  35. #include "ui/gfx/geometry/vector2d.h"
  36. #include "url/gurl.h"
  37. #if BUILDFLAG(IS_CHROMEOS_ASH)
  38. #include "ash/drag_drop/drag_drop_controller.h"
  39. #include "chromeos/ui/base/window_properties.h"
  40. #include "components/exo/extended_drag_source.h"
  41. #include "ui/base/data_transfer_policy/data_transfer_endpoint_serializer.h"
  42. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  43. namespace exo {
  44. namespace {
  45. using ::ui::mojom::DragOperation;
  46. uint32_t DndActionsToDragOperations(const base::flat_set<DndAction>& actions) {
  47. uint32_t dnd_operations = 0;
  48. for (const DndAction action : actions) {
  49. switch (action) {
  50. case DndAction::kNone:
  51. [[fallthrough]];
  52. // We don't support the ask action
  53. case DndAction::kAsk:
  54. break;
  55. case DndAction::kCopy:
  56. dnd_operations |= ui::DragDropTypes::DragOperation::DRAG_COPY;
  57. break;
  58. case DndAction::kMove:
  59. dnd_operations |= ui::DragDropTypes::DragOperation::DRAG_MOVE;
  60. break;
  61. }
  62. }
  63. return dnd_operations;
  64. }
  65. #if BUILDFLAG(IS_CHROMEOS_ASH)
  66. DndAction DragOperationsToPreferredDndAction(int op) {
  67. if (op & ui::DragDropTypes::DragOperation::DRAG_COPY)
  68. return DndAction::kCopy;
  69. if (op & ui::DragDropTypes::DragOperation::DRAG_MOVE)
  70. return DndAction::kMove;
  71. return DndAction::kNone;
  72. }
  73. #endif
  74. DndAction DragOperationToDndAction(DragOperation op) {
  75. switch (op) {
  76. case DragOperation::kNone:
  77. return DndAction::kNone;
  78. case DragOperation::kMove:
  79. return DndAction::kMove;
  80. case DragOperation::kCopy:
  81. return DndAction::kCopy;
  82. case DragOperation::kLink:
  83. return DndAction::kAsk;
  84. default:
  85. NOTREACHED() << op;
  86. return DndAction::kNone;
  87. }
  88. NOTREACHED();
  89. }
  90. } // namespace
  91. // Internal representation of a drag icon surface. Used when a non-null surface
  92. // is passed in wl_data_device::start_drag requests.
  93. // TODO(crbug.com/1119385): Rework icon implementation to avoid frame copies.
  94. class DragDropOperation::IconSurface final : public SurfaceTreeHost,
  95. public ScopedSurface {
  96. public:
  97. IconSurface(Surface* icon, DragDropOperation* operation)
  98. : SurfaceTreeHost("ExoDragIcon"),
  99. ScopedSurface(icon, operation),
  100. operation_(operation) {
  101. DCHECK(operation_);
  102. DCHECK(!icon->HasSurfaceDelegate());
  103. Surface* origin_surface = operation_->origin_->get();
  104. origin_surface->window()->AddChild(host_window());
  105. SetRootSurface(icon);
  106. }
  107. IconSurface(const IconSurface&) = delete;
  108. IconSurface& operator=(const IconSurface&) = delete;
  109. ~IconSurface() override = default;
  110. private:
  111. // SurfaceTreeHost:
  112. void OnSurfaceCommit() override {
  113. SurfaceTreeHost::OnSurfaceCommit();
  114. RequestCaptureIcon();
  115. }
  116. void RequestCaptureIcon() {
  117. SubmitCompositorFrame();
  118. std::unique_ptr<viz::CopyOutputRequest> request =
  119. std::make_unique<viz::CopyOutputRequest>(
  120. viz::CopyOutputRequest::ResultFormat::RGBA,
  121. viz::CopyOutputRequest::ResultDestination::kSystemMemory,
  122. base::BindOnce(&IconSurface::OnCaptured,
  123. weak_ptr_factory_.GetWeakPtr()));
  124. request->set_result_task_runner(base::SequencedTaskRunnerHandle::Get());
  125. host_window()->layer()->RequestCopyOfOutput(std::move(request));
  126. }
  127. void OnCaptured(std::unique_ptr<viz::CopyOutputResult> icon_result) {
  128. // An empty response means the request was deleted before it was completed.
  129. // If this happens, and no operation has yet finished, restart the capture.
  130. if (icon_result->IsEmpty()) {
  131. RequestCaptureIcon();
  132. return;
  133. }
  134. auto scoped_bitmap = icon_result->ScopedAccessSkBitmap();
  135. operation_->OnDragIconCaptured(scoped_bitmap.GetOutScopedBitmap());
  136. }
  137. DragDropOperation* const operation_;
  138. base::WeakPtrFactory<IconSurface> weak_ptr_factory_{this};
  139. };
  140. base::WeakPtr<DragDropOperation> DragDropOperation::Create(
  141. DataExchangeDelegate* data_exchange_delegate,
  142. DataSource* source,
  143. Surface* origin,
  144. Surface* icon,
  145. const gfx::PointF& drag_start_point,
  146. ui::mojom::DragEventSource event_source) {
  147. auto* dnd_op = new DragDropOperation(data_exchange_delegate, source, origin,
  148. icon, drag_start_point, event_source);
  149. return dnd_op->weak_ptr_factory_.GetWeakPtr();
  150. }
  151. DragDropOperation::DragDropOperation(
  152. DataExchangeDelegate* data_exchange_delegate,
  153. DataSource* source,
  154. Surface* origin,
  155. Surface* icon,
  156. const gfx::PointF& drag_start_point,
  157. ui::mojom::DragEventSource event_source)
  158. : source_(std::make_unique<ScopedDataSource>(source, this)),
  159. origin_(std::make_unique<ScopedSurface>(origin, this)),
  160. drag_start_point_(drag_start_point),
  161. os_exchange_data_(std::make_unique<ui::OSExchangeData>()),
  162. event_source_(event_source) {
  163. aura::Window* root_window = origin_->get()->window()->GetRootWindow();
  164. DCHECK(root_window);
  165. #if BUILDFLAG(IS_CHROMEOS_ASH)
  166. drag_drop_controller_ = static_cast<ash::DragDropController*>(
  167. aura::client::GetDragDropClient(root_window));
  168. #else
  169. drag_drop_controller_ = aura::client::GetDragDropClient(root_window);
  170. #endif
  171. DCHECK(drag_drop_controller_);
  172. if (drag_drop_controller_->IsDragDropInProgress())
  173. drag_drop_controller_->DragCancel();
  174. drag_drop_controller_->AddObserver(this);
  175. ui::EndpointType endpoint_type =
  176. data_exchange_delegate->GetDataTransferEndpointType(
  177. origin_->get()->window());
  178. os_exchange_data_->SetSource(
  179. std::make_unique<ui::DataTransferEndpoint>(endpoint_type));
  180. #if BUILDFLAG(IS_CHROMEOS_ASH)
  181. extended_drag_source_ = ExtendedDragSource::Get();
  182. if (extended_drag_source_) {
  183. drag_drop_controller_->set_toplevel_window_drag_delegate(
  184. extended_drag_source_);
  185. extended_drag_source_->AddObserver(this);
  186. }
  187. #endif
  188. int num_additional_callbacks = 0;
  189. #if BUILDFLAG(IS_CHROMEOS_ASH)
  190. // TODO(crbug.com/1298033): Move DTE retrieval into
  191. // DataSource::GetDataForPreferredMimeTypes()
  192. // Lacros sends additional metadata, in a custom MIME type, to sync drag
  193. // source metadata. Hence, the number of callbacks is incremented by one.
  194. if (endpoint_type == ui::EndpointType::kLacros)
  195. ++num_additional_callbacks;
  196. #endif
  197. // When the icon is present, we increment the number of callbacks so we can
  198. // wait for the icon to be captured as well.
  199. if (icon) {
  200. icon_ = std::make_unique<IconSurface>(icon, this);
  201. ++num_additional_callbacks;
  202. }
  203. auto start_op_callback =
  204. base::BindOnce(&DragDropOperation::ScheduleStartDragDropOperation,
  205. weak_ptr_factory_.GetWeakPtr());
  206. counter_ =
  207. base::BarrierClosure(DataSource::kMaxDataTypes + num_additional_callbacks,
  208. std::move(start_op_callback));
  209. #if BUILDFLAG(IS_CHROMEOS_ASH)
  210. // TODO(crbug.com/1298033): Move DTE retrieval into
  211. // DataSource::GetDataForPreferredMimeTypes()
  212. if (endpoint_type == ui::EndpointType::kLacros) {
  213. source->ReadDataTransferEndpoint(
  214. base::BindOnce(&DragDropOperation::OnDataTransferEndpointRead,
  215. weak_ptr_factory_.GetWeakPtr()),
  216. counter_);
  217. }
  218. #endif
  219. source->GetDataForPreferredMimeTypes(
  220. base::BindOnce(&DragDropOperation::OnTextRead,
  221. weak_ptr_factory_.GetWeakPtr()),
  222. DataSource::ReadDataCallback(),
  223. base::BindOnce(&DragDropOperation::OnHTMLRead,
  224. weak_ptr_factory_.GetWeakPtr()),
  225. DataSource::ReadDataCallback(),
  226. base::BindOnce(&DragDropOperation::OnFilenamesRead,
  227. weak_ptr_factory_.GetWeakPtr(), data_exchange_delegate,
  228. origin->window()),
  229. base::BindOnce(&DragDropOperation::OnFileContentsRead,
  230. weak_ptr_factory_.GetWeakPtr()),
  231. base::BindOnce(&DragDropOperation::OnWebCustomDataRead,
  232. weak_ptr_factory_.GetWeakPtr()),
  233. counter_);
  234. }
  235. DragDropOperation::~DragDropOperation() {
  236. drag_drop_controller_->RemoveObserver(this);
  237. if (source_)
  238. source_->get()->Cancelled();
  239. if (drag_drop_controller_->IsDragDropInProgress() && started_by_this_object_)
  240. drag_drop_controller_->DragCancel();
  241. #if BUILDFLAG(IS_CHROMEOS_ASH)
  242. if (extended_drag_source_)
  243. ResetExtendedDragSource();
  244. #endif
  245. }
  246. void DragDropOperation::AbortIfPending() {
  247. if (!started_by_this_object_)
  248. delete this;
  249. }
  250. #if BUILDFLAG(IS_CHROMEOS_ASH)
  251. void DragDropOperation::OnDataTransferEndpointRead(const std::string& mime_type,
  252. std::u16string data) {
  253. DCHECK(os_exchange_data_);
  254. std::string utf8_json = base::UTF16ToUTF8(data);
  255. auto drag_source_dte = ui::ConvertJsonToDataTransferEndpoint(utf8_json);
  256. os_exchange_data_->SetSource(std::move(drag_source_dte));
  257. counter_.Run();
  258. }
  259. #endif
  260. void DragDropOperation::OnTextRead(const std::string& mime_type,
  261. std::u16string data) {
  262. DCHECK(os_exchange_data_);
  263. os_exchange_data_->SetString(std::move(data));
  264. // Prefer to use the HTML MIME type if possible
  265. if (mime_type_.empty())
  266. mime_type_ = mime_type;
  267. counter_.Run();
  268. }
  269. void DragDropOperation::OnHTMLRead(const std::string& mime_type,
  270. std::u16string data) {
  271. DCHECK(os_exchange_data_);
  272. os_exchange_data_->SetHtml(std::move(data), GURL());
  273. mime_type_ = mime_type;
  274. counter_.Run();
  275. }
  276. void DragDropOperation::OnFilenamesRead(
  277. DataExchangeDelegate* data_exchange_delegate,
  278. aura::Window* source,
  279. const std::string& mime_type,
  280. const std::vector<uint8_t>& data) {
  281. DCHECK(os_exchange_data_);
  282. os_exchange_data_->SetFilenames(data_exchange_delegate->GetFilenames(
  283. data_exchange_delegate->GetDataTransferEndpointType(source), data));
  284. mime_type_ = mime_type;
  285. counter_.Run();
  286. }
  287. void DragDropOperation::OnFileContentsRead(const std::string& mime_type,
  288. const base::FilePath& filename,
  289. const std::vector<uint8_t>& data) {
  290. DCHECK(os_exchange_data_);
  291. os_exchange_data_->SetFileContents(filename,
  292. std::string(data.begin(), data.end()));
  293. mime_type_ = mime_type;
  294. counter_.Run();
  295. }
  296. void DragDropOperation::OnWebCustomDataRead(const std::string& mime_type,
  297. const std::vector<uint8_t>& data) {
  298. DCHECK(os_exchange_data_);
  299. base::Pickle pickle(reinterpret_cast<const char*>(data.data()), data.size());
  300. os_exchange_data_->SetPickledData(
  301. ui::ClipboardFormatType::WebCustomDataType(), pickle);
  302. mime_type_ = mime_type;
  303. counter_.Run();
  304. }
  305. void DragDropOperation::OnDragIconCaptured(const SkBitmap& icon_bitmap) {
  306. DCHECK(icon_);
  307. float scale_factor = origin_->get()->window()->layer()->device_scale_factor();
  308. gfx::ImageSkia icon_skia =
  309. gfx::ImageSkia::CreateFromBitmap(icon_bitmap, scale_factor);
  310. gfx::Vector2d icon_offset = -icon_->get()->GetBufferOffset();
  311. if (os_exchange_data_) {
  312. os_exchange_data_->provider().SetDragImage(icon_skia, icon_offset);
  313. } else {
  314. #if BUILDFLAG(IS_CHROMEOS_ASH)
  315. drag_drop_controller_->SetDragImage(icon_skia, icon_offset);
  316. #endif
  317. }
  318. if (!captured_icon_) {
  319. captured_icon_ = true;
  320. counter_.Run();
  321. }
  322. }
  323. void DragDropOperation::ScheduleStartDragDropOperation() {
  324. // StartDragAndDrop uses a nested run loop. When restarting, we a) don't want
  325. // to interrupt the callers task for an arbitrary period of time and b) want
  326. // to let any nested run loops that are currently running to have a chance to
  327. // exit to avoid arbitrarily deep nesting. We can accomplish both of those
  328. // things by posting a new task to actually start the drag and drop operation.
  329. #if BUILDFLAG(IS_CHROMEOS_ASH)
  330. if (extended_drag_source_) {
  331. ShellSurfaceBase* shell_surface = GetShellSurfaceBaseForWindow(
  332. origin_->get()->window()->GetToplevelWindow());
  333. if (shell_surface)
  334. shell_surface->set_in_extended_drag(true);
  335. }
  336. #endif
  337. base::SequencedTaskRunnerHandle::Get()->PostTask(
  338. FROM_HERE, base::BindOnce(&DragDropOperation::StartDragDropOperation,
  339. weak_ptr_factory_.GetWeakPtr()));
  340. }
  341. void DragDropOperation::StartDragDropOperation() {
  342. uint32_t dnd_operations =
  343. DndActionsToDragOperations(source_->get()->GetActions());
  344. base::WeakPtr<DragDropOperation> weak_ptr = weak_ptr_factory_.GetWeakPtr();
  345. started_by_this_object_ = true;
  346. gfx::Point drag_start_point = gfx::ToFlooredPoint(drag_start_point_);
  347. // This triggers a nested run loop that terminates when the drag and drop
  348. // operation is completed.
  349. DragOperation op = drag_drop_controller_->StartDragAndDrop(
  350. std::move(os_exchange_data_), origin_->get()->window()->GetRootWindow(),
  351. origin_->get()->window(), drag_start_point, dnd_operations,
  352. event_source_);
  353. // The instance deleted during StartDragAndDrop's nested RunLoop.
  354. if (!weak_ptr)
  355. return;
  356. #if BUILDFLAG(IS_CHROMEOS_ASH)
  357. // Always reset the in_extended_drag becacuse ExtendedDragSource may be
  358. // destroyed during nested loop.
  359. if (origin_->get()) {
  360. ShellSurfaceBase* shell_surface = GetShellSurfaceBaseForWindow(
  361. origin_->get()->window()->GetToplevelWindow());
  362. if (shell_surface)
  363. shell_surface->set_in_extended_drag(false);
  364. }
  365. #endif
  366. // In tests, drag_drop_controller_ does not create a nested message loop and
  367. // so StartDragAndDrop exits before the drag&drop session finishes. In that
  368. // case the cleanup process shouldn't be made.
  369. if (drag_drop_controller_->IsDragDropInProgress())
  370. return;
  371. if (op != DragOperation::kNone) {
  372. // It is possible that Ash flags the dragged tab to snap its origin, and
  373. // it uses the `chromeos::kIsDeferredTabDraggingTargetWindowKey` property to
  374. // control that. The snap back behavior works as if the drag was cancelled.
  375. bool force_tab_swallow = false;
  376. #if BUILDFLAG(IS_CHROMEOS_ASH)
  377. aura::Window* source_window = origin_->get()->window()->GetToplevelWindow();
  378. force_tab_swallow =
  379. (source_window && source_window->GetProperty(
  380. chromeos::kIsDeferredTabDraggingTargetWindowKey));
  381. source_window->ClearProperty(
  382. chromeos::kIsDeferredTabDraggingTargetWindowKey);
  383. #endif
  384. if (force_tab_swallow) {
  385. source_->get()->Cancelled();
  386. } else {
  387. // Success
  388. // TODO(crbug.com/994065) This is currently not the actual mime type
  389. // used by the recipient, just an arbitrary one we pick out of the
  390. // offered types so we can report back whether or not the drop can
  391. // succeed. This may need to change in the future.
  392. source_->get()->Target(mime_type_);
  393. source_->get()->Action(DragOperationToDndAction(op));
  394. source_->get()->DndDropPerformed();
  395. source_->get()->DndFinished();
  396. }
  397. // Reset |source_| so it the destructor doesn't try to cancel it.
  398. source_.reset();
  399. }
  400. // On failure the destructor will handle canceling the data source.
  401. delete this;
  402. }
  403. void DragDropOperation::OnDragStarted() {
  404. if (!started_by_this_object_)
  405. delete this;
  406. }
  407. #if BUILDFLAG(IS_CHROMEOS_ASH)
  408. void DragDropOperation::OnDragActionsChanged(int actions) {
  409. if (!started_by_this_object_)
  410. return;
  411. DndAction dnd_action = DragOperationsToPreferredDndAction(actions);
  412. // We send a mime type along with the action to indicate to the application
  413. // that dropping is/is not currently possible. We do not currently know of
  414. // any applications that care about the specific mime type until the drop is
  415. // actually performed.
  416. if (dnd_action != DndAction::kNone)
  417. source_->get()->Target(mime_type_);
  418. else
  419. source_->get()->Target(absl::nullopt);
  420. source_->get()->Action(dnd_action);
  421. }
  422. void DragDropOperation::OnExtendedDragSourceDestroying(
  423. ExtendedDragSource* source) {
  424. ResetExtendedDragSource();
  425. }
  426. void DragDropOperation::ResetExtendedDragSource() {
  427. DCHECK(extended_drag_source_);
  428. extended_drag_source_->RemoveObserver(this);
  429. drag_drop_controller_->set_toplevel_window_drag_delegate(nullptr);
  430. extended_drag_source_ = nullptr;
  431. }
  432. #endif
  433. void DragDropOperation::OnSurfaceDestroying(Surface* surface) {
  434. DCHECK(surface == origin_->get() || (icon_ && surface == icon_->get()));
  435. delete this;
  436. }
  437. void DragDropOperation::OnDataSourceDestroying(DataSource* source) {
  438. DCHECK_EQ(source, source_->get());
  439. source_.reset();
  440. delete this;
  441. }
  442. } // namespace exo