ax_assistant_structure.cc 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "ui/accessibility/ax_assistant_structure.h"
  5. #include <utility>
  6. #include "base/logging.h"
  7. #include "base/strings/stringprintf.h"
  8. #include "base/strings/utf_string_conversions.h"
  9. #include "third_party/abseil-cpp/absl/types/optional.h"
  10. #include "ui/accessibility/ax_enums.mojom.h"
  11. #include "ui/accessibility/ax_node.h"
  12. #include "ui/accessibility/ax_role_properties.h"
  13. #include "ui/accessibility/ax_serializable_tree.h"
  14. #include "ui/accessibility/platform/ax_android_constants.h"
  15. #include "ui/gfx/geometry/rect_conversions.h"
  16. #include "ui/gfx/geometry/transform.h"
  17. #include "ui/gfx/range/range.h"
  18. namespace ui {
  19. namespace {
  20. // TODO(muyuanli): share with BrowserAccessibility.
  21. bool IsTextField(const AXNode* node) {
  22. return node->data().IsTextField();
  23. }
  24. bool IsRichTextEditable(const AXNode* node) {
  25. const AXNode* parent = node->GetUnignoredParent();
  26. return node->HasState(ax::mojom::State::kRichlyEditable) &&
  27. (!parent || !parent->HasState(ax::mojom::State::kRichlyEditable));
  28. }
  29. bool IsAtomicTextField(const AXNode* node) {
  30. const std::string& html_tag =
  31. node->GetStringAttribute(ax::mojom::StringAttribute::kHtmlTag);
  32. if (html_tag == "input") {
  33. std::string input_type;
  34. if (!node->GetHtmlAttribute("type", &input_type))
  35. return true;
  36. return input_type.empty() || input_type == "email" ||
  37. input_type == "password" || input_type == "search" ||
  38. input_type == "tel" || input_type == "text" || input_type == "url" ||
  39. input_type == "number";
  40. }
  41. return html_tag == "textarea";
  42. }
  43. bool IsLeaf(const AXNode* node) {
  44. if (node->children().empty())
  45. return true;
  46. if (IsAtomicTextField(node) || node->IsText()) {
  47. return true;
  48. }
  49. switch (node->GetRole()) {
  50. case ax::mojom::Role::kImage:
  51. case ax::mojom::Role::kMeter:
  52. case ax::mojom::Role::kScrollBar:
  53. case ax::mojom::Role::kSlider:
  54. case ax::mojom::Role::kSplitter:
  55. case ax::mojom::Role::kProgressIndicator:
  56. case ax::mojom::Role::kDate:
  57. case ax::mojom::Role::kDateTime:
  58. case ax::mojom::Role::kInputTime:
  59. return true;
  60. default:
  61. return false;
  62. }
  63. }
  64. std::u16string GetInnerText(const AXNode* node) {
  65. if (node->IsText()) {
  66. return node->GetString16Attribute(ax::mojom::StringAttribute::kName);
  67. }
  68. std::u16string text;
  69. for (auto iter = node->UnignoredChildrenBegin();
  70. iter != node->UnignoredChildrenEnd(); ++iter) {
  71. text += GetInnerText(iter.get());
  72. }
  73. return text;
  74. }
  75. std::u16string GetValue(const AXNode* node) {
  76. std::u16string value =
  77. node->GetString16Attribute(ax::mojom::StringAttribute::kValue);
  78. if (value.empty() && (IsTextField(node) || IsRichTextEditable(node)) &&
  79. !IsAtomicTextField(node)) {
  80. value = GetInnerText(node);
  81. }
  82. // Always obscure passwords.
  83. if (node->HasState(ax::mojom::State::kProtected))
  84. value = std::u16string(value.size(), kSecurePasswordBullet);
  85. return value;
  86. }
  87. std::u16string GetText(const AXNode* node) {
  88. if (node->GetRole() == ax::mojom::Role::kPdfRoot ||
  89. node->GetRole() == ax::mojom::Role::kIframe ||
  90. node->GetRole() == ax::mojom::Role::kIframePresentational) {
  91. return std::u16string();
  92. }
  93. ax::mojom::NameFrom name_from = node->GetNameFrom();
  94. if (!ui::IsLeaf(node) && name_from == ax::mojom::NameFrom::kContents) {
  95. return std::u16string();
  96. }
  97. std::u16string value = GetValue(node);
  98. if (!value.empty()) {
  99. if (node->HasState(ax::mojom::State::kEditable))
  100. return value;
  101. switch (node->GetRole()) {
  102. case ax::mojom::Role::kComboBoxMenuButton:
  103. case ax::mojom::Role::kTextFieldWithComboBox:
  104. case ax::mojom::Role::kPopUpButton:
  105. case ax::mojom::Role::kTextField:
  106. return value;
  107. default:
  108. break;
  109. }
  110. }
  111. if (node->GetRole() == ax::mojom::Role::kColorWell) {
  112. unsigned int color = static_cast<unsigned int>(
  113. node->GetIntAttribute(ax::mojom::IntAttribute::kColorValue));
  114. unsigned int red = color >> 16 & 0xFF;
  115. unsigned int green = color >> 8 & 0xFF;
  116. unsigned int blue = color >> 0 & 0xFF;
  117. return base::UTF8ToUTF16(
  118. base::StringPrintf("#%02X%02X%02X", red, green, blue));
  119. }
  120. std::u16string text =
  121. node->GetString16Attribute(ax::mojom::StringAttribute::kName);
  122. std::u16string description =
  123. node->GetString16Attribute(ax::mojom::StringAttribute::kDescription);
  124. if (!description.empty()) {
  125. if (!text.empty())
  126. text += u" ";
  127. text += description;
  128. }
  129. if (text.empty())
  130. text = value;
  131. if (node->GetRole() == ax::mojom::Role::kRootWebArea ||
  132. node->GetRole() == ax::mojom::Role::kPdfRoot) {
  133. return text;
  134. }
  135. if (text.empty() && IsLeaf(node)) {
  136. for (auto iter = node->UnignoredChildrenBegin();
  137. iter != node->UnignoredChildrenEnd(); ++iter) {
  138. text += GetInnerText(iter.get());
  139. }
  140. }
  141. if (text.empty() && (ui::IsLink(node->GetRole()) ||
  142. node->GetRole() == ax::mojom::Role::kImage)) {
  143. std::u16string url =
  144. node->GetString16Attribute(ax::mojom::StringAttribute::kUrl);
  145. text = AXUrlBaseText(url);
  146. }
  147. return text;
  148. }
  149. // Get string representation of ax::mojom::Role. We are not using ToString() in
  150. // ax_enums.h since the names are subject to change in the future and
  151. // we are only interested in a subset of the roles.
  152. absl::optional<std::string> AXRoleToString(ax::mojom::Role role) {
  153. switch (role) {
  154. case ax::mojom::Role::kArticle:
  155. return absl::optional<std::string>("article");
  156. case ax::mojom::Role::kBanner:
  157. return absl::optional<std::string>("banner");
  158. case ax::mojom::Role::kCaption:
  159. return absl::optional<std::string>("caption");
  160. case ax::mojom::Role::kComplementary:
  161. return absl::optional<std::string>("complementary");
  162. case ax::mojom::Role::kDate:
  163. return absl::optional<std::string>("date");
  164. case ax::mojom::Role::kDateTime:
  165. return absl::optional<std::string>("date_time");
  166. case ax::mojom::Role::kDefinition:
  167. return absl::optional<std::string>("definition");
  168. case ax::mojom::Role::kDetails:
  169. return absl::optional<std::string>("details");
  170. case ax::mojom::Role::kDocument:
  171. return absl::optional<std::string>("document");
  172. case ax::mojom::Role::kFeed:
  173. return absl::optional<std::string>("feed");
  174. case ax::mojom::Role::kHeading:
  175. return absl::optional<std::string>("heading");
  176. case ax::mojom::Role::kIframe:
  177. return absl::optional<std::string>("iframe");
  178. case ax::mojom::Role::kIframePresentational:
  179. return absl::optional<std::string>("iframe_presentational");
  180. case ax::mojom::Role::kList:
  181. return absl::optional<std::string>("list");
  182. case ax::mojom::Role::kListItem:
  183. return absl::optional<std::string>("list_item");
  184. case ax::mojom::Role::kMain:
  185. return absl::optional<std::string>("main");
  186. case ax::mojom::Role::kParagraph:
  187. return absl::optional<std::string>("paragraph");
  188. default:
  189. return absl::optional<std::string>();
  190. }
  191. }
  192. AssistantNode* AddChild(AssistantTree* tree) {
  193. auto node = std::make_unique<AssistantNode>();
  194. tree->nodes.push_back(std::move(node));
  195. return tree->nodes.back().get();
  196. }
  197. struct WalkAXTreeConfig {
  198. bool should_select_leaf;
  199. };
  200. void WalkAXTreeDepthFirst(const AXNode* node,
  201. const gfx::Rect& rect,
  202. const AXTreeUpdate& update,
  203. const AXTree* tree,
  204. WalkAXTreeConfig* config,
  205. AssistantTree* assistant_tree,
  206. AssistantNode* result) {
  207. result->text = GetText(node);
  208. result->class_name =
  209. AXRoleToAndroidClassName(node->GetRole(), node->GetUnignoredParent());
  210. result->role = AXRoleToString(node->GetRole());
  211. result->text_size = -1.0;
  212. result->bgcolor = 0;
  213. result->color = 0;
  214. result->bold = false;
  215. result->italic = false;
  216. result->line_through = false;
  217. result->underline = false;
  218. if (node->HasFloatAttribute(ax::mojom::FloatAttribute::kFontSize)) {
  219. result->text_size =
  220. node->GetFloatAttribute(ax::mojom::FloatAttribute::kFontSize);
  221. result->color = node->GetIntAttribute(ax::mojom::IntAttribute::kColor);
  222. result->bgcolor =
  223. node->GetIntAttribute(ax::mojom::IntAttribute::kBackgroundColor);
  224. result->bold = node->HasTextStyle(ax::mojom::TextStyle::kBold);
  225. result->italic = node->HasTextStyle(ax::mojom::TextStyle::kItalic);
  226. result->line_through =
  227. node->HasTextStyle(ax::mojom::TextStyle::kLineThrough);
  228. result->underline = node->HasTextStyle(ax::mojom::TextStyle::kUnderline);
  229. }
  230. const gfx::Rect& absolute_rect =
  231. gfx::ToEnclosingRect(tree->GetTreeBounds(node));
  232. gfx::Rect parent_relative_rect = absolute_rect;
  233. bool is_root = !node->GetUnignoredParent();
  234. if (!is_root) {
  235. parent_relative_rect.Offset(-rect.OffsetFromOrigin());
  236. }
  237. result->rect = gfx::Rect(parent_relative_rect.x(), parent_relative_rect.y(),
  238. absolute_rect.width(), absolute_rect.height());
  239. if (IsLeaf(node) && update.has_tree_data) {
  240. int start_selection = 0;
  241. int end_selection = 0;
  242. AXTree::Selection unignored_selection = tree->GetUnignoredSelection();
  243. if (unignored_selection.anchor_object_id == node->id()) {
  244. start_selection = unignored_selection.anchor_offset;
  245. config->should_select_leaf = true;
  246. }
  247. if (config->should_select_leaf) {
  248. end_selection = static_cast<int32_t>(GetText(node).length());
  249. }
  250. if (unignored_selection.focus_object_id == node->id()) {
  251. end_selection = unignored_selection.focus_offset;
  252. config->should_select_leaf = false;
  253. }
  254. if (end_selection > 0)
  255. result->selection =
  256. absl::make_optional<gfx::Range>(start_selection, end_selection);
  257. }
  258. result->html_tag =
  259. node->GetStringAttribute(ax::mojom::StringAttribute::kHtmlTag);
  260. result->css_display =
  261. node->GetStringAttribute(ax::mojom::StringAttribute::kDisplay);
  262. result->html_attributes = node->GetHtmlAttributes();
  263. std::string class_name =
  264. node->GetStringAttribute(ax::mojom::StringAttribute::kClassName);
  265. if (!class_name.empty())
  266. result->html_attributes.push_back({"class", class_name});
  267. for (auto iter = node->UnignoredChildrenBegin();
  268. iter != node->UnignoredChildrenEnd(); ++iter) {
  269. auto* n = AddChild(assistant_tree);
  270. result->children_indices.push_back(assistant_tree->nodes.size() - 1);
  271. WalkAXTreeDepthFirst(iter.get(), absolute_rect, update, tree, config,
  272. assistant_tree, n);
  273. }
  274. }
  275. } // namespace
  276. AssistantNode::AssistantNode() = default;
  277. AssistantNode::AssistantNode(const AssistantNode& other) = default;
  278. AssistantNode::~AssistantNode() = default;
  279. AssistantTree::AssistantTree() = default;
  280. AssistantTree::~AssistantTree() = default;
  281. AssistantTree::AssistantTree(const AssistantTree& other) {
  282. for (const auto& node : other.nodes)
  283. nodes.emplace_back(std::make_unique<AssistantNode>(*node));
  284. }
  285. std::unique_ptr<AssistantTree> CreateAssistantTree(const AXTreeUpdate& update) {
  286. auto tree = std::make_unique<AXSerializableTree>();
  287. auto assistant_tree = std::make_unique<AssistantTree>();
  288. auto* root = AddChild(assistant_tree.get());
  289. if (!tree->Unserialize(update))
  290. LOG(FATAL) << tree->error();
  291. WalkAXTreeConfig config{
  292. false, // should_select_leaf
  293. };
  294. WalkAXTreeDepthFirst(tree->root(), gfx::Rect(), update, tree.get(), &config,
  295. assistant_tree.get(), root);
  296. return assistant_tree;
  297. }
  298. std::u16string AXUrlBaseText(std::u16string url) {
  299. // Given a url like http://foo.com/bar/baz.png, just return the
  300. // base text, e.g., "baz".
  301. int trailing_slashes = 0;
  302. while (url.size() - trailing_slashes > 0 &&
  303. url[url.size() - trailing_slashes - 1] == '/') {
  304. trailing_slashes++;
  305. }
  306. if (trailing_slashes)
  307. url = url.substr(0, url.size() - trailing_slashes);
  308. size_t slash_index = url.rfind('/');
  309. if (slash_index != std::string::npos)
  310. url = url.substr(slash_index + 1);
  311. size_t dot_index = url.rfind('.');
  312. if (dot_index != std::string::npos)
  313. url = url.substr(0, dot_index);
  314. return url;
  315. }
  316. const char* AXRoleToAndroidClassName(ax::mojom::Role role, bool has_parent) {
  317. switch (role) {
  318. case ax::mojom::Role::kSearchBox:
  319. case ax::mojom::Role::kSpinButton:
  320. case ax::mojom::Role::kTextField:
  321. case ax::mojom::Role::kTextFieldWithComboBox:
  322. return kAXEditTextClassname;
  323. case ax::mojom::Role::kSlider:
  324. return kAXSeekBarClassname;
  325. case ax::mojom::Role::kColorWell:
  326. case ax::mojom::Role::kComboBoxMenuButton:
  327. case ax::mojom::Role::kDate:
  328. case ax::mojom::Role::kDateTime:
  329. case ax::mojom::Role::kInputTime:
  330. return kAXSpinnerClassname;
  331. case ax::mojom::Role::kButton:
  332. case ax::mojom::Role::kPdfActionableHighlight:
  333. return kAXButtonClassname;
  334. case ax::mojom::Role::kCheckBox:
  335. return kAXCheckBoxClassname;
  336. case ax::mojom::Role::kRadioButton:
  337. return kAXRadioButtonClassname;
  338. case ax::mojom::Role::kRadioGroup:
  339. return kAXRadioGroupClassname;
  340. case ax::mojom::Role::kSwitch:
  341. case ax::mojom::Role::kToggleButton:
  342. return kAXToggleButtonClassname;
  343. case ax::mojom::Role::kCanvas:
  344. case ax::mojom::Role::kImage:
  345. case ax::mojom::Role::kSvgRoot:
  346. return kAXImageClassname;
  347. case ax::mojom::Role::kMeter:
  348. case ax::mojom::Role::kProgressIndicator:
  349. return kAXProgressBarClassname;
  350. case ax::mojom::Role::kTabList:
  351. return kAXTabWidgetClassname;
  352. case ax::mojom::Role::kGrid:
  353. case ax::mojom::Role::kTreeGrid:
  354. case ax::mojom::Role::kTable:
  355. return kAXGridViewClassname;
  356. case ax::mojom::Role::kList:
  357. case ax::mojom::Role::kListBox:
  358. case ax::mojom::Role::kDescriptionList:
  359. case ax::mojom::Role::kDirectory:
  360. return kAXListViewClassname;
  361. case ax::mojom::Role::kDialog:
  362. return kAXDialogClassname;
  363. case ax::mojom::Role::kRootWebArea:
  364. return has_parent ? kAXViewClassname : kAXWebViewClassname;
  365. case ax::mojom::Role::kMenuItem:
  366. case ax::mojom::Role::kMenuItemCheckBox:
  367. case ax::mojom::Role::kMenuItemRadio:
  368. return kAXMenuItemClassname;
  369. case ax::mojom::Role::kPre:
  370. case ax::mojom::Role::kStaticText:
  371. return kAXTextViewClassname;
  372. default:
  373. return kAXViewClassname;
  374. }
  375. }
  376. } // namespace ui