spellcheck_worditerator_unittest.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/spellcheck/renderer/spellcheck_worditerator.h"
  5. #include <stddef.h>
  6. #include <string>
  7. #include <vector>
  8. #include "base/format_macros.h"
  9. #include "base/i18n/break_iterator.h"
  10. #include "base/strings/string_split.h"
  11. #include "base/strings/stringprintf.h"
  12. #include "base/strings/utf_string_conversions.h"
  13. #include "testing/gtest/include/gtest/gtest.h"
  14. using base::i18n::BreakIterator;
  15. using WordIteratorStatus = SpellcheckWordIterator::WordIteratorStatus;
  16. namespace {
  17. struct TestCase {
  18. const char* language;
  19. bool allow_contraction;
  20. const wchar_t* expected_words;
  21. };
  22. std::u16string GetRulesForLanguage(const std::string& language) {
  23. SpellcheckCharAttribute attribute;
  24. attribute.SetDefaultLanguage(language);
  25. return attribute.GetRuleSet(true);
  26. }
  27. WordIteratorStatus GetNextNonSkippableWord(SpellcheckWordIterator* iterator,
  28. std::u16string* word_string,
  29. size_t* word_start,
  30. size_t* word_length) {
  31. WordIteratorStatus status = SpellcheckWordIterator::IS_SKIPPABLE;
  32. while (status == SpellcheckWordIterator::IS_SKIPPABLE)
  33. status = iterator->GetNextWord(word_string, word_start, word_length);
  34. return status;
  35. }
  36. } // namespace
  37. // Tests whether or not our SpellcheckWordIterator can extract words used by the
  38. // specified language from a multi-language text.
  39. TEST(SpellcheckWordIteratorTest, SplitWord) {
  40. // An input text. This text includes words of several languages. (Some words
  41. // are not separated with whitespace characters.) Our SpellcheckWordIterator
  42. // should extract the words used by the specified language from this text and
  43. // normalize them so our spell-checker can check their spellings. If
  44. // characters are found that are not from the specified language the test
  45. // skips them.
  46. const char16_t kTestText[] =
  47. // Graphic characters
  48. u"!@#$%^&*()"
  49. // Latin (including a contraction character and a ligature).
  50. u"hello:hello a\xFB03x"
  51. // Greek
  52. u"\x03B3\x03B5\x03B9\x03AC\x0020\x03C3\x03BF\x03C5"
  53. // Cyrillic
  54. u"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
  55. u"\x0443\x0439\x0442\x0435"
  56. // Hebrew (including niqquds)
  57. u"\x05e9\x05c1\x05b8\x05dc\x05d5\x05b9\x05dd "
  58. // Hebrew words with U+0027 and U+05F3
  59. u"\x05e6\x0027\x05d9\x05e4\x05e1 \x05e6\x05F3\x05d9\x05e4\x05e1 "
  60. // Hebrew words with U+0022 and U+05F4
  61. u"\x05e6\x05d4\x0022\x05dc \x05e6\x05d4\x05f4\x05dc "
  62. // Hebrew words enclosed with ASCII quotes.
  63. u"\"\x05e6\x05d4\x0022\x05dc\" '\x05e9\x05c1\x05b8\x05dc\x05d5'"
  64. // Arabic (including vowel marks)
  65. u"\x0627\x064e\x0644\x0633\x064e\x0651\x0644\x0627\x0645\x064f "
  66. u"\x0639\x064e\x0644\x064e\x064a\x0652\x0643\x064f\x0645\x0652 "
  67. // Farsi/Persian (including vowel marks)
  68. // Make sure \u064b - \u0652 are removed.
  69. u"\x0647\x0634\x064e\x0631\x062d "
  70. u"\x0647\x062e\x0648\x0627\x0647 "
  71. u"\x0650\x062f\x0631\x062f "
  72. u"\x0631\x0645\x0627\x0646\x0652 "
  73. u"\x0633\x0631\x0651 "
  74. u"\x0646\x0646\x064e\x062c\x064f\x0633 "
  75. u"\x0627\x0644\x062d\x0645\x062f "
  76. // Also make sure that class "Lm" (the \u0640) is filtered out too.
  77. u"\x062c\x062c\x0640\x062c\x062c"
  78. // Hindi
  79. u"\x0930\x093E\x091C\x0927\x093E\x0928"
  80. // Thai
  81. u"\x0e2a\x0e27\x0e31\x0e2a\x0e14\x0e35\x0020\x0e04"
  82. u"\x0e23\x0e31\x0e1a"
  83. // Hiraganas
  84. u"\x3053\x3093\x306B\x3061\x306F"
  85. // CJKV ideographs
  86. u"\x4F60\x597D"
  87. // Hangul Syllables
  88. u"\xC548\xB155\xD558\xC138\xC694"
  89. // Full-width latin : Hello
  90. u"\xFF28\xFF45\xFF4C\xFF4C\xFF4F "
  91. u"e.g.,";
  92. // The languages and expected results used in this test.
  93. static const TestCase kTestCases[] = {
  94. {
  95. // English (keep contraction words)
  96. "en-US", true, L"hello:hello affix Hello e.g"
  97. }, {
  98. // English (split contraction words)
  99. "en-US", false, L"hello hello affix Hello e g"
  100. }, {
  101. // Greek
  102. "el-GR", true,
  103. L"\x03B3\x03B5\x03B9\x03AC\x0020\x03C3\x03BF\x03C5"
  104. }, {
  105. // Russian
  106. "ru-RU", true,
  107. L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
  108. L"\x0443\x0439\x0442\x0435"
  109. }, {
  110. // Hebrew
  111. "he-IL", true,
  112. L"\x05e9\x05dc\x05d5\x05dd "
  113. L"\x05e6\x0027\x05d9\x05e4\x05e1 \x05e6\x05F3\x05d9\x05e4\x05e1 "
  114. L"\x05e6\x05d4\x0022\x05dc \x05e6\x05d4\x05f4\x05dc "
  115. L"\x05e6\x05d4\x0022\x05dc \x05e9\x05dc\x05d5"
  116. }, {
  117. // Arabic
  118. "ar", true,
  119. L"\x0627\x0644\x0633\x0644\x0627\x0645 "
  120. L"\x0639\x0644\x064a\x0643\x0645 "
  121. // Farsi/Persian
  122. L"\x0647\x0634\x0631\x062d "
  123. L"\x0647\x062e\x0648\x0627\x0647 "
  124. L"\x062f\x0631\x062f "
  125. L"\x0631\x0645\x0627\x0646 "
  126. L"\x0633\x0631 "
  127. L"\x0646\x0646\x062c\x0633 "
  128. L"\x0627\x0644\x062d\x0645\x062f "
  129. L"\x062c\x062c\x062c\x062c"
  130. }, {
  131. // Hindi
  132. "hi-IN", true,
  133. L"\x0930\x093E\x091C\x0927\x093E\x0928"
  134. }, {
  135. // Thai
  136. "th-TH", true,
  137. L"\x0e2a\x0e27\x0e31\x0e2a\x0e14\x0e35\x0020\x0e04"
  138. L"\x0e23\x0e31\x0e1a"
  139. }, {
  140. // Korean
  141. "ko-KR", true,
  142. L"\x110b\x1161\x11ab\x1102\x1167\x11bc\x1112\x1161"
  143. L"\x1109\x1166\x110b\x116d"
  144. },
  145. };
  146. for (size_t i = 0; i < std::size(kTestCases); ++i) {
  147. SCOPED_TRACE(base::StringPrintf("kTestCases[%" PRIuS "]: language=%s", i,
  148. kTestCases[i].language));
  149. SpellcheckCharAttribute attributes;
  150. attributes.SetDefaultLanguage(kTestCases[i].language);
  151. std::u16string input(kTestText);
  152. SpellcheckWordIterator iterator;
  153. EXPECT_TRUE(iterator.Initialize(&attributes,
  154. kTestCases[i].allow_contraction));
  155. EXPECT_TRUE(iterator.SetText(input.c_str(), input.length()));
  156. std::vector<std::u16string> expected_words = base::SplitString(
  157. base::WideToUTF16(kTestCases[i].expected_words), std::u16string(1, ' '),
  158. base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
  159. std::u16string actual_word;
  160. size_t actual_start, actual_len;
  161. size_t index = 0;
  162. for (SpellcheckWordIterator::WordIteratorStatus status =
  163. iterator.GetNextWord(&actual_word, &actual_start, &actual_len);
  164. status != SpellcheckWordIterator::IS_END_OF_TEXT;
  165. status =
  166. iterator.GetNextWord(&actual_word, &actual_start, &actual_len)) {
  167. if (status == SpellcheckWordIterator::WordIteratorStatus::IS_SKIPPABLE)
  168. continue;
  169. EXPECT_TRUE(index < expected_words.size());
  170. if (index < expected_words.size())
  171. EXPECT_EQ(expected_words[index], actual_word);
  172. ++index;
  173. }
  174. }
  175. }
  176. // Tests whether our SpellcheckWordIterator extracts an empty word without
  177. // getting stuck in an infinite loop when inputting a Khmer text. (This is a
  178. // regression test for Issue 46278.)
  179. TEST(SpellcheckWordIteratorTest, RuleSetConsistency) {
  180. SpellcheckCharAttribute attributes;
  181. attributes.SetDefaultLanguage("en-US");
  182. const char16_t kTestText[] = u"\x1791\x17c1\x002e";
  183. std::u16string input(kTestText);
  184. SpellcheckWordIterator iterator;
  185. EXPECT_TRUE(iterator.Initialize(&attributes, true));
  186. EXPECT_TRUE(iterator.SetText(input.c_str(), input.length()));
  187. // When SpellcheckWordIterator uses an inconsistent ICU ruleset, the following
  188. // iterator.GetNextWord() calls get stuck in an infinite loop. Therefore, this
  189. // test succeeds if this call returns without timeouts.
  190. std::u16string actual_word;
  191. size_t actual_start, actual_len;
  192. WordIteratorStatus status = GetNextNonSkippableWord(
  193. &iterator, &actual_word, &actual_start, &actual_len);
  194. EXPECT_EQ(SpellcheckWordIterator::WordIteratorStatus::IS_END_OF_TEXT, status);
  195. EXPECT_EQ(0u, actual_start);
  196. EXPECT_EQ(0u, actual_len);
  197. }
  198. // Vertify our SpellcheckWordIterator can treat ASCII numbers as word characters
  199. // on LTR languages. On the other hand, it should not treat ASCII numbers as
  200. // word characters on RTL languages because they change the text direction from
  201. // RTL to LTR.
  202. TEST(SpellcheckWordIteratorTest, TreatNumbersAsWordCharacters) {
  203. // A set of a language, a dummy word, and a text direction used in this test.
  204. // For each language, this test splits a dummy word, which consists of ASCII
  205. // numbers and an alphabet of the language, into words. When ASCII numbers are
  206. // treated as word characters, the split word becomes equal to the dummy word.
  207. // Otherwise, the split word does not include ASCII numbers.
  208. static const struct {
  209. const char* language;
  210. const wchar_t* text;
  211. bool left_to_right;
  212. } kTestCases[] = {
  213. {
  214. // English
  215. "en-US", L"0123456789" L"a", true,
  216. }, {
  217. // Greek
  218. "el-GR", L"0123456789" L"\x03B1", true,
  219. }, {
  220. // Russian
  221. "ru-RU", L"0123456789" L"\x0430", true,
  222. }, {
  223. // Hebrew
  224. "he-IL", L"0123456789" L"\x05D0", false,
  225. }, {
  226. // Arabic
  227. "ar", L"0123456789" L"\x0627", false,
  228. }, {
  229. // Hindi
  230. "hi-IN", L"0123456789" L"\x0905", true,
  231. }, {
  232. // Thai
  233. "th-TH", L"0123456789" L"\x0e01", true,
  234. }, {
  235. // Korean
  236. "ko-KR", L"0123456789" L"\x1100\x1161", true,
  237. },
  238. };
  239. for (size_t i = 0; i < std::size(kTestCases); ++i) {
  240. SCOPED_TRACE(base::StringPrintf("kTestCases[%" PRIuS "]: language=%s", i,
  241. kTestCases[i].language));
  242. SpellcheckCharAttribute attributes;
  243. attributes.SetDefaultLanguage(kTestCases[i].language);
  244. std::u16string input_word(base::WideToUTF16(kTestCases[i].text));
  245. SpellcheckWordIterator iterator;
  246. EXPECT_TRUE(iterator.Initialize(&attributes, true));
  247. EXPECT_TRUE(iterator.SetText(input_word.c_str(), input_word.length()));
  248. std::u16string actual_word;
  249. size_t actual_start, actual_len;
  250. WordIteratorStatus status = GetNextNonSkippableWord(
  251. &iterator, &actual_word, &actual_start, &actual_len);
  252. EXPECT_EQ(SpellcheckWordIterator::WordIteratorStatus::IS_WORD, status);
  253. if (kTestCases[i].left_to_right)
  254. EXPECT_EQ(input_word, actual_word);
  255. else
  256. EXPECT_NE(input_word, actual_word);
  257. }
  258. }
  259. // Verify SpellcheckWordIterator treats typographical apostrophe as a part of
  260. // the word.
  261. TEST(SpellcheckWordIteratorTest, TypographicalApostropheIsPartOfWord) {
  262. static const struct {
  263. const char* language;
  264. const wchar_t* input;
  265. const wchar_t* expected;
  266. } kTestCases[] = {
  267. // Typewriter apostrophe:
  268. {"en-AU", L"you're", L"you're"},
  269. {"en-CA", L"you're", L"you're"},
  270. {"en-GB", L"you're", L"you're"},
  271. {"en-US", L"you're", L"you're"},
  272. {"en-US", L"!!!!you're", L"you're"},
  273. // Typographical apostrophe:
  274. {"en-AU", L"you\x2019re", L"you\x2019re"},
  275. {"en-CA", L"you\x2019re", L"you\x2019re"},
  276. {"en-GB", L"you\x2019re", L"you\x2019re"},
  277. {"en-US", L"you\x2019re", L"you\x2019re"},
  278. {"en-US", L"....you\x2019re", L"you\x2019re"},
  279. };
  280. for (size_t i = 0; i < std::size(kTestCases); ++i) {
  281. SpellcheckCharAttribute attributes;
  282. attributes.SetDefaultLanguage(kTestCases[i].language);
  283. std::u16string input_word(base::WideToUTF16(kTestCases[i].input));
  284. std::u16string expected_word(base::WideToUTF16(kTestCases[i].expected));
  285. SpellcheckWordIterator iterator;
  286. EXPECT_TRUE(iterator.Initialize(&attributes, true));
  287. EXPECT_TRUE(iterator.SetText(input_word.c_str(), input_word.length()));
  288. std::u16string actual_word;
  289. size_t actual_start, actual_len;
  290. WordIteratorStatus status = GetNextNonSkippableWord(
  291. &iterator, &actual_word, &actual_start, &actual_len);
  292. EXPECT_EQ(SpellcheckWordIterator::WordIteratorStatus::IS_WORD, status);
  293. EXPECT_EQ(expected_word, actual_word);
  294. EXPECT_LE(0u, actual_start);
  295. EXPECT_EQ(expected_word.length(), actual_len);
  296. }
  297. }
  298. TEST(SpellcheckWordIteratorTest, Initialization) {
  299. // Test initialization works when a default language is set.
  300. {
  301. SpellcheckCharAttribute attributes;
  302. attributes.SetDefaultLanguage("en-US");
  303. SpellcheckWordIterator iterator;
  304. EXPECT_TRUE(iterator.Initialize(&attributes, true));
  305. }
  306. // Test initialization fails when no default language is set.
  307. {
  308. SpellcheckCharAttribute attributes;
  309. SpellcheckWordIterator iterator;
  310. EXPECT_FALSE(iterator.Initialize(&attributes, true));
  311. }
  312. }
  313. // This test uses English rules to check that different character set
  314. // combinations properly find word breaks and skippable characters.
  315. TEST(SpellcheckWordIteratorTest, FindSkippableWordsEnglish) {
  316. // A string containing the English word "foo", followed by two Khmer
  317. // characters, the English word "Can", and then two Russian characters and
  318. // punctuation.
  319. std::u16string text(u"foo \x1791\x17C1 Can \x041C\x0438...");
  320. BreakIterator iter(text, GetRulesForLanguage("en-US"));
  321. ASSERT_TRUE(iter.Init());
  322. EXPECT_TRUE(iter.Advance());
  323. // Finds "foo".
  324. EXPECT_EQ(u"foo", iter.GetString());
  325. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
  326. EXPECT_TRUE(iter.Advance());
  327. // Finds the space and then the Khmer characters.
  328. EXPECT_EQ(u" ", iter.GetString());
  329. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  330. EXPECT_TRUE(iter.Advance());
  331. EXPECT_EQ(u"\x1791\x17C1", iter.GetString());
  332. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  333. EXPECT_TRUE(iter.Advance());
  334. // Finds the next space and "Can".
  335. EXPECT_EQ(u" ", iter.GetString());
  336. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  337. EXPECT_TRUE(iter.Advance());
  338. EXPECT_EQ(u"Can", iter.GetString());
  339. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
  340. EXPECT_TRUE(iter.Advance());
  341. // Finds the next space and each Russian character.
  342. EXPECT_EQ(u" ", iter.GetString());
  343. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  344. EXPECT_TRUE(iter.Advance());
  345. EXPECT_EQ(u"\x041C", iter.GetString());
  346. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  347. EXPECT_TRUE(iter.Advance());
  348. EXPECT_EQ(u"\x0438", iter.GetString());
  349. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  350. EXPECT_TRUE(iter.Advance());
  351. // Finds the periods at the end.
  352. EXPECT_EQ(u".", iter.GetString());
  353. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  354. EXPECT_TRUE(iter.Advance());
  355. EXPECT_EQ(u".", iter.GetString());
  356. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  357. EXPECT_TRUE(iter.Advance());
  358. EXPECT_EQ(u".", iter.GetString());
  359. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  360. EXPECT_FALSE(iter.Advance());
  361. }
  362. // This test uses Russian rules to check that different character set
  363. // combinations properly find word breaks and skippable characters.
  364. TEST(SpellcheckWordIteratorTest, FindSkippableWordsRussian) {
  365. // A string containing punctuation followed by two Russian characters, the
  366. // English word "Can", and then two Khmer characters.
  367. std::u16string text(u".;\x041C\x0438 Can \x1791\x17C1 ");
  368. BreakIterator iter(text, GetRulesForLanguage("ru-RU"));
  369. ASSERT_TRUE(iter.Init());
  370. EXPECT_TRUE(iter.Advance());
  371. // Finds the period and semicolon.
  372. EXPECT_EQ(u".", iter.GetString());
  373. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  374. EXPECT_TRUE(iter.Advance());
  375. EXPECT_EQ(u";", iter.GetString());
  376. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  377. EXPECT_TRUE(iter.Advance());
  378. // Finds all the Russian characters.
  379. EXPECT_EQ(u"\x041C\x0438", iter.GetString());
  380. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
  381. EXPECT_TRUE(iter.Advance());
  382. // Finds the space and each character in "Can".
  383. EXPECT_EQ(u" ", iter.GetString());
  384. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  385. EXPECT_TRUE(iter.Advance());
  386. EXPECT_EQ(u"C", iter.GetString());
  387. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  388. EXPECT_TRUE(iter.Advance());
  389. EXPECT_EQ(u"a", iter.GetString());
  390. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  391. EXPECT_TRUE(iter.Advance());
  392. EXPECT_EQ(u"n", iter.GetString());
  393. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  394. EXPECT_TRUE(iter.Advance());
  395. // Finds the next space, the Khmer characters, and the last two spaces.
  396. EXPECT_EQ(u" ", iter.GetString());
  397. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  398. EXPECT_TRUE(iter.Advance());
  399. EXPECT_EQ(u"\x1791\x17C1", iter.GetString());
  400. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  401. EXPECT_TRUE(iter.Advance());
  402. EXPECT_EQ(u" ", iter.GetString());
  403. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  404. EXPECT_TRUE(iter.Advance());
  405. EXPECT_EQ(u" ", iter.GetString());
  406. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  407. EXPECT_FALSE(iter.Advance());
  408. }
  409. // This test uses Khmer rules to check that different character set combinations
  410. // properly find word breaks and skippable characters. Khmer does not use spaces
  411. // between words and uses a dictionary to determine word breaks instead.
  412. TEST(SpellcheckWordIteratorTest, FindSkippableWordsKhmer) {
  413. // A string containing two Russian characters followed by two, three, and
  414. // two-character Khmer words, and then English characters and punctuation.
  415. std::u16string text(
  416. u"\x041C\x0438 \x178F\x17BE\x179B\x17C4\x1780\x1798\x1780zoo. ,");
  417. BreakIterator iter(text, GetRulesForLanguage("km"));
  418. ASSERT_TRUE(iter.Init());
  419. EXPECT_TRUE(iter.Advance());
  420. // Finds each Russian character and the space.
  421. EXPECT_EQ(u"\x041C", iter.GetString());
  422. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  423. EXPECT_TRUE(iter.Advance());
  424. EXPECT_EQ(u"\x0438", iter.GetString());
  425. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  426. EXPECT_TRUE(iter.Advance());
  427. EXPECT_EQ(u" ", iter.GetString());
  428. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  429. EXPECT_TRUE(iter.Advance());
  430. // Finds the first two-character Khmer word.
  431. EXPECT_EQ(u"\x178F\x17BE", iter.GetString());
  432. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
  433. EXPECT_TRUE(iter.Advance());
  434. // Finds the three-character Khmer word and then the next two-character word.
  435. // Note: Technically these are two different Khmer words so the Khmer language
  436. // rule should find a break between them but due to the heuristic/statistical
  437. // nature of the Khmer word breaker it does not.
  438. EXPECT_EQ(u"\x179B\x17C4\x1780\x1798\x1780", iter.GetString());
  439. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_WORD_BREAK);
  440. EXPECT_TRUE(iter.Advance());
  441. // Finds each character in "zoo".
  442. EXPECT_EQ(u"z", iter.GetString());
  443. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  444. EXPECT_TRUE(iter.Advance());
  445. EXPECT_EQ(u"o", iter.GetString());
  446. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  447. EXPECT_TRUE(iter.Advance());
  448. EXPECT_EQ(u"o", iter.GetString());
  449. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  450. EXPECT_TRUE(iter.Advance());
  451. // Finds the period, space, and comma.
  452. EXPECT_EQ(u".", iter.GetString());
  453. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  454. EXPECT_TRUE(iter.Advance());
  455. EXPECT_EQ(u" ", iter.GetString());
  456. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  457. EXPECT_TRUE(iter.Advance());
  458. EXPECT_EQ(u",", iter.GetString());
  459. EXPECT_EQ(iter.GetWordBreakStatus(), BreakIterator::IS_SKIPPABLE_WORD);
  460. EXPECT_FALSE(iter.Advance());
  461. }
  462. TEST(SpellcheckCharAttributeTest, IsTextInSameScript) {
  463. struct LanguageWithSampleText {
  464. const char* language;
  465. const wchar_t* sample_text;
  466. };
  467. static const std::vector<LanguageWithSampleText> kLanguagesWithSampleText = {
  468. // Latin
  469. {"fr", L"Libert\x00e9, \x00e9galitt\x00e9, fraternit\x00e9."},
  470. // Greek
  471. {"el", L"\x03B3\x03B5\x03B9\x03AC\x0020\x03C3\x03BF\x03C5"},
  472. // Cyrillic
  473. {"ru",
  474. L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432\x0443\x0439\x0442"
  475. L"\x0435"},
  476. // Hebrew
  477. {"he", L"\x05e9\x05c1\x05b8\x05dc\x05d5\x05b9\x05dd"},
  478. // Arabic
  479. {"ar",
  480. L"\x0627\x064e\x0644\x0633\x064e\x0651\x0644\x0627\x0645\x064f\x0639"
  481. L"\x064e\x0644\x064e\x064a\x0652\x0643\x064f\x0645\x0652 "},
  482. // Hindi
  483. {"hi", L"\x0930\x093E\x091C\x0927\x093E\x0928"},
  484. // Thai
  485. {"th",
  486. L"\x0e2a\x0e27\x0e31\x0e2a\x0e14\x0e35\x0020\x0e04\x0e23\x0e31\x0e1a"},
  487. // Hiragata
  488. {"jp-Hira", L"\x3053\x3093\x306B\x3061\x306F"},
  489. // Katakana
  490. {"jp-Kana", L"\x30b3\x30de\x30fc\x30b9"},
  491. // CJKV ideographs
  492. {"zh-Hani", L"\x4F60\x597D"},
  493. // Hangul Syllables
  494. {"ko", L"\xC548\xB155\xD558\xC138\xC694"},
  495. };
  496. for (const auto& testcase : kLanguagesWithSampleText) {
  497. SpellcheckCharAttribute attribute;
  498. attribute.SetDefaultLanguage(testcase.language);
  499. std::u16string sample_text(base::WideToUTF16(testcase.sample_text));
  500. EXPECT_TRUE(attribute.IsTextInSameScript(sample_text))
  501. << "Language \"" << testcase.language
  502. << "\" fails to identify that sample text in same language is in same "
  503. "script.";
  504. // All other scripts in isolatation or mixed with current script should
  505. // return false.
  506. for (const auto& other_script : kLanguagesWithSampleText) {
  507. if (testcase.language == other_script.language)
  508. continue;
  509. std::u16string other_sample_text(
  510. base::WideToUTF16(other_script.sample_text));
  511. EXPECT_FALSE(attribute.IsTextInSameScript(other_sample_text))
  512. << "Language \"" << testcase.language
  513. << "\" fails to identify that sample text in language \""
  514. << other_script.language << "\" is in different script.";
  515. EXPECT_FALSE(
  516. attribute.IsTextInSameScript(sample_text + other_sample_text))
  517. << "Language \"" << testcase.language
  518. << "\" fails to identify that sample text in language \""
  519. << other_script.language
  520. << "\" is in different script when appended to text in this script.";
  521. EXPECT_FALSE(
  522. attribute.IsTextInSameScript(other_sample_text + sample_text))
  523. << "Language \"" << testcase.language
  524. << "\" fails to identify that sample text in language \""
  525. << other_script.language
  526. << "\" is in different script when prepended to text in this script.";
  527. }
  528. }
  529. }