spellcheck_common.cc 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/spellcheck/common/spellcheck_common.h"
  5. #include "base/check.h"
  6. #include "base/command_line.h"
  7. #include "base/containers/contains.h"
  8. #include "base/files/file_path.h"
  9. #include "base/metrics/field_trial.h"
  10. #include "base/strings/string_piece.h"
  11. #include "base/strings/string_util.h"
  12. #include "third_party/icu/source/common/unicode/uloc.h"
  13. #include "third_party/icu/source/common/unicode/urename.h"
  14. #include "third_party/icu/source/common/unicode/utypes.h"
  15. namespace spellcheck {
  16. struct LanguageRegion {
  17. const char* language; // The language.
  18. const char* language_region; // language & region, used by dictionaries.
  19. };
  20. struct LanguageVersion {
  21. const char* language; // The language input.
  22. const char* version; // The corresponding version.
  23. };
  24. static constexpr LanguageRegion kSupportedSpellCheckerLanguages[] = {
  25. // Several languages are not to be included in the spellchecker list:
  26. // th-TH, vi-VI.
  27. // clang-format off
  28. {"af", "af-ZA"},
  29. {"bg", "bg-BG"},
  30. {"ca", "ca-ES"},
  31. {"cs", "cs-CZ"},
  32. {"cy", "cy-GB"},
  33. {"da", "da-DK"},
  34. {"de", "de-DE"},
  35. {"de-DE", "de-DE"},
  36. {"el", "el-GR"},
  37. {"en", "en-US"},
  38. {"en-AU", "en-AU"},
  39. {"en-CA", "en-CA"},
  40. {"en-GB", "en-GB"},
  41. {"en-GB-oxendict", "en-GB-oxendict"},
  42. {"en-US", "en-US"},
  43. {"es", "es-ES"},
  44. {"es-419", "es-ES"},
  45. {"es-AR", "es-ES"},
  46. {"es-ES", "es-ES"},
  47. {"es-MX", "es-ES"},
  48. {"es-US", "es-ES"},
  49. {"et", "et-EE"},
  50. {"fa", "fa-IR"},
  51. {"fo", "fo-FO"},
  52. {"fr", "fr-FR"},
  53. {"fr-FR", "fr-FR"},
  54. {"he", "he-IL"},
  55. {"hi", "hi-IN"},
  56. {"hr", "hr-HR"},
  57. {"hu", "hu-HU"},
  58. {"hy", "hy"},
  59. {"id", "id-ID"},
  60. {"it", "it-IT"},
  61. {"it-IT", "it-IT"},
  62. {"ko", "ko"},
  63. {"lt", "lt-LT"},
  64. {"lv", "lv-LV"},
  65. {"nb", "nb-NO"},
  66. {"nl", "nl-NL"},
  67. {"pl", "pl-PL"},
  68. {"pt", "pt-BR"}, // based on kAliasMap from ui/base/l10n/l10n_util.cc
  69. {"pt-BR", "pt-BR"},
  70. {"pt-PT", "pt-PT"},
  71. {"ro", "ro-RO"},
  72. {"ru", "ru-RU"},
  73. {"sh", "sh"},
  74. {"sk", "sk-SK"},
  75. {"sl", "sl-SI"},
  76. {"sq", "sq"},
  77. {"sr", "sr"},
  78. {"sv", "sv-SE"},
  79. {"ta", "ta-IN"},
  80. {"tg", "tg-TG"},
  81. {"tr", "tr-TR"},
  82. {"uk", "uk-UA"},
  83. {"vi", "vi-VN"},
  84. // clang-format on
  85. };
  86. bool IsValidRegion(const std::string& region) {
  87. for (const auto& lang_region : kSupportedSpellCheckerLanguages) {
  88. if (lang_region.language_region == region)
  89. return true;
  90. }
  91. return false;
  92. }
  93. // This function returns the language-region version of language name.
  94. // e.g. returns hi-IN for hi.
  95. std::string GetSpellCheckLanguageRegion(base::StringPiece input_language) {
  96. for (const auto& lang_region : kSupportedSpellCheckerLanguages) {
  97. if (lang_region.language == input_language)
  98. return lang_region.language_region;
  99. }
  100. return std::string(input_language);
  101. }
  102. base::FilePath GetVersionedFileName(base::StringPiece input_language,
  103. const base::FilePath& dict_dir) {
  104. // The default dictionary version is 3-0. This version indicates that the bdic
  105. // file contains a checksum.
  106. static const char kDefaultVersionString[] = "-3-0";
  107. // Add non-default version strings here. Use the same version for all the
  108. // dictionaries that you add at the same time. Increment the major version
  109. // number if you're updating either dic or aff files. Increment the minor
  110. // version number if you're updating only dic_delta files.
  111. static constexpr LanguageVersion kSpecialVersionString[] = {
  112. // Jan 9, 2013: Add "FLAG num" to aff to avoid heapcheck crash.
  113. {"tr-TR", "-4-0"},
  114. // Mar 4, 2014: Add Tajik dictionary.
  115. {"tg-TG", "-5-0"},
  116. // Feb 2019: Initial check-in of Welsh.
  117. {"cy-GB", "-1-0"},
  118. // April 2019: Initial check-in of Armenian.
  119. {"hy", "-1-0"},
  120. // November 2019: Update Serbian-Latin and Serbian-Cyrillic
  121. {"sh", "-4-0"},
  122. {"sr", "-4-0"},
  123. // January 2020: Update fa-IR dictionaries from upstream.
  124. {"fa-IR", "-9-0"},
  125. // March 2022: Update en-* dictionaries from upstream and add "Kyiv" to
  126. // those dictionaries.
  127. {"en-AU", "-10-1"},
  128. {"en-CA", "-10-1"},
  129. {"en-GB", "-10-1"},
  130. {"en-GB-oxendict", "-10-1"},
  131. {"en-US", "-10-1"},
  132. // March 2022: Update uk-UA dictionary from upstream.
  133. {"uk-UA", "-5-0"},
  134. };
  135. // Generate the bdict file name using default version string or special
  136. // version string, depending on the language.
  137. std::string language = GetSpellCheckLanguageRegion(input_language);
  138. std::string version = kDefaultVersionString;
  139. for (const auto& lang_ver : kSpecialVersionString) {
  140. if (language == lang_ver.language) {
  141. version = lang_ver.version;
  142. break;
  143. }
  144. }
  145. std::string versioned_bdict_file_name(language + version + ".bdic");
  146. return dict_dir.AppendASCII(versioned_bdict_file_name);
  147. }
  148. std::string GetCorrespondingSpellCheckLanguage(base::StringPiece language) {
  149. std::string best_match;
  150. // Look for exact match in the Spell Check language list.
  151. for (const auto& lang_region : kSupportedSpellCheckerLanguages) {
  152. // First look for exact match in the language region of the list.
  153. if (lang_region.language == language)
  154. return std::string(language);
  155. // Next, look for exact match in the language_region part of the list.
  156. if (lang_region.language_region == language) {
  157. if (best_match.empty())
  158. best_match = lang_region.language;
  159. }
  160. }
  161. // No match found - return best match, if any.
  162. return best_match;
  163. }
  164. std::vector<std::string> SpellCheckLanguages() {
  165. std::vector<std::string> languages;
  166. for (const auto& lang_region : kSupportedSpellCheckerLanguages)
  167. languages.push_back(lang_region.language);
  168. return languages;
  169. }
  170. void GetISOLanguageCountryCodeFromLocale(const std::string& locale,
  171. std::string* language_code,
  172. std::string* country_code) {
  173. DCHECK(language_code);
  174. DCHECK(country_code);
  175. char language[ULOC_LANG_CAPACITY] = ULOC_ENGLISH;
  176. const char* country = "USA";
  177. if (!locale.empty()) {
  178. UErrorCode error = U_ZERO_ERROR;
  179. char id[ULOC_LANG_CAPACITY + ULOC_SCRIPT_CAPACITY + ULOC_COUNTRY_CAPACITY];
  180. uloc_addLikelySubtags(locale.c_str(), id, std::size(id), &error);
  181. error = U_ZERO_ERROR;
  182. uloc_getLanguage(id, language, std::size(language), &error);
  183. country = uloc_getISO3Country(id);
  184. }
  185. *language_code = std::string(language);
  186. *country_code = std::string(country);
  187. }
  188. void FillSuggestions(
  189. const std::vector<std::vector<std::u16string>>& suggestions_list,
  190. std::vector<std::u16string>* optional_suggestions) {
  191. DCHECK(optional_suggestions);
  192. size_t num_languages = suggestions_list.size();
  193. // Compute maximum number of suggestions in a single language.
  194. size_t max_suggestions = 0;
  195. for (const auto& suggestions : suggestions_list)
  196. max_suggestions = std::max(max_suggestions, suggestions.size());
  197. for (size_t count = 0; count < (max_suggestions * num_languages); ++count) {
  198. size_t language = count % num_languages;
  199. size_t index = count / num_languages;
  200. if (suggestions_list[language].size() <= index)
  201. continue;
  202. const std::u16string& suggestion = suggestions_list[language][index];
  203. // Only add the suggestion if it's unique.
  204. if (!base::Contains(*optional_suggestions, suggestion)) {
  205. optional_suggestions->push_back(suggestion);
  206. }
  207. if (optional_suggestions->size() >= kMaxSuggestions) {
  208. break;
  209. }
  210. }
  211. }
  212. } // namespace spellcheck