spellcheck_worditerator.h 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. // Defines an iterator class that enumerates words supported by our spellchecker
  5. // from multi-language text. This class is used for filtering out characters
  6. // not supported by our spellchecker.
  7. #ifndef COMPONENTS_SPELLCHECK_RENDERER_SPELLCHECK_WORDITERATOR_H_
  8. #define COMPONENTS_SPELLCHECK_RENDERER_SPELLCHECK_WORDITERATOR_H_
  9. #include <stddef.h>
  10. #include <memory>
  11. #include <string>
  12. #include "third_party/icu/source/common/unicode/uscript.h"
  13. namespace base {
  14. namespace i18n {
  15. class BreakIterator;
  16. } // namespace i18n
  17. } // namespace base
  18. // A class which encapsulates language-specific operations used by
  19. // SpellcheckWordIterator. When we set the spellchecker language, this class
  20. // creates rule sets that filter out the characters not supported by the
  21. // spellchecker. (Please read the comment in the SpellcheckWordIterator class
  22. // about how to use this class.)
  23. class SpellcheckCharAttribute {
  24. public:
  25. SpellcheckCharAttribute();
  26. SpellcheckCharAttribute(const SpellcheckCharAttribute&) = delete;
  27. SpellcheckCharAttribute& operator=(const SpellcheckCharAttribute&) = delete;
  28. ~SpellcheckCharAttribute();
  29. // Sets the language of the spellchecker. When this function is called with an
  30. // ISO language code, this function creates the custom rule-sets used by
  31. // the ICU break iterator so it can extract only words used by the language.
  32. // GetRuleSet() returns the rule-sets created in this function.
  33. void SetDefaultLanguage(const std::string& language);
  34. // Returns true if all the characters in a text string are in the script
  35. // associated with the spellcheck language.
  36. bool IsTextInSameScript(const std::u16string& text) const;
  37. // Returns a custom rule-set string used by the ICU break iterator. This class
  38. // has two rule-sets, one splits a contraction and the other does not, so we
  39. // can split a concaticated word (e.g. "seven-year-old") into words (e.g.
  40. // "seven", "year", and "old") and check their spellings. The result stirng is
  41. // encoded in UTF-16 since ICU needs UTF-16 strings.
  42. std::u16string GetRuleSet(bool allow_contraction) const;
  43. // Outputs a character only if it is a word character. (Please read the
  44. // comments in CreateRuleSets() why we need this function.)
  45. bool OutputChar(UChar c, std::u16string* output) const;
  46. private:
  47. // Creates the rule-sets that return words possibly used by the given
  48. // language. Unfortunately, these rule-sets are not perfect and have some
  49. // false-positives. For example, they return combined accent marks even though
  50. // we need English words only. We call OutputCharacter() to filter out such
  51. // false-positive characters.
  52. void CreateRuleSets(const std::string& language);
  53. // Outputs a character only if it is one used by the given language. These
  54. // functions are called from OutputChar().
  55. bool OutputArabic(UChar c, std::u16string* output) const;
  56. bool OutputHangul(UChar c, std::u16string* output) const;
  57. bool OutputHebrew(UChar c, std::u16string* output) const;
  58. bool OutputDefault(UChar c, std::u16string* output) const;
  59. // The custom rule-set strings used by ICU break iterator. Since it is not so
  60. // easy to create custom rule-sets from an ISO language code, this class
  61. // saves these rule-set strings created when we set the language.
  62. std::u16string ruleset_allow_contraction_;
  63. std::u16string ruleset_disallow_contraction_;
  64. // The script code used by this language.
  65. UScriptCode script_code_;
  66. };
  67. // A class which extracts words that can be checked for spelling from a
  68. // multi-language string. The ICU word-break iterator does not discard some
  69. // punctuation characters attached to a word. For example, when we set a word
  70. // "_hello_" to a word-break iterator, it just returns "_hello_". Neither does
  71. // it discard characters not used by the language. For example, it returns
  72. // Russian words even though we need English words only. To extract only the
  73. // words that our spellchecker can check their spellings, this class uses custom
  74. // rule-sets created by the SpellcheckCharAttribute class. Also, this class
  75. // normalizes extracted words so our spellchecker can check the spellings of
  76. // words that include ligatures, combined characters, full-width characters,
  77. // etc. This class uses UTF-16 strings as its input and output strings since
  78. // UTF-16 is the native encoding of ICU and avoid unnecessary conversions
  79. // when changing the encoding of this string for our spellchecker. (Chrome can
  80. // use two or more spellcheckers and we cannot assume their encodings.)
  81. // The following snippet is an example that extracts words with this class.
  82. //
  83. // // Creates the language-specific attributes for US English.
  84. // SpellcheckCharAttribute attribute;
  85. // attribute.SetDefaultLanguage("en-US");
  86. //
  87. // // Set up a SpellcheckWordIterator object which extracts English words,
  88. // // and retrieve them.
  89. // SpellcheckWordIterator iterator;
  90. // std::u16string text(u"this is a test.");
  91. // iterator.Initialize(&attribute, true);
  92. // iterator.SetText(text.c_str(), text_.length());
  93. //
  94. // std::u16string word;
  95. // int offset;
  96. // int length;
  97. // while (iterator.GetNextWord(&word, &offset, &length)) {
  98. // ...
  99. // }
  100. //
  101. class SpellcheckWordIterator {
  102. public:
  103. enum WordIteratorStatus {
  104. // The end of a sequence of text that the iterator recognizes as characters
  105. // that can form a word.
  106. IS_WORD,
  107. // Non-word characters that the iterator can skip past, such as punctuation,
  108. // whitespace, and characters from another character set.
  109. IS_SKIPPABLE,
  110. // The end of the text that the iterator is going over.
  111. IS_END_OF_TEXT
  112. };
  113. SpellcheckWordIterator();
  114. SpellcheckWordIterator(const SpellcheckWordIterator&) = delete;
  115. SpellcheckWordIterator& operator=(const SpellcheckWordIterator&) = delete;
  116. ~SpellcheckWordIterator();
  117. // Initializes a word-iterator object with the language-specific attribute. If
  118. // we need to split contractions and concatenated words, call this function
  119. // with its 'allow_contraction' parameter false. (This function uses lots of
  120. // temporal memory to compile a custom word-break rule into an automaton.)
  121. bool Initialize(const SpellcheckCharAttribute* attribute,
  122. bool allow_contraction);
  123. // Returns whether this word iterator is initialized.
  124. bool IsInitialized() const;
  125. // Set text to be iterated. (This text does not have to be NULL-terminated.)
  126. // This function also resets internal state so we can reuse this iterator
  127. // without calling Initialize().
  128. bool SetText(const char16_t* text, size_t length);
  129. // Advances |iterator_| through |text_| and gets the current status of the
  130. // word iterator within |text|:
  131. //
  132. // - Returns IS_WORD if the iterator just found the end of a sequence of word
  133. // characters and it was able to normalize the sequence. This stores the
  134. // normalized string into |word_string| and stores the position and length
  135. // into |word_start| and |word_length| respectively. Keep in mind that
  136. // since this function normalizes the output word, the length of
  137. // |word_string| may be different from the |word_length|. Therefore, when
  138. // we call functions that change the input text, such as
  139. // string16::replace(), we need to use |word_start| and |word_length| as
  140. // listed in the following snippet:
  141. //
  142. // while(iterator.GetNextWord(&word, &offset, &length))
  143. // text.replace(offset, length, word);
  144. //
  145. // - Returns IS_SKIPPABLE if the iterator just found a character that the
  146. // iterator can skip past such as punctuation, whitespace, and characters
  147. // from another character set. This stores the character, position, and
  148. // length into |word_string|, |word_start|, and |word_length| respectively.
  149. //
  150. // - Returns IS_END_OF_TEXT if the iterator has reached the end of |text_|.
  151. SpellcheckWordIterator::WordIteratorStatus GetNextWord(
  152. std::u16string* word_string,
  153. size_t* word_start,
  154. size_t* word_length);
  155. // Releases all the resources attached to this object.
  156. void Reset();
  157. private:
  158. // Normalizes a non-terminated string returned from an ICU word-break
  159. // iterator. A word returned from an ICU break iterator may include characters
  160. // not supported by our spellchecker, e.g. ligatures, combining/ characters,
  161. // full-width letters, etc. This function replaces such characters with
  162. // alternative characters supported by our spellchecker. This function also
  163. // calls SpellcheckWordIterator::OutputChar() to filter out false-positive
  164. // characters.
  165. bool Normalize(size_t input_start,
  166. size_t input_length,
  167. std::u16string* output_string) const;
  168. // The pointer to the input string from which we are extracting words.
  169. const char16_t* text_;
  170. // The language-specific attributes used for filtering out non-word
  171. // characters.
  172. const SpellcheckCharAttribute* attribute_;
  173. // The break iterator.
  174. std::unique_ptr<base::i18n::BreakIterator> iterator_;
  175. };
  176. #endif // COMPONENTS_SPELLCHECK_RENDERER_SPELLCHECK_WORDITERATOR_H_