char_iterator_unittest.cc 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/i18n/char_iterator.h"
  5. #include "base/strings/utf_string_conversions.h"
  6. #include "testing/gtest/include/gtest/gtest.h"
  7. namespace base {
  8. namespace i18n {
  9. // This test string contains 4 characters:
  10. // x
  11. // u with circumflex - 2 bytes in UTF8, 1 codeword in UTF16
  12. // math double-struck A - 4 bytes in UTF8, 2 codewords in UTF16
  13. // z
  14. static const char* const kTestString = "x\u00FB\U0001D538z";
  15. TEST(CharIteratorsTest, TestUTF8) {
  16. std::string empty;
  17. UTF8CharIterator empty_iter(empty);
  18. EXPECT_TRUE(empty_iter.end());
  19. EXPECT_EQ(0u, empty_iter.array_pos());
  20. EXPECT_EQ(0u, empty_iter.char_pos());
  21. EXPECT_FALSE(empty_iter.Advance());
  22. std::string str("s\303\273r"); // [u with circumflex]
  23. UTF8CharIterator iter(str);
  24. EXPECT_FALSE(iter.end());
  25. EXPECT_EQ(0u, iter.array_pos());
  26. EXPECT_EQ(0u, iter.char_pos());
  27. EXPECT_EQ('s', iter.get());
  28. EXPECT_TRUE(iter.Advance());
  29. EXPECT_FALSE(iter.end());
  30. EXPECT_EQ(1u, iter.array_pos());
  31. EXPECT_EQ(1u, iter.char_pos());
  32. EXPECT_EQ(251, iter.get());
  33. EXPECT_TRUE(iter.Advance());
  34. EXPECT_FALSE(iter.end());
  35. EXPECT_EQ(3u, iter.array_pos());
  36. EXPECT_EQ(2u, iter.char_pos());
  37. EXPECT_EQ('r', iter.get());
  38. EXPECT_TRUE(iter.Advance());
  39. EXPECT_TRUE(iter.end());
  40. EXPECT_EQ(4u, iter.array_pos());
  41. EXPECT_EQ(3u, iter.char_pos());
  42. // Don't care what it returns, but this shouldn't crash
  43. iter.get();
  44. EXPECT_FALSE(iter.Advance());
  45. }
  46. TEST(CharIteratorsTest, TestUTF16_Empty) {
  47. std::u16string empty;
  48. UTF16CharIterator empty_iter(empty);
  49. EXPECT_TRUE(empty_iter.end());
  50. EXPECT_TRUE(empty_iter.start());
  51. EXPECT_EQ(0u, empty_iter.array_pos());
  52. EXPECT_EQ(0, empty_iter.char_offset());
  53. EXPECT_FALSE(empty_iter.Advance());
  54. // These shouldn't crash.
  55. empty_iter.get();
  56. empty_iter.NextCodePoint();
  57. empty_iter.PreviousCodePoint();
  58. }
  59. TEST(CharIteratorsTest, TestUTF16) {
  60. std::u16string str = UTF8ToUTF16(kTestString);
  61. UTF16CharIterator iter(str);
  62. EXPECT_FALSE(iter.end());
  63. EXPECT_TRUE(iter.start());
  64. EXPECT_EQ(0u, iter.array_pos());
  65. EXPECT_EQ(0, iter.char_offset());
  66. EXPECT_EQ('x', iter.get());
  67. // This shouldn't crash.
  68. iter.PreviousCodePoint();
  69. EXPECT_EQ(0xFB, iter.NextCodePoint());
  70. EXPECT_TRUE(iter.Advance());
  71. EXPECT_FALSE(iter.end());
  72. EXPECT_FALSE(iter.start());
  73. EXPECT_EQ(1u, iter.array_pos());
  74. EXPECT_EQ(1, iter.char_offset());
  75. EXPECT_EQ(0xFB, iter.get());
  76. EXPECT_EQ('x', iter.PreviousCodePoint());
  77. EXPECT_EQ(0x1D538, iter.NextCodePoint());
  78. EXPECT_TRUE(iter.Advance());
  79. EXPECT_FALSE(iter.end());
  80. EXPECT_FALSE(iter.start());
  81. EXPECT_EQ(2u, iter.array_pos());
  82. EXPECT_EQ(2, iter.char_offset());
  83. EXPECT_EQ(0x1D538, iter.get());
  84. EXPECT_EQ(0xFB, iter.PreviousCodePoint());
  85. EXPECT_EQ('z', iter.NextCodePoint());
  86. EXPECT_TRUE(iter.Advance());
  87. EXPECT_FALSE(iter.end());
  88. EXPECT_FALSE(iter.start());
  89. EXPECT_EQ(4u, iter.array_pos());
  90. EXPECT_EQ(3, iter.char_offset());
  91. EXPECT_EQ('z', iter.get());
  92. EXPECT_EQ(0x1D538, iter.PreviousCodePoint());
  93. // This shouldn't crash.
  94. iter.NextCodePoint();
  95. EXPECT_TRUE(iter.Advance());
  96. EXPECT_TRUE(iter.end());
  97. EXPECT_FALSE(iter.start());
  98. EXPECT_EQ(5u, iter.array_pos());
  99. EXPECT_EQ(4, iter.char_offset());
  100. EXPECT_EQ('z', iter.PreviousCodePoint());
  101. // Don't care what it returns, but these shouldn't crash
  102. iter.get();
  103. iter.NextCodePoint();
  104. EXPECT_FALSE(iter.Advance());
  105. }
  106. TEST(CharIteratorsTest, TestUTF16_Rewind) {
  107. std::u16string str = UTF8ToUTF16(kTestString);
  108. // It is valid for the starting array index to be on the terminating null
  109. // character; in fact, this is where end() reports true. So we'll start on the
  110. // terminator for this test so we can check the behavior of end().
  111. UTF16CharIterator iter = UTF16CharIterator::UpperBound(str, str.length());
  112. EXPECT_TRUE(iter.end());
  113. EXPECT_FALSE(iter.start());
  114. // This is the index of the terminating null character, and the length of the
  115. // string in char16s.
  116. EXPECT_EQ(5u, iter.array_pos());
  117. EXPECT_EQ(0, iter.char_offset());
  118. EXPECT_EQ('z', iter.PreviousCodePoint());
  119. // Don't care what it returns, but these shouldn't crash
  120. iter.get();
  121. iter.NextCodePoint();
  122. EXPECT_TRUE(iter.Rewind());
  123. EXPECT_FALSE(iter.end());
  124. EXPECT_FALSE(iter.start());
  125. EXPECT_EQ(4u, iter.array_pos());
  126. EXPECT_EQ(-1, iter.char_offset());
  127. EXPECT_EQ('z', iter.get());
  128. EXPECT_EQ(0x1D538, iter.PreviousCodePoint());
  129. // This shouldn't crash.
  130. iter.NextCodePoint();
  131. EXPECT_TRUE(iter.Rewind());
  132. EXPECT_FALSE(iter.end());
  133. EXPECT_FALSE(iter.start());
  134. EXPECT_EQ(2u, iter.array_pos());
  135. EXPECT_EQ(-2, iter.char_offset());
  136. EXPECT_EQ(0x1D538, iter.get());
  137. EXPECT_EQ(0xFB, iter.PreviousCodePoint());
  138. EXPECT_EQ('z', iter.NextCodePoint());
  139. EXPECT_TRUE(iter.Rewind());
  140. EXPECT_FALSE(iter.end());
  141. EXPECT_FALSE(iter.start());
  142. EXPECT_EQ(1u, iter.array_pos());
  143. EXPECT_EQ(-3, iter.char_offset());
  144. EXPECT_EQ(0xFB, iter.get());
  145. EXPECT_EQ('x', iter.PreviousCodePoint());
  146. EXPECT_EQ(0x1D538, iter.NextCodePoint());
  147. EXPECT_TRUE(iter.Rewind());
  148. EXPECT_FALSE(iter.end());
  149. EXPECT_TRUE(iter.start());
  150. EXPECT_EQ(0u, iter.array_pos());
  151. EXPECT_EQ(-4, iter.char_offset());
  152. EXPECT_EQ('x', iter.get());
  153. EXPECT_EQ(0xFB, iter.NextCodePoint());
  154. // This shouldn't crash.
  155. iter.PreviousCodePoint();
  156. EXPECT_FALSE(iter.Rewind());
  157. }
  158. TEST(CharIteratorsTest, TestUTF16_UpperBound) {
  159. std::u16string str = UTF8ToUTF16(kTestString);
  160. ASSERT_EQ(0u, UTF16CharIterator::UpperBound(str, 0).array_pos());
  161. ASSERT_EQ(1u, UTF16CharIterator::UpperBound(str, 1).array_pos());
  162. ASSERT_EQ(2u, UTF16CharIterator::UpperBound(str, 2).array_pos());
  163. ASSERT_EQ(4u, UTF16CharIterator::UpperBound(str, 3).array_pos());
  164. ASSERT_EQ(4u, UTF16CharIterator::UpperBound(str, 4).array_pos());
  165. ASSERT_EQ(5u, UTF16CharIterator::UpperBound(str, 5).array_pos());
  166. }
  167. TEST(CharIteratorsTest, TestUTF16_LowerBound) {
  168. std::u16string str = UTF8ToUTF16(kTestString);
  169. ASSERT_EQ(0u, UTF16CharIterator::LowerBound(str, 0).array_pos());
  170. ASSERT_EQ(1u, UTF16CharIterator::LowerBound(str, 1).array_pos());
  171. ASSERT_EQ(2u, UTF16CharIterator::LowerBound(str, 2).array_pos());
  172. ASSERT_EQ(2u, UTF16CharIterator::LowerBound(str, 3).array_pos());
  173. ASSERT_EQ(4u, UTF16CharIterator::LowerBound(str, 4).array_pos());
  174. ASSERT_EQ(5u, UTF16CharIterator::LowerBound(str, 5).array_pos());
  175. }
  176. } // namespace i18n
  177. } // namespace base