utf_offset_string_conversions_unittest.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/strings/utf_offset_string_conversions.h"
  5. #include <stddef.h>
  6. #include <algorithm>
  7. #include "base/strings/string_piece.h"
  8. #include "testing/gtest/include/gtest/gtest.h"
  9. namespace base {
  10. namespace {
  11. static const size_t kNpos = std::u16string::npos;
  12. } // namespace
  13. TEST(UTFOffsetStringConversionsTest, AdjustOffset) {
  14. struct UTF8ToUTF16Case {
  15. const char* utf8;
  16. size_t input_offset;
  17. size_t output_offset;
  18. } utf8_to_utf16_cases[] = {
  19. {"", 0, 0},
  20. {"", kNpos, kNpos},
  21. {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, kNpos},
  22. {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1},
  23. {"\xed\xb0\x80z", 3, 3},
  24. {"A\xF0\x90\x8C\x80z", 1, 1},
  25. {"A\xF0\x90\x8C\x80z", 2, kNpos},
  26. {"A\xF0\x90\x8C\x80z", 5, 3},
  27. {"A\xF0\x90\x8C\x80z", 6, 4},
  28. {"A\xF0\x90\x8C\x80z", kNpos, kNpos},
  29. };
  30. for (const auto& i : utf8_to_utf16_cases) {
  31. const size_t offset = i.input_offset;
  32. std::vector<size_t> offsets;
  33. offsets.push_back(offset);
  34. UTF8ToUTF16AndAdjustOffsets(i.utf8, &offsets);
  35. EXPECT_EQ(i.output_offset, offsets[0]);
  36. }
  37. struct UTF16ToUTF8Case {
  38. char16_t utf16[10];
  39. size_t input_offset;
  40. size_t output_offset;
  41. } utf16_to_utf8_cases[] = {
  42. {{}, 0, 0},
  43. // Converted to 3-byte utf-8 sequences
  44. {{0x5909, 0x63DB}, 3, kNpos},
  45. {{0x5909, 0x63DB}, 2, 6},
  46. {{0x5909, 0x63DB}, 1, 3},
  47. {{0x5909, 0x63DB}, 0, 0},
  48. // Converted to 2-byte utf-8 sequences
  49. {{'A', 0x00bc, 0x00be, 'z'}, 1, 1},
  50. {{'A', 0x00bc, 0x00be, 'z'}, 2, 3},
  51. {{'A', 0x00bc, 0x00be, 'z'}, 3, 5},
  52. {{'A', 0x00bc, 0x00be, 'z'}, 4, 6},
  53. // Surrogate pair
  54. {{'A', 0xd800, 0xdf00, 'z'}, 1, 1},
  55. {{'A', 0xd800, 0xdf00, 'z'}, 2, kNpos},
  56. {{'A', 0xd800, 0xdf00, 'z'}, 3, 5},
  57. {{'A', 0xd800, 0xdf00, 'z'}, 4, 6},
  58. };
  59. for (size_t i = 0; i < std::size(utf16_to_utf8_cases); ++i) {
  60. size_t offset = utf16_to_utf8_cases[i].input_offset;
  61. std::vector<size_t> offsets;
  62. offsets.push_back(offset);
  63. UTF16ToUTF8AndAdjustOffsets(utf16_to_utf8_cases[i].utf16, &offsets);
  64. EXPECT_EQ(utf16_to_utf8_cases[i].output_offset, offsets[0]) << i;
  65. }
  66. }
  67. TEST(UTFOffsetStringConversionsTest, LimitOffsets) {
  68. const OffsetAdjuster::Adjustments kNoAdjustments;
  69. const size_t kLimit = 10;
  70. const size_t kItems = 20;
  71. std::vector<size_t> size_ts;
  72. for (size_t t = 0; t < kItems; ++t) {
  73. size_ts.push_back(t);
  74. OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
  75. }
  76. size_t unlimited_count = 0;
  77. for (auto ti : size_ts) {
  78. if (ti != kNpos)
  79. ++unlimited_count;
  80. }
  81. EXPECT_EQ(11U, unlimited_count);
  82. // Reverse the values in the vector and try again.
  83. size_ts.clear();
  84. for (size_t t = kItems; t > 0; --t) {
  85. size_ts.push_back(t - 1);
  86. OffsetAdjuster::AdjustOffset(kNoAdjustments, &size_ts.back(), kLimit);
  87. }
  88. unlimited_count = 0;
  89. for (auto ti : size_ts) {
  90. if (ti != kNpos)
  91. ++unlimited_count;
  92. }
  93. EXPECT_EQ(11U, unlimited_count);
  94. }
  95. TEST(UTFOffsetStringConversionsTest, AdjustOffsets) {
  96. // Imagine we have strings as shown in the following cases where the
  97. // X's represent encoded characters.
  98. // 1: abcXXXdef ==> abcXdef
  99. {
  100. std::vector<size_t> offsets;
  101. for (size_t t = 0; t <= 9; ++t)
  102. offsets.push_back(t);
  103. OffsetAdjuster::Adjustments adjustments;
  104. adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
  105. OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
  106. size_t expected_1[] = {0, 1, 2, 3, kNpos, kNpos, 4, 5, 6, 7};
  107. EXPECT_EQ(offsets.size(), std::size(expected_1));
  108. for (size_t i = 0; i < std::size(expected_1); ++i)
  109. EXPECT_EQ(expected_1[i], offsets[i]);
  110. }
  111. // 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
  112. {
  113. std::vector<size_t> offsets;
  114. for (size_t t = 0; t <= 23; ++t)
  115. offsets.push_back(t);
  116. OffsetAdjuster::Adjustments adjustments;
  117. adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
  118. adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
  119. adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
  120. adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
  121. OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
  122. size_t expected_2[] = {
  123. 0, kNpos, kNpos, 1, 2, kNpos, kNpos, kNpos, 4, 5, 6, kNpos, kNpos, kNpos,
  124. kNpos, kNpos, kNpos, 10, 11, 12, 13, kNpos, kNpos, 14
  125. };
  126. EXPECT_EQ(offsets.size(), std::size(expected_2));
  127. for (size_t i = 0; i < std::size(expected_2); ++i)
  128. EXPECT_EQ(expected_2[i], offsets[i]);
  129. }
  130. // 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
  131. {
  132. std::vector<size_t> offsets;
  133. for (size_t t = 0; t <= 17; ++t)
  134. offsets.push_back(t);
  135. OffsetAdjuster::Adjustments adjustments;
  136. adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
  137. adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
  138. adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
  139. adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
  140. OffsetAdjuster::AdjustOffsets(adjustments, &offsets);
  141. size_t expected_3[] = {
  142. 0, kNpos, kNpos, 0, 1, kNpos, kNpos, kNpos, 5, 6, 7, 8, kNpos, kNpos, 11,
  143. 12, kNpos, 12
  144. };
  145. EXPECT_EQ(offsets.size(), std::size(expected_3));
  146. for (size_t i = 0; i < std::size(expected_3); ++i)
  147. EXPECT_EQ(expected_3[i], offsets[i]);
  148. }
  149. }
  150. TEST(UTFOffsetStringConversionsTest, UnadjustOffsets) {
  151. // Imagine we have strings as shown in the following cases where the
  152. // X's represent encoded characters.
  153. // 1: abcXXXdef ==> abcXdef
  154. {
  155. std::vector<size_t> offsets;
  156. for (size_t t = 0; t <= 7; ++t)
  157. offsets.push_back(t);
  158. OffsetAdjuster::Adjustments adjustments;
  159. adjustments.push_back(OffsetAdjuster::Adjustment(3, 3, 1));
  160. OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
  161. size_t expected_1[] = {0, 1, 2, 3, 6, 7, 8, 9};
  162. EXPECT_EQ(offsets.size(), std::size(expected_1));
  163. for (size_t i = 0; i < std::size(expected_1); ++i)
  164. EXPECT_EQ(expected_1[i], offsets[i]);
  165. }
  166. // 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX
  167. {
  168. std::vector<size_t> offsets;
  169. for (size_t t = 0; t <= 14; ++t)
  170. offsets.push_back(t);
  171. OffsetAdjuster::Adjustments adjustments;
  172. adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 1));
  173. adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 2));
  174. adjustments.push_back(OffsetAdjuster::Adjustment(10, 7, 4));
  175. adjustments.push_back(OffsetAdjuster::Adjustment(20, 3, 1));
  176. OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
  177. size_t expected_2[] = {
  178. 0, 3, 4, kNpos, 8, 9, 10, kNpos, kNpos, kNpos, 17, 18, 19, 20, 23
  179. };
  180. EXPECT_EQ(offsets.size(), std::size(expected_2));
  181. for (size_t i = 0; i < std::size(expected_2); ++i)
  182. EXPECT_EQ(expected_2[i], offsets[i]);
  183. }
  184. // 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe
  185. {
  186. std::vector<size_t> offsets;
  187. for (size_t t = 0; t <= 12; ++t)
  188. offsets.push_back(t);
  189. OffsetAdjuster::Adjustments adjustments;
  190. adjustments.push_back(OffsetAdjuster::Adjustment(0, 3, 0));
  191. adjustments.push_back(OffsetAdjuster::Adjustment(4, 4, 4));
  192. adjustments.push_back(OffsetAdjuster::Adjustment(11, 3, 3));
  193. adjustments.push_back(OffsetAdjuster::Adjustment(15, 2, 0));
  194. OffsetAdjuster::UnadjustOffsets(adjustments, &offsets);
  195. size_t expected_3[] = {
  196. 0, // this could just as easily be 3
  197. 4, kNpos, kNpos, kNpos, 8, 9, 10, 11, kNpos, kNpos, 14,
  198. 15 // this could just as easily be 17
  199. };
  200. EXPECT_EQ(offsets.size(), std::size(expected_3));
  201. for (size_t i = 0; i < std::size(expected_3); ++i)
  202. EXPECT_EQ(expected_3[i], offsets[i]);
  203. }
  204. }
  205. // MergeSequentialAdjustments is used by net/base/escape.{h,cc} and
  206. // net/base/net_util.{h,cc}. The two tests EscapeTest.AdjustOffset and
  207. // NetUtilTest.FormatUrlWithOffsets test its behavior extensively. This
  208. // is simply a short, additional test.
  209. TEST(UTFOffsetStringConversionsTest, MergeSequentialAdjustments) {
  210. // Pretend the input string is "abcdefghijklmnopqrstuvwxyz".
  211. // Set up |first_adjustments| to
  212. // - remove the leading "a"
  213. // - combine the "bc" into one character (call it ".")
  214. // - remove the "f"
  215. // - remove the "tuv"
  216. // The resulting string should be ".deghijklmnopqrswxyz".
  217. OffsetAdjuster::Adjustments first_adjustments;
  218. first_adjustments.push_back(OffsetAdjuster::Adjustment(0, 1, 0));
  219. first_adjustments.push_back(OffsetAdjuster::Adjustment(1, 2, 1));
  220. first_adjustments.push_back(OffsetAdjuster::Adjustment(5, 1, 0));
  221. first_adjustments.push_back(OffsetAdjuster::Adjustment(19, 3, 0));
  222. // Set up |adjustments_on_adjusted_string| to
  223. // - combine the "." character that replaced "bc" with "d" into one character
  224. // (call it "?")
  225. // - remove the "egh"
  226. // - expand the "i" into two characters (call them "12")
  227. // - combine the "jkl" into one character (call it "@")
  228. // - expand the "z" into two characters (call it "34")
  229. // The resulting string should be "?12@mnopqrswxy34".
  230. OffsetAdjuster::Adjustments adjustments_on_adjusted_string;
  231. adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
  232. 0, 2, 1));
  233. adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
  234. 2, 3, 0));
  235. adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
  236. 5, 1, 2));
  237. adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
  238. 6, 3, 1));
  239. adjustments_on_adjusted_string.push_back(OffsetAdjuster::Adjustment(
  240. 19, 1, 2));
  241. // Now merge the adjustments and check the results.
  242. OffsetAdjuster::MergeSequentialAdjustments(first_adjustments,
  243. &adjustments_on_adjusted_string);
  244. // The merged adjustments should look like
  245. // - combine abcd into "?"
  246. // - note: it's also reasonable for the Merge function to instead produce
  247. // two adjustments instead of this, one to remove a and another to
  248. // combine bcd into "?". This test verifies the current behavior.
  249. // - remove efgh
  250. // - expand i into "12"
  251. // - combine jkl into "@"
  252. // - remove tuv
  253. // - expand z into "34"
  254. ASSERT_EQ(6u, adjustments_on_adjusted_string.size());
  255. EXPECT_EQ(0u, adjustments_on_adjusted_string[0].original_offset);
  256. EXPECT_EQ(4u, adjustments_on_adjusted_string[0].original_length);
  257. EXPECT_EQ(1u, adjustments_on_adjusted_string[0].output_length);
  258. EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_offset);
  259. EXPECT_EQ(4u, adjustments_on_adjusted_string[1].original_length);
  260. EXPECT_EQ(0u, adjustments_on_adjusted_string[1].output_length);
  261. EXPECT_EQ(8u, adjustments_on_adjusted_string[2].original_offset);
  262. EXPECT_EQ(1u, adjustments_on_adjusted_string[2].original_length);
  263. EXPECT_EQ(2u, adjustments_on_adjusted_string[2].output_length);
  264. EXPECT_EQ(9u, adjustments_on_adjusted_string[3].original_offset);
  265. EXPECT_EQ(3u, adjustments_on_adjusted_string[3].original_length);
  266. EXPECT_EQ(1u, adjustments_on_adjusted_string[3].output_length);
  267. EXPECT_EQ(19u, adjustments_on_adjusted_string[4].original_offset);
  268. EXPECT_EQ(3u, adjustments_on_adjusted_string[4].original_length);
  269. EXPECT_EQ(0u, adjustments_on_adjusted_string[4].output_length);
  270. EXPECT_EQ(25u, adjustments_on_adjusted_string[5].original_offset);
  271. EXPECT_EQ(1u, adjustments_on_adjusted_string[5].original_length);
  272. EXPECT_EQ(2u, adjustments_on_adjusted_string[5].output_length);
  273. }
  274. } // namespace base