icu_string_conversions_unittest.cc 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/i18n/icu_string_conversions.h"
  5. #include <math.h>
  6. #include <stdarg.h>
  7. #include <stddef.h>
  8. #include <limits>
  9. #include <sstream>
  10. #include "base/check_op.h"
  11. #include "base/format_macros.h"
  12. #include "base/strings/string_piece.h"
  13. #include "base/strings/stringprintf.h"
  14. #include "base/strings/utf_string_conversions.h"
  15. #include "build/build_config.h"
  16. #include "testing/gtest/include/gtest/gtest.h"
  17. namespace base {
  18. namespace {
  19. // Given a null-terminated string of wchar_t with each wchar_t representing
  20. // a UTF-16 code unit, returns a std::u16string made up of wchar_t's in the
  21. // input. Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
  22. // should be represented as a surrogate pair (two UTF-16 units)
  23. // *even* where wchar_t is 32-bit (Linux and Mac).
  24. //
  25. // This is to help write tests for functions with std::u16string params until
  26. // the C++ 0x UTF-16 literal is well-supported by compilers.
  27. std::u16string BuildString16(const wchar_t* s) {
  28. #if defined(WCHAR_T_IS_UTF16)
  29. return WideToUTF16(s);
  30. #elif defined(WCHAR_T_IS_UTF32)
  31. std::u16string u16;
  32. while (*s != 0) {
  33. DCHECK_LE(static_cast<unsigned int>(*s), 0xFFFFu);
  34. u16.push_back(*s++);
  35. }
  36. return u16;
  37. #endif
  38. }
  39. } // namespace
  40. // kConverterCodepageCases is not comprehensive. There are a number of cases
  41. // to add if we really want to have a comprehensive coverage of various
  42. // codepages and their 'idiosyncrasies'. Currently, the only implementation
  43. // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive
  44. // set of tests for the charset conversion. So, we can get away with a
  45. // relatively small number of cases listed below.
  46. //
  47. // Note about |u16_wide| in the following struct.
  48. // On Windows, the field is always identical to |wide|. On Mac and Linux,
  49. // it's identical as long as there's no character outside the
  50. // BMP (<= U+FFFF). When there is, it is different from |wide| and
  51. // is not a real wide string (UTF-32 string) in that each wchar_t in
  52. // the string is a UTF-16 code unit zero-extended to be 32-bit
  53. // even when the code unit belongs to a surrogate pair.
  54. // For instance, a Unicode string (U+0041 U+010000) is represented as
  55. // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000".
  56. // To avoid the clutter, |u16_wide| will be set to NULL
  57. // if it's identical to |wide| on *all* platforms.
  58. static const struct {
  59. const char* codepage_name;
  60. const char* encoded;
  61. OnStringConversionError::Type on_error;
  62. bool success;
  63. const wchar_t* wide;
  64. const wchar_t* u16_wide;
  65. } kConvertCodepageCases[] = {
  66. // Test a case where the input cannot be decoded, using SKIP, FAIL
  67. // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't.
  68. {"big5", "\xA7\x41\xA6", OnStringConversionError::FAIL, false, L"",
  69. nullptr},
  70. {"big5", "\xA7\x41\xA6", OnStringConversionError::SKIP, true, L"\x4F60",
  71. nullptr},
  72. {"big5", "\xA7\x41\xA6", OnStringConversionError::SUBSTITUTE, true,
  73. L"\x4F60\xFFFD", nullptr},
  74. // Arabic (ISO-8859)
  75. {"iso-8859-6",
  76. "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF"
  77. " "
  78. "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2",
  79. OnStringConversionError::FAIL, true,
  80. L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F"
  81. L" "
  82. L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652",
  83. nullptr},
  84. // Chinese Simplified (GB2312)
  85. {"gb2312", "\xC4\xE3\xBA\xC3", OnStringConversionError::FAIL, true,
  86. L"\x4F60\x597D", nullptr},
  87. // Chinese (GB18030) : 4 byte sequences mapped to BMP characters
  88. {"gb18030", "\x81\x30\x84\x36\xA1\xA7", OnStringConversionError::FAIL, true,
  89. L"\x00A5\x00A8", nullptr},
  90. // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000)
  91. {"gb18030", "\x95\x32\x82\x36\xD2\xBB", OnStringConversionError::FAIL, true,
  92. #if defined(WCHAR_T_IS_UTF16)
  93. L"\xD840\xDC00\x4E00",
  94. #elif defined(WCHAR_T_IS_UTF32)
  95. L"\x20000\x4E00",
  96. #endif
  97. L"\xD840\xDC00\x4E00"},
  98. {"big5", "\xA7\x41\xA6\x6E", OnStringConversionError::FAIL, true,
  99. L"\x4F60\x597D", nullptr},
  100. // Greek (ISO-8859)
  101. {"iso-8859-7",
  102. "\xE3\xE5\xE9\xDC"
  103. " "
  104. "\xF3\xEF\xF5",
  105. OnStringConversionError::FAIL, true,
  106. L"\x03B3\x03B5\x03B9\x03AC"
  107. L" "
  108. L"\x03C3\x03BF\x03C5",
  109. nullptr},
  110. // Hebrew (Windows)
  111. {"windows-1255", "\xF9\xD1\xC8\xEC\xE5\xC9\xED",
  112. OnStringConversionError::FAIL, true,
  113. L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD", nullptr},
  114. // Korean (EUC)
  115. {"euc-kr", "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4",
  116. OnStringConversionError::FAIL, true, L"\xC548\xB155\xD558\xC138\xC694",
  117. nullptr},
  118. // Japanese (EUC)
  119. {"euc-jp", "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8E\xA6",
  120. OnStringConversionError::FAIL, true,
  121. L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", nullptr},
  122. // Japanese (ISO-2022)
  123. {"iso-2022-jp",
  124. "\x1B$B"
  125. "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C"
  126. "\x1B(B"
  127. "ab"
  128. "\x1B(J"
  129. "\x5C\x7E#$"
  130. "\x1B(B",
  131. OnStringConversionError::FAIL, true,
  132. L"\x3053\x3093\x306B\x3061\x306F\x4E00"
  133. L"ab\x00A5\x203E#$",
  134. nullptr},
  135. // Japanese (Shift-JIS)
  136. {"sjis", "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6",
  137. OnStringConversionError::FAIL, true,
  138. L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", nullptr},
  139. // Russian (KOI8)
  140. {"koi8-r", "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5",
  141. OnStringConversionError::FAIL, true,
  142. L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
  143. L"\x0443\x0439\x0442\x0435",
  144. nullptr},
  145. // Thai (windows-874)
  146. {"windows-874",
  147. "\xCA\xC7\xD1\xCA\xB4\xD5"
  148. "\xA4\xC3\xD1\xBA",
  149. OnStringConversionError::FAIL, true,
  150. L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35"
  151. L"\x0E04\x0E23\x0e31\x0E1A",
  152. nullptr},
  153. };
  154. TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) {
  155. for (size_t i = 0; i < std::size(kConvertCodepageCases); ++i) {
  156. SCOPED_TRACE(base::StringPrintf(
  157. "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
  158. kConvertCodepageCases[i].encoded,
  159. kConvertCodepageCases[i].codepage_name));
  160. std::u16string utf16;
  161. bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded,
  162. kConvertCodepageCases[i].codepage_name,
  163. kConvertCodepageCases[i].on_error,
  164. &utf16);
  165. std::u16string utf16_expected;
  166. if (kConvertCodepageCases[i].u16_wide == nullptr)
  167. utf16_expected = BuildString16(kConvertCodepageCases[i].wide);
  168. else
  169. utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide);
  170. EXPECT_EQ(kConvertCodepageCases[i].success, success);
  171. EXPECT_EQ(utf16_expected, utf16);
  172. // When decoding was successful and nothing was skipped, we also check the
  173. // reverse conversion. See also the corresponding comment in
  174. // ConvertBetweenCodepageAndWide.
  175. if (success &&
  176. kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) {
  177. std::string encoded;
  178. success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name,
  179. kConvertCodepageCases[i].on_error, &encoded);
  180. EXPECT_EQ(kConvertCodepageCases[i].success, success);
  181. EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
  182. }
  183. }
  184. }
  185. static const struct {
  186. const char* encoded;
  187. const char* codepage_name;
  188. bool expected_success;
  189. const char* expected_value;
  190. } kConvertAndNormalizeCases[] = {
  191. {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"},
  192. {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"},
  193. {"foo-\xe4.html", "foo-bar", false, ""},
  194. // HTML Encoding spec treats US-ASCII as synonymous with windows-1252
  195. {"foo-\xff.html", "ascii", true, "foo-\xc3\xbf.html"},
  196. {"foo.html", "ascii", true, "foo.html"},
  197. {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"},
  198. {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"},
  199. {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"},
  200. // Windows-1258 does have a combining character at xD2 (which is U+0309).
  201. // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9.
  202. {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"},
  203. {"", "iso-8859-1", true, ""},
  204. };
  205. TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) {
  206. std::string result;
  207. for (size_t i = 0; i < std::size(kConvertAndNormalizeCases); ++i) {
  208. SCOPED_TRACE(base::StringPrintf(
  209. "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
  210. kConvertAndNormalizeCases[i].encoded,
  211. kConvertAndNormalizeCases[i].codepage_name));
  212. bool success = ConvertToUtf8AndNormalize(
  213. kConvertAndNormalizeCases[i].encoded,
  214. kConvertAndNormalizeCases[i].codepage_name, &result);
  215. EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success);
  216. EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result);
  217. }
  218. }
  219. } // namespace base