url_canon_icu_unittest.cc 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "url/url_canon_icu.h"
  5. #include <stddef.h>
  6. #include "base/logging.h"
  7. #include "base/memory/raw_ptr.h"
  8. #include "testing/gtest/include/gtest/gtest.h"
  9. #include "third_party/icu/source/common/unicode/ucnv.h"
  10. #include "url/url_canon.h"
  11. #include "url/url_canon_stdstring.h"
  12. #include "url/url_test_utils.h"
  13. namespace url {
  14. namespace {
  15. // Wrapper around a UConverter object that managers creation and destruction.
  16. class UConvScoper {
  17. public:
  18. explicit UConvScoper(const char* charset_name) {
  19. UErrorCode err = U_ZERO_ERROR;
  20. converter_ = ucnv_open(charset_name, &err);
  21. if (!converter_) {
  22. LOG(ERROR) << "Failed to open charset " << charset_name << ": "
  23. << u_errorName(err);
  24. }
  25. }
  26. ~UConvScoper() {
  27. if (converter_)
  28. ucnv_close(converter_);
  29. }
  30. // Returns the converter object, may be NULL.
  31. UConverter* converter() const { return converter_; }
  32. private:
  33. raw_ptr<UConverter> converter_;
  34. };
  35. TEST(URLCanonIcuTest, ICUCharsetConverter) {
  36. struct ICUCase {
  37. const wchar_t* input;
  38. const char* encoding;
  39. const char* expected;
  40. } icu_cases[] = {
  41. // UTF-8.
  42. {L"Hello, world", "utf-8", "Hello, world"},
  43. {L"\x4f60\x597d", "utf-8", "\xe4\xbd\xa0\xe5\xa5\xbd"},
  44. // Non-BMP UTF-8.
  45. {L"!\xd800\xdf00!", "utf-8", "!\xf0\x90\x8c\x80!"},
  46. // Big5
  47. {L"\x4f60\x597d", "big5", "\xa7\x41\xa6\x6e"},
  48. // Unrepresentable character in the destination set.
  49. {L"hello\x4f60\x06de\x597dworld", "big5",
  50. "hello\xa7\x41%26%231758%3B\xa6\x6eworld"},
  51. };
  52. for (size_t i = 0; i < std::size(icu_cases); i++) {
  53. UConvScoper conv(icu_cases[i].encoding);
  54. ASSERT_TRUE(conv.converter() != NULL);
  55. ICUCharsetConverter converter(conv.converter());
  56. std::string str;
  57. StdStringCanonOutput output(&str);
  58. std::u16string input_str(
  59. test_utils::TruncateWStringToUTF16(icu_cases[i].input));
  60. int input_len = static_cast<int>(input_str.length());
  61. converter.ConvertFromUTF16(input_str.c_str(), input_len, &output);
  62. output.Complete();
  63. EXPECT_STREQ(icu_cases[i].expected, str.c_str());
  64. }
  65. // Test string sizes around the resize boundary for the output to make sure
  66. // the converter resizes as needed.
  67. const int static_size = 16;
  68. UConvScoper conv("utf-8");
  69. ASSERT_TRUE(conv.converter());
  70. ICUCharsetConverter converter(conv.converter());
  71. for (int i = static_size - 2; i <= static_size + 2; i++) {
  72. // Make a string with the appropriate length.
  73. std::u16string input;
  74. for (int ch = 0; ch < i; ch++)
  75. input.push_back('a');
  76. RawCanonOutput<static_size> output;
  77. converter.ConvertFromUTF16(input.c_str(), static_cast<int>(input.length()),
  78. &output);
  79. EXPECT_EQ(input.length(), output.length());
  80. }
  81. }
  82. TEST(URLCanonIcuTest, QueryWithConverter) {
  83. struct QueryCase {
  84. const char* input8;
  85. const wchar_t* input16;
  86. const char* encoding;
  87. const char* expected;
  88. } query_cases[] = {
  89. // Regular ASCII case in some different encodings.
  90. {"foo=bar", L"foo=bar", "utf-8", "?foo=bar"},
  91. {"foo=bar", L"foo=bar", "shift_jis", "?foo=bar"},
  92. {"foo=bar", L"foo=bar", "gb2312", "?foo=bar"},
  93. // Chinese input/output
  94. {"q=\xe4\xbd\xa0\xe5\xa5\xbd", L"q=\x4f60\x597d", "gb2312",
  95. "?q=%C4%E3%BA%C3"},
  96. {"q=\xe4\xbd\xa0\xe5\xa5\xbd", L"q=\x4f60\x597d", "big5", "?q=%A7A%A6n"},
  97. // Unencodable character in the destination character set should be
  98. // escaped. The escape sequence unescapes to be the entity name:
  99. // "?q=&#20320;"
  100. {"q=Chinese\xef\xbc\xa7", L"q=Chinese\xff27", "iso-8859-1",
  101. "?q=Chinese%26%2365319%3B"},
  102. };
  103. for (size_t i = 0; i < std::size(query_cases); i++) {
  104. Component out_comp;
  105. UConvScoper conv(query_cases[i].encoding);
  106. ASSERT_TRUE(!query_cases[i].encoding || conv.converter());
  107. ICUCharsetConverter converter(conv.converter());
  108. if (query_cases[i].input8) {
  109. int len = static_cast<int>(strlen(query_cases[i].input8));
  110. Component in_comp(0, len);
  111. std::string out_str;
  112. StdStringCanonOutput output(&out_str);
  113. CanonicalizeQuery(query_cases[i].input8, in_comp, &converter, &output,
  114. &out_comp);
  115. output.Complete();
  116. EXPECT_EQ(query_cases[i].expected, out_str);
  117. }
  118. if (query_cases[i].input16) {
  119. std::u16string input16(
  120. test_utils::TruncateWStringToUTF16(query_cases[i].input16));
  121. int len = static_cast<int>(input16.length());
  122. Component in_comp(0, len);
  123. std::string out_str;
  124. StdStringCanonOutput output(&out_str);
  125. CanonicalizeQuery(input16.c_str(), in_comp, &converter, &output,
  126. &out_comp);
  127. output.Complete();
  128. EXPECT_EQ(query_cases[i].expected, out_str);
  129. }
  130. }
  131. // Extra test for input with embedded NULL;
  132. std::string out_str;
  133. StdStringCanonOutput output(&out_str);
  134. Component out_comp;
  135. CanonicalizeQuery("a \x00z\x01", Component(0, 5), NULL, &output, &out_comp);
  136. output.Complete();
  137. EXPECT_EQ("?a%20%00z%01", out_str);
  138. }
  139. } // namespace
  140. } // namespace url