icu_string_conversions.cc 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/i18n/icu_string_conversions.h"
  5. #include <stddef.h>
  6. #include <stdint.h>
  7. #include <memory>
  8. #include "base/check.h"
  9. #include "base/notreached.h"
  10. #include "base/strings/string_util.h"
  11. #include "base/strings/utf_string_conversions.h"
  12. #include "third_party/icu/source/common/unicode/normalizer2.h"
  13. #include "third_party/icu/source/common/unicode/ucnv.h"
  14. #include "third_party/icu/source/common/unicode/ucnv_cb.h"
  15. #include "third_party/icu/source/common/unicode/ucnv_err.h"
  16. #include "third_party/icu/source/common/unicode/ustring.h"
  17. namespace base {
  18. namespace {
  19. // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
  20. // in source/common/ucnv_err.c.
  21. // Copyright (c) 1995-2006 International Business Machines Corporation
  22. // and others
  23. //
  24. // All rights reserved.
  25. //
  26. // Permission is hereby granted, free of charge, to any person obtaining a
  27. // copy of this software and associated documentation files (the "Software"),
  28. // to deal in the Software without restriction, including without limitation
  29. // the rights to use, copy, modify, merge, publish, distribute, and/or
  30. // sell copies of the Software, and to permit persons to whom the Software
  31. // is furnished to do so, provided that the above copyright notice(s) and
  32. // this permission notice appear in all copies of the Software and that
  33. // both the above copyright notice(s) and this permission notice appear in
  34. // supporting documentation.
  35. //
  36. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  37. // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  38. // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
  39. // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
  40. // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
  41. // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
  42. // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
  43. // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
  44. // OR PERFORMANCE OF THIS SOFTWARE.
  45. //
  46. // Except as contained in this notice, the name of a copyright holder
  47. // shall not be used in advertising or otherwise to promote the sale, use
  48. // or other dealings in this Software without prior written authorization
  49. // of the copyright holder.
  50. // ___________________________________________________________________________
  51. //
  52. // All trademarks and registered trademarks mentioned herein are the property
  53. // of their respective owners.
  54. void ToUnicodeCallbackSubstitute(const void* context,
  55. UConverterToUnicodeArgs *to_args,
  56. const char* code_units,
  57. int32_t length,
  58. UConverterCallbackReason reason,
  59. UErrorCode * err) {
  60. static const UChar kReplacementChar = 0xFFFD;
  61. if (reason <= UCNV_IRREGULAR) {
  62. if (context == nullptr ||
  63. (*(reinterpret_cast<const char*>(context)) == 'i' &&
  64. reason == UCNV_UNASSIGNED)) {
  65. *err = U_ZERO_ERROR;
  66. ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
  67. }
  68. // else the caller must have set the error code accordingly.
  69. }
  70. // else ignore the reset, close and clone calls.
  71. }
  72. bool ConvertFromUTF16(UConverter* converter,
  73. base::StringPiece16 src,
  74. OnStringConversionError::Type on_error,
  75. std::string* encoded) {
  76. int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(
  77. src.length(), ucnv_getMaxCharSize(converter));
  78. encoded->resize(encoded_max_length);
  79. UErrorCode status = U_ZERO_ERROR;
  80. // Setup our error handler.
  81. switch (on_error) {
  82. case OnStringConversionError::FAIL:
  83. ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, nullptr,
  84. nullptr, nullptr, &status);
  85. break;
  86. case OnStringConversionError::SKIP:
  87. case OnStringConversionError::SUBSTITUTE:
  88. ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, nullptr,
  89. nullptr, nullptr, &status);
  90. break;
  91. default:
  92. NOTREACHED();
  93. }
  94. // ucnv_fromUChars returns size not including terminating null
  95. int actual_size =
  96. ucnv_fromUChars(converter, &(*encoded)[0], encoded_max_length, src.data(),
  97. src.length(), &status);
  98. encoded->resize(actual_size);
  99. ucnv_close(converter);
  100. if (U_SUCCESS(status))
  101. return true;
  102. encoded->clear(); // Make sure the output is empty on error.
  103. return false;
  104. }
  105. // Set up our error handler for ToUTF-16 converters
  106. void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
  107. UConverter* converter, UErrorCode* status) {
  108. switch (on_error) {
  109. case OnStringConversionError::FAIL:
  110. ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr,
  111. nullptr, status);
  112. break;
  113. case OnStringConversionError::SKIP:
  114. ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, nullptr, nullptr,
  115. nullptr, status);
  116. break;
  117. case OnStringConversionError::SUBSTITUTE:
  118. ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, nullptr,
  119. nullptr, nullptr, status);
  120. break;
  121. default:
  122. NOTREACHED();
  123. }
  124. }
  125. } // namespace
  126. // Codepage <-> Wide/UTF-16 ---------------------------------------------------
  127. bool UTF16ToCodepage(base::StringPiece16 utf16,
  128. const char* codepage_name,
  129. OnStringConversionError::Type on_error,
  130. std::string* encoded) {
  131. encoded->clear();
  132. UErrorCode status = U_ZERO_ERROR;
  133. UConverter* converter = ucnv_open(codepage_name, &status);
  134. if (!U_SUCCESS(status))
  135. return false;
  136. return ConvertFromUTF16(converter, utf16, on_error, encoded);
  137. }
  138. bool CodepageToUTF16(base::StringPiece encoded,
  139. const char* codepage_name,
  140. OnStringConversionError::Type on_error,
  141. std::u16string* utf16) {
  142. utf16->clear();
  143. UErrorCode status = U_ZERO_ERROR;
  144. UConverter* converter = ucnv_open(codepage_name, &status);
  145. if (!U_SUCCESS(status))
  146. return false;
  147. // Even in the worst case, the maximum length in 2-byte units of UTF-16
  148. // output would be at most the same as the number of bytes in input. There
  149. // is no single-byte encoding in which a character is mapped to a
  150. // non-BMP character requiring two 2-byte units.
  151. //
  152. // Moreover, non-BMP characters in legacy multibyte encodings
  153. // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
  154. // BOCU and SCSU, but we don't care about them.
  155. size_t uchar_max_length = encoded.length() + 1;
  156. SetUpErrorHandlerForToUChars(on_error, converter, &status);
  157. std::unique_ptr<char16_t[]> buffer(new char16_t[uchar_max_length]);
  158. int actual_size = ucnv_toUChars(
  159. converter, buffer.get(), static_cast<int>(uchar_max_length),
  160. encoded.data(), static_cast<int>(encoded.length()), &status);
  161. ucnv_close(converter);
  162. if (!U_SUCCESS(status)) {
  163. utf16->clear(); // Make sure the output is empty on error.
  164. return false;
  165. }
  166. utf16->assign(buffer.get(), actual_size);
  167. return true;
  168. }
  169. bool ConvertToUtf8AndNormalize(base::StringPiece text,
  170. const std::string& charset,
  171. std::string* result) {
  172. result->clear();
  173. std::u16string utf16;
  174. if (!CodepageToUTF16(text, charset.c_str(), OnStringConversionError::FAIL,
  175. &utf16))
  176. return false;
  177. UErrorCode status = U_ZERO_ERROR;
  178. const icu::Normalizer2* normalizer = icu::Normalizer2::getNFCInstance(status);
  179. DCHECK(U_SUCCESS(status));
  180. if (U_FAILURE(status))
  181. return false;
  182. int32_t utf16_length = static_cast<int32_t>(utf16.length());
  183. icu::UnicodeString normalized(utf16.data(), utf16_length);
  184. int32_t normalized_prefix_length =
  185. normalizer->spanQuickCheckYes(normalized, status);
  186. if (normalized_prefix_length < utf16_length) {
  187. icu::UnicodeString un_normalized(normalized, normalized_prefix_length);
  188. normalized.truncate(normalized_prefix_length);
  189. normalizer->normalizeSecondAndAppend(normalized, un_normalized, status);
  190. }
  191. if (U_FAILURE(status))
  192. return false;
  193. normalized.toUTF8String(*result);
  194. return true;
  195. }
  196. } // namespace base