123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224 |
- // Copyright (c) 2012 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "base/i18n/icu_string_conversions.h"
- #include <stddef.h>
- #include <stdint.h>
- #include <memory>
- #include "base/check.h"
- #include "base/notreached.h"
- #include "base/strings/string_util.h"
- #include "base/strings/utf_string_conversions.h"
- #include "third_party/icu/source/common/unicode/normalizer2.h"
- #include "third_party/icu/source/common/unicode/ucnv.h"
- #include "third_party/icu/source/common/unicode/ucnv_cb.h"
- #include "third_party/icu/source/common/unicode/ucnv_err.h"
- #include "third_party/icu/source/common/unicode/ustring.h"
- namespace base {
- namespace {
- // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
- // in source/common/ucnv_err.c.
- // Copyright (c) 1995-2006 International Business Machines Corporation
- // and others
- //
- // All rights reserved.
- //
- // Permission is hereby granted, free of charge, to any person obtaining a
- // copy of this software and associated documentation files (the "Software"),
- // to deal in the Software without restriction, including without limitation
- // the rights to use, copy, modify, merge, publish, distribute, and/or
- // sell copies of the Software, and to permit persons to whom the Software
- // is furnished to do so, provided that the above copyright notice(s) and
- // this permission notice appear in all copies of the Software and that
- // both the above copyright notice(s) and this permission notice appear in
- // supporting documentation.
- //
- // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
- // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
- // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
- // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
- // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
- // OR PERFORMANCE OF THIS SOFTWARE.
- //
- // Except as contained in this notice, the name of a copyright holder
- // shall not be used in advertising or otherwise to promote the sale, use
- // or other dealings in this Software without prior written authorization
- // of the copyright holder.
- // ___________________________________________________________________________
- //
- // All trademarks and registered trademarks mentioned herein are the property
- // of their respective owners.
- void ToUnicodeCallbackSubstitute(const void* context,
- UConverterToUnicodeArgs *to_args,
- const char* code_units,
- int32_t length,
- UConverterCallbackReason reason,
- UErrorCode * err) {
- static const UChar kReplacementChar = 0xFFFD;
- if (reason <= UCNV_IRREGULAR) {
- if (context == nullptr ||
- (*(reinterpret_cast<const char*>(context)) == 'i' &&
- reason == UCNV_UNASSIGNED)) {
- *err = U_ZERO_ERROR;
- ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
- }
- // else the caller must have set the error code accordingly.
- }
- // else ignore the reset, close and clone calls.
- }
- bool ConvertFromUTF16(UConverter* converter,
- base::StringPiece16 src,
- OnStringConversionError::Type on_error,
- std::string* encoded) {
- int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(
- src.length(), ucnv_getMaxCharSize(converter));
- encoded->resize(encoded_max_length);
- UErrorCode status = U_ZERO_ERROR;
- // Setup our error handler.
- switch (on_error) {
- case OnStringConversionError::FAIL:
- ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, nullptr,
- nullptr, nullptr, &status);
- break;
- case OnStringConversionError::SKIP:
- case OnStringConversionError::SUBSTITUTE:
- ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, nullptr,
- nullptr, nullptr, &status);
- break;
- default:
- NOTREACHED();
- }
- // ucnv_fromUChars returns size not including terminating null
- int actual_size =
- ucnv_fromUChars(converter, &(*encoded)[0], encoded_max_length, src.data(),
- src.length(), &status);
- encoded->resize(actual_size);
- ucnv_close(converter);
- if (U_SUCCESS(status))
- return true;
- encoded->clear(); // Make sure the output is empty on error.
- return false;
- }
- // Set up our error handler for ToUTF-16 converters
- void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
- UConverter* converter, UErrorCode* status) {
- switch (on_error) {
- case OnStringConversionError::FAIL:
- ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr,
- nullptr, status);
- break;
- case OnStringConversionError::SKIP:
- ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, nullptr, nullptr,
- nullptr, status);
- break;
- case OnStringConversionError::SUBSTITUTE:
- ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, nullptr,
- nullptr, nullptr, status);
- break;
- default:
- NOTREACHED();
- }
- }
- } // namespace
- // Codepage <-> Wide/UTF-16 ---------------------------------------------------
- bool UTF16ToCodepage(base::StringPiece16 utf16,
- const char* codepage_name,
- OnStringConversionError::Type on_error,
- std::string* encoded) {
- encoded->clear();
- UErrorCode status = U_ZERO_ERROR;
- UConverter* converter = ucnv_open(codepage_name, &status);
- if (!U_SUCCESS(status))
- return false;
- return ConvertFromUTF16(converter, utf16, on_error, encoded);
- }
- bool CodepageToUTF16(base::StringPiece encoded,
- const char* codepage_name,
- OnStringConversionError::Type on_error,
- std::u16string* utf16) {
- utf16->clear();
- UErrorCode status = U_ZERO_ERROR;
- UConverter* converter = ucnv_open(codepage_name, &status);
- if (!U_SUCCESS(status))
- return false;
- // Even in the worst case, the maximum length in 2-byte units of UTF-16
- // output would be at most the same as the number of bytes in input. There
- // is no single-byte encoding in which a character is mapped to a
- // non-BMP character requiring two 2-byte units.
- //
- // Moreover, non-BMP characters in legacy multibyte encodings
- // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
- // BOCU and SCSU, but we don't care about them.
- size_t uchar_max_length = encoded.length() + 1;
- SetUpErrorHandlerForToUChars(on_error, converter, &status);
- std::unique_ptr<char16_t[]> buffer(new char16_t[uchar_max_length]);
- int actual_size = ucnv_toUChars(
- converter, buffer.get(), static_cast<int>(uchar_max_length),
- encoded.data(), static_cast<int>(encoded.length()), &status);
- ucnv_close(converter);
- if (!U_SUCCESS(status)) {
- utf16->clear(); // Make sure the output is empty on error.
- return false;
- }
- utf16->assign(buffer.get(), actual_size);
- return true;
- }
- bool ConvertToUtf8AndNormalize(base::StringPiece text,
- const std::string& charset,
- std::string* result) {
- result->clear();
- std::u16string utf16;
- if (!CodepageToUTF16(text, charset.c_str(), OnStringConversionError::FAIL,
- &utf16))
- return false;
- UErrorCode status = U_ZERO_ERROR;
- const icu::Normalizer2* normalizer = icu::Normalizer2::getNFCInstance(status);
- DCHECK(U_SUCCESS(status));
- if (U_FAILURE(status))
- return false;
- int32_t utf16_length = static_cast<int32_t>(utf16.length());
- icu::UnicodeString normalized(utf16.data(), utf16_length);
- int32_t normalized_prefix_length =
- normalizer->spanQuickCheckYes(normalized, status);
- if (normalized_prefix_length < utf16_length) {
- icu::UnicodeString un_normalized(normalized, normalized_prefix_length);
- normalized.truncate(normalized_prefix_length);
- normalizer->normalizeSecondAndAppend(normalized, un_normalized, status);
- }
- if (U_FAILURE(status))
- return false;
- normalized.toUTF8String(*result);
- return true;
- }
- } // namespace base
|