utf_string_conversions.cc 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362
  1. // Copyright (c) 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/strings/utf_string_conversions.h"
  5. #include <limits.h>
  6. #include <stdint.h>
  7. #include <ostream>
  8. #include <type_traits>
  9. #include "base/strings/string_piece.h"
  10. #include "base/strings/string_util.h"
  11. #include "base/strings/utf_string_conversion_utils.h"
  12. #include "base/third_party/icu/icu_utf.h"
  13. #include "build/build_config.h"
  14. namespace base {
  15. namespace {
  16. constexpr base_icu::UChar32 kErrorCodePoint = 0xFFFD;
  17. // Size coefficient ----------------------------------------------------------
  18. // The maximum number of codeunits in the destination encoding corresponding to
  19. // one codeunit in the source encoding.
  20. template <typename SrcChar, typename DestChar>
  21. struct SizeCoefficient {
  22. static_assert(sizeof(SrcChar) < sizeof(DestChar),
  23. "Default case: from a smaller encoding to the bigger one");
  24. // ASCII symbols are encoded by one codeunit in all encodings.
  25. static constexpr int value = 1;
  26. };
  27. template <>
  28. struct SizeCoefficient<char16_t, char> {
  29. // One UTF-16 codeunit corresponds to at most 3 codeunits in UTF-8.
  30. static constexpr int value = 3;
  31. };
  32. #if defined(WCHAR_T_IS_UTF32)
  33. template <>
  34. struct SizeCoefficient<wchar_t, char> {
  35. // UTF-8 uses at most 4 codeunits per character.
  36. static constexpr int value = 4;
  37. };
  38. template <>
  39. struct SizeCoefficient<wchar_t, char16_t> {
  40. // UTF-16 uses at most 2 codeunits per character.
  41. static constexpr int value = 2;
  42. };
  43. #endif // defined(WCHAR_T_IS_UTF32)
  44. template <typename SrcChar, typename DestChar>
  45. constexpr int size_coefficient_v =
  46. SizeCoefficient<std::decay_t<SrcChar>, std::decay_t<DestChar>>::value;
  47. // UnicodeAppendUnsafe --------------------------------------------------------
  48. // Function overloads that write code_point to the output string. Output string
  49. // has to have enough space for the codepoint.
  50. // Convenience typedef that checks whether the passed in type is integral (i.e.
  51. // bool, char, int or their extended versions) and is of the correct size.
  52. template <typename Char, size_t N>
  53. using EnableIfBitsAre = std::enable_if_t<std::is_integral<Char>::value &&
  54. CHAR_BIT * sizeof(Char) == N,
  55. bool>;
  56. template <typename Char, EnableIfBitsAre<Char, 8> = true>
  57. void UnicodeAppendUnsafe(Char* out,
  58. size_t* size,
  59. base_icu::UChar32 code_point) {
  60. CBU8_APPEND_UNSAFE(reinterpret_cast<uint8_t*>(out), *size, code_point);
  61. }
  62. template <typename Char, EnableIfBitsAre<Char, 16> = true>
  63. void UnicodeAppendUnsafe(Char* out,
  64. size_t* size,
  65. base_icu::UChar32 code_point) {
  66. CBU16_APPEND_UNSAFE(out, *size, code_point);
  67. }
  68. template <typename Char, EnableIfBitsAre<Char, 32> = true>
  69. void UnicodeAppendUnsafe(Char* out,
  70. size_t* size,
  71. base_icu::UChar32 code_point) {
  72. out[(*size)++] = static_cast<Char>(code_point);
  73. }
  74. // DoUTFConversion ------------------------------------------------------------
  75. // Main driver of UTFConversion specialized for different Src encodings.
  76. // dest has to have enough room for the converted text.
  77. template <typename DestChar>
  78. bool DoUTFConversion(const char* src,
  79. size_t src_len,
  80. DestChar* dest,
  81. size_t* dest_len) {
  82. bool success = true;
  83. for (size_t i = 0; i < src_len;) {
  84. base_icu::UChar32 code_point;
  85. CBU8_NEXT(reinterpret_cast<const uint8_t*>(src), i, src_len, code_point);
  86. if (!IsValidCodepoint(code_point)) {
  87. success = false;
  88. code_point = kErrorCodePoint;
  89. }
  90. UnicodeAppendUnsafe(dest, dest_len, code_point);
  91. }
  92. return success;
  93. }
  94. template <typename DestChar>
  95. bool DoUTFConversion(const char16_t* src,
  96. size_t src_len,
  97. DestChar* dest,
  98. size_t* dest_len) {
  99. bool success = true;
  100. auto ConvertSingleChar = [&success](char16_t in) -> base_icu::UChar32 {
  101. if (!CBU16_IS_SINGLE(in) || !IsValidCodepoint(in)) {
  102. success = false;
  103. return kErrorCodePoint;
  104. }
  105. return in;
  106. };
  107. size_t i = 0;
  108. // Always have another symbol in order to avoid checking boundaries in the
  109. // middle of the surrogate pair.
  110. while (i + 1 < src_len) {
  111. base_icu::UChar32 code_point;
  112. if (CBU16_IS_LEAD(src[i]) && CBU16_IS_TRAIL(src[i + 1])) {
  113. code_point = CBU16_GET_SUPPLEMENTARY(src[i], src[i + 1]);
  114. if (!IsValidCodepoint(code_point)) {
  115. code_point = kErrorCodePoint;
  116. success = false;
  117. }
  118. i += 2;
  119. } else {
  120. code_point = ConvertSingleChar(src[i]);
  121. ++i;
  122. }
  123. UnicodeAppendUnsafe(dest, dest_len, code_point);
  124. }
  125. if (i < src_len) {
  126. UnicodeAppendUnsafe(dest, dest_len, ConvertSingleChar(src[i]));
  127. }
  128. return success;
  129. }
  130. #if defined(WCHAR_T_IS_UTF32)
  131. template <typename DestChar>
  132. bool DoUTFConversion(const wchar_t* src,
  133. size_t src_len,
  134. DestChar* dest,
  135. size_t* dest_len) {
  136. bool success = true;
  137. for (size_t i = 0; i < src_len; ++i) {
  138. auto code_point = static_cast<base_icu::UChar32>(src[i]);
  139. if (!IsValidCodepoint(code_point)) {
  140. success = false;
  141. code_point = kErrorCodePoint;
  142. }
  143. UnicodeAppendUnsafe(dest, dest_len, code_point);
  144. }
  145. return success;
  146. }
  147. #endif // defined(WCHAR_T_IS_UTF32)
  148. // UTFConversion --------------------------------------------------------------
  149. // Function template for generating all UTF conversions.
  150. template <typename InputString, typename DestString>
  151. bool UTFConversion(const InputString& src_str, DestString* dest_str) {
  152. if (IsStringASCII(src_str)) {
  153. dest_str->assign(src_str.begin(), src_str.end());
  154. return true;
  155. }
  156. dest_str->resize(src_str.length() *
  157. size_coefficient_v<typename InputString::value_type,
  158. typename DestString::value_type>);
  159. // Empty string is ASCII => it OK to call operator[].
  160. auto* dest = &(*dest_str)[0];
  161. // ICU requires 32 bit numbers.
  162. size_t src_len = src_str.length();
  163. size_t dest_len = 0;
  164. bool res = DoUTFConversion(src_str.data(), src_len, dest, &dest_len);
  165. dest_str->resize(dest_len);
  166. dest_str->shrink_to_fit();
  167. return res;
  168. }
  169. } // namespace
  170. // UTF16 <-> UTF8 --------------------------------------------------------------
  171. bool UTF8ToUTF16(const char* src, size_t src_len, std::u16string* output) {
  172. return UTFConversion(StringPiece(src, src_len), output);
  173. }
  174. std::u16string UTF8ToUTF16(StringPiece utf8) {
  175. std::u16string ret;
  176. // Ignore the success flag of this call, it will do the best it can for
  177. // invalid input, which is what we want here.
  178. UTF8ToUTF16(utf8.data(), utf8.size(), &ret);
  179. return ret;
  180. }
  181. bool UTF16ToUTF8(const char16_t* src, size_t src_len, std::string* output) {
  182. return UTFConversion(StringPiece16(src, src_len), output);
  183. }
  184. std::string UTF16ToUTF8(StringPiece16 utf16) {
  185. std::string ret;
  186. // Ignore the success flag of this call, it will do the best it can for
  187. // invalid input, which is what we want here.
  188. UTF16ToUTF8(utf16.data(), utf16.length(), &ret);
  189. return ret;
  190. }
  191. // UTF-16 <-> Wide -------------------------------------------------------------
  192. #if defined(WCHAR_T_IS_UTF16)
  193. // When wide == UTF-16 the conversions are a NOP.
  194. bool WideToUTF16(const wchar_t* src, size_t src_len, std::u16string* output) {
  195. output->assign(src, src + src_len);
  196. return true;
  197. }
  198. std::u16string WideToUTF16(WStringPiece wide) {
  199. return std::u16string(wide.begin(), wide.end());
  200. }
  201. bool UTF16ToWide(const char16_t* src, size_t src_len, std::wstring* output) {
  202. output->assign(src, src + src_len);
  203. return true;
  204. }
  205. std::wstring UTF16ToWide(StringPiece16 utf16) {
  206. return std::wstring(utf16.begin(), utf16.end());
  207. }
  208. #elif defined(WCHAR_T_IS_UTF32)
  209. bool WideToUTF16(const wchar_t* src, size_t src_len, std::u16string* output) {
  210. return UTFConversion(base::WStringPiece(src, src_len), output);
  211. }
  212. std::u16string WideToUTF16(WStringPiece wide) {
  213. std::u16string ret;
  214. // Ignore the success flag of this call, it will do the best it can for
  215. // invalid input, which is what we want here.
  216. WideToUTF16(wide.data(), wide.length(), &ret);
  217. return ret;
  218. }
  219. bool UTF16ToWide(const char16_t* src, size_t src_len, std::wstring* output) {
  220. return UTFConversion(StringPiece16(src, src_len), output);
  221. }
  222. std::wstring UTF16ToWide(StringPiece16 utf16) {
  223. std::wstring ret;
  224. // Ignore the success flag of this call, it will do the best it can for
  225. // invalid input, which is what we want here.
  226. UTF16ToWide(utf16.data(), utf16.length(), &ret);
  227. return ret;
  228. }
  229. #endif // defined(WCHAR_T_IS_UTF32)
  230. // UTF-8 <-> Wide --------------------------------------------------------------
  231. // UTF8ToWide is the same code, regardless of whether wide is 16 or 32 bits
  232. bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output) {
  233. return UTFConversion(StringPiece(src, src_len), output);
  234. }
  235. std::wstring UTF8ToWide(StringPiece utf8) {
  236. std::wstring ret;
  237. // Ignore the success flag of this call, it will do the best it can for
  238. // invalid input, which is what we want here.
  239. UTF8ToWide(utf8.data(), utf8.length(), &ret);
  240. return ret;
  241. }
  242. #if defined(WCHAR_T_IS_UTF16)
  243. // Easy case since we can use the "utf" versions we already wrote above.
  244. bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output) {
  245. return UTF16ToUTF8(as_u16cstr(src), src_len, output);
  246. }
  247. std::string WideToUTF8(WStringPiece wide) {
  248. return UTF16ToUTF8(StringPiece16(as_u16cstr(wide), wide.size()));
  249. }
  250. #elif defined(WCHAR_T_IS_UTF32)
  251. bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output) {
  252. return UTFConversion(WStringPiece(src, src_len), output);
  253. }
  254. std::string WideToUTF8(WStringPiece wide) {
  255. std::string ret;
  256. // Ignore the success flag of this call, it will do the best it can for
  257. // invalid input, which is what we want here.
  258. WideToUTF8(wide.data(), wide.length(), &ret);
  259. return ret;
  260. }
  261. #endif // defined(WCHAR_T_IS_UTF32)
  262. std::u16string ASCIIToUTF16(StringPiece ascii) {
  263. DCHECK(IsStringASCII(ascii)) << ascii;
  264. return std::u16string(ascii.begin(), ascii.end());
  265. }
  266. std::string UTF16ToASCII(StringPiece16 utf16) {
  267. DCHECK(IsStringASCII(utf16)) << UTF16ToUTF8(utf16);
  268. return std::string(utf16.begin(), utf16.end());
  269. }
  270. #if defined(WCHAR_T_IS_UTF16)
  271. std::wstring ASCIIToWide(StringPiece ascii) {
  272. DCHECK(IsStringASCII(ascii)) << ascii;
  273. return std::wstring(ascii.begin(), ascii.end());
  274. }
  275. std::string WideToASCII(WStringPiece wide) {
  276. DCHECK(IsStringASCII(wide)) << wide;
  277. return std::string(wide.begin(), wide.end());
  278. }
  279. #endif // defined(WCHAR_T_IS_UTF16)
  280. } // namespace base