utf_string_conversion_utils.h 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_
  5. #define BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_
  6. // Low-level UTF handling functions. Most code will want to use the functions
  7. // in utf_string_conversions.h
  8. #include <stddef.h>
  9. #include <stdint.h>
  10. #include <string>
  11. #include "base/base_export.h"
  12. #include "base/third_party/icu/icu_utf.h"
  13. #include "build/build_config.h"
  14. namespace base {
  15. inline bool IsValidCodepoint(base_icu::UChar32 code_point) {
  16. // Excludes code points that are not Unicode scalar values, i.e.
  17. // surrogate code points ([0xD800, 0xDFFF]). Additionally, excludes
  18. // code points larger than 0x10FFFF (the highest codepoint allowed).
  19. // Non-characters and unassigned code points are allowed.
  20. // https://unicode.org/glossary/#unicode_scalar_value
  21. return (code_point >= 0 && code_point < 0xD800) ||
  22. (code_point >= 0xE000 && code_point <= 0x10FFFF);
  23. }
  24. inline bool IsValidCharacter(base_icu::UChar32 code_point) {
  25. // Excludes non-characters (U+FDD0..U+FDEF, and all code points
  26. // ending in 0xFFFE or 0xFFFF) from the set of valid code points.
  27. // https://unicode.org/faq/private_use.html#nonchar1
  28. return (code_point >= 0 && code_point < 0xD800) ||
  29. (code_point >= 0xE000 && code_point < 0xFDD0) ||
  30. (code_point > 0xFDEF && code_point <= 0x10FFFF &&
  31. (code_point & 0xFFFE) != 0xFFFE);
  32. }
  33. // ReadUnicodeCharacter --------------------------------------------------------
  34. // Reads a UTF-8 stream, placing the next code point into the given output
  35. // |*code_point|. |src| represents the entire string to read, and |*char_index|
  36. // is the character offset within the string to start reading at. |*char_index|
  37. // will be updated to index the last character read, such that incrementing it
  38. // (as in a for loop) will take the reader to the next character.
  39. //
  40. // Returns true on success. On false, |*code_point| will be invalid.
  41. BASE_EXPORT bool ReadUnicodeCharacter(const char* src,
  42. size_t src_len,
  43. size_t* char_index,
  44. base_icu::UChar32* code_point_out);
  45. // Reads a UTF-16 character. The usage is the same as the 8-bit version above.
  46. BASE_EXPORT bool ReadUnicodeCharacter(const char16_t* src,
  47. size_t src_len,
  48. size_t* char_index,
  49. base_icu::UChar32* code_point);
  50. #if defined(WCHAR_T_IS_UTF32)
  51. // Reads UTF-32 character. The usage is the same as the 8-bit version above.
  52. BASE_EXPORT bool ReadUnicodeCharacter(const wchar_t* src,
  53. size_t src_len,
  54. size_t* char_index,
  55. base_icu::UChar32* code_point);
  56. #endif // defined(WCHAR_T_IS_UTF32)
  57. // WriteUnicodeCharacter -------------------------------------------------------
  58. // Appends a UTF-8 character to the given 8-bit string. Returns the number of
  59. // bytes written.
  60. BASE_EXPORT size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
  61. std::string* output);
  62. // Appends the given code point as a UTF-16 character to the given 16-bit
  63. // string. Returns the number of 16-bit values written.
  64. BASE_EXPORT size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
  65. std::u16string* output);
  66. #if defined(WCHAR_T_IS_UTF32)
  67. // Appends the given UTF-32 character to the given 32-bit string. Returns the
  68. // number of 32-bit values written.
  69. inline size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
  70. std::wstring* output) {
  71. // This is the easy case, just append the character.
  72. output->push_back(static_cast<wchar_t>(code_point));
  73. return 1;
  74. }
  75. #endif // defined(WCHAR_T_IS_UTF32)
  76. // Generalized Unicode converter -----------------------------------------------
  77. // Guesses the length of the output in UTF-8 in bytes, clears that output
  78. // string, and reserves that amount of space. We assume that the input
  79. // character types are unsigned, which will be true for UTF-16 and -32 on our
  80. // systems.
  81. template<typename CHAR>
  82. void PrepareForUTF8Output(const CHAR* src, size_t src_len, std::string* output);
  83. // Prepares an output buffer (containing either UTF-16 or -32 data) given some
  84. // UTF-8 input that will be converted to it. See PrepareForUTF8Output().
  85. template<typename STRING>
  86. void PrepareForUTF16Or32Output(const char* src, size_t src_len, STRING* output);
  87. } // namespace base
  88. #endif // BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_