123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510 |
- /* Copyright (C) 2010-2018 The RetroArch team
- *
- * ---------------------------------------------------------------------------------------
- * The following license statement only applies to this file (encoding_utf.c).
- * ---------------------------------------------------------------------------------------
- *
- * Permission is hereby granted, free of charge,
- * to any person obtaining a copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
- * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
- #include <stdint.h>
- #include <stdlib.h>
- #include <stddef.h>
- #include <string.h>
- #include <boolean.h>
- #include <compat/strl.h>
- #include <retro_inline.h>
- #include <encodings/utf.h>
- #if defined(_WIN32) && !defined(_XBOX)
- #include <windows.h>
- #elif defined(_XBOX)
- #include <xtl.h>
- #endif
- static unsigned leading_ones(uint8_t c)
- {
- unsigned ones = 0;
- while (c & 0x80)
- {
- ones++;
- c <<= 1;
- }
- return ones;
- }
- /* Simple implementation. Assumes the sequence is
- * properly synchronized and terminated. */
- size_t utf8_conv_utf32(uint32_t *out, size_t out_chars,
- const char *in, size_t in_size)
- {
- unsigned i;
- size_t ret = 0;
- while (in_size && out_chars)
- {
- unsigned extra, shift;
- uint32_t c;
- uint8_t first = *in++;
- unsigned ones = leading_ones(first);
- if (ones > 6 || ones == 1) /* Invalid or desync. */
- break;
- extra = ones ? ones - 1 : ones;
- if (1 + extra > in_size) /* Overflow. */
- break;
- shift = (extra - 1) * 6;
- c = (first & ((1 << (7 - ones)) - 1)) << (6 * extra);
- for (i = 0; i < extra; i++, in++, shift -= 6)
- c |= (*in & 0x3f) << shift;
- *out++ = c;
- in_size -= 1 + extra;
- out_chars--;
- ret++;
- }
- return ret;
- }
- bool utf16_conv_utf8(uint8_t *out, size_t *out_chars,
- const uint16_t *in, size_t in_size)
- {
- static uint8_t kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
- size_t out_pos = 0;
- size_t in_pos = 0;
- for (;;)
- {
- unsigned numAdds;
- uint32_t value;
- if (in_pos == in_size)
- {
- *out_chars = out_pos;
- return true;
- }
- value = in[in_pos++];
- if (value < 0x80)
- {
- if (out)
- out[out_pos] = (char)value;
- out_pos++;
- continue;
- }
- if (value >= 0xD800 && value < 0xE000)
- {
- uint32_t c2;
- if (value >= 0xDC00 || in_pos == in_size)
- break;
- c2 = in[in_pos++];
- if (c2 < 0xDC00 || c2 >= 0xE000)
- break;
- value = (((value - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000;
- }
- for (numAdds = 1; numAdds < 5; numAdds++)
- if (value < (((uint32_t)1) << (numAdds * 5 + 6)))
- break;
- if (out)
- out[out_pos] = (char)(kUtf8Limits[numAdds - 1]
- + (value >> (6 * numAdds)));
- out_pos++;
- do
- {
- numAdds--;
- if (out)
- out[out_pos] = (char)(0x80
- + ((value >> (6 * numAdds)) & 0x3F));
- out_pos++;
- }while (numAdds != 0);
- }
- *out_chars = out_pos;
- return false;
- }
- /* Acts mostly like strlcpy.
- *
- * Copies the given number of UTF-8 characters,
- * but at most d_len bytes.
- *
- * Always NULL terminates.
- * Does not copy half a character.
- *
- * Returns number of bytes. 's' is assumed valid UTF-8.
- * Use only if 'chars' is considerably less than 'd_len'. */
- size_t utf8cpy(char *d, size_t d_len, const char *s, size_t chars)
- {
- const uint8_t *sb = (const uint8_t*)s;
- const uint8_t *sb_org = sb;
- if (!s)
- return 0;
- while (*sb && chars-- > 0)
- {
- sb++;
- while ((*sb & 0xC0) == 0x80) sb++;
- }
- if ((size_t)(sb - sb_org) > d_len-1 /* NUL */)
- {
- sb = sb_org + d_len-1;
- while ((*sb & 0xC0) == 0x80) sb--;
- }
- memcpy(d, sb_org, sb-sb_org);
- d[sb-sb_org] = '\0';
- return sb-sb_org;
- }
- const char *utf8skip(const char *str, size_t chars)
- {
- const uint8_t *strb = (const uint8_t*)str;
- if (!chars)
- return str;
- do
- {
- strb++;
- while ((*strb & 0xC0)==0x80) strb++;
- chars--;
- } while(chars);
- return (const char*)strb;
- }
- size_t utf8len(const char *string)
- {
- size_t ret = 0;
- if (!string)
- return 0;
- while (*string)
- {
- if ((*string & 0xC0) != 0x80)
- ret++;
- string++;
- }
- return ret;
- }
- #define utf8_walkbyte(string) (*((*(string))++))
- /* Does not validate the input, returns garbage if it's not UTF-8. */
- uint32_t utf8_walk(const char **string)
- {
- uint8_t first = utf8_walkbyte(string);
- uint32_t ret = 0;
- if (first < 128)
- return first;
- ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
- if (first >= 0xE0)
- {
- ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
- if (first >= 0xF0)
- {
- ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
- return ret | (first & 7) << 18;
- }
- return ret | (first & 15) << 12;
- }
- return ret | (first & 31) << 6;
- }
- static bool utf16_to_char(uint8_t **utf_data,
- size_t *dest_len, const uint16_t *in)
- {
- unsigned len = 0;
- while (in[len] != '\0')
- len++;
- utf16_conv_utf8(NULL, dest_len, in, len);
- *dest_len += 1;
- *utf_data = (uint8_t*)malloc(*dest_len);
- if (*utf_data == 0)
- return false;
- return utf16_conv_utf8(*utf_data, dest_len, in, len);
- }
- bool utf16_to_char_string(const uint16_t *in, char *s, size_t len)
- {
- size_t dest_len = 0;
- uint8_t *utf16_data = NULL;
- bool ret = utf16_to_char(&utf16_data, &dest_len, in);
- if (ret)
- {
- utf16_data[dest_len] = 0;
- strlcpy(s, (const char*)utf16_data, len);
- }
- free(utf16_data);
- utf16_data = NULL;
- return ret;
- }
- #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
- /* Returned pointer MUST be freed by the caller if non-NULL. */
- static char *mb_to_mb_string_alloc(const char *str,
- enum CodePage cp_in, enum CodePage cp_out)
- {
- char *path_buf = NULL;
- wchar_t *path_buf_wide = NULL;
- int path_buf_len = 0;
- int path_buf_wide_len = MultiByteToWideChar(cp_in, 0, str, -1, NULL, 0);
- /* Windows 95 will return 0 from these functions with
- * a UTF8 codepage set without MSLU.
- *
- * From an unknown MSDN version (others omit this info):
- * - CP_UTF8 Windows 98/Me, Windows NT 4.0 and later:
- * Translate using UTF-8. When this is set, dwFlags must be zero.
- * - Windows 95: Under the Microsoft Layer for Unicode,
- * MultiByteToWideChar also supports CP_UTF7 and CP_UTF8.
- */
- if (path_buf_wide_len)
- {
- path_buf_wide = (wchar_t*)
- calloc(path_buf_wide_len + sizeof(wchar_t), sizeof(wchar_t));
- if (path_buf_wide)
- {
- MultiByteToWideChar(cp_in, 0,
- str, -1, path_buf_wide, path_buf_wide_len);
- if (*path_buf_wide)
- {
- path_buf_len = WideCharToMultiByte(cp_out, 0,
- path_buf_wide, -1, NULL, 0, NULL, NULL);
- if (path_buf_len)
- {
- path_buf = (char*)
- calloc(path_buf_len + sizeof(char), sizeof(char));
- if (path_buf)
- {
- WideCharToMultiByte(cp_out, 0,
- path_buf_wide, -1, path_buf,
- path_buf_len, NULL, NULL);
- free(path_buf_wide);
- if (*path_buf)
- return path_buf;
- free(path_buf);
- return NULL;
- }
- }
- else
- {
- free(path_buf_wide);
- return strdup(str);
- }
- }
- }
- }
- else
- return strdup(str);
- if (path_buf_wide)
- free(path_buf_wide);
- return NULL;
- }
- #endif
- /* Returned pointer MUST be freed by the caller if non-NULL. */
- char* utf8_to_local_string_alloc(const char *str)
- {
- if (str && *str)
- {
- #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
- return mb_to_mb_string_alloc(str, CODEPAGE_UTF8, CODEPAGE_LOCAL);
- #else
- /* assume string needs no modification if not on Windows */
- return strdup(str);
- #endif
- }
- return NULL;
- }
- /* Returned pointer MUST be freed by the caller if non-NULL. */
- char* local_to_utf8_string_alloc(const char *str)
- {
- if (str && *str)
- {
- #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
- return mb_to_mb_string_alloc(str, CODEPAGE_LOCAL, CODEPAGE_UTF8);
- #else
- /* assume string needs no modification if not on Windows */
- return strdup(str);
- #endif
- }
- return NULL;
- }
- /* Returned pointer MUST be freed by the caller if non-NULL. */
- wchar_t* utf8_to_utf16_string_alloc(const char *str)
- {
- #ifdef _WIN32
- int len = 0;
- int out_len = 0;
- #else
- size_t len = 0;
- size_t out_len = 0;
- #endif
- wchar_t *buf = NULL;
- if (!str || !*str)
- return NULL;
- #ifdef _WIN32
- len = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
- if (len)
- {
- buf = (wchar_t*)calloc(len, sizeof(wchar_t));
- if (!buf)
- return NULL;
- out_len = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, len);
- }
- else
- {
- /* fallback to ANSI codepage instead */
- len = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0);
- if (len)
- {
- buf = (wchar_t*)calloc(len, sizeof(wchar_t));
- if (!buf)
- return NULL;
- out_len = MultiByteToWideChar(CP_ACP, 0, str, -1, buf, len);
- }
- }
- if (out_len < 0)
- {
- free(buf);
- return NULL;
- }
- #else
- /* NOTE: For now, assume non-Windows platforms' locale is already UTF-8. */
- len = mbstowcs(NULL, str, 0) + 1;
- if (len)
- {
- buf = (wchar_t*)calloc(len, sizeof(wchar_t));
- if (!buf)
- return NULL;
- out_len = mbstowcs(buf, str, len);
- }
- if (out_len == (size_t)-1)
- {
- free(buf);
- return NULL;
- }
- #endif
- return buf;
- }
- /* Returned pointer MUST be freed by the caller if non-NULL. */
- char* utf16_to_utf8_string_alloc(const wchar_t *str)
- {
- #ifdef _WIN32
- int len = 0;
- #else
- size_t len = 0;
- #endif
- char *buf = NULL;
- if (!str || !*str)
- return NULL;
- #ifdef _WIN32
- {
- UINT code_page = CP_UTF8;
- len = WideCharToMultiByte(code_page,
- 0, str, -1, NULL, 0, NULL, NULL);
- /* fallback to ANSI codepage instead */
- if (!len)
- {
- code_page = CP_ACP;
- len = WideCharToMultiByte(code_page,
- 0, str, -1, NULL, 0, NULL, NULL);
- }
- buf = (char*)calloc(len, sizeof(char));
- if (!buf)
- return NULL;
- if (WideCharToMultiByte(code_page,
- 0, str, -1, buf, len, NULL, NULL) < 0)
- {
- free(buf);
- return NULL;
- }
- }
- #else
- /* NOTE: For now, assume non-Windows platforms'
- * locale is already UTF-8. */
- len = wcstombs(NULL, str, 0) + 1;
- if (len)
- {
- buf = (char*)calloc(len, sizeof(char));
- if (!buf)
- return NULL;
- if (wcstombs(buf, str, len) == (size_t)-1)
- {
- free(buf);
- return NULL;
- }
- }
- #endif
- return buf;
- }
|