encoding_utf.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510
  1. /* Copyright (C) 2010-2018 The RetroArch team
  2. *
  3. * ---------------------------------------------------------------------------------------
  4. * The following license statement only applies to this file (encoding_utf.c).
  5. * ---------------------------------------------------------------------------------------
  6. *
  7. * Permission is hereby granted, free of charge,
  8. * to any person obtaining a copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation the rights to
  10. * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
  11. * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
  14. *
  15. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
  16. * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  18. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  19. * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  21. */
  22. #include <stdint.h>
  23. #include <stdlib.h>
  24. #include <stddef.h>
  25. #include <string.h>
  26. #include <boolean.h>
  27. #include <compat/strl.h>
  28. #include <retro_inline.h>
  29. #include <encodings/utf.h>
  30. #if defined(_WIN32) && !defined(_XBOX)
  31. #include <windows.h>
  32. #elif defined(_XBOX)
  33. #include <xtl.h>
  34. #endif
  35. static unsigned leading_ones(uint8_t c)
  36. {
  37. unsigned ones = 0;
  38. while (c & 0x80)
  39. {
  40. ones++;
  41. c <<= 1;
  42. }
  43. return ones;
  44. }
  45. /* Simple implementation. Assumes the sequence is
  46. * properly synchronized and terminated. */
  47. size_t utf8_conv_utf32(uint32_t *out, size_t out_chars,
  48. const char *in, size_t in_size)
  49. {
  50. unsigned i;
  51. size_t ret = 0;
  52. while (in_size && out_chars)
  53. {
  54. unsigned extra, shift;
  55. uint32_t c;
  56. uint8_t first = *in++;
  57. unsigned ones = leading_ones(first);
  58. if (ones > 6 || ones == 1) /* Invalid or desync. */
  59. break;
  60. extra = ones ? ones - 1 : ones;
  61. if (1 + extra > in_size) /* Overflow. */
  62. break;
  63. shift = (extra - 1) * 6;
  64. c = (first & ((1 << (7 - ones)) - 1)) << (6 * extra);
  65. for (i = 0; i < extra; i++, in++, shift -= 6)
  66. c |= (*in & 0x3f) << shift;
  67. *out++ = c;
  68. in_size -= 1 + extra;
  69. out_chars--;
  70. ret++;
  71. }
  72. return ret;
  73. }
  74. bool utf16_conv_utf8(uint8_t *out, size_t *out_chars,
  75. const uint16_t *in, size_t in_size)
  76. {
  77. static uint8_t kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
  78. size_t out_pos = 0;
  79. size_t in_pos = 0;
  80. for (;;)
  81. {
  82. unsigned numAdds;
  83. uint32_t value;
  84. if (in_pos == in_size)
  85. {
  86. *out_chars = out_pos;
  87. return true;
  88. }
  89. value = in[in_pos++];
  90. if (value < 0x80)
  91. {
  92. if (out)
  93. out[out_pos] = (char)value;
  94. out_pos++;
  95. continue;
  96. }
  97. if (value >= 0xD800 && value < 0xE000)
  98. {
  99. uint32_t c2;
  100. if (value >= 0xDC00 || in_pos == in_size)
  101. break;
  102. c2 = in[in_pos++];
  103. if (c2 < 0xDC00 || c2 >= 0xE000)
  104. break;
  105. value = (((value - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000;
  106. }
  107. for (numAdds = 1; numAdds < 5; numAdds++)
  108. if (value < (((uint32_t)1) << (numAdds * 5 + 6)))
  109. break;
  110. if (out)
  111. out[out_pos] = (char)(kUtf8Limits[numAdds - 1]
  112. + (value >> (6 * numAdds)));
  113. out_pos++;
  114. do
  115. {
  116. numAdds--;
  117. if (out)
  118. out[out_pos] = (char)(0x80
  119. + ((value >> (6 * numAdds)) & 0x3F));
  120. out_pos++;
  121. }while (numAdds != 0);
  122. }
  123. *out_chars = out_pos;
  124. return false;
  125. }
  126. /* Acts mostly like strlcpy.
  127. *
  128. * Copies the given number of UTF-8 characters,
  129. * but at most d_len bytes.
  130. *
  131. * Always NULL terminates.
  132. * Does not copy half a character.
  133. *
  134. * Returns number of bytes. 's' is assumed valid UTF-8.
  135. * Use only if 'chars' is considerably less than 'd_len'. */
  136. size_t utf8cpy(char *d, size_t d_len, const char *s, size_t chars)
  137. {
  138. const uint8_t *sb = (const uint8_t*)s;
  139. const uint8_t *sb_org = sb;
  140. if (!s)
  141. return 0;
  142. while (*sb && chars-- > 0)
  143. {
  144. sb++;
  145. while ((*sb & 0xC0) == 0x80) sb++;
  146. }
  147. if ((size_t)(sb - sb_org) > d_len-1 /* NUL */)
  148. {
  149. sb = sb_org + d_len-1;
  150. while ((*sb & 0xC0) == 0x80) sb--;
  151. }
  152. memcpy(d, sb_org, sb-sb_org);
  153. d[sb-sb_org] = '\0';
  154. return sb-sb_org;
  155. }
  156. const char *utf8skip(const char *str, size_t chars)
  157. {
  158. const uint8_t *strb = (const uint8_t*)str;
  159. if (!chars)
  160. return str;
  161. do
  162. {
  163. strb++;
  164. while ((*strb & 0xC0)==0x80) strb++;
  165. chars--;
  166. } while(chars);
  167. return (const char*)strb;
  168. }
  169. size_t utf8len(const char *string)
  170. {
  171. size_t ret = 0;
  172. if (!string)
  173. return 0;
  174. while (*string)
  175. {
  176. if ((*string & 0xC0) != 0x80)
  177. ret++;
  178. string++;
  179. }
  180. return ret;
  181. }
  182. #define utf8_walkbyte(string) (*((*(string))++))
  183. /* Does not validate the input, returns garbage if it's not UTF-8. */
  184. uint32_t utf8_walk(const char **string)
  185. {
  186. uint8_t first = utf8_walkbyte(string);
  187. uint32_t ret = 0;
  188. if (first < 128)
  189. return first;
  190. ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
  191. if (first >= 0xE0)
  192. {
  193. ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
  194. if (first >= 0xF0)
  195. {
  196. ret = (ret << 6) | (utf8_walkbyte(string) & 0x3F);
  197. return ret | (first & 7) << 18;
  198. }
  199. return ret | (first & 15) << 12;
  200. }
  201. return ret | (first & 31) << 6;
  202. }
  203. static bool utf16_to_char(uint8_t **utf_data,
  204. size_t *dest_len, const uint16_t *in)
  205. {
  206. unsigned len = 0;
  207. while (in[len] != '\0')
  208. len++;
  209. utf16_conv_utf8(NULL, dest_len, in, len);
  210. *dest_len += 1;
  211. *utf_data = (uint8_t*)malloc(*dest_len);
  212. if (*utf_data == 0)
  213. return false;
  214. return utf16_conv_utf8(*utf_data, dest_len, in, len);
  215. }
  216. bool utf16_to_char_string(const uint16_t *in, char *s, size_t len)
  217. {
  218. size_t dest_len = 0;
  219. uint8_t *utf16_data = NULL;
  220. bool ret = utf16_to_char(&utf16_data, &dest_len, in);
  221. if (ret)
  222. {
  223. utf16_data[dest_len] = 0;
  224. strlcpy(s, (const char*)utf16_data, len);
  225. }
  226. free(utf16_data);
  227. utf16_data = NULL;
  228. return ret;
  229. }
  230. #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
  231. /* Returned pointer MUST be freed by the caller if non-NULL. */
  232. static char *mb_to_mb_string_alloc(const char *str,
  233. enum CodePage cp_in, enum CodePage cp_out)
  234. {
  235. char *path_buf = NULL;
  236. wchar_t *path_buf_wide = NULL;
  237. int path_buf_len = 0;
  238. int path_buf_wide_len = MultiByteToWideChar(cp_in, 0, str, -1, NULL, 0);
  239. /* Windows 95 will return 0 from these functions with
  240. * a UTF8 codepage set without MSLU.
  241. *
  242. * From an unknown MSDN version (others omit this info):
  243. * - CP_UTF8 Windows 98/Me, Windows NT 4.0 and later:
  244. * Translate using UTF-8. When this is set, dwFlags must be zero.
  245. * - Windows 95: Under the Microsoft Layer for Unicode,
  246. * MultiByteToWideChar also supports CP_UTF7 and CP_UTF8.
  247. */
  248. if (path_buf_wide_len)
  249. {
  250. path_buf_wide = (wchar_t*)
  251. calloc(path_buf_wide_len + sizeof(wchar_t), sizeof(wchar_t));
  252. if (path_buf_wide)
  253. {
  254. MultiByteToWideChar(cp_in, 0,
  255. str, -1, path_buf_wide, path_buf_wide_len);
  256. if (*path_buf_wide)
  257. {
  258. path_buf_len = WideCharToMultiByte(cp_out, 0,
  259. path_buf_wide, -1, NULL, 0, NULL, NULL);
  260. if (path_buf_len)
  261. {
  262. path_buf = (char*)
  263. calloc(path_buf_len + sizeof(char), sizeof(char));
  264. if (path_buf)
  265. {
  266. WideCharToMultiByte(cp_out, 0,
  267. path_buf_wide, -1, path_buf,
  268. path_buf_len, NULL, NULL);
  269. free(path_buf_wide);
  270. if (*path_buf)
  271. return path_buf;
  272. free(path_buf);
  273. return NULL;
  274. }
  275. }
  276. else
  277. {
  278. free(path_buf_wide);
  279. return strdup(str);
  280. }
  281. }
  282. }
  283. }
  284. else
  285. return strdup(str);
  286. if (path_buf_wide)
  287. free(path_buf_wide);
  288. return NULL;
  289. }
  290. #endif
  291. /* Returned pointer MUST be freed by the caller if non-NULL. */
  292. char* utf8_to_local_string_alloc(const char *str)
  293. {
  294. if (str && *str)
  295. {
  296. #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
  297. return mb_to_mb_string_alloc(str, CODEPAGE_UTF8, CODEPAGE_LOCAL);
  298. #else
  299. /* assume string needs no modification if not on Windows */
  300. return strdup(str);
  301. #endif
  302. }
  303. return NULL;
  304. }
  305. /* Returned pointer MUST be freed by the caller if non-NULL. */
  306. char* local_to_utf8_string_alloc(const char *str)
  307. {
  308. if (str && *str)
  309. {
  310. #if defined(_WIN32) && !defined(_XBOX) && !defined(UNICODE)
  311. return mb_to_mb_string_alloc(str, CODEPAGE_LOCAL, CODEPAGE_UTF8);
  312. #else
  313. /* assume string needs no modification if not on Windows */
  314. return strdup(str);
  315. #endif
  316. }
  317. return NULL;
  318. }
  319. /* Returned pointer MUST be freed by the caller if non-NULL. */
  320. wchar_t* utf8_to_utf16_string_alloc(const char *str)
  321. {
  322. #ifdef _WIN32
  323. int len = 0;
  324. int out_len = 0;
  325. #else
  326. size_t len = 0;
  327. size_t out_len = 0;
  328. #endif
  329. wchar_t *buf = NULL;
  330. if (!str || !*str)
  331. return NULL;
  332. #ifdef _WIN32
  333. len = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
  334. if (len)
  335. {
  336. buf = (wchar_t*)calloc(len, sizeof(wchar_t));
  337. if (!buf)
  338. return NULL;
  339. out_len = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, len);
  340. }
  341. else
  342. {
  343. /* fallback to ANSI codepage instead */
  344. len = MultiByteToWideChar(CP_ACP, 0, str, -1, NULL, 0);
  345. if (len)
  346. {
  347. buf = (wchar_t*)calloc(len, sizeof(wchar_t));
  348. if (!buf)
  349. return NULL;
  350. out_len = MultiByteToWideChar(CP_ACP, 0, str, -1, buf, len);
  351. }
  352. }
  353. if (out_len < 0)
  354. {
  355. free(buf);
  356. return NULL;
  357. }
  358. #else
  359. /* NOTE: For now, assume non-Windows platforms' locale is already UTF-8. */
  360. len = mbstowcs(NULL, str, 0) + 1;
  361. if (len)
  362. {
  363. buf = (wchar_t*)calloc(len, sizeof(wchar_t));
  364. if (!buf)
  365. return NULL;
  366. out_len = mbstowcs(buf, str, len);
  367. }
  368. if (out_len == (size_t)-1)
  369. {
  370. free(buf);
  371. return NULL;
  372. }
  373. #endif
  374. return buf;
  375. }
  376. /* Returned pointer MUST be freed by the caller if non-NULL. */
  377. char* utf16_to_utf8_string_alloc(const wchar_t *str)
  378. {
  379. #ifdef _WIN32
  380. int len = 0;
  381. #else
  382. size_t len = 0;
  383. #endif
  384. char *buf = NULL;
  385. if (!str || !*str)
  386. return NULL;
  387. #ifdef _WIN32
  388. {
  389. UINT code_page = CP_UTF8;
  390. len = WideCharToMultiByte(code_page,
  391. 0, str, -1, NULL, 0, NULL, NULL);
  392. /* fallback to ANSI codepage instead */
  393. if (!len)
  394. {
  395. code_page = CP_ACP;
  396. len = WideCharToMultiByte(code_page,
  397. 0, str, -1, NULL, 0, NULL, NULL);
  398. }
  399. buf = (char*)calloc(len, sizeof(char));
  400. if (!buf)
  401. return NULL;
  402. if (WideCharToMultiByte(code_page,
  403. 0, str, -1, buf, len, NULL, NULL) < 0)
  404. {
  405. free(buf);
  406. return NULL;
  407. }
  408. }
  409. #else
  410. /* NOTE: For now, assume non-Windows platforms'
  411. * locale is already UTF-8. */
  412. len = wcstombs(NULL, str, 0) + 1;
  413. if (len)
  414. {
  415. buf = (char*)calloc(len, sizeof(char));
  416. if (!buf)
  417. return NULL;
  418. if (wcstombs(buf, str, len) == (size_t)-1)
  419. {
  420. free(buf);
  421. return NULL;
  422. }
  423. }
  424. #endif
  425. return buf;
  426. }