unicode_ut.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Unit tests for Unicode functions
  4. *
  5. * Copyright (c) 2018 Heinrich Schuchardt <xypron.glpk@gmx.de>
  6. */
  7. #include <common.h>
  8. #include <charset.h>
  9. #include <command.h>
  10. #include <errno.h>
  11. #include <test/test.h>
  12. #include <test/suites.h>
  13. #include <test/ut.h>
  14. /* Linker list entry for a Unicode test */
  15. #define UNICODE_TEST(_name) UNIT_TEST(_name, 0, unicode_test)
  16. /* Constants c1-c4 and d1-d4 encode the same letters */
  17. /* Six characters translating to one utf-8 byte each. */
  18. static const u16 c1[] = {0x55, 0x2d, 0x42, 0x6f, 0x6f, 0x74, 0x00};
  19. /* One character translating to two utf-8 bytes */
  20. static const u16 c2[] = {0x6b, 0x61, 0x66, 0x62, 0xe1, 0x74, 0x75, 0x72, 0x00};
  21. /* Three characters translating to three utf-8 bytes each */
  22. static const u16 c3[] = {0x6f5c, 0x6c34, 0x8266, 0x00};
  23. /* Three letters translating to four utf-8 bytes each */
  24. static const u16 c4[] = {0xd801, 0xdc8d, 0xd801, 0xdc96, 0xd801, 0xdc87,
  25. 0x0000};
  26. /* Illegal utf-16 strings */
  27. static const u16 i1[] = {0x69, 0x31, 0xdc87, 0x6c, 0x00};
  28. static const u16 i2[] = {0x69, 0x32, 0xd801, 0xd801, 0x6c, 0x00};
  29. static const u16 i3[] = {0x69, 0x33, 0xd801, 0x00};
  30. /* Six characters translating to one utf-16 word each. */
  31. static const char d1[] = {0x55, 0x2d, 0x42, 0x6f, 0x6f, 0x74, 0x00};
  32. /* Eight characters translating to one utf-16 word each */
  33. static const char d2[] = {0x6b, 0x61, 0x66, 0x62, 0xc3, 0xa1, 0x74, 0x75,
  34. 0x72, 0x00};
  35. /* Three characters translating to one utf-16 word each */
  36. static const char d3[] = {0xe6, 0xbd, 0x9c, 0xe6, 0xb0, 0xb4, 0xe8, 0x89,
  37. 0xa6, 0x00};
  38. /* Three letters translating to two utf-16 word each */
  39. static const char d4[] = {0xf0, 0x90, 0x92, 0x8d, 0xf0, 0x90, 0x92, 0x96,
  40. 0xf0, 0x90, 0x92, 0x87, 0x00};
  41. /* Illegal utf-8 strings */
  42. static const char j1[] = {0x6a, 0x31, 0xa1, 0x6c, 0x00};
  43. static const char j2[] = {0x6a, 0x32, 0xc3, 0xc3, 0x6c, 0x00};
  44. static const char j3[] = {0x6a, 0x33, 0xf0, 0x90, 0xf0, 0x00};
  45. /* U-Boot uses UTF-16 strings in the EFI context only. */
  46. #if CONFIG_IS_ENABLED(EFI_LOADER) && !defined(API_BUILD)
  47. static int ut_string16(struct unit_test_state *uts)
  48. {
  49. char buf[20];
  50. /* Test length and precision */
  51. memset(buf, 0xff, sizeof(buf));
  52. sprintf(buf, "%8.6ls", c2);
  53. ut_asserteq(' ', buf[1]);
  54. ut_assert(!strncmp(&buf[2], d2, 7));
  55. ut_assert(!buf[9]);
  56. memset(buf, 0xff, sizeof(buf));
  57. sprintf(buf, "%8.6ls", c4);
  58. ut_asserteq(' ', buf[4]);
  59. ut_assert(!strncmp(&buf[5], d4, 12));
  60. ut_assert(!buf[17]);
  61. memset(buf, 0xff, sizeof(buf));
  62. sprintf(buf, "%-8.2ls", c4);
  63. ut_asserteq(' ', buf[8]);
  64. ut_assert(!strncmp(buf, d4, 8));
  65. ut_assert(!buf[14]);
  66. /* Test handling of illegal utf-16 sequences */
  67. memset(buf, 0xff, sizeof(buf));
  68. sprintf(buf, "%ls", i1);
  69. ut_asserteq_str("i1?l", buf);
  70. memset(buf, 0xff, sizeof(buf));
  71. sprintf(buf, "%ls", i2);
  72. ut_asserteq_str("i2?l", buf);
  73. memset(buf, 0xff, sizeof(buf));
  74. sprintf(buf, "%ls", i3);
  75. ut_asserteq_str("i3?", buf);
  76. return 0;
  77. }
  78. UNICODE_TEST(ut_string16);
  79. #endif
  80. static int ut_utf8_get(struct unit_test_state *uts)
  81. {
  82. const char *s;
  83. s32 code;
  84. int i;
  85. /* Check characters less than 0x800 */
  86. s = d2;
  87. for (i = 0; i < 8; ++i) {
  88. code = utf8_get((const char **)&s);
  89. /* c2 is the utf-8 encoding of d2 */
  90. ut_asserteq(c2[i], code);
  91. if (!code)
  92. break;
  93. }
  94. ut_asserteq_ptr(s, d2 + 9)
  95. /* Check characters less than 0x10000 */
  96. s = d3;
  97. for (i = 0; i < 4; ++i) {
  98. code = utf8_get((const char **)&s);
  99. /* c3 is the utf-8 encoding of d3 */
  100. ut_asserteq(c3[i], code);
  101. if (!code)
  102. break;
  103. }
  104. ut_asserteq_ptr(s, d3 + 9)
  105. /* Check character greater 0xffff */
  106. s = d4;
  107. code = utf8_get((const char **)&s);
  108. ut_asserteq(0x0001048d, code);
  109. ut_asserteq_ptr(s, d4 + 4);
  110. return 0;
  111. }
  112. UNICODE_TEST(ut_utf8_get);
  113. static int ut_utf8_put(struct unit_test_state *uts)
  114. {
  115. char buffer[8] = { 0, };
  116. char *pos;
  117. /* Commercial at, translates to one character */
  118. pos = buffer;
  119. ut_assert(!utf8_put('@', &pos))
  120. ut_asserteq(1, pos - buffer);
  121. ut_asserteq('@', buffer[0]);
  122. ut_assert(!buffer[1]);
  123. /* Latin letter G with acute, translates to two charactes */
  124. pos = buffer;
  125. ut_assert(!utf8_put(0x1f4, &pos));
  126. ut_asserteq(2, pos - buffer);
  127. ut_asserteq_str("\xc7\xb4", buffer);
  128. /* Tagalog letter i, translates to three characters */
  129. pos = buffer;
  130. ut_assert(!utf8_put(0x1701, &pos));
  131. ut_asserteq(3, pos - buffer);
  132. ut_asserteq_str("\xe1\x9c\x81", buffer);
  133. /* Hamster face, translates to four characters */
  134. pos = buffer;
  135. ut_assert(!utf8_put(0x1f439, &pos));
  136. ut_asserteq(4, pos - buffer);
  137. ut_asserteq_str("\xf0\x9f\x90\xb9", buffer);
  138. /* Illegal code */
  139. pos = buffer;
  140. ut_asserteq(-1, utf8_put(0xd888, &pos));
  141. return 0;
  142. }
  143. UNICODE_TEST(ut_utf8_put);
  144. static int ut_utf8_utf16_strlen(struct unit_test_state *uts)
  145. {
  146. ut_asserteq(6, utf8_utf16_strlen(d1));
  147. ut_asserteq(8, utf8_utf16_strlen(d2));
  148. ut_asserteq(3, utf8_utf16_strlen(d3));
  149. ut_asserteq(6, utf8_utf16_strlen(d4));
  150. /* illegal utf-8 sequences */
  151. ut_asserteq(4, utf8_utf16_strlen(j1));
  152. ut_asserteq(5, utf8_utf16_strlen(j2));
  153. ut_asserteq(3, utf8_utf16_strlen(j3));
  154. return 0;
  155. }
  156. UNICODE_TEST(ut_utf8_utf16_strlen);
  157. static int ut_utf8_utf16_strnlen(struct unit_test_state *uts)
  158. {
  159. ut_asserteq(3, utf8_utf16_strnlen(d1, 3));
  160. ut_asserteq(6, utf8_utf16_strnlen(d1, 13));
  161. ut_asserteq(6, utf8_utf16_strnlen(d2, 6));
  162. ut_asserteq(2, utf8_utf16_strnlen(d3, 2));
  163. ut_asserteq(4, utf8_utf16_strnlen(d4, 2));
  164. ut_asserteq(6, utf8_utf16_strnlen(d4, 3));
  165. /* illegal utf-8 sequences */
  166. ut_asserteq(4, utf8_utf16_strnlen(j1, 16));
  167. ut_asserteq(5, utf8_utf16_strnlen(j2, 16));
  168. ut_asserteq(3, utf8_utf16_strnlen(j3, 16));
  169. return 0;
  170. }
  171. UNICODE_TEST(ut_utf8_utf16_strnlen);
  172. /**
  173. * ut_u16_strcmp() - Compare to u16 strings.
  174. *
  175. * @a1: first string
  176. * @a2: second string
  177. * @count: number of u16 to compare
  178. * Return: -1 if a1 < a2, 0 if a1 == a2, 1 if a1 > a2
  179. */
  180. static int ut_u16_strcmp(const u16 *a1, const u16 *a2, size_t count)
  181. {
  182. for (; (*a1 || *a2) && count; ++a1, ++a2, --count) {
  183. if (*a1 < *a2)
  184. return -1;
  185. if (*a1 > *a2)
  186. return 1;
  187. }
  188. return 0;
  189. }
  190. static int ut_utf8_utf16_strcpy(struct unit_test_state *uts)
  191. {
  192. u16 buf[16];
  193. u16 *pos;
  194. pos = buf;
  195. utf8_utf16_strcpy(&pos, d1);
  196. ut_asserteq(6, pos - buf);
  197. ut_assert(!ut_u16_strcmp(buf, c1, SIZE_MAX));
  198. pos = buf;
  199. utf8_utf16_strcpy(&pos, d2);
  200. ut_asserteq(8, pos - buf);
  201. ut_assert(!ut_u16_strcmp(buf, c2, SIZE_MAX));
  202. pos = buf;
  203. utf8_utf16_strcpy(&pos, d3);
  204. ut_asserteq(3, pos - buf);
  205. ut_assert(!ut_u16_strcmp(buf, c3, SIZE_MAX));
  206. pos = buf;
  207. utf8_utf16_strcpy(&pos, d4);
  208. ut_asserteq(6, pos - buf);
  209. ut_assert(!ut_u16_strcmp(buf, c4, SIZE_MAX));
  210. /* Illegal utf-8 strings */
  211. pos = buf;
  212. utf8_utf16_strcpy(&pos, j1);
  213. ut_asserteq(4, pos - buf);
  214. ut_assert(!ut_u16_strcmp(buf, L"j1?l", SIZE_MAX));
  215. pos = buf;
  216. utf8_utf16_strcpy(&pos, j2);
  217. ut_asserteq(5, pos - buf);
  218. ut_assert(!ut_u16_strcmp(buf, L"j2??l", SIZE_MAX));
  219. pos = buf;
  220. utf8_utf16_strcpy(&pos, j3);
  221. ut_asserteq(3, pos - buf);
  222. ut_assert(!ut_u16_strcmp(buf, L"j3?", SIZE_MAX));
  223. return 0;
  224. }
  225. UNICODE_TEST(ut_utf8_utf16_strcpy);
  226. int ut_utf8_utf16_strncpy(struct unit_test_state *uts)
  227. {
  228. u16 buf[16];
  229. u16 *pos;
  230. pos = buf;
  231. memset(buf, 0, sizeof(buf));
  232. utf8_utf16_strncpy(&pos, d1, 4);
  233. ut_asserteq(4, pos - buf);
  234. ut_assert(!buf[4]);
  235. ut_assert(!ut_u16_strcmp(buf, c1, 4));
  236. pos = buf;
  237. memset(buf, 0, sizeof(buf));
  238. utf8_utf16_strncpy(&pos, d2, 10);
  239. ut_asserteq(8, pos - buf);
  240. ut_assert(buf[4]);
  241. ut_assert(!ut_u16_strcmp(buf, c2, SIZE_MAX));
  242. pos = buf;
  243. memset(buf, 0, sizeof(buf));
  244. utf8_utf16_strncpy(&pos, d3, 2);
  245. ut_asserteq(2, pos - buf);
  246. ut_assert(!buf[2]);
  247. ut_assert(!ut_u16_strcmp(buf, c3, 2));
  248. pos = buf;
  249. memset(buf, 0, sizeof(buf));
  250. utf8_utf16_strncpy(&pos, d4, 2);
  251. ut_asserteq(4, pos - buf);
  252. ut_assert(!buf[4]);
  253. ut_assert(!ut_u16_strcmp(buf, c4, 4));
  254. pos = buf;
  255. memset(buf, 0, sizeof(buf));
  256. utf8_utf16_strncpy(&pos, d4, 10);
  257. ut_asserteq(6, pos - buf);
  258. ut_assert(buf[5]);
  259. ut_assert(!ut_u16_strcmp(buf, c4, SIZE_MAX));
  260. return 0;
  261. }
  262. UNICODE_TEST(ut_utf8_utf16_strncpy);
  263. static int ut_utf16_get(struct unit_test_state *uts)
  264. {
  265. const u16 *s;
  266. s32 code;
  267. int i;
  268. /* Check characters less than 0x10000 */
  269. s = c2;
  270. for (i = 0; i < 9; ++i) {
  271. code = utf16_get((const u16 **)&s);
  272. ut_asserteq(c2[i], code);
  273. if (!code)
  274. break;
  275. }
  276. ut_asserteq_ptr(c2 + 8, s);
  277. /* Check character greater 0xffff */
  278. s = c4;
  279. code = utf16_get((const u16 **)&s);
  280. ut_asserteq(0x0001048d, code);
  281. ut_asserteq_ptr(c4 + 2, s);
  282. return 0;
  283. }
  284. UNICODE_TEST(ut_utf16_get);
  285. static int ut_utf16_put(struct unit_test_state *uts)
  286. {
  287. u16 buffer[4] = { 0, };
  288. u16 *pos;
  289. /* Commercial at, translates to one word */
  290. pos = buffer;
  291. ut_assert(!utf16_put('@', &pos));
  292. ut_asserteq(1, pos - buffer);
  293. ut_asserteq((u16)'@', buffer[0]);
  294. ut_assert(!buffer[1]);
  295. /* Hamster face, translates to two words */
  296. pos = buffer;
  297. ut_assert(!utf16_put(0x1f439, &pos));
  298. ut_asserteq(2, pos - buffer);
  299. ut_asserteq((u16)0xd83d, buffer[0]);
  300. ut_asserteq((u16)0xdc39, buffer[1]);
  301. ut_assert(!buffer[2]);
  302. /* Illegal code */
  303. pos = buffer;
  304. ut_asserteq(-1, utf16_put(0xd888, &pos));
  305. return 0;
  306. }
  307. UNICODE_TEST(ut_utf16_put);
  308. int ut_utf16_strnlen(struct unit_test_state *uts)
  309. {
  310. ut_asserteq(3, utf16_strnlen(c1, 3));
  311. ut_asserteq(6, utf16_strnlen(c1, 13));
  312. ut_asserteq(6, utf16_strnlen(c2, 6));
  313. ut_asserteq(2, utf16_strnlen(c3, 2));
  314. ut_asserteq(2, utf16_strnlen(c4, 2));
  315. ut_asserteq(3, utf16_strnlen(c4, 3));
  316. /* illegal utf-16 word sequences */
  317. ut_asserteq(4, utf16_strnlen(i1, 16));
  318. ut_asserteq(4, utf16_strnlen(i2, 16));
  319. ut_asserteq(3, utf16_strnlen(i3, 16));
  320. return 0;
  321. }
  322. UNICODE_TEST(ut_utf16_strnlen);
  323. int ut_utf16_utf8_strlen(struct unit_test_state *uts)
  324. {
  325. ut_asserteq(6, utf16_utf8_strlen(c1));
  326. ut_asserteq(9, utf16_utf8_strlen(c2));
  327. ut_asserteq(9, utf16_utf8_strlen(c3));
  328. ut_asserteq(12, utf16_utf8_strlen(c4));
  329. /* illegal utf-16 word sequences */
  330. ut_asserteq(4, utf16_utf8_strlen(i1));
  331. ut_asserteq(4, utf16_utf8_strlen(i2));
  332. ut_asserteq(3, utf16_utf8_strlen(i3));
  333. return 0;
  334. }
  335. UNICODE_TEST(ut_utf16_utf8_strlen);
  336. int ut_utf16_utf8_strnlen(struct unit_test_state *uts)
  337. {
  338. ut_asserteq(3, utf16_utf8_strnlen(c1, 3));
  339. ut_asserteq(6, utf16_utf8_strnlen(c1, 13));
  340. ut_asserteq(7, utf16_utf8_strnlen(c2, 6));
  341. ut_asserteq(6, utf16_utf8_strnlen(c3, 2));
  342. ut_asserteq(8, utf16_utf8_strnlen(c4, 2));
  343. ut_asserteq(12, utf16_utf8_strnlen(c4, 3));
  344. return 0;
  345. }
  346. UNICODE_TEST(ut_utf16_utf8_strnlen);
  347. int ut_utf16_utf8_strcpy(struct unit_test_state *uts)
  348. {
  349. char buf[16];
  350. char *pos;
  351. pos = buf;
  352. utf16_utf8_strcpy(&pos, c1);
  353. ut_asserteq(6, pos - buf);
  354. ut_asserteq_str(d1, buf);
  355. pos = buf;
  356. utf16_utf8_strcpy(&pos, c2);
  357. ut_asserteq(9, pos - buf);
  358. ut_asserteq_str(d2, buf);
  359. pos = buf;
  360. utf16_utf8_strcpy(&pos, c3);
  361. ut_asserteq(9, pos - buf);
  362. ut_asserteq_str(d3, buf);
  363. pos = buf;
  364. utf16_utf8_strcpy(&pos, c4);
  365. ut_asserteq(12, pos - buf);
  366. ut_asserteq_str(d4, buf);
  367. /* Illegal utf-16 strings */
  368. pos = buf;
  369. utf16_utf8_strcpy(&pos, i1);
  370. ut_asserteq(4, pos - buf);
  371. ut_asserteq_str("i1?l", buf);
  372. pos = buf;
  373. utf16_utf8_strcpy(&pos, i2);
  374. ut_asserteq(4, pos - buf);
  375. ut_asserteq_str("i2?l", buf);
  376. pos = buf;
  377. utf16_utf8_strcpy(&pos, i3);
  378. ut_asserteq(3, pos - buf);
  379. ut_asserteq_str("i3?", buf);
  380. return 0;
  381. }
  382. UNICODE_TEST(ut_utf16_utf8_strcpy);
  383. int ut_utf16_utf8_strncpy(struct unit_test_state *uts)
  384. {
  385. char buf[16];
  386. char *pos;
  387. pos = buf;
  388. memset(buf, 0, sizeof(buf));
  389. utf16_utf8_strncpy(&pos, c1, 4);
  390. ut_asserteq(4, pos - buf);
  391. ut_assert(!buf[4]);
  392. ut_assert(!strncmp(buf, d1, 4));
  393. pos = buf;
  394. memset(buf, 0, sizeof(buf));
  395. utf16_utf8_strncpy(&pos, c2, 10);
  396. ut_asserteq(9, pos - buf);
  397. ut_assert(buf[4]);
  398. ut_assert(!strncmp(buf, d2, SIZE_MAX));
  399. pos = buf;
  400. memset(buf, 0, sizeof(buf));
  401. utf16_utf8_strncpy(&pos, c3, 2);
  402. ut_asserteq(6, pos - buf);
  403. ut_assert(!buf[6]);
  404. ut_assert(!strncmp(buf, d3, 6));
  405. pos = buf;
  406. memset(buf, 0, sizeof(buf));
  407. utf16_utf8_strncpy(&pos, c4, 2);
  408. ut_asserteq(8, pos - buf);
  409. ut_assert(!buf[8]);
  410. ut_assert(!strncmp(buf, d4, 8));
  411. pos = buf;
  412. memset(buf, 0, sizeof(buf));
  413. utf16_utf8_strncpy(&pos, c4, 10);
  414. ut_asserteq(12, pos - buf);
  415. ut_assert(buf[5]);
  416. ut_assert(!strncmp(buf, d4, SIZE_MAX));
  417. return 0;
  418. }
  419. UNICODE_TEST(ut_utf16_utf8_strncpy);
  420. int ut_utf_to_lower(struct unit_test_state *uts)
  421. {
  422. ut_asserteq('@', utf_to_lower('@'));
  423. ut_asserteq('a', utf_to_lower('A'));
  424. ut_asserteq('z', utf_to_lower('Z'));
  425. ut_asserteq('[', utf_to_lower('['));
  426. ut_asserteq('m', utf_to_lower('m'));
  427. /* Latin letter O with diaresis (umlaut) */
  428. ut_asserteq(0x00f6, utf_to_lower(0x00d6));
  429. #ifdef CONFIG_EFI_UNICODE_CAPITALIZATION
  430. /* Cyrillic letter I*/
  431. ut_asserteq(0x0438, utf_to_lower(0x0418));
  432. #endif
  433. return 0;
  434. }
  435. UNICODE_TEST(ut_utf_to_lower);
  436. int ut_utf_to_upper(struct unit_test_state *uts)
  437. {
  438. ut_asserteq('`', utf_to_upper('`'));
  439. ut_asserteq('A', utf_to_upper('a'));
  440. ut_asserteq('Z', utf_to_upper('z'));
  441. ut_asserteq('{', utf_to_upper('{'));
  442. ut_asserteq('M', utf_to_upper('M'));
  443. /* Latin letter O with diaresis (umlaut) */
  444. ut_asserteq(0x00d6, utf_to_upper(0x00f6));
  445. #ifdef CONFIG_EFI_UNICODE_CAPITALIZATION
  446. /* Cyrillic letter I */
  447. ut_asserteq(0x0418, utf_to_upper(0x0438));
  448. #endif
  449. return 0;
  450. }
  451. UNICODE_TEST(ut_utf_to_upper);
  452. int do_ut_unicode(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[])
  453. {
  454. struct unit_test *tests = ll_entry_start(struct unit_test, unicode_test);
  455. const int n_ents = ll_entry_count(struct unit_test, unicode_test);
  456. return cmd_ut_category("Unicode", tests, n_ents, argc, argv);
  457. }