json_test.cc 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753
  1. // Copyright 2018 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "json.h"
  5. #include <array>
  6. #include <clocale>
  7. #include <cmath>
  8. #include <cstdlib>
  9. #include <cstring>
  10. #include <iomanip>
  11. #include <iostream>
  12. #include <sstream>
  13. #include <string>
  14. #include "cbor.h"
  15. #include "parser_handler.h"
  16. #include "span.h"
  17. #include "status.h"
  18. #include "status_test_support.h"
  19. #include "test_platform.h"
  20. namespace crdtp {
  21. namespace json {
  22. // =============================================================================
  23. // json::NewJSONEncoder - for encoding streaming parser events as JSON
  24. // =============================================================================
  25. void WriteUTF8AsUTF16(ParserHandler* writer, const std::string& utf8) {
  26. writer->HandleString16(SpanFrom(UTF8ToUTF16(SpanFrom(utf8))));
  27. }
  28. TEST(JsonEncoder, OverlongEncodings) {
  29. std::string out;
  30. Status status;
  31. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  32. // We encode 0x7f, which is the DEL ascii character, as a 4 byte UTF8
  33. // sequence. This is called an overlong encoding, because only 1 byte
  34. // is needed to represent 0x7f as UTF8.
  35. std::vector<uint8_t> chars = {
  36. 0xf0, // Starts 4 byte utf8 sequence
  37. 0x80, // continuation byte
  38. 0x81, // continuation byte w/ payload bit 7 set to 1.
  39. 0xbf, // continuation byte w/ payload bits 0-6 set to 11111.
  40. };
  41. writer->HandleString8(SpanFrom(chars));
  42. EXPECT_EQ("\"\"", out); // Empty string means that 0x7f was rejected (good).
  43. }
  44. TEST(JsonEncoder, NotAContinuationByte) {
  45. std::string out;
  46. Status status;
  47. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  48. // |world| encodes the globe as a 4 byte UTF8 sequence. So, naturally, it'll
  49. // have a start byte, followed by three continuation bytes.
  50. std::string world = "🌎";
  51. ASSERT_EQ(4u, world.size());
  52. ASSERT_EQ(world[1] & 0xc0, 0x80); // checks for continuation byte
  53. ASSERT_EQ(world[2] & 0xc0, 0x80);
  54. ASSERT_EQ(world[3] & 0xc0, 0x80);
  55. // Now create a corrupted UTF8 string, starting with the first two bytes from
  56. // |world|, followed by an ASCII message. Upon encountering '!', our decoder
  57. // will realize that it's not a continuation byte; it'll skip to the end of
  58. // this UTF8 sequence and continue with the next character. In this case, the
  59. // 'H', of "Hello".
  60. std::vector<uint8_t> chars;
  61. chars.push_back(world[0]);
  62. chars.push_back(world[1]);
  63. chars.push_back('!');
  64. chars.push_back('?');
  65. chars.push_back('H');
  66. chars.push_back('e');
  67. chars.push_back('l');
  68. chars.push_back('l');
  69. chars.push_back('o');
  70. writer->HandleString8(SpanFrom(chars));
  71. EXPECT_EQ("\"Hello\"", out); // "Hello" shows we restarted at 'H'.
  72. }
  73. TEST(JsonEncoder, EscapesLoneHighSurrogates) {
  74. // This tests that the JSON encoder escapes lone high surrogates, i.e.
  75. // invalid code points in the range from 0xD800 to 0xDBFF. In
  76. // unescaped form, these cannot be represented in well-formed UTF-8 or
  77. // UTF-16.
  78. std::vector<uint16_t> chars = {'a', 0xd800, 'b', 0xdada, 'c', 0xdbff, 'd'};
  79. std::string out;
  80. Status status;
  81. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  82. writer->HandleString16(span<uint16_t>(chars.data(), chars.size()));
  83. EXPECT_EQ("\"a\\ud800b\\udadac\\udbffd\"", out);
  84. }
  85. TEST(JsonEncoder, EscapesLoneLowSurrogates) {
  86. // This tests that the JSON encoder escapes lone low surrogates, i.e.
  87. // invalid code points in the range from 0xDC00 to 0xDFFF. In
  88. // unescaped form, these cannot be represented in well-formed UTF-8 or
  89. // UTF-16.
  90. std::vector<uint16_t> chars = {'a', 0xdc00, 'b', 0xdede, 'c', 0xdfff, 'd'};
  91. std::string out;
  92. Status status;
  93. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  94. writer->HandleString16(span<uint16_t>(chars.data(), chars.size()));
  95. EXPECT_EQ("\"a\\udc00b\\udedec\\udfffd\"", out);
  96. }
  97. TEST(JsonEncoder, EscapesFFFF) {
  98. // This tests that the JSON encoder will escape the UTF16 input 0xffff as
  99. // \uffff; useful to check this since it's an edge case.
  100. std::vector<uint16_t> chars = {'a', 'b', 'c', 0xffff, 'd'};
  101. std::string out;
  102. Status status;
  103. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  104. writer->HandleString16(span<uint16_t>(chars.data(), chars.size()));
  105. EXPECT_EQ("\"abc\\uffffd\"", out);
  106. }
  107. TEST(JsonEncoder, Passes0x7FString8) {
  108. std::vector<uint8_t> chars = {'a', 0x7f, 'b'};
  109. std::string out;
  110. Status status;
  111. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  112. writer->HandleString8(span<uint8_t>(chars.data(), chars.size()));
  113. EXPECT_EQ(
  114. "\"a\x7f"
  115. "b\"",
  116. out);
  117. }
  118. TEST(JsonEncoder, Passes0x7FString16) {
  119. std::vector<uint16_t> chars16 = {'a', 0x7f, 'b'};
  120. std::string out;
  121. Status status;
  122. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  123. writer->HandleString16(span<uint16_t>(chars16.data(), chars16.size()));
  124. EXPECT_EQ(
  125. "\"a\x7f"
  126. "b\"",
  127. out);
  128. }
  129. TEST(JsonEncoder, IncompleteUtf8Sequence) {
  130. std::string out;
  131. Status status;
  132. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  133. writer->HandleArrayBegin(); // This emits [, which starts an array.
  134. { // 🌎 takes four bytes to encode in UTF-8. We test with the first three;
  135. // This means we're trying to emit a string that consists solely of an
  136. // incomplete UTF-8 sequence. So the string in the JSON output is empty.
  137. std::string world_utf8 = "🌎";
  138. ASSERT_EQ(4u, world_utf8.size());
  139. std::vector<uint8_t> chars(world_utf8.begin(), world_utf8.begin() + 3);
  140. writer->HandleString8(SpanFrom(chars));
  141. EXPECT_EQ("[\"\"", out); // Incomplete sequence rejected: empty string.
  142. }
  143. { // This time, the incomplete sequence is at the end of the string.
  144. std::string msg = "Hello, \xF0\x9F\x8C";
  145. std::vector<uint8_t> chars(msg.begin(), msg.end());
  146. writer->HandleString8(SpanFrom(chars));
  147. EXPECT_EQ("[\"\",\"Hello, \"", out); // Incomplete sequence dropped at end.
  148. }
  149. }
  150. TEST(JsonStdStringWriterTest, HelloWorld) {
  151. std::string out;
  152. Status status;
  153. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  154. writer->HandleMapBegin();
  155. WriteUTF8AsUTF16(writer.get(), "msg1");
  156. WriteUTF8AsUTF16(writer.get(), "Hello, 🌎.");
  157. std::string key = "msg1-as-utf8";
  158. std::string value = "Hello, 🌎.";
  159. writer->HandleString8(SpanFrom(key));
  160. writer->HandleString8(SpanFrom(value));
  161. WriteUTF8AsUTF16(writer.get(), "msg2");
  162. WriteUTF8AsUTF16(writer.get(), "\\\b\r\n\t\f\"");
  163. WriteUTF8AsUTF16(writer.get(), "nested");
  164. writer->HandleMapBegin();
  165. WriteUTF8AsUTF16(writer.get(), "double");
  166. writer->HandleDouble(3.1415);
  167. WriteUTF8AsUTF16(writer.get(), "int");
  168. writer->HandleInt32(-42);
  169. WriteUTF8AsUTF16(writer.get(), "bool");
  170. writer->HandleBool(false);
  171. WriteUTF8AsUTF16(writer.get(), "null");
  172. writer->HandleNull();
  173. writer->HandleMapEnd();
  174. WriteUTF8AsUTF16(writer.get(), "array");
  175. writer->HandleArrayBegin();
  176. writer->HandleInt32(1);
  177. writer->HandleInt32(2);
  178. writer->HandleInt32(3);
  179. writer->HandleArrayEnd();
  180. writer->HandleMapEnd();
  181. EXPECT_TRUE(status.ok());
  182. EXPECT_EQ(
  183. "{\"msg1\":\"Hello, \\ud83c\\udf0e.\","
  184. "\"msg1-as-utf8\":\"Hello, \\ud83c\\udf0e.\","
  185. "\"msg2\":\"\\\\\\b\\r\\n\\t\\f\\\"\","
  186. "\"nested\":{\"double\":3.1415,\"int\":-42,"
  187. "\"bool\":false,\"null\":null},\"array\":[1,2,3]}",
  188. out);
  189. }
  190. TEST(JsonStdStringWriterTest, ScalarsAreRenderedAsInt) {
  191. // Test that Number.MIN_SAFE_INTEGER / Number.MAX_SAFE_INTEGER from Javascript
  192. // are rendered as integers (no decimal point / rounding), even when we
  193. // encode them from double. Javascript's Number is an IEE754 double, so
  194. // it has 53 bits to represent integers.
  195. std::string out;
  196. Status status;
  197. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  198. writer->HandleMapBegin();
  199. writer->HandleString8(SpanFrom("Number.MIN_SAFE_INTEGER"));
  200. EXPECT_EQ(-0x1fffffffffffff, -9007199254740991); // 53 bits for integers.
  201. writer->HandleDouble(-9007199254740991); // Note HandleDouble here.
  202. writer->HandleString8(SpanFrom("Number.MAX_SAFE_INTEGER"));
  203. EXPECT_EQ(0x1fffffffffffff, 9007199254740991); // 53 bits for integers.
  204. writer->HandleDouble(9007199254740991); // Note HandleDouble here.
  205. writer->HandleMapEnd();
  206. EXPECT_TRUE(status.ok());
  207. EXPECT_EQ(
  208. "{\"Number.MIN_SAFE_INTEGER\":-9007199254740991,"
  209. "\"Number.MAX_SAFE_INTEGER\":9007199254740991}",
  210. out);
  211. }
  212. TEST(JsonStdStringWriterTest, RepresentingNonFiniteValuesAsNull) {
  213. // JSON can't represent +Infinity, -Infinity, or NaN.
  214. // So in practice it's mapped to null.
  215. std::string out;
  216. Status status;
  217. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  218. writer->HandleMapBegin();
  219. writer->HandleString8(SpanFrom("Infinity"));
  220. writer->HandleDouble(std::numeric_limits<double>::infinity());
  221. writer->HandleString8(SpanFrom("-Infinity"));
  222. writer->HandleDouble(-std::numeric_limits<double>::infinity());
  223. writer->HandleString8(SpanFrom("NaN"));
  224. writer->HandleDouble(std::numeric_limits<double>::quiet_NaN());
  225. writer->HandleMapEnd();
  226. EXPECT_TRUE(status.ok());
  227. EXPECT_EQ("{\"Infinity\":null,\"-Infinity\":null,\"NaN\":null}", out);
  228. }
  229. TEST(JsonStdStringWriterTest, BinaryEncodedAsJsonString) {
  230. // The encoder emits binary submitted to ParserHandler::HandleBinary
  231. // as base64. The following three examples are taken from
  232. // https://en.wikipedia.org/wiki/Base64.
  233. {
  234. std::string out;
  235. Status status;
  236. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  237. writer->HandleBinary(SpanFrom(std::vector<uint8_t>({'M', 'a', 'n'})));
  238. EXPECT_TRUE(status.ok());
  239. EXPECT_EQ("\"TWFu\"", out);
  240. }
  241. {
  242. std::string out;
  243. Status status;
  244. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  245. writer->HandleBinary(SpanFrom(std::vector<uint8_t>({'M', 'a'})));
  246. EXPECT_TRUE(status.ok());
  247. EXPECT_EQ("\"TWE=\"", out);
  248. }
  249. {
  250. std::string out;
  251. Status status;
  252. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  253. writer->HandleBinary(SpanFrom(std::vector<uint8_t>({'M'})));
  254. EXPECT_TRUE(status.ok());
  255. EXPECT_EQ("\"TQ==\"", out);
  256. }
  257. { // "Hello, world.", verified with base64decode.org.
  258. std::string out;
  259. Status status;
  260. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  261. writer->HandleBinary(SpanFrom(std::vector<uint8_t>(
  262. {'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '.'})));
  263. EXPECT_TRUE(status.ok());
  264. EXPECT_EQ("\"SGVsbG8sIHdvcmxkLg==\"", out);
  265. }
  266. }
  267. TEST(JsonStdStringWriterTest, HandlesErrors) {
  268. // When an error is sent via HandleError, it saves it in the provided
  269. // status and clears the output.
  270. std::string out;
  271. Status status;
  272. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  273. writer->HandleMapBegin();
  274. WriteUTF8AsUTF16(writer.get(), "msg1");
  275. writer->HandleError(Status{Error::JSON_PARSER_VALUE_EXPECTED, 42});
  276. EXPECT_THAT(status, StatusIs(Error::JSON_PARSER_VALUE_EXPECTED, 42u));
  277. EXPECT_EQ("", out);
  278. }
  279. TEST(JsonStdStringWriterTest, DoubleToString_LeadingZero) {
  280. // In JSON, .1 must be rendered as 0.1, and -.7 must be rendered as -0.7.
  281. std::string out;
  282. Status status;
  283. std::unique_ptr<ParserHandler> writer = NewJSONEncoder(&out, &status);
  284. writer->HandleArrayBegin();
  285. writer->HandleDouble(.1);
  286. writer->HandleDouble(-.7);
  287. writer->HandleArrayEnd();
  288. EXPECT_EQ("[0.1,-0.7]", out);
  289. }
  290. // =============================================================================
  291. // json::ParseJSON - for receiving streaming parser events for JSON
  292. // =============================================================================
  293. class Log : public ParserHandler {
  294. public:
  295. void HandleMapBegin() override { log_ << "map begin\n"; }
  296. void HandleMapEnd() override { log_ << "map end\n"; }
  297. void HandleArrayBegin() override { log_ << "array begin\n"; }
  298. void HandleArrayEnd() override { log_ << "array end\n"; }
  299. void HandleString8(span<uint8_t> chars) override {
  300. log_ << "string8: " << std::string(chars.begin(), chars.end()) << "\n";
  301. }
  302. void HandleString16(span<uint16_t> chars) override {
  303. raw_log_string16_.emplace_back(chars.begin(), chars.end());
  304. log_ << "string16: " << UTF16ToUTF8(chars) << "\n";
  305. }
  306. void HandleBinary(span<uint8_t> bytes) override {
  307. // JSON doesn't have native support for arbitrary bytes, so our parser will
  308. // never call this.
  309. CHECK(false);
  310. }
  311. void HandleDouble(double value) override {
  312. log_ << "double: " << value << "\n";
  313. }
  314. void HandleInt32(int32_t value) override { log_ << "int: " << value << "\n"; }
  315. void HandleBool(bool value) override { log_ << "bool: " << value << "\n"; }
  316. void HandleNull() override { log_ << "null\n"; }
  317. void HandleError(Status status) override { status_ = status; }
  318. std::string str() const { return status_.ok() ? log_.str() : ""; }
  319. std::vector<std::vector<uint16_t>> raw_log_string16() const {
  320. return raw_log_string16_;
  321. }
  322. Status status() const { return status_; }
  323. private:
  324. std::ostringstream log_;
  325. std::vector<std::vector<uint16_t>> raw_log_string16_;
  326. Status status_;
  327. };
  328. class JsonParserTest : public ::testing::Test {
  329. protected:
  330. Log log_;
  331. };
  332. TEST_F(JsonParserTest, SimpleDictionary) {
  333. std::string json = "{\"foo\": 42}";
  334. ParseJSON(SpanFrom(json), &log_);
  335. EXPECT_TRUE(log_.status().ok());
  336. EXPECT_EQ(
  337. "map begin\n"
  338. "string16: foo\n"
  339. "int: 42\n"
  340. "map end\n",
  341. log_.str());
  342. }
  343. TEST_F(JsonParserTest, UsAsciiDelCornerCase) {
  344. // DEL (0x7f) is a 7 bit US-ASCII character, and while it is a control
  345. // character according to Unicode, it's not considered a control
  346. // character in https://tools.ietf.org/html/rfc7159#section-7, so
  347. // it can be placed directly into the JSON string, without JSON escaping.
  348. std::string json = "{\"foo\": \"a\x7f\"}";
  349. ParseJSON(SpanFrom(json), &log_);
  350. EXPECT_TRUE(log_.status().ok());
  351. EXPECT_EQ(
  352. "map begin\n"
  353. "string16: foo\n"
  354. "string16: a\x7f\n"
  355. "map end\n",
  356. log_.str());
  357. // We've seen an implementation of UTF16ToUTF8 which would replace the DEL
  358. // character with ' ', so this simple roundtrip tests the routines in
  359. // encoding_test_helper.h, to make test failures of the above easier to
  360. // diagnose.
  361. std::vector<uint16_t> utf16 = UTF8ToUTF16(SpanFrom(json));
  362. EXPECT_EQ(json, UTF16ToUTF8(SpanFrom(utf16)));
  363. }
  364. TEST_F(JsonParserTest, Whitespace) {
  365. std::string json = "\n {\n\"msg\"\n: \v\"Hello, world.\"\t\r}\t";
  366. ParseJSON(SpanFrom(json), &log_);
  367. EXPECT_TRUE(log_.status().ok());
  368. EXPECT_EQ(
  369. "map begin\n"
  370. "string16: msg\n"
  371. "string16: Hello, world.\n"
  372. "map end\n",
  373. log_.str());
  374. }
  375. TEST_F(JsonParserTest, NestedDictionary) {
  376. std::string json = "{\"foo\": {\"bar\": {\"baz\": 1}, \"bar2\": 2}}";
  377. ParseJSON(SpanFrom(json), &log_);
  378. EXPECT_TRUE(log_.status().ok());
  379. EXPECT_EQ(
  380. "map begin\n"
  381. "string16: foo\n"
  382. "map begin\n"
  383. "string16: bar\n"
  384. "map begin\n"
  385. "string16: baz\n"
  386. "int: 1\n"
  387. "map end\n"
  388. "string16: bar2\n"
  389. "int: 2\n"
  390. "map end\n"
  391. "map end\n",
  392. log_.str());
  393. }
  394. TEST_F(JsonParserTest, Doubles) {
  395. std::string json = "{\"foo\": 3.1415, \"bar\": 31415e-4}";
  396. ParseJSON(SpanFrom(json), &log_);
  397. EXPECT_TRUE(log_.status().ok());
  398. EXPECT_EQ(
  399. "map begin\n"
  400. "string16: foo\n"
  401. "double: 3.1415\n"
  402. "string16: bar\n"
  403. "double: 3.1415\n"
  404. "map end\n",
  405. log_.str());
  406. }
  407. TEST_F(JsonParserTest, Unicode) {
  408. // Globe character. 0xF0 0x9F 0x8C 0x8E in utf8, 0xD83C 0xDF0E in utf16.
  409. std::string json = "{\"msg\": \"Hello, \\uD83C\\uDF0E.\"}";
  410. ParseJSON(SpanFrom(json), &log_);
  411. EXPECT_TRUE(log_.status().ok());
  412. EXPECT_EQ(
  413. "map begin\n"
  414. "string16: msg\n"
  415. "string16: Hello, 🌎.\n"
  416. "map end\n",
  417. log_.str());
  418. }
  419. TEST_F(JsonParserTest, Unicode_ParseUtf16) {
  420. // Globe character. utf8: 0xF0 0x9F 0x8C 0x8E; utf16: 0xD83C 0xDF0E.
  421. // Crescent moon character. utf8: 0xF0 0x9F 0x8C 0x99; utf16: 0xD83C 0xDF19.
  422. // We provide the moon with json escape, but the earth as utf16 input.
  423. // Either way they arrive as utf8 (after decoding in log_.str()).
  424. std::vector<uint16_t> json =
  425. UTF8ToUTF16(SpanFrom("{\"space\": \"🌎 \\uD83C\\uDF19.\"}"));
  426. ParseJSON(SpanFrom(json), &log_);
  427. EXPECT_TRUE(log_.status().ok());
  428. EXPECT_EQ(
  429. "map begin\n"
  430. "string16: space\n"
  431. "string16: 🌎 🌙.\n"
  432. "map end\n",
  433. log_.str());
  434. }
  435. TEST_F(JsonParserTest, Unicode_ParseUtf16_SingleEscapeUpToFFFF) {
  436. // 0xFFFF is the max codepoint that can be represented as a single \u escape.
  437. // One way to write this is \uffff, another way is to encode it as a 3 byte
  438. // UTF-8 sequence (0xef 0xbf 0xbf). Both are equivalent.
  439. // Example with both ways of encoding code point 0xFFFF in a JSON string.
  440. std::string json = "{\"escape\": \"\xef\xbf\xbf or \\uffff\"}";
  441. ParseJSON(SpanFrom(json), &log_);
  442. EXPECT_TRUE(log_.status().ok());
  443. // Shows both inputs result in equivalent output once converted to UTF-8.
  444. EXPECT_EQ(
  445. "map begin\n"
  446. "string16: escape\n"
  447. "string16: \xEF\xBF\xBF or \xEF\xBF\xBF\n"
  448. "map end\n",
  449. log_.str());
  450. // Make an even stronger assertion: The parser represents \xffff as a single
  451. // UTF-16 char.
  452. ASSERT_EQ(2u, log_.raw_log_string16().size());
  453. std::vector<uint16_t> expected = {0xffff, ' ', 'o', 'r', ' ', 0xffff};
  454. EXPECT_EQ(expected, log_.raw_log_string16()[1]);
  455. }
  456. TEST_F(JsonParserTest, Unicode_ParseUtf8) {
  457. // Used below:
  458. // гласность - example for 2 byte utf8, Russian word "glasnost"
  459. // 屋 - example for 3 byte utf8, Chinese word for "house"
  460. // 🌎 - example for 4 byte utf8: 0xF0 0x9F 0x8C 0x8E; utf16: 0xD83C 0xDF0E.
  461. // 🌙 - example for escapes: utf8: 0xF0 0x9F 0x8C 0x99; utf16: 0xD83C 0xDF19.
  462. // We provide the moon with json escape, but the earth as utf8 input.
  463. // Either way they arrive as utf8 (after decoding in log_.str()).
  464. std::string json =
  465. "{"
  466. "\"escapes\": \"\\uD83C\\uDF19\","
  467. "\"2 byte\":\"гласность\","
  468. "\"3 byte\":\"屋\","
  469. "\"4 byte\":\"🌎\""
  470. "}";
  471. ParseJSON(SpanFrom(json), &log_);
  472. EXPECT_TRUE(log_.status().ok());
  473. EXPECT_EQ(
  474. "map begin\n"
  475. "string16: escapes\n"
  476. "string16: 🌙\n"
  477. "string16: 2 byte\n"
  478. "string16: гласность\n"
  479. "string16: 3 byte\n"
  480. "string16: 屋\n"
  481. "string16: 4 byte\n"
  482. "string16: 🌎\n"
  483. "map end\n",
  484. log_.str());
  485. }
  486. TEST_F(JsonParserTest, UnprocessedInputRemainsError) {
  487. // Trailing junk after the valid JSON.
  488. std::string json = "{\"foo\": 3.1415} junk";
  489. size_t junk_idx = json.find("junk");
  490. EXPECT_NE(junk_idx, std::string::npos);
  491. ParseJSON(SpanFrom(json), &log_);
  492. EXPECT_THAT(log_.status(),
  493. StatusIs(Error::JSON_PARSER_UNPROCESSED_INPUT_REMAINS, junk_idx));
  494. EXPECT_EQ("", log_.str());
  495. }
  496. std::string MakeNestedJson(int depth) {
  497. std::string json;
  498. for (int ii = 0; ii < depth; ++ii)
  499. json += "{\"foo\":";
  500. json += "42";
  501. for (int ii = 0; ii < depth; ++ii)
  502. json += "}";
  503. return json;
  504. }
  505. TEST_F(JsonParserTest, StackLimitExceededError_BelowLimit) {
  506. // kStackLimit is 300 (see json_parser.cc). First let's
  507. // try with a small nested example.
  508. std::string json_3 = MakeNestedJson(3);
  509. ParseJSON(SpanFrom(json_3), &log_);
  510. EXPECT_TRUE(log_.status().ok());
  511. EXPECT_EQ(
  512. "map begin\n"
  513. "string16: foo\n"
  514. "map begin\n"
  515. "string16: foo\n"
  516. "map begin\n"
  517. "string16: foo\n"
  518. "int: 42\n"
  519. "map end\n"
  520. "map end\n"
  521. "map end\n",
  522. log_.str());
  523. }
  524. TEST_F(JsonParserTest, StackLimitExceededError_AtLimit) {
  525. // Now with kStackLimit (300).
  526. std::string json_limit = MakeNestedJson(300);
  527. ParseJSON(span<uint8_t>(reinterpret_cast<const uint8_t*>(json_limit.data()),
  528. json_limit.size()),
  529. &log_);
  530. EXPECT_THAT(log_.status(), StatusIsOk());
  531. }
  532. TEST_F(JsonParserTest, StackLimitExceededError_AboveLimit) {
  533. // Now with kStackLimit + 1 (301) - it exceeds in the innermost instance.
  534. std::string exceeded = MakeNestedJson(301);
  535. ParseJSON(SpanFrom(exceeded), &log_);
  536. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED,
  537. strlen("{\"foo\":") * 301));
  538. }
  539. TEST_F(JsonParserTest, StackLimitExceededError_WayAboveLimit) {
  540. // Now way past the limit. Still, the point of exceeding is 301.
  541. std::string far_out = MakeNestedJson(320);
  542. ParseJSON(SpanFrom(far_out), &log_);
  543. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_STACK_LIMIT_EXCEEDED,
  544. strlen("{\"foo\":") * 301));
  545. }
  546. TEST_F(JsonParserTest, NoInputError) {
  547. std::string json = "";
  548. ParseJSON(SpanFrom(json), &log_);
  549. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_NO_INPUT, 0u));
  550. EXPECT_EQ("", log_.str());
  551. }
  552. TEST_F(JsonParserTest, InvalidTokenError) {
  553. std::string json = "|";
  554. ParseJSON(SpanFrom(json), &log_);
  555. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_INVALID_TOKEN, 0u));
  556. EXPECT_EQ("", log_.str());
  557. }
  558. TEST_F(JsonParserTest, InvalidNumberError) {
  559. // Mantissa exceeds max (the constant used here is int64_t max).
  560. std::string json = "1E9223372036854775807";
  561. ParseJSON(SpanFrom(json), &log_);
  562. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_INVALID_NUMBER, 0u));
  563. EXPECT_EQ("", log_.str());
  564. }
  565. TEST_F(JsonParserTest, InvalidStringError) {
  566. // \x22 is an unsupported escape sequence
  567. std::string json = "\"foo\\x22\"";
  568. ParseJSON(SpanFrom(json), &log_);
  569. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_INVALID_STRING, 0u));
  570. EXPECT_EQ("", log_.str());
  571. }
  572. TEST_F(JsonParserTest, UnexpectedArrayEndError) {
  573. std::string json = "[1,2,]";
  574. ParseJSON(SpanFrom(json), &log_);
  575. EXPECT_THAT(log_.status(),
  576. StatusIs(Error::JSON_PARSER_UNEXPECTED_ARRAY_END, 5u));
  577. EXPECT_EQ("", log_.str());
  578. }
  579. TEST_F(JsonParserTest, CommaOrArrayEndExpectedError) {
  580. std::string json = "[1,2 2";
  581. ParseJSON(SpanFrom(json), &log_);
  582. EXPECT_THAT(log_.status(),
  583. StatusIs(Error::JSON_PARSER_COMMA_OR_ARRAY_END_EXPECTED, 5u));
  584. EXPECT_EQ("", log_.str());
  585. }
  586. TEST_F(JsonParserTest, StringLiteralExpectedError) {
  587. // There's an error because the key bar, a string, is not terminated.
  588. std::string json = "{\"foo\": 3.1415, \"bar: 31415e-4}";
  589. ParseJSON(SpanFrom(json), &log_);
  590. EXPECT_THAT(log_.status(),
  591. StatusIs(Error::JSON_PARSER_STRING_LITERAL_EXPECTED, 16u));
  592. EXPECT_EQ("", log_.str());
  593. }
  594. TEST_F(JsonParserTest, ColonExpectedError) {
  595. std::string json = "{\"foo\", 42}";
  596. ParseJSON(SpanFrom(json), &log_);
  597. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_COLON_EXPECTED, 6u));
  598. EXPECT_EQ("", log_.str());
  599. }
  600. TEST_F(JsonParserTest, UnexpectedMapEndError) {
  601. std::string json = "{\"foo\": 42, }";
  602. ParseJSON(SpanFrom(json), &log_);
  603. EXPECT_THAT(log_.status(),
  604. StatusIs(Error::JSON_PARSER_UNEXPECTED_MAP_END, 12u));
  605. EXPECT_EQ("", log_.str());
  606. }
  607. TEST_F(JsonParserTest, CommaOrMapEndExpectedError) {
  608. // The second separator should be a comma.
  609. std::string json = "{\"foo\": 3.1415: \"bar\": 0}";
  610. ParseJSON(SpanFrom(json), &log_);
  611. EXPECT_THAT(log_.status(),
  612. StatusIs(Error::JSON_PARSER_COMMA_OR_MAP_END_EXPECTED, 14u));
  613. EXPECT_EQ("", log_.str());
  614. }
  615. TEST_F(JsonParserTest, ValueExpectedError) {
  616. std::string json = "}";
  617. ParseJSON(SpanFrom(json), &log_);
  618. EXPECT_THAT(log_.status(), StatusIs(Error::JSON_PARSER_VALUE_EXPECTED, 0u));
  619. EXPECT_EQ("", log_.str());
  620. }
  621. template <typename T>
  622. class ConvertJSONToCBORTest : public ::testing::Test {};
  623. using ContainerTestTypes = ::testing::Types<std::vector<uint8_t>, std::string>;
  624. TYPED_TEST_SUITE(ConvertJSONToCBORTest, ContainerTestTypes);
  625. TYPED_TEST(ConvertJSONToCBORTest, RoundTripValidJson) {
  626. for (const std::string& json_in : {
  627. "{\"msg\":\"Hello, world.\",\"lst\":[1,2,3]}",
  628. "3.1415",
  629. "false",
  630. "true",
  631. "\"Hello, world.\"",
  632. "[1,2,3]",
  633. "[]",
  634. }) {
  635. SCOPED_TRACE(json_in);
  636. TypeParam json(json_in.begin(), json_in.end());
  637. std::vector<uint8_t> cbor;
  638. {
  639. Status status = ConvertJSONToCBOR(SpanFrom(json), &cbor);
  640. EXPECT_THAT(status, StatusIsOk());
  641. }
  642. TypeParam roundtrip_json;
  643. {
  644. Status status = ConvertCBORToJSON(SpanFrom(cbor), &roundtrip_json);
  645. EXPECT_THAT(status, StatusIsOk());
  646. }
  647. EXPECT_EQ(json, roundtrip_json);
  648. }
  649. }
  650. TYPED_TEST(ConvertJSONToCBORTest, RoundTripValidJson16) {
  651. std::vector<uint16_t> json16 = {
  652. '{', '"', 'm', 's', 'g', '"', ':', '"', 'H', 'e', 'l', 'l',
  653. 'o', ',', ' ', 0xd83c, 0xdf0e, '.', '"', ',', '"', 'l', 's', 't',
  654. '"', ':', '[', '1', ',', '2', ',', '3', ']', '}'};
  655. std::vector<uint8_t> cbor;
  656. {
  657. Status status =
  658. ConvertJSONToCBOR(span<uint16_t>(json16.data(), json16.size()), &cbor);
  659. EXPECT_THAT(status, StatusIsOk());
  660. }
  661. TypeParam roundtrip_json;
  662. {
  663. Status status = ConvertCBORToJSON(SpanFrom(cbor), &roundtrip_json);
  664. EXPECT_THAT(status, StatusIsOk());
  665. }
  666. std::string json = "{\"msg\":\"Hello, \\ud83c\\udf0e.\",\"lst\":[1,2,3]}";
  667. TypeParam expected_json(json.begin(), json.end());
  668. EXPECT_EQ(expected_json, roundtrip_json);
  669. }
  670. } // namespace json
  671. } // namespace crdtp