decoder.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. """Implementation of JSONDecoder
  2. """
  3. import re
  4. import sys
  5. import struct
  6. from simplejson.scanner import make_scanner
  7. def _import_c_scanstring():
  8. try:
  9. from simplejson._speedups import scanstring
  10. return scanstring
  11. except ImportError:
  12. return None
  13. c_scanstring = _import_c_scanstring()
  14. __all__ = ['JSONDecoder']
  15. FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  16. def _floatconstants():
  17. _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
  18. # The struct module in Python 2.4 would get frexp() out of range here
  19. # when an endian is specified in the format string. Fixed in Python 2.5+
  20. if sys.byteorder != 'big':
  21. _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
  22. nan, inf = struct.unpack('dd', _BYTES)
  23. return nan, inf, -inf
  24. NaN, PosInf, NegInf = _floatconstants()
  25. class JSONDecodeError(ValueError):
  26. """Subclass of ValueError with the following additional properties:
  27. msg: The unformatted error message
  28. doc: The JSON document being parsed
  29. pos: The start index of doc where parsing failed
  30. end: The end index of doc where parsing failed (may be None)
  31. lineno: The line corresponding to pos
  32. colno: The column corresponding to pos
  33. endlineno: The line corresponding to end (may be None)
  34. endcolno: The column corresponding to end (may be None)
  35. """
  36. def __init__(self, msg, doc, pos, end=None):
  37. ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
  38. self.msg = msg
  39. self.doc = doc
  40. self.pos = pos
  41. self.end = end
  42. self.lineno, self.colno = linecol(doc, pos)
  43. if end is not None:
  44. self.endlineno, self.endcolno = linecol(doc, end)
  45. else:
  46. self.endlineno, self.endcolno = None, None
  47. def linecol(doc, pos):
  48. lineno = doc.count('\n', 0, pos) + 1
  49. if lineno == 1:
  50. colno = pos
  51. else:
  52. colno = pos - doc.rindex('\n', 0, pos)
  53. return lineno, colno
  54. def errmsg(msg, doc, pos, end=None):
  55. # Note that this function is called from _speedups
  56. lineno, colno = linecol(doc, pos)
  57. if end is None:
  58. #fmt = '{0}: line {1} column {2} (char {3})'
  59. #return fmt.format(msg, lineno, colno, pos)
  60. fmt = '%s: line %d column %d (char %d)'
  61. return fmt % (msg, lineno, colno, pos)
  62. endlineno, endcolno = linecol(doc, end)
  63. #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
  64. #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
  65. fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
  66. return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
  67. _CONSTANTS = {
  68. '-Infinity': NegInf,
  69. 'Infinity': PosInf,
  70. 'NaN': NaN,
  71. }
  72. STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  73. BACKSLASH = {
  74. '"': u'"', '\\': u'\\', '/': u'/',
  75. 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
  76. }
  77. DEFAULT_ENCODING = "utf-8"
  78. def py_scanstring(s, end, encoding=None, strict=True,
  79. _b=BACKSLASH, _m=STRINGCHUNK.match):
  80. """Scan the string s for a JSON string. End is the index of the
  81. character in s after the quote that started the JSON string.
  82. Unescapes all valid JSON string escape sequences and raises ValueError
  83. on attempt to decode an invalid string. If strict is False then literal
  84. control characters are allowed in the string.
  85. Returns a tuple of the decoded string and the index of the character in s
  86. after the end quote."""
  87. if encoding is None:
  88. encoding = DEFAULT_ENCODING
  89. chunks = []
  90. _append = chunks.append
  91. begin = end - 1
  92. while 1:
  93. chunk = _m(s, end)
  94. if chunk is None:
  95. raise JSONDecodeError(
  96. "Unterminated string starting at", s, begin)
  97. end = chunk.end()
  98. content, terminator = chunk.groups()
  99. # Content is contains zero or more unescaped string characters
  100. if content:
  101. if not isinstance(content, unicode):
  102. content = unicode(content, encoding)
  103. _append(content)
  104. # Terminator is the end of string, a literal control character,
  105. # or a backslash denoting that an escape sequence follows
  106. if terminator == '"':
  107. break
  108. elif terminator != '\\':
  109. if strict:
  110. msg = "Invalid control character %r at" % (terminator,)
  111. #msg = "Invalid control character {0!r} at".format(terminator)
  112. raise JSONDecodeError(msg, s, end)
  113. else:
  114. _append(terminator)
  115. continue
  116. try:
  117. esc = s[end]
  118. except IndexError:
  119. raise JSONDecodeError(
  120. "Unterminated string starting at", s, begin)
  121. # If not a unicode escape sequence, must be in the lookup table
  122. if esc != 'u':
  123. try:
  124. char = _b[esc]
  125. except KeyError:
  126. msg = "Invalid \\escape: " + repr(esc)
  127. raise JSONDecodeError(msg, s, end)
  128. end += 1
  129. else:
  130. # Unicode escape sequence
  131. esc = s[end + 1:end + 5]
  132. next_end = end + 5
  133. if len(esc) != 4:
  134. msg = "Invalid \\uXXXX escape"
  135. raise JSONDecodeError(msg, s, end)
  136. uni = int(esc, 16)
  137. # Check for surrogate pair on UCS-4 systems
  138. if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
  139. msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
  140. if not s[end + 5:end + 7] == '\\u':
  141. raise JSONDecodeError(msg, s, end)
  142. esc2 = s[end + 7:end + 11]
  143. if len(esc2) != 4:
  144. raise JSONDecodeError(msg, s, end)
  145. uni2 = int(esc2, 16)
  146. uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
  147. next_end += 6
  148. char = unichr(uni)
  149. end = next_end
  150. # Append the unescaped character
  151. _append(char)
  152. return u''.join(chunks), end
  153. # Use speedup if available
  154. scanstring = c_scanstring or py_scanstring
  155. WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
  156. WHITESPACE_STR = ' \t\n\r'
  157. def JSONObject((s, end), encoding, strict, scan_once, object_hook,
  158. object_pairs_hook, memo=None,
  159. _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  160. # Backwards compatibility
  161. if memo is None:
  162. memo = {}
  163. memo_get = memo.setdefault
  164. pairs = []
  165. # Use a slice to prevent IndexError from being raised, the following
  166. # check will raise a more specific ValueError if the string is empty
  167. nextchar = s[end:end + 1]
  168. # Normally we expect nextchar == '"'
  169. if nextchar != '"':
  170. if nextchar in _ws:
  171. end = _w(s, end).end()
  172. nextchar = s[end:end + 1]
  173. # Trivial empty object
  174. if nextchar == '}':
  175. if object_pairs_hook is not None:
  176. result = object_pairs_hook(pairs)
  177. return result, end + 1
  178. pairs = {}
  179. if object_hook is not None:
  180. pairs = object_hook(pairs)
  181. return pairs, end + 1
  182. elif nextchar != '"':
  183. raise JSONDecodeError(
  184. "Expecting property name enclosed in double quotes",
  185. s, end)
  186. end += 1
  187. while True:
  188. key, end = scanstring(s, end, encoding, strict)
  189. key = memo_get(key, key)
  190. # To skip some function call overhead we optimize the fast paths where
  191. # the JSON key separator is ": " or just ":".
  192. if s[end:end + 1] != ':':
  193. end = _w(s, end).end()
  194. if s[end:end + 1] != ':':
  195. raise JSONDecodeError("Expecting ':' delimiter", s, end)
  196. end += 1
  197. try:
  198. if s[end] in _ws:
  199. end += 1
  200. if s[end] in _ws:
  201. end = _w(s, end + 1).end()
  202. except IndexError:
  203. pass
  204. try:
  205. value, end = scan_once(s, end)
  206. except StopIteration:
  207. raise JSONDecodeError("Expecting object", s, end)
  208. pairs.append((key, value))
  209. try:
  210. nextchar = s[end]
  211. if nextchar in _ws:
  212. end = _w(s, end + 1).end()
  213. nextchar = s[end]
  214. except IndexError:
  215. nextchar = ''
  216. end += 1
  217. if nextchar == '}':
  218. break
  219. elif nextchar != ',':
  220. raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
  221. try:
  222. nextchar = s[end]
  223. if nextchar in _ws:
  224. end += 1
  225. nextchar = s[end]
  226. if nextchar in _ws:
  227. end = _w(s, end + 1).end()
  228. nextchar = s[end]
  229. except IndexError:
  230. nextchar = ''
  231. end += 1
  232. if nextchar != '"':
  233. raise JSONDecodeError(
  234. "Expecting property name enclosed in double quotes",
  235. s, end - 1)
  236. if object_pairs_hook is not None:
  237. result = object_pairs_hook(pairs)
  238. return result, end
  239. pairs = dict(pairs)
  240. if object_hook is not None:
  241. pairs = object_hook(pairs)
  242. return pairs, end
  243. def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  244. values = []
  245. nextchar = s[end:end + 1]
  246. if nextchar in _ws:
  247. end = _w(s, end + 1).end()
  248. nextchar = s[end:end + 1]
  249. # Look-ahead for trivial empty array
  250. if nextchar == ']':
  251. return values, end + 1
  252. _append = values.append
  253. while True:
  254. try:
  255. value, end = scan_once(s, end)
  256. except StopIteration:
  257. raise JSONDecodeError("Expecting object", s, end)
  258. _append(value)
  259. nextchar = s[end:end + 1]
  260. if nextchar in _ws:
  261. end = _w(s, end + 1).end()
  262. nextchar = s[end:end + 1]
  263. end += 1
  264. if nextchar == ']':
  265. break
  266. elif nextchar != ',':
  267. raise JSONDecodeError("Expecting ',' delimiter", s, end)
  268. try:
  269. if s[end] in _ws:
  270. end += 1
  271. if s[end] in _ws:
  272. end = _w(s, end + 1).end()
  273. except IndexError:
  274. pass
  275. return values, end
  276. class JSONDecoder(object):
  277. """Simple JSON <http://json.org> decoder
  278. Performs the following translations in decoding by default:
  279. +---------------+-------------------+
  280. | JSON | Python |
  281. +===============+===================+
  282. | object | dict |
  283. +---------------+-------------------+
  284. | array | list |
  285. +---------------+-------------------+
  286. | string | unicode |
  287. +---------------+-------------------+
  288. | number (int) | int, long |
  289. +---------------+-------------------+
  290. | number (real) | float |
  291. +---------------+-------------------+
  292. | true | True |
  293. +---------------+-------------------+
  294. | false | False |
  295. +---------------+-------------------+
  296. | null | None |
  297. +---------------+-------------------+
  298. It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
  299. their corresponding ``float`` values, which is outside the JSON spec.
  300. """
  301. def __init__(self, encoding=None, object_hook=None, parse_float=None,
  302. parse_int=None, parse_constant=None, strict=True,
  303. object_pairs_hook=None):
  304. """
  305. *encoding* determines the encoding used to interpret any
  306. :class:`str` objects decoded by this instance (``'utf-8'`` by
  307. default). It has no effect when decoding :class:`unicode` objects.
  308. Note that currently only encodings that are a superset of ASCII work,
  309. strings of other encodings should be passed in as :class:`unicode`.
  310. *object_hook*, if specified, will be called with the result of every
  311. JSON object decoded and its return value will be used in place of the
  312. given :class:`dict`. This can be used to provide custom
  313. deserializations (e.g. to support JSON-RPC class hinting).
  314. *object_pairs_hook* is an optional function that will be called with
  315. the result of any object literal decode with an ordered list of pairs.
  316. The return value of *object_pairs_hook* will be used instead of the
  317. :class:`dict`. This feature can be used to implement custom decoders
  318. that rely on the order that the key and value pairs are decoded (for
  319. example, :func:`collections.OrderedDict` will remember the order of
  320. insertion). If *object_hook* is also defined, the *object_pairs_hook*
  321. takes priority.
  322. *parse_float*, if specified, will be called with the string of every
  323. JSON float to be decoded. By default, this is equivalent to
  324. ``float(num_str)``. This can be used to use another datatype or parser
  325. for JSON floats (e.g. :class:`decimal.Decimal`).
  326. *parse_int*, if specified, will be called with the string of every
  327. JSON int to be decoded. By default, this is equivalent to
  328. ``int(num_str)``. This can be used to use another datatype or parser
  329. for JSON integers (e.g. :class:`float`).
  330. *parse_constant*, if specified, will be called with one of the
  331. following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This
  332. can be used to raise an exception if invalid JSON numbers are
  333. encountered.
  334. *strict* controls the parser's behavior when it encounters an
  335. invalid control character in a string. The default setting of
  336. ``True`` means that unescaped control characters are parse errors, if
  337. ``False`` then control characters will be allowed in strings.
  338. """
  339. self.encoding = encoding
  340. self.object_hook = object_hook
  341. self.object_pairs_hook = object_pairs_hook
  342. self.parse_float = parse_float or float
  343. self.parse_int = parse_int or int
  344. self.parse_constant = parse_constant or _CONSTANTS.__getitem__
  345. self.strict = strict
  346. self.parse_object = JSONObject
  347. self.parse_array = JSONArray
  348. self.parse_string = scanstring
  349. self.memo = {}
  350. self.scan_once = make_scanner(self)
  351. def decode(self, s, _w=WHITESPACE.match):
  352. """Return the Python representation of ``s`` (a ``str`` or ``unicode``
  353. instance containing a JSON document)
  354. """
  355. obj, end = self.raw_decode(s)
  356. end = _w(s, end).end()
  357. if end != len(s):
  358. raise JSONDecodeError("Extra data", s, end, len(s))
  359. return obj
  360. def raw_decode(self, s, idx=0, _w=WHITESPACE.match):
  361. """Decode a JSON document from ``s`` (a ``str`` or ``unicode``
  362. beginning with a JSON document) and return a 2-tuple of the Python
  363. representation and the index in ``s`` where the document ended.
  364. Optionally, ``idx`` can be used to specify an offset in ``s`` where
  365. the JSON document begins.
  366. This can be used to decode a JSON document from a string that may
  367. have extraneous data at the end.
  368. """
  369. try:
  370. obj, end = self.scan_once(s, idx=_w(s, idx).end())
  371. except StopIteration:
  372. raise JSONDecodeError("No JSON object could be decoded", s, idx)
  373. return obj, end