llex.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458
  1. /*
  2. ** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $
  3. ** Lexical Analyzer
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define llex_c
  7. #define LUA_CORE
  8. #define LUAC_CROSS_FILE
  9. #include "lua.h"
  10. #include <ctype.h>
  11. #include <locale.h>
  12. #include <string.h>
  13. #include "ldo.h"
  14. #include "llex.h"
  15. #include "lobject.h"
  16. #include "lparser.h"
  17. #include "lstate.h"
  18. #include "lstring.h"
  19. #include "ltable.h"
  20. #include "lzio.h"
  21. #define next(ls) (ls->current = zgetc(ls->z))
  22. #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
  23. /* ORDER RESERVED */
  24. const char *const luaX_tokens [] = {
  25. "and", "break", "do", "else", "elseif",
  26. "end", "false", "for", "function", "if",
  27. "in", "local", "nil", "not", "or", "repeat",
  28. "return", "then", "true", "until", "while",
  29. "..", "...", "==", ">=", "<=", "~=",
  30. "<number>", "<name>", "<string>", "<eof>",
  31. NULL
  32. };
  33. #define save_and_next(ls) (save(ls, ls->current), next(ls))
  34. static void save (LexState *ls, int c) {
  35. Mbuffer *b = ls->buff;
  36. if (b->n + 1 > b->buffsize) {
  37. size_t newsize;
  38. if (b->buffsize >= MAX_SIZET/2)
  39. luaX_lexerror(ls, "lexical element too long", 0);
  40. newsize = b->buffsize * 2;
  41. luaZ_resizebuffer(ls->L, b, newsize);
  42. }
  43. b->buffer[b->n++] = cast(char, c);
  44. }
  45. void luaX_init (lua_State *L) {
  46. }
  47. #define MAXSRC 80
  48. const char *luaX_token2str (LexState *ls, int token) {
  49. if (token < FIRST_RESERVED) {
  50. lua_assert(token == cast(unsigned char, token));
  51. return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  52. luaO_pushfstring(ls->L, "%c", token);
  53. }
  54. else
  55. return luaX_tokens[token-FIRST_RESERVED];
  56. }
  57. static const char *txtToken (LexState *ls, int token) {
  58. switch (token) {
  59. case TK_NAME:
  60. case TK_STRING:
  61. case TK_NUMBER:
  62. save(ls, '\0');
  63. return luaZ_buffer(ls->buff);
  64. default:
  65. return luaX_token2str(ls, token);
  66. }
  67. }
  68. void luaX_lexerror (LexState *ls, const char *msg, int token) {
  69. char buff[MAXSRC];
  70. luaO_chunkid(buff, getstr(ls->source), MAXSRC);
  71. msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
  72. if (token)
  73. luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
  74. luaD_throw(ls->L, LUA_ERRSYNTAX);
  75. }
  76. void luaX_syntaxerror (LexState *ls, const char *msg) {
  77. luaX_lexerror(ls, msg, ls->t.token);
  78. }
  79. TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
  80. lua_State *L = ls->L;
  81. TString *ts = luaS_newlstr(L, str, l);
  82. TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */
  83. if (ttisnil(o)) {
  84. setbvalue(o, 1); /* make sure `str' will not be collected */
  85. luaC_checkGC(L);
  86. }
  87. return ts;
  88. }
  89. static void inclinenumber (LexState *ls) {
  90. int old = ls->current;
  91. lua_assert(currIsNewline(ls));
  92. next(ls); /* skip `\n' or `\r' */
  93. if (currIsNewline(ls) && ls->current != old)
  94. next(ls); /* skip `\n\r' or `\r\n' */
  95. if (++ls->linenumber >= MAX_INT)
  96. luaX_syntaxerror(ls, "chunk has too many lines");
  97. }
  98. void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
  99. ls->decpoint = '.';
  100. ls->L = L;
  101. ls->lookahead.token = TK_EOS; /* no look-ahead token */
  102. ls->z = z;
  103. ls->fs = NULL;
  104. ls->linenumber = 1;
  105. ls->lastline = 1;
  106. ls->source = source;
  107. luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
  108. next(ls); /* read first char */
  109. }
  110. /*
  111. ** =======================================================
  112. ** LEXICAL ANALYZER
  113. ** =======================================================
  114. */
  115. static int check_next (LexState *ls, const char *set) {
  116. if (!strchr(set, ls->current))
  117. return 0;
  118. save_and_next(ls);
  119. return 1;
  120. }
  121. static void buffreplace (LexState *ls, char from, char to) {
  122. size_t n = luaZ_bufflen(ls->buff);
  123. char *p = luaZ_buffer(ls->buff);
  124. while (n--)
  125. if (p[n] == from) p[n] = to;
  126. }
  127. static void trydecpoint (LexState *ls, SemInfo *seminfo) {
  128. /* format error: try to update decimal point separator */
  129. struct lconv *cv = localeconv();
  130. char old = ls->decpoint;
  131. ls->decpoint = (cv ? cv->decimal_point[0] : '.');
  132. buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */
  133. if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
  134. /* format error with correct decimal point: no more options */
  135. buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */
  136. luaX_lexerror(ls, "malformed number", TK_NUMBER);
  137. }
  138. }
  139. /* LUA_NUMBER */
  140. static void read_numeral (LexState *ls, SemInfo *seminfo) {
  141. lua_assert(isdigit(ls->current));
  142. do {
  143. save_and_next(ls);
  144. } while (isdigit(ls->current) || ls->current == '.');
  145. if (check_next(ls, "Ee")) /* `E'? */
  146. check_next(ls, "+-"); /* optional exponent sign */
  147. while (isalnum(ls->current) || ls->current == '_')
  148. save_and_next(ls);
  149. save(ls, '\0');
  150. buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */
  151. if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */
  152. trydecpoint(ls, seminfo); /* try to update decimal point separator */
  153. }
  154. static int skip_sep (LexState *ls) {
  155. int count = 0;
  156. int s = ls->current;
  157. lua_assert(s == '[' || s == ']');
  158. save_and_next(ls);
  159. while (ls->current == '=') {
  160. save_and_next(ls);
  161. count++;
  162. }
  163. return (ls->current == s) ? count : (-count) - 1;
  164. }
  165. static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
  166. int cont = 0;
  167. (void)(cont); /* avoid warnings when `cont' is not used */
  168. save_and_next(ls); /* skip 2nd `[' */
  169. if (currIsNewline(ls)) /* string starts with a newline? */
  170. inclinenumber(ls); /* skip it */
  171. for (;;) {
  172. switch (ls->current) {
  173. case EOZ:
  174. luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
  175. "unfinished long comment", TK_EOS);
  176. break; /* to avoid warnings */
  177. #if defined(LUA_COMPAT_LSTR)
  178. case '[': {
  179. if (skip_sep(ls) == sep) {
  180. save_and_next(ls); /* skip 2nd `[' */
  181. cont++;
  182. #if LUA_COMPAT_LSTR == 1
  183. if (sep == 0)
  184. luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
  185. #endif
  186. }
  187. break;
  188. }
  189. #endif
  190. case ']': {
  191. if (skip_sep(ls) == sep) {
  192. save_and_next(ls); /* skip 2nd `]' */
  193. #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
  194. cont--;
  195. if (sep == 0 && cont >= 0) break;
  196. #endif
  197. goto endloop;
  198. }
  199. break;
  200. }
  201. case '\n':
  202. case '\r': {
  203. save(ls, '\n');
  204. inclinenumber(ls);
  205. if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
  206. break;
  207. }
  208. default: {
  209. if (seminfo) save_and_next(ls);
  210. else next(ls);
  211. }
  212. }
  213. } endloop:
  214. if (seminfo)
  215. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
  216. luaZ_bufflen(ls->buff) - 2*(2 + sep));
  217. }
  218. static void read_string (LexState *ls, int del, SemInfo *seminfo) {
  219. save_and_next(ls);
  220. while (ls->current != del) {
  221. switch (ls->current) {
  222. case EOZ:
  223. luaX_lexerror(ls, "unfinished string", TK_EOS);
  224. continue; /* to avoid warnings */
  225. case '\n':
  226. case '\r':
  227. luaX_lexerror(ls, "unfinished string", TK_STRING);
  228. continue; /* to avoid warnings */
  229. case '\\': {
  230. int c;
  231. next(ls); /* do not save the `\' */
  232. switch (ls->current) {
  233. case 'a': c = '\a'; break;
  234. case 'b': c = '\b'; break;
  235. case 'f': c = '\f'; break;
  236. case 'n': c = '\n'; break;
  237. case 'r': c = '\r'; break;
  238. case 't': c = '\t'; break;
  239. case 'v': c = '\v'; break;
  240. case '\n': /* go through */
  241. case '\r': save(ls, '\n'); inclinenumber(ls); continue;
  242. case EOZ: continue; /* will raise an error next loop */
  243. default: {
  244. if (!isdigit(ls->current))
  245. save_and_next(ls); /* handles \\, \", \', and \? */
  246. else { /* \xxx */
  247. int i = 0;
  248. c = 0;
  249. do {
  250. c = 10*c + (ls->current-'0');
  251. next(ls);
  252. } while (++i<3 && isdigit(ls->current));
  253. if (c > UCHAR_MAX)
  254. luaX_lexerror(ls, "escape sequence too large", TK_STRING);
  255. save(ls, c);
  256. }
  257. continue;
  258. }
  259. }
  260. save(ls, c);
  261. next(ls);
  262. continue;
  263. }
  264. default:
  265. save_and_next(ls);
  266. }
  267. }
  268. save_and_next(ls); /* skip delimiter */
  269. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
  270. luaZ_bufflen(ls->buff) - 2);
  271. }
  272. static int llex (LexState *ls, SemInfo *seminfo) {
  273. luaZ_resetbuffer(ls->buff);
  274. for (;;) {
  275. switch (ls->current) {
  276. case '\n':
  277. case '\r': {
  278. inclinenumber(ls);
  279. continue;
  280. }
  281. case '-': {
  282. next(ls);
  283. if (ls->current != '-') return '-';
  284. /* else is a comment */
  285. next(ls);
  286. if (ls->current == '[') {
  287. int sep = skip_sep(ls);
  288. luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */
  289. if (sep >= 0) {
  290. read_long_string(ls, NULL, sep); /* long comment */
  291. luaZ_resetbuffer(ls->buff);
  292. continue;
  293. }
  294. }
  295. /* else short comment */
  296. while (!currIsNewline(ls) && ls->current != EOZ)
  297. next(ls);
  298. continue;
  299. }
  300. case '[': {
  301. int sep = skip_sep(ls);
  302. if (sep >= 0) {
  303. read_long_string(ls, seminfo, sep);
  304. return TK_STRING;
  305. }
  306. else if (sep == -1) return '[';
  307. else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
  308. }
  309. case '=': {
  310. next(ls);
  311. if (ls->current != '=') return '=';
  312. else { next(ls); return TK_EQ; }
  313. }
  314. case '<': {
  315. next(ls);
  316. if (ls->current != '=') return '<';
  317. else { next(ls); return TK_LE; }
  318. }
  319. case '>': {
  320. next(ls);
  321. if (ls->current != '=') return '>';
  322. else { next(ls); return TK_GE; }
  323. }
  324. case '~': {
  325. next(ls);
  326. if (ls->current != '=') return '~';
  327. else { next(ls); return TK_NE; }
  328. }
  329. case '"':
  330. case '\'': {
  331. read_string(ls, ls->current, seminfo);
  332. return TK_STRING;
  333. }
  334. case '.': {
  335. save_and_next(ls);
  336. if (check_next(ls, ".")) {
  337. if (check_next(ls, "."))
  338. return TK_DOTS; /* ... */
  339. else return TK_CONCAT; /* .. */
  340. }
  341. else if (!isdigit(ls->current)) return '.';
  342. else {
  343. read_numeral(ls, seminfo);
  344. return TK_NUMBER;
  345. }
  346. }
  347. case EOZ: {
  348. return TK_EOS;
  349. }
  350. default: {
  351. if (isspace(ls->current)) {
  352. lua_assert(!currIsNewline(ls));
  353. next(ls);
  354. continue;
  355. }
  356. else if (isdigit(ls->current)) {
  357. read_numeral(ls, seminfo);
  358. return TK_NUMBER;
  359. }
  360. else if (isalpha(ls->current) || ls->current == '_') {
  361. /* identifier or reserved word */
  362. TString *ts;
  363. int i;
  364. do {
  365. save_and_next(ls);
  366. } while (isalnum(ls->current) || ls->current == '_');
  367. /* look for reserved word */
  368. save(ls, '\0');
  369. for (i = 0; i < NUM_RESERVED; i++)
  370. if (!strcmp(luaX_tokens[i], luaZ_buffer(ls->buff)))
  371. return i + FIRST_RESERVED;
  372. ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
  373. luaZ_bufflen(ls->buff) - 1);
  374. seminfo->ts = ts;
  375. return TK_NAME;
  376. }
  377. else {
  378. int c = ls->current;
  379. next(ls);
  380. return c; /* single-char tokens (+ - / ...) */
  381. }
  382. }
  383. }
  384. }
  385. }
  386. void luaX_next (LexState *ls) {
  387. ls->lastline = ls->linenumber;
  388. if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
  389. ls->t = ls->lookahead; /* use this one */
  390. ls->lookahead.token = TK_EOS; /* and discharge it */
  391. }
  392. else
  393. ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
  394. }
  395. void luaX_lookahead (LexState *ls) {
  396. lua_assert(ls->lookahead.token == TK_EOS);
  397. ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  398. }