llex.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. /*
  2. ** $Id: llex.c,v 2.96.1.1 2017/04/19 17:20:42 roberto Exp $
  3. ** Lexical Analyzer
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define llex_c
  7. #define LUA_CORE
  8. #include "lprefix.h"
  9. #include <locale.h>
  10. #include <string.h>
  11. #include "lua.h"
  12. #include "lctype.h"
  13. #include "ldebug.h"
  14. #include "ldo.h"
  15. #include "lgc.h"
  16. #include "llex.h"
  17. #include "lobject.h"
  18. #include "lparser.h"
  19. #include "lstate.h"
  20. #include "lstring.h"
  21. #include "ltable.h"
  22. #include "lzio.h"
  23. #define next(ls) (ls->current = zgetc(ls->z))
  24. #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
  25. /* ORDER RESERVED */
  26. static const char *const luaX_tokens [] = {
  27. "and", "break", "do", "else", "elseif",
  28. "end", "false", "for", "function", "goto", "if",
  29. "in", "local", "nil", "not", "or", "repeat",
  30. "return", "then", "true", "until", "while",
  31. "//", "..", "...", "==", ">=", "<=", "~=",
  32. "<<", ">>", "::", "<eof>",
  33. "<number>", "<integer>", "<name>", "<string>"
  34. };
  35. #define save_and_next(ls) (save(ls, ls->current), next(ls))
  36. static l_noret lexerror (LexState *ls, const char *msg, int token);
  37. static void save (LexState *ls, int c) {
  38. Mbuffer *b = ls->buff;
  39. if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
  40. size_t newsize;
  41. if (luaZ_sizebuffer(b) >= MAX_SIZE/2)
  42. lexerror(ls, "lexical element too long", 0);
  43. newsize = luaZ_sizebuffer(b) * 2;
  44. luaZ_resizebuffer(ls->L, b, newsize);
  45. }
  46. b->buffer[luaZ_bufflen(b)++] = cast(char, c);
  47. }
  48. void luaX_init (lua_State *L) {
  49. int i;
  50. TString *e = luaS_newliteral(L, LUA_ENV); /* create env name */
  51. luaC_fix(L, obj2gco(e)); /* never collect this name */
  52. for (i=0; i<NUM_RESERVED; i++) {
  53. TString *ts = luaS_new(L, luaX_tokens[i]);
  54. luaC_fix(L, obj2gco(ts)); /* reserved words are never collected */
  55. if (!isLFSobj(ts)) /* if in LFS then this has been done already */
  56. ts->extra = cast_byte(i+1); /* reserved word */
  57. else
  58. lua_assert(ts->extra == cast_byte(i+1)); /* LFS version should match */
  59. }
  60. }
  61. /* Access method to expose luaX_fixed strings */
  62. const char *luaX_getstr (unsigned int i, int *extra) {
  63. if (i == sizeof(luaX_tokens)/sizeof(*luaX_tokens))
  64. return NULL;
  65. if (extra)
  66. *extra = (i<NUM_RESERVED) ? i+1 : 0;
  67. return luaX_tokens[i];
  68. }
  69. const char *luaX_token2str (LexState *ls, int token) {
  70. if (token < FIRST_RESERVED) { /* single-byte symbols? */
  71. lua_assert(token == cast_uchar(token));
  72. return luaO_pushfstring(ls->L, "'%c'", token);
  73. }
  74. else {
  75. const char *s = luaX_tokens[token - FIRST_RESERVED];
  76. if (token < TK_EOS) /* fixed format (symbols and reserved words)? */
  77. return luaO_pushfstring(ls->L, "'%s'", s);
  78. else /* names, strings, and numerals */
  79. return s;
  80. }
  81. }
  82. static const char *txtToken (LexState *ls, int token) {
  83. switch (token) {
  84. case TK_NAME: case TK_STRING:
  85. case TK_FLT: case TK_INT:
  86. save(ls, '\0');
  87. return luaO_pushfstring(ls->L, "'%s'", luaZ_buffer(ls->buff));
  88. default:
  89. return luaX_token2str(ls, token);
  90. }
  91. }
  92. static l_noret lexerror (LexState *ls, const char *msg, int token) {
  93. msg = luaG_addinfo(ls->L, msg, ls->source, ls->linenumber);
  94. if (token)
  95. luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
  96. luaD_throw(ls->L, LUA_ERRSYNTAX);
  97. }
  98. l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
  99. lexerror(ls, msg, ls->t.token);
  100. }
  101. /*
  102. ** creates a new string and anchors it in scanner's table so that
  103. ** it will not be collected until the end of the compilation
  104. ** (by that time it should be anchored somewhere)
  105. */
  106. TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
  107. lua_State *L = ls->L;
  108. TValue *o; /* entry for 'str' */
  109. TString *ts = luaS_newlstr(L, str, l); /* create new string */
  110. setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
  111. o = luaH_set(L, ls->h, L->top - 1);
  112. if (ttisnil(o)) { /* not in use yet? */
  113. /* boolean value does not need GC barrier;
  114. table has no metatable, so it does not need to invalidate cache */
  115. setbvalue(o, 1); /* t[string] = true */
  116. luaC_checkGC(L);
  117. }
  118. else { /* string already present */
  119. ts = tsvalue(keyfromval(o)); /* re-use value previously stored */
  120. }
  121. L->top--; /* remove string from stack */
  122. return ts;
  123. }
  124. /*
  125. ** increment line number and skips newline sequence (any of
  126. ** \n, \r, \n\r, or \r\n)
  127. */
  128. static void inclinenumber (LexState *ls) {
  129. int old = ls->current;
  130. lua_assert(currIsNewline(ls));
  131. next(ls); /* skip '\n' or '\r' */
  132. if (currIsNewline(ls) && ls->current != old)
  133. next(ls); /* skip '\n\r' or '\r\n' */
  134. if (++ls->linenumber >= MAX_INT)
  135. lexerror(ls, "chunk has too many lines", 0);
  136. }
  137. void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
  138. int firstchar) {
  139. ls->t.token = 0;
  140. ls->L = L;
  141. ls->current = firstchar;
  142. ls->lookahead.token = TK_EOS; /* no look-ahead token */
  143. ls->z = z;
  144. ls->fs = NULL;
  145. ls->linenumber = 1;
  146. ls->lastline = 1;
  147. ls->source = source;
  148. ls->envn = luaS_newliteral(L, LUA_ENV); /* get env name */
  149. luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
  150. }
  151. /*
  152. ** =======================================================
  153. ** LEXICAL ANALYZER
  154. ** =======================================================
  155. */
  156. static int check_next1 (LexState *ls, int c) {
  157. if (ls->current == c) {
  158. next(ls);
  159. return 1;
  160. }
  161. else return 0;
  162. }
  163. /*
  164. ** Check whether current char is in set 'set' (with two chars) and
  165. ** saves it
  166. */
  167. static int check_next2 (LexState *ls, const char *set) {
  168. lua_assert(set[2] == '\0');
  169. if (ls->current == set[0] || ls->current == set[1]) {
  170. save_and_next(ls);
  171. return 1;
  172. }
  173. else return 0;
  174. }
  175. /* LUA_NUMBER */
  176. /*
  177. ** this function is quite liberal in what it accepts, as 'luaO_str2num'
  178. ** will reject ill-formed numerals.
  179. */
  180. static int read_numeral (LexState *ls, SemInfo *seminfo) {
  181. TValue obj;
  182. const char *expo = "Ee";
  183. int first = ls->current;
  184. lua_assert(lisdigit(ls->current));
  185. save_and_next(ls);
  186. if (first == '0' && check_next2(ls, "xX")) /* hexadecimal? */
  187. expo = "Pp";
  188. for (;;) {
  189. if (check_next2(ls, expo)) /* exponent part? */
  190. check_next2(ls, "-+"); /* optional exponent sign */
  191. if (lisxdigit(ls->current))
  192. save_and_next(ls);
  193. else if (ls->current == '.')
  194. save_and_next(ls);
  195. else break;
  196. }
  197. save(ls, '\0');
  198. if (luaO_str2num(luaZ_buffer(ls->buff), &obj) == 0) /* format error? */
  199. lexerror(ls, "malformed number", TK_FLT);
  200. if (ttisinteger(&obj)) {
  201. seminfo->i = ivalue(&obj);
  202. return TK_INT;
  203. }
  204. else {
  205. lua_assert(ttisfloat(&obj));
  206. seminfo->r = fltvalue(&obj);
  207. return TK_FLT;
  208. }
  209. }
  210. /*
  211. ** skip a sequence '[=*[' or ']=*]'; if sequence is well formed, return
  212. ** its number of '='s; otherwise, return a negative number (-1 iff there
  213. ** are no '='s after initial bracket)
  214. */
  215. static int skip_sep (LexState *ls) {
  216. int count = 0;
  217. int s = ls->current;
  218. lua_assert(s == '[' || s == ']');
  219. save_and_next(ls);
  220. while (ls->current == '=') {
  221. save_and_next(ls);
  222. count++;
  223. }
  224. return (ls->current == s) ? count : (-count) - 1;
  225. }
  226. static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
  227. int line = ls->linenumber; /* initial line (for error message) */
  228. save_and_next(ls); /* skip 2nd '[' */
  229. if (currIsNewline(ls)) /* string starts with a newline? */
  230. inclinenumber(ls); /* skip it */
  231. for (;;) {
  232. switch (ls->current) {
  233. case EOZ: { /* error */
  234. const char *what = (seminfo ? "string" : "comment");
  235. const char *msg = luaO_pushfstring(ls->L,
  236. "unfinished long %s (starting at line %d)", what, line);
  237. lexerror(ls, msg, TK_EOS);
  238. break; /* to avoid warnings */
  239. }
  240. case ']': {
  241. if (skip_sep(ls) == sep) {
  242. save_and_next(ls); /* skip 2nd ']' */
  243. goto endloop;
  244. }
  245. break;
  246. }
  247. case '\n': case '\r': {
  248. save(ls, '\n');
  249. inclinenumber(ls);
  250. if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
  251. break;
  252. }
  253. default: {
  254. if (seminfo) save_and_next(ls);
  255. else next(ls);
  256. }
  257. }
  258. } endloop:
  259. if (seminfo)
  260. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
  261. luaZ_bufflen(ls->buff) - 2*(2 + sep));
  262. }
  263. static void esccheck (LexState *ls, int c, const char *msg) {
  264. if (!c) {
  265. if (ls->current != EOZ)
  266. save_and_next(ls); /* add current to buffer for error message */
  267. lexerror(ls, msg, TK_STRING);
  268. }
  269. }
  270. static int gethexa (LexState *ls) {
  271. save_and_next(ls);
  272. esccheck (ls, lisxdigit(ls->current), "hexadecimal digit expected");
  273. return luaO_hexavalue(ls->current);
  274. }
  275. static int readhexaesc (LexState *ls) {
  276. int r = gethexa(ls);
  277. r = (r << 4) + gethexa(ls);
  278. luaZ_buffremove(ls->buff, 2); /* remove saved chars from buffer */
  279. return r;
  280. }
  281. static unsigned long readutf8esc (LexState *ls) {
  282. unsigned long r;
  283. int i = 4; /* chars to be removed: '\', 'u', '{', and first digit */
  284. save_and_next(ls); /* skip 'u' */
  285. esccheck(ls, ls->current == '{', "missing '{'");
  286. r = gethexa(ls); /* must have at least one digit */
  287. while ((save_and_next(ls), lisxdigit(ls->current))) {
  288. i++;
  289. r = (r << 4) + luaO_hexavalue(ls->current);
  290. esccheck(ls, r <= 0x10FFFF, "UTF-8 value too large");
  291. }
  292. esccheck(ls, ls->current == '}', "missing '}'");
  293. next(ls); /* skip '}' */
  294. luaZ_buffremove(ls->buff, i); /* remove saved chars from buffer */
  295. return r;
  296. }
  297. static void utf8esc (LexState *ls) {
  298. char buff[UTF8BUFFSZ];
  299. int n = luaO_utf8esc(buff, readutf8esc(ls));
  300. for (; n > 0; n--) /* add 'buff' to string */
  301. save(ls, buff[UTF8BUFFSZ - n]);
  302. }
  303. static int readdecesc (LexState *ls) {
  304. int i;
  305. int r = 0; /* result accumulator */
  306. for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */
  307. r = 10*r + ls->current - '0';
  308. save_and_next(ls);
  309. }
  310. esccheck(ls, r <= UCHAR_MAX, "decimal escape too large");
  311. luaZ_buffremove(ls->buff, i); /* remove read digits from buffer */
  312. return r;
  313. }
  314. static void read_string (LexState *ls, int del, SemInfo *seminfo) {
  315. save_and_next(ls); /* keep delimiter (for error messages) */
  316. while (ls->current != del) {
  317. switch (ls->current) {
  318. case EOZ:
  319. lexerror(ls, "unfinished string", TK_EOS);
  320. break; /* to avoid warnings */
  321. case '\n':
  322. case '\r':
  323. lexerror(ls, "unfinished string", TK_STRING);
  324. break; /* to avoid warnings */
  325. case '\\': { /* escape sequences */
  326. int c; /* final character to be saved */
  327. save_and_next(ls); /* keep '\\' for error messages */
  328. switch (ls->current) {
  329. case 'a': c = '\a'; goto read_save;
  330. case 'b': c = '\b'; goto read_save;
  331. case 'f': c = '\f'; goto read_save;
  332. case 'n': c = '\n'; goto read_save;
  333. case 'r': c = '\r'; goto read_save;
  334. case 't': c = '\t'; goto read_save;
  335. case 'v': c = '\v'; goto read_save;
  336. case 'x': c = readhexaesc(ls); goto read_save;
  337. case 'u': utf8esc(ls); goto no_save;
  338. case '\n': case '\r':
  339. inclinenumber(ls); c = '\n'; goto only_save;
  340. case '\\': case '\"': case '\'':
  341. c = ls->current; goto read_save;
  342. case EOZ: goto no_save; /* will raise an error next loop */
  343. case 'z': { /* zap following span of spaces */
  344. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  345. next(ls); /* skip the 'z' */
  346. while (lisspace(ls->current)) {
  347. if (currIsNewline(ls)) inclinenumber(ls);
  348. else next(ls);
  349. }
  350. goto no_save;
  351. }
  352. default: {
  353. esccheck(ls, lisdigit(ls->current), "invalid escape sequence");
  354. c = readdecesc(ls); /* digital escape '\ddd' */
  355. goto only_save;
  356. }
  357. }
  358. read_save:
  359. next(ls);
  360. /* go through */
  361. only_save:
  362. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  363. save(ls, c);
  364. /* go through */
  365. no_save: break;
  366. }
  367. default:
  368. save_and_next(ls);
  369. }
  370. }
  371. save_and_next(ls); /* skip delimiter */
  372. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
  373. luaZ_bufflen(ls->buff) - 2);
  374. }
  375. static int llex (LexState *ls, SemInfo *seminfo) {
  376. luaZ_resetbuffer(ls->buff);
  377. for (;;) {
  378. switch (ls->current) {
  379. case '\n': case '\r': { /* line breaks */
  380. inclinenumber(ls);
  381. break;
  382. }
  383. case ' ': case '\f': case '\t': case '\v': { /* spaces */
  384. next(ls);
  385. break;
  386. }
  387. case '-': { /* '-' or '--' (comment) */
  388. next(ls);
  389. if (ls->current != '-') return '-';
  390. /* else is a comment */
  391. next(ls);
  392. if (ls->current == '[') { /* long comment? */
  393. int sep = skip_sep(ls);
  394. luaZ_resetbuffer(ls->buff); /* 'skip_sep' may dirty the buffer */
  395. if (sep >= 0) {
  396. read_long_string(ls, NULL, sep); /* skip long comment */
  397. luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */
  398. break;
  399. }
  400. }
  401. /* else short comment */
  402. while (!currIsNewline(ls) && ls->current != EOZ)
  403. next(ls); /* skip until end of line (or end of file) */
  404. break;
  405. }
  406. case '[': { /* long string or simply '[' */
  407. int sep = skip_sep(ls);
  408. if (sep >= 0) {
  409. read_long_string(ls, seminfo, sep);
  410. return TK_STRING;
  411. }
  412. else if (sep != -1) /* '[=...' missing second bracket */
  413. lexerror(ls, "invalid long string delimiter", TK_STRING);
  414. return '[';
  415. }
  416. case '=': {
  417. next(ls);
  418. if (check_next1(ls, '=')) return TK_EQ;
  419. else return '=';
  420. }
  421. case '<': {
  422. next(ls);
  423. if (check_next1(ls, '=')) return TK_LE;
  424. else if (check_next1(ls, '<')) return TK_SHL;
  425. else return '<';
  426. }
  427. case '>': {
  428. next(ls);
  429. if (check_next1(ls, '=')) return TK_GE;
  430. else if (check_next1(ls, '>')) return TK_SHR;
  431. else return '>';
  432. }
  433. case '/': {
  434. next(ls);
  435. if (check_next1(ls, '/')) return TK_IDIV;
  436. else return '/';
  437. }
  438. case '~': {
  439. next(ls);
  440. if (check_next1(ls, '=')) return TK_NE;
  441. else return '~';
  442. }
  443. case ':': {
  444. next(ls);
  445. if (check_next1(ls, ':')) return TK_DBCOLON;
  446. else return ':';
  447. }
  448. case '"': case '\'': { /* short literal strings */
  449. read_string(ls, ls->current, seminfo);
  450. return TK_STRING;
  451. }
  452. case '.': { /* '.', '..', '...', or number */
  453. save_and_next(ls);
  454. if (check_next1(ls, '.')) {
  455. if (check_next1(ls, '.'))
  456. return TK_DOTS; /* '...' */
  457. else return TK_CONCAT; /* '..' */
  458. }
  459. else if (!lisdigit(ls->current)) return '.';
  460. else return read_numeral(ls, seminfo);
  461. }
  462. case '0': case '1': case '2': case '3': case '4':
  463. case '5': case '6': case '7': case '8': case '9': {
  464. return read_numeral(ls, seminfo);
  465. }
  466. case EOZ: {
  467. return TK_EOS;
  468. }
  469. default: {
  470. if (lislalpha(ls->current)) { /* identifier or reserved word? */
  471. TString *ts;
  472. do {
  473. save_and_next(ls);
  474. } while (lislalnum(ls->current));
  475. ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
  476. luaZ_bufflen(ls->buff));
  477. seminfo->ts = ts;
  478. if (isreserved(ts)) /* reserved word? */
  479. return ts->extra - 1 + FIRST_RESERVED;
  480. else {
  481. return TK_NAME;
  482. }
  483. }
  484. else { /* single-char tokens (+ - / ...) */
  485. int c = ls->current;
  486. next(ls);
  487. return c;
  488. }
  489. }
  490. }
  491. }
  492. }
  493. void luaX_next (LexState *ls) {
  494. ls->lastline = ls->linenumber;
  495. if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
  496. ls->t = ls->lookahead; /* use this one */
  497. ls->lookahead.token = TK_EOS; /* and discharge it */
  498. }
  499. else
  500. ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
  501. }
  502. int luaX_lookahead (LexState *ls) {
  503. lua_assert(ls->lookahead.token == TK_EOS);
  504. ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  505. return ls->lookahead.token;
  506. }