llex.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457
  1. /*
  2. ** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $
  3. ** Lexical Analyzer
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define llex_c
  7. #define LUA_CORE
  8. #include "lua.h"
  9. #include <ctype.h>
  10. #include <locale.h>
  11. #include <string.h>
  12. #include "ldo.h"
  13. #include "llex.h"
  14. #include "lobject.h"
  15. #include "lparser.h"
  16. #include "lstate.h"
  17. #include "lstring.h"
  18. #include "ltable.h"
  19. #include "lzio.h"
  20. #define next(ls) (ls->current = zgetc(ls->z))
  21. #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
  22. /* ORDER RESERVED */
  23. const char *const luaX_tokens [] = {
  24. "and", "break", "do", "else", "elseif",
  25. "end", "false", "for", "function", "if",
  26. "in", "local", "nil", "not", "or", "repeat",
  27. "return", "then", "true", "until", "while",
  28. "..", "...", "==", ">=", "<=", "~=",
  29. "<number>", "<name>", "<string>", "<eof>",
  30. NULL
  31. };
  32. #define save_and_next(ls) (save(ls, ls->current), next(ls))
  33. static void save (LexState *ls, int c) {
  34. Mbuffer *b = ls->buff;
  35. if (b->n + 1 > b->buffsize) {
  36. size_t newsize;
  37. if (b->buffsize >= MAX_SIZET/2)
  38. luaX_lexerror(ls, "lexical element too long", 0);
  39. newsize = b->buffsize * 2;
  40. luaZ_resizebuffer(ls->L, b, newsize);
  41. }
  42. b->buffer[b->n++] = cast(char, c);
  43. }
  44. void luaX_init (lua_State *L) {
  45. }
  46. #define MAXSRC 80
  47. const char *luaX_token2str (LexState *ls, int token) {
  48. if (token < FIRST_RESERVED) {
  49. lua_assert(token == cast(unsigned char, token));
  50. return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) :
  51. luaO_pushfstring(ls->L, "%c", token);
  52. }
  53. else
  54. return luaX_tokens[token-FIRST_RESERVED];
  55. }
  56. static const char *txtToken (LexState *ls, int token) {
  57. switch (token) {
  58. case TK_NAME:
  59. case TK_STRING:
  60. case TK_NUMBER:
  61. save(ls, '\0');
  62. return luaZ_buffer(ls->buff);
  63. default:
  64. return luaX_token2str(ls, token);
  65. }
  66. }
  67. void luaX_lexerror (LexState *ls, const char *msg, int token) {
  68. char buff[MAXSRC];
  69. luaO_chunkid(buff, getstr(ls->source), MAXSRC);
  70. msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
  71. if (token)
  72. luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token));
  73. luaD_throw(ls->L, LUA_ERRSYNTAX);
  74. }
  75. void luaX_syntaxerror (LexState *ls, const char *msg) {
  76. luaX_lexerror(ls, msg, ls->t.token);
  77. }
  78. TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
  79. lua_State *L = ls->L;
  80. TString *ts = luaS_newlstr(L, str, l);
  81. TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */
  82. if (ttisnil(o)) {
  83. setbvalue(o, 1); /* make sure `str' will not be collected */
  84. luaC_checkGC(L);
  85. }
  86. return ts;
  87. }
  88. static void inclinenumber (LexState *ls) {
  89. int old = ls->current;
  90. lua_assert(currIsNewline(ls));
  91. next(ls); /* skip `\n' or `\r' */
  92. if (currIsNewline(ls) && ls->current != old)
  93. next(ls); /* skip `\n\r' or `\r\n' */
  94. if (++ls->linenumber >= MAX_INT)
  95. luaX_syntaxerror(ls, "chunk has too many lines");
  96. }
  97. void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
  98. ls->decpoint = '.';
  99. ls->L = L;
  100. ls->lookahead.token = TK_EOS; /* no look-ahead token */
  101. ls->z = z;
  102. ls->fs = NULL;
  103. ls->linenumber = 1;
  104. ls->lastline = 1;
  105. ls->source = source;
  106. luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
  107. next(ls); /* read first char */
  108. }
  109. /*
  110. ** =======================================================
  111. ** LEXICAL ANALYZER
  112. ** =======================================================
  113. */
  114. static int check_next (LexState *ls, const char *set) {
  115. if (!strchr(set, ls->current))
  116. return 0;
  117. save_and_next(ls);
  118. return 1;
  119. }
  120. static void buffreplace (LexState *ls, char from, char to) {
  121. size_t n = luaZ_bufflen(ls->buff);
  122. char *p = luaZ_buffer(ls->buff);
  123. while (n--)
  124. if (p[n] == from) p[n] = to;
  125. }
  126. static void trydecpoint (LexState *ls, SemInfo *seminfo) {
  127. /* format error: try to update decimal point separator */
  128. struct lconv *cv = localeconv();
  129. char old = ls->decpoint;
  130. ls->decpoint = (cv ? cv->decimal_point[0] : '.');
  131. buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */
  132. if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) {
  133. /* format error with correct decimal point: no more options */
  134. buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */
  135. luaX_lexerror(ls, "malformed number", TK_NUMBER);
  136. }
  137. }
  138. /* LUA_NUMBER */
  139. static void read_numeral (LexState *ls, SemInfo *seminfo) {
  140. lua_assert(isdigit(ls->current));
  141. do {
  142. save_and_next(ls);
  143. } while (isdigit(ls->current) || ls->current == '.');
  144. if (check_next(ls, "Ee")) /* `E'? */
  145. check_next(ls, "+-"); /* optional exponent sign */
  146. while (isalnum(ls->current) || ls->current == '_')
  147. save_and_next(ls);
  148. save(ls, '\0');
  149. buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */
  150. if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */
  151. trydecpoint(ls, seminfo); /* try to update decimal point separator */
  152. }
  153. static int skip_sep (LexState *ls) {
  154. int count = 0;
  155. int s = ls->current;
  156. lua_assert(s == '[' || s == ']');
  157. save_and_next(ls);
  158. while (ls->current == '=') {
  159. save_and_next(ls);
  160. count++;
  161. }
  162. return (ls->current == s) ? count : (-count) - 1;
  163. }
  164. static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
  165. int cont = 0;
  166. (void)(cont); /* avoid warnings when `cont' is not used */
  167. save_and_next(ls); /* skip 2nd `[' */
  168. if (currIsNewline(ls)) /* string starts with a newline? */
  169. inclinenumber(ls); /* skip it */
  170. for (;;) {
  171. switch (ls->current) {
  172. case EOZ:
  173. luaX_lexerror(ls, (seminfo) ? "unfinished long string" :
  174. "unfinished long comment", TK_EOS);
  175. break; /* to avoid warnings */
  176. #if defined(LUA_COMPAT_LSTR)
  177. case '[': {
  178. if (skip_sep(ls) == sep) {
  179. save_and_next(ls); /* skip 2nd `[' */
  180. cont++;
  181. #if LUA_COMPAT_LSTR == 1
  182. if (sep == 0)
  183. luaX_lexerror(ls, "nesting of [[...]] is deprecated", '[');
  184. #endif
  185. }
  186. break;
  187. }
  188. #endif
  189. case ']': {
  190. if (skip_sep(ls) == sep) {
  191. save_and_next(ls); /* skip 2nd `]' */
  192. #if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2
  193. cont--;
  194. if (sep == 0 && cont >= 0) break;
  195. #endif
  196. goto endloop;
  197. }
  198. break;
  199. }
  200. case '\n':
  201. case '\r': {
  202. save(ls, '\n');
  203. inclinenumber(ls);
  204. if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
  205. break;
  206. }
  207. default: {
  208. if (seminfo) save_and_next(ls);
  209. else next(ls);
  210. }
  211. }
  212. } endloop:
  213. if (seminfo)
  214. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
  215. luaZ_bufflen(ls->buff) - 2*(2 + sep));
  216. }
  217. static void read_string (LexState *ls, int del, SemInfo *seminfo) {
  218. save_and_next(ls);
  219. while (ls->current != del) {
  220. switch (ls->current) {
  221. case EOZ:
  222. luaX_lexerror(ls, "unfinished string", TK_EOS);
  223. continue; /* to avoid warnings */
  224. case '\n':
  225. case '\r':
  226. luaX_lexerror(ls, "unfinished string", TK_STRING);
  227. continue; /* to avoid warnings */
  228. case '\\': {
  229. int c;
  230. next(ls); /* do not save the `\' */
  231. switch (ls->current) {
  232. case 'a': c = '\a'; break;
  233. case 'b': c = '\b'; break;
  234. case 'f': c = '\f'; break;
  235. case 'n': c = '\n'; break;
  236. case 'r': c = '\r'; break;
  237. case 't': c = '\t'; break;
  238. case 'v': c = '\v'; break;
  239. case '\n': /* go through */
  240. case '\r': save(ls, '\n'); inclinenumber(ls); continue;
  241. case EOZ: continue; /* will raise an error next loop */
  242. default: {
  243. if (!isdigit(ls->current))
  244. save_and_next(ls); /* handles \\, \", \', and \? */
  245. else { /* \xxx */
  246. int i = 0;
  247. c = 0;
  248. do {
  249. c = 10*c + (ls->current-'0');
  250. next(ls);
  251. } while (++i<3 && isdigit(ls->current));
  252. if (c > UCHAR_MAX)
  253. luaX_lexerror(ls, "escape sequence too large", TK_STRING);
  254. save(ls, c);
  255. }
  256. continue;
  257. }
  258. }
  259. save(ls, c);
  260. next(ls);
  261. continue;
  262. }
  263. default:
  264. save_and_next(ls);
  265. }
  266. }
  267. save_and_next(ls); /* skip delimiter */
  268. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
  269. luaZ_bufflen(ls->buff) - 2);
  270. }
  271. static int llex (LexState *ls, SemInfo *seminfo) {
  272. luaZ_resetbuffer(ls->buff);
  273. for (;;) {
  274. switch (ls->current) {
  275. case '\n':
  276. case '\r': {
  277. inclinenumber(ls);
  278. continue;
  279. }
  280. case '-': {
  281. next(ls);
  282. if (ls->current != '-') return '-';
  283. /* else is a comment */
  284. next(ls);
  285. if (ls->current == '[') {
  286. int sep = skip_sep(ls);
  287. luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */
  288. if (sep >= 0) {
  289. read_long_string(ls, NULL, sep); /* long comment */
  290. luaZ_resetbuffer(ls->buff);
  291. continue;
  292. }
  293. }
  294. /* else short comment */
  295. while (!currIsNewline(ls) && ls->current != EOZ)
  296. next(ls);
  297. continue;
  298. }
  299. case '[': {
  300. int sep = skip_sep(ls);
  301. if (sep >= 0) {
  302. read_long_string(ls, seminfo, sep);
  303. return TK_STRING;
  304. }
  305. else if (sep == -1) return '[';
  306. else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING);
  307. }
  308. case '=': {
  309. next(ls);
  310. if (ls->current != '=') return '=';
  311. else { next(ls); return TK_EQ; }
  312. }
  313. case '<': {
  314. next(ls);
  315. if (ls->current != '=') return '<';
  316. else { next(ls); return TK_LE; }
  317. }
  318. case '>': {
  319. next(ls);
  320. if (ls->current != '=') return '>';
  321. else { next(ls); return TK_GE; }
  322. }
  323. case '~': {
  324. next(ls);
  325. if (ls->current != '=') return '~';
  326. else { next(ls); return TK_NE; }
  327. }
  328. case '"':
  329. case '\'': {
  330. read_string(ls, ls->current, seminfo);
  331. return TK_STRING;
  332. }
  333. case '.': {
  334. save_and_next(ls);
  335. if (check_next(ls, ".")) {
  336. if (check_next(ls, "."))
  337. return TK_DOTS; /* ... */
  338. else return TK_CONCAT; /* .. */
  339. }
  340. else if (!isdigit(ls->current)) return '.';
  341. else {
  342. read_numeral(ls, seminfo);
  343. return TK_NUMBER;
  344. }
  345. }
  346. case EOZ: {
  347. return TK_EOS;
  348. }
  349. default: {
  350. if (isspace(ls->current)) {
  351. lua_assert(!currIsNewline(ls));
  352. next(ls);
  353. continue;
  354. }
  355. else if (isdigit(ls->current)) {
  356. read_numeral(ls, seminfo);
  357. return TK_NUMBER;
  358. }
  359. else if (isalpha(ls->current) || ls->current == '_') {
  360. /* identifier or reserved word */
  361. TString *ts;
  362. int i;
  363. do {
  364. save_and_next(ls);
  365. } while (isalnum(ls->current) || ls->current == '_');
  366. /* look for reserved word */
  367. save(ls, '\0');
  368. for (i = 0; i < NUM_RESERVED; i++)
  369. if (!strcmp(luaX_tokens[i], luaZ_buffer(ls->buff)))
  370. return i + FIRST_RESERVED;
  371. ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
  372. luaZ_bufflen(ls->buff) - 1);
  373. seminfo->ts = ts;
  374. return TK_NAME;
  375. }
  376. else {
  377. int c = ls->current;
  378. next(ls);
  379. return c; /* single-char tokens (+ - / ...) */
  380. }
  381. }
  382. }
  383. }
  384. }
  385. void luaX_next (LexState *ls) {
  386. ls->lastline = ls->linenumber;
  387. if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
  388. ls->t = ls->lookahead; /* use this one */
  389. ls->lookahead.token = TK_EOS; /* and discharge it */
  390. }
  391. else
  392. ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
  393. }
  394. void luaX_lookahead (LexState *ls) {
  395. lua_assert(ls->lookahead.token == TK_EOS);
  396. ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  397. }