LLlex.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Header$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "lint.h"
  8. #include <alloc.h>
  9. #include "nofloat.h"
  10. #include "idfsize.h"
  11. #include "numsize.h"
  12. #include "debug.h"
  13. #include "strsize.h"
  14. #include "nopp.h"
  15. #include "input.h"
  16. #include "arith.h"
  17. #include "def.h"
  18. #include "idf.h"
  19. #include "LLlex.h"
  20. #include "Lpars.h"
  21. #include "class.h"
  22. #include "assert.h"
  23. #include "sizes.h"
  24. /* Data about the token yielded */
  25. struct token dot, ahead, aside;
  26. #ifndef NOPP
  27. int ReplaceMacros = 1; /* replacing macros */
  28. int AccDefined = 0; /* accept "defined(...)" */
  29. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  30. int Unstacked = 0; /* an unstack is done */
  31. #endif
  32. int SkipEscNewline = 0; /* how to interpret backslash-newline */
  33. int AccFileSpecifier = 0; /* return filespecifier <...> */
  34. int EoiForNewline = 0; /* return EOI upon encountering newline */
  35. int File_Inserted = 0; /* a file has just been inserted */
  36. #define MAX_LL_DEPTH 2
  37. static struct token LexStack[MAX_LL_DEPTH];
  38. static LexSP = 0;
  39. /* In PushLex() the actions are taken in order to initialise or
  40. re-initialise the lexical scanner.
  41. E.g. at the invocation of a sub-parser that uses LLlex(), the
  42. state of the current parser should be saved.
  43. */
  44. PushLex()
  45. {
  46. ASSERT(LexSP < 2);
  47. ASSERT(ASIDE == 0); /* ASIDE = 0; */
  48. GetToken(&ahead);
  49. LexStack[LexSP++] = dot;
  50. }
  51. PopLex()
  52. {
  53. ASSERT(LexSP > 0);
  54. dot = LexStack[--LexSP];
  55. }
  56. int
  57. LLlex()
  58. {
  59. /* LLlex() plays the role of Lexical Analyzer for the C parser.
  60. The look-ahead and putting aside of tokens are taken into
  61. account.
  62. */
  63. if (ASIDE) { /* a token is put aside */
  64. dot = aside;
  65. ASIDE = 0;
  66. }
  67. else { /* read ahead and return the old one */
  68. #ifdef LINT
  69. move_NOT2s();
  70. #endif LINT
  71. dot = ahead;
  72. /* the following test is performed due to the dual
  73. task of LLlex(): it is also called for parsing the
  74. restricted constant expression following a #if or
  75. #elif. The newline character causes EOF to be
  76. returned in this case to stop the LLgen parsing task.
  77. */
  78. if (DOT != EOI)
  79. GetToken(&ahead);
  80. else
  81. DOT = EOF;
  82. }
  83. return DOT;
  84. }
  85. char *string_token();
  86. int
  87. GetToken(ptok)
  88. register struct token *ptok;
  89. {
  90. /* GetToken() is the actual token recognizer. It calls the
  91. control line interpreter if it encounters a "\n#"
  92. combination. Macro replacement is also performed if it is
  93. needed.
  94. */
  95. char buf[(IDFSIZE > NUMSIZE ? IDFSIZE : NUMSIZE) + 1];
  96. register int ch, nch;
  97. if (File_Inserted) {
  98. File_Inserted = 0;
  99. goto firstline;
  100. }
  101. again: /* rescan the input after an error or replacement */
  102. #ifndef NOPP
  103. if (Unstacked) EnableMacros();
  104. #endif
  105. LoadChar(ch);
  106. go_on: /* rescan, the following character has been read */
  107. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  108. fatal("non-ascii '\\%03o' read", ch & 0377);
  109. /* keep track of the place of the token in the file */
  110. ptok->tk_file = FileName;
  111. ptok->tk_line = LineNumber;
  112. switch (class(ch)) { /* detect character class */
  113. case STNL: /* newline, vertical space or formfeed */
  114. firstline:
  115. LineNumber++; /* also at vs and ff */
  116. ptok->tk_file = FileName;
  117. ptok->tk_line = LineNumber;
  118. if (EoiForNewline) /* called in control line */
  119. /* a newline in a control line indicates the
  120. end-of-information of the line.
  121. */
  122. return ptok->tk_symb = EOI;
  123. while (LoadChar(ch), ch == '#') { /* a control line follows */
  124. domacro();
  125. if (File_Inserted) {
  126. File_Inserted = 0;
  127. goto firstline;
  128. }
  129. }
  130. /* We have to loop here, because in
  131. `domacro' the nl, vt or ff is read. The
  132. character following it may again be a `#'.
  133. */
  134. goto go_on;
  135. case STSKIP: /* just skip the skip characters */
  136. goto again;
  137. case STGARB: /* garbage character */
  138. #ifndef NOPP
  139. if (SkipEscNewline && (ch == '\\')) {
  140. /* a '\\' is allowed in #if/#elif expression */
  141. LoadChar(ch);
  142. if (class(ch) == STNL) { /* vt , ff ? */
  143. ++LineNumber;
  144. goto again;
  145. }
  146. PushBack();
  147. ch = '\\';
  148. }
  149. #endif NOPP
  150. if (040 < ch && ch < 0177)
  151. lexerror("garbage char %c", ch);
  152. else
  153. lexerror("garbage char \\%03o", ch);
  154. goto again;
  155. case STSIMP: /* a simple character, no part of compound token*/
  156. if (ch == '/') { /* probably the start of comment */
  157. LoadChar(ch);
  158. if (ch == '*') { /* start of comment */
  159. skipcomment();
  160. goto again;
  161. }
  162. else {
  163. PushBack();
  164. ch = '/'; /* restore ch */
  165. }
  166. }
  167. return ptok->tk_symb = ch;
  168. case STCOMP: /* maybe the start of a compound token */
  169. LoadChar(nch); /* character lookahead */
  170. switch (ch) {
  171. case '!':
  172. if (nch == '=')
  173. return ptok->tk_symb = NOTEQUAL;
  174. PushBack();
  175. return ptok->tk_symb = ch;
  176. case '&':
  177. if (nch == '&')
  178. return ptok->tk_symb = AND;
  179. PushBack();
  180. return ptok->tk_symb = ch;
  181. case '+':
  182. if (nch == '+')
  183. return ptok->tk_symb = PLUSPLUS;
  184. PushBack();
  185. return ptok->tk_symb = ch;
  186. case '-':
  187. if (nch == '-')
  188. return ptok->tk_symb = MINMIN;
  189. if (nch == '>')
  190. return ptok->tk_symb = ARROW;
  191. PushBack();
  192. return ptok->tk_symb = ch;
  193. case '<':
  194. if (AccFileSpecifier) {
  195. PushBack(); /* pushback nch */
  196. ptok->tk_bts = string_token("file specifier",
  197. '>', &(ptok->tk_len));
  198. return ptok->tk_symb = FILESPECIFIER;
  199. }
  200. if (nch == '<')
  201. return ptok->tk_symb = LEFT;
  202. if (nch == '=')
  203. return ptok->tk_symb = LESSEQ;
  204. PushBack();
  205. return ptok->tk_symb = ch;
  206. case '=':
  207. if (nch == '=')
  208. return ptok->tk_symb = EQUAL;
  209. /* The following piece of code tries to recognise
  210. old-fashioned assignment operators `=op'
  211. */
  212. switch (nch) {
  213. case '+':
  214. return ptok->tk_symb = PLUSAB;
  215. case '-':
  216. return ptok->tk_symb = MINAB;
  217. case '*':
  218. return ptok->tk_symb = TIMESAB;
  219. case '/':
  220. return ptok->tk_symb = DIVAB;
  221. case '%':
  222. return ptok->tk_symb = MODAB;
  223. case '>':
  224. case '<':
  225. LoadChar(ch);
  226. if (ch != nch) {
  227. PushBack();
  228. lexerror("illegal operator '=%c%c'",
  229. nch, ch);
  230. }
  231. return ptok->tk_symb =
  232. nch == '<' ? LEFTAB : RIGHTAB;
  233. case '&':
  234. return ptok->tk_symb = ANDAB;
  235. case '^':
  236. return ptok->tk_symb = XORAB;
  237. case '|':
  238. return ptok->tk_symb = ORAB;
  239. }
  240. PushBack();
  241. return ptok->tk_symb = ch;
  242. case '>':
  243. if (nch == '=')
  244. return ptok->tk_symb = GREATEREQ;
  245. if (nch == '>')
  246. return ptok->tk_symb = RIGHT;
  247. PushBack();
  248. return ptok->tk_symb = ch;
  249. case '|':
  250. if (nch == '|')
  251. return ptok->tk_symb = OR;
  252. PushBack();
  253. return ptok->tk_symb = ch;
  254. }
  255. case STIDF:
  256. {
  257. register char *tg = &buf[0];
  258. register int pos = -1;
  259. register int hash;
  260. register struct idf *idef;
  261. extern int idfsize; /* ??? */
  262. hash = STARTHASH();
  263. do { /* read the identifier */
  264. if (++pos < idfsize) {
  265. #ifndef NOPP
  266. if (Unstacked) EnableMacros();
  267. #endif
  268. *tg++ = ch;
  269. hash = ENHASH(hash, ch, pos);
  270. }
  271. LoadChar(ch);
  272. } while (in_idf(ch));
  273. hash = STOPHASH(hash);
  274. if (ch != EOI)
  275. PushBack();
  276. *tg++ = '\0'; /* mark the end of the identifier */
  277. idef = ptok->tk_idf = idf_hashed(buf, tg - buf, hash);
  278. idef->id_file = ptok->tk_file;
  279. idef->id_line = ptok->tk_line;
  280. #ifndef NOPP
  281. if (idef->id_macro && ReplaceMacros && replace(idef))
  282. /* macro replacement should be performed */
  283. goto again;
  284. if (UnknownIdIsZero && idef->id_reserved != SIZEOF) {
  285. ptok->tk_ival = (arith)0;
  286. ptok->tk_fund = INT;
  287. return ptok->tk_symb = INTEGER;
  288. }
  289. #endif NOPP
  290. ptok->tk_symb = (
  291. idef->id_reserved ? idef->id_reserved
  292. : idef->id_def && idef->id_def->df_sc == TYPEDEF ?
  293. TYPE_IDENTIFIER
  294. : IDENTIFIER
  295. );
  296. return IDENTIFIER;
  297. }
  298. case STCHAR: /* character constant */
  299. {
  300. register arith val = 0;
  301. int size = 0;
  302. LoadChar(ch);
  303. if (ch == '\'')
  304. lexerror("character constant too short");
  305. else
  306. while (ch != '\'') {
  307. if (ch == '\n') {
  308. lexerror("newline in character constant");
  309. LineNumber++;
  310. break;
  311. }
  312. if (ch == '\\') {
  313. LoadChar(ch);
  314. if (ch == '\n')
  315. LineNumber++;
  316. ch = quoted(ch);
  317. }
  318. if (ch >= 128) ch -= 256;
  319. val = val*256 + ch;
  320. size++;
  321. LoadChar(ch);
  322. }
  323. if (size > (int)int_size)
  324. lexerror("character constant too long");
  325. ptok->tk_ival = val;
  326. ptok->tk_fund = INT;
  327. return ptok->tk_symb = INTEGER;
  328. }
  329. case STSTR: /* string */
  330. ptok->tk_bts = string_token("string", '"', &(ptok->tk_len));
  331. return ptok->tk_symb = STRING;
  332. case STNUM: /* a numeric constant */
  333. {
  334. /* It should be noted that 099 means 81(decimal) and
  335. 099.5 means 99.5 . This severely limits the tricks
  336. we can use to scan a numeric value.
  337. */
  338. register char *np = &buf[1];
  339. register int base = 10;
  340. register int vch;
  341. register arith val = 0;
  342. if (ch == '.') { /* an embarrassing ambiguity */
  343. #ifndef NOFLOAT
  344. LoadChar(vch);
  345. PushBack();
  346. if (!is_dig(vch)) /* just a `.' */
  347. return ptok->tk_symb = ch;
  348. *np++ = '0';
  349. /* in the rest of the compiler, all floats
  350. have to start with a digit.
  351. */
  352. #else NOFLOAT
  353. return ptok->tk_symb = ch;
  354. #endif NOFLOAT
  355. }
  356. if (ch == '0') {
  357. *np++ = ch;
  358. LoadChar(ch);
  359. if (ch == 'x' || ch == 'X') {
  360. base = 16;
  361. LoadChar(ch);
  362. }
  363. else
  364. base = 8;
  365. }
  366. while (vch = val_in_base(ch, base), vch >= 0) {
  367. val = val*base + vch;
  368. if (np < &buf[NUMSIZE])
  369. *np++ = ch;
  370. LoadChar(ch);
  371. }
  372. if (ch == 'l' || ch == 'L') {
  373. ptok->tk_ival = val;
  374. ptok->tk_fund = LONG;
  375. return ptok->tk_symb = INTEGER;
  376. }
  377. #ifndef NOFLOAT
  378. if (base == 16 || !(ch == '.' || ch == 'e' || ch == 'E'))
  379. #endif NOFLOAT
  380. {
  381. PushBack();
  382. ptok->tk_ival = val;
  383. /* The semantic analyser must know if the
  384. integral constant is given in octal/hexa-
  385. decimal form, in which case its type is
  386. UNSIGNED, or in decimal form, in which case
  387. its type is signed, indicated by
  388. the fund INTEGER.
  389. */
  390. ptok->tk_fund =
  391. (base == 10 || (base == 8 && val == (arith)0))
  392. ? INTEGER : UNSIGNED;
  393. return ptok->tk_symb = INTEGER;
  394. }
  395. /* where's the test for the length of the integral ??? */
  396. #ifndef NOFLOAT
  397. if (ch == '.'){
  398. if (np < &buf[NUMSIZE])
  399. *np++ = ch;
  400. LoadChar(ch);
  401. }
  402. while (is_dig(ch)){
  403. if (np < &buf[NUMSIZE])
  404. *np++ = ch;
  405. LoadChar(ch);
  406. }
  407. if (ch == 'e' || ch == 'E') {
  408. if (np < &buf[NUMSIZE])
  409. *np++ = ch;
  410. LoadChar(ch);
  411. if (ch == '+' || ch == '-') {
  412. if (np < &buf[NUMSIZE])
  413. *np++ = ch;
  414. LoadChar(ch);
  415. }
  416. if (!is_dig(ch)) {
  417. lexerror("malformed floating constant");
  418. if (np < &buf[NUMSIZE])
  419. *np++ = ch;
  420. }
  421. while (is_dig(ch)) {
  422. if (np < &buf[NUMSIZE])
  423. *np++ = ch;
  424. LoadChar(ch);
  425. }
  426. }
  427. PushBack();
  428. *np++ = '\0';
  429. buf[0] = '-'; /* good heavens... */
  430. if (np == &buf[NUMSIZE+1]) {
  431. lexerror("floating constant too long");
  432. ptok->tk_fval = Salloc("0.0",(unsigned) 5) + 1;
  433. }
  434. else
  435. ptok->tk_fval = Salloc(buf,(unsigned) (np - buf)) + 1;
  436. return ptok->tk_symb = FLOATING;
  437. #endif NOFLOAT
  438. }
  439. case STEOI: /* end of text on source file */
  440. return ptok->tk_symb = EOI;
  441. default: /* this cannot happen */
  442. crash("bad class for char 0%o", ch);
  443. }
  444. /*NOTREACHED*/
  445. }
  446. skipcomment()
  447. {
  448. /* The last character read has been the '*' of '/_*'. The
  449. characters, except NL and EOI, between '/_*' and the first
  450. occurring '*_/' are not interpreted.
  451. NL only affects the LineNumber. EOI is not legal.
  452. Important note: it is not possible to stop skipping comment
  453. beyond the end-of-file of an included file.
  454. EOI is returned by LoadChar only on encountering EOF of the
  455. top-level file...
  456. */
  457. register int c;
  458. NoUnstack++;
  459. LoadChar(c);
  460. #ifdef LINT
  461. lint_comment(-2);
  462. lint_comment(c);
  463. #endif LINT
  464. do {
  465. while (c != '*') {
  466. if (class(c) == STNL)
  467. ++LineNumber;
  468. else
  469. if (c == EOI) {
  470. NoUnstack--;
  471. return;
  472. }
  473. LoadChar(c);
  474. #ifdef LINT
  475. lint_comment(c);
  476. #endif LINT
  477. } /* last Character seen was '*' */
  478. LoadChar(c);
  479. #ifdef LINT
  480. lint_comment(c);
  481. #endif LINT
  482. } while (c != '/');
  483. NoUnstack--;
  484. }
  485. char *
  486. string_token(nm, stop_char, plen)
  487. char *nm;
  488. int *plen;
  489. {
  490. register int ch;
  491. register int str_size;
  492. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  493. register int pos = 0;
  494. LoadChar(ch);
  495. while (ch != stop_char) {
  496. if (ch == '\n') {
  497. lexerror("newline in %s", nm);
  498. LineNumber++;
  499. break;
  500. }
  501. if (ch == EOI) {
  502. lexerror("end-of-file inside %s", nm);
  503. break;
  504. }
  505. if (ch == '\\') {
  506. LoadChar(ch);
  507. if (ch == '\n') {
  508. LineNumber++;
  509. LoadChar(ch);
  510. continue;
  511. }
  512. ch = quoted(ch);
  513. }
  514. str[pos++] = ch;
  515. if (pos == str_size)
  516. str = Srealloc(str, (unsigned) (str_size += RSTRSIZE));
  517. LoadChar(ch);
  518. }
  519. str[pos++] = '\0'; /* for filenames etc. */
  520. *plen = pos;
  521. return str;
  522. }
  523. int
  524. quoted(ch)
  525. register int ch;
  526. {
  527. /* quoted() replaces an escaped character sequence by the
  528. character meant.
  529. */
  530. /* first char after backslash already in ch */
  531. if (!is_oct(ch)) { /* a quoted char */
  532. switch (ch) {
  533. case 'n':
  534. ch = '\n';
  535. break;
  536. case 't':
  537. ch = '\t';
  538. break;
  539. case 'b':
  540. ch = '\b';
  541. break;
  542. case 'r':
  543. ch = '\r';
  544. break;
  545. case 'f':
  546. ch = '\f';
  547. break;
  548. }
  549. }
  550. else { /* a quoted octal */
  551. register int oct = 0, cnt = 0;
  552. do {
  553. oct = oct*8 + (ch-'0');
  554. LoadChar(ch);
  555. } while (is_oct(ch) && ++cnt < 3);
  556. PushBack();
  557. ch = oct;
  558. }
  559. return ch&0377;
  560. }
  561. /* provisional */
  562. int
  563. val_in_base(ch, base)
  564. register int ch;
  565. {
  566. return
  567. is_dig(ch) ? ch - '0'
  568. : base != 16 ? -1
  569. : is_hex(ch) ? (ch - 'a' + 10) & 017
  570. : -1;
  571. }