LLlex.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Id$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "lint.h"
  8. #include <alloc.h>
  9. #include "nofloat.h"
  10. #include "idfsize.h"
  11. #include "numsize.h"
  12. #include "debug.h"
  13. #include "strsize.h"
  14. #include "nopp.h"
  15. #include "input.h"
  16. #include "arith.h"
  17. #include "def.h"
  18. #include "idf.h"
  19. #include "LLlex.h"
  20. #include "Lpars.h"
  21. #include "class.h"
  22. #include "assert.h"
  23. #include "sizes.h"
  24. /* Data about the token yielded */
  25. struct token dot, ahead, aside;
  26. int token_nmb = 0; /* number of the ahead token */
  27. int tk_nmb_at_last_syn_err = -5/*ERR_SHADOW*/;
  28. /* token number at last syntax error */
  29. #ifndef NOPP
  30. int ReplaceMacros = 1; /* replacing macros */
  31. int AccDefined = 0; /* accept "defined(...)" */
  32. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  33. int Unstacked = 0; /* an unstack is done */
  34. #endif
  35. int SkipEscNewline = 0; /* how to interpret backslash-newline */
  36. int AccFileSpecifier = 0; /* return filespecifier <...> */
  37. int EoiForNewline = 0; /* return EOI upon encountering newline */
  38. int File_Inserted = 0; /* a file has just been inserted */
  39. #ifdef LINT
  40. extern int lint_skip_comment;
  41. #endif
  42. #define MAX_LL_DEPTH 2
  43. static struct token LexStack[MAX_LL_DEPTH];
  44. static LexSP = 0;
  45. /* In PushLex() the actions are taken in order to initialise or
  46. re-initialise the lexical scanner.
  47. E.g. at the invocation of a sub-parser that uses LLlex(), the
  48. state of the current parser should be saved.
  49. */
  50. PushLex()
  51. {
  52. ASSERT(LexSP < 2);
  53. ASSERT(ASIDE == 0); /* ASIDE = 0; */
  54. GetToken(&ahead);
  55. LexStack[LexSP++] = dot;
  56. }
  57. PopLex()
  58. {
  59. ASSERT(LexSP > 0);
  60. dot = LexStack[--LexSP];
  61. }
  62. int
  63. LLlex()
  64. {
  65. /* LLlex() plays the role of Lexical Analyzer for the C parser.
  66. The look-ahead and putting aside of tokens are taken into
  67. account.
  68. */
  69. if (ASIDE) { /* a token is put aside */
  70. dot = aside;
  71. ASIDE = 0;
  72. }
  73. else { /* read ahead and return the old one */
  74. #ifdef LINT
  75. lint_comment_ahead();
  76. #endif /* LINT */
  77. dot = ahead;
  78. /* the following test is performed due to the dual
  79. task of LLlex(): it is also called for parsing the
  80. restricted constant expression following a #if or
  81. #elif. The newline character causes EOF to be
  82. returned in this case to stop the LLgen parsing task.
  83. */
  84. if (DOT != EOI)
  85. GetToken(&ahead);
  86. else
  87. DOT = EOF;
  88. }
  89. return DOT;
  90. }
  91. char *string_token();
  92. int
  93. GetToken(ptok)
  94. register struct token *ptok;
  95. {
  96. /* GetToken() is the actual token recognizer. It calls the
  97. control line interpreter if it encounters a "\n#"
  98. combination. Macro replacement is also performed if it is
  99. needed.
  100. */
  101. char buf[(IDFSIZE > NUMSIZE ? IDFSIZE : NUMSIZE) + 1];
  102. register int ch, nch;
  103. token_nmb++;
  104. if (File_Inserted) {
  105. File_Inserted = 0;
  106. goto firstline;
  107. }
  108. again: /* rescan the input after an error or replacement */
  109. #ifndef NOPP
  110. if (Unstacked) EnableMacros();
  111. #endif
  112. LoadChar(ch);
  113. go_on: /* rescan, the following character has been read */
  114. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  115. fatal("non-ascii '\\%03o' read", ch & 0377);
  116. /* keep track of the place of the token in the file */
  117. ptok->tk_file = FileName;
  118. ptok->tk_line = LineNumber;
  119. switch (class(ch)) { /* detect character class */
  120. case STNL: /* newline, vertical space or formfeed */
  121. firstline:
  122. LineNumber++; /* also at vs and ff */
  123. ptok->tk_file = FileName;
  124. ptok->tk_line = LineNumber;
  125. if (EoiForNewline) /* called in control line */
  126. /* a newline in a control line indicates the
  127. end-of-information of the line.
  128. */
  129. return ptok->tk_symb = EOI;
  130. while (LoadChar(ch), ch == '#') { /* a control line follows */
  131. domacro();
  132. if (File_Inserted) {
  133. File_Inserted = 0;
  134. goto firstline;
  135. }
  136. }
  137. /* We have to loop here, because in
  138. `domacro' the nl, vt or ff is read. The
  139. character following it may again be a `#'.
  140. */
  141. goto go_on;
  142. case STSKIP: /* just skip the skip characters */
  143. goto again;
  144. case STGARB: /* garbage character */
  145. #ifndef NOPP
  146. if (SkipEscNewline && (ch == '\\')) {
  147. /* a '\\' is allowed in #if/#elif expression */
  148. LoadChar(ch);
  149. if (class(ch) == STNL) { /* vt , ff ? */
  150. ++LineNumber;
  151. goto again;
  152. }
  153. PushBack();
  154. ch = '\\';
  155. }
  156. #endif /* NOPP */
  157. if (040 < ch && ch < 0177)
  158. lexerror("garbage char %c", ch);
  159. else
  160. lexerror("garbage char \\%03o", ch);
  161. goto again;
  162. case STSIMP: /* a simple character, no part of compound token*/
  163. if (ch == '/') { /* probably the start of comment */
  164. LoadChar(ch);
  165. if (ch == '*') { /* start of comment */
  166. skipcomment();
  167. goto again;
  168. }
  169. else {
  170. PushBack();
  171. ch = '/'; /* restore ch */
  172. }
  173. }
  174. return ptok->tk_symb = ch;
  175. case STCOMP: /* maybe the start of a compound token */
  176. LoadChar(nch); /* character lookahead */
  177. switch (ch) {
  178. case '!':
  179. if (nch == '=')
  180. return ptok->tk_symb = NOTEQUAL;
  181. PushBack();
  182. return ptok->tk_symb = ch;
  183. case '&':
  184. if (nch == '&')
  185. return ptok->tk_symb = AND;
  186. PushBack();
  187. return ptok->tk_symb = ch;
  188. case '+':
  189. if (nch == '+')
  190. return ptok->tk_symb = PLUSPLUS;
  191. PushBack();
  192. return ptok->tk_symb = ch;
  193. case '-':
  194. if (nch == '-')
  195. return ptok->tk_symb = MINMIN;
  196. if (nch == '>')
  197. return ptok->tk_symb = ARROW;
  198. PushBack();
  199. return ptok->tk_symb = ch;
  200. case '<':
  201. if (AccFileSpecifier) {
  202. PushBack(); /* pushback nch */
  203. ptok->tk_bts = string_token("file specifier",
  204. '>', &(ptok->tk_len));
  205. return ptok->tk_symb = FILESPECIFIER;
  206. }
  207. if (nch == '<')
  208. return ptok->tk_symb = LEFT;
  209. if (nch == '=')
  210. return ptok->tk_symb = LESSEQ;
  211. PushBack();
  212. return ptok->tk_symb = ch;
  213. case '=':
  214. if (nch == '=')
  215. return ptok->tk_symb = EQUAL;
  216. PushBack();
  217. return ptok->tk_symb = ch;
  218. case '>':
  219. if (nch == '=')
  220. return ptok->tk_symb = GREATEREQ;
  221. if (nch == '>')
  222. return ptok->tk_symb = RIGHT;
  223. PushBack();
  224. return ptok->tk_symb = ch;
  225. case '|':
  226. if (nch == '|')
  227. return ptok->tk_symb = OR;
  228. PushBack();
  229. return ptok->tk_symb = ch;
  230. }
  231. case STIDF:
  232. {
  233. register char *tg = &buf[0];
  234. register int pos = -1;
  235. register int hash;
  236. register struct idf *idef;
  237. extern int idfsize; /* ??? */
  238. hash = STARTHASH();
  239. do { /* read the identifier */
  240. if (++pos < idfsize) {
  241. #ifndef NOPP
  242. if (Unstacked) EnableMacros();
  243. #endif
  244. *tg++ = ch;
  245. hash = ENHASH(hash, ch, pos);
  246. }
  247. LoadChar(ch);
  248. } while (in_idf(ch));
  249. hash = STOPHASH(hash);
  250. if (ch != EOI)
  251. PushBack();
  252. *tg++ = '\0'; /* mark the end of the identifier */
  253. idef = ptok->tk_idf = idf_hashed(buf, tg - buf, hash);
  254. idef->id_file = ptok->tk_file;
  255. idef->id_line = ptok->tk_line;
  256. #ifndef NOPP
  257. if (idef->id_macro && ReplaceMacros && replace(idef))
  258. /* macro replacement should be performed */
  259. goto again;
  260. if (UnknownIdIsZero && idef->id_reserved != SIZEOF) {
  261. ptok->tk_ival = (arith)0;
  262. ptok->tk_fund = INT;
  263. return ptok->tk_symb = INTEGER;
  264. }
  265. #endif /* NOPP */
  266. ptok->tk_symb = (
  267. idef->id_reserved ? idef->id_reserved
  268. : idef->id_def && idef->id_def->df_sc == TYPEDEF ?
  269. TYPE_IDENTIFIER
  270. : IDENTIFIER
  271. );
  272. return IDENTIFIER;
  273. }
  274. case STCHAR: /* character constant */
  275. {
  276. register arith val = 0;
  277. int size = 0;
  278. LoadChar(ch);
  279. if (ch == '\'')
  280. lexerror("character constant too short");
  281. else
  282. while (ch != '\'') {
  283. if (ch == '\n') {
  284. lexerror("newline in character constant");
  285. PushBack();
  286. break;
  287. }
  288. if (ch == '\\') {
  289. LoadChar(ch);
  290. if (ch == '\n')
  291. LineNumber++;
  292. ch = quoted(ch);
  293. }
  294. if (ch >= 128) ch -= 256;
  295. val = val*256 + ch;
  296. size++;
  297. LoadChar(ch);
  298. }
  299. if (size > (int)int_size)
  300. lexerror("character constant too long");
  301. ptok->tk_ival = val;
  302. ptok->tk_fund = INT;
  303. return ptok->tk_symb = INTEGER;
  304. }
  305. case STSTR: /* string */
  306. ptok->tk_bts = string_token("string", '"', &(ptok->tk_len));
  307. return ptok->tk_symb = STRING;
  308. case STNUM: /* a numeric constant */
  309. {
  310. /* It should be noted that 099 means 81(decimal) and
  311. 099.5 means 99.5 . This severely limits the tricks
  312. we can use to scan a numeric value.
  313. */
  314. register char *np = &buf[1];
  315. register int base = 10;
  316. register int vch;
  317. register arith val = 0;
  318. if (ch == '.') { /* an embarrassing ambiguity */
  319. #ifndef NOFLOAT
  320. LoadChar(vch);
  321. PushBack();
  322. if (!is_dig(vch)) /* just a `.' */
  323. return ptok->tk_symb = ch;
  324. *np++ = '0';
  325. /* in the rest of the compiler, all floats
  326. have to start with a digit.
  327. */
  328. #else /* NOFLOAT */
  329. return ptok->tk_symb = ch;
  330. #endif /* NOFLOAT */
  331. }
  332. if (ch == '0') {
  333. *np++ = ch;
  334. LoadChar(ch);
  335. if (ch == 'x' || ch == 'X') {
  336. base = 16;
  337. LoadChar(ch);
  338. }
  339. else
  340. base = 8;
  341. }
  342. while (vch = val_in_base(ch, base), vch >= 0) {
  343. val = val*base + vch;
  344. if (np < &buf[NUMSIZE])
  345. *np++ = ch;
  346. LoadChar(ch);
  347. }
  348. if (ch == 'l' || ch == 'L') {
  349. ptok->tk_ival = val;
  350. ptok->tk_fund = LONG;
  351. return ptok->tk_symb = INTEGER;
  352. }
  353. #ifndef NOFLOAT
  354. if (base == 16 || !(ch == '.' || ch == 'e' || ch == 'E'))
  355. #endif /* NOFLOAT */
  356. {
  357. PushBack();
  358. ptok->tk_ival = val;
  359. /* The semantic analyser must know if the
  360. integral constant is given in octal/hexa-
  361. decimal form, in which case its type is
  362. UNSIGNED, or in decimal form, in which case
  363. its type is signed, indicated by
  364. the fund INTEGER.
  365. */
  366. ptok->tk_fund =
  367. (base == 10 || (base == 8 && val == (arith)0))
  368. ? INTEGER : UNSIGNED;
  369. return ptok->tk_symb = INTEGER;
  370. }
  371. /* where's the test for the length of the integral ??? */
  372. #ifndef NOFLOAT
  373. if (ch == '.'){
  374. if (np < &buf[NUMSIZE])
  375. *np++ = ch;
  376. LoadChar(ch);
  377. }
  378. while (is_dig(ch)){
  379. if (np < &buf[NUMSIZE])
  380. *np++ = ch;
  381. LoadChar(ch);
  382. }
  383. if (ch == 'e' || ch == 'E') {
  384. if (np < &buf[NUMSIZE])
  385. *np++ = ch;
  386. LoadChar(ch);
  387. if (ch == '+' || ch == '-') {
  388. if (np < &buf[NUMSIZE])
  389. *np++ = ch;
  390. LoadChar(ch);
  391. }
  392. if (!is_dig(ch)) {
  393. lexerror("malformed floating constant");
  394. if (np < &buf[NUMSIZE])
  395. *np++ = ch;
  396. }
  397. while (is_dig(ch)) {
  398. if (np < &buf[NUMSIZE])
  399. *np++ = ch;
  400. LoadChar(ch);
  401. }
  402. }
  403. PushBack();
  404. *np++ = '\0';
  405. buf[0] = '-'; /* good heavens... */
  406. if (np == &buf[NUMSIZE+1]) {
  407. lexerror("floating constant too long");
  408. ptok->tk_fval = Salloc("-0.0",(unsigned) 5) + 1;
  409. }
  410. else
  411. ptok->tk_fval = Salloc(buf,(unsigned) (np - buf)) + 1;
  412. return ptok->tk_symb = FLOATING;
  413. #endif /* NOFLOAT */
  414. }
  415. case STEOI: /* end of text on source file */
  416. return ptok->tk_symb = EOI;
  417. default: /* this cannot happen */
  418. crash("bad class for char 0%o", ch);
  419. }
  420. /*NOTREACHED*/
  421. }
  422. skipcomment()
  423. {
  424. /* The last character read has been the '*' of '/_*'. The
  425. characters, except NL and EOI, between '/_*' and the first
  426. occurring '*_/' are not interpreted.
  427. NL only affects the LineNumber. EOI is not legal.
  428. Important note: it is not possible to stop skipping comment
  429. beyond the end-of-file of an included file.
  430. EOI is returned by LoadChar only on encountering EOF of the
  431. top-level file...
  432. */
  433. register int c;
  434. NoUnstack++;
  435. LoadChar(c);
  436. #ifdef LINT
  437. if (! lint_skip_comment) {
  438. lint_start_comment();
  439. lint_comment_char(c);
  440. }
  441. #endif /* LINT */
  442. do {
  443. while (c != '*') {
  444. if (class(c) == STNL)
  445. ++LineNumber;
  446. else
  447. if (c == EOI) {
  448. NoUnstack--;
  449. #ifdef LINT
  450. if (! lint_skip_comment) lint_end_comment();
  451. #endif /* LINT */
  452. return;
  453. }
  454. LoadChar(c);
  455. #ifdef LINT
  456. if (! lint_skip_comment) lint_comment_char(c);
  457. #endif /* LINT */
  458. } /* last Character seen was '*' */
  459. LoadChar(c);
  460. #ifdef LINT
  461. if (! lint_skip_comment) lint_comment_char(c);
  462. #endif /* LINT */
  463. } while (c != '/');
  464. #ifdef LINT
  465. if (! lint_skip_comment) lint_end_comment();
  466. #endif /* LINT */
  467. NoUnstack--;
  468. }
  469. char *
  470. string_token(nm, stop_char, plen)
  471. char *nm;
  472. int *plen;
  473. {
  474. register int ch;
  475. register int str_size;
  476. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  477. register int pos = 0;
  478. LoadChar(ch);
  479. while (ch != stop_char) {
  480. if (ch == '\n') {
  481. lexerror("newline in %s", nm);
  482. PushBack();
  483. break;
  484. }
  485. if (ch == EOI) {
  486. lexerror("end-of-file inside %s", nm);
  487. break;
  488. }
  489. if (ch == '\\') {
  490. LoadChar(ch);
  491. if (ch == '\n') {
  492. LineNumber++;
  493. LoadChar(ch);
  494. continue;
  495. }
  496. ch = quoted(ch);
  497. }
  498. str[pos++] = ch;
  499. if (pos == str_size)
  500. str = Srealloc(str, (unsigned) (str_size += RSTRSIZE));
  501. LoadChar(ch);
  502. }
  503. str[pos++] = '\0'; /* for filenames etc. */
  504. *plen = pos;
  505. return str;
  506. }
  507. int
  508. quoted(ch)
  509. register int ch;
  510. {
  511. /* quoted() replaces an escaped character sequence by the
  512. character meant.
  513. */
  514. /* first char after backslash already in ch */
  515. if (!is_oct(ch)) { /* a quoted char */
  516. switch (ch) {
  517. case 'n':
  518. ch = '\n';
  519. break;
  520. case 't':
  521. ch = '\t';
  522. break;
  523. case 'b':
  524. ch = '\b';
  525. break;
  526. case 'r':
  527. ch = '\r';
  528. break;
  529. case 'f':
  530. ch = '\f';
  531. break;
  532. }
  533. }
  534. else { /* a quoted octal */
  535. register int oct = 0, cnt = 0;
  536. do {
  537. oct = oct*8 + (ch-'0');
  538. LoadChar(ch);
  539. } while (is_oct(ch) && ++cnt < 3);
  540. PushBack();
  541. ch = oct;
  542. }
  543. return ch&0377;
  544. }
  545. /* provisional */
  546. int
  547. val_in_base(ch, base)
  548. register int ch;
  549. {
  550. return
  551. is_dig(ch) ? ch - '0'
  552. : base != 16 ? -1
  553. : is_hex(ch) ? (ch - 'a' + 10) & 017
  554. : -1;
  555. }