LLlex.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Header$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "lint.h"
  8. #include <alloc.h>
  9. #include "nofloat.h"
  10. #include "idfsize.h"
  11. #include "numsize.h"
  12. #include "debug.h"
  13. #include "strsize.h"
  14. #include "nopp.h"
  15. #include "input.h"
  16. #include "arith.h"
  17. #include "def.h"
  18. #include "idf.h"
  19. #include "LLlex.h"
  20. #include "Lpars.h"
  21. #include "class.h"
  22. #include "assert.h"
  23. #include "sizes.h"
  24. /* Data about the token yielded */
  25. struct token dot, ahead, aside;
  26. int token_nmb = 0; /* number of the ahead token */
  27. int tk_nmb_at_last_syn_err = -5/*ERR_SHADOW*/;
  28. /* token number at last syntax error */
  29. #ifndef NOPP
  30. int ReplaceMacros = 1; /* replacing macros */
  31. int AccDefined = 0; /* accept "defined(...)" */
  32. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  33. int Unstacked = 0; /* an unstack is done */
  34. #endif
  35. int SkipEscNewline = 0; /* how to interpret backslash-newline */
  36. int AccFileSpecifier = 0; /* return filespecifier <...> */
  37. int EoiForNewline = 0; /* return EOI upon encountering newline */
  38. int File_Inserted = 0; /* a file has just been inserted */
  39. #define MAX_LL_DEPTH 2
  40. static struct token LexStack[MAX_LL_DEPTH];
  41. static LexSP = 0;
  42. /* In PushLex() the actions are taken in order to initialise or
  43. re-initialise the lexical scanner.
  44. E.g. at the invocation of a sub-parser that uses LLlex(), the
  45. state of the current parser should be saved.
  46. */
  47. PushLex()
  48. {
  49. ASSERT(LexSP < 2);
  50. ASSERT(ASIDE == 0); /* ASIDE = 0; */
  51. GetToken(&ahead);
  52. LexStack[LexSP++] = dot;
  53. }
  54. PopLex()
  55. {
  56. ASSERT(LexSP > 0);
  57. dot = LexStack[--LexSP];
  58. }
  59. int
  60. LLlex()
  61. {
  62. /* LLlex() plays the role of Lexical Analyzer for the C parser.
  63. The look-ahead and putting aside of tokens are taken into
  64. account.
  65. */
  66. if (ASIDE) { /* a token is put aside */
  67. dot = aside;
  68. ASIDE = 0;
  69. }
  70. else { /* read ahead and return the old one */
  71. #ifdef LINT
  72. lint_comment_ahead();
  73. #endif LINT
  74. dot = ahead;
  75. /* the following test is performed due to the dual
  76. task of LLlex(): it is also called for parsing the
  77. restricted constant expression following a #if or
  78. #elif. The newline character causes EOF to be
  79. returned in this case to stop the LLgen parsing task.
  80. */
  81. if (DOT != EOI)
  82. GetToken(&ahead);
  83. else
  84. DOT = EOF;
  85. }
  86. return DOT;
  87. }
  88. char *string_token();
  89. int
  90. GetToken(ptok)
  91. register struct token *ptok;
  92. {
  93. /* GetToken() is the actual token recognizer. It calls the
  94. control line interpreter if it encounters a "\n#"
  95. combination. Macro replacement is also performed if it is
  96. needed.
  97. */
  98. char buf[(IDFSIZE > NUMSIZE ? IDFSIZE : NUMSIZE) + 1];
  99. register int ch, nch;
  100. token_nmb++;
  101. if (File_Inserted) {
  102. File_Inserted = 0;
  103. goto firstline;
  104. }
  105. again: /* rescan the input after an error or replacement */
  106. #ifndef NOPP
  107. if (Unstacked) EnableMacros();
  108. #endif
  109. LoadChar(ch);
  110. go_on: /* rescan, the following character has been read */
  111. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  112. fatal("non-ascii '\\%03o' read", ch & 0377);
  113. /* keep track of the place of the token in the file */
  114. ptok->tk_file = FileName;
  115. ptok->tk_line = LineNumber;
  116. switch (class(ch)) { /* detect character class */
  117. case STNL: /* newline, vertical space or formfeed */
  118. firstline:
  119. LineNumber++; /* also at vs and ff */
  120. ptok->tk_file = FileName;
  121. ptok->tk_line = LineNumber;
  122. if (EoiForNewline) /* called in control line */
  123. /* a newline in a control line indicates the
  124. end-of-information of the line.
  125. */
  126. return ptok->tk_symb = EOI;
  127. while (LoadChar(ch), ch == '#') { /* a control line follows */
  128. domacro();
  129. if (File_Inserted) {
  130. File_Inserted = 0;
  131. goto firstline;
  132. }
  133. }
  134. /* We have to loop here, because in
  135. `domacro' the nl, vt or ff is read. The
  136. character following it may again be a `#'.
  137. */
  138. goto go_on;
  139. case STSKIP: /* just skip the skip characters */
  140. goto again;
  141. case STGARB: /* garbage character */
  142. #ifndef NOPP
  143. if (SkipEscNewline && (ch == '\\')) {
  144. /* a '\\' is allowed in #if/#elif expression */
  145. LoadChar(ch);
  146. if (class(ch) == STNL) { /* vt , ff ? */
  147. ++LineNumber;
  148. goto again;
  149. }
  150. PushBack();
  151. ch = '\\';
  152. }
  153. #endif NOPP
  154. if (040 < ch && ch < 0177)
  155. lexerror("garbage char %c", ch);
  156. else
  157. lexerror("garbage char \\%03o", ch);
  158. goto again;
  159. case STSIMP: /* a simple character, no part of compound token*/
  160. if (ch == '/') { /* probably the start of comment */
  161. LoadChar(ch);
  162. if (ch == '*') { /* start of comment */
  163. skipcomment();
  164. goto again;
  165. }
  166. else {
  167. PushBack();
  168. ch = '/'; /* restore ch */
  169. }
  170. }
  171. return ptok->tk_symb = ch;
  172. case STCOMP: /* maybe the start of a compound token */
  173. LoadChar(nch); /* character lookahead */
  174. switch (ch) {
  175. case '!':
  176. if (nch == '=')
  177. return ptok->tk_symb = NOTEQUAL;
  178. PushBack();
  179. return ptok->tk_symb = ch;
  180. case '&':
  181. if (nch == '&')
  182. return ptok->tk_symb = AND;
  183. PushBack();
  184. return ptok->tk_symb = ch;
  185. case '+':
  186. if (nch == '+')
  187. return ptok->tk_symb = PLUSPLUS;
  188. PushBack();
  189. return ptok->tk_symb = ch;
  190. case '-':
  191. if (nch == '-')
  192. return ptok->tk_symb = MINMIN;
  193. if (nch == '>')
  194. return ptok->tk_symb = ARROW;
  195. PushBack();
  196. return ptok->tk_symb = ch;
  197. case '<':
  198. if (AccFileSpecifier) {
  199. PushBack(); /* pushback nch */
  200. ptok->tk_bts = string_token("file specifier",
  201. '>', &(ptok->tk_len));
  202. return ptok->tk_symb = FILESPECIFIER;
  203. }
  204. if (nch == '<')
  205. return ptok->tk_symb = LEFT;
  206. if (nch == '=')
  207. return ptok->tk_symb = LESSEQ;
  208. PushBack();
  209. return ptok->tk_symb = ch;
  210. case '=':
  211. if (nch == '=')
  212. return ptok->tk_symb = EQUAL;
  213. PushBack();
  214. return ptok->tk_symb = ch;
  215. case '>':
  216. if (nch == '=')
  217. return ptok->tk_symb = GREATEREQ;
  218. if (nch == '>')
  219. return ptok->tk_symb = RIGHT;
  220. PushBack();
  221. return ptok->tk_symb = ch;
  222. case '|':
  223. if (nch == '|')
  224. return ptok->tk_symb = OR;
  225. PushBack();
  226. return ptok->tk_symb = ch;
  227. }
  228. case STIDF:
  229. {
  230. register char *tg = &buf[0];
  231. register int pos = -1;
  232. register int hash;
  233. register struct idf *idef;
  234. extern int idfsize; /* ??? */
  235. hash = STARTHASH();
  236. do { /* read the identifier */
  237. if (++pos < idfsize) {
  238. #ifndef NOPP
  239. if (Unstacked) EnableMacros();
  240. #endif
  241. *tg++ = ch;
  242. hash = ENHASH(hash, ch, pos);
  243. }
  244. LoadChar(ch);
  245. } while (in_idf(ch));
  246. hash = STOPHASH(hash);
  247. if (ch != EOI)
  248. PushBack();
  249. *tg++ = '\0'; /* mark the end of the identifier */
  250. idef = ptok->tk_idf = idf_hashed(buf, tg - buf, hash);
  251. idef->id_file = ptok->tk_file;
  252. idef->id_line = ptok->tk_line;
  253. #ifndef NOPP
  254. if (idef->id_macro && ReplaceMacros && replace(idef))
  255. /* macro replacement should be performed */
  256. goto again;
  257. if (UnknownIdIsZero && idef->id_reserved != SIZEOF) {
  258. ptok->tk_ival = (arith)0;
  259. ptok->tk_fund = INT;
  260. return ptok->tk_symb = INTEGER;
  261. }
  262. #endif NOPP
  263. ptok->tk_symb = (
  264. idef->id_reserved ? idef->id_reserved
  265. : idef->id_def && idef->id_def->df_sc == TYPEDEF ?
  266. TYPE_IDENTIFIER
  267. : IDENTIFIER
  268. );
  269. return IDENTIFIER;
  270. }
  271. case STCHAR: /* character constant */
  272. {
  273. register arith val = 0;
  274. int size = 0;
  275. LoadChar(ch);
  276. if (ch == '\'')
  277. lexerror("character constant too short");
  278. else
  279. while (ch != '\'') {
  280. if (ch == '\n') {
  281. lexerror("newline in character constant");
  282. PushBack();
  283. break;
  284. }
  285. if (ch == '\\') {
  286. LoadChar(ch);
  287. if (ch == '\n')
  288. LineNumber++;
  289. ch = quoted(ch);
  290. }
  291. if (ch >= 128) ch -= 256;
  292. val = val*256 + ch;
  293. size++;
  294. LoadChar(ch);
  295. }
  296. if (size > (int)int_size)
  297. lexerror("character constant too long");
  298. ptok->tk_ival = val;
  299. ptok->tk_fund = INT;
  300. return ptok->tk_symb = INTEGER;
  301. }
  302. case STSTR: /* string */
  303. ptok->tk_bts = string_token("string", '"', &(ptok->tk_len));
  304. return ptok->tk_symb = STRING;
  305. case STNUM: /* a numeric constant */
  306. {
  307. /* It should be noted that 099 means 81(decimal) and
  308. 099.5 means 99.5 . This severely limits the tricks
  309. we can use to scan a numeric value.
  310. */
  311. register char *np = &buf[1];
  312. register int base = 10;
  313. register int vch;
  314. register arith val = 0;
  315. if (ch == '.') { /* an embarrassing ambiguity */
  316. #ifndef NOFLOAT
  317. LoadChar(vch);
  318. PushBack();
  319. if (!is_dig(vch)) /* just a `.' */
  320. return ptok->tk_symb = ch;
  321. *np++ = '0';
  322. /* in the rest of the compiler, all floats
  323. have to start with a digit.
  324. */
  325. #else NOFLOAT
  326. return ptok->tk_symb = ch;
  327. #endif NOFLOAT
  328. }
  329. if (ch == '0') {
  330. *np++ = ch;
  331. LoadChar(ch);
  332. if (ch == 'x' || ch == 'X') {
  333. base = 16;
  334. LoadChar(ch);
  335. }
  336. else
  337. base = 8;
  338. }
  339. while (vch = val_in_base(ch, base), vch >= 0) {
  340. val = val*base + vch;
  341. if (np < &buf[NUMSIZE])
  342. *np++ = ch;
  343. LoadChar(ch);
  344. }
  345. if (ch == 'l' || ch == 'L') {
  346. ptok->tk_ival = val;
  347. ptok->tk_fund = LONG;
  348. return ptok->tk_symb = INTEGER;
  349. }
  350. #ifndef NOFLOAT
  351. if (base == 16 || !(ch == '.' || ch == 'e' || ch == 'E'))
  352. #endif NOFLOAT
  353. {
  354. PushBack();
  355. ptok->tk_ival = val;
  356. /* The semantic analyser must know if the
  357. integral constant is given in octal/hexa-
  358. decimal form, in which case its type is
  359. UNSIGNED, or in decimal form, in which case
  360. its type is signed, indicated by
  361. the fund INTEGER.
  362. */
  363. ptok->tk_fund =
  364. (base == 10 || (base == 8 && val == (arith)0))
  365. ? INTEGER : UNSIGNED;
  366. return ptok->tk_symb = INTEGER;
  367. }
  368. /* where's the test for the length of the integral ??? */
  369. #ifndef NOFLOAT
  370. if (ch == '.'){
  371. if (np < &buf[NUMSIZE])
  372. *np++ = ch;
  373. LoadChar(ch);
  374. }
  375. while (is_dig(ch)){
  376. if (np < &buf[NUMSIZE])
  377. *np++ = ch;
  378. LoadChar(ch);
  379. }
  380. if (ch == 'e' || ch == 'E') {
  381. if (np < &buf[NUMSIZE])
  382. *np++ = ch;
  383. LoadChar(ch);
  384. if (ch == '+' || ch == '-') {
  385. if (np < &buf[NUMSIZE])
  386. *np++ = ch;
  387. LoadChar(ch);
  388. }
  389. if (!is_dig(ch)) {
  390. lexerror("malformed floating constant");
  391. if (np < &buf[NUMSIZE])
  392. *np++ = ch;
  393. }
  394. while (is_dig(ch)) {
  395. if (np < &buf[NUMSIZE])
  396. *np++ = ch;
  397. LoadChar(ch);
  398. }
  399. }
  400. PushBack();
  401. *np++ = '\0';
  402. buf[0] = '-'; /* good heavens... */
  403. if (np == &buf[NUMSIZE+1]) {
  404. lexerror("floating constant too long");
  405. ptok->tk_fval = Salloc("-0.0",(unsigned) 5) + 1;
  406. }
  407. else
  408. ptok->tk_fval = Salloc(buf,(unsigned) (np - buf)) + 1;
  409. return ptok->tk_symb = FLOATING;
  410. #endif NOFLOAT
  411. }
  412. case STEOI: /* end of text on source file */
  413. return ptok->tk_symb = EOI;
  414. default: /* this cannot happen */
  415. crash("bad class for char 0%o", ch);
  416. }
  417. /*NOTREACHED*/
  418. }
  419. skipcomment()
  420. {
  421. /* The last character read has been the '*' of '/_*'. The
  422. characters, except NL and EOI, between '/_*' and the first
  423. occurring '*_/' are not interpreted.
  424. NL only affects the LineNumber. EOI is not legal.
  425. Important note: it is not possible to stop skipping comment
  426. beyond the end-of-file of an included file.
  427. EOI is returned by LoadChar only on encountering EOF of the
  428. top-level file...
  429. */
  430. register int c;
  431. NoUnstack++;
  432. LoadChar(c);
  433. #ifdef LINT
  434. lint_start_comment();
  435. lint_comment_char(c);
  436. #endif LINT
  437. do {
  438. while (c != '*') {
  439. if (class(c) == STNL)
  440. ++LineNumber;
  441. else
  442. if (c == EOI) {
  443. NoUnstack--;
  444. #ifdef LINT
  445. lint_end_comment();
  446. #endif LINT
  447. return;
  448. }
  449. LoadChar(c);
  450. #ifdef LINT
  451. lint_comment_char(c);
  452. #endif LINT
  453. } /* last Character seen was '*' */
  454. LoadChar(c);
  455. #ifdef LINT
  456. lint_comment_char(c);
  457. #endif LINT
  458. } while (c != '/');
  459. #ifdef LINT
  460. lint_end_comment();
  461. #endif LINT
  462. NoUnstack--;
  463. }
  464. char *
  465. string_token(nm, stop_char, plen)
  466. char *nm;
  467. int *plen;
  468. {
  469. register int ch;
  470. register int str_size;
  471. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  472. register int pos = 0;
  473. LoadChar(ch);
  474. while (ch != stop_char) {
  475. if (ch == '\n') {
  476. lexerror("newline in %s", nm);
  477. PushBack();
  478. break;
  479. }
  480. if (ch == EOI) {
  481. lexerror("end-of-file inside %s", nm);
  482. break;
  483. }
  484. if (ch == '\\') {
  485. LoadChar(ch);
  486. if (ch == '\n') {
  487. LineNumber++;
  488. LoadChar(ch);
  489. continue;
  490. }
  491. ch = quoted(ch);
  492. }
  493. str[pos++] = ch;
  494. if (pos == str_size)
  495. str = Srealloc(str, (unsigned) (str_size += RSTRSIZE));
  496. LoadChar(ch);
  497. }
  498. str[pos++] = '\0'; /* for filenames etc. */
  499. *plen = pos;
  500. return str;
  501. }
  502. int
  503. quoted(ch)
  504. register int ch;
  505. {
  506. /* quoted() replaces an escaped character sequence by the
  507. character meant.
  508. */
  509. /* first char after backslash already in ch */
  510. if (!is_oct(ch)) { /* a quoted char */
  511. switch (ch) {
  512. case 'n':
  513. ch = '\n';
  514. break;
  515. case 't':
  516. ch = '\t';
  517. break;
  518. case 'b':
  519. ch = '\b';
  520. break;
  521. case 'r':
  522. ch = '\r';
  523. break;
  524. case 'f':
  525. ch = '\f';
  526. break;
  527. }
  528. }
  529. else { /* a quoted octal */
  530. register int oct = 0, cnt = 0;
  531. do {
  532. oct = oct*8 + (ch-'0');
  533. LoadChar(ch);
  534. } while (is_oct(ch) && ++cnt < 3);
  535. PushBack();
  536. ch = oct;
  537. }
  538. return ch&0377;
  539. }
  540. /* provisional */
  541. int
  542. val_in_base(ch, base)
  543. register int ch;
  544. {
  545. return
  546. is_dig(ch) ? ch - '0'
  547. : base != 16 ? -1
  548. : is_hex(ch) ? (ch - 'a' + 10) & 017
  549. : -1;
  550. }