LLlex.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564
  1. /* $Header$ */
  2. /* L E X I C A L A N A L Y Z E R */
  3. #include <alloc.h>
  4. #include "nofloat.h"
  5. #include "idfsize.h"
  6. #include "numsize.h"
  7. #include "debug.h"
  8. #include "strsize.h"
  9. #include "nopp.h"
  10. #include "input.h"
  11. #include "arith.h"
  12. #include "def.h"
  13. #include "idf.h"
  14. #include "LLlex.h"
  15. #include "Lpars.h"
  16. #include "class.h"
  17. #include "assert.h"
  18. #include "sizes.h"
  19. /* Data about the token yielded */
  20. struct token dot, ahead, aside;
  21. int ReplaceMacros = 1; /* replacing macros */
  22. int EoiForNewline = 0; /* return EOI upon encountering newline */
  23. int PreProcKeys = 0; /* return preprocessor key */
  24. int AccFileSpecifier = 0; /* return filespecifier <...> */
  25. int AccDefined = 0; /* accept "defined(...)" */
  26. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  27. int SkipEscNewline = 0; /* how to interpret backslash-newline */
  28. int Unstacked = 0; /* an unstack is done */
  29. #define MAX_LL_DEPTH 2
  30. static struct token LexStack[MAX_LL_DEPTH];
  31. static LexSP = 0;
  32. /* In PushLex() the actions are taken in order to initialise or
  33. re-initialise the lexical scanner.
  34. E.g. at the invocation of a sub-parser that uses LLlex(), the
  35. state of the current parser should be saved.
  36. */
  37. PushLex()
  38. {
  39. ASSERT(LexSP < 2);
  40. ASSERT(ASIDE == 0); /* ASIDE = 0; */
  41. GetToken(&ahead);
  42. ahead.tk_line = LineNumber;
  43. ahead.tk_file = FileName;
  44. LexStack[LexSP++] = dot;
  45. }
  46. PopLex()
  47. {
  48. ASSERT(LexSP > 0);
  49. dot = LexStack[--LexSP];
  50. }
  51. int
  52. LLlex()
  53. {
  54. /* LLlex() plays the role of Lexical Analyzer for the C parser.
  55. The look-ahead and putting aside of tokens are taken into
  56. account.
  57. */
  58. if (ASIDE) { /* a token is put aside */
  59. dot = aside;
  60. ASIDE = 0;
  61. }
  62. else { /* read ahead and return the old one */
  63. dot = ahead;
  64. /* the following test is performed due to the dual
  65. task of LLlex(): it is also called for parsing the
  66. restricted constant expression following a #if or
  67. #elif. The newline character causes EOF to be
  68. returned in this case to stop the LLgen parsing task.
  69. */
  70. if (DOT != EOI)
  71. GetToken(&ahead);
  72. else
  73. DOT = EOF;
  74. }
  75. /* keep track of the place of the token in the file */
  76. ahead.tk_file = FileName;
  77. ahead.tk_line = LineNumber;
  78. return DOT;
  79. }
  80. char *string_token();
  81. int
  82. GetToken(ptok)
  83. register struct token *ptok;
  84. {
  85. /* GetToken() is the actual token recognizer. It calls the
  86. control line interpreter if it encounters a "\n#"
  87. combination. Macro replacement is also performed if it is
  88. needed.
  89. */
  90. char buf[(IDFSIZE > NUMSIZE ? IDFSIZE : NUMSIZE) + 1];
  91. register int ch, nch;
  92. if (! LineNumber) goto firstline;
  93. again: /* rescan the input after an error or replacement */
  94. #ifndef NOPP
  95. if (Unstacked) EnableMacros();
  96. #endif
  97. LoadChar(ch);
  98. go_on: /* rescan, the following character has been read */
  99. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  100. fatal("non-ascii '\\%03o' read", ch & 0377);
  101. switch (class(ch)) { /* detect character class */
  102. case STNL: /* newline, vertical space or formfeed */
  103. firstline:
  104. LineNumber++; /* also at vs and ff */
  105. if (EoiForNewline) /* called in control line */
  106. /* a newline in a control line indicates the
  107. end-of-information of the line.
  108. */
  109. return ptok->tk_symb = EOI;
  110. while (LoadChar(ch), ch == '#') { /* a control line follows */
  111. domacro();
  112. if (!LineNumber) goto firstline;
  113. }
  114. /* We have to loop here, because in
  115. `domacro' the nl, vt or ff is read. The
  116. character following it may again be a `#'.
  117. */
  118. goto go_on;
  119. case STSKIP: /* just skip the skip characters */
  120. goto again;
  121. case STGARB: /* garbage character */
  122. #ifndef NOPP
  123. if (SkipEscNewline && (ch == '\\')) {
  124. /* a '\\' is allowed in #if/#elif expression */
  125. LoadChar(ch);
  126. if (class(ch) == STNL) { /* vt , ff ? */
  127. ++LineNumber;
  128. goto again;
  129. }
  130. PushBack();
  131. ch = '\\';
  132. }
  133. #endif NOPP
  134. if (040 < ch && ch < 0177)
  135. lexerror("garbage char %c", ch);
  136. else
  137. lexerror("garbage char \\%03o", ch);
  138. goto again;
  139. case STSIMP: /* a simple character, no part of compound token*/
  140. if (ch == '/') { /* probably the start of comment */
  141. LoadChar(ch);
  142. if (ch == '*') { /* start of comment */
  143. skipcomment();
  144. goto again;
  145. }
  146. else {
  147. PushBack();
  148. ch = '/'; /* restore ch */
  149. }
  150. }
  151. return ptok->tk_symb = ch;
  152. case STCOMP: /* maybe the start of a compound token */
  153. LoadChar(nch); /* character lookahead */
  154. switch (ch) {
  155. case '!':
  156. if (nch == '=')
  157. return ptok->tk_symb = NOTEQUAL;
  158. PushBack();
  159. return ptok->tk_symb = ch;
  160. case '&':
  161. if (nch == '&')
  162. return ptok->tk_symb = AND;
  163. PushBack();
  164. return ptok->tk_symb = ch;
  165. case '+':
  166. if (nch == '+')
  167. return ptok->tk_symb = PLUSPLUS;
  168. PushBack();
  169. return ptok->tk_symb = ch;
  170. case '-':
  171. if (nch == '-')
  172. return ptok->tk_symb = MINMIN;
  173. if (nch == '>')
  174. return ptok->tk_symb = ARROW;
  175. PushBack();
  176. return ptok->tk_symb = ch;
  177. case '<':
  178. if (AccFileSpecifier) {
  179. PushBack(); /* pushback nch */
  180. ptok->tk_bts = string_token("file specifier",
  181. '>', &(ptok->tk_len));
  182. return ptok->tk_symb = FILESPECIFIER;
  183. }
  184. if (nch == '<')
  185. return ptok->tk_symb = LEFT;
  186. if (nch == '=')
  187. return ptok->tk_symb = LESSEQ;
  188. PushBack();
  189. return ptok->tk_symb = ch;
  190. case '=':
  191. if (nch == '=')
  192. return ptok->tk_symb = EQUAL;
  193. /* The following piece of code tries to recognise
  194. old-fashioned assignment operators `=op'
  195. */
  196. switch (nch) {
  197. case '+':
  198. return ptok->tk_symb = PLUSAB;
  199. case '-':
  200. return ptok->tk_symb = MINAB;
  201. case '*':
  202. return ptok->tk_symb = TIMESAB;
  203. case '/':
  204. return ptok->tk_symb = DIVAB;
  205. case '%':
  206. return ptok->tk_symb = MODAB;
  207. case '>':
  208. case '<':
  209. LoadChar(ch);
  210. if (ch != nch) {
  211. PushBack();
  212. lexerror("illegal combination '=%c'",
  213. nch);
  214. }
  215. return ptok->tk_symb =
  216. nch == '<' ? LEFTAB : RIGHTAB;
  217. case '&':
  218. return ptok->tk_symb = ANDAB;
  219. case '^':
  220. return ptok->tk_symb = XORAB;
  221. case '|':
  222. return ptok->tk_symb = ORAB;
  223. }
  224. PushBack();
  225. return ptok->tk_symb = ch;
  226. case '>':
  227. if (nch == '=')
  228. return ptok->tk_symb = GREATEREQ;
  229. if (nch == '>')
  230. return ptok->tk_symb = RIGHT;
  231. PushBack();
  232. return ptok->tk_symb = ch;
  233. case '|':
  234. if (nch == '|')
  235. return ptok->tk_symb = OR;
  236. PushBack();
  237. return ptok->tk_symb = ch;
  238. }
  239. case STIDF:
  240. {
  241. register char *tg = &buf[0];
  242. register int pos = -1;
  243. register int hash;
  244. register struct idf *idef;
  245. extern int idfsize; /* ??? */
  246. hash = STARTHASH();
  247. do { /* read the identifier */
  248. if (++pos < idfsize) {
  249. #ifndef NOPP
  250. if (Unstacked) EnableMacros();
  251. #endif
  252. *tg++ = ch;
  253. hash = ENHASH(hash, ch, pos);
  254. }
  255. LoadChar(ch);
  256. } while (in_idf(ch));
  257. hash = STOPHASH(hash);
  258. if (ch != EOI)
  259. PushBack();
  260. *tg++ = '\0'; /* mark the end of the identifier */
  261. idef = ptok->tk_idf = idf_hashed(buf, tg - buf, hash);
  262. #ifndef NOPP
  263. if (idef->id_macro && ReplaceMacros && replace(idef))
  264. /* macro replacement should be performed */
  265. goto again;
  266. if (UnknownIdIsZero && idef->id_reserved != SIZEOF) {
  267. ptok->tk_ival = (arith)0;
  268. ptok->tk_fund = INT;
  269. return ptok->tk_symb = INTEGER;
  270. }
  271. #endif NOPP
  272. ptok->tk_symb = (
  273. idef->id_reserved ? idef->id_reserved
  274. : idef->id_def && idef->id_def->df_sc == TYPEDEF ?
  275. TYPE_IDENTIFIER
  276. : IDENTIFIER
  277. );
  278. return IDENTIFIER;
  279. }
  280. case STCHAR: /* character constant */
  281. {
  282. register arith val = 0, size = 0;
  283. LoadChar(ch);
  284. if (ch == '\'')
  285. lexerror("character constant too short");
  286. else
  287. while (ch != '\'') {
  288. if (ch == '\n') {
  289. lexerror("newline in character constant");
  290. LineNumber++;
  291. break;
  292. }
  293. if (ch == '\\') {
  294. LoadChar(ch);
  295. if (ch == '\n')
  296. LineNumber++;
  297. ch = quoted(ch);
  298. }
  299. if (ch >= 128) ch -= 256;
  300. val = val*256 + ch;
  301. size++;
  302. LoadChar(ch);
  303. }
  304. if (size > int_size)
  305. lexerror("character constant too long");
  306. ptok->tk_ival = val;
  307. ptok->tk_fund = INT;
  308. return ptok->tk_symb = INTEGER;
  309. }
  310. case STSTR: /* string */
  311. ptok->tk_bts = string_token("string", '"', &(ptok->tk_len));
  312. return ptok->tk_symb = STRING;
  313. case STNUM: /* a numeric constant */
  314. {
  315. /* It should be noted that 099 means 81(decimal) and
  316. 099.5 means 99.5 . This severely limits the tricks
  317. we can use to scan a numeric value.
  318. */
  319. register char *np = &buf[1];
  320. register int base = 10;
  321. register int vch;
  322. register arith val = 0;
  323. if (ch == '.') { /* an embarrassing ambiguity */
  324. #ifndef NOFLOAT
  325. LoadChar(vch);
  326. PushBack();
  327. if (!is_dig(vch)) /* just a `.' */
  328. return ptok->tk_symb = ch;
  329. *np++ = '0';
  330. /* in the rest of the compiler, all floats
  331. have to start with a digit.
  332. */
  333. #else NOFLOAT
  334. return ptok->tk_symb = ch;
  335. #endif NOFLOAT
  336. }
  337. if (ch == '0') {
  338. *np++ = ch;
  339. LoadChar(ch);
  340. if (ch == 'x' || ch == 'X') {
  341. base = 16;
  342. LoadChar(ch);
  343. }
  344. else
  345. base = 8;
  346. }
  347. while (vch = val_in_base(ch, base), vch >= 0) {
  348. val = val*base + vch;
  349. if (np < &buf[NUMSIZE])
  350. *np++ = ch;
  351. LoadChar(ch);
  352. }
  353. if (ch == 'l' || ch == 'L') {
  354. ptok->tk_ival = val;
  355. ptok->tk_fund = LONG;
  356. return ptok->tk_symb = INTEGER;
  357. }
  358. #ifndef NOFLOAT
  359. if (base == 16 || !(ch == '.' || ch == 'e' || ch == 'E'))
  360. #endif NOFLOAT
  361. {
  362. PushBack();
  363. ptok->tk_ival = val;
  364. /* The semantic analyser must know if the
  365. integral constant is given in octal/hexa-
  366. decimal form, in which case its type is
  367. UNSIGNED, or in decimal form, in which case
  368. its type is signed, indicated by
  369. the fund INTEGER.
  370. */
  371. ptok->tk_fund =
  372. (base == 10 || (base == 8 && val == (arith)0))
  373. ? INTEGER : UNSIGNED;
  374. return ptok->tk_symb = INTEGER;
  375. }
  376. /* where's the test for the length of the integral ??? */
  377. #ifndef NOFLOAT
  378. if (ch == '.'){
  379. if (np < &buf[NUMSIZE])
  380. *np++ = ch;
  381. LoadChar(ch);
  382. }
  383. while (is_dig(ch)){
  384. if (np < &buf[NUMSIZE])
  385. *np++ = ch;
  386. LoadChar(ch);
  387. }
  388. if (ch == 'e' || ch == 'E') {
  389. if (np < &buf[NUMSIZE])
  390. *np++ = ch;
  391. LoadChar(ch);
  392. if (ch == '+' || ch == '-') {
  393. if (np < &buf[NUMSIZE])
  394. *np++ = ch;
  395. LoadChar(ch);
  396. }
  397. if (!is_dig(ch)) {
  398. lexerror("malformed floating constant");
  399. if (np < &buf[NUMSIZE])
  400. *np++ = ch;
  401. }
  402. while (is_dig(ch)) {
  403. if (np < &buf[NUMSIZE])
  404. *np++ = ch;
  405. LoadChar(ch);
  406. }
  407. }
  408. PushBack();
  409. *np++ = '\0';
  410. buf[0] = '-'; /* good heavens... */
  411. if (np == &buf[NUMSIZE+1]) {
  412. lexerror("floating constant too long");
  413. ptok->tk_fval = Salloc("0.0", 5) + 1;
  414. }
  415. else
  416. ptok->tk_fval = Salloc(buf, np - buf) + 1;
  417. return ptok->tk_symb = FLOATING;
  418. #endif NOFLOAT
  419. }
  420. case STEOI: /* end of text on source file */
  421. return ptok->tk_symb = EOI;
  422. default: /* this cannot happen */
  423. crash("bad class for char 0%o", ch);
  424. }
  425. /*NOTREACHED*/
  426. }
  427. skipcomment()
  428. {
  429. /* The last character read has been the '*' of '/_*'. The
  430. characters, except NL and EOI, between '/_*' and the first
  431. occurring '*_/' are not interpreted.
  432. NL only affects the LineNumber. EOI is not legal.
  433. Important note: it is not possible to stop skipping comment
  434. beyond the end-of-file of an included file.
  435. EOI is returned by LoadChar only on encountering EOF of the
  436. top-level file...
  437. */
  438. register int c;
  439. NoUnstack++;
  440. LoadChar(c);
  441. do {
  442. while (c != '*') {
  443. if (class(c) == STNL)
  444. ++LineNumber;
  445. else
  446. if (c == EOI) {
  447. NoUnstack--;
  448. return;
  449. }
  450. LoadChar(c);
  451. } /* last Character seen was '*' */
  452. LoadChar(c);
  453. } while (c != '/');
  454. NoUnstack--;
  455. }
  456. char *
  457. string_token(nm, stop_char, plen)
  458. char *nm;
  459. int *plen;
  460. {
  461. register int ch;
  462. register int str_size;
  463. register char *str = Malloc(str_size = ISTRSIZE);
  464. register int pos = 0;
  465. LoadChar(ch);
  466. while (ch != stop_char) {
  467. if (ch == '\n') {
  468. lexerror("newline in %s", nm);
  469. LineNumber++;
  470. break;
  471. }
  472. if (ch == EOI) {
  473. lexerror("end-of-file inside %s", nm);
  474. break;
  475. }
  476. if (ch == '\\') {
  477. LoadChar(ch);
  478. if (ch == '\n') {
  479. LineNumber++;
  480. LoadChar(ch);
  481. continue;
  482. }
  483. ch = quoted(ch);
  484. }
  485. str[pos++] = ch;
  486. if (pos == str_size)
  487. str = Srealloc(str, str_size += RSTRSIZE);
  488. LoadChar(ch);
  489. }
  490. str[pos++] = '\0'; /* for filenames etc. */
  491. *plen = pos;
  492. return str;
  493. }
  494. int
  495. quoted(ch)
  496. register int ch;
  497. {
  498. /* quoted() replaces an escaped character sequence by the
  499. character meant.
  500. */
  501. /* first char after backslash already in ch */
  502. if (!is_oct(ch)) { /* a quoted char */
  503. switch (ch) {
  504. case 'n':
  505. ch = '\n';
  506. break;
  507. case 't':
  508. ch = '\t';
  509. break;
  510. case 'b':
  511. ch = '\b';
  512. break;
  513. case 'r':
  514. ch = '\r';
  515. break;
  516. case 'f':
  517. ch = '\f';
  518. break;
  519. }
  520. }
  521. else { /* a quoted octal */
  522. register int oct = 0, cnt = 0;
  523. do {
  524. oct = oct*8 + (ch-'0');
  525. LoadChar(ch);
  526. } while (is_oct(ch) && ++cnt < 3);
  527. PushBack();
  528. ch = oct;
  529. }
  530. return ch&0377;
  531. }
  532. /* provisional */
  533. int
  534. val_in_base(ch, base)
  535. register int ch;
  536. {
  537. return
  538. is_dig(ch) ? ch - '0'
  539. : base != 16 ? -1
  540. : is_hex(ch) ? (ch - 'a' + 10) & 017
  541. : -1;
  542. }