LLlex.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Id$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "idfsize.h"
  8. #include "numsize.h"
  9. #include "strsize.h"
  10. #include <alloc.h>
  11. #include "input.h"
  12. #include "arith.h"
  13. #include "macro.h"
  14. #include "idf.h"
  15. #include "LLlex.h"
  16. #include "Lpars.h"
  17. #include "class.h"
  18. #include "bits.h"
  19. #define BUFSIZ 1024
  20. struct token dot;
  21. int ReplaceMacros = 1; /* replacing macros */
  22. int AccDefined = 0; /* accept "defined(...)" */
  23. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  24. int Unstacked = 0; /* an unstack is done */
  25. int AccFileSpecifier = 0; /* return filespecifier <...> */
  26. int LexSave = 0; /* last character read by GetChar */
  27. extern int InputLevel; /* # of current macro expansions */
  28. extern char *string_token();
  29. extern char *strcpy();
  30. extern arith char_constant();
  31. #define FLG_ESEEN 0x01 /* possibly a floating point number */
  32. #define FLG_DOTSEEN 0x02 /* certainly a floating point number */
  33. int LLlex()
  34. {
  35. return (DOT != EOF) ? GetToken(&dot) : EOF;
  36. }
  37. int GetToken(struct token *ptok)
  38. {
  39. /* GetToken() is the actual token recognizer. It calls the
  40. control line interpreter if it encounters a "\n{w}*#"
  41. combination. Macro replacement is also performed if it is
  42. needed.
  43. */
  44. char buf[BUFSIZ];
  45. register int ch, nch;
  46. again: /* rescan the input after an error or replacement */
  47. ch = GetChar();
  48. /* rescan, the following character has been read */
  49. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  50. fatal("non-ascii '\\%03o' read", ch & 0377);
  51. /* keep track of the place of the token in the file */
  52. switch (class(ch)) { /* detect character class */
  53. case STNL: /* newline, vertical space or formfeed */
  54. LineNumber++;
  55. return ptok->tk_symb = EOF;
  56. case STSKIP: /* just skip the skip characters */
  57. goto again;
  58. case STGARB: /* garbage character */
  59. garbage:
  60. if (040 < ch && ch < 0177)
  61. error("garbage char %c", ch);
  62. else
  63. error("garbage char \\%03o", ch);
  64. goto again;
  65. case STSIMP: /* a simple character, no part of compound token*/
  66. return ptok->tk_symb = ch;
  67. case STCOMP: /* maybe the start of a compound token */
  68. nch = GetChar(); /* character lookahead */
  69. switch (ch) {
  70. case '!':
  71. if (nch == '=')
  72. return ptok->tk_symb = NOTEQUAL;
  73. UnGetChar();
  74. return ptok->tk_symb = ch;
  75. case '&':
  76. if (nch == '&')
  77. return ptok->tk_symb = AND;
  78. else if (nch == '=')
  79. return ptok->tk_symb = ANDAB;
  80. UnGetChar();
  81. return ptok->tk_symb = ch;
  82. case '+':
  83. if (nch == '+')
  84. return ptok->tk_symb = PLUSPLUS;
  85. else if (nch == '=')
  86. return ptok->tk_symb = PLUSAB;
  87. UnGetChar();
  88. return ptok->tk_symb = ch;
  89. case '-':
  90. if (nch == '-')
  91. return ptok->tk_symb = MINMIN;
  92. else if (nch == '>')
  93. return ptok->tk_symb = ARROW;
  94. else if (nch == '=')
  95. return ptok->tk_symb = MINAB;
  96. UnGetChar();
  97. return ptok->tk_symb = ch;
  98. case '<':
  99. if (AccFileSpecifier) {
  100. UnGetChar(); /* pushback nch */
  101. ptok->tk_str =
  102. string_token("file specifier", '>');
  103. return ptok->tk_symb = FILESPECIFIER;
  104. } else if (nch == '<') {
  105. if ((nch = GetChar()) == '=')
  106. return ptok->tk_symb = LEFTAB;
  107. UnGetChar();
  108. return ptok->tk_symb = LEFT;
  109. } else if (nch == '=')
  110. return ptok->tk_symb = LESSEQ;
  111. UnGetChar();
  112. return ptok->tk_symb = ch;
  113. case '=':
  114. if (nch == '=')
  115. return ptok->tk_symb = EQUAL;
  116. UnGetChar();
  117. return ptok->tk_symb = ch;
  118. case '>':
  119. if (nch == '=')
  120. return ptok->tk_symb = GREATEREQ;
  121. else if (nch == '>') {
  122. if ((nch = GetChar()) == '=')
  123. return ptok->tk_symb = RIGHTAB;
  124. UnGetChar();
  125. return ptok->tk_symb = RIGHT;
  126. }
  127. UnGetChar();
  128. return ptok->tk_symb = ch;
  129. case '|':
  130. if (nch == '|')
  131. return ptok->tk_symb = OR;
  132. else if (nch == '=')
  133. return ptok->tk_symb = ORAB;
  134. UnGetChar();
  135. return ptok->tk_symb = ch;
  136. case '%':
  137. if (nch == '=')
  138. return ptok->tk_symb = MODAB;
  139. UnGetChar();
  140. return ptok->tk_symb = ch;
  141. case '*':
  142. if (nch == '=')
  143. return ptok->tk_symb = TIMESAB;
  144. UnGetChar();
  145. return ptok->tk_symb = ch;
  146. case '^':
  147. if (nch == '=')
  148. return ptok->tk_symb = XORAB;
  149. UnGetChar();
  150. return ptok->tk_symb = ch;
  151. case '/':
  152. if (nch == '*' && !InputLevel) {
  153. skipcomment();
  154. goto again;
  155. }
  156. else if (nch == '=')
  157. return ptok->tk_symb = DIVAB;
  158. UnGetChar();
  159. return ptok->tk_symb = ch;
  160. default:
  161. crash("bad class for char 0%o", ch);
  162. /* NOTREACHED */
  163. }
  164. case STCHAR: /* character constant */
  165. ptok->tk_val = char_constant("character");
  166. return ptok->tk_symb = INTEGER;
  167. case STSTR: /* string */
  168. ptok->tk_str = string_token("string", '"');
  169. return ptok->tk_symb = STRING;
  170. case STELL: /* wide character constant/string prefix */
  171. nch = GetChar();
  172. if (nch == '"') {
  173. ptok->tk_str =
  174. string_token("wide character string", '"');
  175. return ptok->tk_symb = STRING;
  176. } else if (nch == '\'') {
  177. ptok->tk_val = char_constant("wide character");
  178. return ptok->tk_symb = INTEGER;
  179. }
  180. UnGetChar();
  181. /* fallthrough */
  182. case STIDF:
  183. {
  184. extern int idfsize; /* ??? */
  185. register char *tg = &buf[0];
  186. register char *maxpos = &buf[idfsize];
  187. int NoExpandNext = 0;
  188. #define tstmac(bx) if (!(bits[ch] & bx)) goto nomac
  189. #define cpy *tg++ = ch
  190. #define load (ch = GetChar()); if (!in_idf(ch)) goto endidf
  191. if (Unstacked) EnableMacros(); /* unstack macro's when allowed. */
  192. if (ch == NOEXPM) {
  193. NoExpandNext = 1;
  194. ch = GetChar();
  195. }
  196. #ifdef DOBITS
  197. cpy; tstmac(bit0); load;
  198. cpy; tstmac(bit1); load;
  199. cpy; tstmac(bit2); load;
  200. cpy; tstmac(bit3); load;
  201. cpy; tstmac(bit4); load;
  202. cpy; tstmac(bit5); load;
  203. cpy; tstmac(bit6); load;
  204. cpy; tstmac(bit7); load;
  205. #endif
  206. for(;;) {
  207. if (tg < maxpos) {
  208. cpy;
  209. }
  210. load;
  211. }
  212. endidf:
  213. /*if (ch != EOI) UnGetChar();*/
  214. UnGetChar();
  215. *tg++ = '\0'; /* mark the end of the identifier */
  216. if (ReplaceMacros) {
  217. register struct idf *idef = findidf(buf);
  218. if (idef && idef->id_macro && !NoExpandNext) {
  219. if (replace(idef))
  220. goto again;
  221. }
  222. }
  223. nomac: /* buf can already be null-terminated. soit */
  224. ch = GetChar();
  225. while (in_idf(ch)) {
  226. if (tg < maxpos) *tg++ = ch;
  227. ch = GetChar();
  228. }
  229. UnGetChar();
  230. *tg++ = '\0'; /* mark the end of the identifier */
  231. NoExpandNext = 0;
  232. if (UnknownIdIsZero) {
  233. ptok->tk_val = (arith)0;
  234. return ptok->tk_symb = INTEGER;
  235. }
  236. ptok->tk_str = Malloc((unsigned)(tg - buf));
  237. strcpy(ptok->tk_str, buf);
  238. return IDENTIFIER;
  239. }
  240. case STNUM: /* a numeric constant */
  241. { /* it may only be an integer constant */
  242. register int base = 10, vch;
  243. register arith val = 0;
  244. int ovfl = 0;
  245. arith ubound = ~(1<<(sizeof(arith)*8-1))/(base/2);
  246. /* Since the preprocessor only knows integers and has
  247. * nothing to do with ellipsis we just return when the
  248. * pp-number starts with a '.'
  249. */
  250. if (ch == '.') {
  251. return ptok->tk_symb = ch;
  252. }
  253. if (ch == '0') {
  254. ch = GetChar();
  255. if (ch == 'x' || ch == 'X') {
  256. base = 16;
  257. ch = GetChar();
  258. } else {
  259. base = 8;
  260. }
  261. }
  262. while ((vch = val_in_base(ch, base)) >= 0) {
  263. if (val < 0 || val > ubound) ovfl++;
  264. val *= base;
  265. if (val < 0 && val + vch >= 0) ovfl++;
  266. val += vch;
  267. ch = GetChar();
  268. }
  269. ptok->tk_unsigned = 0;
  270. if (ch == 'u' || ch == 'U') {
  271. ptok->tk_unsigned = 1;
  272. ch = GetChar();
  273. if (ch == 'l' || ch == 'L') {
  274. ch = GetChar();
  275. }
  276. }
  277. else if (ch == 'l' || ch == 'L') {
  278. ch = GetChar();
  279. if (ch == 'u' || ch == 'U') {
  280. ptok->tk_unsigned = 1;
  281. ch = GetChar();
  282. }
  283. }
  284. if (ovfl) {
  285. warning("overflow in constant");
  286. ptok->tk_unsigned = 1;
  287. }
  288. else if (val < 0) {
  289. /* give warning??? */
  290. ptok->tk_unsigned = 1;
  291. }
  292. UnGetChar();
  293. ptok->tk_val = val;
  294. return ptok->tk_symb = INTEGER;
  295. }
  296. case STEOI: /* end of text on source file */
  297. return ptok->tk_symb = EOF;
  298. case STMSPEC:
  299. if (!InputLevel) goto garbage;
  300. if (ch == TOKSEP) goto again;
  301. /* fallthrough shouldn't happen */
  302. default: /* this cannot happen */
  303. crash("bad class for char 0%o", ch);
  304. }
  305. /*NOTREACHED*/
  306. return -1;
  307. }
  308. void skipcomment()
  309. {
  310. /* The last character read has been the '*' of '/_*'. The
  311. characters, except NL and EOI, between '/_*' and the first
  312. occurring '*_/' are not interpreted.
  313. NL only affects the LineNumber. EOI is not legal.
  314. Important note: it is not possible to stop skipping comment
  315. beyond the end-of-file of an included file.
  316. EOI is returned by LoadChar only on encountering EOF of the
  317. top-level file...
  318. */
  319. register int c;
  320. NoUnstack++;
  321. c = GetChar();
  322. do {
  323. while (c != '*') {
  324. if (class(c) == STNL) {
  325. ++LineNumber;
  326. } else if (c == EOI) {
  327. NoUnstack--;
  328. return;
  329. }
  330. c = GetChar();
  331. } /* last Character seen was '*' */
  332. c = GetChar();
  333. } while (c != '/');
  334. NoUnstack--;
  335. }
  336. arith char_constant(char *nm)
  337. {
  338. register arith val = 0;
  339. register int ch;
  340. int size = 0;
  341. ch = GetChar();
  342. if (ch == '\'')
  343. error("%s constant too short", nm);
  344. else
  345. while (ch != '\'') {
  346. if (ch == '\n') {
  347. error("newline in %s constant", nm);
  348. LineNumber++;
  349. break;
  350. }
  351. if (ch == '\\')
  352. ch = quoted(GetChar());
  353. if (ch >= 128) ch -= 256;
  354. if (size < sizeof(arith))
  355. val |= ch << (8 * size);
  356. size++;
  357. ch = GetChar();
  358. }
  359. if (size > sizeof(arith))
  360. error("%s constant too long", nm);
  361. else if (size > 1)
  362. strict("%s constant includes more than one character", nm);
  363. return val;
  364. }
  365. char *string_token(char *nm, int stop_char)
  366. {
  367. register int ch;
  368. register int str_size;
  369. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  370. register int pos = 0;
  371. ch = GetChar();
  372. while (ch != stop_char) {
  373. if (ch == '\n') {
  374. error("newline in %s", nm);
  375. LineNumber++;
  376. break;
  377. }
  378. if (ch == EOI) {
  379. error("end-of-file inside %s", nm);
  380. break;
  381. }
  382. if (ch == '\\' && !AccFileSpecifier)
  383. ch = quoted(GetChar());
  384. str[pos++] = ch;
  385. if (pos == str_size)
  386. str = Realloc(str, (unsigned)(str_size <<= 1));
  387. ch = GetChar();
  388. }
  389. str[pos++] = '\0'; /* for filenames etc. */
  390. str = Realloc(str, (unsigned)pos);
  391. return str;
  392. }
  393. int quoted(int ch)
  394. {
  395. /* quoted() replaces an escaped character sequence by the
  396. character meant.
  397. */
  398. /* first char after backslash already in ch */
  399. if (!is_oct(ch)) { /* a quoted char */
  400. switch (ch) {
  401. case 'n':
  402. ch = '\n';
  403. break;
  404. case 't':
  405. ch = '\t';
  406. break;
  407. case 'b':
  408. ch = '\b';
  409. break;
  410. case 'r':
  411. ch = '\r';
  412. break;
  413. case 'f':
  414. ch = '\f';
  415. break;
  416. case 'a': /* alert */
  417. ch = '\007';
  418. break;
  419. case 'v': /* vertical tab */
  420. ch = '\013';
  421. break;
  422. case 'x': /* quoted hex */
  423. {
  424. register int hex = 0;
  425. register int vch;
  426. for (;;) {
  427. ch = GetChar();
  428. if (vch = val_in_base(ch, 16), vch == -1)
  429. break;
  430. hex = hex * 16 + vch;
  431. }
  432. UnGetChar();
  433. ch = hex;
  434. }
  435. }
  436. } else { /* a quoted octal */
  437. register int oct = 0, cnt = 0;
  438. do {
  439. oct = oct*8 + (ch-'0');
  440. ch = GetChar();
  441. } while (is_oct(ch) && ++cnt < 3);
  442. UnGetChar();
  443. ch = oct;
  444. }
  445. return ch&0377;
  446. }
  447. int val_in_base(int ch, int base)
  448. {
  449. switch (base) {
  450. case 8:
  451. return (is_dig(ch) && ch < '9') ? ch - '0' : -1;
  452. case 10:
  453. return is_dig(ch) ? ch - '0' : -1;
  454. case 16:
  455. return is_dig(ch) ? ch - '0'
  456. : is_hex(ch) ? (ch - 'a' + 10) & 017
  457. : -1;
  458. default:
  459. fatal("(val_in_base) illegal base value %d", base);
  460. /* NOTREACHED */
  461. }
  462. return -1;
  463. }
  464. int GetChar()
  465. {
  466. /* The routines GetChar and trigraph parses the trigraph
  467. sequences and removes occurences of \\\n.
  468. */
  469. register int ch;
  470. again:
  471. LoadChar(ch);
  472. /* possible trigraph sequence */
  473. if (ch == '?')
  474. ch = trigraph();
  475. /* \\\n are removed from the input stream */
  476. if (ch == '\\') {
  477. LoadChar(ch);
  478. if (ch == '\n') {
  479. ++LineNumber;
  480. goto again;
  481. }
  482. PushBack();
  483. ch = '\\';
  484. }
  485. return(LexSave = ch);
  486. }
  487. int trigraph()
  488. {
  489. register int ch;
  490. LoadChar(ch);
  491. if (ch == '?') {
  492. LoadChar(ch);
  493. switch (ch) { /* its a trigraph */
  494. case '=':
  495. ch = '#';
  496. return(ch);
  497. case '(':
  498. ch = '[';
  499. return(ch);
  500. case '/':
  501. ch = '\\';
  502. return(ch);
  503. case ')':
  504. ch = ']';
  505. return(ch);
  506. case '\'':
  507. ch = '^';
  508. return(ch);
  509. case '<':
  510. ch = '{';
  511. return(ch);
  512. case '!':
  513. ch = '|';
  514. return(ch);
  515. case '>':
  516. ch = '}';
  517. return(ch);
  518. case '-':
  519. ch = '~';
  520. return(ch);
  521. }
  522. PushBack();
  523. }
  524. PushBack();
  525. return('?');
  526. }