LLlex.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Id$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "idfsize.h"
  8. #include "numsize.h"
  9. #include "strsize.h"
  10. #include <alloc.h>
  11. #include "input.h"
  12. #include "arith.h"
  13. #include "macro.h"
  14. #include "idf.h"
  15. #include "LLlex.h"
  16. #include "Lpars.h"
  17. #include "class.h"
  18. #include "bits.h"
  19. #define BUFSIZ 1024
  20. struct token dot;
  21. int ReplaceMacros = 1; /* replacing macros */
  22. int AccDefined = 0; /* accept "defined(...)" */
  23. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  24. int Unstacked = 0; /* an unstack is done */
  25. int AccFileSpecifier = 0; /* return filespecifier <...> */
  26. int LexSave = 0; /* last character read by GetChar */
  27. extern int InputLevel; /* # of current macro expansions */
  28. extern char *string_token();
  29. extern char *strcpy();
  30. extern arith char_constant();
  31. #define FLG_ESEEN 0x01 /* possibly a floating point number */
  32. #define FLG_DOTSEEN 0x02 /* certainly a floating point number */
  33. int
  34. LLlex()
  35. {
  36. return (DOT != EOF) ? GetToken(&dot) : EOF;
  37. }
  38. int
  39. GetToken(ptok)
  40. register struct token *ptok;
  41. {
  42. /* GetToken() is the actual token recognizer. It calls the
  43. control line interpreter if it encounters a "\n{w}*#"
  44. combination. Macro replacement is also performed if it is
  45. needed.
  46. */
  47. char buf[BUFSIZ];
  48. register int ch, nch;
  49. again: /* rescan the input after an error or replacement */
  50. ch = GetChar();
  51. /* rescan, the following character has been read */
  52. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  53. fatal("non-ascii '\\%03o' read", ch & 0377);
  54. /* keep track of the place of the token in the file */
  55. switch (class(ch)) { /* detect character class */
  56. case STNL: /* newline, vertical space or formfeed */
  57. LineNumber++;
  58. return ptok->tk_symb = EOF;
  59. case STSKIP: /* just skip the skip characters */
  60. goto again;
  61. case STGARB: /* garbage character */
  62. garbage:
  63. if (040 < ch && ch < 0177)
  64. error("garbage char %c", ch);
  65. else
  66. error("garbage char \\%03o", ch);
  67. goto again;
  68. case STSIMP: /* a simple character, no part of compound token*/
  69. return ptok->tk_symb = ch;
  70. case STCOMP: /* maybe the start of a compound token */
  71. nch = GetChar(); /* character lookahead */
  72. switch (ch) {
  73. case '!':
  74. if (nch == '=')
  75. return ptok->tk_symb = NOTEQUAL;
  76. UnGetChar();
  77. return ptok->tk_symb = ch;
  78. case '&':
  79. if (nch == '&')
  80. return ptok->tk_symb = AND;
  81. else if (nch == '=')
  82. return ptok->tk_symb = ANDAB;
  83. UnGetChar();
  84. return ptok->tk_symb = ch;
  85. case '+':
  86. if (nch == '+')
  87. return ptok->tk_symb = PLUSPLUS;
  88. else if (nch == '=')
  89. return ptok->tk_symb = PLUSAB;
  90. UnGetChar();
  91. return ptok->tk_symb = ch;
  92. case '-':
  93. if (nch == '-')
  94. return ptok->tk_symb = MINMIN;
  95. else if (nch == '>')
  96. return ptok->tk_symb = ARROW;
  97. else if (nch == '=')
  98. return ptok->tk_symb = MINAB;
  99. UnGetChar();
  100. return ptok->tk_symb = ch;
  101. case '<':
  102. if (AccFileSpecifier) {
  103. UnGetChar(); /* pushback nch */
  104. ptok->tk_str =
  105. string_token("file specifier", '>');
  106. return ptok->tk_symb = FILESPECIFIER;
  107. } else if (nch == '<') {
  108. if ((nch = GetChar()) == '=')
  109. return ptok->tk_symb = LEFTAB;
  110. UnGetChar();
  111. return ptok->tk_symb = LEFT;
  112. } else if (nch == '=')
  113. return ptok->tk_symb = LESSEQ;
  114. UnGetChar();
  115. return ptok->tk_symb = ch;
  116. case '=':
  117. if (nch == '=')
  118. return ptok->tk_symb = EQUAL;
  119. UnGetChar();
  120. return ptok->tk_symb = ch;
  121. case '>':
  122. if (nch == '=')
  123. return ptok->tk_symb = GREATEREQ;
  124. else if (nch == '>') {
  125. if ((nch = GetChar()) == '=')
  126. return ptok->tk_symb = RIGHTAB;
  127. UnGetChar();
  128. return ptok->tk_symb = RIGHT;
  129. }
  130. UnGetChar();
  131. return ptok->tk_symb = ch;
  132. case '|':
  133. if (nch == '|')
  134. return ptok->tk_symb = OR;
  135. else if (nch == '=')
  136. return ptok->tk_symb = ORAB;
  137. UnGetChar();
  138. return ptok->tk_symb = ch;
  139. case '%':
  140. if (nch == '=')
  141. return ptok->tk_symb = MODAB;
  142. UnGetChar();
  143. return ptok->tk_symb = ch;
  144. case '*':
  145. if (nch == '=')
  146. return ptok->tk_symb = TIMESAB;
  147. UnGetChar();
  148. return ptok->tk_symb = ch;
  149. case '^':
  150. if (nch == '=')
  151. return ptok->tk_symb = XORAB;
  152. UnGetChar();
  153. return ptok->tk_symb = ch;
  154. case '/':
  155. if (nch == '*' && !InputLevel) {
  156. skipcomment();
  157. goto again;
  158. }
  159. else if (nch == '=')
  160. return ptok->tk_symb = DIVAB;
  161. UnGetChar();
  162. return ptok->tk_symb = ch;
  163. default:
  164. crash("bad class for char 0%o", ch);
  165. /* NOTREACHED */
  166. }
  167. case STCHAR: /* character constant */
  168. ptok->tk_val = char_constant("character");
  169. return ptok->tk_symb = INTEGER;
  170. case STSTR: /* string */
  171. ptok->tk_str = string_token("string", '"');
  172. return ptok->tk_symb = STRING;
  173. case STELL: /* wide character constant/string prefix */
  174. nch = GetChar();
  175. if (nch == '"') {
  176. ptok->tk_str =
  177. string_token("wide character string", '"');
  178. return ptok->tk_symb = STRING;
  179. } else if (nch == '\'') {
  180. ptok->tk_val = char_constant("wide character");
  181. return ptok->tk_symb = INTEGER;
  182. }
  183. UnGetChar();
  184. /* fallthrough */
  185. case STIDF:
  186. {
  187. extern int idfsize; /* ??? */
  188. register char *tg = &buf[0];
  189. register char *maxpos = &buf[idfsize];
  190. int NoExpandNext = 0;
  191. #define tstmac(bx) if (!(bits[ch] & bx)) goto nomac
  192. #define cpy *tg++ = ch
  193. #define load (ch = GetChar()); if (!in_idf(ch)) goto endidf
  194. if (Unstacked) EnableMacros(); /* unstack macro's when allowed. */
  195. if (ch == NOEXPM) {
  196. NoExpandNext = 1;
  197. ch = GetChar();
  198. }
  199. #ifdef DOBITS
  200. cpy; tstmac(bit0); load;
  201. cpy; tstmac(bit1); load;
  202. cpy; tstmac(bit2); load;
  203. cpy; tstmac(bit3); load;
  204. cpy; tstmac(bit4); load;
  205. cpy; tstmac(bit5); load;
  206. cpy; tstmac(bit6); load;
  207. cpy; tstmac(bit7); load;
  208. #endif
  209. for(;;) {
  210. if (tg < maxpos) {
  211. cpy;
  212. }
  213. load;
  214. }
  215. endidf:
  216. /*if (ch != EOI) UnGetChar();*/
  217. UnGetChar();
  218. *tg++ = '\0'; /* mark the end of the identifier */
  219. if (ReplaceMacros) {
  220. register struct idf *idef = findidf(buf);
  221. if (idef && idef->id_macro && !NoExpandNext) {
  222. if (replace(idef))
  223. goto again;
  224. }
  225. }
  226. nomac: /* buf can already be null-terminated. soit */
  227. ch = GetChar();
  228. while (in_idf(ch)) {
  229. if (tg < maxpos) *tg++ = ch;
  230. ch = GetChar();
  231. }
  232. UnGetChar();
  233. *tg++ = '\0'; /* mark the end of the identifier */
  234. NoExpandNext = 0;
  235. if (UnknownIdIsZero) {
  236. ptok->tk_val = (arith)0;
  237. return ptok->tk_symb = INTEGER;
  238. }
  239. ptok->tk_str = Malloc((unsigned)(tg - buf));
  240. strcpy(ptok->tk_str, buf);
  241. return IDENTIFIER;
  242. }
  243. case STNUM: /* a numeric constant */
  244. { /* it may only be an integer constant */
  245. register int base = 10, vch;
  246. register arith val = 0;
  247. int ovfl = 0;
  248. arith ubound = ~(1<<(sizeof(arith)*8-1))/(base/2);
  249. /* Since the preprocessor only knows integers and has
  250. * nothing to do with ellipsis we just return when the
  251. * pp-number starts with a '.'
  252. */
  253. if (ch == '.') {
  254. return ptok->tk_symb = ch;
  255. }
  256. if (ch == '0') {
  257. ch = GetChar();
  258. if (ch == 'x' || ch == 'X') {
  259. base = 16;
  260. ch = GetChar();
  261. } else {
  262. base = 8;
  263. }
  264. }
  265. while ((vch = val_in_base(ch, base)) >= 0) {
  266. if (val < 0 || val > ubound) ovfl++;
  267. val *= base;
  268. if (val < 0 && val + vch >= 0) ovfl++;
  269. val += vch;
  270. ch = GetChar();
  271. }
  272. ptok->tk_unsigned = 0;
  273. if (ch == 'u' || ch == 'U') {
  274. ptok->tk_unsigned = 1;
  275. ch = GetChar();
  276. if (ch == 'l' || ch == 'L') {
  277. ch = GetChar();
  278. }
  279. }
  280. else if (ch == 'l' || ch == 'L') {
  281. ch = GetChar();
  282. if (ch == 'u' || ch == 'U') {
  283. ptok->tk_unsigned = 1;
  284. ch = GetChar();
  285. }
  286. }
  287. if (ovfl) {
  288. warning("overflow in constant");
  289. ptok->tk_unsigned = 1;
  290. }
  291. else if (val < 0) {
  292. /* give warning??? */
  293. ptok->tk_unsigned = 1;
  294. }
  295. UnGetChar();
  296. ptok->tk_val = val;
  297. return ptok->tk_symb = INTEGER;
  298. }
  299. case STEOI: /* end of text on source file */
  300. return ptok->tk_symb = EOF;
  301. case STMSPEC:
  302. if (!InputLevel) goto garbage;
  303. if (ch == TOKSEP) goto again;
  304. /* fallthrough shouldn't happen */
  305. default: /* this cannot happen */
  306. crash("bad class for char 0%o", ch);
  307. }
  308. /*NOTREACHED*/
  309. }
  310. skipcomment()
  311. {
  312. /* The last character read has been the '*' of '/_*'. The
  313. characters, except NL and EOI, between '/_*' and the first
  314. occurring '*_/' are not interpreted.
  315. NL only affects the LineNumber. EOI is not legal.
  316. Important note: it is not possible to stop skipping comment
  317. beyond the end-of-file of an included file.
  318. EOI is returned by LoadChar only on encountering EOF of the
  319. top-level file...
  320. */
  321. register int c;
  322. NoUnstack++;
  323. c = GetChar();
  324. do {
  325. while (c != '*') {
  326. if (class(c) == STNL) {
  327. ++LineNumber;
  328. } else if (c == EOI) {
  329. NoUnstack--;
  330. return;
  331. }
  332. c = GetChar();
  333. } /* last Character seen was '*' */
  334. c = GetChar();
  335. } while (c != '/');
  336. NoUnstack--;
  337. }
  338. arith
  339. char_constant(nm)
  340. char *nm;
  341. {
  342. register arith val = 0;
  343. register int ch;
  344. int size = 0;
  345. ch = GetChar();
  346. if (ch == '\'')
  347. error("%s constant too short", nm);
  348. else
  349. while (ch != '\'') {
  350. if (ch == '\n') {
  351. error("newline in %s constant", nm);
  352. LineNumber++;
  353. break;
  354. }
  355. if (ch == '\\')
  356. ch = quoted(GetChar());
  357. if (ch >= 128) ch -= 256;
  358. if (size < sizeof(arith))
  359. val |= ch << (8 * size);
  360. size++;
  361. ch = GetChar();
  362. }
  363. if (size > sizeof(arith))
  364. error("%s constant too long", nm);
  365. else if (size > 1)
  366. strict("%s constant includes more than one character", nm);
  367. return val;
  368. }
  369. char *
  370. string_token(nm, stop_char)
  371. char *nm;
  372. {
  373. register int ch;
  374. register int str_size;
  375. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  376. register int pos = 0;
  377. ch = GetChar();
  378. while (ch != stop_char) {
  379. if (ch == '\n') {
  380. error("newline in %s", nm);
  381. LineNumber++;
  382. break;
  383. }
  384. if (ch == EOI) {
  385. error("end-of-file inside %s", nm);
  386. break;
  387. }
  388. if (ch == '\\' && !AccFileSpecifier)
  389. ch = quoted(GetChar());
  390. str[pos++] = ch;
  391. if (pos == str_size)
  392. str = Realloc(str, (unsigned)(str_size <<= 1));
  393. ch = GetChar();
  394. }
  395. str[pos++] = '\0'; /* for filenames etc. */
  396. str = Realloc(str, (unsigned)pos);
  397. return str;
  398. }
  399. int
  400. quoted(ch)
  401. register int ch;
  402. {
  403. /* quoted() replaces an escaped character sequence by the
  404. character meant.
  405. */
  406. /* first char after backslash already in ch */
  407. if (!is_oct(ch)) { /* a quoted char */
  408. switch (ch) {
  409. case 'n':
  410. ch = '\n';
  411. break;
  412. case 't':
  413. ch = '\t';
  414. break;
  415. case 'b':
  416. ch = '\b';
  417. break;
  418. case 'r':
  419. ch = '\r';
  420. break;
  421. case 'f':
  422. ch = '\f';
  423. break;
  424. case 'a': /* alert */
  425. ch = '\007';
  426. break;
  427. case 'v': /* vertical tab */
  428. ch = '\013';
  429. break;
  430. case 'x': /* quoted hex */
  431. {
  432. register int hex = 0;
  433. register int vch;
  434. for (;;) {
  435. ch = GetChar();
  436. if (vch = val_in_base(ch, 16), vch == -1)
  437. break;
  438. hex = hex * 16 + vch;
  439. }
  440. UnGetChar();
  441. ch = hex;
  442. }
  443. }
  444. } else { /* a quoted octal */
  445. register int oct = 0, cnt = 0;
  446. do {
  447. oct = oct*8 + (ch-'0');
  448. ch = GetChar();
  449. } while (is_oct(ch) && ++cnt < 3);
  450. UnGetChar();
  451. ch = oct;
  452. }
  453. return ch&0377;
  454. }
  455. int
  456. val_in_base(ch, base)
  457. register int ch;
  458. {
  459. switch (base) {
  460. case 8:
  461. return (is_dig(ch) && ch < '9') ? ch - '0' : -1;
  462. case 10:
  463. return is_dig(ch) ? ch - '0' : -1;
  464. case 16:
  465. return is_dig(ch) ? ch - '0'
  466. : is_hex(ch) ? (ch - 'a' + 10) & 017
  467. : -1;
  468. default:
  469. fatal("(val_in_base) illegal base value %d", base);
  470. /* NOTREACHED */
  471. }
  472. }
  473. int
  474. GetChar()
  475. {
  476. /* The routines GetChar and trigraph parses the trigraph
  477. sequences and removes occurences of \\\n.
  478. */
  479. register int ch;
  480. again:
  481. LoadChar(ch);
  482. /* possible trigraph sequence */
  483. if (ch == '?')
  484. ch = trigraph();
  485. /* \\\n are removed from the input stream */
  486. if (ch == '\\') {
  487. LoadChar(ch);
  488. if (ch == '\n') {
  489. ++LineNumber;
  490. goto again;
  491. }
  492. PushBack();
  493. ch = '\\';
  494. }
  495. return(LexSave = ch);
  496. }
  497. int
  498. trigraph()
  499. {
  500. register int ch;
  501. LoadChar(ch);
  502. if (ch == '?') {
  503. LoadChar(ch);
  504. switch (ch) { /* its a trigraph */
  505. case '=':
  506. ch = '#';
  507. return(ch);
  508. case '(':
  509. ch = '[';
  510. return(ch);
  511. case '/':
  512. ch = '\\';
  513. return(ch);
  514. case ')':
  515. ch = ']';
  516. return(ch);
  517. case '\'':
  518. ch = '^';
  519. return(ch);
  520. case '<':
  521. ch = '{';
  522. return(ch);
  523. case '!':
  524. ch = '|';
  525. return(ch);
  526. case '>':
  527. ch = '}';
  528. return(ch);
  529. case '-':
  530. ch = '~';
  531. return(ch);
  532. }
  533. PushBack();
  534. }
  535. PushBack();
  536. return('?');
  537. }