LLlex.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549
  1. /*
  2. * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
  3. * See the copyright notice in the ACK home directory, in the file "Copyright".
  4. */
  5. /* $Header$ */
  6. /* L E X I C A L A N A L Y Z E R */
  7. #include "idfsize.h"
  8. #include "numsize.h"
  9. #include "strsize.h"
  10. #include <alloc.h>
  11. #include "input.h"
  12. #include "arith.h"
  13. #include "macro.h"
  14. #include "idf.h"
  15. #include "LLlex.h"
  16. #include "Lpars.h"
  17. #include "class.h"
  18. #include "bits.h"
  19. #define BUFSIZ 1024
  20. struct token dot;
  21. int ReplaceMacros = 1; /* replacing macros */
  22. int AccDefined = 0; /* accept "defined(...)" */
  23. int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
  24. int Unstacked = 0; /* an unstack is done */
  25. int AccFileSpecifier = 0; /* return filespecifier <...> */
  26. int LexSave = 0; /* last character read by GetChar */
  27. extern int InputLevel; /* # of current macro expansions */
  28. char *string_token();
  29. arith char_constant();
  30. #define FLG_ESEEN 0x01 /* possibly a floating point number */
  31. #define FLG_DOTSEEN 0x02 /* certainly a floating point number */
  32. int
  33. LLlex()
  34. {
  35. return (DOT != EOF) ? GetToken(&dot) : EOF;
  36. }
  37. int
  38. GetToken(ptok)
  39. register struct token *ptok;
  40. {
  41. /* GetToken() is the actual token recognizer. It calls the
  42. control line interpreter if it encounters a "\n{w}*#"
  43. combination. Macro replacement is also performed if it is
  44. needed.
  45. */
  46. char buf[BUFSIZ];
  47. register int ch, nch;
  48. again: /* rescan the input after an error or replacement */
  49. ch = GetChar();
  50. go_on: /* rescan, the following character has been read */
  51. if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
  52. fatal("non-ascii '\\%03o' read", ch & 0377);
  53. /* keep track of the place of the token in the file */
  54. switch (class(ch)) { /* detect character class */
  55. case STNL: /* newline, vertical space or formfeed */
  56. LineNumber++;
  57. return ptok->tk_symb = EOF;
  58. case STSKIP: /* just skip the skip characters */
  59. goto again;
  60. case STGARB: /* garbage character */
  61. garbage:
  62. if (040 < ch && ch < 0177)
  63. error("garbage char %c", ch);
  64. else
  65. error("garbage char \\%03o", ch);
  66. goto again;
  67. case STSIMP: /* a simple character, no part of compound token*/
  68. return ptok->tk_symb = ch;
  69. case STCOMP: /* maybe the start of a compound token */
  70. nch = GetChar(); /* character lookahead */
  71. switch (ch) {
  72. case '!':
  73. if (nch == '=')
  74. return ptok->tk_symb = NOTEQUAL;
  75. UnGetChar();
  76. return ptok->tk_symb = ch;
  77. case '&':
  78. if (nch == '&')
  79. return ptok->tk_symb = AND;
  80. else if (nch == '=')
  81. return ptok->tk_symb = ANDAB;
  82. UnGetChar();
  83. return ptok->tk_symb = ch;
  84. case '+':
  85. if (nch == '+')
  86. return ptok->tk_symb = PLUSPLUS;
  87. else if (nch == '=')
  88. return ptok->tk_symb = PLUSAB;
  89. UnGetChar();
  90. return ptok->tk_symb = ch;
  91. case '-':
  92. if (nch == '-')
  93. return ptok->tk_symb = MINMIN;
  94. else if (nch == '>')
  95. return ptok->tk_symb = ARROW;
  96. else if (nch == '=')
  97. return ptok->tk_symb = MINAB;
  98. UnGetChar();
  99. return ptok->tk_symb = ch;
  100. case '<':
  101. if (AccFileSpecifier) {
  102. UnGetChar(); /* pushback nch */
  103. ptok->tk_str =
  104. string_token("file specifier", '>');
  105. return ptok->tk_symb = FILESPECIFIER;
  106. } else if (nch == '<') {
  107. if ((nch = GetChar()) == '=')
  108. return ptok->tk_symb = LEFTAB;
  109. UnGetChar();
  110. return ptok->tk_symb = LEFT;
  111. } else if (nch == '=')
  112. return ptok->tk_symb = LESSEQ;
  113. UnGetChar();
  114. return ptok->tk_symb = ch;
  115. case '=':
  116. if (nch == '=')
  117. return ptok->tk_symb = EQUAL;
  118. UnGetChar();
  119. return ptok->tk_symb = ch;
  120. case '>':
  121. if (nch == '=')
  122. return ptok->tk_symb = GREATEREQ;
  123. else if (nch == '>') {
  124. if ((nch = GetChar()) == '=')
  125. return ptok->tk_symb = RIGHTAB;
  126. UnGetChar();
  127. return ptok->tk_symb = RIGHT;
  128. }
  129. UnGetChar();
  130. return ptok->tk_symb = ch;
  131. case '|':
  132. if (nch == '|')
  133. return ptok->tk_symb = OR;
  134. else if (nch == '=')
  135. return ptok->tk_symb = ORAB;
  136. UnGetChar();
  137. return ptok->tk_symb = ch;
  138. case '%':
  139. if (nch == '=')
  140. return ptok->tk_symb = MODAB;
  141. UnGetChar();
  142. return ptok->tk_symb = ch;
  143. case '*':
  144. if (nch == '=')
  145. return ptok->tk_symb = TIMESAB;
  146. UnGetChar();
  147. return ptok->tk_symb = ch;
  148. case '^':
  149. if (nch == '=')
  150. return ptok->tk_symb = XORAB;
  151. UnGetChar();
  152. return ptok->tk_symb = ch;
  153. case '/':
  154. if (nch == '*' && !InputLevel) {
  155. skipcomment();
  156. goto again;
  157. }
  158. else if (nch == '=')
  159. return ptok->tk_symb = DIVAB;
  160. UnGetChar();
  161. return ptok->tk_symb = ch;
  162. default:
  163. crash("bad class for char 0%o", ch);
  164. /* NOTREACHED */
  165. }
  166. case STCHAR: /* character constant */
  167. ptok->tk_val = char_constant("character");
  168. return ptok->tk_symb = INTEGER;
  169. case STSTR: /* string */
  170. ptok->tk_str = string_token("string", '"');
  171. return ptok->tk_symb = STRING;
  172. case STELL: /* wide character constant/string prefix */
  173. nch = GetChar();
  174. if (nch == '"') {
  175. ptok->tk_str =
  176. string_token("wide character string", '"');
  177. return ptok->tk_symb = STRING;
  178. } else if (nch == '\'') {
  179. ptok->tk_val = char_constant("wide character");
  180. return ptok->tk_symb = INTEGER;
  181. }
  182. UnGetChar();
  183. /* fallthrough */
  184. case STIDF:
  185. {
  186. extern int idfsize; /* ??? */
  187. register char *tg = &buf[0];
  188. register char *maxpos = &buf[idfsize];
  189. int NoExpandNext = 0;
  190. #define tstmac(bx) if (!(bits[ch] & bx)) goto nomac
  191. #define cpy *tg++ = ch
  192. #define load (ch = GetChar()); if (!in_idf(ch)) goto endidf
  193. if (Unstacked) EnableMacros(); /* unstack macro's when allowed. */
  194. if (ch == NOEXPM) {
  195. NoExpandNext = 1;
  196. ch = GetChar();
  197. }
  198. #ifdef DOBITS
  199. cpy; tstmac(bit0); load;
  200. cpy; tstmac(bit1); load;
  201. cpy; tstmac(bit2); load;
  202. cpy; tstmac(bit3); load;
  203. cpy; tstmac(bit4); load;
  204. cpy; tstmac(bit5); load;
  205. cpy; tstmac(bit6); load;
  206. cpy; tstmac(bit7); load;
  207. #endif
  208. for(;;) {
  209. if (tg < maxpos) {
  210. cpy;
  211. }
  212. load;
  213. }
  214. endidf:
  215. /*if (ch != EOI) UnGetChar();*/
  216. UnGetChar();
  217. *tg++ = '\0'; /* mark the end of the identifier */
  218. if (ReplaceMacros) {
  219. register struct idf *idef = findidf(buf);
  220. if (idef && idef->id_macro && !NoExpandNext) {
  221. if (replace(idef))
  222. goto again;
  223. }
  224. }
  225. nomac: /* buf can already be null-terminated. soit */
  226. ch = GetChar();
  227. while (in_idf(ch)) {
  228. if (tg < maxpos) *tg++ = ch;
  229. ch = GetChar();
  230. }
  231. UnGetChar();
  232. *tg++ = '\0'; /* mark the end of the identifier */
  233. NoExpandNext = 0;
  234. if (UnknownIdIsZero) {
  235. ptok->tk_val = (arith)0;
  236. return ptok->tk_symb = INTEGER;
  237. }
  238. ptok->tk_str = Malloc(tg - buf);
  239. strcpy(ptok->tk_str, buf);
  240. return IDENTIFIER;
  241. }
  242. case STNUM: /* a numeric constant */
  243. { /* it may only be an integer constant */
  244. register int base = 10, val = 0, vch;
  245. /* Since the preprocessor only knows integers and has
  246. * nothing to do with ellipsis we just return when the
  247. * pp-number starts with a '.'
  248. */
  249. if (ch == '.') {
  250. return ptok->tk_symb = ch;
  251. }
  252. if (ch == '0') {
  253. ch = GetChar();
  254. if (ch == 'x' || ch == 'X') {
  255. base = 16;
  256. ch = GetChar();
  257. } else {
  258. base = 8;
  259. }
  260. }
  261. while ((vch = val_in_base(ch, base)) >= 0) {
  262. val = val * base + vch; /* overflow? nah */
  263. ch = GetChar();
  264. }
  265. while (ch == 'l' || ch == 'L' || ch == 'u' || ch == 'U')
  266. ch = GetChar();
  267. UnGetChar();
  268. ptok->tk_val = val;
  269. return ptok->tk_symb = INTEGER;
  270. }
  271. case STEOI: /* end of text on source file */
  272. return ptok->tk_symb = EOF;
  273. case STMSPEC:
  274. if (!InputLevel) goto garbage;
  275. if (ch == TOKSEP) goto again;
  276. /* fallthrough shouldn't happen */
  277. default: /* this cannot happen */
  278. crash("bad class for char 0%o", ch);
  279. }
  280. /*NOTREACHED*/
  281. }
  282. skipcomment()
  283. {
  284. /* The last character read has been the '*' of '/_*'. The
  285. characters, except NL and EOI, between '/_*' and the first
  286. occurring '*_/' are not interpreted.
  287. NL only affects the LineNumber. EOI is not legal.
  288. Important note: it is not possible to stop skipping comment
  289. beyond the end-of-file of an included file.
  290. EOI is returned by LoadChar only on encountering EOF of the
  291. top-level file...
  292. */
  293. register int c;
  294. NoUnstack++;
  295. c = GetChar();
  296. do {
  297. while (c != '*') {
  298. if (class(c) == STNL) {
  299. ++LineNumber;
  300. } else if (c == EOI) {
  301. NoUnstack--;
  302. return;
  303. }
  304. c = GetChar();
  305. } /* last Character seen was '*' */
  306. c = GetChar();
  307. } while (c != '/');
  308. NoUnstack--;
  309. }
  310. arith
  311. char_constant(nm)
  312. char *nm;
  313. {
  314. register arith val = 0;
  315. register int ch;
  316. int size = 0;
  317. ch = GetChar();
  318. if (ch == '\'')
  319. error("%s constant too short", nm);
  320. else
  321. while (ch != '\'') {
  322. if (ch == '\n') {
  323. error("newline in %s constant", nm);
  324. LineNumber++;
  325. break;
  326. }
  327. if (ch == '\\')
  328. ch = quoted(GetChar());
  329. if (ch >= 128) ch -= 256;
  330. if (size < (int)size)
  331. val |= ch << 8 * size;
  332. size++;
  333. ch = GetChar();
  334. }
  335. if (size > 1)
  336. strict("%s constant includes more than one character", nm);
  337. if (size > sizeof(arith))
  338. error("%s constant too long", nm);
  339. return val;
  340. }
  341. char *
  342. string_token(nm, stop_char)
  343. char *nm;
  344. {
  345. register int ch;
  346. register int str_size;
  347. register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
  348. register int pos = 0;
  349. ch = GetChar();
  350. while (ch != stop_char) {
  351. if (ch == '\n') {
  352. error("newline in %s", nm);
  353. LineNumber++;
  354. break;
  355. }
  356. if (ch == EOI) {
  357. error("end-of-file inside %s", nm);
  358. break;
  359. }
  360. if (ch == '\\' && !AccFileSpecifier)
  361. ch = quoted(GetChar());
  362. str[pos++] = ch;
  363. if (pos == str_size)
  364. str = Realloc(str, str_size <<= 1);
  365. ch = GetChar();
  366. }
  367. str[pos++] = '\0'; /* for filenames etc. */
  368. str = Realloc(str, pos);
  369. return str;
  370. }
  371. int
  372. quoted(ch)
  373. register int ch;
  374. {
  375. /* quoted() replaces an escaped character sequence by the
  376. character meant.
  377. */
  378. /* first char after backslash already in ch */
  379. if (!is_oct(ch)) { /* a quoted char */
  380. switch (ch) {
  381. case 'n':
  382. ch = '\n';
  383. break;
  384. case 't':
  385. ch = '\t';
  386. break;
  387. case 'b':
  388. ch = '\b';
  389. break;
  390. case 'r':
  391. ch = '\r';
  392. break;
  393. case 'f':
  394. ch = '\f';
  395. break;
  396. case 'a': /* alert */
  397. ch = '\007';
  398. break;
  399. case 'v': /* vertical tab */
  400. ch = '\013';
  401. break;
  402. case 'x': /* quoted hex */
  403. {
  404. register int hex = 0;
  405. register int vch;
  406. for (;;) {
  407. ch = GetChar();
  408. if (vch = val_in_base(ch, 16), vch == -1)
  409. break;
  410. hex = hex * 16 + vch;
  411. }
  412. UnGetChar();
  413. ch = hex;
  414. }
  415. }
  416. }
  417. else { /* a quoted octal */
  418. register int oct = 0, cnt = 0;
  419. do {
  420. oct = oct*8 + (ch-'0');
  421. ch = GetChar();
  422. } while (is_oct(ch) && ++cnt < 3);
  423. UnGetChar();
  424. ch = oct;
  425. }
  426. return ch&0377;
  427. }
  428. int
  429. val_in_base(ch, base)
  430. register int ch;
  431. {
  432. switch (base) {
  433. case 8:
  434. return (is_dig(ch) && ch < '9') ? ch - '0' : -1;
  435. case 10:
  436. return is_dig(ch) ? ch - '0' : -1;
  437. case 16:
  438. return is_dig(ch) ? ch - '0'
  439. : is_hex(ch) ? (ch - 'a' + 10) & 017
  440. : -1;
  441. default:
  442. fatal("(val_in_base) illegal base value %d", base);
  443. /* NOTREACHED */
  444. }
  445. }
  446. int
  447. GetChar()
  448. {
  449. /* The routines GetChar and trigraph parses the trigraph
  450. sequences and removes occurences of \\\n.
  451. */
  452. register int ch;
  453. again:
  454. LoadChar(ch);
  455. /* possible trigraph sequence */
  456. if (ch == '?')
  457. ch = trigraph();
  458. /* \\\n are removed from the input stream */
  459. if (ch == '\\') {
  460. LoadChar(ch);
  461. if (ch == '\n') {
  462. ++LineNumber;
  463. goto again;
  464. }
  465. PushBack();
  466. ch = '\\';
  467. }
  468. return(LexSave = ch);
  469. }
  470. int
  471. trigraph()
  472. {
  473. register int ch;
  474. LoadChar(ch);
  475. if (ch == '?') {
  476. LoadChar(ch);
  477. switch (ch) { /* its a trigraph */
  478. case '=':
  479. ch = '#';
  480. return(ch);
  481. case '(':
  482. ch = '[';
  483. return(ch);
  484. case '/':
  485. ch = '\\';
  486. return(ch);
  487. case ')':
  488. ch = ']';
  489. return(ch);
  490. case '\'':
  491. ch = '^';
  492. return(ch);
  493. case '<':
  494. ch = '{';
  495. return(ch);
  496. case '!':
  497. ch = '|';
  498. return(ch);
  499. case '>':
  500. ch = '}';
  501. return(ch);
  502. case '-':
  503. ch = '~';
  504. return(ch);
  505. }
  506. PushBack();
  507. }
  508. PushBack();
  509. return('?');
  510. }