123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570 |
- /*
- * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
- * See the copyright notice in the ACK home directory, in the file "Copyright".
- */
- /* $Id$ */
- /* L E X I C A L A N A L Y Z E R */
- #include <string.h>
- #include "idfsize.h"
- #include "numsize.h"
- #include "strsize.h"
- #include <alloc.h>
- #include "input.h"
- #include "arith.h"
- #include "macro.h"
- #include "idf.h"
- #include "LLlex.h"
- #include "Lpars.h"
- #include "class.h"
- #include "bits.h"
- #define BUFSIZ 1024
- struct token dot;
- char *string_token(char *nm, int stop_char);
- arith char_constant(char *nm);
- int val_in_base(int ch, int base);
- int quoted(int ch);
- int trigraph();
- int ReplaceMacros = 1; /* replacing macros */
- int AccDefined = 0; /* accept "defined(...)" */
- int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
- int Unstacked = 0; /* an unstack is done */
- int AccFileSpecifier = 0; /* return filespecifier <...> */
- int LexSave = 0; /* last character read by GetChar */
- extern int InputLevel; /* # of current macro expansions */
- #define FLG_ESEEN 0x01 /* possibly a floating point number */
- #define FLG_DOTSEEN 0x02 /* certainly a floating point number */
- int LLlex()
- {
- return (DOT != EOF) ? GetToken(&dot) : EOF;
- }
- int GetToken(struct token *ptok)
- {
- /* GetToken() is the actual token recognizer. It calls the
- control line interpreter if it encounters a "\n{w}*#"
- combination. Macro replacement is also performed if it is
- needed.
- */
- char buf[BUFSIZ];
- register int ch, nch;
- again: /* rescan the input after an error or replacement */
- ch = GetChar();
- /* rescan, the following character has been read */
- if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
- fatal("non-ascii '\\%03o' read", ch & 0377);
- /* keep track of the place of the token in the file */
- switch (class(ch)) { /* detect character class */
- case STNL: /* newline, vertical space or formfeed */
- LineNumber++;
- return ptok->tk_symb = EOF;
- case STSKIP: /* just skip the skip characters */
- goto again;
- case STGARB: /* garbage character */
- garbage:
- if (040 < ch && ch < 0177)
- error("garbage char %c", ch);
- else
- error("garbage char \\%03o", ch);
- goto again;
- case STSIMP: /* a simple character, no part of compound token*/
- return ptok->tk_symb = ch;
- case STCOMP: /* maybe the start of a compound token */
- nch = GetChar(); /* character lookahead */
- switch (ch) {
- case '!':
- if (nch == '=')
- return ptok->tk_symb = NOTEQUAL;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '&':
- if (nch == '&')
- return ptok->tk_symb = AND;
- else if (nch == '=')
- return ptok->tk_symb = ANDAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '+':
- if (nch == '+')
- return ptok->tk_symb = PLUSPLUS;
- else if (nch == '=')
- return ptok->tk_symb = PLUSAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '-':
- if (nch == '-')
- return ptok->tk_symb = MINMIN;
- else if (nch == '>')
- return ptok->tk_symb = ARROW;
- else if (nch == '=')
- return ptok->tk_symb = MINAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '<':
- if (AccFileSpecifier) {
- UnGetChar(); /* pushback nch */
- ptok->tk_str =
- string_token("file specifier", '>');
- return ptok->tk_symb = FILESPECIFIER;
- } else if (nch == '<') {
- if ((nch = GetChar()) == '=')
- return ptok->tk_symb = LEFTAB;
- UnGetChar();
- return ptok->tk_symb = LEFT;
- } else if (nch == '=')
- return ptok->tk_symb = LESSEQ;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '=':
- if (nch == '=')
- return ptok->tk_symb = EQUAL;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '>':
- if (nch == '=')
- return ptok->tk_symb = GREATEREQ;
- else if (nch == '>') {
- if ((nch = GetChar()) == '=')
- return ptok->tk_symb = RIGHTAB;
- UnGetChar();
- return ptok->tk_symb = RIGHT;
- }
- UnGetChar();
- return ptok->tk_symb = ch;
- case '|':
- if (nch == '|')
- return ptok->tk_symb = OR;
- else if (nch == '=')
- return ptok->tk_symb = ORAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '%':
- if (nch == '=')
- return ptok->tk_symb = MODAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '*':
- if (nch == '=')
- return ptok->tk_symb = TIMESAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '^':
- if (nch == '=')
- return ptok->tk_symb = XORAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- case '/':
- if (nch == '*' && !InputLevel) {
- skipcomment();
- goto again;
- }
- else if (nch == '=')
- return ptok->tk_symb = DIVAB;
- UnGetChar();
- return ptok->tk_symb = ch;
- default:
- crash("bad class for char 0%o", ch);
- /* NOTREACHED */
- }
- case STCHAR: /* character constant */
- ptok->tk_val = char_constant("character");
- return ptok->tk_symb = INTEGER;
- case STSTR: /* string */
- ptok->tk_str = string_token("string", '"');
- return ptok->tk_symb = STRING;
- case STELL: /* wide character constant/string prefix */
- nch = GetChar();
- if (nch == '"') {
- ptok->tk_str =
- string_token("wide character string", '"');
- return ptok->tk_symb = STRING;
- } else if (nch == '\'') {
- ptok->tk_val = char_constant("wide character");
- return ptok->tk_symb = INTEGER;
- }
- UnGetChar();
- /* fallthrough */
- case STIDF:
- {
- extern int idfsize; /* ??? */
- register char *tg = &buf[0];
- register char *maxpos = &buf[idfsize];
- int NoExpandNext = 0;
- #define tstmac(bx) if (!(bits[ch] & bx)) goto nomac
- #define cpy *tg++ = ch
- #define load (ch = GetChar()); if (!in_idf(ch)) goto endidf
- if (Unstacked) EnableMacros(); /* unstack macro's when allowed. */
- if (ch == NOEXPM) {
- NoExpandNext = 1;
- ch = GetChar();
- }
- #ifdef DOBITS
- cpy; tstmac(bit0); load;
- cpy; tstmac(bit1); load;
- cpy; tstmac(bit2); load;
- cpy; tstmac(bit3); load;
- cpy; tstmac(bit4); load;
- cpy; tstmac(bit5); load;
- cpy; tstmac(bit6); load;
- cpy; tstmac(bit7); load;
- #endif
- for(;;) {
- if (tg < maxpos) {
- cpy;
- }
- load;
- }
- endidf:
- /*if (ch != EOI) UnGetChar();*/
- UnGetChar();
- *tg++ = '\0'; /* mark the end of the identifier */
- if (ReplaceMacros) {
- register struct idf *idef = findidf(buf);
- if (idef && idef->id_macro && !NoExpandNext) {
- if (replace(idef))
- goto again;
- }
- }
- nomac: /* buf can already be null-terminated. soit */
- ch = GetChar();
- while (in_idf(ch)) {
- if (tg < maxpos) *tg++ = ch;
- ch = GetChar();
- }
- UnGetChar();
- *tg++ = '\0'; /* mark the end of the identifier */
- NoExpandNext = 0;
- if (UnknownIdIsZero) {
- ptok->tk_val = (arith)0;
- return ptok->tk_symb = INTEGER;
- }
- ptok->tk_str = Malloc((unsigned)(tg - buf));
- strcpy(ptok->tk_str, buf);
- return IDENTIFIER;
- }
- case STNUM: /* a numeric constant */
- { /* it may only be an integer constant */
- int base = 10, vch;
- arith val = 0;
- int ovfl = 0;
- arith ubound = ~(1<<(sizeof(arith)*8-1))/(base/2);
- /* Since the preprocessor only knows integers and has
- * nothing to do with ellipsis we just return when the
- * pp-number starts with a '.'
- */
- if (ch == '.') {
- return ptok->tk_symb = ch;
- }
- if (ch == '0') {
- ch = GetChar();
- if (ch == 'x' || ch == 'X') {
- base = 16;
- ch = GetChar();
- } else {
- base = 8;
- }
- }
- while ((vch = val_in_base(ch, base)) >= 0) {
- if (val < 0 || val > ubound) ovfl++;
- val *= base;
- if (val < 0 && val + vch >= 0) ovfl++;
- val += vch;
- ch = GetChar();
- }
- ptok->tk_unsigned = 0;
- if (ch == 'u' || ch == 'U') {
- ptok->tk_unsigned = 1;
- ch = GetChar();
- if (ch == 'l' || ch == 'L') {
- ch = GetChar();
- }
- }
- else if (ch == 'l' || ch == 'L') {
- ch = GetChar();
- if (ch == 'u' || ch == 'U') {
- ptok->tk_unsigned = 1;
- ch = GetChar();
- }
- }
- if (ovfl) {
- warning("overflow in constant");
- ptok->tk_unsigned = 1;
- }
- else if (val < 0) {
- /* give warning??? */
- ptok->tk_unsigned = 1;
- }
- UnGetChar();
- ptok->tk_val = val;
- return ptok->tk_symb = INTEGER;
- }
- case STEOI: /* end of text on source file */
- return ptok->tk_symb = EOF;
- case STMSPEC:
- if (!InputLevel) goto garbage;
- if (ch == TOKSEP) goto again;
- /* fallthrough shouldn't happen */
- default: /* this cannot happen */
- crash("bad class for char 0%o", ch);
- }
- /*NOTREACHED*/
- return -1;
- }
- void skipcomment()
- {
- /* The last character read has been the '*' of '/_*'. The
- characters, except NL and EOI, between '/_*' and the first
- occurring '*_/' are not interpreted.
- NL only affects the LineNumber. EOI is not legal.
- Important note: it is not possible to stop skipping comment
- beyond the end-of-file of an included file.
- EOI is returned by LoadChar only on encountering EOF of the
- top-level file...
- */
- register int c;
- NoUnstack++;
- c = GetChar();
- do {
- while (c != '*') {
- if (class(c) == STNL) {
- ++LineNumber;
- } else if (c == EOI) {
- NoUnstack--;
- return;
- }
- c = GetChar();
- } /* last Character seen was '*' */
- c = GetChar();
- } while (c != '/');
- NoUnstack--;
- }
- arith char_constant(char *nm)
- {
- register arith val = 0;
- register int ch;
- int size = 0;
- ch = GetChar();
- if (ch == '\'')
- error("%s constant too short", nm);
- else
- while (ch != '\'') {
- if (ch == '\n') {
- error("newline in %s constant", nm);
- LineNumber++;
- break;
- }
- if (ch == '\\')
- ch = quoted(GetChar());
- if (ch >= 128) ch -= 256;
- if (size < sizeof(arith))
- val |= ch << (8 * size);
- size++;
- ch = GetChar();
- }
- if (size > sizeof(arith))
- error("%s constant too long", nm);
- else if (size > 1)
- strict("%s constant includes more than one character", nm);
- return val;
- }
- char *string_token(char *nm, int stop_char)
- {
- register int ch;
- register int str_size;
- register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
- register int pos = 0;
-
- ch = GetChar();
- while (ch != stop_char) {
- if (ch == '\n') {
- error("newline in %s", nm);
- LineNumber++;
- break;
- }
- if (ch == EOI) {
- error("end-of-file inside %s", nm);
- break;
- }
- if (ch == '\\' && !AccFileSpecifier)
- ch = quoted(GetChar());
- str[pos++] = ch;
- if (pos == str_size)
- str = Realloc(str, (unsigned)(str_size <<= 1));
- ch = GetChar();
- }
- str[pos++] = '\0'; /* for filenames etc. */
- str = Realloc(str, (unsigned)pos);
- return str;
- }
- int quoted(int ch)
- {
- /* quoted() replaces an escaped character sequence by the
- character meant.
- */
- /* first char after backslash already in ch */
- if (!is_oct(ch)) { /* a quoted char */
- switch (ch) {
- case 'n':
- ch = '\n';
- break;
- case 't':
- ch = '\t';
- break;
- case 'b':
- ch = '\b';
- break;
- case 'r':
- ch = '\r';
- break;
- case 'f':
- ch = '\f';
- break;
- case 'a': /* alert */
- ch = '\007';
- break;
- case 'v': /* vertical tab */
- ch = '\013';
- break;
- case 'x': /* quoted hex */
- {
- register int hex = 0;
- register int vch;
- for (;;) {
- ch = GetChar();
- if (vch = val_in_base(ch, 16), vch == -1)
- break;
- hex = hex * 16 + vch;
- }
- UnGetChar();
- ch = hex;
- }
- }
- } else { /* a quoted octal */
- register int oct = 0, cnt = 0;
- do {
- oct = oct*8 + (ch-'0');
- ch = GetChar();
- } while (is_oct(ch) && ++cnt < 3);
- UnGetChar();
- ch = oct;
- }
- return ch&0377;
- }
- int val_in_base(int ch, int base)
- {
- switch (base) {
- case 8:
- return (is_dig(ch) && ch < '9') ? ch - '0' : -1;
- case 10:
- return is_dig(ch) ? ch - '0' : -1;
- case 16:
- return is_dig(ch) ? ch - '0'
- : is_hex(ch) ? (ch - 'a' + 10) & 017
- : -1;
- default:
- fatal("(val_in_base) illegal base value %d", base);
- /* NOTREACHED */
- }
- return -1;
- }
- int GetChar()
- {
- /* The routines GetChar and trigraph parses the trigraph
- sequences and removes occurences of \\\n.
- */
- register int ch;
- again:
- LoadChar(ch);
- /* possible trigraph sequence */
- if (ch == '?')
- ch = trigraph();
- /* \\\n are removed from the input stream */
- if (ch == '\\') {
- LoadChar(ch);
- if (ch == '\n') {
- ++LineNumber;
- goto again;
- }
- PushBack();
- ch = '\\';
- }
- return(LexSave = ch);
- }
- int trigraph()
- {
- register int ch;
- LoadChar(ch);
- if (ch == '?') {
- LoadChar(ch);
- switch (ch) { /* its a trigraph */
- case '=':
- ch = '#';
- return(ch);
- case '(':
- ch = '[';
- return(ch);
- case '/':
- ch = '\\';
- return(ch);
- case ')':
- ch = ']';
- return(ch);
- case '\'':
- ch = '^';
- return(ch);
- case '<':
- ch = '{';
- return(ch);
- case '!':
- ch = '|';
- return(ch);
- case '>':
- ch = '}';
- return(ch);
- case '-':
- ch = '~';
- return(ch);
- }
- PushBack();
- }
- PushBack();
- return('?');
- }
|