123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575 |
- /*
- * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
- * See the copyright notice in the ACK home directory, in the file "Copyright".
- */
- /* $Header$ */
- /* L E X I C A L A N A L Y Z E R */
- #include "lint.h"
- #include <alloc.h>
- #include "nofloat.h"
- #include "idfsize.h"
- #include "numsize.h"
- #include "debug.h"
- #include "strsize.h"
- #include "nopp.h"
- #include "input.h"
- #include "arith.h"
- #include "def.h"
- #include "idf.h"
- #include "LLlex.h"
- #include "Lpars.h"
- #include "class.h"
- #include "assert.h"
- #include "sizes.h"
- /* Data about the token yielded */
- struct token dot, ahead, aside;
- int token_nmb = 0; /* number of the ahead token */
- int tk_nmb_at_last_syn_err = -5/*ERR_SHADOW*/;
- /* token number at last syntax error */
- #ifndef NOPP
- int ReplaceMacros = 1; /* replacing macros */
- int AccDefined = 0; /* accept "defined(...)" */
- int UnknownIdIsZero = 0; /* interpret unknown id as integer 0 */
- int Unstacked = 0; /* an unstack is done */
- #endif
- int SkipEscNewline = 0; /* how to interpret backslash-newline */
- int AccFileSpecifier = 0; /* return filespecifier <...> */
- int EoiForNewline = 0; /* return EOI upon encountering newline */
- int File_Inserted = 0; /* a file has just been inserted */
- #define MAX_LL_DEPTH 2
- static struct token LexStack[MAX_LL_DEPTH];
- static LexSP = 0;
- /* In PushLex() the actions are taken in order to initialise or
- re-initialise the lexical scanner.
- E.g. at the invocation of a sub-parser that uses LLlex(), the
- state of the current parser should be saved.
- */
- PushLex()
- {
- ASSERT(LexSP < 2);
- ASSERT(ASIDE == 0); /* ASIDE = 0; */
- GetToken(&ahead);
- LexStack[LexSP++] = dot;
- }
- PopLex()
- {
- ASSERT(LexSP > 0);
- dot = LexStack[--LexSP];
- }
- int
- LLlex()
- {
- /* LLlex() plays the role of Lexical Analyzer for the C parser.
- The look-ahead and putting aside of tokens are taken into
- account.
- */
- if (ASIDE) { /* a token is put aside */
- dot = aside;
- ASIDE = 0;
- }
- else { /* read ahead and return the old one */
- #ifdef LINT
- lint_comment_ahead();
- #endif LINT
- dot = ahead;
- /* the following test is performed due to the dual
- task of LLlex(): it is also called for parsing the
- restricted constant expression following a #if or
- #elif. The newline character causes EOF to be
- returned in this case to stop the LLgen parsing task.
- */
- if (DOT != EOI)
- GetToken(&ahead);
- else
- DOT = EOF;
- }
- return DOT;
- }
- char *string_token();
- int
- GetToken(ptok)
- register struct token *ptok;
- {
- /* GetToken() is the actual token recognizer. It calls the
- control line interpreter if it encounters a "\n#"
- combination. Macro replacement is also performed if it is
- needed.
- */
- char buf[(IDFSIZE > NUMSIZE ? IDFSIZE : NUMSIZE) + 1];
- register int ch, nch;
- token_nmb++;
- if (File_Inserted) {
- File_Inserted = 0;
- goto firstline;
- }
- again: /* rescan the input after an error or replacement */
- #ifndef NOPP
- if (Unstacked) EnableMacros();
- #endif
- LoadChar(ch);
- go_on: /* rescan, the following character has been read */
- if ((ch & 0200) && ch != EOI) /* stop on non-ascii character */
- fatal("non-ascii '\\%03o' read", ch & 0377);
- /* keep track of the place of the token in the file */
- ptok->tk_file = FileName;
- ptok->tk_line = LineNumber;
- switch (class(ch)) { /* detect character class */
- case STNL: /* newline, vertical space or formfeed */
- firstline:
- LineNumber++; /* also at vs and ff */
- ptok->tk_file = FileName;
- ptok->tk_line = LineNumber;
- if (EoiForNewline) /* called in control line */
- /* a newline in a control line indicates the
- end-of-information of the line.
- */
- return ptok->tk_symb = EOI;
- while (LoadChar(ch), ch == '#') { /* a control line follows */
- domacro();
- if (File_Inserted) {
- File_Inserted = 0;
- goto firstline;
- }
- }
- /* We have to loop here, because in
- `domacro' the nl, vt or ff is read. The
- character following it may again be a `#'.
- */
- goto go_on;
- case STSKIP: /* just skip the skip characters */
- goto again;
- case STGARB: /* garbage character */
- #ifndef NOPP
- if (SkipEscNewline && (ch == '\\')) {
- /* a '\\' is allowed in #if/#elif expression */
- LoadChar(ch);
- if (class(ch) == STNL) { /* vt , ff ? */
- ++LineNumber;
- goto again;
- }
- PushBack();
- ch = '\\';
- }
- #endif NOPP
- if (040 < ch && ch < 0177)
- lexerror("garbage char %c", ch);
- else
- lexerror("garbage char \\%03o", ch);
- goto again;
- case STSIMP: /* a simple character, no part of compound token*/
- if (ch == '/') { /* probably the start of comment */
- LoadChar(ch);
- if (ch == '*') { /* start of comment */
- skipcomment();
- goto again;
- }
- else {
- PushBack();
- ch = '/'; /* restore ch */
- }
- }
- return ptok->tk_symb = ch;
- case STCOMP: /* maybe the start of a compound token */
- LoadChar(nch); /* character lookahead */
- switch (ch) {
- case '!':
- if (nch == '=')
- return ptok->tk_symb = NOTEQUAL;
- PushBack();
- return ptok->tk_symb = ch;
- case '&':
- if (nch == '&')
- return ptok->tk_symb = AND;
- PushBack();
- return ptok->tk_symb = ch;
- case '+':
- if (nch == '+')
- return ptok->tk_symb = PLUSPLUS;
- PushBack();
- return ptok->tk_symb = ch;
- case '-':
- if (nch == '-')
- return ptok->tk_symb = MINMIN;
- if (nch == '>')
- return ptok->tk_symb = ARROW;
- PushBack();
- return ptok->tk_symb = ch;
- case '<':
- if (AccFileSpecifier) {
- PushBack(); /* pushback nch */
- ptok->tk_bts = string_token("file specifier",
- '>', &(ptok->tk_len));
- return ptok->tk_symb = FILESPECIFIER;
- }
- if (nch == '<')
- return ptok->tk_symb = LEFT;
- if (nch == '=')
- return ptok->tk_symb = LESSEQ;
- PushBack();
- return ptok->tk_symb = ch;
- case '=':
- if (nch == '=')
- return ptok->tk_symb = EQUAL;
- PushBack();
- return ptok->tk_symb = ch;
- case '>':
- if (nch == '=')
- return ptok->tk_symb = GREATEREQ;
- if (nch == '>')
- return ptok->tk_symb = RIGHT;
- PushBack();
- return ptok->tk_symb = ch;
- case '|':
- if (nch == '|')
- return ptok->tk_symb = OR;
- PushBack();
- return ptok->tk_symb = ch;
- }
- case STIDF:
- {
- register char *tg = &buf[0];
- register int pos = -1;
- register int hash;
- register struct idf *idef;
- extern int idfsize; /* ??? */
- hash = STARTHASH();
- do { /* read the identifier */
- if (++pos < idfsize) {
- #ifndef NOPP
- if (Unstacked) EnableMacros();
- #endif
- *tg++ = ch;
- hash = ENHASH(hash, ch, pos);
- }
- LoadChar(ch);
- } while (in_idf(ch));
- hash = STOPHASH(hash);
- if (ch != EOI)
- PushBack();
- *tg++ = '\0'; /* mark the end of the identifier */
- idef = ptok->tk_idf = idf_hashed(buf, tg - buf, hash);
- idef->id_file = ptok->tk_file;
- idef->id_line = ptok->tk_line;
- #ifndef NOPP
- if (idef->id_macro && ReplaceMacros && replace(idef))
- /* macro replacement should be performed */
- goto again;
- if (UnknownIdIsZero && idef->id_reserved != SIZEOF) {
- ptok->tk_ival = (arith)0;
- ptok->tk_fund = INT;
- return ptok->tk_symb = INTEGER;
- }
- #endif NOPP
- ptok->tk_symb = (
- idef->id_reserved ? idef->id_reserved
- : idef->id_def && idef->id_def->df_sc == TYPEDEF ?
- TYPE_IDENTIFIER
- : IDENTIFIER
- );
- return IDENTIFIER;
- }
- case STCHAR: /* character constant */
- {
- register arith val = 0;
- int size = 0;
- LoadChar(ch);
- if (ch == '\'')
- lexerror("character constant too short");
- else
- while (ch != '\'') {
- if (ch == '\n') {
- lexerror("newline in character constant");
- PushBack();
- break;
- }
- if (ch == '\\') {
- LoadChar(ch);
- if (ch == '\n')
- LineNumber++;
- ch = quoted(ch);
- }
- if (ch >= 128) ch -= 256;
- val = val*256 + ch;
- size++;
- LoadChar(ch);
- }
- if (size > (int)int_size)
- lexerror("character constant too long");
- ptok->tk_ival = val;
- ptok->tk_fund = INT;
- return ptok->tk_symb = INTEGER;
- }
- case STSTR: /* string */
- ptok->tk_bts = string_token("string", '"', &(ptok->tk_len));
- return ptok->tk_symb = STRING;
- case STNUM: /* a numeric constant */
- {
- /* It should be noted that 099 means 81(decimal) and
- 099.5 means 99.5 . This severely limits the tricks
- we can use to scan a numeric value.
- */
- register char *np = &buf[1];
- register int base = 10;
- register int vch;
- register arith val = 0;
- if (ch == '.') { /* an embarrassing ambiguity */
- #ifndef NOFLOAT
- LoadChar(vch);
- PushBack();
- if (!is_dig(vch)) /* just a `.' */
- return ptok->tk_symb = ch;
- *np++ = '0';
- /* in the rest of the compiler, all floats
- have to start with a digit.
- */
- #else NOFLOAT
- return ptok->tk_symb = ch;
- #endif NOFLOAT
- }
- if (ch == '0') {
- *np++ = ch;
- LoadChar(ch);
- if (ch == 'x' || ch == 'X') {
- base = 16;
- LoadChar(ch);
- }
- else
- base = 8;
- }
- while (vch = val_in_base(ch, base), vch >= 0) {
- val = val*base + vch;
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- }
- if (ch == 'l' || ch == 'L') {
- ptok->tk_ival = val;
- ptok->tk_fund = LONG;
- return ptok->tk_symb = INTEGER;
- }
- #ifndef NOFLOAT
- if (base == 16 || !(ch == '.' || ch == 'e' || ch == 'E'))
- #endif NOFLOAT
- {
- PushBack();
- ptok->tk_ival = val;
- /* The semantic analyser must know if the
- integral constant is given in octal/hexa-
- decimal form, in which case its type is
- UNSIGNED, or in decimal form, in which case
- its type is signed, indicated by
- the fund INTEGER.
- */
- ptok->tk_fund =
- (base == 10 || (base == 8 && val == (arith)0))
- ? INTEGER : UNSIGNED;
- return ptok->tk_symb = INTEGER;
- }
- /* where's the test for the length of the integral ??? */
- #ifndef NOFLOAT
- if (ch == '.'){
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- }
- while (is_dig(ch)){
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- }
- if (ch == 'e' || ch == 'E') {
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- if (ch == '+' || ch == '-') {
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- }
- if (!is_dig(ch)) {
- lexerror("malformed floating constant");
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- }
- while (is_dig(ch)) {
- if (np < &buf[NUMSIZE])
- *np++ = ch;
- LoadChar(ch);
- }
- }
- PushBack();
- *np++ = '\0';
- buf[0] = '-'; /* good heavens... */
- if (np == &buf[NUMSIZE+1]) {
- lexerror("floating constant too long");
- ptok->tk_fval = Salloc("-0.0",(unsigned) 5) + 1;
- }
- else
- ptok->tk_fval = Salloc(buf,(unsigned) (np - buf)) + 1;
- return ptok->tk_symb = FLOATING;
- #endif NOFLOAT
- }
- case STEOI: /* end of text on source file */
- return ptok->tk_symb = EOI;
- default: /* this cannot happen */
- crash("bad class for char 0%o", ch);
- }
- /*NOTREACHED*/
- }
- skipcomment()
- {
- /* The last character read has been the '*' of '/_*'. The
- characters, except NL and EOI, between '/_*' and the first
- occurring '*_/' are not interpreted.
- NL only affects the LineNumber. EOI is not legal.
- Important note: it is not possible to stop skipping comment
- beyond the end-of-file of an included file.
- EOI is returned by LoadChar only on encountering EOF of the
- top-level file...
- */
- register int c;
- NoUnstack++;
- LoadChar(c);
- #ifdef LINT
- lint_start_comment();
- lint_comment_char(c);
- #endif LINT
- do {
- while (c != '*') {
- if (class(c) == STNL)
- ++LineNumber;
- else
- if (c == EOI) {
- NoUnstack--;
- #ifdef LINT
- lint_end_comment();
- #endif LINT
- return;
- }
- LoadChar(c);
- #ifdef LINT
- lint_comment_char(c);
- #endif LINT
- } /* last Character seen was '*' */
- LoadChar(c);
- #ifdef LINT
- lint_comment_char(c);
- #endif LINT
- } while (c != '/');
- #ifdef LINT
- lint_end_comment();
- #endif LINT
- NoUnstack--;
- }
- char *
- string_token(nm, stop_char, plen)
- char *nm;
- int *plen;
- {
- register int ch;
- register int str_size;
- register char *str = Malloc((unsigned) (str_size = ISTRSIZE));
- register int pos = 0;
-
- LoadChar(ch);
- while (ch != stop_char) {
- if (ch == '\n') {
- lexerror("newline in %s", nm);
- PushBack();
- break;
- }
- if (ch == EOI) {
- lexerror("end-of-file inside %s", nm);
- break;
- }
- if (ch == '\\') {
- LoadChar(ch);
- if (ch == '\n') {
- LineNumber++;
- LoadChar(ch);
- continue;
- }
- ch = quoted(ch);
- }
- str[pos++] = ch;
- if (pos == str_size)
- str = Srealloc(str, (unsigned) (str_size += RSTRSIZE));
- LoadChar(ch);
- }
- str[pos++] = '\0'; /* for filenames etc. */
- *plen = pos;
- return str;
- }
- int
- quoted(ch)
- register int ch;
- {
- /* quoted() replaces an escaped character sequence by the
- character meant.
- */
- /* first char after backslash already in ch */
- if (!is_oct(ch)) { /* a quoted char */
- switch (ch) {
- case 'n':
- ch = '\n';
- break;
- case 't':
- ch = '\t';
- break;
- case 'b':
- ch = '\b';
- break;
- case 'r':
- ch = '\r';
- break;
- case 'f':
- ch = '\f';
- break;
- }
- }
- else { /* a quoted octal */
- register int oct = 0, cnt = 0;
- do {
- oct = oct*8 + (ch-'0');
- LoadChar(ch);
- } while (is_oct(ch) && ++cnt < 3);
- PushBack();
- ch = oct;
- }
- return ch&0377;
- }
- /* provisional */
- int
- val_in_base(ch, base)
- register int ch;
- {
- return
- is_dig(ch) ? ch - '0'
- : base != 16 ? -1
- : is_hex(ch) ? (ch - 'a' + 10) & 017
- : -1;
- }
|