tokens.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. /* LLgen generated code from source tokens.g */
  2. #include "Lpars.h"
  3. #define LL_LEXI scanner
  4. #define LLNOFIRSTS
  5. #if __STDC__ || __cplusplus
  6. #define LL_ANSI_C 1
  7. #endif
  8. #define LL_LEXI scanner
  9. /* $Id$ */
  10. #ifdef LL_DEBUG
  11. #include <assert.h>
  12. #include <stdio.h>
  13. #define LL_assert(x) assert(x)
  14. #else
  15. #define LL_assert(x) /* nothing */
  16. #endif
  17. extern int LLsymb;
  18. #define LL_SAFE(x) /* Nothing */
  19. #define LL_SSCANDONE(x) if (LLsymb != x) LLsafeerror(x)
  20. #define LL_SCANDONE(x) if (LLsymb != x) LLerror(x)
  21. #define LL_NOSCANDONE(x) LLscan(x)
  22. #ifdef LL_FASTER
  23. #define LLscan(x) if ((LLsymb = LL_LEXI()) != x) LLerror(x)
  24. #endif
  25. extern unsigned int LLscnt[];
  26. extern unsigned int LLtcnt[];
  27. extern int LLcsymb;
  28. #if LL_NON_CORR
  29. extern int LLstartsymb;
  30. #endif
  31. #define LLsdecr(d) {LL_assert(LLscnt[d] > 0); LLscnt[d]--;}
  32. #define LLtdecr(d) {LL_assert(LLtcnt[d] > 0); LLtcnt[d]--;}
  33. #define LLsincr(d) LLscnt[d]++
  34. #define LLtincr(d) LLtcnt[d]++
  35. #if LL_ANSI_C
  36. extern int LL_LEXI(void);
  37. extern void LLread(void);
  38. extern int LLskip(void);
  39. extern int LLnext(int);
  40. extern void LLerror(int);
  41. extern void LLsafeerror(int);
  42. extern void LLnewlevel(unsigned int *);
  43. extern void LLoldlevel(unsigned int *);
  44. #ifndef LL_FASTER
  45. extern void LLscan(int);
  46. #endif
  47. #ifndef LLNOFIRSTS
  48. extern int LLfirst(int, int);
  49. #endif
  50. #if LL_NON_CORR
  51. extern void LLnc_recover(void);
  52. #endif
  53. #else /* not LL_ANSI_C */
  54. extern LLread();
  55. extern int LLskip();
  56. extern int LLnext();
  57. extern LLerror();
  58. extern LLsafeerror();
  59. extern LLnewlevel();
  60. extern LLoldlevel();
  61. #ifndef LL_FASTER
  62. extern LLscan();
  63. #endif
  64. #ifndef LLNOFIRSTS
  65. extern int LLfirst();
  66. #endif
  67. #if LL_NON_CORR
  68. extern LLnc_recover();
  69. #endif
  70. #endif /* not LL_ANSI_C */
  71. # line 20 "tokens.g"
  72. #include <stdio.h>
  73. #include <string.h>
  74. # include "types.h"
  75. # include "io.h"
  76. # include "extern.h"
  77. # include "assert.h"
  78. # include "cclass.h"
  79. #include "LLgen.h"
  80. static int nparams;
  81. # line 63 "tokens.g"
  82. /*
  83. * Structure for a keyword
  84. */
  85. typedef struct keyword {
  86. string w_word;
  87. int w_value;
  88. } t_keyw, *p_keyw;
  89. /*
  90. * The list of keywords, the most often used keywords come first.
  91. * Linear search is used, as there are not many keywords
  92. */
  93. static t_keyw resword[] = {
  94. { "token", C_TOKEN },
  95. { "avoid", C_AVOID },
  96. { "prefer", C_PREFER },
  97. { "persistent", C_PERSISTENT },
  98. { "default", C_DEFAULT },
  99. { "if", C_IF },
  100. { "while", C_WHILE },
  101. { "first", C_FIRST },
  102. { "start", C_START },
  103. { "lexical", C_LEXICAL },
  104. { "onerror", C_ONERROR },
  105. { "prefix", C_PREFIX },
  106. #ifdef NON_CORRECTING
  107. { "substart", C_SUBSTART },
  108. { "erroneous", C_ERRONEOUS },
  109. { "illegal", C_ILLEGAL },
  110. #endif
  111. { 0, 0 }
  112. };
  113. static t_token savedtok; /* to save lextoken in case of an insertion */
  114. # ifdef LINE_DIRECTIVE
  115. static int nostartline; /* = 0 if at the start of a line */
  116. # endif
  117. STATIC void copyact(char ch1, char ch2, int flag, int level) {
  118. /*
  119. * Copy an action to file f. Opening bracket is ch1, closing bracket
  120. * is ch2.
  121. * If flag & 1, copy opening and closing parameters too.
  122. * If flag & 2, don't allow ','.
  123. */
  124. static int text_seen = 0;
  125. register FILE *f;
  126. register ch; /* Current char */
  127. register match; /* used to read strings */
  128. int saved = linecount;
  129. /* save linecount */
  130. int sav_strip = strip_grammar;
  131. f = fact;
  132. if (ch1 == '{' || flag != 1) strip_grammar = 0;
  133. if (!level) {
  134. text_seen = 0;
  135. nparams = 0; /* count comma's */
  136. putc('\0',f);
  137. fprintf(f,"# line %d \"%s\"\n", linecount,f_input);
  138. }
  139. if (level || (flag & 1)) putc(ch1,f);
  140. for (;;) {
  141. ch = input();
  142. if (ch == ch2) {
  143. if (!level) {
  144. if (text_seen) nparams++;
  145. }
  146. if (level || (flag & 1)) putc(ch,f);
  147. if (strip_grammar != sav_strip) {
  148. if (ch1 == '{' || flag != 1) putchar(ch);
  149. }
  150. strip_grammar = sav_strip;
  151. return;
  152. }
  153. switch(ch) {
  154. case ')':
  155. case '}':
  156. case ']':
  157. error(linecount,"Parentheses mismatch", NULL, NULL);
  158. break;
  159. case '(':
  160. text_seen = 1;
  161. copyact('(',')',flag,level+1);
  162. continue;
  163. case '{':
  164. text_seen = 1;
  165. copyact('{','}',flag,level+1);
  166. continue;
  167. case '[':
  168. text_seen = 1;
  169. copyact('[',']',flag,level+1);
  170. continue;
  171. case '/':
  172. ch = input();
  173. unput(ch);
  174. if (ch == '*') {
  175. putc('/', f);
  176. skipcomment(1);
  177. continue;
  178. }
  179. ch = '/';
  180. text_seen = 1;
  181. break;
  182. case ';':
  183. case ',':
  184. if (! level && text_seen) {
  185. text_seen = 0;
  186. nparams++;
  187. if (ch == ',' && (flag & 2)) {
  188. warning(linecount, "Parameters may not be separated with a ','", NULL, NULL);
  189. ch = ';';
  190. }
  191. }
  192. break;
  193. case '\'':
  194. case '"' :
  195. /*
  196. * watch out for brackets in strings, they do not
  197. * count !
  198. */
  199. text_seen = 1;
  200. match = ch;
  201. putc(ch,f);
  202. while((ch = input())) {
  203. if (ch == match) break;
  204. if (ch == '\\') {
  205. putc(ch,f);
  206. ch = input();
  207. }
  208. if (ch == '\n') {
  209. error(linecount,"Newline in string", NULL, NULL);
  210. unput(match);
  211. }
  212. putc(ch,f);
  213. }
  214. if (ch == match) break;
  215. /* Fall through */
  216. case EOF :
  217. if (!level) error(saved,"Action does not terminate", NULL, NULL);
  218. strip_grammar = sav_strip;
  219. return;
  220. default:
  221. if (c_class[ch] != ISSPA) text_seen = 1;
  222. }
  223. putc(ch,f);
  224. }
  225. }
  226. int scanner() {
  227. /*
  228. * Lexical analyser, what else
  229. */
  230. register int ch; /* Current char */
  231. register char *p = ltext;
  232. int reserved = 0; /* reserved word? */
  233. char *max = &ltext[LTEXTSZ - 1];
  234. static int nextexpr;
  235. int expect_expr = nextexpr;
  236. long off;
  237. nextexpr = 0;
  238. if (savedtok.t_tokno) {
  239. /* A token has been inserted.
  240. * Now deliver the last lextoken again
  241. */
  242. lextoken = savedtok;
  243. savedtok.t_tokno = 0;
  244. return lextoken.t_tokno;
  245. }
  246. for (;;) {
  247. ch = input();
  248. if (ch == EOF) return ch;
  249. # ifdef LINE_DIRECTIVE
  250. if (ch == '#' && !nostartline) {
  251. linedirective();
  252. continue;
  253. }
  254. # endif
  255. switch(c_class[ch]) {
  256. case ISACT :
  257. if (ch == '{') {
  258. copyact('{', '}', in_production, 0);
  259. return C_ACTION;
  260. }
  261. assert(ch == '(');
  262. if (expect_expr) {
  263. copyact('(', ')', 1, 0);
  264. return C_EXPR;
  265. }
  266. off = ftell(fact);
  267. copyact('(', ')', in_production != 0 ? 0 : 2, 0);
  268. if (nparams == 0) fseek(fact, off, 0);
  269. lextoken.t_num = nparams;
  270. return C_PARAMS;
  271. case ISLIT :
  272. for (;;) {
  273. ch = input();
  274. if (ch == '\n' || ch == EOF) {
  275. error(linecount,"Missing '", NULL, NULL);
  276. break;
  277. }
  278. if (ch == '\'') break;
  279. if (ch == '\\') {
  280. *p++ = ch;
  281. ch = input();
  282. }
  283. *p++ = ch;
  284. if (p > max) p--;
  285. }
  286. *p = '\0';
  287. lextoken.t_string = ltext;
  288. return C_LITERAL;
  289. case ISCOM :
  290. skipcomment(0);
  291. /* Fall through */
  292. case ISSPA :
  293. continue;
  294. case ISDIG : {
  295. register i = 0;
  296. do {
  297. i = 10 * i + (ch - '0');
  298. ch= input();
  299. } while (c_class[ch] == ISDIG);
  300. lextoken.t_num = i;
  301. unput(ch);
  302. return C_NUMBER; }
  303. default:
  304. return ch;
  305. case ISKEY :
  306. reserved = 1;
  307. ch = input();
  308. /* Fall through */
  309. case ISLET :
  310. do {
  311. if (reserved && ch >= 'A' && ch <= 'Z') {
  312. ch += 'a' - 'A';
  313. }
  314. *p++ = ch;
  315. if (p > max) p--;
  316. ch = input();
  317. } while (c_class[ch] == ISDIG || c_class[ch] == ISLET);
  318. unput(ch);
  319. *p = '\0';
  320. if (reserved) { /*
  321. * Now search for the keyword
  322. */
  323. register p_keyw w;
  324. w = resword;
  325. while (w->w_word) {
  326. if (! strcmp(ltext,w->w_word)) {
  327. /*
  328. * Return token number.
  329. */
  330. if (w->w_value == C_IF ||
  331. w->w_value == C_WHILE) {
  332. nextexpr = 1;
  333. }
  334. return w->w_value;
  335. }
  336. w++;
  337. }
  338. error(linecount,"Illegal reserved word", NULL, NULL);
  339. }
  340. lextoken.t_string = ltext;
  341. return C_IDENT;
  342. }
  343. }
  344. }
  345. static int backupc; /* for unput() */
  346. static int nonline; /* = 1 if last char read was a newline */
  347. int input() {
  348. /*
  349. * Low level input routine, used by all other input routines
  350. */
  351. int c;
  352. if ((c = backupc)) {
  353. /* Last char was "unput()". Deliver it again
  354. */
  355. backupc = 0;
  356. return c;
  357. }
  358. if ((c = getc(finput)) == EOF) {
  359. nonline = 0;
  360. return c;
  361. }
  362. # ifdef LINE_DIRECTIVE
  363. nostartline = 1;
  364. # endif
  365. if (!nonline) {
  366. linecount++;
  367. # ifdef LINE_DIRECTIVE
  368. nostartline = 0;
  369. # endif
  370. nonline = 1;
  371. }
  372. if (c == '\n') nonline = 0;
  373. if (strip_grammar) putchar(c);
  374. return c;
  375. }
  376. void unput(c) {
  377. /*
  378. * "unread" c
  379. */
  380. backupc = c;
  381. }
  382. void skipcomment(flag) {
  383. /*
  384. * Skip comment. If flag != 0, the comment is inside a fragment
  385. * of C-code, so keep it.
  386. */
  387. register int ch;
  388. int saved; /* line count on which comment starts */
  389. saved = linecount;
  390. if (input() != '*') error(linecount,"Illegal comment", NULL, NULL);
  391. if (flag) putc('*', fact);
  392. do {
  393. ch = input();
  394. if (flag) putc(ch, fact);
  395. while (ch == '*') {
  396. ch = input();
  397. if (flag) putc(ch, fact);
  398. if (ch == '/') return;
  399. }
  400. } while (ch != EOF);
  401. error(saved,"Comment does not terminate", NULL, NULL);
  402. }
  403. # ifdef LINE_DIRECTIVE
  404. STATIC void linedirective() {
  405. /*
  406. * Read a line directive
  407. */
  408. register int ch;
  409. register int i;
  410. string s_error = "Illegal line directive";
  411. string store();
  412. register string c;
  413. do { /*
  414. * Skip to next digit
  415. * Do not skip newlines
  416. */
  417. ch = input();
  418. } while (ch != '\n' && c_class[ch] != ISDIG);
  419. if (ch == '\n') {
  420. error(linecount,s_error, NULL, NULL);
  421. return;
  422. }
  423. i = 0;
  424. do {
  425. i = i*10 + (ch - '0');
  426. ch = input();
  427. } while (c_class[ch] == ISDIG);
  428. while (ch != '\n' && ch != '"') ch = input();
  429. if (ch == '"') {
  430. c = ltext;
  431. do {
  432. *c++ = ch = input();
  433. } while (ch != '"' && ch != '\n');
  434. if (ch == '\n') {
  435. error(linecount,s_error, NULL, NULL);
  436. return;
  437. }
  438. *--c = '\0';
  439. do {
  440. ch = input();
  441. } while (ch != '\n');
  442. /*
  443. * Remember the file name
  444. */
  445. if (strcmp(f_input,ltext)) f_input = store(ltext);
  446. }
  447. linecount = i;
  448. }
  449. # endif
  450. STATIC string vallookup(s) {
  451. /*
  452. * Look up the keyword that has token number s
  453. */
  454. register p_keyw p = resword;
  455. while (p->w_value) {
  456. if (p->w_value == s) return p->w_word;
  457. p++;
  458. }
  459. return 0;
  460. }
  461. STATIC string cpy(int s,string p, int inserted) {
  462. /*
  463. * Create a piece of error message for token s and put it at p.
  464. * inserted = 0 if the token s was deleted (in which case we have
  465. * attributes), else it was inserted
  466. */
  467. register string t = 0;
  468. switch(s) {
  469. case C_IDENT :
  470. if (!inserted) t = lextoken.t_string;
  471. else t = "identifier";
  472. break;
  473. case C_NUMBER :
  474. t = "number";
  475. break;
  476. case C_LITERAL :
  477. if (!inserted) {
  478. *p++ = '\'';
  479. t = lextoken.t_string;
  480. break;
  481. }
  482. t = "literal";
  483. break;
  484. case C_ACTION:
  485. t = "C action";
  486. break;
  487. case C_PARAMS:
  488. t = "C parameter section";
  489. break;
  490. case C_EXPR:
  491. t = "C expression";
  492. break;
  493. case EOFILE :
  494. t = "end-of-file";
  495. break;
  496. }
  497. if (!t && (t = vallookup(s))) {
  498. *p++ = '%';
  499. }
  500. if (t) { /*
  501. * We have a string for the token. Copy it
  502. */
  503. while (*t) *p++ = *t++;
  504. if (s == C_LITERAL && !inserted) {
  505. *p++ = '\'';
  506. }
  507. return p;
  508. }
  509. /*
  510. * The token is a literal
  511. */
  512. *p++ = '\'';
  513. if (s >= 040 && s <= 0176) *p++ = s;
  514. else {
  515. *p++ = '\\';
  516. switch(s) {
  517. case '\b' : *p++ = 'b'; break;
  518. case '\f' : *p++ = 'f'; break;
  519. case '\n' : *p++ = 'n'; break;
  520. case '\r' : *p++ = 'r'; break;
  521. case '\t' : *p++ = 't'; break;
  522. default : *p++='0'+((s&0377)>>6); *p++='0'+((s>>3)&07);
  523. *p++='0'+(s&07);
  524. }
  525. }
  526. *p++ = '\'';
  527. return p;
  528. }
  529. void LLmessage(int d) {
  530. /*
  531. * d is either 0, in which case the current token has been deleted,
  532. * or non-zero, in which case it represents a token that is inserted
  533. * before the current token
  534. */
  535. register string s,t;
  536. char buf[128];
  537. nerrors++;
  538. s = buf;
  539. if (d < 0) {
  540. strcpy(buf, "end-of-file expected");
  541. }
  542. else if (d == 0) {
  543. #ifdef LLNONCORR
  544. t = " unexpected";
  545. #else
  546. t = " deleted";
  547. #endif
  548. s = cpy(LLsymb,s,0);
  549. do *s++ = *t; while (*t++);
  550. } else {
  551. s = cpy(d,s,1);
  552. t = " inserted in front of ";
  553. do *s++ = *t++; while (*t);
  554. s = cpy(LLsymb,s,0);
  555. *s = '\0';
  556. }
  557. if (d > 0) { /*
  558. * Save the current token and make up some
  559. * attributes for the inserted token
  560. */
  561. savedtok = lextoken;
  562. savedtok.t_tokno = LLsymb;
  563. if (d == C_IDENT) lextoken.t_string = "dummy_identifier";
  564. else if (d == C_LITERAL) lextoken.t_string = "dummy_literal";
  565. else if (d == C_NUMBER) lextoken.t_num = 1;
  566. }
  567. #ifdef LLNONCORR
  568. else
  569. #endif
  570. error(linecount, "%s", buf, NULL);
  571. /* Don't change this line to
  572. * error(linecount, buf).
  573. * The string in "buf" might contain '%' ...
  574. */
  575. #ifdef LLNONCORR
  576. in_production = 1;
  577. /* To prevent warnings from copyact */
  578. #endif
  579. }