tokens.g 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604
  1. /* Copyright (c) 1991 by the Vrije Universiteit, Amsterdam, the Netherlands.
  2. * For full copyright and restrictions on use see the file COPYING in the top
  3. * level of the LLgen tree.
  4. */
  5. /*
  6. * L L G E N
  7. *
  8. * An Extended LL(1) Parser Generator
  9. *
  10. * Author : Ceriel J.H. Jacobs
  11. */
  12. /*
  13. * tokens.g
  14. * Defines the tokens for the grammar of LLgen.
  15. * The lexical analyser and LLmessage are also included here.
  16. */
  17. {
  18. # include "types.h"
  19. # include "io.h"
  20. # include "extern.h"
  21. # include "assert.h"
  22. # include "cclass.h"
  23. # ifndef NORCSID
  24. static string rcsidc = "$Id$";
  25. # endif
  26. /* Here are defined : */
  27. extern int scanner();
  28. extern LLmessage();
  29. extern int input();
  30. extern unput();
  31. extern skipcomment();
  32. # ifdef LINE_DIRECTIVE
  33. STATIC linedirective();
  34. # endif
  35. STATIC string cpy();
  36. STATIC string vallookup();
  37. STATIC copyact();
  38. static int nparams;
  39. }
  40. /* Classes */
  41. %token C_IDENT ; /* lextoken.t_string contains the identifier read */
  42. %token C_NUMBER ; /* lextoken.t_num contains the number read */
  43. %token C_LITERAL ; /* lextoken.t_string contains the literal read */
  44. %token C_EXPR ; /* A C expression (%if or %while) */
  45. %token C_PARAMS ; /* formal or actual parameters */
  46. %token C_ACTION ; /* a C action */
  47. /* Keywords */
  48. %token C_TOKEN ;
  49. %token C_START ;
  50. %token C_IF ;
  51. %token C_WHILE ;
  52. %token C_PERSISTENT ;
  53. %token C_FIRST ;
  54. %token C_LEXICAL ;
  55. %token C_PREFIX ;
  56. %token C_ONERROR ;
  57. %token C_AVOID ;
  58. %token C_PREFER ;
  59. %token C_DEFAULT ;
  60. %token C_SUBSTART ;
  61. %token C_ERRONEOUS ;
  62. %token C_ILLEGAL ;
  63. %lexical scanner ;
  64. {
  65. /*
  66. * Structure for a keyword
  67. */
  68. typedef struct keyword {
  69. string w_word;
  70. int w_value;
  71. } t_keyw, *p_keyw;
  72. /*
  73. * The list of keywords, the most often used keywords come first.
  74. * Linear search is used, as there are not many keywords
  75. */
  76. static t_keyw resword[] = {
  77. { "token", C_TOKEN },
  78. { "avoid", C_AVOID },
  79. { "prefer", C_PREFER },
  80. { "persistent", C_PERSISTENT },
  81. { "default", C_DEFAULT },
  82. { "if", C_IF },
  83. { "while", C_WHILE },
  84. { "first", C_FIRST },
  85. { "start", C_START },
  86. { "lexical", C_LEXICAL },
  87. { "onerror", C_ONERROR },
  88. { "prefix", C_PREFIX },
  89. #ifdef NON_CORRECTING
  90. { "substart", C_SUBSTART },
  91. { "erroneous", C_ERRONEOUS },
  92. { "illegal", C_ILLEGAL },
  93. #endif
  94. { 0, 0 }
  95. };
  96. static t_token savedtok; /* to save lextoken in case of an insertion */
  97. # ifdef LINE_DIRECTIVE
  98. static int nostartline; /* = 0 if at the start of a line */
  99. # endif
  100. STATIC
  101. copyact(ch1,ch2,flag,level) char ch1,ch2; {
  102. /*
  103. * Copy an action to file f. Opening bracket is ch1, closing bracket
  104. * is ch2.
  105. * If flag & 1, copy opening and closing parameters too.
  106. * If flag & 2, don't allow ','.
  107. */
  108. static int text_seen = 0;
  109. register FILE *f;
  110. register ch; /* Current char */
  111. register match; /* used to read strings */
  112. int saved = linecount;
  113. /* save linecount */
  114. int sav_strip = strip_grammar;
  115. f = fact;
  116. if (ch1 == '{' || flag != 1) strip_grammar = 0;
  117. if (!level) {
  118. text_seen = 0;
  119. nparams = 0; /* count comma's */
  120. putc('\0',f);
  121. fprintf(f,"# line %d \"%s\"\n", linecount,f_input);
  122. }
  123. if (level || (flag & 1)) putc(ch1,f);
  124. for (;;) {
  125. ch = input();
  126. if (ch == ch2) {
  127. if (!level) {
  128. if (text_seen) nparams++;
  129. }
  130. if (level || (flag & 1)) putc(ch,f);
  131. if (strip_grammar != sav_strip) {
  132. if (ch1 == '{' || flag != 1) putchar(ch);
  133. }
  134. strip_grammar = sav_strip;
  135. return;
  136. }
  137. switch(ch) {
  138. case ')':
  139. case '}':
  140. case ']':
  141. error(linecount,"Parentheses mismatch");
  142. break;
  143. case '(':
  144. text_seen = 1;
  145. copyact('(',')',flag,level+1);
  146. continue;
  147. case '{':
  148. text_seen = 1;
  149. copyact('{','}',flag,level+1);
  150. continue;
  151. case '[':
  152. text_seen = 1;
  153. copyact('[',']',flag,level+1);
  154. continue;
  155. case '/':
  156. ch = input();
  157. unput(ch);
  158. if (ch == '*') {
  159. putc('/', f);
  160. skipcomment(1);
  161. continue;
  162. }
  163. ch = '/';
  164. text_seen = 1;
  165. break;
  166. case ';':
  167. case ',':
  168. if (! level && text_seen) {
  169. text_seen = 0;
  170. nparams++;
  171. if (ch == ',' && (flag & 2)) {
  172. warning(linecount, "Parameters may not be separated with a ','");
  173. ch = ';';
  174. }
  175. }
  176. break;
  177. case '\'':
  178. case '"' :
  179. /*
  180. * watch out for brackets in strings, they do not
  181. * count !
  182. */
  183. text_seen = 1;
  184. match = ch;
  185. putc(ch,f);
  186. while((ch = input())) {
  187. if (ch == match) break;
  188. if (ch == '\\') {
  189. putc(ch,f);
  190. ch = input();
  191. }
  192. if (ch == '\n') {
  193. error(linecount,"Newline in string");
  194. unput(match);
  195. }
  196. putc(ch,f);
  197. }
  198. if (ch == match) break;
  199. /* Fall through */
  200. case EOF :
  201. if (!level) error(saved,"Action does not terminate");
  202. strip_grammar = sav_strip;
  203. return;
  204. default:
  205. if (c_class[ch] != ISSPA) text_seen = 1;
  206. }
  207. putc(ch,f);
  208. }
  209. }
  210. scanner() {
  211. /*
  212. * Lexical analyser, what else
  213. */
  214. register int ch; /* Current char */
  215. register char *p = ltext;
  216. int reserved = 0; /* reserved word? */
  217. char *max = &ltext[LTEXTSZ - 1];
  218. static int nextexpr;
  219. int expect_expr = nextexpr;
  220. long off;
  221. nextexpr = 0;
  222. if (savedtok.t_tokno) {
  223. /* A token has been inserted.
  224. * Now deliver the last lextoken again
  225. */
  226. lextoken = savedtok;
  227. savedtok.t_tokno = 0;
  228. return lextoken.t_tokno;
  229. }
  230. for (;;) {
  231. ch = input();
  232. if (ch == EOF) return ch;
  233. # ifdef LINE_DIRECTIVE
  234. if (ch == '#' && !nostartline) {
  235. linedirective();
  236. continue;
  237. }
  238. # endif
  239. switch(c_class[ch]) {
  240. case ISACT :
  241. if (ch == '{') {
  242. copyact('{', '}', in_production, 0);
  243. return C_ACTION;
  244. }
  245. assert(ch == '(');
  246. if (expect_expr) {
  247. copyact('(', ')', 1, 0);
  248. return C_EXPR;
  249. }
  250. off = ftell(fact);
  251. copyact('(', ')', in_production != 0 ? 0 : 2, 0);
  252. if (nparams == 0) fseek(fact, off, 0);
  253. lextoken.t_num = nparams;
  254. return C_PARAMS;
  255. case ISLIT :
  256. for (;;) {
  257. ch = input();
  258. if (ch == '\n' || ch == EOF) {
  259. error(linecount,"Missing '");
  260. break;
  261. }
  262. if (ch == '\'') break;
  263. if (ch == '\\') {
  264. *p++ = ch;
  265. ch = input();
  266. }
  267. *p++ = ch;
  268. if (p > max) p--;
  269. }
  270. *p = '\0';
  271. lextoken.t_string = ltext;
  272. return C_LITERAL;
  273. case ISCOM :
  274. skipcomment(0);
  275. /* Fall through */
  276. case ISSPA :
  277. continue;
  278. case ISDIG : {
  279. register i = 0;
  280. do {
  281. i = 10 * i + (ch - '0');
  282. ch= input();
  283. } while (c_class[ch] == ISDIG);
  284. lextoken.t_num = i;
  285. unput(ch);
  286. return C_NUMBER; }
  287. default:
  288. return ch;
  289. case ISKEY :
  290. reserved = 1;
  291. ch = input();
  292. /* Fall through */
  293. case ISLET :
  294. do {
  295. if (reserved && ch >= 'A' && ch <= 'Z') {
  296. ch += 'a' - 'A';
  297. }
  298. *p++ = ch;
  299. if (p > max) p--;
  300. ch = input();
  301. } while (c_class[ch] == ISDIG || c_class[ch] == ISLET);
  302. unput(ch);
  303. *p = '\0';
  304. if (reserved) { /*
  305. * Now search for the keyword
  306. */
  307. register p_keyw w;
  308. w = resword;
  309. while (w->w_word) {
  310. if (! strcmp(ltext,w->w_word)) {
  311. /*
  312. * Return token number.
  313. */
  314. if (w->w_value == C_IF ||
  315. w->w_value == C_WHILE) {
  316. nextexpr = 1;
  317. }
  318. return w->w_value;
  319. }
  320. w++;
  321. }
  322. error(linecount,"Illegal reserved word");
  323. }
  324. lextoken.t_string = ltext;
  325. return C_IDENT;
  326. }
  327. }
  328. }
  329. static int backupc; /* for unput() */
  330. static int nonline; /* = 1 if last char read was a newline */
  331. input() {
  332. /*
  333. * Low level input routine, used by all other input routines
  334. */
  335. register c;
  336. if (c = backupc) {
  337. /* Last char was "unput()". Deliver it again
  338. */
  339. backupc = 0;
  340. return c;
  341. }
  342. if ((c = getc(finput)) == EOF) {
  343. nonline = 0;
  344. return c;
  345. }
  346. # ifdef LINE_DIRECTIVE
  347. nostartline = 1;
  348. # endif
  349. if (!nonline) {
  350. linecount++;
  351. # ifdef LINE_DIRECTIVE
  352. nostartline = 0;
  353. # endif
  354. nonline = 1;
  355. }
  356. if (c == '\n') nonline = 0;
  357. if (strip_grammar) putchar(c);
  358. return c;
  359. }
  360. unput(c) {
  361. /*
  362. * "unread" c
  363. */
  364. backupc = c;
  365. }
  366. skipcomment(flag) {
  367. /*
  368. * Skip comment. If flag != 0, the comment is inside a fragment
  369. * of C-code, so keep it.
  370. */
  371. register int ch;
  372. int saved; /* line count on which comment starts */
  373. saved = linecount;
  374. if (input() != '*') error(linecount,"Illegal comment");
  375. if (flag) putc('*', fact);
  376. do {
  377. ch = input();
  378. if (flag) putc(ch, fact);
  379. while (ch == '*') {
  380. ch = input();
  381. if (flag) putc(ch, fact);
  382. if (ch == '/') return;
  383. }
  384. } while (ch != EOF);
  385. error(saved,"Comment does not terminate");
  386. }
  387. # ifdef LINE_DIRECTIVE
  388. STATIC
  389. linedirective() {
  390. /*
  391. * Read a line directive
  392. */
  393. register int ch;
  394. register int i;
  395. string s_error = "Illegal line directive";
  396. string store();
  397. register string c;
  398. do { /*
  399. * Skip to next digit
  400. * Do not skip newlines
  401. */
  402. ch = input();
  403. } while (ch != '\n' && c_class[ch] != ISDIG);
  404. if (ch == '\n') {
  405. error(linecount,s_error);
  406. return;
  407. }
  408. i = 0;
  409. do {
  410. i = i*10 + (ch - '0');
  411. ch = input();
  412. } while (c_class[ch] == ISDIG);
  413. while (ch != '\n' && ch != '"') ch = input();
  414. if (ch == '"') {
  415. c = ltext;
  416. do {
  417. *c++ = ch = input();
  418. } while (ch != '"' && ch != '\n');
  419. if (ch == '\n') {
  420. error(linecount,s_error);
  421. return;
  422. }
  423. *--c = '\0';
  424. do {
  425. ch = input();
  426. } while (ch != '\n');
  427. /*
  428. * Remember the file name
  429. */
  430. if (strcmp(f_input,ltext)) f_input = store(ltext);
  431. }
  432. linecount = i;
  433. }
  434. # endif
  435. STATIC string
  436. vallookup(s) {
  437. /*
  438. * Look up the keyword that has token number s
  439. */
  440. register p_keyw p = resword;
  441. while (p->w_value) {
  442. if (p->w_value == s) return p->w_word;
  443. p++;
  444. }
  445. return 0;
  446. }
  447. STATIC string
  448. cpy(s,p,inserted) register string p; {
  449. /*
  450. * Create a piece of error message for token s and put it at p.
  451. * inserted = 0 if the token s was deleted (in which case we have
  452. * attributes), else it was inserted
  453. */
  454. register string t = 0;
  455. switch(s) {
  456. case C_IDENT :
  457. if (!inserted) t = lextoken.t_string;
  458. else t = "identifier";
  459. break;
  460. case C_NUMBER :
  461. t = "number";
  462. break;
  463. case C_LITERAL :
  464. if (!inserted) {
  465. *p++ = '\'';
  466. t = lextoken.t_string;
  467. break;
  468. }
  469. t = "literal";
  470. break;
  471. case C_ACTION:
  472. t = "C action";
  473. break;
  474. case C_PARAMS:
  475. t = "C parameter section";
  476. break;
  477. case C_EXPR:
  478. t = "C expression";
  479. break;
  480. case EOFILE :
  481. t = "end-of-file";
  482. break;
  483. }
  484. if (!t && (t = vallookup(s))) {
  485. *p++ = '%';
  486. }
  487. if (t) { /*
  488. * We have a string for the token. Copy it
  489. */
  490. while (*t) *p++ = *t++;
  491. if (s == C_LITERAL && !inserted) {
  492. *p++ = '\'';
  493. }
  494. return p;
  495. }
  496. /*
  497. * The token is a literal
  498. */
  499. *p++ = '\'';
  500. if (s >= 040 && s <= 0176) *p++ = s;
  501. else {
  502. *p++ = '\\';
  503. switch(s) {
  504. case '\b' : *p++ = 'b'; break;
  505. case '\f' : *p++ = 'f'; break;
  506. case '\n' : *p++ = 'n'; break;
  507. case '\r' : *p++ = 'r'; break;
  508. case '\t' : *p++ = 't'; break;
  509. default : *p++='0'+((s&0377)>>6); *p++='0'+((s>>3)&07);
  510. *p++='0'+(s&07);
  511. }
  512. }
  513. *p++ = '\'';
  514. return p;
  515. }
  516. string strcpy();
  517. LLmessage(d) {
  518. /*
  519. * d is either 0, in which case the current token has been deleted,
  520. * or non-zero, in which case it represents a token that is inserted
  521. * before the current token
  522. */
  523. register string s,t;
  524. char buf[128];
  525. nerrors++;
  526. s = buf;
  527. if (d < 0) {
  528. strcpy(buf, "end-of-file expected");
  529. }
  530. else if (d == 0) {
  531. #ifdef LLNONCORR
  532. t = " unexpected";
  533. #else
  534. t = " deleted";
  535. #endif
  536. s = cpy(LLsymb,s,0);
  537. do *s++ = *t; while (*t++);
  538. } else {
  539. s = cpy(d,s,1);
  540. t = " inserted in front of ";
  541. do *s++ = *t++; while (*t);
  542. s = cpy(LLsymb,s,0);
  543. *s = '\0';
  544. }
  545. if (d > 0) { /*
  546. * Save the current token and make up some
  547. * attributes for the inserted token
  548. */
  549. savedtok = lextoken;
  550. savedtok.t_tokno = LLsymb;
  551. if (d == C_IDENT) lextoken.t_string = "dummy_identifier";
  552. else if (d == C_LITERAL) lextoken.t_string = "dummy_literal";
  553. else if (d == C_NUMBER) lextoken.t_num = 1;
  554. }
  555. #ifdef LLNONCORR
  556. else
  557. #endif
  558. error(linecount, "%s", buf);
  559. /* Don't change this line to
  560. * error(linecount, buf).
  561. * The string in "buf" might contain '%' ...
  562. */
  563. #ifdef LLNONCORR
  564. in_production = 1;
  565. /* To prevent warnings from copyact */
  566. #endif
  567. }
  568. }