tokens.g 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. /* Copyright (c) 1991 by the Vrije Universiteit, Amsterdam, the Netherlands.
  2. * For full copyright and restrictions on use see the file COPYING in the top
  3. * level of the LLgen tree.
  4. */
  5. /*
  6. * L L G E N
  7. *
  8. * An Extended LL(1) Parser Generator
  9. *
  10. * Author : Ceriel J.H. Jacobs
  11. */
  12. /*
  13. * tokens.g
  14. * Defines the tokens for the grammar of LLgen.
  15. * The lexical analyser and LLmessage are also included here.
  16. */
  17. {
  18. # include "types.h"
  19. # include "io.h"
  20. # include "extern.h"
  21. # include "assert.h"
  22. # include "cclass.h"
  23. # ifndef NORCSID
  24. static string rcsidc = "$Id$";
  25. # endif
  26. /* Here are defined : */
  27. extern int scanner();
  28. extern LLmessage();
  29. extern int input();
  30. extern unput();
  31. extern skipcomment();
  32. # ifdef LINE_DIRECTIVE
  33. STATIC linedirective();
  34. # endif
  35. STATIC string cpy();
  36. STATIC string vallookup();
  37. }
  38. /* Classes */
  39. %token C_IDENT ; /* lextoken.t_string contains the identifier read */
  40. %token C_NUMBER ; /* lextoken.t_num contains the number read */
  41. %token C_LITERAL ; /* lextoken.t_string contains the literal read */
  42. /* Keywords */
  43. %token C_TOKEN ;
  44. %token C_START ;
  45. %token C_IF ;
  46. %token C_WHILE ;
  47. %token C_PERSISTENT ;
  48. %token C_FIRST ;
  49. %token C_LEXICAL ;
  50. %token C_PREFIX ;
  51. %token C_ONERROR ;
  52. %token C_AVOID ;
  53. %token C_PREFER ;
  54. %token C_DEFAULT ;
  55. %lexical scanner ;
  56. {
  57. /*
  58. * Structure for a keyword
  59. */
  60. typedef struct keyword {
  61. string w_word;
  62. int w_value;
  63. } t_keyw, *p_keyw;
  64. /*
  65. * The list of keywords, the most often used keywords come first.
  66. * Linear search is used, as there are not many keywords
  67. */
  68. static t_keyw resword[] = {
  69. { "token", C_TOKEN },
  70. { "avoid", C_AVOID },
  71. { "prefer", C_PREFER },
  72. { "persistent", C_PERSISTENT },
  73. { "default", C_DEFAULT },
  74. { "if", C_IF },
  75. { "while", C_WHILE },
  76. { "first", C_FIRST },
  77. { "start", C_START },
  78. { "lexical", C_LEXICAL },
  79. { "onerror", C_ONERROR },
  80. { "prefix", C_PREFIX },
  81. { 0, 0 }
  82. };
  83. static t_token savedtok; /* to save lextoken in case of an insertion */
  84. # ifdef LINE_DIRECTIVE
  85. static int nostartline; /* = 0 if at the start of a line */
  86. # endif
  87. scanner() {
  88. /*
  89. * Lexical analyser, what else
  90. */
  91. register int ch; /* Current char */
  92. register char *p = ltext;
  93. int reserved = 0; /* reserved word? */
  94. char *max = &ltext[LTEXTSZ - 1];
  95. if (savedtok.t_tokno) {
  96. /* A token has been inserted.
  97. * Now deliver the last lextoken again
  98. */
  99. lextoken = savedtok;
  100. savedtok.t_tokno = 0;
  101. return lextoken.t_tokno;
  102. }
  103. for (;;) {
  104. ch = input();
  105. if (ch == EOF) return ch;
  106. # ifdef LINE_DIRECTIVE
  107. if (ch == '#' && !nostartline) {
  108. linedirective();
  109. continue;
  110. }
  111. # endif
  112. switch(c_class[ch]) {
  113. case ISLIT :
  114. for (;;) {
  115. ch = input();
  116. if (ch == '\n' || ch == EOF) {
  117. error(linecount,"Missing '");
  118. break;
  119. }
  120. if (ch == '\'') break;
  121. if (ch == '\\') {
  122. *p++ = ch;
  123. ch = input();
  124. }
  125. *p++ = ch;
  126. if (p > max) p--;
  127. }
  128. *p = '\0';
  129. lextoken.t_string = ltext;
  130. return C_LITERAL;
  131. case ISCOM :
  132. skipcomment(0);
  133. /* Fall through */
  134. case ISSPA :
  135. continue;
  136. case ISDIG : {
  137. register i = 0;
  138. do {
  139. i = 10 * i + (ch - '0');
  140. ch= input();
  141. } while (c_class[ch] == ISDIG);
  142. lextoken.t_num = i;
  143. unput(ch);
  144. return C_NUMBER; }
  145. default:
  146. return ch;
  147. case ISKEY :
  148. reserved = 1;
  149. ch = input();
  150. /* Fall through */
  151. case ISLET :
  152. do {
  153. if (reserved && ch >= 'A' && ch <= 'Z') {
  154. ch += 'a' - 'A';
  155. }
  156. *p++ = ch;
  157. if (p > max) p--;
  158. ch = input();
  159. } while (c_class[ch] == ISDIG || c_class[ch] == ISLET);
  160. unput(ch);
  161. *p = '\0';
  162. if (reserved) { /*
  163. * Now search for the keyword
  164. */
  165. register p_keyw w;
  166. w = resword;
  167. while (w->w_word) {
  168. if (! strcmp(ltext,w->w_word)) {
  169. /*
  170. * Return token number.
  171. */
  172. return w->w_value;
  173. }
  174. w++;
  175. }
  176. error(linecount,"Illegal reserved word");
  177. }
  178. lextoken.t_string = ltext;
  179. return C_IDENT;
  180. }
  181. }
  182. }
  183. static int backupc; /* for unput() */
  184. static int nonline; /* = 1 if last char read was a newline */
  185. input() {
  186. /*
  187. * Low level input routine, used by all other input routines
  188. */
  189. register c;
  190. if (c = backupc) {
  191. /* Last char was "unput()". Deliver it again
  192. */
  193. backupc = 0;
  194. return c;
  195. }
  196. if ((c = getc(finput)) == EOF) return c;
  197. # ifdef LINE_DIRECTIVE
  198. nostartline = 1;
  199. # endif
  200. if (!nonline) {
  201. linecount++;
  202. # ifdef LINE_DIRECTIVE
  203. nostartline = 0;
  204. # endif
  205. nonline = 1;
  206. }
  207. if (c == '\n') nonline = 0;
  208. if (strip_grammar) putchar(c);
  209. return c;
  210. }
  211. unput(c) {
  212. /*
  213. * "unread" c
  214. */
  215. backupc = c;
  216. }
  217. skipcomment(flag) {
  218. /*
  219. * Skip comment. If flag != 0, the comment is inside a fragment
  220. * of C-code, so keep it.
  221. */
  222. register int ch;
  223. int saved; /* line count on which comment starts */
  224. saved = linecount;
  225. if (input() != '*') error(linecount,"Illegal comment");
  226. if (flag) putc('*', fact);
  227. do {
  228. ch = input();
  229. if (flag) putc(ch, fact);
  230. while (ch == '*') {
  231. ch = input();
  232. if (flag) putc(ch, fact);
  233. if (ch == '/') return;
  234. }
  235. } while (ch != EOF);
  236. error(saved,"Comment does not terminate");
  237. }
  238. # ifdef LINE_DIRECTIVE
  239. STATIC
  240. linedirective() {
  241. /*
  242. * Read a line directive
  243. */
  244. register int ch;
  245. register int i;
  246. string s_error = "Illegal line directive";
  247. string store();
  248. register string c;
  249. do { /*
  250. * Skip to next digit
  251. * Do not skip newlines
  252. */
  253. ch = input();
  254. } while (ch != '\n' && c_class[ch] != ISDIG);
  255. if (ch == '\n') {
  256. error(linecount,s_error);
  257. return;
  258. }
  259. i = 0;
  260. do {
  261. i = i*10 + (ch - '0');
  262. ch = input();
  263. } while (c_class[ch] == ISDIG);
  264. while (ch != '\n' && ch != '"') ch = input();
  265. if (ch == '"') {
  266. c = ltext;
  267. do {
  268. *c++ = ch = input();
  269. } while (ch != '"' && ch != '\n');
  270. if (ch == '\n') {
  271. error(linecount,s_error);
  272. return;
  273. }
  274. *--c = '\0';
  275. do {
  276. ch = input();
  277. } while (ch != '\n');
  278. /*
  279. * Remember the file name
  280. */
  281. if (strcmp(f_input,ltext)) f_input = store(ltext);
  282. }
  283. linecount = i;
  284. }
  285. # endif
  286. STATIC string
  287. vallookup(s) {
  288. /*
  289. * Look up the keyword that has token number s
  290. */
  291. register p_keyw p = resword;
  292. while (p->w_value) {
  293. if (p->w_value == s) return p->w_word;
  294. p++;
  295. }
  296. return 0;
  297. }
  298. STATIC string
  299. cpy(s,p,inserted) register string p; {
  300. /*
  301. * Create a piece of error message for token s and put it at p.
  302. * inserted = 0 if the token s was deleted (in which case we have
  303. * attributes), else it was inserted
  304. */
  305. register string t = 0;
  306. switch(s) {
  307. case C_IDENT :
  308. if (!inserted) t = lextoken.t_string;
  309. else t = "identifier";
  310. break;
  311. case C_NUMBER :
  312. t = "number";
  313. break;
  314. case C_LITERAL :
  315. if (!inserted) {
  316. *p++ = '\'';
  317. t = lextoken.t_string;
  318. break;
  319. }
  320. t = "literal";
  321. break;
  322. case EOFILE :
  323. t = "endoffile";
  324. break;
  325. }
  326. if (!t && (t = vallookup(s))) {
  327. *p++ = '%';
  328. }
  329. if (t) { /*
  330. * We have a string for the token. Copy it
  331. */
  332. while (*t) *p++ = *t++;
  333. if (s == C_LITERAL && !inserted) {
  334. *p++ = '\'';
  335. }
  336. return p;
  337. }
  338. /*
  339. * The token is a literal
  340. */
  341. *p++ = '\'';
  342. if (s >= 040 && s <= 0176) *p++ = s;
  343. else {
  344. *p++ = '\\';
  345. switch(s) {
  346. case '\b' : *p++ = 'b'; break;
  347. case '\f' : *p++ = 'f'; break;
  348. case '\n' : *p++ = 'n'; break;
  349. case '\r' : *p++ = 'r'; break;
  350. case '\t' : *p++ = 't'; break;
  351. default : *p++='0'+((s&0377)>>6); *p++='0'+((s>>3)&07);
  352. *p++='0'+(s&07);
  353. }
  354. }
  355. *p++ = '\'';
  356. return p;
  357. }
  358. LLmessage(d) {
  359. /*
  360. * d is either 0, in which case the current token has been deleted,
  361. * or non-zero, in which case it represents a token that is inserted
  362. * before the current token
  363. */
  364. register string s,t;
  365. char buf[128];
  366. nerrors++;
  367. s = buf;
  368. if (d == 0) {
  369. s = cpy(LLsymb,s,0);
  370. t = " deleted";
  371. do *s++ = *t; while (*t++);
  372. } else {
  373. s = cpy(d,s,1);
  374. t = " inserted in front of ";
  375. do *s++ = *t++; while (*t);
  376. s = cpy(LLsymb,s,0);
  377. *s = '\0';
  378. }
  379. error(linecount, "%s", buf);
  380. /* Don't change this line to
  381. * error(linecount, buf).
  382. * The string in "buf" might contain '%' ...
  383. */
  384. if (d) { /*
  385. * Save the current token and make up some
  386. * attributes for the inserted token
  387. */
  388. savedtok = lextoken;
  389. savedtok.t_tokno = LLsymb;
  390. if (d == C_IDENT) lextoken.t_string = "dummy_identifier";
  391. else if (d == C_LITERAL) lextoken.t_string = "dummy_literal";
  392. else if (d == C_NUMBER) lextoken.t_num = 1;
  393. }
  394. }
  395. }