Main.cpp 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. /*
  2. * Copyright 2017 Google Inc.
  3. *
  4. * Use of this source code is governed by a BSD-style license that can be
  5. * found in the LICENSE file.
  6. */
  7. #include "src/sksl/lex/NFAtoDFA.h"
  8. #include "src/sksl/lex/RegexParser.h"
  9. #include <fstream>
  10. #include <sstream>
  11. #include <string>
  12. /**
  13. * Processes a .lex file and produces .h and .cpp files which implement a lexical analyzer. The .lex
  14. * file is a text file with one token definition per line. Each line is of the form:
  15. * <TOKEN_NAME> = <pattern>
  16. * where <pattern> is either a regular expression (e.g [0-9]) or a double-quoted literal string.
  17. */
  18. static constexpr const char* HEADER =
  19. "/*\n"
  20. " * Copyright 2017 Google Inc.\n"
  21. " *\n"
  22. " * Use of this source code is governed by a BSD-style license that can be\n"
  23. " * found in the LICENSE file.\n"
  24. " */\n"
  25. "/*****************************************************************************************\n"
  26. " ******************** This file was generated by sksllex. Do not edit. *******************\n"
  27. " *****************************************************************************************/\n";
  28. void writeH(const DFA& dfa, const char* lexer, const char* token,
  29. const std::vector<std::string>& tokens, const char* hPath) {
  30. std::ofstream out(hPath);
  31. SkASSERT(out.good());
  32. out << HEADER;
  33. out << "#ifndef SKSL_" << lexer << "\n";
  34. out << "#define SKSL_" << lexer << "\n";
  35. out << "#include <cstddef>\n";
  36. out << "#include <cstdint>\n";
  37. out << "namespace SkSL {\n";
  38. out << "\n";
  39. out << "struct " << token << " {\n";
  40. out << " enum Kind {\n";
  41. for (const std::string& t : tokens) {
  42. out << " #undef " << t << "\n";
  43. out << " " << t << ",\n";
  44. }
  45. out << " };\n";
  46. out << "\n";
  47. out << " " << token << "()\n";
  48. out << " : fKind(Kind::INVALID)\n";
  49. out << " , fOffset(-1)\n";
  50. out << " , fLength(-1) {}\n";
  51. out << "\n";
  52. out << " " << token << "(Kind kind, int32_t offset, int32_t length)\n";
  53. out << " : fKind(kind)\n";
  54. out << " , fOffset(offset)\n";
  55. out << " , fLength(length) {}\n";
  56. out << "\n";
  57. out << " Kind fKind;\n";
  58. out << " int fOffset;\n";
  59. out << " int fLength;\n";
  60. out << "};\n";
  61. out << "\n";
  62. out << "class " << lexer << " {\n";
  63. out << "public:\n";
  64. out << " void start(const char* text, int32_t length) {\n";
  65. out << " fText = text;\n";
  66. out << " fLength = length;\n";
  67. out << " fOffset = 0;\n";
  68. out << " }\n";
  69. out << "\n";
  70. out << " " << token << " next();\n";
  71. out << "\n";
  72. out << "private:\n";
  73. out << " const char* fText;\n";
  74. out << " int32_t fLength;\n";
  75. out << " int32_t fOffset;\n";
  76. out << "};\n";
  77. out << "\n";
  78. out << "} // namespace\n";
  79. out << "#endif\n";
  80. }
  81. void writeCPP(const DFA& dfa, const char* lexer, const char* token, const char* include,
  82. const char* cppPath) {
  83. std::ofstream out(cppPath);
  84. SkASSERT(out.good());
  85. out << HEADER;
  86. out << "#include \"" << include << "\"\n";
  87. out << "\n";
  88. out << "namespace SkSL {\n";
  89. out << "\n";
  90. size_t states = 0;
  91. for (const auto& row : dfa.fTransitions) {
  92. states = std::max(states, row.size());
  93. }
  94. // arbitrarily-chosen character which is greater than START_CHAR and should not appear in actual
  95. // input
  96. out << "static const uint8_t INVALID_CHAR = 18;";
  97. out << "static int8_t mappings[" << dfa.fCharMappings.size() << "] = {\n ";
  98. const char* separator = "";
  99. for (int m : dfa.fCharMappings) {
  100. out << separator << std::to_string(m);
  101. separator = ", ";
  102. }
  103. out << "\n};\n";
  104. out << "static int16_t transitions[" << dfa.fTransitions.size() << "][" << states << "] = {\n";
  105. for (size_t c = 0; c < dfa.fTransitions.size(); ++c) {
  106. out << " {";
  107. for (size_t j = 0; j < states; ++j) {
  108. if ((size_t) c < dfa.fTransitions.size() && j < dfa.fTransitions[c].size()) {
  109. out << " " << dfa.fTransitions[c][j] << ",";
  110. } else {
  111. out << " 0,";
  112. }
  113. }
  114. out << " },\n";
  115. }
  116. out << "};\n";
  117. out << "\n";
  118. out << "static int8_t accepts[" << states << "] = {";
  119. for (size_t i = 0; i < states; ++i) {
  120. if (i < dfa.fAccepts.size()) {
  121. out << " " << dfa.fAccepts[i] << ",";
  122. } else {
  123. out << " " << INVALID << ",";
  124. }
  125. }
  126. out << " };\n";
  127. out << "\n";
  128. out << token << " " << lexer << "::next() {\n";
  129. out << " // note that we cheat here: normally a lexer needs to worry about the case\n";
  130. out << " // where a token has a prefix which is not itself a valid token - for instance, \n";
  131. out << " // maybe we have a valid token 'while', but 'w', 'wh', etc. are not valid\n";
  132. out << " // tokens. Our grammar doesn't have this property, so we can simplify the logic\n";
  133. out << " // a bit.\n";
  134. out << " int32_t startOffset = fOffset;\n";
  135. out << " if (startOffset == fLength) {\n";
  136. out << " return " << token << "(" << token << "::END_OF_FILE, startOffset, 0);\n";
  137. out << " }\n";
  138. out << " int16_t state = 1;\n";
  139. out << " for (;;) {\n";
  140. out << " if (fOffset >= fLength) {\n";
  141. out << " if (accepts[state] == -1) {\n";
  142. out << " return Token(Token::END_OF_FILE, startOffset, 0);\n";
  143. out << " }\n";
  144. out << " break;\n";
  145. out << " }\n";
  146. out << " uint8_t c = (uint8_t) fText[fOffset];";
  147. out << " if (c <= 8 || c >= " << dfa.fCharMappings.size() << ") {";
  148. out << " c = INVALID_CHAR;";
  149. out << " }";
  150. out << " int16_t newState = transitions[mappings[c]][state];\n";
  151. out << " if (!newState) {\n";
  152. out << " break;\n";
  153. out << " }\n";
  154. out << " state = newState;";
  155. out << " ++fOffset;\n";
  156. out << " }\n";
  157. out << " Token::Kind kind = (" << token << "::Kind) accepts[state];\n";
  158. out << " return " << token << "(kind, startOffset, fOffset - startOffset);\n";
  159. out << "}\n";
  160. out << "\n";
  161. out << "} // namespace\n";
  162. }
  163. void process(const char* inPath, const char* lexer, const char* token, const char* hPath,
  164. const char* cppPath) {
  165. NFA nfa;
  166. std::vector<std::string> tokens;
  167. tokens.push_back("END_OF_FILE");
  168. std::string line;
  169. std::ifstream in(inPath);
  170. while (std::getline(in, line)) {
  171. std::istringstream split(line);
  172. std::string name, delimiter, pattern;
  173. if (split >> name >> delimiter >> pattern) {
  174. SkASSERT(split.eof());
  175. SkASSERT(name != "");
  176. SkASSERT(delimiter == "=");
  177. SkASSERT(pattern != "");
  178. tokens.push_back(name);
  179. if (pattern[0] == '"') {
  180. SkASSERT(pattern.size() > 2 && pattern[pattern.size() - 1] == '"');
  181. RegexNode node = RegexNode(RegexNode::kChar_Kind, pattern[1]);
  182. for (size_t i = 2; i < pattern.size() - 1; ++i) {
  183. node = RegexNode(RegexNode::kConcat_Kind, node,
  184. RegexNode(RegexNode::kChar_Kind, pattern[i]));
  185. }
  186. nfa.addRegex(node);
  187. }
  188. else {
  189. nfa.addRegex(RegexParser().parse(pattern));
  190. }
  191. }
  192. }
  193. NFAtoDFA converter(&nfa);
  194. DFA dfa = converter.convert();
  195. writeH(dfa, lexer, token, tokens, hPath);
  196. writeCPP(dfa, lexer, token, (std::string("src/sksl/SkSL") + lexer + ".h").c_str(), cppPath);
  197. }
  198. int main(int argc, const char** argv) {
  199. if (argc != 6) {
  200. printf("usage: sksllex <input.lex> <lexername> <tokenname> <output.h> <output.cpp>\n");
  201. exit(1);
  202. }
  203. process(argv[1], argv[2], argv[3], argv[4], argv[5]);
  204. return 0;
  205. }