flexdoc.1 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446
  1. .TH FLEX 1 "26 May 1990" "Version 2.3"
  2. .SH NAME
  3. flex - fast lexical analyzer generator
  4. .SH SYNOPSIS
  5. .B flex
  6. .B [-bcdfinpstvFILT8 -C[efmF] -Sskeleton]
  7. .I [filename ...]
  8. .SH DESCRIPTION
  9. .I flex
  10. is a tool for generating
  11. .I scanners:
  12. programs which recognized lexical patterns in text.
  13. .I flex
  14. reads
  15. the given input files, or its standard input if no file names are given,
  16. for a description of a scanner to generate. The description is in
  17. the form of pairs
  18. of regular expressions and C code, called
  19. .I rules. flex
  20. generates as output a C source file,
  21. .B lex.yy.c,
  22. which defines a routine
  23. .B yylex().
  24. This file is compiled and linked with the
  25. .B -lfl
  26. library to produce an executable. When the executable is run,
  27. it analyzes its input for occurrences
  28. of the regular expressions. Whenever it finds one, it executes
  29. the corresponding C code.
  30. .SH SOME SIMPLE EXAMPLES
  31. .LP
  32. First some simple examples to get the flavor of how one uses
  33. .I flex.
  34. The following
  35. .I flex
  36. input specifies a scanner which whenever it encounters the string
  37. "username" will replace it with the user's login name:
  38. .nf
  39. %%
  40. username printf( "%s", getlogin() );
  41. .fi
  42. By default, any text not matched by a
  43. .I flex
  44. scanner
  45. is copied to the output, so the net effect of this scanner is
  46. to copy its input file to its output with each occurrence
  47. of "username" expanded.
  48. In this input, there is just one rule. "username" is the
  49. .I pattern
  50. and the "printf" is the
  51. .I action.
  52. The "%%" marks the beginning of the rules.
  53. .LP
  54. Here's another simple example:
  55. .nf
  56. int num_lines = 0, num_chars = 0;
  57. %%
  58. \\n ++num_lines; ++num_chars;
  59. . ++num_chars;
  60. %%
  61. main()
  62. {
  63. yylex();
  64. printf( "# of lines = %d, # of chars = %d\\n",
  65. num_lines, num_chars );
  66. }
  67. .fi
  68. This scanner counts the number of characters and the number
  69. of lines in its input (it produces no output other than the
  70. final report on the counts). The first line
  71. declares two globals, "num_lines" and "num_chars", which are accessible
  72. both inside
  73. .B yylex()
  74. and in the
  75. .B main()
  76. routine declared after the second "%%". There are two rules, one
  77. which matches a newline ("\\n") and increments both the line count and
  78. the character count, and one which matches any character other than
  79. a newline (indicated by the "." regular expression).
  80. .LP
  81. A somewhat more complicated example:
  82. .nf
  83. /* scanner for a toy Pascal-like language */
  84. %{
  85. /* need this for the call to atof() below */
  86. #include <math.h>
  87. %}
  88. DIGIT [0-9]
  89. ID [a-z][a-z0-9]*
  90. %%
  91. {DIGIT}+ {
  92. printf( "An integer: %s (%d)\\n", yytext,
  93. atoi( yytext ) );
  94. }
  95. {DIGIT}+"."{DIGIT}* {
  96. printf( "A float: %s (%g)\\n", yytext,
  97. atof( yytext ) );
  98. }
  99. if|then|begin|end|procedure|function {
  100. printf( "A keyword: %s\\n", yytext );
  101. }
  102. {ID} printf( "An identifier: %s\\n", yytext );
  103. "+"|"-"|"*"|"/" printf( "An operator: %s\\n", yytext );
  104. "{"[^}\\n]*"}" /* eat up one-line comments */
  105. [ \\t\\n]+ /* eat up whitespace */
  106. . printf( "Unrecognized character: %s\\n", yytext );
  107. %%
  108. main( argc, argv )
  109. int argc;
  110. char **argv;
  111. {
  112. ++argv, --argc; /* skip over program name */
  113. if ( argc > 0 )
  114. yyin = fopen( argv[0], "r" );
  115. else
  116. yyin = stdin;
  117. yylex();
  118. }
  119. .fi
  120. This is the beginnings of a simple scanner for a language like
  121. Pascal. It identifies different types of
  122. .I tokens
  123. and reports on what it has seen.
  124. .LP
  125. The details of this example will be explained in the following
  126. sections.
  127. .SH FORMAT OF THE INPUT FILE
  128. The
  129. .I flex
  130. input file consists of three sections, separated by a line with just
  131. .B %%
  132. in it:
  133. .nf
  134. definitions
  135. %%
  136. rules
  137. %%
  138. user code
  139. .fi
  140. The
  141. .I definitions
  142. section contains declarations of simple
  143. .I name
  144. definitions to simplify the scanner specification, and declarations of
  145. .I start conditions,
  146. which are explained in a later section.
  147. .LP
  148. Name definitions have the form:
  149. .nf
  150. name definition
  151. .fi
  152. The "name" is a word beginning with a letter or an underscore ('_')
  153. followed by zero or more letters, digits, '_', or '-' (dash).
  154. The definition is taken to begin at the first non-white-space character
  155. following the name and continuing to the end of the line.
  156. The definition can subsequently be referred to using "{name}", which
  157. will expand to "(definition)". For example,
  158. .nf
  159. DIGIT [0-9]
  160. ID [a-z][a-z0-9]*
  161. .fi
  162. defines "DIGIT" to be a regular expression which matches a
  163. single digit, and
  164. "ID" to be a regular expression which matches a letter
  165. followed by zero-or-more letters-or-digits.
  166. A subsequent reference to
  167. .nf
  168. {DIGIT}+"."{DIGIT}*
  169. .fi
  170. is identical to
  171. .nf
  172. ([0-9])+"."([0-9])*
  173. .fi
  174. and matches one-or-more digits followed by a '.' followed
  175. by zero-or-more digits.
  176. .LP
  177. The
  178. .I rules
  179. section of the
  180. .I flex
  181. input contains a series of rules of the form:
  182. .nf
  183. pattern action
  184. .fi
  185. where the pattern must be unindented and the action must begin
  186. on the same line.
  187. .LP
  188. See below for a further description of patterns and actions.
  189. .LP
  190. Finally, the user code section is simply copied to
  191. .B lex.yy.c
  192. verbatim.
  193. It is used for companion routines which call or are called
  194. by the scanner. The presence of this section is optional;
  195. if it is missing, the second
  196. .B %%
  197. in the input file may be skipped, too.
  198. .LP
  199. In the definitions and rules sections, any
  200. .I indented
  201. text or text enclosed in
  202. .B %{
  203. and
  204. .B %}
  205. is copied verbatim to the output (with the %{}'s removed).
  206. The %{}'s must appear unindented on lines by themselves.
  207. .LP
  208. In the rules section,
  209. any indented or %{} text appearing before the
  210. first rule may be used to declare variables
  211. which are local to the scanning routine and (after the declarations)
  212. code which is to be executed whenever the scanning routine is entered.
  213. Other indented or %{} text in the rule section is still copied to the output,
  214. but its meaning is not well-defined and it may well cause compile-time
  215. errors (this feature is present for
  216. .I POSIX
  217. compliance; see below for other such features).
  218. .LP
  219. In the definitions section, an unindented comment (i.e., a line
  220. beginning with "/*") is also copied verbatim to the output up
  221. to the next "*/". Also, any line in the definitions section
  222. beginning with '#' is ignored, though this style of comment is
  223. deprecated and may go away in the future.
  224. .SH PATTERNS
  225. The patterns in the input are written using an extended set of regular
  226. expressions. These are:
  227. .nf
  228. x match the character 'x'
  229. . any character except newline
  230. [xyz] a "character class"; in this case, the pattern
  231. matches either an 'x', a 'y', or a 'z'
  232. [abj-oZ] a "character class" with a range in it; matches
  233. an 'a', a 'b', any letter from 'j' through 'o',
  234. or a 'Z'
  235. [^A-Z] a "negated character class", i.e., any character
  236. but those in the class. In this case, any
  237. character EXCEPT an uppercase letter.
  238. [^A-Z\\n] any character EXCEPT an uppercase letter or
  239. a newline
  240. r* zero or more r's, where r is any regular expression
  241. r+ one or more r's
  242. r? zero or one r's (that is, "an optional r")
  243. r{2,5} anywhere from two to five r's
  244. r{2,} two or more r's
  245. r{4} exactly 4 r's
  246. {name} the expansion of the "name" definition
  247. (see above)
  248. "[xyz]\\"foo"
  249. the literal string: [xyz]"foo
  250. \\X if X is an 'a', 'b', 'f', 'n', 'r', 't', or 'v',
  251. then the ANSI-C interpretation of \\x.
  252. Otherwise, a literal 'X' (used to escape
  253. operators such as '*')
  254. \\123 the character with octal value 123
  255. \\x2a the character with hexadecimal value 2a
  256. (r) match an r; parentheses are used to override
  257. precedence (see below)
  258. rs the regular expression r followed by the
  259. regular expression s; called "concatenation"
  260. r|s either an r or an s
  261. r/s an r but only if it is followed by an s. The
  262. s is not part of the matched text. This type
  263. of pattern is called as "trailing context".
  264. ^r an r, but only at the beginning of a line
  265. r$ an r, but only at the end of a line. Equivalent
  266. to "r/\\n".
  267. <s>r an r, but only in start condition s (see
  268. below for discussion of start conditions)
  269. <s1,s2,s3>r
  270. same, but in any of start conditions s1,
  271. s2, or s3
  272. <<EOF>> an end-of-file
  273. <s1,s2><<EOF>>
  274. an end-of-file when in start condition s1 or s2
  275. .fi
  276. The regular expressions listed above are grouped according to
  277. precedence, from highest precedence at the top to lowest at the bottom.
  278. Those grouped together have equal precedence. For example,
  279. .nf
  280. foo|bar*
  281. .fi
  282. is the same as
  283. .nf
  284. (foo)|(ba(r*))
  285. .fi
  286. since the '*' operator has higher precedence than concatenation,
  287. and concatenation higher than alternation ('|'). This pattern
  288. therefore matches
  289. .I either
  290. the string "foo"
  291. .I or
  292. the string "ba" followed by zero-or-more r's.
  293. To match "foo" or zero-or-more "bar"'s, use:
  294. .nf
  295. foo|(bar)*
  296. .fi
  297. and to match zero-or-more "foo"'s-or-"bar"'s:
  298. .nf
  299. (foo|bar)*
  300. .fi
  301. .LP
  302. Some notes on patterns:
  303. .IP -
  304. A negated character class such as the example "[^A-Z]"
  305. above
  306. .I will match a newline
  307. unless "\\n" (or an equivalent escape sequence) is one of the
  308. characters explicitly present in the negated character class
  309. (e.g., "[^A-Z\\n]"). This is unlike how many other regular
  310. expression tools treat negated character classes, but unfortunately
  311. the inconsistency is historically entrenched.
  312. Matching newlines means that a pattern like [^"]* can match an entire
  313. input (overflowing the scanner's input buffer) unless there's another
  314. quote in the input.
  315. .IP -
  316. A rule can have at most one instance of trailing context (the '/' operator
  317. or the '$' operator). The start condition, '^', and "<<EOF>>" patterns
  318. can only occur at the beginning of a pattern, and, as well as with '/' and '$',
  319. cannot be grouped inside parentheses. A '^' which does not occur at
  320. the beginning of a rule or a '$' which does not occur at the end of
  321. a rule loses its special properties and is treated as a normal character.
  322. .IP
  323. The following are illegal:
  324. .nf
  325. foo/bar$
  326. <sc1>foo<sc2>bar
  327. .fi
  328. Note that the first of these, can be written "foo/bar\\n".
  329. .IP
  330. The following will result in '$' or '^' being treated as a normal character:
  331. .nf
  332. foo|(bar$)
  333. foo|^bar
  334. .fi
  335. If what's wanted is a "foo" or a bar-followed-by-a-newline, the following
  336. could be used (the special '|' action is explained below):
  337. .nf
  338. foo |
  339. bar$ /* action goes here */
  340. .fi
  341. A similar trick will work for matching a foo or a
  342. bar-at-the-beginning-of-a-line.
  343. .SH HOW THE INPUT IS MATCHED
  344. When the generated scanner is run, it analyzes its input looking
  345. for strings which match any of its patterns. If it finds more than
  346. one match, it takes the one matching the most text (for trailing
  347. context rules, this includes the length of the trailing part, even
  348. though it will then be returned to the input). If it finds two
  349. or more matches of the same length, the
  350. rule listed first in the
  351. .I flex
  352. input file is chosen.
  353. .LP
  354. Once the match is determined, the text corresponding to the match
  355. (called the
  356. .I token)
  357. is made available in the global character pointer
  358. .B yytext,
  359. and its length in the global integer
  360. .B yyleng.
  361. The
  362. .I action
  363. corresponding to the matched pattern is then executed (a more
  364. detailed description of actions follows), and then the remaining
  365. input is scanned for another match.
  366. .LP
  367. If no match is found, then the
  368. .I default rule
  369. is executed: the next character in the input is considered matched and
  370. copied to the standard output. Thus, the simplest legal
  371. .I flex
  372. input is:
  373. .nf
  374. %%
  375. .fi
  376. which generates a scanner that simply copies its input (one character
  377. at a time) to its output.
  378. .SH ACTIONS
  379. Each pattern in a rule has a corresponding action, which can be any
  380. arbitrary C statement. The pattern ends at the first non-escaped
  381. whitespace character; the remainder of the line is its action. If the
  382. action is empty, then when the pattern is matched the input token
  383. is simply discarded. For example, here is the specification for a program
  384. which deletes all occurrences of "zap me" from its input:
  385. .nf
  386. %%
  387. "zap me"
  388. .fi
  389. (It will copy all other characters in the input to the output since
  390. they will be matched by the default rule.)
  391. .LP
  392. Here is a program which compresses multiple blanks and tabs down to
  393. a single blank, and throws away whitespace found at the end of a line:
  394. .nf
  395. %%
  396. [ \\t]+ putchar( ' ' );
  397. [ \\t]+$ /* ignore this token */
  398. .fi
  399. .LP
  400. If the action contains a '{', then the action spans till the balancing '}'
  401. is found, and the action may cross multiple lines.
  402. .I flex
  403. knows about C strings and comments and won't be fooled by braces found
  404. within them, but also allows actions to begin with
  405. .B %{
  406. and will consider the action to be all the text up to the next
  407. .B %}
  408. (regardless of ordinary braces inside the action).
  409. .LP
  410. An action consisting solely of a vertical bar ('|') means "same as
  411. the action for the next rule." See below for an illustration.
  412. .LP
  413. Actions can include arbitrary C code, including
  414. .B return
  415. statements to return a value to whatever routine called
  416. .B yylex().
  417. Each time
  418. .B yylex()
  419. is called it continues processing tokens from where it last left
  420. off until it either reaches
  421. the end of the file or executes a return. Once it reaches an end-of-file,
  422. however, then any subsequent call to
  423. .B yylex()
  424. will simply immediately return, unless
  425. .B yyrestart()
  426. is first called (see below).
  427. .LP
  428. Actions are not allowed to modify yytext or yyleng.
  429. .LP
  430. There are a number of special directives which can be included within
  431. an action:
  432. .IP -
  433. .B ECHO
  434. copies yytext to the scanner's output.
  435. .IP -
  436. .B BEGIN
  437. followed by the name of a start condition places the scanner in the
  438. corresponding start condition (see below).
  439. .IP -
  440. .B REJECT
  441. directs the scanner to proceed on to the "second best" rule which matched the
  442. input (or a prefix of the input). The rule is chosen as described
  443. above in "How the Input is Matched", and
  444. .B yytext
  445. and
  446. .B yyleng
  447. set up appropriately.
  448. It may either be one which matched as much text
  449. as the originally chosen rule but came later in the
  450. .I flex
  451. input file, or one which matched less text.
  452. For example, the following will both count the
  453. words in the input and call the routine special() whenever "frob" is seen:
  454. .nf
  455. int word_count = 0;
  456. %%
  457. frob special(); REJECT;
  458. [^ \\t\\n]+ ++word_count;
  459. .fi
  460. Without the
  461. .B REJECT,
  462. any "frob"'s in the input would not be counted as words, since the
  463. scanner normally executes only one action per token.
  464. Multiple
  465. .B REJECT's
  466. are allowed, each one finding the next best choice to the currently
  467. active rule. For example, when the following scanner scans the token
  468. "abcd", it will write "abcdabcaba" to the output:
  469. .nf
  470. %%
  471. a |
  472. ab |
  473. abc |
  474. abcd ECHO; REJECT;
  475. .|\\n /* eat up any unmatched character */
  476. .fi
  477. (The first three rules share the fourth's action since they use
  478. the special '|' action.)
  479. .B REJECT
  480. is a particularly expensive feature in terms scanner performance;
  481. if it is used in
  482. .I any
  483. of the scanner's actions it will slow down
  484. .I all
  485. of the scanner's matching. Furthermore,
  486. .B REJECT
  487. cannot be used with the
  488. .I -f
  489. or
  490. .I -F
  491. options (see below).
  492. .IP
  493. Note also that unlike the other special actions,
  494. .B REJECT
  495. is a
  496. .I branch;
  497. code immediately following it in the action will
  498. .I not
  499. be executed.
  500. .IP -
  501. .B yymore()
  502. tells the scanner that the next time it matches a rule, the corresponding
  503. token should be
  504. .I appended
  505. onto the current value of
  506. .B yytext
  507. rather than replacing it. For example, given the input "mega-kludge"
  508. the following will write "mega-mega-kludge" to the output:
  509. .nf
  510. %%
  511. mega- ECHO; yymore();
  512. kludge ECHO;
  513. .fi
  514. First "mega-" is matched and echoed to the output. Then "kludge"
  515. is matched, but the previous "mega-" is still hanging around at the
  516. beginning of
  517. .B yytext
  518. so the
  519. .B ECHO
  520. for the "kludge" rule will actually write "mega-kludge".
  521. The presence of
  522. .B yymore()
  523. in the scanner's action entails a minor performance penalty in the
  524. scanner's matching speed.
  525. .IP -
  526. .B yyless(n)
  527. returns all but the first
  528. .I n
  529. characters of the current token back to the input stream, where they
  530. will be rescanned when the scanner looks for the next match.
  531. .B yytext
  532. and
  533. .B yyleng
  534. are adjusted appropriately (e.g.,
  535. .B yyleng
  536. will now be equal to
  537. .I n
  538. ). For example, on the input "foobar" the following will write out
  539. "foobarbar":
  540. .nf
  541. %%
  542. foobar ECHO; yyless(3);
  543. [a-z]+ ECHO;
  544. .fi
  545. An argument of 0 to
  546. .B yyless
  547. will cause the entire current input string to be scanned again. Unless you've
  548. changed how the scanner will subsequently process its input (using
  549. .B BEGIN,
  550. for example), this will result in an endless loop.
  551. .IP -
  552. .B unput(c)
  553. puts the character
  554. .I c
  555. back onto the input stream. It will be the next character scanned.
  556. The following action will take the current token and cause it
  557. to be rescanned enclosed in parentheses.
  558. .nf
  559. {
  560. int i;
  561. unput( ')' );
  562. for ( i = yyleng - 1; i >= 0; --i )
  563. unput( yytext[i] );
  564. unput( '(' );
  565. }
  566. .fi
  567. Note that since each
  568. .B unput()
  569. puts the given character back at the
  570. .I beginning
  571. of the input stream, pushing back strings must be done back-to-front.
  572. .IP -
  573. .B input()
  574. reads the next character from the input stream. For example,
  575. the following is one way to eat up C comments:
  576. .nf
  577. %%
  578. "/*" {
  579. register int c;
  580. for ( ; ; )
  581. {
  582. while ( (c = input()) != '*' &&
  583. c != EOF )
  584. ; /* eat up text of comment */
  585. if ( c == '*' )
  586. {
  587. while ( (c = input()) == '*' )
  588. ;
  589. if ( c == '/' )
  590. break; /* found the end */
  591. }
  592. if ( c == EOF )
  593. {
  594. error( "EOF in comment" );
  595. break;
  596. }
  597. }
  598. }
  599. .fi
  600. (Note that if the scanner is compiled using
  601. .B C++,
  602. then
  603. .B input()
  604. is instead referred to as
  605. .B yyinput(),
  606. in order to avoid a name clash with the
  607. .B C++
  608. stream by the name of
  609. .I input.)
  610. .IP -
  611. .B yyterminate()
  612. can be used in lieu of a return statement in an action. It terminates
  613. the scanner and returns a 0 to the scanner's caller, indicating "all done".
  614. Subsequent calls to the scanner will immediately return unless preceded
  615. by a call to
  616. .B yyrestart()
  617. (see below).
  618. By default,
  619. .B yyterminate()
  620. is also called when an end-of-file is encountered. It is a macro and
  621. may be redefined.
  622. .SH THE GENERATED SCANNER
  623. The output of
  624. .I flex
  625. is the file
  626. .B lex.yy.c,
  627. which contains the scanning routine
  628. .B yylex(),
  629. a number of tables used by it for matching tokens, and a number
  630. of auxiliary routines and macros. By default,
  631. .B yylex()
  632. is declared as follows:
  633. .nf
  634. int yylex()
  635. {
  636. ... various definitions and the actions in here ...
  637. }
  638. .fi
  639. (If your environment supports function prototypes, then it will
  640. be "int yylex( void )".) This definition may be changed by redefining
  641. the "YY_DECL" macro. For example, you could use:
  642. .nf
  643. #undef YY_DECL
  644. #define YY_DECL float lexscan( a, b ) float a, b;
  645. .fi
  646. to give the scanning routine the name
  647. .I lexscan,
  648. returning a float, and taking two floats as arguments. Note that
  649. if you give arguments to the scanning routine using a
  650. K&R-style/non-prototyped function declaration, you must terminate
  651. the definition with a semi-colon (;).
  652. .LP
  653. Whenever
  654. .B yylex()
  655. is called, it scans tokens from the global input file
  656. .I yyin
  657. (which defaults to stdin). It continues until it either reaches
  658. an end-of-file (at which point it returns the value 0) or
  659. one of its actions executes a
  660. .I return
  661. statement.
  662. In the former case, when called again the scanner will immediately
  663. return unless
  664. .B yyrestart()
  665. is called to point
  666. .I yyin
  667. at the new input file. (
  668. .B yyrestart()
  669. takes one argument, a
  670. .B FILE *
  671. pointer.)
  672. In the latter case (i.e., when an action
  673. executes a return), the scanner may then be called again and it
  674. will resume scanning where it left off.
  675. .LP
  676. By default (and for purposes of efficiency), the scanner uses
  677. block-reads rather than simple
  678. .I getc()
  679. calls to read characters from
  680. .I yyin.
  681. The nature of how it gets its input can be controlled by redefining the
  682. .B YY_INPUT
  683. macro.
  684. YY_INPUT's calling sequence is "YY_INPUT(buf,result,max_size)". Its
  685. action is to place up to
  686. .I max_size
  687. characters in the character array
  688. .I buf
  689. and return in the integer variable
  690. .I result
  691. either the
  692. number of characters read or the constant YY_NULL (0 on Unix systems)
  693. to indicate EOF. The default YY_INPUT reads from the
  694. global file-pointer "yyin".
  695. .LP
  696. A sample redefinition of YY_INPUT (in the definitions
  697. section of the input file):
  698. .nf
  699. %{
  700. #undef YY_INPUT
  701. #define YY_INPUT(buf,result,max_size) \\
  702. { \\
  703. int c = getchar(); \\
  704. result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \\
  705. }
  706. %}
  707. .fi
  708. This definition will change the input processing to occur
  709. one character at a time.
  710. .LP
  711. You also can add in things like keeping track of the
  712. input line number this way; but don't expect your scanner to
  713. go very fast.
  714. .LP
  715. When the scanner receives an end-of-file indication from YY_INPUT,
  716. it then checks the
  717. .B yywrap()
  718. function. If
  719. .B yywrap()
  720. returns false (zero), then it is assumed that the
  721. function has gone ahead and set up
  722. .I yyin
  723. to point to another input file, and scanning continues. If it returns
  724. true (non-zero), then the scanner terminates, returning 0 to its
  725. caller.
  726. .LP
  727. The default
  728. .B yywrap()
  729. always returns 1. Presently, to redefine it you must first
  730. "#undef yywrap", as it is currently implemented as a macro. As indicated
  731. by the hedging in the previous sentence, it may be changed to
  732. a true function in the near future.
  733. .LP
  734. The scanner writes its
  735. .B ECHO
  736. output to the
  737. .I yyout
  738. global (default, stdout), which may be redefined by the user simply
  739. by assigning it to some other
  740. .B FILE
  741. pointer.
  742. .SH START CONDITIONS
  743. .I flex
  744. provides a mechanism for conditionally activating rules. Any rule
  745. whose pattern is prefixed with "<sc>" will only be active when
  746. the scanner is in the start condition named "sc". For example,
  747. .nf
  748. <STRING>[^"]* { /* eat up the string body ... */
  749. ...
  750. }
  751. .fi
  752. will be active only when the scanner is in the "STRING" start
  753. condition, and
  754. .nf
  755. <INITIAL,STRING,QUOTE>\\. { /* handle an escape ... */
  756. ...
  757. }
  758. .fi
  759. will be active only when the current start condition is
  760. either "INITIAL", "STRING", or "QUOTE".
  761. .LP
  762. Start conditions
  763. are declared in the definitions (first) section of the input
  764. using unindented lines beginning with either
  765. .B %s
  766. or
  767. .B %x
  768. followed by a list of names.
  769. The former declares
  770. .I inclusive
  771. start conditions, the latter
  772. .I exclusive
  773. start conditions. A start condition is activated using the
  774. .B BEGIN
  775. action. Until the next
  776. .B BEGIN
  777. action is executed, rules with the given start
  778. condition will be active and
  779. rules with other start conditions will be inactive.
  780. If the start condition is
  781. .I inclusive,
  782. then rules with no start conditions at all will also be active.
  783. If it is
  784. .I exclusive,
  785. then
  786. .I only
  787. rules qualified with the start condition will be active.
  788. A set of rules contingent on the same exclusive start condition
  789. describe a scanner which is independent of any of the other rules in the
  790. .I flex
  791. input. Because of this,
  792. exclusive start conditions make it easy to specify "mini-scanners"
  793. which scan portions of the input that are syntactically different
  794. from the rest (e.g., comments).
  795. .LP
  796. If the distinction between inclusive and exclusive start conditions
  797. is still a little vague, here's a simple example illustrating the
  798. connection between the two. The set of rules:
  799. .nf
  800. %s example
  801. %%
  802. <example>foo /* do something */
  803. .fi
  804. is equivalent to
  805. .nf
  806. %x example
  807. %%
  808. <INITIAL,example>foo /* do something */
  809. .fi
  810. .LP
  811. The default rule (to
  812. .B ECHO
  813. any unmatched character) remains active in start conditions.
  814. .LP
  815. .B BEGIN(0)
  816. returns to the original state where only the rules with
  817. no start conditions are active. This state can also be
  818. referred to as the start-condition "INITIAL", so
  819. .B BEGIN(INITIAL)
  820. is equivalent to
  821. .B BEGIN(0).
  822. (The parentheses around the start condition name are not required but
  823. are considered good style.)
  824. .LP
  825. .B BEGIN
  826. actions can also be given as indented code at the beginning
  827. of the rules section. For example, the following will cause
  828. the scanner to enter the "SPECIAL" start condition whenever
  829. .I yylex()
  830. is called and the global variable
  831. .I enter_special
  832. is true:
  833. .nf
  834. int enter_special;
  835. %x SPECIAL
  836. %%
  837. if ( enter_special )
  838. BEGIN(SPECIAL);
  839. <SPECIAL>blahblahblah
  840. ...more rules follow...
  841. .fi
  842. .LP
  843. To illustrate the uses of start conditions,
  844. here is a scanner which provides two different interpretations
  845. of a string like "123.456". By default it will treat it as
  846. as three tokens, the integer "123", a dot ('.'), and the integer "456".
  847. But if the string is preceded earlier in the line by the string
  848. "expect-floats"
  849. it will treat it as a single token, the floating-point number
  850. 123.456:
  851. .nf
  852. %{
  853. #include <math.h>
  854. %}
  855. %s expect
  856. %%
  857. expect-floats BEGIN(expect);
  858. <expect>[0-9]+"."[0-9]+ {
  859. printf( "found a float, = %f\\n",
  860. atof( yytext ) );
  861. }
  862. <expect>\\n {
  863. /* that's the end of the line, so
  864. * we need another "expect-number"
  865. * before we'll recognize any more
  866. * numbers
  867. */
  868. BEGIN(INITIAL);
  869. }
  870. [0-9]+ {
  871. printf( "found an integer, = %d\\n",
  872. atoi( yytext ) );
  873. }
  874. "." printf( "found a dot\\n" );
  875. .fi
  876. Here is a scanner which recognizes (and discards) C comments while
  877. maintaining a count of the current input line.
  878. .nf
  879. %x comment
  880. %%
  881. int line_num = 1;
  882. "/*" BEGIN(comment);
  883. <comment>[^*\\n]* /* eat anything that's not a '*' */
  884. <comment>"*"+[^*/\\n]* /* eat up '*'s not followed by '/'s */
  885. <comment>\\n ++line_num;
  886. <comment>"*"+"/" BEGIN(INITIAL);
  887. .fi
  888. Note that start-conditions names are really integer values and
  889. can be stored as such. Thus, the above could be extended in the
  890. following fashion:
  891. .nf
  892. %x comment foo
  893. %%
  894. int line_num = 1;
  895. int comment_caller;
  896. "/*" {
  897. comment_caller = INITIAL;
  898. BEGIN(comment);
  899. }
  900. ...
  901. <foo>"/*" {
  902. comment_caller = foo;
  903. BEGIN(comment);
  904. }
  905. <comment>[^*\\n]* /* eat anything that's not a '*' */
  906. <comment>"*"+[^*/\\n]* /* eat up '*'s not followed by '/'s */
  907. <comment>\\n ++line_num;
  908. <comment>"*"+"/" BEGIN(comment_caller);
  909. .fi
  910. One can then implement a "stack" of start conditions using an
  911. array of integers. (It is likely that such stacks will become
  912. a full-fledged
  913. .I flex
  914. feature in the future.) Note, though, that
  915. start conditions do not have their own name-space; %s's and %x's
  916. declare names in the same fashion as #define's.
  917. .SH MULTIPLE INPUT BUFFERS
  918. Some scanners (such as those which support "include" files)
  919. require reading from several input streams. As
  920. .I flex
  921. scanners do a large amount of buffering, one cannot control
  922. where the next input will be read from by simply writing a
  923. .B YY_INPUT
  924. which is sensitive to the scanning context.
  925. .B YY_INPUT
  926. is only called when the scanner reaches the end of its buffer, which
  927. may be a long time after scanning a statement such as an "include"
  928. which requires switching the input source.
  929. .LP
  930. To negotiate these sorts of problems,
  931. .I flex
  932. provides a mechanism for creating and switching between multiple
  933. input buffers. An input buffer is created by using:
  934. .nf
  935. YY_BUFFER_STATE yy_create_buffer( FILE *file, int size )
  936. .fi
  937. which takes a
  938. .I FILE
  939. pointer and a size and creates a buffer associated with the given
  940. file and large enough to hold
  941. .I size
  942. characters (when in doubt, use
  943. .B YY_BUF_SIZE
  944. for the size). It returns a
  945. .B YY_BUFFER_STATE
  946. handle, which may then be passed to other routines:
  947. .nf
  948. void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer )
  949. .fi
  950. switches the scanner's input buffer so subsequent tokens will
  951. come from
  952. .I new_buffer.
  953. Note that
  954. .B yy_switch_to_buffer()
  955. may be used by yywrap() to sets things up for continued scanning, instead
  956. of opening a new file and pointing
  957. .I yyin
  958. at it.
  959. .nf
  960. void yy_delete_buffer( YY_BUFFER_STATE buffer )
  961. .fi
  962. is used to reclaim the storage associated with a buffer.
  963. .LP
  964. .B yy_new_buffer()
  965. is an alias for
  966. .B yy_create_buffer(),
  967. provided for compatibility with the C++ use of
  968. .I new
  969. and
  970. .I delete
  971. for creating and destroying dynamic objects.
  972. .LP
  973. Finally, the
  974. .B YY_CURRENT_BUFFER
  975. macro returns a
  976. .B YY_BUFFER_STATE
  977. handle to the current buffer.
  978. .LP
  979. Here is an example of using these features for writing a scanner
  980. which expands include files (the
  981. .B <<EOF>>
  982. feature is discussed below):
  983. .nf
  984. /* the "incl" state is used for picking up the name
  985. * of an include file
  986. */
  987. %x incl
  988. %{
  989. #define MAX_INCLUDE_DEPTH 10
  990. YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
  991. int include_stack_ptr = 0;
  992. %}
  993. %%
  994. include BEGIN(incl);
  995. [a-z]+ ECHO;
  996. [^a-z\\n]*\\n? ECHO;
  997. <incl>[ \\t]* /* eat the whitespace */
  998. <incl>[^ \\t\\n]+ { /* got the include file name */
  999. if ( include_stack_ptr >= MAX_INCLUDE_DEPTH )
  1000. {
  1001. fprintf( stderr, "Includes nested too deeply" );
  1002. exit( 1 );
  1003. }
  1004. include_stack[include_stack_ptr++] =
  1005. YY_CURRENT_BUFFER;
  1006. yyin = fopen( yytext, "r" );
  1007. if ( ! yyin )
  1008. error( ... );
  1009. yy_switch_to_buffer(
  1010. yy_create_buffer( yyin, YY_BUF_SIZE ) );
  1011. BEGIN(INITIAL);
  1012. }
  1013. <<EOF>> {
  1014. if ( --include_stack_ptr < 0 )
  1015. {
  1016. yyterminate();
  1017. }
  1018. else
  1019. yy_switch_to_buffer(
  1020. include_stack[include_stack_ptr] );
  1021. }
  1022. .fi
  1023. .SH END-OF-FILE RULES
  1024. The special rule "<<EOF>>" indicates
  1025. actions which are to be taken when an end-of-file is
  1026. encountered and yywrap() returns non-zero (i.e., indicates
  1027. no further files to process). The action must finish
  1028. by doing one of four things:
  1029. .IP -
  1030. the special
  1031. .B YY_NEW_FILE
  1032. action, if
  1033. .I yyin
  1034. has been pointed at a new file to process;
  1035. .IP -
  1036. a
  1037. .I return
  1038. statement;
  1039. .IP -
  1040. the special
  1041. .B yyterminate()
  1042. action;
  1043. .IP -
  1044. or, switching to a new buffer using
  1045. .B yy_switch_to_buffer()
  1046. as shown in the example above.
  1047. .LP
  1048. <<EOF>> rules may not be used with other
  1049. patterns; they may only be qualified with a list of start
  1050. conditions. If an unqualified <<EOF>> rule is given, it
  1051. applies to
  1052. .I all
  1053. start conditions which do not already have <<EOF>> actions. To
  1054. specify an <<EOF>> rule for only the initial start condition, use
  1055. .nf
  1056. <INITIAL><<EOF>>
  1057. .fi
  1058. .LP
  1059. These rules are useful for catching things like unclosed comments.
  1060. An example:
  1061. .nf
  1062. %x quote
  1063. %%
  1064. ...other rules for dealing with quotes...
  1065. <quote><<EOF>> {
  1066. error( "unterminated quote" );
  1067. yyterminate();
  1068. }
  1069. <<EOF>> {
  1070. if ( *++filelist )
  1071. {
  1072. yyin = fopen( *filelist, "r" );
  1073. YY_NEW_FILE;
  1074. }
  1075. else
  1076. yyterminate();
  1077. }
  1078. .fi
  1079. .SH MISCELLANEOUS MACROS
  1080. The macro
  1081. .bd
  1082. YY_USER_ACTION
  1083. can be redefined to provide an action
  1084. which is always executed prior to the matched rule's action. For example,
  1085. it could be #define'd to call a routine to convert yytext to lower-case.
  1086. .LP
  1087. The macro
  1088. .B YY_USER_INIT
  1089. may be redefined to provide an action which is always executed before
  1090. the first scan (and before the scanner's internal initializations are done).
  1091. For example, it could be used to call a routine to read
  1092. in a data table or open a logging file.
  1093. .LP
  1094. In the generated scanner, the actions are all gathered in one large
  1095. switch statement and separated using
  1096. .B YY_BREAK,
  1097. which may be redefined. By default, it is simply a "break", to separate
  1098. each rule's action from the following rule's.
  1099. Redefining
  1100. .B YY_BREAK
  1101. allows, for example, C++ users to
  1102. #define YY_BREAK to do nothing (while being very careful that every
  1103. rule ends with a "break" or a "return"!) to avoid suffering from
  1104. unreachable statement warnings where because a rule's action ends with
  1105. "return", the
  1106. .B YY_BREAK
  1107. is inaccessible.
  1108. .SH INTERFACING WITH YACC
  1109. One of the main uses of
  1110. .I flex
  1111. is as a companion to the
  1112. .I yacc
  1113. parser-generator.
  1114. .I yacc
  1115. parsers expect to call a routine named
  1116. .B yylex()
  1117. to find the next input token. The routine is supposed to
  1118. return the type of the next token as well as putting any associated
  1119. value in the global
  1120. .B yylval.
  1121. To use
  1122. .I flex
  1123. with
  1124. .I yacc,
  1125. one specifies the
  1126. .B -d
  1127. option to
  1128. .I yacc
  1129. to instruct it to generate the file
  1130. .B y.tab.h
  1131. containing definitions of all the
  1132. .B %tokens
  1133. appearing in the
  1134. .I yacc
  1135. input. This file is then included in the
  1136. .I flex
  1137. scanner. For example, if one of the tokens is "TOK_NUMBER",
  1138. part of the scanner might look like:
  1139. .nf
  1140. %{
  1141. #include "y.tab.h"
  1142. %}
  1143. %%
  1144. [0-9]+ yylval = atoi( yytext ); return TOK_NUMBER;
  1145. .fi
  1146. .SH TRANSLATION TABLE
  1147. In the name of POSIX compliance,
  1148. .I flex
  1149. supports a
  1150. .I translation table
  1151. for mapping input characters into groups.
  1152. The table is specified in the first section, and its format looks like:
  1153. .nf
  1154. %t
  1155. 1 abcd
  1156. 2 ABCDEFGHIJKLMNOPQRSTUVWXYZ
  1157. 52 0123456789
  1158. 6 \\t\\ \\n
  1159. %t
  1160. .fi
  1161. This example specifies that the characters 'a', 'b', 'c', and 'd'
  1162. are to all be lumped into group #1, upper-case letters
  1163. in group #2, digits in group #52, tabs, blanks, and newlines into
  1164. group #6, and
  1165. .I
  1166. no other characters will appear in the patterns.
  1167. The group numbers are actually disregarded by
  1168. .I flex;
  1169. .B %t
  1170. serves, though, to lump characters together. Given the above
  1171. table, for example, the pattern "a(AA)*5" is equivalent to "d(ZQ)*0".
  1172. They both say, "match any character in group #1, followed by
  1173. zero-or-more pairs of characters
  1174. from group #2, followed by a character from group #52." Thus
  1175. .B %t
  1176. provides a crude way for introducing equivalence classes into
  1177. the scanner specification.
  1178. .LP
  1179. Note that the
  1180. .B -i
  1181. option (see below) coupled with the equivalence classes which
  1182. .I flex
  1183. automatically generates take care of virtually all the instances
  1184. when one might consider using
  1185. .B %t.
  1186. But what the hell, it's there if you want it.
  1187. .SH OPTIONS
  1188. .I flex
  1189. has the following options:
  1190. .TP
  1191. .B -b
  1192. Generate backtracking information to
  1193. .I lex.backtrack.
  1194. This is a list of scanner states which require backtracking
  1195. and the input characters on which they do so. By adding rules one
  1196. can remove backtracking states. If all backtracking states
  1197. are eliminated and
  1198. .B -f
  1199. or
  1200. .B -F
  1201. is used, the generated scanner will run faster (see the
  1202. .B -p
  1203. flag). Only users who wish to squeeze every last cycle out of their
  1204. scanners need worry about this option. (See the section on PERFORMANCE
  1205. CONSIDERATIONS below.)
  1206. .TP
  1207. .B -c
  1208. is a do-nothing, deprecated option included for POSIX compliance.
  1209. .IP
  1210. .B NOTE:
  1211. in previous releases of
  1212. .I flex
  1213. .B -c
  1214. specified table-compression options. This functionality is
  1215. now given by the
  1216. .B -C
  1217. flag. To ease the the impact of this change, when
  1218. .I flex
  1219. encounters
  1220. .B -c,
  1221. it currently issues a warning message and assumes that
  1222. .B -C
  1223. was desired instead. In the future this "promotion" of
  1224. .B -c
  1225. to
  1226. .B -C
  1227. will go away in the name of full POSIX compliance (unless
  1228. the POSIX meaning is removed first).
  1229. .TP
  1230. .B -d
  1231. makes the generated scanner run in
  1232. .I debug
  1233. mode. Whenever a pattern is recognized and the global
  1234. .B yy_flex_debug
  1235. is non-zero (which is the default),
  1236. the scanner will write to
  1237. .I stderr
  1238. a line of the form:
  1239. .nf
  1240. --accepting rule at line 53 ("the matched text")
  1241. .fi
  1242. The line number refers to the location of the rule in the file
  1243. defining the scanner (i.e., the file that was fed to flex). Messages
  1244. are also generated when the scanner backtracks, accepts the
  1245. default rule, reaches the end of its input buffer (or encounters
  1246. a NUL; at this point, the two look the same as far as the scanner's concerned),
  1247. or reaches an end-of-file.
  1248. .TP
  1249. .B -f
  1250. specifies (take your pick)
  1251. .I full table
  1252. or
  1253. .I fast scanner.
  1254. No table compression is done. The result is large but fast.
  1255. This option is equivalent to
  1256. .B -Cf
  1257. (see below).
  1258. .TP
  1259. .B -i
  1260. instructs
  1261. .I flex
  1262. to generate a
  1263. .I case-insensitive
  1264. scanner. The case of letters given in the
  1265. .I flex
  1266. input patterns will
  1267. be ignored, and tokens in the input will be matched regardless of case. The
  1268. matched text given in
  1269. .I yytext
  1270. will have the preserved case (i.e., it will not be folded).
  1271. .TP
  1272. .B -n
  1273. is another do-nothing, deprecated option included only for
  1274. POSIX compliance.
  1275. .TP
  1276. .B -p
  1277. generates a performance report to stderr. The report
  1278. consists of comments regarding features of the
  1279. .I flex
  1280. input file which will cause a loss of performance in the resulting scanner.
  1281. Note that the use of
  1282. .I REJECT
  1283. and variable trailing context (see the BUGS section in flex(1))
  1284. entails a substantial performance penalty; use of
  1285. .I yymore(),
  1286. the
  1287. .B ^
  1288. operator,
  1289. and the
  1290. .B -I
  1291. flag entail minor performance penalties.
  1292. .TP
  1293. .B -s
  1294. causes the
  1295. .I default rule
  1296. (that unmatched scanner input is echoed to
  1297. .I stdout)
  1298. to be suppressed. If the scanner encounters input that does not
  1299. match any of its rules, it aborts with an error. This option is
  1300. useful for finding holes in a scanner's rule set.
  1301. .TP
  1302. .B -t
  1303. instructs
  1304. .I flex
  1305. to write the scanner it generates to standard output instead
  1306. of
  1307. .B lex.yy.c.
  1308. .TP
  1309. .B -v
  1310. specifies that
  1311. .I flex
  1312. should write to
  1313. .I stderr
  1314. a summary of statistics regarding the scanner it generates.
  1315. Most of the statistics are meaningless to the casual
  1316. .I flex
  1317. user, but the
  1318. first line identifies the version of
  1319. .I flex,
  1320. which is useful for figuring
  1321. out where you stand with respect to patches and new releases,
  1322. and the next two lines give the date when the scanner was created
  1323. and a summary of the flags which were in effect.
  1324. .TP
  1325. .B -F
  1326. specifies that the
  1327. .ul
  1328. fast
  1329. scanner table representation should be used. This representation is
  1330. about as fast as the full table representation
  1331. .ul
  1332. (-f),
  1333. and for some sets of patterns will be considerably smaller (and for
  1334. others, larger). In general, if the pattern set contains both "keywords"
  1335. and a catch-all, "identifier" rule, such as in the set:
  1336. .nf
  1337. "case" return TOK_CASE;
  1338. "switch" return TOK_SWITCH;
  1339. ...
  1340. "default" return TOK_DEFAULT;
  1341. [a-z]+ return TOK_ID;
  1342. .fi
  1343. then you're better off using the full table representation. If only
  1344. the "identifier" rule is present and you then use a hash table or some such
  1345. to detect the keywords, you're better off using
  1346. .ul
  1347. -F.
  1348. .IP
  1349. This option is equivalent to
  1350. .B -CF
  1351. (see below).
  1352. .TP
  1353. .B -I
  1354. instructs
  1355. .I flex
  1356. to generate an
  1357. .I interactive
  1358. scanner. Normally, scanners generated by
  1359. .I flex
  1360. always look ahead one
  1361. character before deciding that a rule has been matched. At the cost of
  1362. some scanning overhead,
  1363. .I flex
  1364. will generate a scanner which only looks ahead
  1365. when needed. Such scanners are called
  1366. .I interactive
  1367. because if you want to write a scanner for an interactive system such as a
  1368. command shell, you will probably want the user's input to be terminated
  1369. with a newline, and without
  1370. .B -I
  1371. the user will have to type a character in addition to the newline in order
  1372. to have the newline recognized. This leads to dreadful interactive
  1373. performance.
  1374. .IP
  1375. If all this seems to confusing, here's the general rule: if a human will
  1376. be typing in input to your scanner, use
  1377. .B -I,
  1378. otherwise don't; if you don't care about squeezing the utmost performance
  1379. from your scanner and you
  1380. don't want to make any assumptions about the input to your scanner,
  1381. use
  1382. .B -I.
  1383. .IP
  1384. Note,
  1385. .B -I
  1386. cannot be used in conjunction with
  1387. .I full
  1388. or
  1389. .I fast tables,
  1390. i.e., the
  1391. .B -f, -F, -Cf,
  1392. or
  1393. .B -CF
  1394. flags.
  1395. .TP
  1396. .B -L
  1397. instructs
  1398. .I flex
  1399. not to generate
  1400. .B #line
  1401. directives. Without this option,
  1402. .I flex
  1403. peppers the generated scanner
  1404. with #line directives so error messages in the actions will be correctly
  1405. located with respect to the original
  1406. .I flex
  1407. input file, and not to
  1408. the fairly meaningless line numbers of
  1409. .B lex.yy.c.
  1410. (Unfortunately
  1411. .I flex
  1412. does not presently generate the necessary directives
  1413. to "retarget" the line numbers for those parts of
  1414. .B lex.yy.c
  1415. which it generated. So if there is an error in the generated code,
  1416. a meaningless line number is reported.)
  1417. .TP
  1418. .B -T
  1419. makes
  1420. .I flex
  1421. run in
  1422. .I trace
  1423. mode. It will generate a lot of messages to
  1424. .I stdout
  1425. concerning
  1426. the form of the input and the resultant non-deterministic and deterministic
  1427. finite automata. This option is mostly for use in maintaining
  1428. .I flex.
  1429. .TP
  1430. .B -8
  1431. instructs
  1432. .I flex
  1433. to generate an 8-bit scanner, i.e., one which can recognize 8-bit
  1434. characters. On some sites,
  1435. .I flex
  1436. is installed with this option as the default. On others, the default
  1437. is 7-bit characters. To see which is the case, check the verbose
  1438. .B (-v)
  1439. output for "equivalence classes created". If the denominator of
  1440. the number shown is 128, then by default
  1441. .I flex
  1442. is generating 7-bit characters. If it is 256, then the default is
  1443. 8-bit characters and the
  1444. .B -8
  1445. flag is not required (but may be a good idea to keep the scanner
  1446. specification portable). Feeding a 7-bit scanner 8-bit characters
  1447. will result in infinite loops, bus errors, or other such fireworks,
  1448. so when in doubt, use the flag. Note that if equivalence classes
  1449. are used, 8-bit scanners take only slightly more table space than
  1450. 7-bit scanners (128 bytes, to be exact); if equivalence classes are
  1451. not used, however, then the tables may grow up to twice their
  1452. 7-bit size.
  1453. .TP
  1454. .B -C[efmF]
  1455. controls the degree of table compression.
  1456. .IP
  1457. .B -Ce
  1458. directs
  1459. .I flex
  1460. to construct
  1461. .I equivalence classes,
  1462. i.e., sets of characters
  1463. which have identical lexical properties (for example, if the only
  1464. appearance of digits in the
  1465. .I flex
  1466. input is in the character class
  1467. "[0-9]" then the digits '0', '1', ..., '9' will all be put
  1468. in the same equivalence class). Equivalence classes usually give
  1469. dramatic reductions in the final table/object file sizes (typically
  1470. a factor of 2-5) and are pretty cheap performance-wise (one array
  1471. look-up per character scanned).
  1472. .IP
  1473. .B -Cf
  1474. specifies that the
  1475. .I full
  1476. scanner tables should be generated -
  1477. .I flex
  1478. should not compress the
  1479. tables by taking advantages of similar transition functions for
  1480. different states.
  1481. .IP
  1482. .B -CF
  1483. specifies that the alternate fast scanner representation (described
  1484. above under the
  1485. .B -F
  1486. flag)
  1487. should be used.
  1488. .IP
  1489. .B -Cm
  1490. directs
  1491. .I flex
  1492. to construct
  1493. .I meta-equivalence classes,
  1494. which are sets of equivalence classes (or characters, if equivalence
  1495. classes are not being used) that are commonly used together. Meta-equivalence
  1496. classes are often a big win when using compressed tables, but they
  1497. have a moderate performance impact (one or two "if" tests and one
  1498. array look-up per character scanned).
  1499. .IP
  1500. A lone
  1501. .B -C
  1502. specifies that the scanner tables should be compressed but neither
  1503. equivalence classes nor meta-equivalence classes should be used.
  1504. .IP
  1505. The options
  1506. .B -Cf
  1507. or
  1508. .B -CF
  1509. and
  1510. .B -Cm
  1511. do not make sense together - there is no opportunity for meta-equivalence
  1512. classes if the table is not being compressed. Otherwise the options
  1513. may be freely mixed.
  1514. .IP
  1515. The default setting is
  1516. .B -Cem,
  1517. which specifies that
  1518. .I flex
  1519. should generate equivalence classes
  1520. and meta-equivalence classes. This setting provides the highest
  1521. degree of table compression. You can trade off
  1522. faster-executing scanners at the cost of larger tables with
  1523. the following generally being true:
  1524. .nf
  1525. slowest & smallest
  1526. -Cem
  1527. -Cm
  1528. -Ce
  1529. -C
  1530. -C{f,F}e
  1531. -C{f,F}
  1532. fastest & largest
  1533. .fi
  1534. Note that scanners with the smallest tables are usually generated and
  1535. compiled the quickest, so
  1536. during development you will usually want to use the default, maximal
  1537. compression.
  1538. .IP
  1539. .B -Cfe
  1540. is often a good compromise between speed and size for production
  1541. scanners.
  1542. .IP
  1543. .B -C
  1544. options are not cumulative; whenever the flag is encountered, the
  1545. previous -C settings are forgotten.
  1546. .TP
  1547. .B -Sskeleton_file
  1548. overrides the default skeleton file from which
  1549. .I flex
  1550. constructs its scanners. You'll never need this option unless you are doing
  1551. .I flex
  1552. maintenance or development.
  1553. .SH PERFORMANCE CONSIDERATIONS
  1554. The main design goal of
  1555. .I flex
  1556. is that it generate high-performance scanners. It has been optimized
  1557. for dealing well with large sets of rules. Aside from the effects
  1558. of table compression on scanner speed outlined above,
  1559. there are a number of options/actions which degrade performance. These
  1560. are, from most expensive to least:
  1561. .nf
  1562. REJECT
  1563. pattern sets that require backtracking
  1564. arbitrary trailing context
  1565. '^' beginning-of-line operator
  1566. yymore()
  1567. .fi
  1568. with the first three all being quite expensive and the last two
  1569. being quite cheap.
  1570. .LP
  1571. .B REJECT
  1572. should be avoided at all costs when performance is important.
  1573. It is a particularly expensive option.
  1574. .LP
  1575. Getting rid of backtracking is messy and often may be an enormous
  1576. amount of work for a complicated scanner. In principal, one begins
  1577. by using the
  1578. .B -b
  1579. flag to generate a
  1580. .I lex.backtrack
  1581. file. For example, on the input
  1582. .nf
  1583. %%
  1584. foo return TOK_KEYWORD;
  1585. foobar return TOK_KEYWORD;
  1586. .fi
  1587. the file looks like:
  1588. .nf
  1589. State #6 is non-accepting -
  1590. associated rule line numbers:
  1591. 2 3
  1592. out-transitions: [ o ]
  1593. jam-transitions: EOF [ \\001-n p-\\177 ]
  1594. State #8 is non-accepting -
  1595. associated rule line numbers:
  1596. 3
  1597. out-transitions: [ a ]
  1598. jam-transitions: EOF [ \\001-` b-\\177 ]
  1599. State #9 is non-accepting -
  1600. associated rule line numbers:
  1601. 3
  1602. out-transitions: [ r ]
  1603. jam-transitions: EOF [ \\001-q s-\\177 ]
  1604. Compressed tables always backtrack.
  1605. .fi
  1606. The first few lines tell us that there's a scanner state in
  1607. which it can make a transition on an 'o' but not on any other
  1608. character, and that in that state the currently scanned text does not match
  1609. any rule. The state occurs when trying to match the rules found
  1610. at lines 2 and 3 in the input file.
  1611. If the scanner is in that state and then reads
  1612. something other than an 'o', it will have to backtrack to find
  1613. a rule which is matched. With
  1614. a bit of headscratching one can see that this must be the
  1615. state it's in when it has seen "fo". When this has happened,
  1616. if anything other than another 'o' is seen, the scanner will
  1617. have to back up to simply match the 'f' (by the default rule).
  1618. .LP
  1619. The comment regarding State #8 indicates there's a problem
  1620. when "foob" has been scanned. Indeed, on any character other
  1621. than a 'b', the scanner will have to back up to accept "foo".
  1622. Similarly, the comment for State #9 concerns when "fooba" has
  1623. been scanned.
  1624. .LP
  1625. The final comment reminds us that there's no point going to
  1626. all the trouble of removing backtracking from the rules unless
  1627. we're using
  1628. .B -f
  1629. or
  1630. .B -F,
  1631. since there's no performance gain doing so with compressed scanners.
  1632. .LP
  1633. The way to remove the backtracking is to add "error" rules:
  1634. .nf
  1635. %%
  1636. foo return TOK_KEYWORD;
  1637. foobar return TOK_KEYWORD;
  1638. fooba |
  1639. foob |
  1640. fo {
  1641. /* false alarm, not really a keyword */
  1642. return TOK_ID;
  1643. }
  1644. .fi
  1645. .LP
  1646. Eliminating backtracking among a list of keywords can also be
  1647. done using a "catch-all" rule:
  1648. .nf
  1649. %%
  1650. foo return TOK_KEYWORD;
  1651. foobar return TOK_KEYWORD;
  1652. [a-z]+ return TOK_ID;
  1653. .fi
  1654. This is usually the best solution when appropriate.
  1655. .LP
  1656. Backtracking messages tend to cascade.
  1657. With a complicated set of rules it's not uncommon to get hundreds
  1658. of messages. If one can decipher them, though, it often
  1659. only takes a dozen or so rules to eliminate the backtracking (though
  1660. it's easy to make a mistake and have an error rule accidentally match
  1661. a valid token. A possible future
  1662. .I flex
  1663. feature will be to automatically add rules to eliminate backtracking).
  1664. .LP
  1665. .I Variable
  1666. trailing context (where both the leading and trailing parts do not have
  1667. a fixed length) entails almost the same performance loss as
  1668. .I REJECT
  1669. (i.e., substantial). So when possible a rule like:
  1670. .nf
  1671. %%
  1672. mouse|rat/(cat|dog) run();
  1673. .fi
  1674. is better written:
  1675. .nf
  1676. %%
  1677. mouse/cat|dog run();
  1678. rat/cat|dog run();
  1679. .fi
  1680. or as
  1681. .nf
  1682. %%
  1683. mouse|rat/cat run();
  1684. mouse|rat/dog run();
  1685. .fi
  1686. Note that here the special '|' action does
  1687. .I not
  1688. provide any savings, and can even make things worse (see
  1689. .B BUGS
  1690. in flex(1)).
  1691. .LP
  1692. Another area where the user can increase a scanner's performance
  1693. (and one that's easier to implement) arises from the fact that
  1694. the longer the tokens matched, the faster the scanner will run.
  1695. This is because with long tokens the processing of most input
  1696. characters takes place in the (short) inner scanning loop, and
  1697. does not often have to go through the additional work of setting up
  1698. the scanning environment (e.g.,
  1699. .B yytext)
  1700. for the action. Recall the scanner for C comments:
  1701. .nf
  1702. %x comment
  1703. %%
  1704. int line_num = 1;
  1705. "/*" BEGIN(comment);
  1706. <comment>[^*\\n]*
  1707. <comment>"*"+[^*/\\n]*
  1708. <comment>\\n ++line_num;
  1709. <comment>"*"+"/" BEGIN(INITIAL);
  1710. .fi
  1711. This could be sped up by writing it as:
  1712. .nf
  1713. %x comment
  1714. %%
  1715. int line_num = 1;
  1716. "/*" BEGIN(comment);
  1717. <comment>[^*\\n]*
  1718. <comment>[^*\\n]*\\n ++line_num;
  1719. <comment>"*"+[^*/\\n]*
  1720. <comment>"*"+[^*/\\n]*\\n ++line_num;
  1721. <comment>"*"+"/" BEGIN(INITIAL);
  1722. .fi
  1723. Now instead of each newline requiring the processing of another
  1724. action, recognizing the newlines is "distributed" over the other rules
  1725. to keep the matched text as long as possible. Note that
  1726. .I adding
  1727. rules does
  1728. .I not
  1729. slow down the scanner! The speed of the scanner is independent
  1730. of the number of rules or (modulo the considerations given at the
  1731. beginning of this section) how complicated the rules are with
  1732. regard to operators such as '*' and '|'.
  1733. .LP
  1734. A final example in speeding up a scanner: suppose you want to scan
  1735. through a file containing identifiers and keywords, one per line
  1736. and with no other extraneous characters, and recognize all the
  1737. keywords. A natural first approach is:
  1738. .nf
  1739. %%
  1740. asm |
  1741. auto |
  1742. break |
  1743. ... etc ...
  1744. volatile |
  1745. while /* it's a keyword */
  1746. .|\\n /* it's not a keyword */
  1747. .fi
  1748. To eliminate the back-tracking, introduce a catch-all rule:
  1749. .nf
  1750. %%
  1751. asm |
  1752. auto |
  1753. break |
  1754. ... etc ...
  1755. volatile |
  1756. while /* it's a keyword */
  1757. [a-z]+ |
  1758. .|\\n /* it's not a keyword */
  1759. .fi
  1760. Now, if it's guaranteed that there's exactly one word per line,
  1761. then we can reduce the total number of matches by a half by
  1762. merging in the recognition of newlines with that of the other
  1763. tokens:
  1764. .nf
  1765. %%
  1766. asm\\n |
  1767. auto\\n |
  1768. break\\n |
  1769. ... etc ...
  1770. volatile\\n |
  1771. while\\n /* it's a keyword */
  1772. [a-z]+\\n |
  1773. .|\\n /* it's not a keyword */
  1774. .fi
  1775. One has to be careful here, as we have now reintroduced backtracking
  1776. into the scanner. In particular, while
  1777. .I we
  1778. know that there will never be any characters in the input stream
  1779. other than letters or newlines,
  1780. .I flex
  1781. can't figure this out, and it will plan for possibly needing backtracking
  1782. when it has scanned a token like "auto" and then the next character
  1783. is something other than a newline or a letter. Previously it would
  1784. then just match the "auto" rule and be done, but now it has no "auto"
  1785. rule, only a "auto\\n" rule. To eliminate the possibility of backtracking,
  1786. we could either duplicate all rules but without final newlines, or,
  1787. since we never expect to encounter such an input and therefore don't
  1788. how it's classified, we can introduce one more catch-all rule, this
  1789. one which doesn't include a newline:
  1790. .nf
  1791. %%
  1792. asm\\n |
  1793. auto\\n |
  1794. break\\n |
  1795. ... etc ...
  1796. volatile\\n |
  1797. while\\n /* it's a keyword */
  1798. [a-z]+\\n |
  1799. [a-z]+ |
  1800. .|\\n /* it's not a keyword */
  1801. .fi
  1802. Compiled with
  1803. .B -Cf,
  1804. this is about as fast as one can get a
  1805. .I flex
  1806. scanner to go for this particular problem.
  1807. .LP
  1808. A final note:
  1809. .I flex
  1810. is slow when matching NUL's, particularly when a token contains
  1811. multiple NUL's.
  1812. It's best to write rules which match
  1813. .I short
  1814. amounts of text if it's anticipated that the text will often include NUL's.
  1815. .SH INCOMPATIBILITIES WITH LEX AND POSIX
  1816. .I flex
  1817. is a rewrite of the Unix
  1818. .I lex
  1819. tool (the two implementations do not share any code, though),
  1820. with some extensions and incompatibilities, both of which
  1821. are of concern to those who wish to write scanners acceptable
  1822. to either implementation. At present, the POSIX
  1823. .I lex
  1824. draft is
  1825. very close to the original
  1826. .I lex
  1827. implementation, so some of these
  1828. incompatibilities are also in conflict with the POSIX draft. But
  1829. the intent is that except as noted below,
  1830. .I flex
  1831. as it presently stands will
  1832. ultimately be POSIX conformant (i.e., that those areas of conflict with
  1833. the POSIX draft will be resolved in
  1834. .I flex's
  1835. favor). Please bear in
  1836. mind that all the comments which follow are with regard to the POSIX
  1837. .I draft
  1838. standard of Summer 1989, and not the final document (or subsequent
  1839. drafts); they are included so
  1840. .I flex
  1841. users can be aware of the standardization issues and those areas where
  1842. .I flex
  1843. may in the near future undergo changes incompatible with
  1844. its current definition.
  1845. .LP
  1846. .I flex
  1847. is fully compatible with
  1848. .I lex
  1849. with the following exceptions:
  1850. .IP -
  1851. The undocumented
  1852. .I lex
  1853. scanner internal variable
  1854. .B yylineno
  1855. is not supported. It is difficult to support this option efficiently,
  1856. since it requires examining every character scanned and reexamining
  1857. the characters when the scanner backs up.
  1858. Things get more complicated when the end of buffer or file is reached or a
  1859. NUL is scanned (since the scan must then be restarted with the proper line
  1860. number count), or the user uses the yyless(), unput(), or REJECT actions,
  1861. or the multiple input buffer functions.
  1862. .IP
  1863. The fix is to add rules which, upon seeing a newline, increment
  1864. yylineno. This is usually an easy process, though it can be a drag if some
  1865. of the patterns can match multiple newlines along with other characters.
  1866. .IP
  1867. yylineno is not part of the POSIX draft.
  1868. .IP -
  1869. The
  1870. .B input()
  1871. routine is not redefinable, though it may be called to read characters
  1872. following whatever has been matched by a rule. If
  1873. .B input()
  1874. encounters an end-of-file the normal
  1875. .B yywrap()
  1876. processing is done. A ``real'' end-of-file is returned by
  1877. .B input()
  1878. as
  1879. .I EOF.
  1880. .IP
  1881. Input is instead controlled by redefining the
  1882. .B YY_INPUT
  1883. macro.
  1884. .IP
  1885. The
  1886. .I flex
  1887. restriction that
  1888. .B input()
  1889. cannot be redefined is in accordance with the POSIX draft, but
  1890. .B YY_INPUT
  1891. has not yet been accepted into the draft (and probably won't; it looks
  1892. like the draft will simply not specify any way of controlling the
  1893. scanner's input other than by making an initial assignment to
  1894. .I yyin).
  1895. .IP -
  1896. .I flex
  1897. scanners do not use stdio for input. Because of this, when writing an
  1898. interactive scanner one must explicitly call fflush() on the
  1899. stream associated with the terminal after writing out a prompt.
  1900. With
  1901. .I lex
  1902. such writes are automatically flushed since
  1903. .I lex
  1904. scanners use
  1905. .B getchar()
  1906. for their input. Also, when writing interactive scanners with
  1907. .I flex,
  1908. the
  1909. .B -I
  1910. flag must be used.
  1911. .IP -
  1912. .I flex
  1913. scanners are not as reentrant as
  1914. .I lex
  1915. scanners. In particular, if you have an interactive scanner and
  1916. an interrupt handler which long-jumps out of the scanner, and
  1917. the scanner is subsequently called again, you may get the following
  1918. message:
  1919. .nf
  1920. fatal flex scanner internal error--end of buffer missed
  1921. .fi
  1922. To reenter the scanner, first use
  1923. .nf
  1924. yyrestart( yyin );
  1925. .fi
  1926. .IP -
  1927. .B output()
  1928. is not supported.
  1929. Output from the
  1930. .B ECHO
  1931. macro is done to the file-pointer
  1932. .I yyout
  1933. (default
  1934. .I stdout).
  1935. .IP
  1936. The POSIX draft mentions that an
  1937. .B output()
  1938. routine exists but currently gives no details as to what it does.
  1939. .IP -
  1940. .I lex
  1941. does not support exclusive start conditions (%x), though they
  1942. are in the current POSIX draft.
  1943. .IP -
  1944. When definitions are expanded,
  1945. .I flex
  1946. encloses them in parentheses.
  1947. With lex, the following:
  1948. .nf
  1949. NAME [A-Z][A-Z0-9]*
  1950. %%
  1951. foo{NAME}? printf( "Found it\\n" );
  1952. %%
  1953. .fi
  1954. will not match the string "foo" because when the macro
  1955. is expanded the rule is equivalent to "foo[A-Z][A-Z0-9]*?"
  1956. and the precedence is such that the '?' is associated with
  1957. "[A-Z0-9]*". With
  1958. .I flex,
  1959. the rule will be expanded to
  1960. "foo([A-Z][A-Z0-9]*)?" and so the string "foo" will match.
  1961. Note that because of this, the
  1962. .B ^, $, <s>, /,
  1963. and
  1964. .B <<EOF>>
  1965. operators cannot be used in a
  1966. .I flex
  1967. definition.
  1968. .IP
  1969. The POSIX draft interpretation is the same as
  1970. .I flex's.
  1971. .IP -
  1972. To specify a character class which matches anything but a left bracket (']'),
  1973. in
  1974. .I lex
  1975. one can use "[^]]" but with
  1976. .I flex
  1977. one must use "[^\\]]". The latter works with
  1978. .I lex,
  1979. too.
  1980. .IP -
  1981. The
  1982. .I lex
  1983. .B %r
  1984. (generate a Ratfor scanner) option is not supported. It is not part
  1985. of the POSIX draft.
  1986. .IP -
  1987. If you are providing your own yywrap() routine, you must include a
  1988. "#undef yywrap" in the definitions section (section 1). Note that
  1989. the "#undef" will have to be enclosed in %{}'s.
  1990. .IP
  1991. The POSIX draft
  1992. specifies that yywrap() is a function and this is very unlikely to change; so
  1993. .I flex users are warned
  1994. that
  1995. .B yywrap()
  1996. is likely to be changed to a function in the near future.
  1997. .IP -
  1998. After a call to
  1999. .B unput(),
  2000. .I yytext
  2001. and
  2002. .I yyleng
  2003. are undefined until the next token is matched. This is not the case with
  2004. .I lex
  2005. or the present POSIX draft.
  2006. .IP -
  2007. The precedence of the
  2008. .B {}
  2009. (numeric range) operator is different.
  2010. .I lex
  2011. interprets "abc{1,3}" as "match one, two, or
  2012. three occurrences of 'abc'", whereas
  2013. .I flex
  2014. interprets it as "match 'ab'
  2015. followed by one, two, or three occurrences of 'c'". The latter is
  2016. in agreement with the current POSIX draft.
  2017. .IP -
  2018. The precedence of the
  2019. .B ^
  2020. operator is different.
  2021. .I lex
  2022. interprets "^foo|bar" as "match either 'foo' at the beginning of a line,
  2023. or 'bar' anywhere", whereas
  2024. .I flex
  2025. interprets it as "match either 'foo' or 'bar' if they come at the beginning
  2026. of a line". The latter is in agreement with the current POSIX draft.
  2027. .IP -
  2028. To refer to yytext outside of the scanner source file,
  2029. the correct definition with
  2030. .I flex
  2031. is "extern char *yytext" rather than "extern char yytext[]".
  2032. This is contrary to the current POSIX draft but a point on which
  2033. .I flex
  2034. will not be changing, as the array representation entails a
  2035. serious performance penalty. It is hoped that the POSIX draft will
  2036. be emended to support the
  2037. .I flex
  2038. variety of declaration (as this is a fairly painless change to
  2039. require of
  2040. .I lex
  2041. users).
  2042. .IP -
  2043. .I yyin
  2044. is
  2045. .I initialized
  2046. by
  2047. .I lex
  2048. to be
  2049. .I stdin;
  2050. .I flex,
  2051. on the other hand,
  2052. initializes
  2053. .I yyin
  2054. to NULL
  2055. and then
  2056. .I assigns
  2057. it to
  2058. .I stdin
  2059. the first time the scanner is called, providing
  2060. .I yyin
  2061. has not already been assigned to a non-NULL value. The difference is
  2062. subtle, but the net effect is that with
  2063. .I flex
  2064. scanners,
  2065. .I yyin
  2066. does not have a valid value until the scanner has been called.
  2067. .IP -
  2068. The special table-size declarations such as
  2069. .B %a
  2070. supported by
  2071. .I lex
  2072. are not required by
  2073. .I flex
  2074. scanners;
  2075. .I flex
  2076. ignores them.
  2077. .IP -
  2078. The name
  2079. .bd
  2080. FLEX_SCANNER
  2081. is #define'd so scanners may be written for use with either
  2082. .I flex
  2083. or
  2084. .I lex.
  2085. .LP
  2086. The following
  2087. .I flex
  2088. features are not included in
  2089. .I lex
  2090. or the POSIX draft standard:
  2091. .nf
  2092. yyterminate()
  2093. <<EOF>>
  2094. YY_DECL
  2095. #line directives
  2096. %{}'s around actions
  2097. yyrestart()
  2098. comments beginning with '#' (deprecated)
  2099. multiple actions on a line
  2100. .fi
  2101. This last feature refers to the fact that with
  2102. .I flex
  2103. you can put multiple actions on the same line, separated with
  2104. semi-colons, while with
  2105. .I lex,
  2106. the following
  2107. .nf
  2108. foo handle_foo(); ++num_foos_seen;
  2109. .fi
  2110. is (rather surprisingly) truncated to
  2111. .nf
  2112. foo handle_foo();
  2113. .fi
  2114. .I flex
  2115. does not truncate the action. Actions that are not enclosed in
  2116. braces are simply terminated at the end of the line.
  2117. .SH DIAGNOSTICS
  2118. .I reject_used_but_not_detected undefined
  2119. or
  2120. .I yymore_used_but_not_detected undefined -
  2121. These errors can occur at compile time. They indicate that the
  2122. scanner uses
  2123. .B REJECT
  2124. or
  2125. .B yymore()
  2126. but that
  2127. .I flex
  2128. failed to notice the fact, meaning that
  2129. .I flex
  2130. scanned the first two sections looking for occurrences of these actions
  2131. and failed to find any, but somehow you snuck some in (via a #include
  2132. file, for example). Make an explicit reference to the action in your
  2133. .I flex
  2134. input file. (Note that previously
  2135. .I flex
  2136. supported a
  2137. .B %used/%unused
  2138. mechanism for dealing with this problem; this feature is still supported
  2139. but now deprecated, and will go away soon unless the author hears from
  2140. people who can argue compellingly that they need it.)
  2141. .LP
  2142. .I flex scanner jammed -
  2143. a scanner compiled with
  2144. .B -s
  2145. has encountered an input string which wasn't matched by
  2146. any of its rules.
  2147. .LP
  2148. .I flex input buffer overflowed -
  2149. a scanner rule matched a string long enough to overflow the
  2150. scanner's internal input buffer (16K bytes by default - controlled by
  2151. .B YY_BUF_SIZE
  2152. in "flex.skel". Note that to redefine this macro, you must first
  2153. .B #undefine
  2154. it).
  2155. .LP
  2156. .I scanner requires -8 flag -
  2157. Your scanner specification includes recognizing 8-bit characters and
  2158. you did not specify the -8 flag (and your site has not installed flex
  2159. with -8 as the default).
  2160. .LP
  2161. .I
  2162. fatal flex scanner internal error--end of buffer missed -
  2163. This can occur in an scanner which is reentered after a long-jump
  2164. has jumped out (or over) the scanner's activation frame. Before
  2165. reentering the scanner, use:
  2166. .nf
  2167. yyrestart( yyin );
  2168. .fi
  2169. .LP
  2170. .I too many %t classes! -
  2171. You managed to put every single character into its own %t class.
  2172. .I flex
  2173. requires that at least one of the classes share characters.
  2174. .SH DEFICIENCIES / BUGS
  2175. See flex(1).
  2176. .SH "SEE ALSO"
  2177. .LP
  2178. flex(1), lex(1), yacc(1), sed(1), awk(1).
  2179. .LP
  2180. M. E. Lesk and E. Schmidt,
  2181. .I LEX - Lexical Analyzer Generator
  2182. .SH AUTHOR
  2183. Vern Paxson, with the help of many ideas and much inspiration from
  2184. Van Jacobson. Original version by Jef Poskanzer. The fast table
  2185. representation is a partial implementation of a design done by Van
  2186. Jacobson. The implementation was done by Kevin Gong and Vern Paxson.
  2187. .LP
  2188. Thanks to the many
  2189. .I flex
  2190. beta-testers, feedbackers, and contributors, especially Casey
  2191. Leedom, benson@odi.com, Keith Bostic,
  2192. Frederic Brehm, Nick Christopher, Jason Coughlin,
  2193. Scott David Daniels, Leo Eskin,
  2194. Chris Faylor, Eric Goldman, Eric
  2195. Hughes, Jeffrey R. Jones, Kevin B. Kenny, Ronald Lamprecht,
  2196. Greg Lee, Craig Leres, Mohamed el Lozy, Jim Meyering, Marc Nozell, Esmond Pitt,
  2197. Jef Poskanzer, Jim Roskind,
  2198. Dave Tallman, Frank Whaley, Ken Yap, and those whose names
  2199. have slipped my marginal mail-archiving skills but whose contributions
  2200. are appreciated all the same.
  2201. .LP
  2202. Thanks to Keith Bostic, John Gilmore, Craig Leres, Bob
  2203. Mulcahy, Rich Salz, and Richard Stallman for help with various distribution
  2204. headaches.
  2205. .LP
  2206. Thanks to Esmond Pitt and Earle Horton for 8-bit character support;
  2207. to Benson Margulies and Fred
  2208. Burke for C++ support; to Ove Ewerlid for the basics of support for
  2209. NUL's; and to Eric Hughes for the basics of support for multiple buffers.
  2210. .LP
  2211. Work is being done on extending
  2212. .I flex
  2213. to generate scanners in which the
  2214. state machine is directly represented in C code rather than tables.
  2215. These scanners may well be substantially faster than those generated
  2216. using -f or -F. If you are working in this area and are interested
  2217. in comparing notes and seeing whether redundant work can be avoided,
  2218. contact Ove Ewerlid (ewerlid@mizar.DoCS.UU.SE).
  2219. .LP
  2220. This work was primarily done when I was at the Real Time Systems Group
  2221. at the Lawrence Berkeley Laboratory in Berkeley, CA. Many thanks to all there
  2222. for the support I received.
  2223. .LP
  2224. Send comments to:
  2225. .nf
  2226. Vern Paxson
  2227. Computer Science Department
  2228. 4126 Upson Hall
  2229. Cornell University
  2230. Ithaca, NY 14853-7501
  2231. vern@cs.cornell.edu
  2232. decvax!cornell!vern
  2233. .fi