CHANGES_FROM_133_BEFORE_MR13.txt 140 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666
  1. ------------------------------------------------------------
  2. This is the second part of a two part file.
  3. This is a list of changes to pccts 1.33 prior to MR13
  4. For more recent information see CHANGES_FROM_133.txt
  5. ------------------------------------------------------------
  6. DISCLAIMER
  7. The software and these notes are provided "as is". They may include
  8. typographical or technical errors and their authors disclaims all
  9. liability of any kind or nature for damages due to error, fault,
  10. defect, or deficiency regardless of cause. All warranties of any
  11. kind, either express or implied, including, but not limited to, the
  12. implied warranties of merchantability and fitness for a particular
  13. purpose are disclaimed.
  14. #153. (Changed in MR12b) Bug in computation of -mrhoist suppression set
  15. Consider the following grammar with k=1 and "-mrhoist on":
  16. r1 : (A)? => ((p>>? x /* l1 */
  17. | r2 /* l2 */
  18. ;
  19. r2 : A /* l4 */
  20. | (B)? => <<q>>? y /* l5 */
  21. ;
  22. In earlier versions the mrhoist routine would see that both l1 and
  23. l2 contained predicates and would assume that this prevented either
  24. from acting to suppress the other predicate. In the example above
  25. it didn't realize the A at line l4 is capable of suppressing the
  26. predicate at l1 even though alt l2 contains (indirectly) a predicate.
  27. This is fixed in MR12b.
  28. Reported by Reinier van den Born (reinier@vnet.ibm.com)
  29. #153. (Changed in MR12a) Bug in computation of -mrhoist suppression set
  30. An oversight similar to that described in Item #152 appeared in
  31. the computation of the set that "covered" a predicate. If a
  32. predicate expression included a term such as p=AND(q,r) the context
  33. of p was taken to be context(q) & context(r), when it should have
  34. been context(q) | context(r). This is fixed in MR12a.
  35. #152. (Changed in MR12) Bug in generation of predicate expressions
  36. The primary purpose for MR12 is to make quite clear that MR11 is
  37. obsolete and to fix the bug related to predicate expressions.
  38. In MR10 code was added to optimize the code generated for
  39. predicate expression tests. Unfortunately, there was a
  40. significant oversight in the code which resulted in a bug in
  41. the generation of code for predicate expression tests which
  42. contained predicates combined using AND:
  43. r0 : (r1)* "@" ;
  44. r1 : (AAA)? => <<p LATEXT(1)>>? r2 ;
  45. r2 : (BBB)? => <<q LATEXT(1)>>? Q
  46. | (BBB)? => <<r LATEXT(1)>>? Q
  47. ;
  48. In MR11 (and MR10 when using "-mrhoist on") the code generated
  49. for r0 to predict r1 would be equivalent to:
  50. if ( LA(1)==Q &&
  51. (LA(1)==AAA && LA(1)==BBB) &&
  52. ( p && ( q || r )) ) {
  53. This is incorrect because it expresses the idea that LA(1)
  54. *must* be AAA in order to attempt r1, and *must* be BBB to
  55. attempt r2. The result was that r1 became unreachable since
  56. both condition can not be simultaneously true.
  57. The general philosophy of code generation for predicates
  58. can be summarized as follows:
  59. a. If the context is true don't enter an alt
  60. for which the corresponding predicate is false.
  61. If the context is false then it is okay to enter
  62. the alt without evaluating the predicate at all.
  63. b. A predicate created by ORing of predicates has
  64. context which is the OR of their individual contexts.
  65. c. A predicate created by ANDing of predicates has
  66. (surprise) context which is the OR of their individual
  67. contexts.
  68. d. Apply these rules recursively.
  69. e. Remember rule (a)
  70. The correct code should express the idea that *if* LA(1) is
  71. AAA then p must be true to attempt r1, but if LA(1) is *not*
  72. AAA then it is okay to attempt r1, provided that *if* LA(1) is
  73. BBB then one of q or r must be true.
  74. if ( LA(1)==Q &&
  75. ( !(LA(1)==AAA || LA(1)==BBB) ||
  76. ( ! LA(1) == AAA || p) &&
  77. ( ! LA(1) == BBB || q || r ) ) ) {
  78. I believe this is fixed in MR12.
  79. Reported by Reinier van den Born (reinier@vnet.ibm.com)
  80. #151a. (Changed in MR12) ANTLRParser::getLexer()
  81. As a result of several requests, I have added public methods to
  82. get a pointer to the lexer belonging to a parser.
  83. ANTLRTokenStream *ANTLRParser::getLexer() const
  84. Returns a pointer to the lexer being used by the
  85. parser. ANTLRTokenStream is the base class of
  86. DLGLexer
  87. ANTLRTokenStream *ANTLRTokenBuffer::getLexer() const
  88. Returns a pointer to the lexer being used by the
  89. ANTLRTokenBuffer. ANTLRTokenStream is the base
  90. class of DLGLexer
  91. You must manually cast the ANTLRTokenStream to your program's
  92. lexer class. Because the name of the lexer's class is not fixed.
  93. Thus it is impossible to incorporate it into the DLGLexerBase
  94. class.
  95. #151b.(Changed in MR12) ParserBlackBox member getLexer()
  96. The template class ParserBlackBox now has a member getLexer()
  97. which returns a pointer to the lexer.
  98. #150. (Changed in MR12) syntaxErrCount and lexErrCount now public
  99. See Item #127 for more information.
  100. #149. (Changed in MR12) antlr option -info o (letter o for orphan)
  101. If there is more than one rule which is not referenced by any
  102. other rule then all such rules are listed. This is useful for
  103. alerting one to rules which are not used, but which can still
  104. contribute to ambiguity. For example:
  105. start : a Z ;
  106. unused: a A ;
  107. a : (A)+ ;
  108. will cause an ambiguity report for rule "a" which will be
  109. difficult to understand if the user forgets about rule "unused"
  110. simply because it is not used in the grammar.
  111. #148. (Changed in MR11) #token names appearing in zztokens,token_tbl
  112. In a #token statement like the following:
  113. #token Plus "\+"
  114. the string "Plus" appears in the zztokens array (C mode) and
  115. token_tbl (C++ mode). This string is used in most error
  116. messages. In MR11 one has the option of using some other string,
  117. (e.g. "+") in those tables.
  118. In MR11 one can write:
  119. #token Plus ("+") "\+"
  120. #token RP ("(") "\("
  121. #token COM ("comment begin") "/\*"
  122. A #token statement is allowed to appear in more than one #lexclass
  123. with different regular expressions. However, the token name appears
  124. only once in the zztokens/token_tbl array. This means that only
  125. one substitute can be specified for a given #token name. The second
  126. attempt to define a substitute name (different from the first) will
  127. result in an error message.
  128. #147. (Changed in MR11) Bug in follow set computation
  129. There is a bug in 1.33 vanilla and all maintenance releases
  130. prior to MR11 in the computation of the follow set. The bug is
  131. different than that described in Item #82 and probably more
  132. common. It was discovered in the ansi.g grammar while testing
  133. the "ambiguity aid" (Item #119). The search for a bug started
  134. when the ambiguity aid was unable to discover the actual source
  135. of an ambiguity reported by antlr.
  136. The problem appears when an optimization of the follow set
  137. computation is used inappropriately. The result is that the
  138. follow set used is the "worst case". In other words, the error
  139. can lead to false reports of ambiguity. The good news is that
  140. if you have a grammar in which you have addressed all reported
  141. ambiguities you are ok. The bad news is that you may have spent
  142. time fixing ambiguities that were not real, or used k=2 when
  143. ck=2 might have been sufficient, and so on.
  144. The following grammar demonstrates the problem:
  145. ------------------------------------------------------------
  146. expr : ID ;
  147. start : stmt SEMI ;
  148. stmt : CASE expr COLON
  149. | expr SEMI
  150. | plain_stmt
  151. ;
  152. plain_stmt : ID COLON ;
  153. ------------------------------------------------------------
  154. When compiled with k=1 and ck=2 it will report:
  155. warning: alts 2 and 3 of the rule itself ambiguous upon
  156. { IDENTIFIER }, { COLON }
  157. When antlr analyzes "stmt" it computes the first[1] set of all
  158. alternatives. It finds an ambiguity between alts 2 and 3 for ID.
  159. It then computes the first[2] set for alternatives 2 and 3 to resolve
  160. the ambiguity. In computing the first[2] set of "expr" (which is
  161. only one token long) it needs to determine what could follow "expr".
  162. Under a certain combination of circumstances antlr forgets that it
  163. is trying to analyze "stmt" which can only be followed by SEMI and
  164. adds to the first[2] set of "expr" the "global" follow set (including
  165. "COLON") which could follow "expr" (under other conditions) in the
  166. phrase "CASE expr COLON".
  167. #146. (Changed in MR11) Option -treport for locating "difficult" alts
  168. It can be difficult to determine which alternatives are causing
  169. pccts to work hard to resolve an ambiguity. In some cases the
  170. ambiguity is successfully resolved after much CPU time so there
  171. is no message at all.
  172. A rough measure of the amount of work being peformed which is
  173. independent of the CPU speed and system load is the number of
  174. tnodes created. Using "-info t" gives information about the
  175. total number of tnodes created and the peak number of tnodes.
  176. Tree Nodes: peak 1300k created 1416k lost 0
  177. It also puts in the generated C or C++ file the number of tnodes
  178. created for a rule (at the end of the rule). However this
  179. information is not sufficient to locate the alternatives within
  180. a rule which are causing the creation of tnodes.
  181. Using:
  182. antlr -treport 100000 ....
  183. causes antlr to list on stdout any alternatives which require the
  184. creation of more than 100,000 tnodes, along with the lookahead sets
  185. for those alternatives.
  186. The following is a trivial case from the ansi.g grammar which shows
  187. the format of the report. This report might be of more interest
  188. in cases where 1,000,000 tuples were created to resolve the ambiguity.
  189. -------------------------------------------------------------------------
  190. There were 0 tuples whose ambiguity could not be resolved
  191. by full lookahead
  192. There were 157 tnodes created to resolve ambiguity between:
  193. Choice 1: statement/2 line 475 file ansi.g
  194. Choice 2: statement/3 line 476 file ansi.g
  195. Intersection of lookahead[1] sets:
  196. IDENTIFIER
  197. Intersection of lookahead[2] sets:
  198. LPARENTHESIS COLON AMPERSAND MINUS
  199. STAR PLUSPLUS MINUSMINUS ONESCOMPLEMENT
  200. NOT SIZEOF OCTALINT DECIMALINT
  201. HEXADECIMALINT FLOATONE FLOATTWO IDENTIFIER
  202. STRING CHARACTER
  203. -------------------------------------------------------------------------
  204. #145. (Documentation) Generation of Expression Trees
  205. Item #99 was misleading because it implied that the optimization
  206. for tree expressions was available only for trees created by
  207. predicate expressions and neglected to mention that it required
  208. the use of "-mrhoist on". The optimization applies to tree
  209. expressions created for grammars with k>1 and for predicates with
  210. lookahead depth >1.
  211. In MR11 the optimized version is always used so the -mrhoist on
  212. option need not be specified.
  213. #144. (Changed in MR11) Incorrect test for exception group
  214. In testing for a rule's exception group the label a pointer
  215. is compared against '\0'. The intention is "*pointer".
  216. Reported by Jeffrey C. Fried (Jeff@Fried.net).
  217. #143. (Changed in MR11) Optional ";" at end of #token statement
  218. Fixes problem of:
  219. #token X "x"
  220. <<
  221. parser action
  222. >>
  223. Being confused with:
  224. #token X "x" <<lexical action>>
  225. #142. (Changed in MR11) class BufFileInput subclass of DLGInputStream
  226. Alexey Demakov (demakov@kazbek.ispras.ru) has supplied class
  227. BufFileInput derived from DLGInputStream which provides a
  228. function lookahead(char *string) to test characters in the
  229. input stream more than one character ahead.
  230. The default amount of lookahead is specified by the constructor
  231. and defaults to 8 characters. This does *not* include the one
  232. character of lookahead maintained internally by DLG in member "ch"
  233. and which is not available for testing via BufFileInput::lookahead().
  234. This is a useful class for overcoming the one-character-lookahead
  235. limitation of DLG without resorting to a lexer capable of
  236. backtracking (like flex) which is not integrated with antlr as is
  237. DLG.
  238. There are no restrictions on copying or using BufFileInput.* except
  239. that the authorship and related information must be retained in the
  240. source code.
  241. The class is located in pccts/h/BufFileInput.* of the kit.
  242. #141. (Changed in MR11) ZZDEBUG_CONSUME for ANTLRParser::consume()
  243. A debug aid has been added to file ANTLRParser::consume() in
  244. file AParser.cpp:
  245. #ifdef ZZDEBUG_CONSUME_ACTION
  246. zzdebug_consume_action();
  247. #endif
  248. Suggested by Sramji Ramanathan (ps@kumaran.com).
  249. #140. (Changed in MR11) #pred to define predicates
  250. +---------------------------------------------------+
  251. | Note: Assume "-prc on" for this entire discussion |
  252. +---------------------------------------------------+
  253. A problem with predicates is that each one is regarded as
  254. unique and capable of disambiguating cases where two
  255. alternatives have identical lookahead. For example:
  256. rule : <<pred(LATEXT(1))>>? A
  257. | <<pred(LATEXT(1))>>? A
  258. ;
  259. will not cause any error messages or warnings to be issued
  260. by earlier versions of pccts. To compare the text of the
  261. predicates is an incomplete solution.
  262. In 1.33MR11 I am introducing the #pred statement in order to
  263. solve some problems with predicates. The #pred statement allows
  264. one to give a symbolic name to a "predicate literal" or a
  265. "predicate expression" in order to refer to it in other predicate
  266. expressions or in the rules of the grammar.
  267. The predicate literal associated with a predicate symbol is C
  268. or C++ code which can be used to test the condition. A
  269. predicate expression defines a predicate symbol in terms of other
  270. predicate symbols using "!", "&&", and "||". A predicate symbol
  271. can be defined in terms of a predicate literal, a predicate
  272. expression, or *both*.
  273. When a predicate symbol is defined with both a predicate literal
  274. and a predicate expression, the predicate literal is used to generate
  275. code, but the predicate expression is used to check for two
  276. alternatives with identical predicates in both alternatives.
  277. Here are some examples of #pred statements:
  278. #pred IsLabel <<isLabel(LATEXT(1))>>?
  279. #pred IsLocalVar <<isLocalVar(LATEXT(1))>>?
  280. #pred IsGlobalVar <<isGlobalVar(LATEXT(1)>>?
  281. #pred IsVar <<isVar(LATEXT(1))>>? IsLocalVar || IsGlobalVar
  282. #pred IsScoped <<isScoped(LATEXT(1))>>? IsLabel || IsLocalVar
  283. I hope that the use of EBNF notation to describe the syntax of the
  284. #pred statement will not cause problems for my readers (joke).
  285. predStatement : "#pred"
  286. CapitalizedName
  287. (
  288. "<<predicate_literal>>?"
  289. | "<<predicate_literal>>?" predOrExpr
  290. | predOrExpr
  291. )
  292. ;
  293. predOrExpr : predAndExpr ( "||" predAndExpr ) * ;
  294. predAndExpr : predPrimary ( "&&" predPrimary ) * ;
  295. predPrimary : CapitalizedName
  296. | "!" predPrimary
  297. | "(" predOrExpr ")"
  298. ;
  299. What is the purpose of this nonsense ?
  300. To understand how predicate symbols help, you need to realize that
  301. predicate symbols are used in two different ways with two different
  302. goals.
  303. a. Allow simplification of predicates which have been combined
  304. during predicate hoisting.
  305. b. Allow recognition of identical predicates which can't disambiguate
  306. alternatives with common lookahead.
  307. First we will discuss goal (a). Consider the following rule:
  308. rule0: rule1
  309. | ID
  310. | ...
  311. ;
  312. rule1: rule2
  313. | rule3
  314. ;
  315. rule2: <<isX(LATEXT(1))>>? ID ;
  316. rule3: <<!isX(LATEXT(1)>>? ID ;
  317. When the predicates in rule2 and rule3 are combined by hoisting
  318. to create a prediction expression for rule1 the result is:
  319. if ( LA(1)==ID
  320. && ( isX(LATEXT(1) || !isX(LATEXT(1) ) ) { rule1(); ...
  321. This is inefficient, but more importantly, can lead to false
  322. assumptions that the predicate expression distinguishes the rule1
  323. alternative with some other alternative with lookahead ID. In
  324. MR11 one can write:
  325. #pred IsX <<isX(LATEXT(1))>>?
  326. ...
  327. rule2: <<IsX>>? ID ;
  328. rule3: <<!IsX>>? ID ;
  329. During hoisting MR11 recognizes this as a special case and
  330. eliminates the predicates. The result is a prediction
  331. expression like the following:
  332. if ( LA(1)==ID ) { rule1(); ...
  333. Please note that the following cases which appear to be equivalent
  334. *cannot* be simplified by MR11 during hoisting because the hoisting
  335. logic only checks for a "!" in the predicate action, not in the
  336. predicate expression for a predicate symbol.
  337. *Not* equivalent and is not simplified during hoisting:
  338. #pred IsX <<isX(LATEXT(1))>>?
  339. #pred NotX <<!isX(LATEXT(1))>>?
  340. ...
  341. rule2: <<IsX>>? ID ;
  342. rule3: <<NotX>>? ID ;
  343. *Not* equivalent and is not simplified during hoisting:
  344. #pred IsX <<isX(LATEXT(1))>>?
  345. #pred NotX !IsX
  346. ...
  347. rule2: <<IsX>>? ID ;
  348. rule3: <<NotX>>? ID ;
  349. Now we will discuss goal (b).
  350. When antlr discovers that there is a lookahead ambiguity between
  351. two alternatives it attempts to resolve the ambiguity by searching
  352. for predicates in both alternatives. In the past any predicate
  353. would do, even if the same one appeared in both alternatives:
  354. rule: <<p(LATEXT(1))>>? X
  355. | <<p(LATEXT(1))>>? X
  356. ;
  357. The #pred statement is a start towards solving this problem.
  358. During ambiguity resolution (*not* predicate hoisting) the
  359. predicates for the two alternatives are expanded and compared.
  360. Consider the following example:
  361. #pred Upper <<isUpper(LATEXT(1))>>?
  362. #pred Lower <<isLower(LATEXT(1))>>?
  363. #pred Alpha <<isAlpha(LATEXT(1))>>? Upper || Lower
  364. rule0: rule1
  365. | <<Alpha>>? ID
  366. ;
  367. rule1:
  368. | rule2
  369. | rule3
  370. ...
  371. ;
  372. rule2: <<Upper>>? ID;
  373. rule3: <<Lower>>? ID;
  374. The definition of #pred Alpha expresses:
  375. a. to test the predicate use the C code "isAlpha(LATEXT(1))"
  376. b. to analyze the predicate use the information that
  377. Alpha is equivalent to the union of Upper and Lower,
  378. During ambiguity resolution the definition of Alpha is expanded
  379. into "Upper || Lower" and compared with the predicate in the other
  380. alternative, which is also "Upper || Lower". Because they are
  381. identical MR11 will report a problem.
  382. -------------------------------------------------------------------------
  383. t10.g, line 5: warning: the predicates used to disambiguate rule rule0
  384. (file t10.g alt 1 line 5 and alt 2 line 6)
  385. are identical when compared without context and may have no
  386. resolving power for some lookahead sequences.
  387. -------------------------------------------------------------------------
  388. If you use the "-info p" option the output file will contain:
  389. +----------------------------------------------------------------------+
  390. |#if 0 |
  391. | |
  392. |The following predicates are identical when compared without |
  393. | lookahead context information. For some ambiguous lookahead |
  394. | sequences they may not have any power to resolve the ambiguity. |
  395. | |
  396. |Choice 1: rule0/1 alt 1 line 5 file t10.g |
  397. | |
  398. | The original predicate for choice 1 with available context |
  399. | information: |
  400. | |
  401. | OR expr |
  402. | |
  403. | pred << Upper>>? |
  404. | depth=k=1 rule rule2 line 14 t10.g |
  405. | set context: |
  406. | ID |
  407. | |
  408. | pred << Lower>>? |
  409. | depth=k=1 rule rule3 line 15 t10.g |
  410. | set context: |
  411. | ID |
  412. | |
  413. | The predicate for choice 1 after expansion (but without context |
  414. | information): |
  415. | |
  416. | OR expr |
  417. | |
  418. | pred << isUpper(LATEXT(1))>>? |
  419. | depth=k=1 rule line 1 t10.g |
  420. | |
  421. | pred << isLower(LATEXT(1))>>? |
  422. | depth=k=1 rule line 2 t10.g |
  423. | |
  424. | |
  425. |Choice 2: rule0/2 alt 2 line 6 file t10.g |
  426. | |
  427. | The original predicate for choice 2 with available context |
  428. | information: |
  429. | |
  430. | pred << Alpha>>? |
  431. | depth=k=1 rule rule0 line 6 t10.g |
  432. | set context: |
  433. | ID |
  434. | |
  435. | The predicate for choice 2 after expansion (but without context |
  436. | information): |
  437. | |
  438. | OR expr |
  439. | |
  440. | pred << isUpper(LATEXT(1))>>? |
  441. | depth=k=1 rule line 1 t10.g |
  442. | |
  443. | pred << isLower(LATEXT(1))>>? |
  444. | depth=k=1 rule line 2 t10.g |
  445. | |
  446. | |
  447. |#endif |
  448. +----------------------------------------------------------------------+
  449. The comparison of the predicates for the two alternatives takes
  450. place without context information, which means that in some cases
  451. the predicates will be considered identical even though they operate
  452. on disjoint lookahead sets. Consider:
  453. #pred Alpha
  454. rule1: <<Alpha>>? ID
  455. | <<Alpha>>? Label
  456. ;
  457. Because the comparison of predicates takes place without context
  458. these will be considered identical. The reason for comparing
  459. without context is that otherwise it would be necessary to re-evaluate
  460. the entire predicate expression for each possible lookahead sequence.
  461. This would require more code to be written and more CPU time during
  462. grammar analysis, and it is not yet clear whether anyone will even make
  463. use of the new #pred facility.
  464. A temporary workaround might be to use different #pred statements
  465. for predicates you know have different context. This would avoid
  466. extraneous warnings.
  467. The above example might be termed a "false positive". Comparison
  468. without context will also lead to "false negatives". Consider the
  469. following example:
  470. #pred Alpha
  471. #pred Beta
  472. rule1: <<Alpha>>? A
  473. | rule2
  474. ;
  475. rule2: <<Alpha>>? A
  476. | <<Beta>>? B
  477. ;
  478. The predicate used for alt 2 of rule1 is (Alpha || Beta). This
  479. appears to be different than the predicate Alpha used for alt1.
  480. However, the context of Beta is B. Thus when the lookahead is A
  481. Beta will have no resolving power and Alpha will be used for both
  482. alternatives. Using the same predicate for both alternatives isn't
  483. very helpful, but this will not be detected with 1.33MR11.
  484. To properly handle this the predicate expression would have to be
  485. evaluated for each distinct lookahead context.
  486. To determine whether two predicate expressions are identical is
  487. difficult. The routine may fail to identify identical predicates.
  488. The #pred feature also compares predicates to see if a choice between
  489. alternatives which is resolved by a predicate which makes the second
  490. choice unreachable. Consider the following example:
  491. #pred A <<A(LATEXT(1)>>?
  492. #pred B <<B(LATEXT(1)>>?
  493. #pred A_or_B A || B
  494. r : s
  495. | t
  496. ;
  497. s : <<A_or_B>>? ID
  498. ;
  499. t : <<A>>? ID
  500. ;
  501. ----------------------------------------------------------------------------
  502. t11.g, line 5: warning: the predicate used to disambiguate the
  503. first choice of rule r
  504. (file t11.g alt 1 line 5 and alt 2 line 6)
  505. appears to "cover" the second predicate when compared without context.
  506. The second predicate may have no resolving power for some lookahead
  507. sequences.
  508. ----------------------------------------------------------------------------
  509. #139. (Changed in MR11) Problem with -gp in C++ mode
  510. The -gp option to add a prefix to rule names did not work in
  511. C++ mode. This has been fixed.
  512. Reported by Alexey Demakov (demakov@kazbek.ispras.ru).
  513. #138. (Changed in MR11) Additional makefiles for non-MSVC++ MS systems
  514. Sramji Ramanathan (ps@kumaran.com) has supplied makefiles for
  515. building antlr and dlg with Win95/NT development tools that
  516. are not based on MSVC5. They are pccts/antlr/AntlrMS.mak and
  517. pccts/dlg/DlgMS.mak.
  518. The first line of the makefiles require a definition of PCCTS_HOME.
  519. These are in addition to the AntlrMSVC50.* and DlgMSVC50.*
  520. supplied by Jeff Vincent (JVincent@novell.com).
  521. #137. (Changed in MR11) Token getType(), getText(), getLine() const members
  522. --------------------------------------------------------------------
  523. If you use ANTLRCommonToken this change probably does not affect you.
  524. --------------------------------------------------------------------
  525. For a long time it has bothered me that these accessor functions
  526. in ANTLRAbstractToken were not const member functions. I have
  527. refrained from changing them because it require users to modify
  528. existing token class definitions which are derived directly
  529. from ANTLRAbstractToken. I think it is now time.
  530. For those who are not used to C++, a "const member function" is a
  531. member function which does not modify its own object - the thing
  532. to which "this" points. This is quite different from a function
  533. which does not modify its arguments
  534. Most token definitions based on ANTLRAbstractToken have something like
  535. the following in order to create concrete definitions of the pure
  536. virtual methods in ANTLRAbstractToken:
  537. class MyToken : public ANTLRAbstractToken {
  538. ...
  539. ANTLRTokenType getType() {return _type; }
  540. int getLine() {return _line; }
  541. ANTLRChar * getText() {return _text; }
  542. ...
  543. }
  544. The required change is simply to put "const" following the function
  545. prototype in the header (.h file) and the definition file (.cpp if
  546. it is not inline):
  547. class MyToken : public ANTLRAbstractToken {
  548. ...
  549. ANTLRTokenType getType() const {return _type; }
  550. int getLine() const {return _line; }
  551. ANTLRChar * getText() const {return _text; }
  552. ...
  553. }
  554. This was originally proposed a long time ago by Bruce
  555. Guenter (bruceg@qcc.sk.ca).
  556. #136. (Changed in MR11) Added getLength() to ANTLRCommonToken
  557. Classes ANTLRCommonToken and ANTLRCommonTokenNoRefCountToken
  558. now have a member function:
  559. int getLength() const { return strlen(getText()) }
  560. Suggested by Sramji Ramanathan (ps@kumaran.com).
  561. #135. (Changed in MR11) Raised antlr's own default ZZLEXBUFSIZE to 8k
  562. #134a. (ansi_mr10.zip) T.J. Parr's ANSI C grammar made 1.33MR11 compatible
  563. There is a typographical error in the definition of BITWISEOREQ:
  564. #token BITWISEOREQ "!=" should be "\|="
  565. When this change is combined with the bugfix to the follow set cache
  566. problem (Item #147) and a minor rearrangement of the grammar
  567. (Item #134b) it becomes a k=1 ck=2 grammar.
  568. #134b. (ansi_mr10.zip) T.J. Parr's ANSI C grammar made 1.33MR11 compatible
  569. The following changes were made in the ansi.g grammar (along with
  570. using -mrhoist on):
  571. ansi.g
  572. ======
  573. void tracein(char *) ====> void tracein(const char *)
  574. void traceout(char *) ====> void traceout(const char *)
  575. <LT(1)->getType()==IDENTIFIER ? isTypeName(LT(1)->getText()) : 1>>?
  576. ====> <<isTypeName(LT(1)->getText())>>?
  577. <<(LT(1)->getType()==LPARENTHESIS && LT(2)->getType()==IDENTIFIER) ? \
  578. isTypeName(LT(2)->getText()) : 1>>?
  579. ====> (LPARENTHESIS IDENTIFIER)? => <<isTypeName(LT(2)->getText())>>?
  580. <<(LT(1)->getType()==LPARENTHESIS && LT(2)->getType()==IDENTIFIER) ? \
  581. isTypeName(LT(2)->getText()) : 1>>?
  582. ====> (LPARENTHESIS IDENTIFIER)? => <<isTypeName(LT(2)->getText())>>?
  583. added to init(): traceOptionValueDefault=0;
  584. added to init(): traceOption(-1);
  585. change rule "statement":
  586. statement
  587. : plain_label_statement
  588. | case_label_statement
  589. | <<;>> expression SEMICOLON
  590. | compound_statement
  591. | selection_statement
  592. | iteration_statement
  593. | jump_statement
  594. | SEMICOLON
  595. ;
  596. plain_label_statement
  597. : IDENTIFIER COLON statement
  598. ;
  599. case_label_statement
  600. : CASE constant_expression COLON statement
  601. | DEFAULT COLON statement
  602. ;
  603. support.cpp
  604. ===========
  605. void tracein(char *) ====> void tracein(const char *)
  606. void traceout(char *) ====> void traceout(const char *)
  607. added to tracein(): ANTLRParser::tracein(r); // call superclass method
  608. added to traceout(): ANTLRParser::traceout(r); // call superclass method
  609. Makefile
  610. ========
  611. added to AFLAGS: -mrhoist on -prc on
  612. #133. (Changed in 1.33MR11) Make trace options public in ANTLRParser
  613. In checking T.J. Parr's ANSI C grammar for compatibility with
  614. 1.33MR11 discovered that it was inconvenient to have the
  615. trace facilities with protected access.
  616. #132. (Changed in 1.33MR11) Recognition of identical predicates in alts
  617. Prior to 1.33MR11, there would be no ambiguity warning when the
  618. very same predicate was used to disambiguate both alternatives:
  619. test: ref B
  620. | ref C
  621. ;
  622. ref : <<pred(LATEXT(1)>>? A
  623. In 1.33MR11 this will cause the warning:
  624. warning: the predicates used to disambiguate rule test
  625. (file v98.g alt 1 line 1 and alt 2 line 2)
  626. are identical and have no resolving power
  627. ----------------- Note -----------------
  628. This is different than the following case
  629. test: <<pred(LATEXT(1))>>? A B
  630. | <<pred(LATEXT(1)>>? A C
  631. ;
  632. In this case there are two distinct predicates
  633. which have exactly the same text. In the first
  634. example there are two references to the same
  635. predicate. The problem represented by this
  636. grammar will be addressed later.
  637. #131. (Changed in 1.33MR11) Case insensitive command line options
  638. Command line switches like "-CC" and keywords like "on", "off",
  639. and "stdin" are no longer case sensitive in antlr, dlg, and sorcerer.
  640. #130. (Changed in 1.33MR11) Changed ANTLR_VERSION to int from string
  641. The ANTLR_VERSION was not an integer, making it difficult to
  642. perform conditional compilation based on the antlr version.
  643. Henceforth, ANTLR_VERSION will be:
  644. (base_version * 10000) + release number
  645. thus 1.33MR11 will be: 133*100+11 = 13311
  646. Suggested by Rainer Janssen (Rainer.Janssen@Informatik.Uni-Oldenburg.DE).
  647. #129. (Changed in 1.33MR11) Addition of ANTLR_VERSION to <parserName>.h
  648. The following code is now inserted into <parserName>.h amd
  649. stdpccts.h:
  650. #ifndef ANTLR_VERSION
  651. #define ANTLR_VERSION 13311
  652. #endif
  653. Suggested by Rainer Janssen (Rainer.Janssen@Informatik.Uni-Oldenburg.DE)
  654. #128. (Changed in 1.33MR11) Redundant predicate code in (<<pred>>? ...)+
  655. Prior to 1.33MR11, the following grammar would generate
  656. redundant tests for the "while" condition.
  657. rule2 : (<<pred>>? X)+ X
  658. | B
  659. ;
  660. The code would resemble:
  661. if (LA(1)==X) {
  662. if (pred) {
  663. do {
  664. if (!pred) {zzfailed_pred(" pred");}
  665. zzmatch(X); zzCONSUME;
  666. } while (LA(1)==X && pred && pred);
  667. } else {...
  668. With 1.33MR11 the redundant predicate test is omitted.
  669. #127. (Changed in 1.33MR11)
  670. Count Syntax Errors Count DLG Errors
  671. ------------------- ----------------
  672. C++ mode ANTLRParser:: DLGLexerBase::
  673. syntaxErrCount lexErrCount
  674. C mode zzSyntaxErrCount zzLexErrCount
  675. The C mode variables are global and initialized to 0.
  676. They are *not* reset to 0 automatically when antlr is
  677. restarted.
  678. The C++ mode variables are public. They are initialized
  679. to 0 by the constructors. They are *not* reset to 0 by the
  680. ANTLRParser::init() method.
  681. Suggested by Reinier van den Born (reinier@vnet.ibm.com).
  682. #126. (Changed in 1.33MR11) Addition of #first <<...>>
  683. The #first <<...>> inserts the specified text in the output
  684. files before any other #include statements required by pccts.
  685. The only things before the #first text are comments and
  686. a #define ANTLR_VERSION.
  687. Requested by and Esa Pulkkinen (esap@cs.tut.fi) and Alexin
  688. Zoltan (alexin@inf.u-szeged.hu).
  689. #125. (Changed in 1.33MR11) Lookahead for (guard)? && <<p>>? predicates
  690. When implementing the new style of guard predicate (Item #113)
  691. in 1.33MR10 I decided to temporarily ignore the problem of
  692. computing the "narrowest" lookahead context.
  693. Consider the following k=1 grammar:
  694. start : a
  695. | b
  696. ;
  697. a : (A)? && <<pred1(LATEXT(1))>>? ab ;
  698. b : (B)? && <<pred2(LATEXT(1))>>? ab ;
  699. ab : A | B ;
  700. In MR10 the context for both "a" and "b" was {A B} because this is
  701. the first set of rule "ab". Normally, this is not a problem because
  702. the predicate which follows the guard inhibits any ambiguity report
  703. by antlr.
  704. In MR11 the first set for rule "a" is {A} and for rule "b" it is {B}.
  705. #124. A Note on the New "&&" Style Guarded Predicates
  706. I've been asked several times, "What is the difference between
  707. the old "=>" style guard predicates and the new style "&&" guard
  708. predicates, and how do you choose one over the other" ?
  709. The main difference is that the "=>" does not apply the
  710. predicate if the context guard doesn't match, whereas
  711. the && form always does. What is the significance ?
  712. If you have a predicate which is not on the "leading edge"
  713. it cannot be hoisted. Suppose you need a predicate that
  714. looks at LA(2). You must introduce it manually. The
  715. classic example is:
  716. castExpr :
  717. LP typeName RP
  718. | ....
  719. ;
  720. typeName : <<isTypeName(LATEXT(1))>>? ID
  721. | STRUCT ID
  722. ;
  723. The problem is that typeName isn't on the leading edge
  724. of castExpr, so the predicate isTypeName won't be hoisted into
  725. castExpr to help make a decision on which production to choose.
  726. The *first* attempt to fix it is this:
  727. castExpr :
  728. <<isTypeName(LATEXT(2))>>?
  729. LP typeName RP
  730. | ....
  731. ;
  732. Unfortunately, this won't work because it ignores
  733. the problem of STRUCT. The solution is to apply
  734. isTypeName() in castExpr if LA(2) is an ID and
  735. don't apply it when LA(2) is STRUCT:
  736. castExpr :
  737. (LP ID)? => <<isTypeName(LATEXT(2))>>?
  738. LP typeName RP
  739. | ....
  740. ;
  741. In conclusion, the "=>" style guarded predicate is
  742. useful when:
  743. a. the tokens required for the predicate
  744. are not on the leading edge
  745. b. there are alternatives in the expression
  746. selected by the predicate for which the
  747. predicate is inappropriate
  748. If (b) were false, then one could use a simple
  749. predicate (assuming "-prc on"):
  750. castExpr :
  751. <<isTypeName(LATEXT(2))>>?
  752. LP typeName RP
  753. | ....
  754. ;
  755. typeName : <<isTypeName(LATEXT(1))>>? ID
  756. ;
  757. So, when do you use the "&&" style guarded predicate ?
  758. The new-style "&&" predicate should always be used with
  759. predicate context. The context guard is in ADDITION to
  760. the automatically computed context. Thus it useful for
  761. predicates which depend on the token type for reasons
  762. other than context.
  763. The following example is contributed by Reinier van den Born
  764. (reinier@vnet.ibm.com).
  765. +-------------------------------------------------------------------------+
  766. | This grammar has two ways to call functions: |
  767. | |
  768. | - a "standard" call syntax with parens and comma separated args |
  769. | - a shell command like syntax (no parens and spacing separated args) |
  770. | |
  771. | The former also allows a variable to hold the name of the function, |
  772. | the latter can also be used to call external commands. |
  773. | |
  774. | The grammar (simplified) looks like this: |
  775. | |
  776. | fun_call : ID "(" { expr ("," expr)* } ")" |
  777. | /* ID is function name */ |
  778. | | "@" ID "(" { expr ("," expr)* } ")" |
  779. | /* ID is var containing fun name */ |
  780. | ; |
  781. | |
  782. | command : ID expr* /* ID is function name */ |
  783. | | path expr* /* path is external command name */ |
  784. | ; |
  785. | |
  786. | path : ID /* left out slashes and such */ |
  787. | | "@" ID /* ID is environment var */ |
  788. | ; |
  789. | |
  790. | expr : .... |
  791. | | "(" expr ")"; |
  792. | |
  793. | call : fun_call |
  794. | | command |
  795. | ; |
  796. | |
  797. | Obviously the call is wildly ambiguous. This is more or less how this |
  798. | is to be resolved: |
  799. | |
  800. | A call begins with an ID or an @ followed by an ID. |
  801. | |
  802. | If it is an ID and if it is an ext. command name -> command |
  803. | if followed by a paren -> fun_call |
  804. | otherwise -> command |
  805. | |
  806. | If it is an @ and if the ID is a var name -> fun_call |
  807. | otherwise -> command |
  808. | |
  809. | One can implement these rules quite neatly using && predicates: |
  810. | |
  811. | call : ("@" ID)? && <<isVarName(LT(2))>>? fun_call |
  812. | | (ID)? && <<isExtCmdName>>? command |
  813. | | (ID "(")? fun_call |
  814. | | command |
  815. | ; |
  816. | |
  817. | This can be done better, so it is not an ideal example, but it |
  818. | conveys the principle. |
  819. +-------------------------------------------------------------------------+
  820. #123. (Changed in 1.33MR11) Correct definition of operators in ATokPtr.h
  821. The return value of operators in ANTLRTokenPtr:
  822. changed: unsigned ... operator !=(...)
  823. to: int ... operator != (...)
  824. changed: unsigned ... operator ==(...)
  825. to: int ... operator == (...)
  826. Suggested by R.A. Nelson (cowboy@VNET.IBM.COM)
  827. #122. (Changed in 1.33MR11) Member functions to reset DLG in C++ mode
  828. void DLGFileReset(FILE *f) { input = f; found_eof = 0; }
  829. void DLGStringReset(DLGChar *s) { input = s; p = &input[0]; }
  830. Supplied by R.A. Nelson (cowboy@VNET.IBM.COM)
  831. #121. (Changed in 1.33MR11) Another attempt to fix -o (output dir) option
  832. Another attempt is made to improve the -o option of antlr, dlg,
  833. and sorcerer. This one by JVincent (JVincent@novell.com).
  834. The current rule:
  835. a. If -o is not specified than any explicit directory
  836. names are retained.
  837. b. If -o is specified than the -o directory name overrides any
  838. explicit directory names.
  839. c. The directory name of the grammar file is *not* stripped
  840. to create the main output file. However it is stil subject
  841. to override by the -o directory name.
  842. #120. (Changed in 1.33MR11) "-info f" output to stdout rather than stderr
  843. Added option 0 (e.g. "-info 0") which is a noop.
  844. #119. (Changed in 1.33MR11) Ambiguity aid for grammars
  845. The user can ask for additional information on ambiguities reported
  846. by antlr to stdout. At the moment, only one ambiguity report can
  847. be created in an antlr run.
  848. This feature is enabled using the "-aa" (Ambiguity Aid) option.
  849. The following options control the reporting of ambiguities:
  850. -aa ruleName Selects reporting by name of rule
  851. -aa lineNumber Selects reporting by line number
  852. (file name not compared)
  853. -aam Selects "multiple" reporting for a token
  854. in the intersection set of the
  855. alternatives.
  856. For instance, the token ID may appear dozens
  857. of times in various paths as the program
  858. explores the rules which are reachable from
  859. the point of an ambiguity. With option -aam
  860. every possible path the search program
  861. encounters is reported.
  862. Without -aam only the first encounter is
  863. reported. This may result in incomplete
  864. information, but the information may be
  865. sufficient and much shorter.
  866. -aad depth Selects the depth of the search.
  867. The default value is 1.
  868. The number of paths to be searched, and the
  869. size of the report can grow geometrically
  870. with the -ck value if a full search for all
  871. contributions to the source of the ambiguity
  872. is explored.
  873. The depth represents the number of tokens
  874. in the lookahead set which are matched against
  875. the set of ambiguous tokens. A depth of 1
  876. means that the search stops when a lookahead
  877. sequence of just one token is matched.
  878. A k=1 ck=6 grammar might generate 5,000 items
  879. in a report if a full depth 6 search is made
  880. with the Ambiguity Aid. The source of the
  881. problem may be in the first token and obscured
  882. by the volume of data - I hesitate to call
  883. it information.
  884. When the user selects a depth > 1, the search
  885. is first performed at depth=1 for both
  886. alternatives, then depth=2 for both alternatives,
  887. etc.
  888. Sample output for rule grammar in antlr.g itself:
  889. +---------------------------------------------------------------------+
  890. | Ambiguity Aid |
  891. | |
  892. | Choice 1: grammar/70 line 632 file a.g |
  893. | Choice 2: grammar/82 line 644 file a.g |
  894. | |
  895. | Intersection of lookahead[1] sets: |
  896. | |
  897. | "\}" "class" "#errclass" "#tokclass" |
  898. | |
  899. | Choice:1 Depth:1 Group:1 ("#errclass") |
  900. | 1 in (...)* block grammar/70 line 632 a.g |
  901. | 2 to error grammar/73 line 635 a.g |
  902. | 3 error error/1 line 894 a.g |
  903. | 4 #token "#errclass" error/2 line 895 a.g |
  904. | |
  905. | Choice:1 Depth:1 Group:2 ("#tokclass") |
  906. | 2 to tclass grammar/74 line 636 a.g |
  907. | 3 tclass tclass/1 line 937 a.g |
  908. | 4 #token "#tokclass" tclass/2 line 938 a.g |
  909. | |
  910. | Choice:1 Depth:1 Group:3 ("class") |
  911. | 2 to class_def grammar/75 line 637 a.g |
  912. | 3 class_def class_def/1 line 669 a.g |
  913. | 4 #token "class" class_def/3 line 671 a.g |
  914. | |
  915. | Choice:1 Depth:1 Group:4 ("\}") |
  916. | 2 #token "\}" grammar/76 line 638 a.g |
  917. | |
  918. | Choice:2 Depth:1 Group:5 ("#errclass") |
  919. | 1 in (...)* block grammar/83 line 645 a.g |
  920. | 2 to error grammar/93 line 655 a.g |
  921. | 3 error error/1 line 894 a.g |
  922. | 4 #token "#errclass" error/2 line 895 a.g |
  923. | |
  924. | Choice:2 Depth:1 Group:6 ("#tokclass") |
  925. | 2 to tclass grammar/94 line 656 a.g |
  926. | 3 tclass tclass/1 line 937 a.g |
  927. | 4 #token "#tokclass" tclass/2 line 938 a.g |
  928. | |
  929. | Choice:2 Depth:1 Group:7 ("class") |
  930. | 2 to class_def grammar/95 line 657 a.g |
  931. | 3 class_def class_def/1 line 669 a.g |
  932. | 4 #token "class" class_def/3 line 671 a.g |
  933. | |
  934. | Choice:2 Depth:1 Group:8 ("\}") |
  935. | 2 #token "\}" grammar/96 line 658 a.g |
  936. +---------------------------------------------------------------------+
  937. For a linear lookahead set ambiguity (where k=1 or for k>1 but
  938. when all lookahead sets [i] with i<k all have degree one) the
  939. reports appear in the following order:
  940. for (depth=1 ; depth <= "-aad depth" ; depth++) {
  941. for (alternative=1; alternative <=2 ; alternative++) {
  942. while (matches-are-found) {
  943. group++;
  944. print-report
  945. };
  946. };
  947. };
  948. For reporting a k-tuple ambiguity, the reports appear in the
  949. following order:
  950. for (depth=1 ; depth <= "-aad depth" ; depth++) {
  951. while (matches-are-found) {
  952. for (alternative=1; alternative <=2 ; alternative++) {
  953. group++;
  954. print-report
  955. };
  956. };
  957. };
  958. This is because matches are generated in different ways for
  959. linear lookahead and k-tuples.
  960. #118. (Changed in 1.33MR11) DEC VMS makefile and VMS related changes
  961. Revised makefiles for DEC/VMS operating system for antlr, dlg,
  962. and sorcerer.
  963. Reduced names of routines with external linkage to less than 32
  964. characters to conform to DEC/VMS linker limitations.
  965. Jean-Francois Pieronne discovered problems with dlg and antlr
  966. due to the VMS linker not being case sensitive for names with
  967. external linkage. In dlg the problem was with "className" and
  968. "ClassName". In antlr the problem was with "GenExprSets" and
  969. "genExprSets".
  970. Added genmms, a version of genmk for the DEC/VMS version of make.
  971. The source is in directory pccts/support/DECmms.
  972. All VMS contributions by Jean-Francois Pieronne (jfp@iname.com).
  973. #117. (Changed in 1.33MR10) new EXPERIMENTAL predicate hoisting code
  974. The hoisting of predicates into rules to create prediction
  975. expressions is a problem in antlr. Consider the following
  976. example (k=1 with -prc on):
  977. start : (a)* "@" ;
  978. a : b | c ;
  979. b : <<isUpper(LATEXT(1))>>? A ;
  980. c : A ;
  981. Prior to 1.33MR10 the code generated for "start" would resemble:
  982. while {
  983. if (LA(1)==A &&
  984. (!LA(1)==A || isUpper())) {
  985. a();
  986. }
  987. };
  988. This code is wrong because it makes rule "c" unreachable from
  989. "start". The essence of the problem is that antlr fails to
  990. recognize that there can be a valid alternative within "a" even
  991. when the predicate <<isUpper(LATEXT(1))>>? is false.
  992. In 1.33MR10 with -mrhoist the hoisting of the predicate into
  993. "start" is suppressed because it recognizes that "c" can
  994. cover all the cases where the predicate is false:
  995. while {
  996. if (LA(1)==A) {
  997. a();
  998. }
  999. };
  1000. With the antlr "-info p" switch the user will receive information
  1001. about the predicate suppression in the generated file:
  1002. --------------------------------------------------------------
  1003. #if 0
  1004. Hoisting of predicate suppressed by alternative without predicate.
  1005. The alt without the predicate includes all cases where
  1006. the predicate is false.
  1007. WITH predicate: line 7 v1.g
  1008. WITHOUT predicate: line 7 v1.g
  1009. The context set for the predicate:
  1010. A
  1011. The lookahead set for the alt WITHOUT the semantic predicate:
  1012. A
  1013. The predicate:
  1014. pred << isUpper(LATEXT(1))>>?
  1015. depth=k=1 rule b line 9 v1.g
  1016. set context:
  1017. A
  1018. tree context: null
  1019. Chain of referenced rules:
  1020. #0 in rule start (line 5 v1.g) to rule a
  1021. #1 in rule a (line 7 v1.g)
  1022. #endif
  1023. --------------------------------------------------------------
  1024. A predicate can be suppressed by a combination of alternatives
  1025. which, taken together, cover a predicate:
  1026. start : (a)* "@" ;
  1027. a : b | ca | cb | cc ;
  1028. b : <<isUpper(LATEXT(1))>>? ( A | B | C ) ;
  1029. ca : A ;
  1030. cb : B ;
  1031. cc : C ;
  1032. Consider a more complex example in which "c" covers only part of
  1033. a predicate:
  1034. start : (a)* "@" ;
  1035. a : b
  1036. | c
  1037. ;
  1038. b : <<isUpper(LATEXT(1))>>?
  1039. ( A
  1040. | X
  1041. );
  1042. c : A
  1043. ;
  1044. Prior to 1.33MR10 the code generated for "start" would resemble:
  1045. while {
  1046. if ( (LA(1)==A || LA(1)==X) &&
  1047. (! (LA(1)==A || LA(1)==X) || isUpper()) {
  1048. a();
  1049. }
  1050. };
  1051. With 1.33MR10 and -mrhoist the predicate context is restricted to
  1052. the non-covered lookahead. The code resembles:
  1053. while {
  1054. if ( (LA(1)==A || LA(1)==X) &&
  1055. (! (LA(1)==X) || isUpper()) {
  1056. a();
  1057. }
  1058. };
  1059. With the antlr "-info p" switch the user will receive information
  1060. about the predicate restriction in the generated file:
  1061. --------------------------------------------------------------
  1062. #if 0
  1063. Restricting the context of a predicate because of overlap
  1064. in the lookahead set between the alternative with the
  1065. semantic predicate and one without
  1066. Without this restriction the alternative without the predicate
  1067. could not be reached when input matched the context of the
  1068. predicate and the predicate was false.
  1069. WITH predicate: line 11 v4.g
  1070. WITHOUT predicate: line 12 v4.g
  1071. The original context set for the predicate:
  1072. A X
  1073. The lookahead set for the alt WITHOUT the semantic predicate:
  1074. A
  1075. The intersection of the two sets
  1076. A
  1077. The original predicate:
  1078. pred << isUpper(LATEXT(1))>>?
  1079. depth=k=1 rule b line 15 v4.g
  1080. set context:
  1081. A X
  1082. tree context: null
  1083. The new (modified) form of the predicate:
  1084. pred << isUpper(LATEXT(1))>>?
  1085. depth=k=1 rule b line 15 v4.g
  1086. set context:
  1087. X
  1088. tree context: null
  1089. #endif
  1090. --------------------------------------------------------------
  1091. The bad news about -mrhoist:
  1092. (a) -mrhoist does not analyze predicates with lookahead
  1093. depth > 1.
  1094. (b) -mrhoist does not look past a guarded predicate to
  1095. find context which might cover other predicates.
  1096. For these cases you might want to use syntactic predicates.
  1097. When a semantic predicate fails during guess mode the guess
  1098. fails and the next alternative is tried.
  1099. Limitation (a) is illustrated by the following example:
  1100. start : (stmt)* EOF ;
  1101. stmt : cast
  1102. | expr
  1103. ;
  1104. cast : <<isTypename(LATEXT(2))>>? LP ID RP ;
  1105. expr : LP ID RP ;
  1106. This is not much different from the first example, except that
  1107. it requires two tokens of lookahead context to determine what
  1108. to do. This predicate is NOT suppressed because the current version
  1109. is unable to handle predicates with depth > 1.
  1110. A predicate can be combined with other predicates during hoisting.
  1111. In those cases the depth=1 predicates are still handled. Thus,
  1112. in the following example the isUpper() predicate will be suppressed
  1113. by line #4 when hoisted from "bizarre" into "start", but will still
  1114. be present in "bizarre" in order to predict "stmt".
  1115. start : (bizarre)* EOF ; // #1
  1116. // #2
  1117. bizarre : stmt // #3
  1118. | A // #4
  1119. ;
  1120. stmt : cast
  1121. | expr
  1122. ;
  1123. cast : <<isTypename(LATEXT(2))>>? LP ID RP ;
  1124. expr : LP ID RP ;
  1125. | <<isUpper(LATEXT(1))>>? A
  1126. Limitation (b) is illustrated by the following example of a
  1127. context guarded predicate:
  1128. rule : (A)? <<p>>? // #1
  1129. (A // #2
  1130. |B // #3
  1131. ) // #4
  1132. | <<q>> B // #5
  1133. ;
  1134. Recall that this means that when the lookahead is NOT A then
  1135. the predicate "p" is ignored and it attempts to match "A|B".
  1136. Ideally, the "B" at line #3 should suppress predicate "q".
  1137. However, the current version does not attempt to look past
  1138. the guard predicate to find context which might suppress other
  1139. predicates.
  1140. In some cases -mrhoist will lead to the reporting of ambiguities
  1141. which were not visible before:
  1142. start : (a)* "@";
  1143. a : bc | d;
  1144. bc : b | c ;
  1145. b : <<isUpper(LATEXT(1))>>? A;
  1146. c : A ;
  1147. d : A ;
  1148. In this case there is a true ambiguity in "a" between "bc" and "d"
  1149. which can both match "A". Without -mrhoist the predicate in "b"
  1150. is hoisted into "a" and there is no ambiguity reported. However,
  1151. with -mrhoist, the predicate in "b" is suppressed by "c" (as it
  1152. should be) making the ambiguity in "a" apparent.
  1153. The motivations for these changes were hoisting problems reported
  1154. by Reinier van den Born (reinier@vnet.ibm.com) and several others.
  1155. #116. (Changed in 1.33MR10) C++ mode: tracein/traceout rule name is (const char *)
  1156. The prototype for C++ mode routine tracein (and traceout) has changed from
  1157. "char *" to "const char *".
  1158. #115. (Changed in 1.33MR10) Using guess mode with exception handlers in C mode
  1159. The definition of the C mode macros zzmatch_wsig and zzsetmatch_wsig
  1160. neglected to consider guess mode. When control passed to the rule's
  1161. parse exception handler the routine would exit without ever closing the
  1162. guess block. This would lead to unpredictable behavior.
  1163. In 1.33MR10 the behavior of exceptions in C mode and C++ mode should be
  1164. identical.
  1165. #114. (Changed in 1.33MR10) difference in [zz]resynch() between C and C++ modes
  1166. There was a slight difference in the way C and C++ mode resynchronized
  1167. following a parsing error. The C routine would sometimes skip an extra
  1168. token before attempting to resynchronize.
  1169. The C routine was changed to match the C++ routine.
  1170. #113. (Changed in 1.33MR10) new context guarded pred: (g)? && <<p>>? expr
  1171. The existing context guarded predicate:
  1172. rule : (guard)? => <<p>>? expr
  1173. | next_alternative
  1174. ;
  1175. generates code which resembles:
  1176. if (lookahead(expr) && (!guard || pred)) {
  1177. expr()
  1178. } else ....
  1179. This is not suitable for some applications because it allows
  1180. expr() to be invoked when the predicate is false. This is
  1181. intentional because it is meant to mimic automatically computed
  1182. predicate context.
  1183. The new context guarded predicate uses the guard information
  1184. differently because it has a different goal. Consider:
  1185. rule : (guard)? && <<p>>? expr
  1186. | next_alternative
  1187. ;
  1188. The new style of context guarded predicate is equivalent to:
  1189. rule : <<guard==true && pred>>? expr
  1190. | next_alternative
  1191. ;
  1192. It generates code which resembles:
  1193. if (lookahead(expr) && guard && pred) {
  1194. expr();
  1195. } else ...
  1196. Both forms of guarded predicates severely restrict the form of
  1197. the context guard: it can contain no rule references, no
  1198. (...)*, no (...)+, and no {...}. It may contain token and
  1199. token class references, and alternation ("|").
  1200. Addition for 1.33MR11: in the token expression all tokens must
  1201. be at the same height of the token tree:
  1202. (A ( B | C))? && ... is ok (all height 2)
  1203. (A ( B | ))? && ... is not ok (some 1, some 2)
  1204. (A B C D | E F G H)? && ... is ok (all height 4)
  1205. (A B C D | E )? && ... is not ok (some 4, some 1)
  1206. This restriction is required in order to properly compute the lookahead
  1207. set for expressions like:
  1208. rule1 : (A B C)? && <<pred>>? rule2 ;
  1209. rule2 : (A|X) (B|Y) (C|Z);
  1210. This addition was suggested by Rienier van den Born (reinier@vnet.ibm.com)
  1211. #112. (Changed in 1.33MR10) failed validation predicate in C guess mode
  1212. John Lilley (jlilley@empathy.com) suggested that failed validation
  1213. predicates abort a guess rather than reporting a failed error.
  1214. This was installed in C++ mode (Item #4). Only now was it noticed
  1215. that the fix was never installed for C mode.
  1216. #111. (Changed in 1.33MR10) moved zzTRACEIN to before init action
  1217. When the antlr -gd switch is present antlr generates calls to
  1218. zzTRACEIN at the start of a rule and zzTRACEOUT at the exit
  1219. from a rule. Prior to 1.33MR10 Tthe call to zzTRACEIN was
  1220. after the init-action, which could cause confusion because the
  1221. init-actions were reported with the name of the enclosing rule,
  1222. rather than the active rule.
  1223. #110. (Changed in 1.33MR10) antlr command line copied to generated file
  1224. The antlr command line is now copied to the generated file near
  1225. the start.
  1226. #109. (Changed in 1.33MR10) improved trace information
  1227. The quality of the trace information provided by the "-gd"
  1228. switch has been improved significantly. Here is an example
  1229. of the output from a test program. It shows the rule name,
  1230. the first token of lookahead, the call depth, and the guess
  1231. status:
  1232. exit rule gusxx {"?"} depth 2
  1233. enter rule gusxx {"?"} depth 2
  1234. enter rule gus1 {"o"} depth 3 guessing
  1235. guess done - returning to rule gus1 {"o"} at depth 3
  1236. (guess mode continues - an enclosing guess is still active)
  1237. guess done - returning to rule gus1 {"Z"} at depth 3
  1238. (guess mode continues - an enclosing guess is still active)
  1239. exit rule gus1 {"Z"} depth 3 guessing
  1240. guess done - returning to rule gusxx {"o"} at depth 2 (guess mode ends)
  1241. enter rule gus1 {"o"} depth 3
  1242. guess done - returning to rule gus1 {"o"} at depth 3 (guess mode ends)
  1243. guess done - returning to rule gus1 {"Z"} at depth 3 (guess mode ends)
  1244. exit rule gus1 {"Z"} depth 3
  1245. line 1: syntax error at "Z" missing SC
  1246. ...
  1247. Rule trace reporting is controlled by the value of the integer
  1248. [zz]traceOptionValue: when it is positive tracing is enabled,
  1249. otherwise it is disabled. Tracing during guess mode is controlled
  1250. by the value of the integer [zz]traceGuessOptionValue. When
  1251. it is positive AND [zz]traceOptionValue is positive rule trace
  1252. is reported in guess mode.
  1253. The values of [zz]traceOptionValue and [zz]traceGuessOptionValue
  1254. can be adjusted by subroutine calls listed below.
  1255. Depending on the presence or absence of the antlr -gd switch
  1256. the variable [zz]traceOptionValueDefault is set to 0 or 1. When
  1257. the parser is initialized or [zz]traceReset() is called the
  1258. value of [zz]traceOptionValueDefault is copied to [zz]traceOptionValue.
  1259. The value of [zz]traceGuessOptionValue is always initialized to 1,
  1260. but, as noted earlier, nothing will be reported unless
  1261. [zz]traceOptionValue is also positive.
  1262. When the parser state is saved/restored the value of the trace
  1263. variables are also saved/restored. If a restore causes a change in
  1264. reporting behavior from on to off or vice versa this will be reported.
  1265. When the -gd option is selected, the macro "#define zzTRACE_RULES"
  1266. is added to appropriate output files.
  1267. C++ mode
  1268. --------
  1269. int traceOption(int delta)
  1270. int traceGuessOption(int delta)
  1271. void traceReset()
  1272. int traceOptionValueDefault
  1273. C mode
  1274. --------
  1275. int zzTraceOption(int delta)
  1276. int zzTraceGuessOption(int delta)
  1277. void zzTraceReset()
  1278. int zzTraceOptionValueDefault
  1279. The argument "delta" is added to the traceOptionValue. To
  1280. turn on trace when inside a particular rule one:
  1281. rule : <<traceOption(+1);>>
  1282. (
  1283. rest-of-rule
  1284. )
  1285. <<traceOption(-1);>>
  1286. ; /* fail clause */ <<traceOption(-1);>>
  1287. One can use the same idea to turn *off* tracing within a
  1288. rule by using a delta of (-1).
  1289. An improvement in the rule trace was suggested by Sramji
  1290. Ramanathan (ps@kumaran.com).
  1291. #108. A Note on Deallocation of Variables Allocated in Guess Mode
  1292. NOTE
  1293. ------------------------------------------------------
  1294. This mechanism only works for heap allocated variables
  1295. ------------------------------------------------------
  1296. The rewrite of the trace provides the machinery necessary
  1297. to properly free variables or undo actions following a
  1298. failed guess.
  1299. The macro zzUSER_GUESS_HOOK(guessSeq,zzrv) is expanded
  1300. as part of the zzGUESS macro. When a guess is opened
  1301. the value of zzrv is 0. When a longjmp() is executed to
  1302. undo the guess, the value of zzrv will be 1.
  1303. The macro zzUSER_GUESS_DONE_HOOK(guessSeq) is expanded
  1304. as part of the zzGUESS_DONE macro. This is executed
  1305. whether the guess succeeds or fails as part of closing
  1306. the guess.
  1307. The guessSeq is a sequence number which is assigned to each
  1308. guess and is incremented by 1 for each guess which becomes
  1309. active. It is needed by the user to associate the start of
  1310. a guess with the failure and/or completion (closing) of a
  1311. guess.
  1312. Guesses are nested. They must be closed in the reverse
  1313. of the order that they are opened.
  1314. In order to free memory used by a variable during a guess
  1315. a user must write a routine which can be called to
  1316. register the variable along with the current guess sequence
  1317. number provided by the zzUSER_GUESS_HOOK macro. If the guess
  1318. fails, all variables tagged with the corresponding guess
  1319. sequence number should be released. This is ugly, but
  1320. it would require a major rewrite of antlr 1.33 to use
  1321. some mechanism other than setjmp()/longjmp().
  1322. The order of calls for a *successful* guess would be:
  1323. zzUSER_GUESS_HOOK(guessSeq,0);
  1324. zzUSER_GUESS_DONE_HOOK(guessSeq);
  1325. The order of calls for a *failed* guess would be:
  1326. zzUSER_GUESS_HOOK(guessSeq,0);
  1327. zzUSER_GUESS_HOOK(guessSeq,1);
  1328. zzUSER_GUESS_DONE_HOOK(guessSeq);
  1329. The default definitions of these macros are empty strings.
  1330. Here is an example in C++ mode. The zzUSER_GUESS_HOOK and
  1331. zzUSER_GUESS_DONE_HOOK macros and myGuessHook() routine
  1332. can be used without change in both C and C++ versions.
  1333. ----------------------------------------------------------------------
  1334. <<
  1335. #include "AToken.h"
  1336. typedef ANTLRCommonToken ANTLRToken;
  1337. #include "DLGLexer.h"
  1338. int main() {
  1339. {
  1340. DLGFileInput in(stdin);
  1341. DLGLexer lexer(&in,2000);
  1342. ANTLRTokenBuffer pipe(&lexer,1);
  1343. ANTLRCommonToken aToken;
  1344. P parser(&pipe);
  1345. lexer.setToken(&aToken);
  1346. parser.init();
  1347. parser.start();
  1348. };
  1349. fclose(stdin);
  1350. fclose(stdout);
  1351. return 0;
  1352. }
  1353. >>
  1354. <<
  1355. char *s=NULL;
  1356. #undef zzUSER_GUESS_HOOK
  1357. #define zzUSER_GUESS_HOOK(guessSeq,zzrv) myGuessHook(guessSeq,zzrv);
  1358. #undef zzUSER_GUESS_DONE_HOOK
  1359. #define zzUSER_GUESS_DONE_HOOK(guessSeq) myGuessHook(guessSeq,2);
  1360. void myGuessHook(int guessSeq,int zzrv) {
  1361. if (zzrv == 0) {
  1362. fprintf(stderr,"User hook: starting guess #%d\n",guessSeq);
  1363. } else if (zzrv == 1) {
  1364. free (s);
  1365. s=NULL;
  1366. fprintf(stderr,"User hook: failed guess #%d\n",guessSeq);
  1367. } else if (zzrv == 2) {
  1368. free (s);
  1369. s=NULL;
  1370. fprintf(stderr,"User hook: ending guess #%d\n",guessSeq);
  1371. };
  1372. }
  1373. >>
  1374. #token A "a"
  1375. #token "[\t \ \n]" <<skip();>>
  1376. class P {
  1377. start : (top)+
  1378. ;
  1379. top : (which) ? <<fprintf(stderr,"%s is a which\n",s); free(s); s=NULL; >>
  1380. | other <<fprintf(stderr,"%s is an other\n",s); free(s); s=NULL; >>
  1381. ; <<if (s != NULL) free(s); s=NULL; >>
  1382. which : which2
  1383. ;
  1384. which2 : which3
  1385. ;
  1386. which3
  1387. : (label)? <<fprintf(stderr,"%s is a label\n",s);>>
  1388. | (global)? <<fprintf(stderr,"%s is a global\n",s);>>
  1389. | (exclamation)? <<fprintf(stderr,"%s is an exclamation\n",s);>>
  1390. ;
  1391. label : <<s=strdup(LT(1)->getText());>> A ":" ;
  1392. global : <<s=strdup(LT(1)->getText());>> A "::" ;
  1393. exclamation : <<s=strdup(LT(1)->getText());>> A "!" ;
  1394. other : <<s=strdup(LT(1)->getText());>> "other" ;
  1395. }
  1396. ----------------------------------------------------------------------
  1397. This is a silly example, but illustrates the idea. For the input
  1398. "a ::" with tracing enabled the output begins:
  1399. ----------------------------------------------------------------------
  1400. enter rule "start" depth 1
  1401. enter rule "top" depth 2
  1402. User hook: starting guess #1
  1403. enter rule "which" depth 3 guessing
  1404. enter rule "which2" depth 4 guessing
  1405. enter rule "which3" depth 5 guessing
  1406. User hook: starting guess #2
  1407. enter rule "label" depth 6 guessing
  1408. guess failed
  1409. User hook: failed guess #2
  1410. guess done - returning to rule "which3" at depth 5 (guess mode continues
  1411. - an enclosing guess is still active)
  1412. User hook: ending guess #2
  1413. User hook: starting guess #3
  1414. enter rule "global" depth 6 guessing
  1415. exit rule "global" depth 6 guessing
  1416. guess done - returning to rule "which3" at depth 5 (guess mode continues
  1417. - an enclosing guess is still active)
  1418. User hook: ending guess #3
  1419. enter rule "global" depth 6 guessing
  1420. exit rule "global" depth 6 guessing
  1421. exit rule "which3" depth 5 guessing
  1422. exit rule "which2" depth 4 guessing
  1423. exit rule "which" depth 3 guessing
  1424. guess done - returning to rule "top" at depth 2 (guess mode ends)
  1425. User hook: ending guess #1
  1426. enter rule "which" depth 3
  1427. .....
  1428. ----------------------------------------------------------------------
  1429. Remember:
  1430. (a) Only init-actions are executed during guess mode.
  1431. (b) A rule can be invoked multiple times during guess mode.
  1432. (c) If the guess succeeds the rule will be called once more
  1433. without guess mode so that normal actions will be executed.
  1434. This means that the init-action might need to distinguish
  1435. between guess mode and non-guess mode using the variable
  1436. [zz]guessing.
  1437. #107. (Changed in 1.33MR10) construction of ASTs in guess mode
  1438. Prior to 1.33MR10, when using automatic AST construction in C++
  1439. mode for a rule, an AST would be constructed for elements of the
  1440. rule even while in guess mode. In MR10 this no longer occurs.
  1441. #106. (Changed in 1.33MR10) guess variable confusion
  1442. In C++ mode a guess which failed always restored the parser state
  1443. using zzGUESS_DONE as part of zzGUESS_FAIL. Prior to 1.33MR10,
  1444. C mode required an explicit call to zzGUESS_DONE after the
  1445. call to zzGUESS_FAIL.
  1446. Consider:
  1447. rule : (alpha)? beta
  1448. | ...
  1449. ;
  1450. The generated code resembles:
  1451. zzGUESS
  1452. if (!zzrv && LA(1)==ID) { <==== line #1
  1453. alpha
  1454. zzGUESS_DONE
  1455. beta
  1456. } else {
  1457. if (! zzrv) zzGUESS_DONE <==== line #2a
  1458. ....
  1459. However, in some cases line #2 was rendered:
  1460. if (guessing) zzGUESS_DONE <==== line #2b
  1461. This would work for simple test cases, but would fail in
  1462. some cases where there was a guess while another guess was active.
  1463. One kind of failure would be to match up the zzGUESS_DONE at line
  1464. #2b with the "outer" guess which was still active. The outer
  1465. guess would "succeed" when only the inner guess should have
  1466. succeeded.
  1467. In 1.33MR10 the behavior of zzGUESS and zzGUESS_FAIL in C and
  1468. and C++ mode should be identical.
  1469. The same problem appears in 1.33 vanilla in some places. For
  1470. example:
  1471. start : { (sub)? } ;
  1472. or:
  1473. start : (
  1474. B
  1475. | ( sub )?
  1476. | C
  1477. )+
  1478. ;
  1479. generates incorrect code.
  1480. The general principle is:
  1481. (a) use [zz]guessing only when deciding between a call to zzFAIL
  1482. or zzGUESS_FAIL
  1483. (b) use zzrv in all other cases
  1484. This problem was discovered while testing changes to item #105.
  1485. I believe this is now fixed. My apologies.
  1486. #105. (Changed in 1.33MR10) guess block as single alt of (...)+
  1487. Prior to 1.33MR10 the following constructs:
  1488. rule_plus : (
  1489. (sub)?
  1490. )+
  1491. ;
  1492. rule_star : (
  1493. (sub)?
  1494. )*
  1495. ;
  1496. generated incorrect code for the guess block (which could result
  1497. in runtime errors) because of an incorrect optimization of a
  1498. block with only a single alternative.
  1499. The fix caused some changes to the fix described in Item #49
  1500. because there are now three code generation sequences for (...)+
  1501. blocks containing a guess block:
  1502. a. single alternative which is a guess block
  1503. b. multiple alternatives in which the last is a guess block
  1504. c. all other cases
  1505. Forms like "rule_star" can have unexpected behavior when there
  1506. is a syntax error: if the subrule "sub" is not matched *exactly*
  1507. then "rule_star" will consume no tokens.
  1508. Reported by Esa Pulkkinen (esap@cs.tut.fi).
  1509. #104. (Changed in 1.33MR10) -o option for dlg
  1510. There was problem with the code added by item #74 to handle the
  1511. -o option of dlg. This should fix it.
  1512. #103. (Changed in 1.33MR10) ANDed semantic predicates
  1513. Rescinded.
  1514. The optimization was a mistake.
  1515. The resulting problem is described in Item #150.
  1516. #102. (Changed in 1.33MR10) allow "class parser : .... {"
  1517. The syntax of the class statement ("class parser-name {")
  1518. has been extended to allow for the specification of base
  1519. classes. An arbitrary number of tokens may now appear
  1520. between the class name and the "{". They are output
  1521. again when the class declaration is generated. For
  1522. example:
  1523. class Parser : public MyBaseClassANTLRparser {
  1524. This was suggested by a user, but I don't have a record
  1525. of who it was.
  1526. #101. (Changed in 1.33MR10) antlr -info command line switch
  1527. -info
  1528. p - extra predicate information in generated file
  1529. t - information about tnode use:
  1530. at the end of each rule in generated file
  1531. summary on stderr at end of program
  1532. m - monitor progress
  1533. prints name of each rule as it is started
  1534. flushes output at start of each rule
  1535. f - first/follow set information to stdout
  1536. 0 - no operation (added in 1.33MR11)
  1537. The options may be combined and may appear in any order.
  1538. For example:
  1539. antlr -info ptm -CC -gt -mrhoist on mygrammar.g
  1540. #100a. (Changed in 1.33MR10) Predicate tree simplification
  1541. When the same predicates can be referenced in more than one
  1542. alternative of a block large predicate trees can be formed.
  1543. The difference that these optimizations make is so dramatic
  1544. that I have decided to use it even when -mrhoist is not selected.
  1545. Consider the following grammar:
  1546. start : ( all )* ;
  1547. all : a
  1548. | d
  1549. | e
  1550. | f
  1551. ;
  1552. a : c A B
  1553. | c A C
  1554. ;
  1555. c : <<AAA(LATEXT(2))>>?
  1556. ;
  1557. d : <<BBB(LATEXT(2))>>? B C
  1558. ;
  1559. e : <<CCC(LATEXT(2))>>? B C
  1560. ;
  1561. f : e X Y
  1562. ;
  1563. In rule "a" there is a reference to rule "c" in both alternatives.
  1564. The length of the predicate AAA is k=2 and it can be followed in
  1565. alternative 1 only by (A B) while in alternative 2 it can be
  1566. followed only by (A C). Thus they do not have identical context.
  1567. In rule "all" the alternatives which refer to rules "e" and "f" allow
  1568. elimination of the duplicate reference to predicate CCC.
  1569. The table below summarized the kind of simplification performed by
  1570. 1.33MR10. In the table, X and Y stand for single predicates
  1571. (not trees).
  1572. (OR X (OR Y (OR Z))) => (OR X Y Z)
  1573. (AND X (AND Y (AND Z))) => (AND X Y Z)
  1574. (OR X (... (OR X Y) ... )) => (OR X (... Y ... ))
  1575. (AND X (... (AND X Y) ... )) => (AND X (... Y ... ))
  1576. (OR X (... (AND X Y) ... )) => (OR X (... ... ))
  1577. (AND X (... (OR X Y) ... )) => (AND X (... ... ))
  1578. (AND X) => X
  1579. (OR X) => X
  1580. In a test with a complex grammar for a real application, a predicate
  1581. tree with six OR nodes and 12 leaves was reduced to "(OR X Y Z)".
  1582. In 1.33MR10 there is a greater effort to release memory used
  1583. by predicates once they are no longer in use.
  1584. #100b. (Changed in 1.33MR10) Suppression of extra predicate tests
  1585. The following optimizations require that -mrhoist be selected.
  1586. It is relatively easy to optimize the code generated for predicate
  1587. gates when they are of the form:
  1588. (AND X Y Z ...)
  1589. or (OR X Y Z ...)
  1590. where X, Y, Z, and "..." represent individual predicates (leaves) not
  1591. predicate trees.
  1592. If the predicate is an AND the contexts of the X, Y, Z, etc. are
  1593. ANDed together to create a single Tree context for the group and
  1594. context tests for the individual predicates are suppressed:
  1595. --------------------------------------------------
  1596. Note: This was incorrect. The contexts should be
  1597. ORed together. This has been fixed. A more
  1598. complete description is available in item #152.
  1599. ---------------------------------------------------
  1600. Optimization 1: (AND X Y Z ...)
  1601. Suppose the context for Xtest is LA(1)==LP and the context for
  1602. Ytest is LA(1)==LP && LA(2)==ID.
  1603. Without the optimization the code would resemble:
  1604. if (lookaheadContext &&
  1605. !(LA(1)==LP && LA(1)==LP && LA(2)==ID) ||
  1606. ( (! LA(1)==LP || Xtest) &&
  1607. (! (LA(1)==LP || LA(2)==ID) || Xtest)
  1608. )) {...
  1609. With the -mrhoist optimization the code would resemble:
  1610. if (lookaheadContext &&
  1611. ! (LA(1)==LP && LA(2)==ID) || (Xtest && Ytest) {...
  1612. Optimization 2: (OR X Y Z ...) with identical contexts
  1613. Suppose the context for Xtest is LA(1)==ID and for Ytest
  1614. the context is also LA(1)==ID.
  1615. Without the optimization the code would resemble:
  1616. if (lookaheadContext &&
  1617. ! (LA(1)==ID || LA(1)==ID) ||
  1618. (LA(1)==ID && Xtest) ||
  1619. (LA(1)==ID && Ytest) {...
  1620. With the -mrhoist optimization the code would resemble:
  1621. if (lookaheadContext &&
  1622. (! LA(1)==ID) || (Xtest || Ytest) {...
  1623. Optimization 3: (OR X Y Z ...) with distinct contexts
  1624. Suppose the context for Xtest is LA(1)==ID and for Ytest
  1625. the context is LA(1)==LP.
  1626. Without the optimization the code would resemble:
  1627. if (lookaheadContext &&
  1628. ! (LA(1)==ID || LA(1)==LP) ||
  1629. (LA(1)==ID && Xtest) ||
  1630. (LA(1)==LP && Ytest) {...
  1631. With the -mrhoist optimization the code would resemble:
  1632. if (lookaheadContext &&
  1633. (zzpf=0,
  1634. (LA(1)==ID && (zzpf=1) && Xtest) ||
  1635. (LA(1)==LP && (zzpf=1) && Ytest) ||
  1636. !zzpf) {
  1637. These may appear to be of similar complexity at first,
  1638. but the non-optimized version contains two tests of each
  1639. context while the optimized version contains only one
  1640. such test, as well as eliminating some of the inverted
  1641. logic (" !(...) || ").
  1642. Optimization 4: Computation of predicate gate trees
  1643. When generating code for the gates of predicate expressions
  1644. antlr 1.33 vanilla uses a recursive procedure to generate
  1645. "&&" and "||" expressions for testing the lookahead. As each
  1646. layer of the predicate tree is exposed a new set of "&&" and
  1647. "||" expressions on the lookahead are generated. In many
  1648. cases the lookahead being tested has already been tested.
  1649. With -mrhoist a lookahead tree is computed for the entire
  1650. lookahead expression. This means that predicates with identical
  1651. context or context which is a subset of another predicate's
  1652. context disappear.
  1653. This is especially important for predicates formed by rules
  1654. like the following:
  1655. upperCaseVowel : <<isUpperCase(LATEXT(1))>>? vowel;
  1656. vowel: : <<isVowel(LATEXT(1))>>? LETTERS;
  1657. These predicates are combined using AND since both must be
  1658. satisfied for rule upperCaseVowel. They have identical
  1659. context which makes this optimization very effective.
  1660. The affect of Items #100a and #100b together can be dramatic. In
  1661. a very large (but real world) grammar one particular predicate
  1662. expression was reduced from an (unreadable) 50 predicate leaves,
  1663. 195 LA(1) terms, and 5500 characters to an (easily comprehensible)
  1664. 3 predicate leaves (all different) and a *single* LA(1) term.
  1665. #99. (Changed in 1.33MR10) Code generation for expression trees
  1666. Expression trees are used for k>1 grammars and predicates with
  1667. lookahead depth >1. This optimization must be enabled using
  1668. "-mrhoist on". (Clarification added for 1.33MR11).
  1669. In the processing of expression trees, antlr can generate long chains
  1670. of token comparisons. Prior to 1.33MR10 there were many redundant
  1671. parenthesis which caused problems for compilers which could handle
  1672. expressions of only limited complexity. For example, to test an
  1673. expression tree (root R A B C D), antlr would generate something
  1674. resembling:
  1675. (LA(1)==R && (LA(2)==A || (LA(2)==B || (LA(2)==C || LA(2)==D)))))
  1676. If there were twenty tokens to test then there would be twenty
  1677. parenthesis at the end of the expression.
  1678. In 1.33MR10 the generated code for tree expressions resembles:
  1679. (LA(1)==R && (LA(2)==A || LA(2)==B || LA(2)==C || LA(2)==D))
  1680. For "complex" expressions the output is indented to reflect the LA
  1681. number being tested:
  1682. (LA(1)==R
  1683. && (LA(2)==A || LA(2)==B || LA(2)==C || LA(2)==D
  1684. || LA(2)==E || LA(2)==F)
  1685. || LA(1)==S
  1686. && (LA(2)==G || LA(2)==H))
  1687. Suggested by S. Bochnak (S.Bochnak@@microTool.com.pl),
  1688. #98. (Changed in 1.33MR10) Option "-info p"
  1689. When the user selects option "-info p" the program will generate
  1690. detailed information about predicates. If the user selects
  1691. "-mrhoist on" additional detail will be provided explaining
  1692. the promotion and suppression of predicates. The output is part
  1693. of the generated file and sandwiched between #if 0/#endif statements.
  1694. Consider the following k=1 grammar:
  1695. start : ( all ) * ;
  1696. all : ( a
  1697. | b
  1698. )
  1699. ;
  1700. a : c B
  1701. ;
  1702. c : <<LATEXT(1)>>?
  1703. | B
  1704. ;
  1705. b : <<LATEXT(1)>>? X
  1706. ;
  1707. Below is an excerpt of the output for rule "start" for the three
  1708. predicate options (off, on, and maintenance release style hoisting).
  1709. For those who do not wish to use the "-mrhoist on" option for code
  1710. generation the option can be used in a "diagnostic" mode to provide
  1711. valuable information:
  1712. a. where one should insert null actions to inhibit hoisting
  1713. b. a chain of rule references which shows where predicates are
  1714. being hoisted
  1715. ======================================================================
  1716. Example of "-info p" with "-mrhoist on"
  1717. ======================================================================
  1718. #if 0
  1719. Hoisting of predicate suppressed by alternative without predicate.
  1720. The alt without the predicate includes all cases where the
  1721. predicate is false.
  1722. WITH predicate: line 11 v36.g
  1723. WITHOUT predicate: line 12 v36.g
  1724. The context set for the predicate:
  1725. B
  1726. The lookahead set for alt WITHOUT the semantic predicate:
  1727. B
  1728. The predicate:
  1729. pred << LATEXT(1)>>? depth=k=1 rule c line 11 v36.g
  1730. set context:
  1731. B
  1732. tree context: null
  1733. Chain of referenced rules:
  1734. #0 in rule start (line 1 v36.g) to rule all
  1735. #1 in rule all (line 3 v36.g) to rule a
  1736. #2 in rule a (line 8 v36.g) to rule c
  1737. #3 in rule c (line 11 v36.g)
  1738. #endif
  1739. &&
  1740. #if 0
  1741. pred << LATEXT(1)>>? depth=k=1 rule b line 15 v36.g
  1742. set context:
  1743. X
  1744. tree context: null
  1745. #endif
  1746. ======================================================================
  1747. Example of "-info p" with the default -prc setting ( "-prc off")
  1748. ======================================================================
  1749. #if 0
  1750. OR
  1751. pred << LATEXT(1)>>? depth=k=1 rule c line 11 v36.g
  1752. set context:
  1753. nil
  1754. tree context: null
  1755. pred << LATEXT(1)>>? depth=k=1 rule b line 15 v36.g
  1756. set context:
  1757. nil
  1758. tree context: null
  1759. #endif
  1760. ======================================================================
  1761. Example of "-info p" with "-prc on" and "-mrhoist off"
  1762. ======================================================================
  1763. #if 0
  1764. OR
  1765. pred << LATEXT(1)>>? depth=k=1 rule c line 11 v36.g
  1766. set context:
  1767. B
  1768. tree context: null
  1769. pred << LATEXT(1)>>? depth=k=1 rule b line 15 v36.g
  1770. set context:
  1771. X
  1772. tree context: null
  1773. #endif
  1774. ======================================================================
  1775. #97. (Fixed in 1.33MR10) "Predicate applied for more than one ... "
  1776. In 1.33 vanilla, the grammar listed below produced this message for
  1777. the first alternative (only) of rule "b":
  1778. warning: predicate applied for >1 lookahead 1-sequences
  1779. [you may only want one lookahead 1-sequence to apply.
  1780. Try using a context guard '(...)? =>'
  1781. In 1.33MR10 the message is issued for both alternatives.
  1782. top : (a)*;
  1783. a : b | c ;
  1784. b : <<PPP(LATEXT(1))>>? ( AAA | BBB )
  1785. | <<QQQ(LATEXT(1))>>? ( XXX | YYY )
  1786. ;
  1787. c : AAA | XXX;
  1788. #96. (Fixed in 1.33MR10) Guard predicates ignored when -prc off
  1789. Prior to 1.33MR10, guard predicate code was not generated unless
  1790. "-prc on" was selected.
  1791. This was incorrect, since "-prc off" (the default) is supposed to
  1792. disable only AUTOMATIC computation of predicate context, not the
  1793. programmer specified context supplied by guard predicates.
  1794. #95. (Fixed in 1.33MR10) Predicate guard context length was k, not max(k,ck)
  1795. Prior to 1.33MR10, predicate guards were computed to k tokens rather
  1796. than max(k,ck). Consider the following grammar:
  1797. a : ( A B C)? => <<AAA(LATEXT(1))>>? (A|X) (B|Y) (C|Z) ;
  1798. The code generated by 1.33 vanilla with "-k 1 -ck 3 -prc on"
  1799. for the predicate in "a" resembles:
  1800. if ( (! LA(1)==A) || AAA(LATEXT(1))) {...
  1801. With 1.33MR10 and the same options the code resembles:
  1802. if ( (! (LA(1)==A && LA(2)==B && LA(3)==C) || AAA(LATEXT(1))) {...
  1803. #94. (Fixed in 1.33MR10) Predicates followed by rule references
  1804. Prior to 1.33MR10, a semantic predicate which referenced a token
  1805. which was off the end of the rule caused an incomplete context
  1806. to be computed (with "-prc on") for the predicate under some circum-
  1807. stances. In some cases this manifested itself as illegal C code
  1808. (e.g. "LA(2)==[Ep](1)" in the k=2 examples below:
  1809. all : ( a ) *;
  1810. a : <<AAA(LATEXT(2))>>? ID X
  1811. | <<BBB(LATEXT(2))>>? Y
  1812. | Z
  1813. ;
  1814. This might also occur when the semantic predicate was followed
  1815. by a rule reference which was shorter than the length of the
  1816. semantic predicate:
  1817. all : ( a ) *;
  1818. a : <<AAA(LATEXT(2))>>? ID X
  1819. | <<BBB(LATEXT(2))>>? y
  1820. | Z
  1821. ;
  1822. y : Y ;
  1823. Depending on circumstance, the resulting context might be too
  1824. generous because it was too short, or too restrictive because
  1825. of missing alternatives.
  1826. #93. (Changed in 1.33MR10) Definition of Purify macro
  1827. Ofer Ben-Ami (gremlin@cs.huji.ac.il) has supplied a definition
  1828. for the Purify macro:
  1829. #define PURIFY(r, s) memset((char *) &(r), '\0', (s));
  1830. Note: This may not be the right thing to do for C++ objects that
  1831. have constructors. Reported by Bonny Rais (bonny@werple.net.au).
  1832. For those cases one should #define PURIFY to an empty macro in the
  1833. #header or #first actions.
  1834. #92. (Fixed in 1.33MR10) Guarded predicates and hoisting
  1835. When a guarded predicate participates in hoisting it is linked into
  1836. a predicate expression tree. Prior to 1.33MR10 this link was never
  1837. cleared and the next time the guard was used to construct a new
  1838. tree the link could contain a spurious reference to another element
  1839. which had previously been joined to it in the semantic predicate tree.
  1840. For example:
  1841. start : ( all ) *;
  1842. all : ( a | b ) ;
  1843. start2 : ( all2 ) *;
  1844. all2 : ( a ) ;
  1845. a : (A)? => <<AAA(LATEXT(1))>>? A ;
  1846. b : (B)? => <<BBB(LATEXT(1))>>? B ;
  1847. Prior to 1.33MR10 the code for "start2" would include a spurious
  1848. reference to the BBB predicate which was left from constructing
  1849. the predicate tree for rule "start" (i.e. or(AAA,BBB) ).
  1850. In 1.33MR10 this problem is avoided by cloning the original guard
  1851. each time it is linked into a predicate tree.
  1852. #91. (Changed in 1.33MR10) Extensive changes to semantic pred hoisting
  1853. ============================================
  1854. This has been rendered obsolete by Item #117
  1855. ============================================
  1856. #90. (Fixed in 1.33MR10) Semantic pred with LT(i) and i>max(k,ck)
  1857. There is a bug in antlr 1.33 vanilla and all maintenance releases
  1858. prior to 1.33MR10 which allows semantic predicates to reference
  1859. an LT(i) or LATEXT(i) where i is larger than max(k,ck). When
  1860. this occurs antlr will attempt to mark the ith element of an array
  1861. in which there are only max(k,ck) elements. The result cannot
  1862. be predicted.
  1863. Using LT(i) or LATEXT(i) for i>max(k,ck) is reported as an error
  1864. in 1.33MR10.
  1865. #89. Rescinded
  1866. #88. (Fixed in 1.33MR10) Tokens used in semantic predicates in guess mode
  1867. Consider the behavior of a semantic predicate during guess mode:
  1868. rule : a:A (
  1869. <<test($a)>>? b:B
  1870. | c:C
  1871. );
  1872. Prior to MR10 the assignment of the token or attribute to
  1873. $a did not occur during guess mode, which would cause the
  1874. semantic predicate to misbehave because $a would be null.
  1875. In 1.33MR10 a semantic predicate with a reference to an
  1876. element label (such as $a) forces the assignment to take
  1877. place even in guess mode.
  1878. In order to work, this fix REQUIRES use of the $label format
  1879. for token pointers and attributes referenced in semantic
  1880. predicates.
  1881. The fix does not apply to semantic predicates using the
  1882. numeric form to refer to attributes (e.g. <<test($1)>>?).
  1883. The user will receive a warning for this case.
  1884. Reported by Rob Trout (trout@mcs.cs.kent.edu).
  1885. #87. (Fixed in 1.33MR10) Malformed guard predicates
  1886. Context guard predicates may contain only references to
  1887. tokens. They may not contain references to (...)+ and
  1888. (...)* blocks. This is now checked. This replaces the
  1889. fatal error message in item #78 with an appropriate
  1890. (non-fatal) error message.
  1891. In theory, context guards should be allowed to reference
  1892. rules. However, I have not had time to fix this.
  1893. Evaluation of the guard takes place before all rules have
  1894. been read, making it difficult to resolve a forward reference
  1895. to rule "zzz" - it hasn't been read yet ! To postpone evaluation
  1896. of the guard until all rules have been read is too much
  1897. for the moment.
  1898. #86. (Fixed in 1.33MR10) Unequal set size in set_sub
  1899. Routine set_sub() in pccts/support/set/set.h did not work
  1900. correctly when the sets were of unequal sizes. Rewrote
  1901. set_equ to make it simpler and remove unnecessary and
  1902. expensive calls to set_deg(). This routine was not used
  1903. in 1.33 vanilla.
  1904. #85. (Changed in 1.33MR10) Allow redefinition of MaxNumFiles
  1905. Raised the maximum number of input files to 99 from 20.
  1906. Put a #ifndef/#endif around the "#define MaxNumFiles 99".
  1907. #84. (Fixed in 1.33MR10) Initialize zzBadTok in macro zzRULE
  1908. Initialize zzBadTok to NULL in zzRULE macro of AParser.h.
  1909. in order to get rid of warning messages.
  1910. #83. (Fixed in 1.33MR10) False warnings with -w2 for #tokclass
  1911. When -w2 is selected antlr gives inappropriate warnings about
  1912. #tokclass names not having any associated regular expressions.
  1913. Since a #tokclass is not a "real" token it will never have an
  1914. associated regular expression and there should be no warning.
  1915. Reported by Derek Pappas (derek.pappas@eng.sun.com)
  1916. #82. (Fixed in 1.33MR10) Computation of follow sets with multiple cycles
  1917. Reinier van den Born (reinier@vnet.ibm.com) reported a problem
  1918. in the computation of follow sets by antlr. The problem (bug)
  1919. exists in 1.33 vanilla and all maintenance releases prior to 1.33MR10.
  1920. The problem involves the computation of follow sets when there are
  1921. cycles - rules which have mutual references. I believe the problem
  1922. is restricted to cases where there is more than one cycle AND
  1923. elements of those cycles have rules in common. Even when this
  1924. occurs it may not affect the code generated - but it might. It
  1925. might also lead to undetected ambiguities.
  1926. There were no changes in antlr or dlg output from the revised version.
  1927. The following fragment demonstrates the problem by giving different
  1928. follow sets (option -pa) for var_access when built with k=1 and ck=2 on
  1929. 1.33 vanilla and 1.33MR10:
  1930. echo_statement : ECHO ( echo_expr )*
  1931. ;
  1932. echo_expr : ( command )?
  1933. | expression
  1934. ;
  1935. command : IDENTIFIER
  1936. { concat }
  1937. ;
  1938. expression : operand ( OPERATOR operand )*
  1939. ;
  1940. operand : value
  1941. | START command END
  1942. ;
  1943. value : concat
  1944. | TYPE operand
  1945. ;
  1946. concat : var_access { CONCAT value }
  1947. ;
  1948. var_access : IDENTIFIER { INDEX }
  1949. ;
  1950. #81. (Changed in 1.33MR10) C mode use of attributes and ASTs
  1951. Reported by Isaac Clark (irclark@mindspring.com).
  1952. C mode code ignores attributes returned by rules which are
  1953. referenced using element labels when ASTs are enabled (-gt option).
  1954. 1. start : r:rule t:Token <<$start=$r;>>
  1955. The $r reference will not work when combined with
  1956. the -gt option.
  1957. 2. start : t:Token <<$start=$t;>>
  1958. The $t reference works in all cases.
  1959. 3. start : rule <<$0=$1;>>
  1960. Numeric labels work in all cases.
  1961. With MR10 the user will receive an error message for case 1 when
  1962. the -gt option is used.
  1963. #80. (Fixed in 1.33MR10) (...)? as last alternative of block
  1964. A construct like the following:
  1965. rule : a
  1966. | (b)?
  1967. ;
  1968. does not make sense because there is no alternative when
  1969. the guess block fails. This is now reported as a warning
  1970. to the user.
  1971. Previously, there was a code generation error for this case:
  1972. the guess block was not "closed" when the guess failed.
  1973. This could cause an infinite loop or other problems. This
  1974. is now fixed.
  1975. Example problem:
  1976. #header<<
  1977. #include <stdio.h>
  1978. #include "charptr.h"
  1979. >>
  1980. <<
  1981. #include "charptr.c"
  1982. main ()
  1983. {
  1984. ANTLR(start(),stdin);
  1985. }
  1986. >>
  1987. #token "[\ \t]+" << zzskip(); >>
  1988. #token "[\n]" << zzline++; zzskip(); >>
  1989. #token Word "[a-z]+"
  1990. #token Number "[0-9]+"
  1991. start : (test1)?
  1992. | (test2)?
  1993. ;
  1994. test1 : (Word Word Word Word)?
  1995. | (Word Word Word Number)?
  1996. ;
  1997. test2 : (Word Word Number Word)?
  1998. | (Word Word Number Number)?
  1999. ;
  2000. Test data which caused infinite loop:
  2001. a 1 a a
  2002. #79. (Changed in 1.33MR10) Use of -fh with multiple parsers
  2003. Previously, antlr always used the pre-processor symbol
  2004. STDPCCTS_H as a gate for the file stdpccts.h. This
  2005. caused problems when there were multiple parsers defined
  2006. because they used the same gate symbol.
  2007. In 1.33MR10, the -fh filename is used to generate the
  2008. gate file for stdpccts.h. For instance:
  2009. antlr -fh std_parser1.h
  2010. generates the pre-processor symbol "STDPCCTS_std_parser1_H".
  2011. Reported by Ramanathan Santhanam (ps@kumaran.com).
  2012. #78. (Changed in 1.33MR9) Guard predicates that refer to rules
  2013. ------------------------
  2014. Please refer to Item #87
  2015. ------------------------
  2016. Guard predicates are processed during an early phase
  2017. of antlr (during parsing) before all data structures
  2018. are completed.
  2019. There is an apparent bug in earlier versions of 1.33
  2020. which caused guard predicates which contained references
  2021. to rules (rather than tokens) to reference a structure
  2022. which hadn't yet been initialized.
  2023. In some cases (perhaps all cases) references to rules
  2024. in guard predicates resulted in the use of "garbage".
  2025. #79. (Changed in 1.33MR9) Jeff Vincent (JVincent@novell.com)
  2026. Previously, the maximum length file name was set
  2027. arbitrarily to 300 characters in antlr, dlg, and sorcerer.
  2028. The config.h file now attempts to define the maximum length
  2029. filename using _MAX_PATH from stdlib.h before falling back
  2030. to using the value 300.
  2031. #78. (Changed in 1.33MR9) Jeff Vincent (JVincent@novell.com)
  2032. Put #ifndef/#endif around definition of ZZLEXBUFSIZE in
  2033. antlr.
  2034. #77. (Changed in 1.33MR9) Arithmetic overflow for very large grammars
  2035. In routine HandleAmbiguities() antlr attempts to compute the
  2036. number of possible elements in a set that is order of
  2037. number-of-tokens raised to the number-of-lookahead-tokens power.
  2038. For large grammars or large lookahead (e.g. -ck 7) this can
  2039. cause arithmetic overflow.
  2040. With 1.33MR9, arithmetic overflow in this computation is reported
  2041. the first time it happens. The program continues to run and
  2042. the program branches based on the assumption that the computed
  2043. value is larger than any number computed by counting actual cases
  2044. because 2**31 is larger than the number of bits in most computers.
  2045. Before 1.33MR9 overflow was not reported. The behavior following
  2046. overflow is not predictable by anyone but the original author.
  2047. NOTE
  2048. In 1.33MR10 the warning message is suppressed.
  2049. The code which detects the overflow allows the
  2050. computation to continue without an error. The
  2051. error message itself made made users worry.
  2052. #76. (Changed in 1.33MR9) Jeff Vincent (JVincent@novell.com)
  2053. Jeff Vincent has convinced me to make ANTLRCommonToken and
  2054. ANTLRCommonNoRefCountToken use variable length strings
  2055. allocated from the heap rather than fixed length strings.
  2056. By suitable definition of setText(), the copy constructor,
  2057. and operator =() it is possible to maintain "copy" semantics.
  2058. By "copy" semantics I mean that when a token is copied from
  2059. an existing token it receives its own, distinct, copy of the
  2060. text allocated from the heap rather than simply a pointer
  2061. to the original token's text.
  2062. ============================================================
  2063. W * A * R * N * I * N * G
  2064. ============================================================
  2065. It is possible that this may cause problems for some users.
  2066. For those users I have included the old version of AToken.h as
  2067. pccts/h/AToken_traditional.h.
  2068. #75. (Changed in 1.33MR9) Bruce Guenter (bruceg@qcc.sk.ca)
  2069. Make DLGStringInput const correct. Since this is infrequently
  2070. subclassed, it should affect few users, I hope.
  2071. #74. (Changed in 1.33MR9) -o (output directory) option
  2072. Antlr does not properly handle the -o output directory option
  2073. when the filename of the grammar contains a directory part. For
  2074. example:
  2075. antlr -o outdir pccts_src/myfile.g
  2076. causes antlr create a file called "outdir/pccts_src/myfile.cpp.
  2077. It SHOULD create outdir/myfile.cpp
  2078. The suggested code fix has been installed in antlr, dlg, and
  2079. Sorcerer.
  2080. #73. (Changed in 1.33MR9) Hoisting of semantic predicates and -mrhoist
  2081. ============================================
  2082. This has been rendered obsolete by Item #117
  2083. ============================================
  2084. #72. (Changed in 1.33MR9) virtual saveState()/restoreState()/guess_XXX
  2085. The following methods in ANTLRParser were made virtual at
  2086. the request of S. Bochnak (S.Bochnak@microTool.com.pl):
  2087. saveState() and restoreState()
  2088. guess(), guess_fail(), and guess_done()
  2089. #71. (Changed in 1.33MR9) Access to omitted command line argument
  2090. If a switch requiring arguments is the last thing on the
  2091. command line, and the argument is omitted, antlr would core.
  2092. antlr test.g -prc
  2093. instead of
  2094. antlr test.g -prc off
  2095. #70. (Changed in 1.33MR9) Addition of MSVC .dsp and .mak build files
  2096. The following MSVC .dsp and .mak files for pccts and sorcerer
  2097. were contributed by Stanislaw Bochnak (S.Bochnak@microTool.com.pl)
  2098. and Jeff Vincent (JVincent@novell.com)
  2099. PCCTS Distribution Kit
  2100. ----------------------
  2101. pccts/PCCTSMSVC50.dsw
  2102. pccts/antlr/AntlrMSVC50.dsp
  2103. pccts/antlr/AntlrMSVC50.mak
  2104. pccts/dlg/DlgMSVC50.dsp
  2105. pccts/dlg/DlgMSVC50.mak
  2106. pccts/support/msvc.dsp
  2107. Sorcerer Distribution Kit
  2108. -------------------------
  2109. pccts/sorcerer/SorcererMSVC50.dsp
  2110. pccts/sorcerer/SorcererMSVC50.mak
  2111. pccts/sorcerer/lib/msvc.dsp
  2112. #69. (Changed in 1.33MR9) Change "unsigned int" to plain "int"
  2113. Declaration of max_token_num in misc.c as "unsigned int"
  2114. caused comparison between signed and unsigned ints giving
  2115. warning message without any special benefit.
  2116. #68. (Changed in 1.33MR9) Add void return for dlg internal_error()
  2117. Get rid of "no return value" message in internal_error()
  2118. in file dlg/support.c and dlg/dlg.h.
  2119. #67. (Changed in Sor) sor.g: lisp() has no return value
  2120. Added a "void" for the return type.
  2121. #66. (Added to Sor) sor.g: ZZLEXBUFSIZE enclosed in #ifndef/#endif
  2122. A user needed to be able to change the ZZLEXBUFSIZE for
  2123. sor. Put the definition of ZZLEXBUFSIZE inside #ifndef/#endif
  2124. #65. (Changed in 1.33MR9) PCCTSAST::deepCopy() and ast_dup() bug
  2125. Jeff Vincent (JVincent@novell.com) found that deepCopy()
  2126. made new copies of only the direct descendents. No new
  2127. copies were made of sibling nodes, Sibling pointers are
  2128. set to zero by shallowCopy().
  2129. PCCTS_AST::deepCopy() has been changed to make a
  2130. deep copy in the traditional sense.
  2131. The deepCopy() routine depends on the behavior of
  2132. shallowCopy(). In all sor examples I've found,
  2133. shallowCopy() zeroes the right and down pointers.
  2134. Original Tree Original deepCopy() Revised deepCopy
  2135. ------------- ------------------- ----------------
  2136. a->b->c A A
  2137. | | |
  2138. d->e->f D D->E->F
  2139. | | |
  2140. g->h->i G G->H->I
  2141. | |
  2142. j->k J->K
  2143. While comparing deepCopy() for C++ mode with ast_dup for
  2144. C mode I found a problem with ast_dup().
  2145. Routine ast_dup() has been changed to make a deep copy
  2146. in the traditional sense.
  2147. Original Tree Original ast_dup() Revised ast_dup()
  2148. ------------- ------------------- ----------------
  2149. a->b->c A->B->C A
  2150. | | |
  2151. d->e->f D->E->F D->E->F
  2152. | | |
  2153. g->h->i G->H->I G->H->I
  2154. | | |
  2155. j->k J->K J->K
  2156. I believe this affects transform mode sorcerer programs only.
  2157. #64. (Changed in 1.33MR9) anltr/hash.h prototype for killHashTable()
  2158. #63. (Changed in 1.33MR8) h/charptr.h does not zero pointer after free
  2159. The charptr.h routine now zeroes the pointer after free().
  2160. Reported by Jens Tingleff (jensting@imaginet.fr)
  2161. #62. (Changed in 1.33MR8) ANTLRParser::resynch had static variable
  2162. The static variable "consumed" in ANTLRParser::resynch was
  2163. changed into an instance variable of the class with the
  2164. name "resynchConsumed".
  2165. Reported by S.Bochnak@microTool.com.pl
  2166. #61. (Changed in 1.33MR8) Using rule>[i,j] when rule has no return values
  2167. Previously, the following code would cause antlr to core when
  2168. it tried to generate code for rule1 because rule2 had no return
  2169. values ("upward inheritance"):
  2170. rule1 : <<int i; int j>>
  2171. rule2 > [i,j]
  2172. ;
  2173. rule2 : Anything ;
  2174. Reported by S.Bochnak@microTool.com.pl
  2175. Verified correct operation of antlr MR8 when missing or extra
  2176. inheritance arguments for all combinations. When there are
  2177. missing or extra arguments code will still be generated even
  2178. though this might cause the invocation of a subroutine with
  2179. the wrong number of arguments.
  2180. #60. (Changed in 1.33MR7) Major changes to exception handling
  2181. There were significant problems in the handling of exceptions
  2182. in 1.33 vanilla. The general problem is that it can only
  2183. process one level of exception handler. For example, a named
  2184. exception handler, an exception handler for an alternative, or
  2185. an exception for a subrule always went to the rule's exception
  2186. handler if there was no "catch" which matched the exception.
  2187. In 1.33MR7 the exception handlers properly "nest". If an
  2188. exception handler does not have a matching "catch" then the
  2189. nextmost outer exception handler is checked for an appropriate
  2190. "catch" clause, and so on until an exception handler with an
  2191. appropriate "catch" is found.
  2192. There are still undesirable features in the way exception
  2193. handlers are implemented, but I do not have time to fix them
  2194. at the moment:
  2195. The exception handlers for alternatives are outside the
  2196. block containing the alternative. This makes it impossible
  2197. to access variables declared in a block or to resume the
  2198. parse by "falling through". The parse can still be easily
  2199. resumed in other ways, but not in the most natural fashion.
  2200. This results in an inconsistency between named exception
  2201. handlers and exception handlers for alternatives. When
  2202. an exception handler for an alternative "falls through"
  2203. it goes to the nextmost outer handler - not the "normal
  2204. action".
  2205. A major difference between 1.33MR7 and 1.33 vanilla is
  2206. the default action after an exception is caught:
  2207. 1.33 Vanilla
  2208. ------------
  2209. In 1.33 vanilla the signal value is set to zero ("NoSignal")
  2210. and the code drops through to the code following the exception.
  2211. For named exception handlers this is the "normal action".
  2212. For alternative exception handlers this is the rule's handler.
  2213. 1.33MR7
  2214. -------
  2215. In 1.33MR7 the signal value is NOT automatically set to zero.
  2216. There are two cases:
  2217. For named exception handlers: if the signal value has been
  2218. set to zero the code drops through to the "normal action".
  2219. For all other cases the code branches to the nextmost outer
  2220. exception handler until it reaches the handler for the rule.
  2221. The following macros have been defined for convenience:
  2222. C/C++ Mode Name
  2223. --------------------
  2224. (zz)suppressSignal
  2225. set signal & return signal arg to 0 ("NoSignal")
  2226. (zz)setSignal(intValue)
  2227. set signal & return signal arg to some value
  2228. (zz)exportSignal
  2229. copy the signal value to the return signal arg
  2230. I'm not sure why PCCTS make a distinction between the local
  2231. signal value and the return signal argument, but I'm loathe
  2232. to change the code. The burden of copying the local signal
  2233. value to the return signal argument can be given to the
  2234. default signal handler, I suppose.
  2235. #59. (Changed in 1.33MR7) Prototypes for some functions
  2236. Added prototypes for the following functions to antlr.h
  2237. zzconsumeUntil()
  2238. zzconsumeUntilToken()
  2239. #58. (Changed in 1.33MR7) Added definition of zzbufsize to dlgauto.h
  2240. #57. (Changed in 1.33MR7) Format of #line directive
  2241. Previously, the -gl directive for line 1234 would
  2242. resemble: "# 1234 filename.g". This caused problems
  2243. for some compilers/pre-processors. In MR7 it generates
  2244. "#line 1234 filename.g".
  2245. #56. (Added in 1.33MR7) Jan Mikkelsen <janm@zeta.org.au>
  2246. Move PURIFY macro invocation to after rule's init action.
  2247. #55. (Fixed in 1.33MR7) Uninitialized variables in ANTLRParser
  2248. Member variables inf_labase and inf_last were not initialized.
  2249. (See item #50.)
  2250. #54. (Fixed in 1.33MR6) Brad Schick (schick@interacess.com)
  2251. Previously, the following constructs generated the same
  2252. code:
  2253. rule1 : (A B C)?
  2254. | something-else
  2255. ;
  2256. rule2 : (A B C)? ()
  2257. | something-else
  2258. ;
  2259. In all versions of pccts rule1 guesses (A B C) and then
  2260. consume all three tokens if the guess succeeds. In MR6
  2261. rule2 guesses (A B C) but consumes NONE of the tokens
  2262. when the guess succeeds because "()" matches epsilon.
  2263. #53. (Explanation for 1.33MR6) What happens after an exception is caught ?
  2264. The Book is silent about what happens after an exception
  2265. is caught.
  2266. The following code fragment prints "Error Action" followed
  2267. by "Normal Action".
  2268. test : Word ex:Number <<printf("Normal Action\n");>>
  2269. exception[ex]
  2270. catch NoViableAlt:
  2271. <<printf("Error Action\n");>>
  2272. ;
  2273. The reason for "Normal Action" is that the normal flow of the
  2274. program after a user-written exception handler is to "drop through".
  2275. In the case of an exception handler for a rule this results in
  2276. the execution of a "return" statement. In the case of an
  2277. exception handler attached to an alternative, rule, or token
  2278. this is the code that would have executed had there been no
  2279. exception.
  2280. The user can achieve the desired result by using a "return"
  2281. statement.
  2282. test : Word ex:Number <<printf("Normal Action\n");>>
  2283. exception[ex]
  2284. catch NoViableAlt:
  2285. <<printf("Error Action\n"); return;>>
  2286. ;
  2287. The most powerful mechanism for recovery from parse errors
  2288. in pccts is syntactic predicates because they provide
  2289. backtracking. Exceptions allow "return", "break",
  2290. "consumeUntil(...)", "goto _handler", "goto _fail", and
  2291. changing the _signal value.
  2292. #52. (Fixed in 1.33MR6) Exceptions without syntactic predicates
  2293. The following generates bad code in 1.33 if no syntactic
  2294. predicates are present in the grammar.
  2295. test : Word ex:Number <<printf("Normal Action\n");>>
  2296. exception[ex]
  2297. catch NoViableAlt:
  2298. <<printf("Error Action\n");>>
  2299. There is a reference to a guess variable. In C mode
  2300. this causes a compiler error. In C++ mode it generates
  2301. an extraneous check on member "guessing".
  2302. In MR6 correct code is generated for both C and C++ mode.
  2303. #51. (Added to 1.33MR6) Exception operator "@" used without exceptions
  2304. In MR6 added a warning when the exception operator "@" is
  2305. used and no exception group is defined. This is probably
  2306. a case where "\@" or "@" is meant.
  2307. #50. (Fixed in 1.33MR6) Gunnar Rxnning (gunnar@candleweb.no)
  2308. http://www.candleweb.no/~gunnar/
  2309. Routines zzsave_antlr_state and zzrestore_antlr_state don't
  2310. save and restore all the data needed when switching states.
  2311. Suggested patch applied to antlr.h and err.h for MR6.
  2312. #49. (Fixed in 1.33MR6) Sinan Karasu (sinan@boeing.com)
  2313. Generated code failed to turn off guess mode when leaving a
  2314. (...)+ block which contained a guess block. The result was
  2315. an infinite loop. For example:
  2316. rule : (
  2317. (x)?
  2318. | y
  2319. )+
  2320. Suggested code fix implemented in MR6. Replaced
  2321. ... else if (zzcnt>1) break;
  2322. with:
  2323. C++ mode:
  2324. ... else if (zzcnt>1) {if (!zzrv) zzGUESS_DONE; break;};
  2325. C mode:
  2326. ... else if (zzcnt>1) {if (zzguessing) zzGUESS_DONE; break;};
  2327. #48. (Fixed in 1.33MR6) Invalid exception element causes core
  2328. A label attached to an invalid construct can cause
  2329. pccts to crash while processing the exception associated
  2330. with the label. For example:
  2331. rule : t:(B C)
  2332. exception[t] catch MismatchedToken: <<printf(...);>>
  2333. Version MR6 generates the message:
  2334. reference in exception handler to undefined label 't'
  2335. #47. (Fixed in 1.33MR6) Manuel Ornato
  2336. Under some circumstances involving a k >1 or ck >1
  2337. grammar and a loop block (i.e. (...)* ) pccts will
  2338. fail to detect a syntax error and loop indefinitely.
  2339. The problem did not exist in 1.20, but has existed
  2340. from 1.23 to the present.
  2341. Fixed in MR6.
  2342. ---------------------------------------------------
  2343. Complete test program
  2344. ---------------------------------------------------
  2345. #header<<
  2346. #include <stdio.h>
  2347. #include "charptr.h"
  2348. >>
  2349. <<
  2350. #include "charptr.c"
  2351. main ()
  2352. {
  2353. ANTLR(global(),stdin);
  2354. }
  2355. >>
  2356. #token "[\ \t]+" << zzskip(); >>
  2357. #token "[\n]" << zzline++; zzskip(); >>
  2358. #token B "b"
  2359. #token C "c"
  2360. #token D "d"
  2361. #token E "e"
  2362. #token LP "\("
  2363. #token RP "\)"
  2364. #token ANTLREOF "@"
  2365. global : (
  2366. (E liste)
  2367. | liste
  2368. | listed
  2369. ) ANTLREOF
  2370. ;
  2371. listeb : LP ( B ( B | C )* ) RP ;
  2372. listec : LP ( C ( B | C )* ) RP ;
  2373. listed : LP ( D ( B | C )* ) RP ;
  2374. liste : ( listeb | listec )* ;
  2375. ---------------------------------------------------
  2376. Sample data causing infinite loop
  2377. ---------------------------------------------------
  2378. e (d c)
  2379. ---------------------------------------------------
  2380. #46. (Fixed in 1.33MR6) Robert Richter
  2381. (Robert.Richter@infotech.tu-chemnitz.de)
  2382. This item from the list of known problems was
  2383. fixed by item #18 (below).
  2384. #45. (Fixed in 1.33MR6) Brad Schick (schick@interaccess.com)
  2385. The dependency scanner in VC++ mistakenly sees a
  2386. reference to an MPW #include file even though properly
  2387. #ifdef/#endif in config.h. The suggested workaround
  2388. has been implemented:
  2389. #ifdef MPW
  2390. .....
  2391. #define MPW_CursorCtl_Header <CursorCtl.h>
  2392. #include MPW_CursorCtl_Header
  2393. .....
  2394. #endif
  2395. #44. (Fixed in 1.33MR6) cast malloc() to (char *) in charptr.c
  2396. Added (char *) cast for systems where malloc returns "void *".
  2397. #43. (Added to 1.33MR6) Bruce Guenter (bruceg@qcc.sk.ca)
  2398. Add setLeft() and setUp methods to ASTDoublyLinkedBase
  2399. for symmetry with setRight() and setDown() methods.
  2400. #42. (Fixed in 1.33MR6) Jeff Katcher (jkatcher@nortel.ca)
  2401. C++ style comment in antlr.c corrected.
  2402. #41. (Added in 1.33MR6) antlr -stdout
  2403. Using "antlr -stdout ..." forces the text that would
  2404. normally go to the grammar.c or grammar.cpp file to
  2405. stdout.
  2406. #40. (Added in 1.33MR6) antlr -tab to change tab stops
  2407. Using "antlr -tab number ..." changes the tab stops
  2408. for the grammar.c or grammar.cpp file. The number
  2409. must be between 0 and 8. Using 0 gives tab characters,
  2410. values between 1 and 8 give the appropriate number of
  2411. space characters.
  2412. #39. (Fixed in 1.33MR5) Jan Mikkelsen <janm@zeta.org.au>
  2413. Commas in function prototype still not correct under
  2414. some circumstances. Suggested code fix installed.
  2415. #38. (Fixed in 1.33MR5) ANTLRTokenBuffer constructor
  2416. Have ANTLRTokenBuffer ctor initialize member "parser" to null.
  2417. #37. (Fixed in 1.33MR4) Bruce Guenter (bruceg@qcc.sk.ca)
  2418. In ANTLRParser::FAIL(int k,...) released memory pointed to by
  2419. f[i] (as well as f itself. Should only free f itself.
  2420. #36. (Fixed in 1.33MR3) Cortland D. Starrett (cort@shay.ecn.purdue.edu)
  2421. Neglected to properly declare isDLGmaxToken() when fixing problem
  2422. reported by Andreas Magnusson.
  2423. Undo "_retv=NULL;" change which caused problems for return values
  2424. from rules whose return values weren't pointers.
  2425. Failed to create bin directory if it didn't exist.
  2426. #35. (Fixed in 1.33MR2) Andreas Magnusson
  2427. (Andreas.Magnusson@mailbox.swipnet.se)
  2428. Repair bug introduced by 1.33MR1 for #tokdefs. The original fix
  2429. placed "DLGmaxToken=9999" and "DLGminToken=0" in the TokenType enum
  2430. in order to fix a problem with an aggressive compiler assigning an 8
  2431. bit enum which might be too narrow. This caused #tokdefs to assume
  2432. that there were 9999 real tokens. The repair to the fix causes antlr to
  2433. ignore TokenTypes "DLGmaxToken" and "DLGminToken" in a #tokdefs file.
  2434. #34. (Added to 1.33MR1) Add public DLGLexerBase::set_line(int newValue)
  2435. Previously there was no public function for changing the line
  2436. number maintained by the lexer.
  2437. #33. (Fixed in 1.33MR1) Franklin Chen (chen@adi.com)
  2438. Accidental use of EXIT_FAILURE rather than PCCTS_EXIT_FAILURE
  2439. in pccts/h/AParser.cpp.
  2440. #32. (Fixed in 1.33MR1) Franklin Chen (chen@adi.com)
  2441. In PCCTSAST.cpp lines 405 and 466: Change
  2442. free (t)
  2443. to
  2444. free ( (char *)t );
  2445. to match prototype.
  2446. #31. (Added to 1.33MR1) Pointer to parser in ANTLRTokenBuffer
  2447. Pointer to parser in DLGLexerBase
  2448. The ANTLRTokenBuffer class now contains a pointer to the
  2449. parser which is using it. This is established by the
  2450. ANTLRParser constructor calling ANTLRTokenBuffer::
  2451. setParser(ANTLRParser *p).
  2452. When ANTLRTokenBuffer::setParser(ANTLRParser *p) is
  2453. called it saves the pointer to the parser and then
  2454. calls ANTLRTokenStream::setParser(ANTLRParser *p)
  2455. so that the lexer can also save a pointer to the
  2456. parser.
  2457. There is also a function getParser() in each class
  2458. with the obvious purpose.
  2459. It is possible that these functions will return NULL
  2460. under some circumstances (e.g. a non-DLG lexer is used).
  2461. #30. (Added to 1.33MR1) function tokenName(int token) standard
  2462. The generated parser class now includes the
  2463. function:
  2464. static const ANTLRChar * tokenName(int token)
  2465. which returns a pointer to the "name" corresponding
  2466. to the token.
  2467. The base class (ANTLRParser) always includes the
  2468. member function:
  2469. const ANTLRChar * parserTokenName(int token)
  2470. which can be accessed by objects which have a pointer
  2471. to an ANTLRParser, but do not know the name of the
  2472. parser class (e.g. ANTLRTokenBuffer and DLGLexerBase).
  2473. #29. (Added to 1.33MR1) Debugging DLG lexers
  2474. If the pre-processor symbol DEBUG_LEXER is defined
  2475. then DLexerBase will include code for printing out
  2476. key information about tokens which are recognized.
  2477. The debug feature of the lexer is controlled by:
  2478. int previousDebugValue=lexer.debugLexer(newValue);
  2479. a value of 0 disables output
  2480. a value of 1 enables output
  2481. Even if the lexer debug code is compiled into DLexerBase
  2482. it must be enabled before any output is generated. For
  2483. example:
  2484. DLGFileInput in(stdin);
  2485. MyDLG lexer(&in,2000);
  2486. lexer.setToken(&aToken);
  2487. #if DEBUG_LEXER
  2488. lexer.debugLexer(1); // enable debug information
  2489. #endif
  2490. #28. (Added to 1.33MR1) More control over DLG header
  2491. Version 1.33MR1 adds the following directives to PCCTS
  2492. for C++ mode:
  2493. #lexprefix <<source code>>
  2494. Adds source code to the DLGLexer.h file
  2495. after the #include "DLexerBase.h" but
  2496. before the start of the class definition.
  2497. #lexmember <<source code>>
  2498. Adds source code to the DLGLexer.h file
  2499. as part of the DLGLexer class body. It
  2500. appears immediately after the start of
  2501. the class and a "public: statement.
  2502. #27. (Fixed in 1.33MR1) Comments in DLG actions
  2503. Previously, DLG would not recognize comments as a special case.
  2504. Thus, ">>" in the comments would cause errors. This is fixed.
  2505. #26. (Fixed in 1.33MR1) Removed static variables from error routines
  2506. Previously, the existence of statically allocated variables
  2507. in some of the parser's member functions posed a danger when
  2508. there was more than one parser active.
  2509. Replaced with dynamically allocated/freed variables in 1.33MR1.
  2510. #25. (Fixed in 1.33MR1) Use of string literals in semantic predicates
  2511. Previously, it was not possible to place a string literal in
  2512. a semantic predicate because it was not properly "stringized"
  2513. for the report of a failed predicate.
  2514. #24. (Fixed in 1.33MR1) Continuation lines for semantic predicates
  2515. Previously, it was not possible to continue semantic
  2516. predicates across a line because it was not properly
  2517. "stringized" for the report of a failed predicate.
  2518. rule : <<ifXYZ()>>?[ a very
  2519. long statement ]
  2520. #23. (Fixed in 1.33MR1) {...} envelope for failed semantic predicates
  2521. Previously, there was a code generation error for failed
  2522. semantic predicates:
  2523. rule : <<xyz()>>?[ stmt1; stmt2; ]
  2524. which generated code which resembled:
  2525. if (! xyz()) stmt1; stmt2;
  2526. It now puts the statements in a {...} envelope:
  2527. if (! xyz()) { stmt1; stmt2; };
  2528. #22. (Fixed in 1.33MR1) Continuation of #token across lines using "\"
  2529. Previously, it was not possible to continue a #token regular
  2530. expression across a line. The trailing "\" and newline caused
  2531. a newline to be inserted into the regular expression by DLG.
  2532. Fixed in 1.33MR1.
  2533. #21. (Fixed in 1.33MR1) Use of ">>" (right shift operator in DLG actions
  2534. It is now possible to use the C++ right shift operator ">>"
  2535. in DLG actions by using the normal escapes:
  2536. #token "shift-right" << value=value \>\> 1;>>
  2537. #20. (Version 1.33/19-Jan-97 Karl Eccleson <karle@microrobotics.co.uk>
  2538. P.A. Keller (P.A.Keller@bath.ac.uk)
  2539. There is a problem due to using exceptions with the -gh option.
  2540. Suggested fix now in 1.33MR1.
  2541. #19. (Fixed in 1.33MR1) Tom Piscotti and John Lilley
  2542. There were problems suppressing messages to stdin and stdout
  2543. when running in a window environment because some functions
  2544. which uses fprint were not virtual.
  2545. Suggested change now in 1.33MR1.
  2546. I believe all functions containing error messages (excluding those
  2547. indicating internal inconsistency) have been placed in functions
  2548. which are virtual.
  2549. #18. (Version 1.33/ 22-Nov-96) John Bair (jbair@iftime.com)
  2550. Under some combination of options a required "return _retv" is
  2551. not generated.
  2552. Suggested fix now in 1.33MR1.
  2553. #17. (Version 1.33/3-Sep-96) Ron House (house@helios.usq.edu.au)
  2554. The routine ASTBase::predorder_action omits two "tree->"
  2555. prefixes, which results in the preorder_action belonging
  2556. to the wrong node to be invoked.
  2557. Suggested fix now in 1.33MR1.
  2558. #16. (Version 1.33/7-Jun-96) Eli Sternheim <eli@interhdl.com>
  2559. Routine consumeUntilToken() does not check for end-of-file
  2560. condition.
  2561. Suggested fix now in 1.33MR1.
  2562. #15. (Version 1.33/8 Apr 96) Asgeir Olafsson <olafsson@cstar.ac.com>
  2563. Problem with tree duplication of doubly linked ASTs in ASTBase.cpp.
  2564. Suggested fix now in 1.33MR1.
  2565. #14. (Version 1.33/28-Feb-96) Andreas.Magnusson@mailbox.swipnet.se
  2566. Problem with definition of operator = (const ANTLRTokenPtr rhs).
  2567. Suggested fix now in 1.33MR1.
  2568. #13. (Version 1.33/13-Feb-96) Franklin Chen (chen@adi.com)
  2569. Sun C++ Compiler 3.0.1 can't compile testcpp/1 due to goto in
  2570. block with destructors.
  2571. Apparently fixed. Can't locate "goto".
  2572. #12. (Version 1.33/10-Nov-95) Minor problems with 1.33 code
  2573. The following items have been fixed in 1.33MR1:
  2574. 1. pccts/antlr/main.c line 142
  2575. "void" appears in classic C code
  2576. 2. no makefile in support/genmk
  2577. 3. EXIT_FAILURE/_SUCCESS instead of PCCTS_EXIT_FAILURE/_SUCCESS
  2578. pccts/h/PCCTSAST.cpp
  2579. pccts/h/DLexerBase.cpp
  2580. pccts/testcpp/6/test.g
  2581. 4. use of "signed int" isn't accepted by AT&T cfront
  2582. pccts/h/PCCTSAST.h line 42
  2583. 5. in call to ANTLRParser::FAIL the var arg err_k is passed as
  2584. "int" but is declared "unsigned int".
  2585. 6. I believe that a failed validation predicate still does not
  2586. get put in a "{...}" envelope, despite the release notes.
  2587. 7. The #token ">>" appearing in the DLG grammar description
  2588. causes DLG to generate the string literal "\>\>" which
  2589. is non-conforming and will cause some compilers to
  2590. complain (scan.c function act10 line 143 of source code).
  2591. #11. (Version 1.32b6) Dave Kuhlman (dkuhlman@netcom.com)
  2592. Problem with file close in gen.c. Already fixed in 1.33.
  2593. #10. (Version 1.32b6/29-Aug-95)
  2594. pccts/antlr/main.c contains a C++ style comments on lines 149
  2595. and 176 which causes problems for most C compilers.
  2596. Already fixed in 1.33.
  2597. #9. (Version 1.32b4/14-Mar-95) dlgauto.h #include "config.h"
  2598. The file pccts/h/dlgauto.h should probably contain a #include
  2599. "config.h" as it uses the #define symbol __USE_PROTOS.
  2600. Added to 1.33MR1.
  2601. #8. (Version 1.32b4/6-Mar-95) Michael T. Richter (mtr@igs.net)
  2602. In C++ output mode anonymous tokens from in-line regular expressions
  2603. can create enum values which are too wide for the datatype of the enum
  2604. assigned by the C++ compiler.
  2605. Fixed in 1.33MR1.
  2606. #7. (Version 1.32b4/6-Mar-95) C++ does not imply __STDC__
  2607. In err.h the combination of # directives assumes that a C++
  2608. compiler has __STDC__ defined. This is not necessarily true.
  2609. This problem also appears in the use of __USE_PROTOS which
  2610. is appropriate for both Standard C and C++ in antlr/gen.c
  2611. and antlr/lex.c
  2612. Fixed in 1.33MR1.
  2613. #6. (Version 1.32 ?/15-Feb-95) Name conflict for "TokenType"
  2614. Already fixed in 1.33.
  2615. #5. (23-Jan-95) Douglas_Cuthbertson.JTIDS@jtids_qmail.hanscom.af.mil
  2616. The fail action following a semantic predicate is not enclosed in
  2617. "{...}". This can lead to problems when the fail action contains
  2618. more than one statement.
  2619. Fixed in 1.33MR1.
  2620. #4 . (Version 1.33/31-Mar-96) jlilley@empathy.com (John Lilley)
  2621. Put briefly, a semantic predicate ought to abort a guess if it fails.
  2622. Correction suggested by J. Lilley has been added to 1.33MR1.
  2623. #3 . (Version 1.33) P.A.Keller@bath.ac.uk
  2624. Extra commas are placed in the K&R style argument list for rules
  2625. when using both exceptions and ASTs.
  2626. Fixed in 1.33MR1.
  2627. #2. (Version 1.32b6/2-Oct-95) Brad Schick <schick@interaccess.com>
  2628. Construct #[] generates zzastnew() in C++ mode.
  2629. Already fixed in 1.33.
  2630. #1. (Version 1.33) Bob Bailey (robert@oakhill.sps.mot.com)
  2631. Previously, config.h assumed that all PC systems required
  2632. "short" file names. The user can now override that
  2633. assumption with "#define LONGFILENAMES".
  2634. Added to 1.33MR1.