generate_buildbot_json_unittest.py 176 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830
  1. #!/usr/bin/env vpython3
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Tests for generate_buildbot_json.py."""
  6. import argparse
  7. import contextlib
  8. import json
  9. import os
  10. import unittest
  11. import generate_buildbot_json
  12. from pyfakefs import fake_filesystem_unittest
  13. # pylint: disable=super-with-arguments
  14. EMPTY_PYL_FILE = """\
  15. {
  16. }
  17. """
  18. # Use this value to refer to the directory containing this code
  19. # The tests use a fake filesystem and python filesystem calls are monkey-patched
  20. # to use the fake filesystem, which affects abspath
  21. THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  22. class TestCase(fake_filesystem_unittest.TestCase):
  23. def setUp(self):
  24. self.setUpPyfakefs()
  25. self.fs.cwd = THIS_DIR
  26. self.args = generate_buildbot_json.BBJSONGenerator.parse_args([])
  27. def override_args(self, **kwargs):
  28. for k, v in kwargs.items():
  29. setattr(self.args, k, v)
  30. def create_testing_buildbot_json_file(self, path, contents):
  31. return self.fs.create_file(os.path.join(THIS_DIR, path), contents=contents)
  32. @contextlib.contextmanager
  33. def dump_on_failure(fbb, dump=True):
  34. try:
  35. yield
  36. except:
  37. if dump:
  38. for l in fbb.printed_lines:
  39. print(l)
  40. raise
  41. class FakeBBGen(generate_buildbot_json.BBJSONGenerator):
  42. def __init__(self,
  43. args,
  44. waterfalls,
  45. test_suites,
  46. luci_milo_cfg,
  47. project_pyl='{"validate_source_side_specs_have_builder": True}',
  48. exceptions=EMPTY_PYL_FILE,
  49. mixins=EMPTY_PYL_FILE,
  50. gn_isolate_map=EMPTY_PYL_FILE,
  51. variants=EMPTY_PYL_FILE):
  52. super(FakeBBGen, self).__init__(args)
  53. pyl_files_dir = args.pyl_files_dir or THIS_DIR
  54. infra_config_dir = args.infra_config_dir
  55. files = {
  56. (pyl_files_dir, 'waterfalls.pyl'): waterfalls,
  57. (pyl_files_dir, 'test_suites.pyl'): test_suites,
  58. (pyl_files_dir, 'test_suite_exceptions.pyl'): exceptions,
  59. (pyl_files_dir, 'mixins.pyl'): mixins,
  60. (pyl_files_dir, 'gn_isolate_map.pyl'): gn_isolate_map,
  61. (pyl_files_dir, 'gn_isolate_map2.pyl'): GPU_TELEMETRY_GN_ISOLATE_MAP,
  62. (pyl_files_dir, 'variants.pyl'): variants,
  63. (infra_config_dir, 'generated/project.pyl'): project_pyl,
  64. (infra_config_dir, 'generated/luci/luci-milo.cfg'): luci_milo_cfg,
  65. (infra_config_dir, 'generated/luci/luci-milo-dev.cfg'): '',
  66. }
  67. for (d, filename), content in files.items():
  68. if content is None:
  69. continue
  70. path = os.path.join(d, filename)
  71. parent = os.path.abspath(os.path.dirname(path))
  72. if not os.path.exists(parent):
  73. os.makedirs(parent)
  74. with open(path, 'w') as f:
  75. f.write(content)
  76. self.printed_lines = []
  77. def print_line(self, line):
  78. self.printed_lines.append(line)
  79. # pragma pylint: disable=arguments-differ
  80. def check_output_file_consistency(self, verbose=False, dump=True):
  81. with dump_on_failure(self, dump=verbose and dump):
  82. super(FakeBBGen, self).check_output_file_consistency(verbose)
  83. # pragma pylint: enable=arguments-differ
  84. FOO_GTESTS_WATERFALL = """\
  85. [
  86. {
  87. 'project': 'chromium',
  88. 'bucket': 'ci',
  89. 'name': 'chromium.test',
  90. 'machines': {
  91. 'Fake Tester': {
  92. 'swarming': {
  93. 'dimension_sets': [
  94. {
  95. 'kvm': '1',
  96. },
  97. ],
  98. },
  99. 'test_suites': {
  100. 'gtest_tests': 'foo_tests',
  101. },
  102. },
  103. },
  104. },
  105. ]
  106. """
  107. FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL = """\
  108. [
  109. {
  110. 'project': 'chromium',
  111. 'bucket': 'ci',
  112. 'name': 'chromium.test',
  113. 'machines': {
  114. 'Fake Tester': {
  115. 'test_suites': {
  116. 'gtest_tests': 'foo_tests',
  117. },
  118. 'args': [
  119. '--enable-features=Baz',
  120. ],
  121. },
  122. },
  123. },
  124. ]
  125. """
  126. FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL = """\
  127. [
  128. {
  129. 'project': 'chromium',
  130. 'bucket': 'ci',
  131. 'name': 'chromium.test',
  132. 'machines': {
  133. 'Fake Tester': {
  134. 'swarming': {
  135. 'dimension_sets': [
  136. {
  137. "device_type": "foo_device",
  138. },
  139. ],
  140. },
  141. 'test_suites': {
  142. 'gtest_tests': 'foo_tests',
  143. },
  144. 'os_type': 'chromeos',
  145. },
  146. },
  147. },
  148. ]
  149. """
  150. FOO_LINUX_GTESTS_WATERFALL = """\
  151. [
  152. {
  153. 'project': 'chromium',
  154. 'bucket': 'ci',
  155. 'name': 'chromium.test',
  156. 'machines': {
  157. 'Fake Tester': {
  158. 'os_type': 'linux',
  159. 'test_suites': {
  160. 'gtest_tests': 'foo_tests',
  161. },
  162. },
  163. },
  164. },
  165. ]
  166. """
  167. COMPOSITION_GTEST_SUITE_WATERFALL = """\
  168. [
  169. {
  170. 'project': 'chromium',
  171. 'bucket': 'ci',
  172. 'name': 'chromium.test',
  173. 'machines': {
  174. 'Fake Tester': {
  175. 'test_suites': {
  176. 'gtest_tests': 'composition_tests',
  177. },
  178. },
  179. },
  180. },
  181. ]
  182. """
  183. COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL = """\
  184. [
  185. {
  186. 'project': 'chromium',
  187. 'bucket': 'ci',
  188. 'name': 'chromium.test',
  189. 'machines': {
  190. 'Fake Tester': {
  191. 'test_suites': {
  192. 'gtest_tests': 'composition_tests',
  193. },
  194. 'args': [
  195. '--this-is-an-argument',
  196. ],
  197. },
  198. },
  199. },
  200. ]
  201. """
  202. FOO_ISOLATED_SCRIPTS_WATERFALL = """\
  203. [
  204. {
  205. 'project': 'chromium',
  206. 'bucket': 'ci',
  207. 'name': 'chromium.test',
  208. 'machines': {
  209. 'Fake Tester': {
  210. 'test_suites': {
  211. 'isolated_scripts': 'composition_tests',
  212. },
  213. },
  214. },
  215. },
  216. ]
  217. """
  218. FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID = """\
  219. [
  220. {
  221. 'project': 'chromium',
  222. 'bucket': 'ci',
  223. 'name': 'chromium.test',
  224. 'machines': {
  225. 'Fake Tester': {
  226. 'os_type': 'android',
  227. 'test_suites': {
  228. 'isolated_scripts': 'composition_tests',
  229. },
  230. 'use_android_presentation': True,
  231. },
  232. },
  233. },
  234. ]
  235. """
  236. FOO_SCRIPT_WATERFALL = """\
  237. [
  238. {
  239. 'project': 'chromium',
  240. 'bucket': 'ci',
  241. 'name': 'chromium.test',
  242. 'machines': {
  243. 'Fake Tester': {
  244. 'test_suites': {
  245. 'scripts': 'foo_scripts',
  246. },
  247. },
  248. },
  249. },
  250. ]
  251. """
  252. FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS = """\
  253. [
  254. {
  255. 'project': 'chromium',
  256. 'bucket': 'ci',
  257. 'name': 'chromium.test',
  258. 'machines': {
  259. 'Fake Tester': {
  260. 'forbid_script_tests': True,
  261. 'test_suites': {
  262. 'scripts': 'foo_scripts',
  263. },
  264. },
  265. },
  266. },
  267. ]
  268. """
  269. FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS = """\
  270. [
  271. {
  272. 'project': 'chromium',
  273. 'bucket': 'ci',
  274. 'name': 'chromium.test',
  275. 'forbid_script_tests': True,
  276. 'machines': {
  277. 'Fake Tester': {
  278. 'test_suites': {
  279. 'scripts': 'foo_scripts',
  280. },
  281. },
  282. },
  283. },
  284. ]
  285. """
  286. FOO_JUNIT_WATERFALL = """\
  287. [
  288. {
  289. 'project': 'chromium',
  290. 'bucket': 'ci',
  291. 'name': 'chromium.test',
  292. 'machines': {
  293. 'Fake Tester': {
  294. 'test_suites': {
  295. 'junit_tests': 'composition_tests',
  296. },
  297. },
  298. },
  299. },
  300. ]
  301. """
  302. FOO_GPU_TELEMETRY_TEST_WATERFALL = """\
  303. [
  304. {
  305. 'project': 'chromium',
  306. 'bucket': 'ci',
  307. 'name': 'chromium.test',
  308. 'machines': {
  309. 'Fake Tester': {
  310. 'os_type': 'win',
  311. 'browser_config': 'release',
  312. 'swarming': {
  313. 'dimension_sets': [
  314. {
  315. 'gpu': '10de:1cb3',
  316. },
  317. ],
  318. },
  319. 'test_suites': {
  320. 'gpu_telemetry_tests': 'composition_tests',
  321. },
  322. },
  323. },
  324. },
  325. ]
  326. """
  327. FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID = """\
  328. [
  329. {
  330. 'project': 'chromium',
  331. 'bucket': 'ci',
  332. 'name': 'chromium.test',
  333. 'machines': {
  334. 'Fake Tester': {
  335. 'os_type': 'android',
  336. 'browser_config': 'android-chromium',
  337. 'swarming': {
  338. 'dimension_sets': [
  339. {
  340. 'device_type': 'bullhead',
  341. },
  342. ],
  343. },
  344. 'test_suites': {
  345. 'gpu_telemetry_tests': 'composition_tests',
  346. },
  347. },
  348. },
  349. },
  350. ]
  351. """
  352. FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW = """\
  353. [
  354. {
  355. 'project': 'chromium',
  356. 'bucket': 'ci',
  357. 'name': 'chromium.test',
  358. 'machines': {
  359. 'Fake Tester': {
  360. 'os_type': 'android',
  361. 'browser_config': 'not-a-real-browser',
  362. 'swarming': {
  363. 'dimension_sets': [
  364. {
  365. 'device_type': 'bullhead',
  366. },
  367. ],
  368. },
  369. 'test_suites': {
  370. 'android_webview_gpu_telemetry_tests': 'composition_tests',
  371. },
  372. },
  373. },
  374. },
  375. ]
  376. """
  377. FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA = """\
  378. [
  379. {
  380. 'project': 'chromium',
  381. 'bucket': 'ci',
  382. 'name': 'chromium.test',
  383. 'machines': {
  384. 'Fake Tester': {
  385. 'os_type': 'fuchsia',
  386. 'browser_config': 'fuchsia-chrome',
  387. 'swarming': {
  388. 'dimension_sets': [
  389. {
  390. 'kvm': '1',
  391. },
  392. ],
  393. },
  394. 'test_suites': {
  395. 'gpu_telemetry_tests': 'composition_tests',
  396. },
  397. },
  398. },
  399. },
  400. ]
  401. """
  402. FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING = """\
  403. [
  404. {
  405. 'project': 'chromium',
  406. 'bucket': 'ci',
  407. 'name': 'chromium.test',
  408. 'machines': {
  409. 'Fake Tester': {
  410. 'os_type': 'fuchsia',
  411. 'browser_config': 'not-a-real-browser',
  412. 'swarming': {
  413. 'dimension_sets': [
  414. {
  415. 'kvm': '1',
  416. },
  417. ],
  418. },
  419. 'test_suites': {
  420. 'cast_streaming_tests': 'composition_tests',
  421. },
  422. },
  423. },
  424. },
  425. ]
  426. """
  427. NVIDIA_GPU_TELEMETRY_TEST_WATERFALL = """\
  428. [
  429. {
  430. 'project': 'chromium',
  431. 'bucket': 'ci',
  432. 'name': 'chromium.test',
  433. 'machines': {
  434. 'Fake Tester': {
  435. 'os_type': 'win',
  436. 'browser_config': 'release',
  437. 'swarming': {
  438. 'dimension_sets': [
  439. {
  440. 'gpu': '10de:1cb3-26.21.14.3102',
  441. },
  442. ],
  443. },
  444. 'test_suites': {
  445. 'gpu_telemetry_tests': 'composition_tests',
  446. },
  447. },
  448. },
  449. },
  450. ]
  451. """
  452. INTEL_GPU_TELEMETRY_TEST_WATERFALL = """\
  453. [
  454. {
  455. 'project': 'chromium',
  456. 'bucket': 'ci',
  457. 'name': 'chromium.test',
  458. 'machines': {
  459. 'Fake Tester': {
  460. 'os_type': 'win',
  461. 'browser_config': 'release',
  462. 'swarming': {
  463. 'dimension_sets': [
  464. {
  465. 'gpu': '8086:5912-24.20.100.6286',
  466. },
  467. ],
  468. },
  469. 'test_suites': {
  470. 'gpu_telemetry_tests': 'composition_tests',
  471. },
  472. },
  473. },
  474. },
  475. ]
  476. """
  477. INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL = """\
  478. [
  479. {
  480. 'project': 'chromium',
  481. 'bucket': 'ci',
  482. 'name': 'chromium.test',
  483. 'machines': {
  484. 'Fake Tester': {
  485. 'os_type': 'win',
  486. 'browser_config': 'release',
  487. 'swarming': {
  488. 'dimension_sets': [
  489. {
  490. 'gpu': '8086:3e92-24.20.100.6286',
  491. },
  492. ],
  493. },
  494. 'test_suites': {
  495. 'gpu_telemetry_tests': 'composition_tests',
  496. },
  497. },
  498. },
  499. },
  500. ]
  501. """
  502. GPU_TELEMETRY_TEST_VARIANTS_WATERFALL = """\
  503. [
  504. {
  505. 'project': 'chromium',
  506. 'bucket': 'ci',
  507. 'name': 'chromium.test',
  508. 'machines': {
  509. 'Fake Tester': {
  510. 'os_type': 'win',
  511. 'browser_config': 'release',
  512. 'swarming': {
  513. 'dimension_sets': [
  514. {
  515. 'gpu': '8086:3e92-24.20.100.6286',
  516. },
  517. ],
  518. },
  519. 'test_suites': {
  520. 'gpu_telemetry_tests': 'matrix_tests',
  521. },
  522. },
  523. },
  524. },
  525. ]
  526. """
  527. UNKNOWN_TEST_SUITE_WATERFALL = """\
  528. [
  529. {
  530. 'project': 'chromium',
  531. 'bucket': 'ci',
  532. 'name': 'chromium.test',
  533. 'machines': {
  534. 'Fake Tester': {
  535. 'test_suites': {
  536. 'gtest_tests': 'baz_tests',
  537. },
  538. },
  539. },
  540. },
  541. ]
  542. """
  543. UNKNOWN_TEST_SUITE_TYPE_WATERFALL = """\
  544. [
  545. {
  546. 'project': 'chromium',
  547. 'bucket': 'ci',
  548. 'name': 'chromium.test',
  549. 'machines': {
  550. 'Fake Tester': {
  551. 'test_suites': {
  552. 'gtest_tests': 'foo_tests',
  553. 'foo_test_type': 'foo_tests',
  554. },
  555. },
  556. },
  557. },
  558. ]
  559. """
  560. ANDROID_WATERFALL = """\
  561. [
  562. {
  563. 'project': 'chromium',
  564. 'bucket': 'ci',
  565. 'name': 'chromium.test',
  566. 'machines': {
  567. 'Android Builder': {
  568. 'additional_compile_targets': [
  569. 'bar_test',
  570. ],
  571. },
  572. 'Fake Android K Tester': {
  573. 'additional_compile_targets': [
  574. 'bar_test',
  575. ],
  576. 'swarming': {
  577. 'dimension_sets': [
  578. {
  579. 'device_os': 'KTU84P',
  580. 'device_type': 'hammerhead',
  581. 'os': 'Android',
  582. },
  583. ],
  584. },
  585. 'os_type': 'android',
  586. 'skip_merge_script': True,
  587. 'test_suites': {
  588. 'gtest_tests': 'foo_tests',
  589. },
  590. },
  591. 'Fake Android L Tester': {
  592. 'swarming': {
  593. 'dimension_sets': [
  594. {
  595. 'device_os': 'LMY41U',
  596. 'device_os_type': 'user',
  597. 'device_type': 'hammerhead',
  598. 'os': 'Android',
  599. },
  600. ],
  601. },
  602. 'os_type': 'android',
  603. 'skip_merge_script': True,
  604. 'skip_output_links': True,
  605. 'test_suites': {
  606. 'gtest_tests': 'foo_tests',
  607. },
  608. },
  609. 'Fake Android M Tester': {
  610. 'swarming': {
  611. 'dimension_sets': [
  612. {
  613. 'device_os': 'MMB29Q',
  614. 'device_type': 'bullhead',
  615. 'os': 'Android',
  616. },
  617. ],
  618. },
  619. 'os_type': 'android',
  620. 'use_swarming': False,
  621. 'test_suites': {
  622. 'gtest_tests': 'foo_tests',
  623. },
  624. },
  625. },
  626. },
  627. ]
  628. """
  629. UNKNOWN_BOT_GTESTS_WATERFALL = """\
  630. [
  631. {
  632. 'project': 'chromium',
  633. 'bucket': 'ci',
  634. 'name': 'chromium.test',
  635. 'machines': {
  636. 'Unknown Bot': {
  637. 'test_suites': {
  638. 'gtest_tests': 'foo_tests',
  639. },
  640. },
  641. },
  642. },
  643. ]
  644. """
  645. MATRIX_GTEST_SUITE_WATERFALL = """\
  646. [
  647. {
  648. 'project': 'chromium',
  649. 'bucket': 'ci',
  650. 'name': 'chromium.test',
  651. 'machines': {
  652. 'Fake Tester': {
  653. 'test_suites': {
  654. 'gtest_tests': 'matrix_tests',
  655. },
  656. },
  657. },
  658. },
  659. ]
  660. """
  661. MATRIX_GTEST_SUITE_WATERFALL_MIXINS = """\
  662. [
  663. {
  664. 'project': 'chromium',
  665. 'bucket': 'ci',
  666. 'name': 'chromium.test',
  667. 'machines': {
  668. 'Fake Tester': {
  669. 'mixins': ['dimension_mixin'],
  670. 'test_suites': {
  671. 'gtest_tests': 'matrix_tests',
  672. },
  673. },
  674. },
  675. },
  676. ]
  677. """
  678. FOO_TEST_SUITE = """\
  679. {
  680. 'basic_suites': {
  681. 'foo_tests': {
  682. 'foo_test': {
  683. 'swarming': {
  684. 'dimension_sets': [
  685. {
  686. 'integrity': 'high',
  687. }
  688. ],
  689. 'expiration': 120,
  690. },
  691. },
  692. },
  693. },
  694. }
  695. """
  696. FOO_TEST_SUITE_NO_DIMENSIONS = """\
  697. {
  698. 'basic_suites': {
  699. 'foo_tests': {
  700. 'foo_test': {
  701. },
  702. },
  703. },
  704. }
  705. """
  706. FOO_TEST_SUITE_NOT_SORTED = """\
  707. {
  708. 'basic_suites': {
  709. 'foo_tests': {
  710. 'foo_test': {},
  711. 'a_test': {},
  712. },
  713. },
  714. }
  715. """
  716. FOO_TEST_SUITE_WITH_ARGS = """\
  717. {
  718. 'basic_suites': {
  719. 'foo_tests': {
  720. 'foo_test': {
  721. 'args': [
  722. '--c_arg',
  723. ],
  724. },
  725. },
  726. },
  727. }
  728. """
  729. FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES = """\
  730. {
  731. 'basic_suites': {
  732. 'foo_tests': {
  733. 'foo_test': {
  734. 'swarming': {
  735. 'named_caches': [
  736. {
  737. 'name': 'cache_in_test',
  738. 'file': 'cache_in_test_file',
  739. },
  740. ],
  741. },
  742. },
  743. },
  744. },
  745. }
  746. """
  747. FOO_TEST_SUITE_WITH_LINUX_ARGS = """\
  748. {
  749. 'basic_suites': {
  750. 'foo_tests': {
  751. 'foo_test': {
  752. 'linux_args': [
  753. '--no-xvfb',
  754. ],
  755. },
  756. },
  757. },
  758. }
  759. """
  760. FOO_TEST_SUITE_WITH_ENABLE_FEATURES = """\
  761. {
  762. 'basic_suites': {
  763. 'foo_tests': {
  764. 'foo_test': {
  765. 'args': [
  766. '--enable-features=Foo,Bar',
  767. ],
  768. },
  769. },
  770. },
  771. }
  772. """
  773. FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN = """\
  774. {
  775. 'basic_suites': {
  776. 'foo_tests': {
  777. 'foo_test': {
  778. 'remove_mixins': ['waterfall_mixin'],
  779. 'swarming': {
  780. 'dimension_sets': [
  781. {
  782. 'integrity': 'high',
  783. }
  784. ],
  785. 'expiration': 120,
  786. },
  787. },
  788. },
  789. },
  790. }
  791. """
  792. FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN = """\
  793. {
  794. 'basic_suites': {
  795. 'foo_tests': {
  796. 'foo_test': {
  797. 'remove_mixins': ['builder_mixin'],
  798. 'swarming': {
  799. 'dimension_sets': [
  800. {
  801. 'integrity': 'high',
  802. }
  803. ],
  804. 'expiration': 120,
  805. },
  806. },
  807. },
  808. },
  809. }
  810. """
  811. FOO_SCRIPT_SUITE = """\
  812. {
  813. 'basic_suites': {
  814. 'foo_scripts': {
  815. 'foo_test': {
  816. 'script': 'foo.py',
  817. },
  818. 'bar_test': {
  819. 'script': 'bar.py',
  820. },
  821. },
  822. },
  823. }
  824. """
  825. GOOD_COMPOSITION_TEST_SUITES = """\
  826. {
  827. 'basic_suites': {
  828. 'bar_tests': {
  829. 'bar_test': {},
  830. },
  831. 'foo_tests': {
  832. 'foo_test': {},
  833. },
  834. },
  835. 'compound_suites': {
  836. 'composition_tests': [
  837. 'foo_tests',
  838. 'bar_tests',
  839. ],
  840. },
  841. }
  842. """
  843. BAD_COMPOSITION_TEST_SUITES = """\
  844. {
  845. 'basic_suites': {
  846. 'bar_tests': {},
  847. 'foo_tests': {},
  848. },
  849. 'compound_suites': {
  850. 'buggy_composition_tests': [
  851. 'bar_tests',
  852. ],
  853. 'composition_tests': [
  854. 'foo_tests',
  855. 'buggy_composition_tests',
  856. ],
  857. },
  858. }
  859. """
  860. CONFLICTING_COMPOSITION_TEST_SUITES = """\
  861. {
  862. 'basic_suites': {
  863. 'bar_tests': {
  864. 'baz_tests': {
  865. 'args': [
  866. '--bar',
  867. ],
  868. }
  869. },
  870. 'foo_tests': {
  871. 'baz_tests': {
  872. 'args': [
  873. '--foo',
  874. ],
  875. }
  876. },
  877. },
  878. 'compound_suites': {
  879. 'foobar_tests': [
  880. 'foo_tests',
  881. 'bar_tests',
  882. ],
  883. },
  884. }
  885. """
  886. DUPLICATES_COMPOSITION_TEST_SUITES = """\
  887. {
  888. 'basic_suites': {
  889. 'bar_tests': {},
  890. 'buggy_composition_tests': {},
  891. 'foo_tests': {},
  892. },
  893. 'compound_suites': {
  894. 'bar_tests': [
  895. 'foo_tests',
  896. ],
  897. 'composition_tests': [
  898. 'foo_tests',
  899. 'buggy_composition_tests',
  900. ],
  901. },
  902. }
  903. """
  904. SCRIPT_SUITE = """\
  905. {
  906. 'basic_suites': {
  907. 'foo_scripts': {
  908. 'foo_test': {
  909. 'script': 'foo.py',
  910. },
  911. },
  912. },
  913. }
  914. """
  915. UNREFED_TEST_SUITE = """\
  916. {
  917. 'basic_suites': {
  918. 'bar_tests': {},
  919. 'foo_tests': {},
  920. },
  921. }
  922. """
  923. REUSING_TEST_WITH_DIFFERENT_NAME = """\
  924. {
  925. 'basic_suites': {
  926. 'foo_tests': {
  927. 'foo_test': {},
  928. 'variation_test': {
  929. 'args': [
  930. '--variation',
  931. ],
  932. 'test': 'foo_test',
  933. },
  934. },
  935. },
  936. }
  937. """
  938. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST = """\
  939. {
  940. 'basic_suites': {
  941. 'foo_tests': {
  942. 'foo': {},
  943. },
  944. 'bar_tests': {
  945. 'bar_test': {},
  946. },
  947. },
  948. 'compound_suites': {
  949. 'composition_tests': [
  950. 'foo_tests',
  951. 'bar_tests',
  952. ],
  953. },
  954. }
  955. """
  956. COMPOSITION_SUITE_WITH_GPU_ARGS = """\
  957. {
  958. 'basic_suites': {
  959. 'foo_tests': {
  960. 'foo': {
  961. 'args': [
  962. '--gpu-vendor-id',
  963. '${gpu_vendor_id}',
  964. '--gpu-device-id',
  965. '${gpu_device_id}',
  966. ],
  967. },
  968. },
  969. 'bar_tests': {
  970. 'bar_test': {},
  971. },
  972. },
  973. 'compound_suites': {
  974. 'composition_tests': [
  975. 'foo_tests',
  976. 'bar_tests',
  977. ],
  978. },
  979. }
  980. """
  981. COMPOSITION_SUITE_WITH_PIXEL_AND_FILTER = """\
  982. {
  983. 'basic_suites': {
  984. 'foo_tests': {
  985. 'pixel': {
  986. 'args': [
  987. '--git-revision=aaaa',
  988. '--test-filter=bar',
  989. ],
  990. },
  991. },
  992. 'bar_tests': {
  993. 'bar_test': {},
  994. },
  995. },
  996. 'compound_suites': {
  997. 'composition_tests': [
  998. 'foo_tests',
  999. 'bar_tests',
  1000. ],
  1001. },
  1002. }
  1003. """
  1004. GTEST_AS_ISOLATED_SCRIPT_SUITE = """\
  1005. {
  1006. 'basic_suites': {
  1007. 'foo_tests': {
  1008. 'foo_test': {
  1009. 'script': 'foo.py',
  1010. 'use_isolated_scripts_api': True,
  1011. },
  1012. },
  1013. },
  1014. }
  1015. """
  1016. SCRIPT_WITH_ARGS_EXCEPTIONS = """\
  1017. {
  1018. 'foo_test': {
  1019. 'modifications': {
  1020. 'Fake Tester': {
  1021. 'args': ['--fake-arg'],
  1022. },
  1023. },
  1024. },
  1025. }
  1026. """
  1027. SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS = """\
  1028. {
  1029. 'foo_test': {
  1030. 'modifications': {
  1031. 'Fake Tester': {
  1032. 'swarming': {
  1033. 'value': 'exception',
  1034. },
  1035. },
  1036. },
  1037. },
  1038. }
  1039. """
  1040. NO_BAR_TEST_EXCEPTIONS = """\
  1041. {
  1042. 'bar_test': {
  1043. 'remove_from': [
  1044. 'Fake Tester',
  1045. ]
  1046. }
  1047. }
  1048. """
  1049. EMPTY_BAR_TEST_EXCEPTIONS = """\
  1050. {
  1051. 'bar_test': {
  1052. }
  1053. }
  1054. """
  1055. EXCEPTIONS_SORTED = """\
  1056. {
  1057. 'suite_c': {
  1058. 'modifications': {
  1059. 'Fake Tester': {
  1060. 'foo': 'bar',
  1061. },
  1062. },
  1063. },
  1064. 'suite_d': {
  1065. 'modifications': {
  1066. 'Fake Tester': {
  1067. 'foo': 'baz',
  1068. },
  1069. },
  1070. },
  1071. }
  1072. """
  1073. EXCEPTIONS_UNSORTED = """\
  1074. {
  1075. 'suite_d': {
  1076. 'modifications': {
  1077. 'Fake Tester': {
  1078. 'foo': 'baz',
  1079. },
  1080. },
  1081. },
  1082. 'suite_c': {
  1083. 'modifications': {
  1084. 'Fake Tester': {
  1085. 'foo': 'bar',
  1086. },
  1087. },
  1088. },
  1089. }
  1090. """
  1091. EXCEPTIONS_PER_TEST_UNSORTED = """\
  1092. {
  1093. 'suite_d': {
  1094. 'modifications': {
  1095. 'Other Tester': {
  1096. 'foo': 'baz',
  1097. },
  1098. 'Fake Tester': {
  1099. 'foo': 'baz',
  1100. },
  1101. },
  1102. },
  1103. }
  1104. """
  1105. EXCEPTIONS_DUPS_REMOVE_FROM = """\
  1106. {
  1107. 'suite_d': {
  1108. 'remove_from': [
  1109. 'Fake Tester',
  1110. 'Fake Tester',
  1111. ],
  1112. 'modifications': {
  1113. 'Fake Tester': {
  1114. 'foo': 'baz',
  1115. },
  1116. },
  1117. },
  1118. }
  1119. """
  1120. FOO_TEST_MODIFICATIONS = """\
  1121. {
  1122. 'foo_test': {
  1123. 'modifications': {
  1124. 'Fake Tester': {
  1125. 'args': [
  1126. '--bar',
  1127. ],
  1128. 'swarming': {
  1129. 'hard_timeout': 600,
  1130. },
  1131. },
  1132. },
  1133. }
  1134. }
  1135. """
  1136. FOO_TEST_EXPLICIT_NONE_EXCEPTIONS = """\
  1137. {
  1138. 'foo_test': {
  1139. 'modifications': {
  1140. 'Fake Tester': {
  1141. 'swarming': {
  1142. 'dimension_sets': [
  1143. {
  1144. 'integrity': None,
  1145. },
  1146. ],
  1147. },
  1148. },
  1149. },
  1150. },
  1151. }
  1152. """
  1153. NONEXISTENT_REMOVAL = """\
  1154. {
  1155. 'foo_test': {
  1156. 'remove_from': [
  1157. 'Nonexistent Tester',
  1158. ]
  1159. }
  1160. }
  1161. """
  1162. NONEXISTENT_MODIFICATION = """\
  1163. {
  1164. 'foo_test': {
  1165. 'modifications': {
  1166. 'Nonexistent Tester': {
  1167. 'args': [],
  1168. },
  1169. },
  1170. }
  1171. }
  1172. """
  1173. COMPOSITION_WATERFALL_OUTPUT = """\
  1174. {
  1175. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1176. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1177. "Fake Tester": {
  1178. "gtest_tests": [
  1179. {
  1180. "merge": {
  1181. "args": [],
  1182. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1183. },
  1184. "swarming": {
  1185. "can_use_on_swarming_builders": true
  1186. },
  1187. "test": "bar_test"
  1188. },
  1189. {
  1190. "merge": {
  1191. "args": [],
  1192. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1193. },
  1194. "swarming": {
  1195. "can_use_on_swarming_builders": true
  1196. },
  1197. "test": "foo_test"
  1198. }
  1199. ]
  1200. }
  1201. }
  1202. """
  1203. COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT = """\
  1204. {
  1205. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1206. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1207. "Fake Tester": {
  1208. "gtest_tests": [
  1209. {
  1210. "args": [
  1211. "--this-is-an-argument"
  1212. ],
  1213. "merge": {
  1214. "args": [],
  1215. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1216. },
  1217. "swarming": {
  1218. "can_use_on_swarming_builders": true
  1219. },
  1220. "test": "bar_test"
  1221. },
  1222. {
  1223. "args": [
  1224. "--this-is-an-argument"
  1225. ],
  1226. "merge": {
  1227. "args": [],
  1228. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1229. },
  1230. "swarming": {
  1231. "can_use_on_swarming_builders": true
  1232. },
  1233. "test": "foo_test"
  1234. }
  1235. ]
  1236. }
  1237. }
  1238. """
  1239. VARIATION_GTEST_OUTPUT = """\
  1240. {
  1241. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1242. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1243. "Fake Tester": {
  1244. "gtest_tests": [
  1245. {
  1246. "merge": {
  1247. "args": [],
  1248. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1249. },
  1250. "swarming": {
  1251. "can_use_on_swarming_builders": true,
  1252. "dimension_sets": [
  1253. {
  1254. "kvm": "1"
  1255. }
  1256. ]
  1257. },
  1258. "test": "foo_test",
  1259. "test_id_prefix": "ninja://chrome/test:foo_test/"
  1260. },
  1261. {
  1262. "args": [
  1263. "--variation"
  1264. ],
  1265. "merge": {
  1266. "args": [],
  1267. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1268. },
  1269. "name": "variation_test",
  1270. "swarming": {
  1271. "can_use_on_swarming_builders": true,
  1272. "dimension_sets": [
  1273. {
  1274. "kvm": "1"
  1275. }
  1276. ]
  1277. },
  1278. "test": "foo_test",
  1279. "test_id_prefix": "ninja://chrome/test:foo_test/"
  1280. }
  1281. ]
  1282. }
  1283. }
  1284. """
  1285. FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT = """\
  1286. {
  1287. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1288. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1289. "Fake Tester": {
  1290. "gtest_tests": [
  1291. {
  1292. "merge": {
  1293. "args": [],
  1294. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1295. },
  1296. "script": "foo.py",
  1297. "swarming": {
  1298. "can_use_on_swarming_builders": true,
  1299. "dimension_sets": [
  1300. {
  1301. "kvm": "1"
  1302. }
  1303. ]
  1304. },
  1305. "test": "foo_test",
  1306. "test_id_prefix": "ninja://chrome/test:foo_test/",
  1307. "use_isolated_scripts_api": true
  1308. }
  1309. ]
  1310. }
  1311. }
  1312. """
  1313. COMPOSITION_WATERFALL_FILTERED_OUTPUT = """\
  1314. {
  1315. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1316. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1317. "Fake Tester": {
  1318. "gtest_tests": [
  1319. {
  1320. "merge": {
  1321. "args": [],
  1322. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1323. },
  1324. "swarming": {
  1325. "can_use_on_swarming_builders": true
  1326. },
  1327. "test": "foo_test"
  1328. }
  1329. ]
  1330. }
  1331. }
  1332. """
  1333. MERGED_ARGS_OUTPUT = """\
  1334. {
  1335. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1336. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1337. "Fake Tester": {
  1338. "gtest_tests": [
  1339. {
  1340. "args": [
  1341. "--c_arg",
  1342. "--bar"
  1343. ],
  1344. "merge": {
  1345. "args": [],
  1346. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1347. },
  1348. "swarming": {
  1349. "can_use_on_swarming_builders": true,
  1350. "dimension_sets": [
  1351. {
  1352. "kvm": "1"
  1353. }
  1354. ],
  1355. "hard_timeout": 600
  1356. },
  1357. "test": "foo_test"
  1358. }
  1359. ]
  1360. }
  1361. }
  1362. """
  1363. LINUX_ARGS_OUTPUT = """\
  1364. {
  1365. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1366. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1367. "Fake Tester": {
  1368. "gtest_tests": [
  1369. {
  1370. "args": [
  1371. "--no-xvfb"
  1372. ],
  1373. "merge": {
  1374. "args": [],
  1375. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1376. },
  1377. "swarming": {
  1378. "can_use_on_swarming_builders": true
  1379. },
  1380. "test": "foo_test"
  1381. }
  1382. ]
  1383. }
  1384. }
  1385. """
  1386. MERGED_ENABLE_FEATURES_OUTPUT = """\
  1387. {
  1388. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1389. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1390. "Fake Tester": {
  1391. "gtest_tests": [
  1392. {
  1393. "args": [
  1394. "--enable-features=Foo,Bar,Baz"
  1395. ],
  1396. "merge": {
  1397. "args": [],
  1398. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1399. },
  1400. "swarming": {
  1401. "can_use_on_swarming_builders": true
  1402. },
  1403. "test": "foo_test"
  1404. }
  1405. ]
  1406. }
  1407. }
  1408. """
  1409. MODIFIED_OUTPUT = """\
  1410. {
  1411. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1412. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1413. "Fake Tester": {
  1414. "gtest_tests": [
  1415. {
  1416. "args": [
  1417. "--bar"
  1418. ],
  1419. "merge": {
  1420. "args": [],
  1421. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1422. },
  1423. "swarming": {
  1424. "can_use_on_swarming_builders": true,
  1425. "dimension_sets": [
  1426. {
  1427. "integrity": "high",
  1428. "kvm": "1"
  1429. }
  1430. ],
  1431. "expiration": 120,
  1432. "hard_timeout": 600
  1433. },
  1434. "test": "foo_test"
  1435. }
  1436. ]
  1437. }
  1438. }
  1439. """
  1440. EXPLICIT_NONE_OUTPUT = """\
  1441. {
  1442. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1443. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1444. "Fake Tester": {
  1445. "gtest_tests": [
  1446. {
  1447. "merge": {
  1448. "args": [],
  1449. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1450. },
  1451. "swarming": {
  1452. "can_use_on_swarming_builders": true,
  1453. "dimension_sets": [
  1454. {
  1455. "kvm": "1"
  1456. }
  1457. ],
  1458. "expiration": 120
  1459. },
  1460. "test": "foo_test"
  1461. }
  1462. ]
  1463. }
  1464. }
  1465. """
  1466. ISOLATED_SCRIPT_OUTPUT = """\
  1467. {
  1468. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1469. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1470. "Fake Tester": {
  1471. "isolated_scripts": [
  1472. {
  1473. "isolate_name": "foo_test",
  1474. "merge": {
  1475. "args": [],
  1476. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1477. },
  1478. "name": "foo_test",
  1479. "swarming": {
  1480. "can_use_on_swarming_builders": true
  1481. }
  1482. }
  1483. ]
  1484. }
  1485. }
  1486. """
  1487. ISOLATED_SCRIPT_OUTPUT_ANDROID = """\
  1488. {
  1489. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1490. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1491. "Fake Tester": {
  1492. "isolated_scripts": [
  1493. {
  1494. "args": [
  1495. "--gs-results-bucket=chromium-result-details"
  1496. ],
  1497. "isolate_name": "foo_test",
  1498. "merge": {
  1499. "args": [
  1500. "--bucket",
  1501. "chromium-result-details",
  1502. "--test-name",
  1503. "foo_test"
  1504. ],
  1505. "script": \
  1506. "//build/android/pylib/results/presentation/test_results_presentation.py"
  1507. },
  1508. "name": "foo_test",
  1509. "swarming": {
  1510. "can_use_on_swarming_builders": true,
  1511. "cipd_packages": [
  1512. {
  1513. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  1514. "location": "bin",
  1515. "revision": \
  1516. "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  1517. }
  1518. ],
  1519. "output_links": [
  1520. {
  1521. "link": [
  1522. "https://luci-logdog.appspot.com/v/?s",
  1523. "=android%2Fswarming%2Flogcats%2F",
  1524. "${TASK_ID}%2F%2B%2Funified_logcats"
  1525. ],
  1526. "name": "shard #${SHARD_INDEX} logcats"
  1527. }
  1528. ]
  1529. }
  1530. }
  1531. ]
  1532. }
  1533. }
  1534. """
  1535. SCRIPT_OUTPUT = """\
  1536. {
  1537. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1538. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1539. "Fake Tester": {
  1540. "scripts": [
  1541. {
  1542. "name": "foo_test",
  1543. "script": "foo.py"
  1544. }
  1545. ]
  1546. }
  1547. }
  1548. """
  1549. SCRIPT_WITH_ARGS_OUTPUT = """\
  1550. {
  1551. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1552. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1553. "Fake Tester": {
  1554. "scripts": [
  1555. {
  1556. "args": [
  1557. "--fake-arg"
  1558. ],
  1559. "name": "foo_test",
  1560. "script": "foo.py"
  1561. }
  1562. ]
  1563. }
  1564. }
  1565. """
  1566. JUNIT_OUTPUT = """\
  1567. {
  1568. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1569. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1570. "Fake Tester": {
  1571. "junit_tests": [
  1572. {
  1573. "name": "foo_test",
  1574. "test": "foo_test"
  1575. }
  1576. ]
  1577. }
  1578. }
  1579. """
  1580. GPU_TELEMETRY_TEST_OUTPUT = """\
  1581. {
  1582. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1583. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1584. "Fake Tester": {
  1585. "isolated_scripts": [
  1586. {
  1587. "args": [
  1588. "foo",
  1589. "--show-stdout",
  1590. "--browser=release",
  1591. "--passthrough",
  1592. "-v",
  1593. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  1594. ],
  1595. "isolate_name": "telemetry_gpu_integration_test",
  1596. "merge": {
  1597. "args": [],
  1598. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1599. },
  1600. "name": "foo_tests",
  1601. "should_retry_with_patch": false,
  1602. "swarming": {
  1603. "can_use_on_swarming_builders": true,
  1604. "dimension_sets": [
  1605. {
  1606. "gpu": "10de:1cb3"
  1607. }
  1608. ],
  1609. "idempotent": false
  1610. },
  1611. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
  1612. }
  1613. ]
  1614. }
  1615. }
  1616. """
  1617. GPU_TELEMETRY_TEST_OUTPUT_ANDROID = """\
  1618. {
  1619. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1620. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1621. "Fake Tester": {
  1622. "isolated_scripts": [
  1623. {
  1624. "args": [
  1625. "foo",
  1626. "--show-stdout",
  1627. "--browser=android-chromium",
  1628. "--passthrough",
  1629. "-v",
  1630. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  1631. ],
  1632. "isolate_name": "telemetry_gpu_integration_test_android_chrome",
  1633. "merge": {
  1634. "args": [],
  1635. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1636. },
  1637. "name": "foo_tests",
  1638. "should_retry_with_patch": false,
  1639. "swarming": {
  1640. "can_use_on_swarming_builders": true,
  1641. "cipd_packages": [
  1642. {
  1643. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  1644. "location": "bin",
  1645. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  1646. }
  1647. ],
  1648. "dimension_sets": [
  1649. {
  1650. "device_type": "bullhead"
  1651. }
  1652. ],
  1653. "idempotent": false
  1654. },
  1655. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/"
  1656. }
  1657. ]
  1658. }
  1659. }
  1660. """
  1661. GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW = """\
  1662. {
  1663. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1664. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1665. "Fake Tester": {
  1666. "isolated_scripts": [
  1667. {
  1668. "args": [
  1669. "foo",
  1670. "--show-stdout",
  1671. "--browser=android-webview-instrumentation",
  1672. "--passthrough",
  1673. "-v",
  1674. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  1675. ],
  1676. "isolate_name": "telemetry_gpu_integration_test_android_webview",
  1677. "merge": {
  1678. "args": [],
  1679. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1680. },
  1681. "name": "foo_tests",
  1682. "should_retry_with_patch": false,
  1683. "swarming": {
  1684. "can_use_on_swarming_builders": true,
  1685. "cipd_packages": [
  1686. {
  1687. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  1688. "location": "bin",
  1689. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  1690. }
  1691. ],
  1692. "dimension_sets": [
  1693. {
  1694. "device_type": "bullhead"
  1695. }
  1696. ],
  1697. "idempotent": false
  1698. },
  1699. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_android_webview/"
  1700. }
  1701. ]
  1702. }
  1703. }
  1704. """
  1705. GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA = """\
  1706. {
  1707. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1708. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1709. "Fake Tester": {
  1710. "isolated_scripts": [
  1711. {
  1712. "args": [
  1713. "foo",
  1714. "--show-stdout",
  1715. "--browser=fuchsia-chrome",
  1716. "--passthrough",
  1717. "-v",
  1718. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  1719. ],
  1720. "isolate_name": "telemetry_gpu_integration_test_fuchsia",
  1721. "merge": {
  1722. "args": [],
  1723. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1724. },
  1725. "name": "foo_tests",
  1726. "should_retry_with_patch": false,
  1727. "swarming": {
  1728. "can_use_on_swarming_builders": true,
  1729. "dimension_sets": [
  1730. {
  1731. "kvm": "1"
  1732. }
  1733. ],
  1734. "idempotent": false
  1735. },
  1736. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_fuchsia/"
  1737. }
  1738. ]
  1739. }
  1740. }
  1741. """
  1742. GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING = """\
  1743. {
  1744. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1745. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1746. "Fake Tester": {
  1747. "isolated_scripts": [
  1748. {
  1749. "args": [
  1750. "foo",
  1751. "--show-stdout",
  1752. "--browser=cast-streaming-shell",
  1753. "--passthrough",
  1754. "-v",
  1755. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  1756. ],
  1757. "isolate_name": "telemetry_gpu_integration_test_fuchsia",
  1758. "merge": {
  1759. "args": [],
  1760. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1761. },
  1762. "name": "foo_tests",
  1763. "should_retry_with_patch": false,
  1764. "swarming": {
  1765. "can_use_on_swarming_builders": true,
  1766. "dimension_sets": [
  1767. {
  1768. "kvm": "1"
  1769. }
  1770. ],
  1771. "idempotent": false
  1772. },
  1773. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_fuchsia/"
  1774. }
  1775. ]
  1776. }
  1777. }
  1778. """
  1779. NVIDIA_GPU_TELEMETRY_TEST_OUTPUT = """\
  1780. {
  1781. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1782. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1783. "Fake Tester": {
  1784. "isolated_scripts": [
  1785. {
  1786. "args": [
  1787. "foo",
  1788. "--show-stdout",
  1789. "--browser=release",
  1790. "--passthrough",
  1791. "-v",
  1792. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
  1793. "--gpu-vendor-id",
  1794. "10de",
  1795. "--gpu-device-id",
  1796. "1cb3"
  1797. ],
  1798. "isolate_name": "telemetry_gpu_integration_test",
  1799. "merge": {
  1800. "args": [],
  1801. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1802. },
  1803. "name": "foo_tests",
  1804. "should_retry_with_patch": false,
  1805. "swarming": {
  1806. "can_use_on_swarming_builders": true,
  1807. "dimension_sets": [
  1808. {
  1809. "gpu": "10de:1cb3-26.21.14.3102"
  1810. }
  1811. ],
  1812. "idempotent": false
  1813. },
  1814. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
  1815. }
  1816. ]
  1817. }
  1818. }
  1819. """
  1820. INTEL_GPU_TELEMETRY_TEST_OUTPUT = """\
  1821. {
  1822. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1823. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1824. "Fake Tester": {
  1825. "isolated_scripts": [
  1826. {
  1827. "args": [
  1828. "foo",
  1829. "--show-stdout",
  1830. "--browser=release",
  1831. "--passthrough",
  1832. "-v",
  1833. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
  1834. "--gpu-vendor-id",
  1835. "8086",
  1836. "--gpu-device-id",
  1837. "5912"
  1838. ],
  1839. "isolate_name": "telemetry_gpu_integration_test",
  1840. "merge": {
  1841. "args": [],
  1842. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1843. },
  1844. "name": "foo_tests",
  1845. "should_retry_with_patch": false,
  1846. "swarming": {
  1847. "can_use_on_swarming_builders": true,
  1848. "dimension_sets": [
  1849. {
  1850. "gpu": "8086:5912-24.20.100.6286"
  1851. }
  1852. ],
  1853. "idempotent": false
  1854. },
  1855. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
  1856. }
  1857. ]
  1858. }
  1859. }
  1860. """
  1861. INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT = """\
  1862. {
  1863. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1864. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1865. "Fake Tester": {
  1866. "isolated_scripts": [
  1867. {
  1868. "args": [
  1869. "foo",
  1870. "--show-stdout",
  1871. "--browser=release",
  1872. "--passthrough",
  1873. "-v",
  1874. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
  1875. "--gpu-vendor-id",
  1876. "8086",
  1877. "--gpu-device-id",
  1878. "3e92"
  1879. ],
  1880. "isolate_name": "telemetry_gpu_integration_test",
  1881. "merge": {
  1882. "args": [],
  1883. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1884. },
  1885. "name": "foo_tests",
  1886. "should_retry_with_patch": false,
  1887. "swarming": {
  1888. "can_use_on_swarming_builders": true,
  1889. "dimension_sets": [
  1890. {
  1891. "gpu": "8086:3e92-24.20.100.6286"
  1892. }
  1893. ],
  1894. "idempotent": false
  1895. },
  1896. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
  1897. }
  1898. ]
  1899. }
  1900. }
  1901. """
  1902. GPU_TELEMETRY_TEST_VARIANTS_OUTPUT = """\
  1903. {
  1904. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1905. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1906. "Fake Tester": {
  1907. "isolated_scripts": [
  1908. {
  1909. "args": [
  1910. "swarming_test",
  1911. "--show-stdout",
  1912. "--browser=release",
  1913. "--passthrough",
  1914. "-v",
  1915. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
  1916. "--platform",
  1917. "device",
  1918. "--version",
  1919. "1"
  1920. ],
  1921. "isolate_name": "telemetry_gpu_integration_test",
  1922. "merge": {
  1923. "args": [],
  1924. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1925. },
  1926. "name": "swarming_test a_variant",
  1927. "should_retry_with_patch": false,
  1928. "swarming": {
  1929. "can_use_on_swarming_builders": true,
  1930. "dimension_sets": [
  1931. {
  1932. "gpu": "8086:3e92-24.20.100.6286"
  1933. }
  1934. ],
  1935. "idempotent": false
  1936. },
  1937. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/",
  1938. "variant_id": "a_variant"
  1939. },
  1940. {
  1941. "args": [
  1942. "swarming_test",
  1943. "--show-stdout",
  1944. "--browser=release",
  1945. "--passthrough",
  1946. "-v",
  1947. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
  1948. "a",
  1949. "b"
  1950. ],
  1951. "isolate_name": "telemetry_gpu_integration_test",
  1952. "merge": {
  1953. "args": [],
  1954. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  1955. },
  1956. "name": "swarming_test ab",
  1957. "should_retry_with_patch": false,
  1958. "swarming": {
  1959. "can_use_on_swarming_builders": true,
  1960. "dimension_sets": [
  1961. {
  1962. "gpu": "8086:3e92-24.20.100.6286"
  1963. }
  1964. ],
  1965. "idempotent": false
  1966. },
  1967. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/",
  1968. "variant_id": "ab"
  1969. }
  1970. ]
  1971. }
  1972. }
  1973. """
  1974. ANDROID_WATERFALL_OUTPUT = """\
  1975. {
  1976. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  1977. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  1978. "Android Builder": {
  1979. "additional_compile_targets": [
  1980. "bar_test"
  1981. ]
  1982. },
  1983. "Fake Android K Tester": {
  1984. "additional_compile_targets": [
  1985. "bar_test"
  1986. ],
  1987. "gtest_tests": [
  1988. {
  1989. "args": [
  1990. "--gs-results-bucket=chromium-result-details",
  1991. "--recover-devices"
  1992. ],
  1993. "merge": {
  1994. "args": [],
  1995. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  1996. },
  1997. "swarming": {
  1998. "can_use_on_swarming_builders": true,
  1999. "cipd_packages": [
  2000. {
  2001. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  2002. "location": "bin",
  2003. "revision": \
  2004. "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  2005. }
  2006. ],
  2007. "dimension_sets": [
  2008. {
  2009. "device_os": "KTU84P",
  2010. "device_os_type": "userdebug",
  2011. "device_type": "hammerhead",
  2012. "integrity": "high",
  2013. "os": "Android"
  2014. }
  2015. ],
  2016. "expiration": 120,
  2017. "output_links": [
  2018. {
  2019. "link": [
  2020. "https://luci-logdog.appspot.com/v/?s",
  2021. "=android%2Fswarming%2Flogcats%2F",
  2022. "${TASK_ID}%2F%2B%2Funified_logcats"
  2023. ],
  2024. "name": "shard #${SHARD_INDEX} logcats"
  2025. }
  2026. ]
  2027. },
  2028. "test": "foo_test"
  2029. }
  2030. ]
  2031. },
  2032. "Fake Android L Tester": {
  2033. "gtest_tests": [
  2034. {
  2035. "args": [
  2036. "--gs-results-bucket=chromium-result-details",
  2037. "--recover-devices"
  2038. ],
  2039. "merge": {
  2040. "args": [],
  2041. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  2042. },
  2043. "swarming": {
  2044. "can_use_on_swarming_builders": true,
  2045. "cipd_packages": [
  2046. {
  2047. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  2048. "location": "bin",
  2049. "revision": \
  2050. "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  2051. }
  2052. ],
  2053. "dimension_sets": [
  2054. {
  2055. "device_os": "LMY41U",
  2056. "device_os_type": "user",
  2057. "device_type": "hammerhead",
  2058. "integrity": "high",
  2059. "os": "Android"
  2060. }
  2061. ],
  2062. "expiration": 120
  2063. },
  2064. "test": "foo_test"
  2065. }
  2066. ]
  2067. },
  2068. "Fake Android M Tester": {
  2069. "gtest_tests": [
  2070. {
  2071. "merge": {
  2072. "args": [],
  2073. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  2074. },
  2075. "swarming": {
  2076. "can_use_on_swarming_builders": false
  2077. },
  2078. "test": "foo_test"
  2079. }
  2080. ]
  2081. }
  2082. }
  2083. """
  2084. CHROMEOS_TRIGGER_SCRIPT_OUTPUT = """\
  2085. {
  2086. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  2087. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  2088. "Fake Tester": {
  2089. "gtest_tests": [
  2090. {
  2091. "merge": {
  2092. "args": [],
  2093. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  2094. },
  2095. "swarming": {
  2096. "can_use_on_swarming_builders": true,
  2097. "dimension_sets": [
  2098. {
  2099. "device_type": "foo_device",
  2100. "integrity": "high"
  2101. }
  2102. ],
  2103. "expiration": 120
  2104. },
  2105. "test": "foo_test",
  2106. "trigger_script": {
  2107. "script": "//testing/trigger_scripts/chromeos_device_trigger.py"
  2108. }
  2109. }
  2110. ]
  2111. }
  2112. }
  2113. """
  2114. GPU_DIMENSIONS_WATERFALL_OUTPUT = """\
  2115. {
  2116. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  2117. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  2118. "Fake Tester": {
  2119. "isolated_scripts": [
  2120. {
  2121. "args": [
  2122. "foo_test",
  2123. "--show-stdout",
  2124. "--browser=release",
  2125. "--passthrough",
  2126. "-v",
  2127. "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
  2128. ],
  2129. "isolate_name": "telemetry_gpu_integration_test",
  2130. "merge": {
  2131. "args": [],
  2132. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  2133. },
  2134. "name": "foo_test",
  2135. "should_retry_with_patch": false,
  2136. "swarming": {
  2137. "can_use_on_swarming_builders": true,
  2138. "dimension_sets": [
  2139. {
  2140. "iama": "mixin",
  2141. "integrity": "high"
  2142. }
  2143. ],
  2144. "expiration": 120,
  2145. "idempotent": false,
  2146. "value": "test"
  2147. },
  2148. "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
  2149. }
  2150. ]
  2151. }
  2152. }
  2153. """
  2154. LUCI_MILO_CFG = """\
  2155. consoles {
  2156. builders {
  2157. name: "buildbucket/luci.chromium.ci/Fake Tester"
  2158. }
  2159. }
  2160. """
  2161. LUCI_MILO_CFG_WATERFALL_SORTING = """\
  2162. consoles {
  2163. builders {
  2164. name: "buildbucket/luci.chromium.ci/Fake Tester"
  2165. name: "buildbucket/luci.chromium.ci/Really Fake Tester"
  2166. }
  2167. }
  2168. """
  2169. TEST_SUITE_SORTING_WATERFALL = """
  2170. [
  2171. {
  2172. 'project': 'chromium',
  2173. 'bucket': 'ci',
  2174. 'name': 'chromium.test',
  2175. 'machines': {
  2176. 'Fake Tester': {
  2177. 'test_suites': {
  2178. 'gtest_tests': 'suite_a',
  2179. 'scripts': 'suite_b',
  2180. },
  2181. },
  2182. },
  2183. },
  2184. ]
  2185. """
  2186. TEST_SUITE_SORTED_WATERFALL = """
  2187. [
  2188. {
  2189. 'project': 'chromium',
  2190. 'bucket': 'ci',
  2191. 'name': 'chromium.test',
  2192. 'machines': {
  2193. 'Fake Tester': {
  2194. 'test_suites': {
  2195. 'gtest_tests': 'suite_a',
  2196. 'scripts': 'suite_b',
  2197. },
  2198. },
  2199. 'Really Fake Tester': {
  2200. 'test_suites': {
  2201. 'gtest_tests': 'suite_a',
  2202. 'scripts': 'suite_b',
  2203. },
  2204. },
  2205. },
  2206. },
  2207. {
  2208. 'project': 'chromium',
  2209. 'bucket': 'ci',
  2210. 'name': 'chromium.zz.test',
  2211. 'machines': {
  2212. 'Fake Tester': {
  2213. 'test_suites': {
  2214. 'gtest_tests': 'suite_a',
  2215. 'scripts': 'suite_b',
  2216. },
  2217. },
  2218. 'Really Fake Tester': {
  2219. 'test_suites': {
  2220. 'gtest_tests': 'suite_a',
  2221. 'scripts': 'suite_b',
  2222. },
  2223. },
  2224. },
  2225. },
  2226. ]
  2227. """
  2228. TEST_SUITE_UNSORTED_WATERFALL_1 = """
  2229. [
  2230. {
  2231. 'project': 'chromium',
  2232. 'bucket': 'ci',
  2233. 'name': 'chromium.zz.test',
  2234. 'machines': {
  2235. 'Fake Tester': {
  2236. 'test_suites': {
  2237. 'gtest_tests': 'suite_a',
  2238. 'scripts': 'suite_b',
  2239. },
  2240. },
  2241. 'Really Fake Tester': {
  2242. 'test_suites': {
  2243. 'gtest_tests': 'suite_a',
  2244. 'scripts': 'suite_b',
  2245. },
  2246. },
  2247. },
  2248. },
  2249. {
  2250. 'project': 'chromium',
  2251. 'bucket': 'ci',
  2252. 'name': 'chromium.test',
  2253. 'machines': {
  2254. 'Fake Tester': {
  2255. 'test_suites': {
  2256. 'gtest_tests': 'suite_a',
  2257. 'scripts': 'suite_b',
  2258. },
  2259. },
  2260. 'Really Fake Tester': {
  2261. 'test_suites': {
  2262. 'gtest_tests': 'suite_a',
  2263. 'scripts': 'suite_b',
  2264. },
  2265. },
  2266. },
  2267. },
  2268. ]
  2269. """
  2270. TEST_SUITE_UNSORTED_WATERFALL_2 = """
  2271. [
  2272. {
  2273. 'project': 'chromium',
  2274. 'bucket': 'ci',
  2275. 'name': 'chromium.test',
  2276. 'machines': {
  2277. 'Really Fake Tester': {
  2278. 'test_suites': {
  2279. 'gtest_tests': 'suite_a',
  2280. 'scripts': 'suite_b',
  2281. },
  2282. },
  2283. 'Fake Tester': {
  2284. 'test_suites': {
  2285. 'gtest_tests': 'suite_a',
  2286. 'scripts': 'suite_b',
  2287. },
  2288. },
  2289. },
  2290. },
  2291. {
  2292. 'project': 'chromium',
  2293. 'bucket': 'ci',
  2294. 'name': 'chromium.zz.test',
  2295. 'machines': {
  2296. 'Fake Tester': {
  2297. 'test_suites': {
  2298. 'gtest_tests': 'suite_a',
  2299. 'scripts': 'suite_b',
  2300. },
  2301. },
  2302. 'Really Fake Tester': {
  2303. 'test_suites': {
  2304. 'gtest_tests': 'suite_a',
  2305. 'scripts': 'suite_b',
  2306. },
  2307. },
  2308. },
  2309. },
  2310. ]
  2311. """
  2312. # Note that the suites in basic_suites would be sorted after the suites in
  2313. # compound_suites. This is valid though, because each set of suites is sorted
  2314. # separately.
  2315. # suite_c is a 'gtest_tests' test
  2316. # suite_d is a 'scripts' test
  2317. TEST_SUITE_SORTED = """\
  2318. {
  2319. 'basic_suites': {
  2320. 'suite_c': {
  2321. 'suite_c': {},
  2322. },
  2323. 'suite_d': {
  2324. 'script': {
  2325. 'script': 'suite_d.py',
  2326. }
  2327. },
  2328. },
  2329. 'compound_suites': {
  2330. 'suite_a': [
  2331. 'suite_c',
  2332. ],
  2333. 'suite_b': [
  2334. 'suite_d',
  2335. ],
  2336. },
  2337. }
  2338. """
  2339. TEST_SUITE_UNSORTED_1 = """\
  2340. {
  2341. 'basic_suites': {
  2342. 'suite_d': {
  2343. 'a': 'b',
  2344. },
  2345. 'suite_c': {
  2346. 'a': 'b',
  2347. },
  2348. },
  2349. 'compound_suites': {
  2350. 'suite_a': [
  2351. 'suite_c',
  2352. ],
  2353. 'suite_b': [
  2354. 'suite_d',
  2355. ],
  2356. },
  2357. }
  2358. """
  2359. TEST_SUITE_UNSORTED_2 = """\
  2360. {
  2361. 'basic_suites': {
  2362. 'suite_c': {
  2363. 'a': 'b',
  2364. },
  2365. 'suite_d': {
  2366. 'a': 'b',
  2367. },
  2368. },
  2369. 'compound_suites': {
  2370. 'suite_b': [
  2371. 'suite_c',
  2372. ],
  2373. 'suite_a': [
  2374. 'suite_d',
  2375. ],
  2376. },
  2377. }
  2378. """
  2379. TEST_SUITE_UNSORTED_3 = """\
  2380. {
  2381. 'basic_suites': {
  2382. 'suite_d': {
  2383. 'a': 'b',
  2384. },
  2385. 'suite_c': {
  2386. 'a': 'b',
  2387. },
  2388. },
  2389. 'compound_suites': {
  2390. 'suite_b': [
  2391. 'suite_c',
  2392. ],
  2393. 'suite_a': [
  2394. 'suite_d',
  2395. ],
  2396. },
  2397. }
  2398. """
  2399. TEST_SUITES_SYNTAX_ERROR = """\
  2400. {
  2401. 'basic_suites': {
  2402. 3: {
  2403. 'suite_c': {},
  2404. },
  2405. },
  2406. 'compound_suites': {},
  2407. }
  2408. """
  2409. GN_ISOLATE_MAP="""\
  2410. {
  2411. 'foo_test': {
  2412. 'label': '//chrome/test:foo_test',
  2413. 'type': 'windowed_test_launcher',
  2414. }
  2415. }
  2416. """
  2417. GPU_TELEMETRY_GN_ISOLATE_MAP="""\
  2418. {
  2419. 'telemetry_gpu_integration_test': {
  2420. 'label': '//chrome/test:telemetry_gpu_integration_test',
  2421. 'type': 'script',
  2422. }
  2423. }
  2424. """
  2425. GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID = """\
  2426. {
  2427. 'telemetry_gpu_integration_test_android_chrome': {
  2428. 'label': '//chrome/test:telemetry_gpu_integration_test_android_chrome',
  2429. 'type': 'script',
  2430. }
  2431. }
  2432. """
  2433. GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW = """\
  2434. {
  2435. 'telemetry_gpu_integration_test_android_webview': {
  2436. 'label': '//chrome/test:telemetry_gpu_integration_test_android_webview',
  2437. 'type': 'script',
  2438. }
  2439. }
  2440. """
  2441. GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA = """\
  2442. {
  2443. 'telemetry_gpu_integration_test_fuchsia': {
  2444. 'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
  2445. 'type': 'script',
  2446. }
  2447. }
  2448. """
  2449. GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING = """\
  2450. {
  2451. 'telemetry_gpu_integration_test_fuchsia': {
  2452. 'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
  2453. 'type': 'script',
  2454. }
  2455. }
  2456. """
  2457. GN_ISOLATE_MAP_KEY_LABEL_MISMATCH="""\
  2458. {
  2459. 'foo_test': {
  2460. 'label': '//chrome/test:foo_test_tmp',
  2461. 'type': 'windowed_test_launcher',
  2462. }
  2463. }
  2464. """
  2465. GN_ISOLATE_MAP_USING_IMPLICIT_NAME="""\
  2466. {
  2467. 'foo_test': {
  2468. 'label': '//chrome/foo_test',
  2469. 'type': 'windowed_test_launcher',
  2470. }
  2471. }
  2472. """
  2473. class UnitTest(TestCase):
  2474. def test_base_generator(self):
  2475. # Only needed for complete code coverage.
  2476. self.assertRaises(NotImplementedError,
  2477. generate_buildbot_json.BaseGenerator(None).generate,
  2478. None, None, None, None)
  2479. self.assertRaises(NotImplementedError,
  2480. generate_buildbot_json.BaseGenerator(None).sort,
  2481. None)
  2482. def test_good_test_suites_are_ok(self):
  2483. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE,
  2484. LUCI_MILO_CFG)
  2485. fbb.check_input_file_consistency(verbose=True)
  2486. self.assertFalse(fbb.printed_lines)
  2487. def test_good_composition_test_suites_are_ok(self):
  2488. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
  2489. GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2490. fbb.check_input_file_consistency(verbose=True)
  2491. self.assertFalse(fbb.printed_lines)
  2492. def test_bad_composition_test_suites_are_caught(self):
  2493. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
  2494. BAD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2495. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2496. 'compound_suites may not refer to.*'):
  2497. fbb.check_input_file_consistency(verbose=True)
  2498. self.assertFalse(fbb.printed_lines)
  2499. def test_composition_test_suites_no_conflicts(self):
  2500. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
  2501. CONFLICTING_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2502. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2503. 'Conflicting test definitions.*'):
  2504. fbb.check_input_file_consistency(verbose=True)
  2505. self.assertFalse(fbb.printed_lines)
  2506. def test_composition_test_suites_no_duplicate_names(self):
  2507. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
  2508. DUPLICATES_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2509. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2510. '.*may not duplicate basic test suite.*'):
  2511. fbb.check_input_file_consistency(verbose=True)
  2512. self.assertFalse(fbb.printed_lines)
  2513. def test_unknown_test_suites_are_caught(self):
  2514. fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_WATERFALL, FOO_TEST_SUITE,
  2515. LUCI_MILO_CFG)
  2516. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2517. 'Test suite baz_tests from machine.*'):
  2518. fbb.check_input_file_consistency(verbose=True)
  2519. self.assertFalse(fbb.printed_lines)
  2520. def test_unknown_test_suite_types_are_caught(self):
  2521. fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_TYPE_WATERFALL,
  2522. FOO_TEST_SUITE, LUCI_MILO_CFG)
  2523. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2524. 'Unknown test suite type foo_test_type.*'):
  2525. fbb.check_input_file_consistency(verbose=True)
  2526. self.assertFalse(fbb.printed_lines)
  2527. def test_unrefed_test_suite_caught(self):
  2528. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, UNREFED_TEST_SUITE,
  2529. LUCI_MILO_CFG)
  2530. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2531. '.*unreferenced.*bar_tests.*'):
  2532. fbb.check_input_file_consistency(verbose=True)
  2533. self.assertFalse(fbb.printed_lines)
  2534. def test_good_waterfall_output(self):
  2535. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
  2536. GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2537. self.create_testing_buildbot_json_file('chromium.test.json',
  2538. COMPOSITION_WATERFALL_OUTPUT)
  2539. fbb.check_output_file_consistency(verbose=True)
  2540. self.assertFalse(fbb.printed_lines)
  2541. def test_reusing_gtest_targets(self):
  2542. fbb = FakeBBGen(self.args,
  2543. FOO_GTESTS_WATERFALL,
  2544. REUSING_TEST_WITH_DIFFERENT_NAME,
  2545. LUCI_MILO_CFG,
  2546. gn_isolate_map=GN_ISOLATE_MAP)
  2547. self.create_testing_buildbot_json_file('chromium.test.json',
  2548. VARIATION_GTEST_OUTPUT)
  2549. fbb.check_output_file_consistency(verbose=True)
  2550. self.assertFalse(fbb.printed_lines)
  2551. def test_load_multiple_isolate_map_files_with_duplicates(self):
  2552. self.args.isolate_map_files = ['gn_isolate_map.pyl']
  2553. fbb = FakeBBGen(self.args,
  2554. FOO_GTESTS_WATERFALL,
  2555. REUSING_TEST_WITH_DIFFERENT_NAME,
  2556. LUCI_MILO_CFG,
  2557. gn_isolate_map=GN_ISOLATE_MAP)
  2558. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2559. 'Duplicate targets in isolate map files.*'):
  2560. fbb.load_configuration_files()
  2561. def test_load_multiple_isolate_map_files_without_duplicates(self):
  2562. self.args.isolate_map_files = ['gn_isolate_map2.pyl']
  2563. fbb = FakeBBGen(self.args,
  2564. FOO_GTESTS_WATERFALL,
  2565. REUSING_TEST_WITH_DIFFERENT_NAME,
  2566. LUCI_MILO_CFG,
  2567. gn_isolate_map=GN_ISOLATE_MAP)
  2568. fbb.load_configuration_files()
  2569. isolate_dict = {}
  2570. isolate_map_1 = fbb.load_pyl_file('gn_isolate_map.pyl')
  2571. isolate_map_2 = fbb.load_pyl_file('gn_isolate_map2.pyl')
  2572. isolate_dict.update(isolate_map_1)
  2573. isolate_dict.update(isolate_map_2)
  2574. self.assertEqual(isolate_dict, fbb.gn_isolate_map)
  2575. def test_gn_isolate_map_with_label_mismatch(self):
  2576. fbb = FakeBBGen(self.args,
  2577. FOO_GTESTS_WATERFALL,
  2578. FOO_TEST_SUITE,
  2579. LUCI_MILO_CFG,
  2580. gn_isolate_map=GN_ISOLATE_MAP_KEY_LABEL_MISMATCH)
  2581. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2582. 'key name.*foo_test.*label.*'
  2583. 'foo_test_tmp.*'):
  2584. fbb.check_input_file_consistency(verbose=True)
  2585. self.assertFalse(fbb.printed_lines)
  2586. def test_gn_isolate_map_using_implicit_gn_name(self):
  2587. fbb = FakeBBGen(self.args,
  2588. FOO_GTESTS_WATERFALL,
  2589. FOO_TEST_SUITE,
  2590. LUCI_MILO_CFG,
  2591. gn_isolate_map=GN_ISOLATE_MAP_USING_IMPLICIT_NAME)
  2592. with self.assertRaisesRegex(
  2593. generate_buildbot_json.BBGenErr,
  2594. 'Malformed.*//chrome/foo_test.*for key.*'
  2595. 'foo_test.*'):
  2596. fbb.check_input_file_consistency(verbose=True)
  2597. self.assertFalse(fbb.printed_lines)
  2598. def test_noop_exception_does_nothing(self):
  2599. fbb = FakeBBGen(self.args,
  2600. COMPOSITION_GTEST_SUITE_WATERFALL,
  2601. GOOD_COMPOSITION_TEST_SUITES,
  2602. LUCI_MILO_CFG,
  2603. exceptions=EMPTY_BAR_TEST_EXCEPTIONS)
  2604. self.create_testing_buildbot_json_file('chromium.test.json',
  2605. COMPOSITION_WATERFALL_OUTPUT)
  2606. fbb.check_output_file_consistency(verbose=True)
  2607. self.assertFalse(fbb.printed_lines)
  2608. def test_test_arg_merges(self):
  2609. fbb = FakeBBGen(self.args,
  2610. FOO_GTESTS_WATERFALL,
  2611. FOO_TEST_SUITE_WITH_ARGS,
  2612. LUCI_MILO_CFG,
  2613. exceptions=FOO_TEST_MODIFICATIONS)
  2614. self.create_testing_buildbot_json_file('chromium.test.json',
  2615. MERGED_ARGS_OUTPUT)
  2616. fbb.check_output_file_consistency(verbose=True)
  2617. self.assertFalse(fbb.printed_lines)
  2618. def test_enable_features_arg_merges(self):
  2619. fbb = FakeBBGen(self.args, FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL,
  2620. FOO_TEST_SUITE_WITH_ENABLE_FEATURES, LUCI_MILO_CFG)
  2621. self.create_testing_buildbot_json_file('chromium.test.json',
  2622. MERGED_ENABLE_FEATURES_OUTPUT)
  2623. fbb.check_output_file_consistency(verbose=True)
  2624. self.assertFalse(fbb.printed_lines)
  2625. def test_linux_args(self):
  2626. fbb = FakeBBGen(self.args, FOO_LINUX_GTESTS_WATERFALL,
  2627. FOO_TEST_SUITE_WITH_LINUX_ARGS, LUCI_MILO_CFG)
  2628. self.create_testing_buildbot_json_file('chromium.test.json',
  2629. LINUX_ARGS_OUTPUT)
  2630. fbb.check_output_file_consistency(verbose=True)
  2631. self.assertFalse(fbb.printed_lines)
  2632. def test_test_filtering(self):
  2633. fbb = FakeBBGen(self.args,
  2634. COMPOSITION_GTEST_SUITE_WATERFALL,
  2635. GOOD_COMPOSITION_TEST_SUITES,
  2636. LUCI_MILO_CFG,
  2637. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2638. self.create_testing_buildbot_json_file(
  2639. 'chromium.test.json', COMPOSITION_WATERFALL_FILTERED_OUTPUT)
  2640. fbb.check_output_file_consistency(verbose=True)
  2641. self.assertFalse(fbb.printed_lines)
  2642. def test_test_modifications(self):
  2643. fbb = FakeBBGen(self.args,
  2644. FOO_GTESTS_WATERFALL,
  2645. FOO_TEST_SUITE,
  2646. LUCI_MILO_CFG,
  2647. exceptions=FOO_TEST_MODIFICATIONS)
  2648. self.create_testing_buildbot_json_file('chromium.test.json',
  2649. MODIFIED_OUTPUT)
  2650. self.create_testing_buildbot_json_file('chromium.ci.json', MODIFIED_OUTPUT)
  2651. fbb.check_output_file_consistency(verbose=True)
  2652. self.assertFalse(fbb.printed_lines)
  2653. def test_test_with_explicit_none(self):
  2654. fbb = FakeBBGen(self.args,
  2655. FOO_GTESTS_WATERFALL,
  2656. FOO_TEST_SUITE,
  2657. LUCI_MILO_CFG,
  2658. exceptions=FOO_TEST_EXPLICIT_NONE_EXCEPTIONS,
  2659. mixins=SWARMING_MIXINS)
  2660. self.create_testing_buildbot_json_file('chromium.test.json',
  2661. EXPLICIT_NONE_OUTPUT)
  2662. self.create_testing_buildbot_json_file('chromium.ci.json',
  2663. EXPLICIT_NONE_OUTPUT)
  2664. fbb.check_output_file_consistency(verbose=True)
  2665. self.assertFalse(fbb.printed_lines)
  2666. def test_isolated_script_tests(self):
  2667. fbb = FakeBBGen(self.args,
  2668. FOO_ISOLATED_SCRIPTS_WATERFALL,
  2669. GOOD_COMPOSITION_TEST_SUITES,
  2670. LUCI_MILO_CFG,
  2671. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2672. self.create_testing_buildbot_json_file('chromium.test.json',
  2673. ISOLATED_SCRIPT_OUTPUT)
  2674. self.create_testing_buildbot_json_file('chromium.ci.json',
  2675. ISOLATED_SCRIPT_OUTPUT)
  2676. fbb.check_output_file_consistency(verbose=True)
  2677. self.assertFalse(fbb.printed_lines)
  2678. def test_isolated_script_tests(self):
  2679. fbb = FakeBBGen(self.args,
  2680. FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID,
  2681. GOOD_COMPOSITION_TEST_SUITES,
  2682. LUCI_MILO_CFG,
  2683. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2684. self.create_testing_buildbot_json_file('chromium.test.json',
  2685. ISOLATED_SCRIPT_OUTPUT_ANDROID)
  2686. self.create_testing_buildbot_json_file('chromium.ci.json',
  2687. ISOLATED_SCRIPT_OUTPUT_ANDROID)
  2688. fbb.check_output_file_consistency(verbose=True)
  2689. self.assertFalse(fbb.printed_lines)
  2690. def test_script_with_args(self):
  2691. fbb = FakeBBGen(self.args,
  2692. FOO_SCRIPT_WATERFALL,
  2693. SCRIPT_SUITE,
  2694. LUCI_MILO_CFG,
  2695. exceptions=SCRIPT_WITH_ARGS_EXCEPTIONS)
  2696. self.create_testing_buildbot_json_file('chromium.test.json',
  2697. SCRIPT_WITH_ARGS_OUTPUT)
  2698. self.create_testing_buildbot_json_file('chromium.ci.json',
  2699. SCRIPT_WITH_ARGS_OUTPUT)
  2700. fbb.check_output_file_consistency(verbose=True)
  2701. self.assertFalse(fbb.printed_lines)
  2702. def test_script(self):
  2703. fbb = FakeBBGen(self.args,
  2704. FOO_SCRIPT_WATERFALL,
  2705. FOO_SCRIPT_SUITE,
  2706. LUCI_MILO_CFG,
  2707. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2708. self.create_testing_buildbot_json_file('chromium.test.json', SCRIPT_OUTPUT)
  2709. self.create_testing_buildbot_json_file('chromium.ci.json', SCRIPT_OUTPUT)
  2710. fbb.check_output_file_consistency(verbose=True)
  2711. self.assertFalse(fbb.printed_lines)
  2712. def test_script_machine_forbids_scripts(self):
  2713. fbb = FakeBBGen(self.args,
  2714. FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS,
  2715. FOO_SCRIPT_SUITE,
  2716. LUCI_MILO_CFG,
  2717. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2718. with self.assertRaisesRegex(
  2719. generate_buildbot_json.BBGenErr,
  2720. 'Attempted to generate a script test on tester.*'):
  2721. fbb.check_output_file_consistency(verbose=True)
  2722. def test_script_waterfall_forbids_scripts(self):
  2723. fbb = FakeBBGen(self.args,
  2724. FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS,
  2725. FOO_SCRIPT_SUITE,
  2726. LUCI_MILO_CFG,
  2727. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2728. with self.assertRaisesRegex(
  2729. generate_buildbot_json.BBGenErr,
  2730. 'Attempted to generate a script test on tester.*'):
  2731. fbb.check_output_file_consistency(verbose=True)
  2732. def test_junit_tests(self):
  2733. fbb = FakeBBGen(self.args,
  2734. FOO_JUNIT_WATERFALL,
  2735. GOOD_COMPOSITION_TEST_SUITES,
  2736. LUCI_MILO_CFG,
  2737. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2738. self.create_testing_buildbot_json_file('chromium.test.json', JUNIT_OUTPUT)
  2739. self.create_testing_buildbot_json_file('chromium.ci.json', JUNIT_OUTPUT)
  2740. fbb.check_output_file_consistency(verbose=True)
  2741. self.assertFalse(fbb.printed_lines)
  2742. def test_gpu_telemetry_tests(self):
  2743. fbb = FakeBBGen(self.args,
  2744. FOO_GPU_TELEMETRY_TEST_WATERFALL,
  2745. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
  2746. LUCI_MILO_CFG,
  2747. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2748. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  2749. self.create_testing_buildbot_json_file('chromium.test.json',
  2750. GPU_TELEMETRY_TEST_OUTPUT)
  2751. self.create_testing_buildbot_json_file('chromium.ci.json',
  2752. GPU_TELEMETRY_TEST_OUTPUT)
  2753. fbb.check_output_file_consistency(verbose=True)
  2754. self.assertFalse(fbb.printed_lines)
  2755. def test_gpu_telemetry_tests_pixel_with_filter(self):
  2756. fbb = FakeBBGen(self.args,
  2757. FOO_GPU_TELEMETRY_TEST_WATERFALL,
  2758. COMPOSITION_SUITE_WITH_PIXEL_AND_FILTER,
  2759. LUCI_MILO_CFG,
  2760. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2761. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  2762. self.create_testing_buildbot_json_file('chromium.test.json',
  2763. GPU_TELEMETRY_TEST_OUTPUT)
  2764. self.create_testing_buildbot_json_file('chromium.ci.json',
  2765. GPU_TELEMETRY_TEST_OUTPUT)
  2766. with self.assertRaises(RuntimeError):
  2767. fbb.check_output_file_consistency(verbose=True)
  2768. def test_gpu_telemetry_tests_android(self):
  2769. fbb = FakeBBGen(self.args,
  2770. FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID,
  2771. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
  2772. LUCI_MILO_CFG,
  2773. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2774. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID)
  2775. self.create_testing_buildbot_json_file('chromium.test.json',
  2776. GPU_TELEMETRY_TEST_OUTPUT_ANDROID)
  2777. self.create_testing_buildbot_json_file('chromium.ci.json',
  2778. GPU_TELEMETRY_TEST_OUTPUT_ANDROID)
  2779. fbb.check_output_file_consistency(verbose=True)
  2780. self.assertFalse(fbb.printed_lines)
  2781. def test_gpu_telemetry_tests_android_webview(self):
  2782. fbb = FakeBBGen(self.args,
  2783. FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW,
  2784. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
  2785. LUCI_MILO_CFG,
  2786. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2787. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW)
  2788. self.create_testing_buildbot_json_file(
  2789. 'chromium.test.json', GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW)
  2790. self.create_testing_buildbot_json_file(
  2791. 'chromium.ci.json', GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW)
  2792. fbb.check_output_file_consistency(verbose=True)
  2793. self.assertFalse(fbb.printed_lines)
  2794. def test_gpu_telemetry_tests_fuchsia(self):
  2795. fbb = FakeBBGen(self.args,
  2796. FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA,
  2797. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
  2798. LUCI_MILO_CFG,
  2799. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2800. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA)
  2801. self.create_testing_buildbot_json_file('chromium.test.json',
  2802. GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA)
  2803. self.create_testing_buildbot_json_file('chromium.ci.json',
  2804. GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA)
  2805. fbb.check_output_file_consistency(verbose=True)
  2806. self.assertFalse(fbb.printed_lines)
  2807. def test_gpu_telemetry_tests_cast_streaming(self):
  2808. fbb = FakeBBGen(self.args,
  2809. FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING,
  2810. COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
  2811. LUCI_MILO_CFG,
  2812. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2813. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING)
  2814. self.create_testing_buildbot_json_file(
  2815. 'chromium.test.json', GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING)
  2816. self.create_testing_buildbot_json_file(
  2817. 'chromium.ci.json', GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING)
  2818. fbb.check_output_file_consistency(verbose=True)
  2819. self.assertFalse(fbb.printed_lines)
  2820. def test_nvidia_gpu_telemetry_tests(self):
  2821. fbb = FakeBBGen(self.args,
  2822. NVIDIA_GPU_TELEMETRY_TEST_WATERFALL,
  2823. COMPOSITION_SUITE_WITH_GPU_ARGS,
  2824. LUCI_MILO_CFG,
  2825. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2826. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  2827. self.create_testing_buildbot_json_file('chromium.test.json',
  2828. NVIDIA_GPU_TELEMETRY_TEST_OUTPUT)
  2829. self.create_testing_buildbot_json_file('chromium.ci.json',
  2830. NVIDIA_GPU_TELEMETRY_TEST_OUTPUT)
  2831. fbb.check_output_file_consistency(verbose=True)
  2832. self.assertFalse(fbb.printed_lines)
  2833. def test_intel_gpu_telemetry_tests(self):
  2834. fbb = FakeBBGen(self.args,
  2835. INTEL_GPU_TELEMETRY_TEST_WATERFALL,
  2836. COMPOSITION_SUITE_WITH_GPU_ARGS,
  2837. LUCI_MILO_CFG,
  2838. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2839. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  2840. self.create_testing_buildbot_json_file('chromium.test.json',
  2841. INTEL_GPU_TELEMETRY_TEST_OUTPUT)
  2842. self.create_testing_buildbot_json_file('chromium.ci.json',
  2843. INTEL_GPU_TELEMETRY_TEST_OUTPUT)
  2844. fbb.check_output_file_consistency(verbose=True)
  2845. self.assertFalse(fbb.printed_lines)
  2846. def test_gpu_telemetry_tests_with_variants(self):
  2847. fbb = FakeBBGen(self.args,
  2848. GPU_TELEMETRY_TEST_VARIANTS_WATERFALL,
  2849. MATRIX_COMPOUND_MIXED_VARIANTS_REF,
  2850. LUCI_MILO_CFG,
  2851. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2852. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP,
  2853. variants=MULTI_VARIANTS_FILE)
  2854. self.create_testing_buildbot_json_file('chromium.test.json',
  2855. GPU_TELEMETRY_TEST_VARIANTS_OUTPUT)
  2856. fbb.check_output_file_consistency(verbose=True)
  2857. self.assertFalse(fbb.printed_lines)
  2858. def test_intel_uhd_gpu_telemetry_tests(self):
  2859. fbb = FakeBBGen(self.args,
  2860. INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL,
  2861. COMPOSITION_SUITE_WITH_GPU_ARGS,
  2862. LUCI_MILO_CFG,
  2863. exceptions=NO_BAR_TEST_EXCEPTIONS,
  2864. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  2865. self.create_testing_buildbot_json_file('chromium.test.json',
  2866. INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT)
  2867. self.create_testing_buildbot_json_file('chromium.ci.json',
  2868. INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT)
  2869. fbb.check_output_file_consistency(verbose=True)
  2870. self.assertFalse(fbb.printed_lines)
  2871. def test_gtest_as_isolated_Script(self):
  2872. fbb = FakeBBGen(self.args,
  2873. FOO_GTESTS_WATERFALL,
  2874. GTEST_AS_ISOLATED_SCRIPT_SUITE,
  2875. LUCI_MILO_CFG,
  2876. gn_isolate_map=GN_ISOLATE_MAP)
  2877. self.create_testing_buildbot_json_file(
  2878. 'chromium.test.json', FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT)
  2879. self.create_testing_buildbot_json_file(
  2880. 'chromium.ci.json', FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT)
  2881. fbb.check_output_file_consistency(verbose=True)
  2882. self.assertFalse(fbb.printed_lines)
  2883. def test_ungenerated_output_files_are_caught(self):
  2884. fbb = FakeBBGen(self.args,
  2885. COMPOSITION_GTEST_SUITE_WATERFALL,
  2886. GOOD_COMPOSITION_TEST_SUITES,
  2887. LUCI_MILO_CFG,
  2888. exceptions=NO_BAR_TEST_EXCEPTIONS)
  2889. self.create_testing_buildbot_json_file(
  2890. 'chromium.test.json', '\n' + COMPOSITION_WATERFALL_FILTERED_OUTPUT)
  2891. with self.assertRaises(generate_buildbot_json.BBGenErr):
  2892. fbb.check_output_file_consistency(verbose=True, dump=False)
  2893. joined_lines = ' '.join(fbb.printed_lines)
  2894. self.assertRegex(
  2895. joined_lines, 'File chromium.test.json did not have the following'
  2896. ' expected contents:.*')
  2897. self.assertRegex(joined_lines, '.*--- expected.*')
  2898. self.assertRegex(joined_lines, '.*\+\+\+ current.*')
  2899. fbb.printed_lines = []
  2900. self.assertFalse(fbb.printed_lines)
  2901. def test_android_output_options(self):
  2902. fbb = FakeBBGen(self.args, ANDROID_WATERFALL, FOO_TEST_SUITE, LUCI_MILO_CFG)
  2903. self.create_testing_buildbot_json_file('chromium.test.json',
  2904. ANDROID_WATERFALL_OUTPUT)
  2905. self.create_testing_buildbot_json_file('chromium.ci.json',
  2906. ANDROID_WATERFALL_OUTPUT)
  2907. fbb.check_output_file_consistency(verbose=True)
  2908. self.assertFalse(fbb.printed_lines)
  2909. def test_nonexistent_removal_raises(self):
  2910. fbb = FakeBBGen(self.args,
  2911. FOO_GTESTS_WATERFALL,
  2912. FOO_TEST_SUITE,
  2913. LUCI_MILO_CFG,
  2914. exceptions=NONEXISTENT_REMOVAL)
  2915. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2916. 'The following nonexistent machines.*'):
  2917. fbb.check_input_file_consistency(verbose=True)
  2918. self.assertFalse(fbb.printed_lines)
  2919. def test_nonexistent_modification_raises(self):
  2920. fbb = FakeBBGen(self.args,
  2921. FOO_GTESTS_WATERFALL,
  2922. FOO_TEST_SUITE,
  2923. LUCI_MILO_CFG,
  2924. exceptions=NONEXISTENT_MODIFICATION)
  2925. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  2926. 'The following nonexistent machines.*'):
  2927. fbb.check_input_file_consistency(verbose=True)
  2928. self.assertFalse(fbb.printed_lines)
  2929. def test_waterfall_args(self):
  2930. fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL,
  2931. GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
  2932. self.create_testing_buildbot_json_file(
  2933. 'chromium.test.json', COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT)
  2934. self.create_testing_buildbot_json_file(
  2935. 'chromium.ci.json', COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT)
  2936. fbb.check_output_file_consistency(verbose=True)
  2937. self.assertFalse(fbb.printed_lines)
  2938. def test_chromeos_trigger_script_output(self):
  2939. fbb = FakeBBGen(self.args, FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL,
  2940. FOO_TEST_SUITE, LUCI_MILO_CFG)
  2941. self.create_testing_buildbot_json_file('chromium.test.json',
  2942. CHROMEOS_TRIGGER_SCRIPT_OUTPUT)
  2943. self.create_testing_buildbot_json_file('chromium.ci.json',
  2944. CHROMEOS_TRIGGER_SCRIPT_OUTPUT)
  2945. fbb.check_output_file_consistency(verbose=True)
  2946. self.assertFalse(fbb.printed_lines)
  2947. def test_relative_pyl_file_dir(self):
  2948. self.override_args(pyl_files_dir='relative/path/', waterfall_filters=[])
  2949. fbb = FakeBBGen(self.args,
  2950. FOO_GTESTS_WATERFALL,
  2951. REUSING_TEST_WITH_DIFFERENT_NAME,
  2952. LUCI_MILO_CFG,
  2953. gn_isolate_map=GN_ISOLATE_MAP)
  2954. fbb.check_input_file_consistency(verbose=True)
  2955. self.create_testing_buildbot_json_file('relative/path/chromium.test.json',
  2956. VARIATION_GTEST_OUTPUT)
  2957. self.create_testing_buildbot_json_file('relative/path/chromium.ci.json',
  2958. VARIATION_GTEST_OUTPUT)
  2959. fbb.check_output_file_consistency(verbose=True)
  2960. self.assertFalse(fbb.printed_lines)
  2961. def test_nonexistent_bot_raises(self):
  2962. fbb = FakeBBGen(self.args, UNKNOWN_BOT_GTESTS_WATERFALL, FOO_TEST_SUITE,
  2963. LUCI_MILO_CFG)
  2964. with self.assertRaises(generate_buildbot_json.BBGenErr):
  2965. fbb.check_input_file_consistency(verbose=True)
  2966. self.assertFalse(fbb.printed_lines)
  2967. def test_nonexistent_bot_raises_when_no_project_pyl_exists(self):
  2968. fbb = FakeBBGen(self.args,
  2969. UNKNOWN_BOT_GTESTS_WATERFALL,
  2970. FOO_TEST_SUITE,
  2971. LUCI_MILO_CFG,
  2972. project_pyl=None)
  2973. with self.assertRaises(generate_buildbot_json.BBGenErr):
  2974. fbb.check_input_file_consistency(verbose=True)
  2975. self.assertFalse(fbb.printed_lines)
  2976. def test_nonexistent_bot_does_not_raise_when_validation_disabled(self):
  2977. fbb = FakeBBGen(
  2978. self.args,
  2979. UNKNOWN_BOT_GTESTS_WATERFALL,
  2980. FOO_TEST_SUITE,
  2981. LUCI_MILO_CFG,
  2982. project_pyl='{"validate_source_side_specs_have_builder": False}')
  2983. fbb.check_input_file_consistency(verbose=True)
  2984. def test_waterfalls_must_be_sorted(self):
  2985. fbb = FakeBBGen(self.args, TEST_SUITE_SORTED_WATERFALL, TEST_SUITE_SORTED,
  2986. LUCI_MILO_CFG_WATERFALL_SORTING)
  2987. fbb.check_input_file_consistency(verbose=True)
  2988. self.assertFalse(fbb.printed_lines)
  2989. fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_1,
  2990. TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
  2991. with self.assertRaisesRegex(
  2992. generate_buildbot_json.BBGenErr,
  2993. 'The following files have invalid keys: waterfalls.pyl'):
  2994. fbb.check_input_file_consistency(verbose=True)
  2995. joined_lines = '\n'.join(fbb.printed_lines)
  2996. self.assertRegex(joined_lines, '.*\+ chromium\..*test.*')
  2997. self.assertRegex(joined_lines, '.*\- chromium\..*test.*')
  2998. fbb.printed_lines = []
  2999. self.assertFalse(fbb.printed_lines)
  3000. fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_2,
  3001. TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
  3002. with self.assertRaisesRegex(
  3003. generate_buildbot_json.BBGenErr,
  3004. 'The following files have invalid keys: waterfalls.pyl'):
  3005. fbb.check_input_file_consistency(verbose=True)
  3006. joined_lines = ' '.join(fbb.printed_lines)
  3007. self.assertRegex(joined_lines, '.*\+.*Fake Tester.*')
  3008. self.assertRegex(joined_lines, '.*\-.*Fake Tester.*')
  3009. fbb.printed_lines = []
  3010. self.assertFalse(fbb.printed_lines)
  3011. def test_test_suite_exceptions_must_be_sorted(self):
  3012. fbb = FakeBBGen(self.args,
  3013. TEST_SUITE_SORTING_WATERFALL,
  3014. TEST_SUITE_SORTED,
  3015. LUCI_MILO_CFG,
  3016. exceptions=EXCEPTIONS_SORTED)
  3017. fbb.check_input_file_consistency(verbose=True)
  3018. self.assertFalse(fbb.printed_lines)
  3019. fbb = FakeBBGen(self.args,
  3020. TEST_SUITE_SORTING_WATERFALL,
  3021. TEST_SUITE_SORTED,
  3022. LUCI_MILO_CFG,
  3023. exceptions=EXCEPTIONS_DUPS_REMOVE_FROM)
  3024. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3025. fbb.check_input_file_consistency(verbose=True)
  3026. joined_lines = ' '.join(fbb.printed_lines)
  3027. self.assertRegex(joined_lines, '.*\- Fake Tester.*')
  3028. fbb.printed_lines = []
  3029. self.assertFalse(fbb.printed_lines)
  3030. def test_test_suite_exceptions_no_dups_remove_from(self):
  3031. fbb = FakeBBGen(self.args,
  3032. TEST_SUITE_SORTING_WATERFALL,
  3033. TEST_SUITE_SORTED,
  3034. LUCI_MILO_CFG,
  3035. exceptions=EXCEPTIONS_SORTED)
  3036. fbb.check_input_file_consistency(verbose=True)
  3037. self.assertFalse(fbb.printed_lines)
  3038. fbb = FakeBBGen(self.args,
  3039. TEST_SUITE_SORTING_WATERFALL,
  3040. TEST_SUITE_SORTED,
  3041. LUCI_MILO_CFG,
  3042. exceptions=EXCEPTIONS_PER_TEST_UNSORTED)
  3043. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3044. fbb.check_input_file_consistency(verbose=True)
  3045. joined_lines = ' '.join(fbb.printed_lines)
  3046. self.assertRegex(joined_lines, '.*\+ Fake Tester.*')
  3047. self.assertRegex(joined_lines, '.*\- Fake Tester.*')
  3048. fbb.printed_lines = []
  3049. self.assertFalse(fbb.printed_lines)
  3050. def test_test_suite_exceptions_per_test_must_be_sorted(self):
  3051. fbb = FakeBBGen(self.args,
  3052. TEST_SUITE_SORTING_WATERFALL,
  3053. TEST_SUITE_SORTED,
  3054. LUCI_MILO_CFG,
  3055. exceptions=EXCEPTIONS_SORTED)
  3056. fbb.check_input_file_consistency(verbose=True)
  3057. self.assertFalse(fbb.printed_lines)
  3058. fbb = FakeBBGen(self.args,
  3059. TEST_SUITE_SORTING_WATERFALL,
  3060. TEST_SUITE_SORTED,
  3061. LUCI_MILO_CFG,
  3062. exceptions=EXCEPTIONS_UNSORTED)
  3063. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3064. fbb.check_input_file_consistency(verbose=True)
  3065. joined_lines = ' '.join(fbb.printed_lines)
  3066. self.assertRegex(joined_lines, '.*\+ suite_.*')
  3067. self.assertRegex(joined_lines, '.*\- suite_.*')
  3068. fbb.printed_lines = []
  3069. self.assertFalse(fbb.printed_lines)
  3070. def test_test_suites_must_be_sorted(self):
  3071. fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, TEST_SUITE_SORTED,
  3072. LUCI_MILO_CFG)
  3073. fbb.check_input_file_consistency(verbose=True)
  3074. self.assertFalse(fbb.printed_lines)
  3075. for unsorted in (
  3076. TEST_SUITE_UNSORTED_1,
  3077. TEST_SUITE_UNSORTED_2,
  3078. TEST_SUITE_UNSORTED_3,
  3079. ):
  3080. fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, unsorted,
  3081. LUCI_MILO_CFG)
  3082. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3083. fbb.check_input_file_consistency(verbose=True)
  3084. joined_lines = ' '.join(fbb.printed_lines)
  3085. self.assertRegex(joined_lines, '.*\+ suite_.*')
  3086. self.assertRegex(joined_lines, '.*\- suite_.*')
  3087. fbb.printed_lines = []
  3088. self.assertFalse(fbb.printed_lines)
  3089. FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
  3090. [
  3091. {
  3092. 'mixins': ['waterfall_mixin'],
  3093. 'project': 'chromium',
  3094. 'bucket': 'ci',
  3095. 'name': 'chromium.test',
  3096. 'machines': {
  3097. 'Fake Tester': {
  3098. 'swarming': {},
  3099. 'test_suites': {
  3100. 'gtest_tests': 'foo_tests',
  3101. },
  3102. },
  3103. },
  3104. },
  3105. ]
  3106. """
  3107. FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
  3108. [
  3109. {
  3110. 'project': 'chromium',
  3111. 'bucket': 'ci',
  3112. 'name': 'chromium.test',
  3113. 'machines': {
  3114. 'Fake Tester': {
  3115. 'mixins': ['builder_mixin'],
  3116. 'swarming': {},
  3117. 'test_suites': {
  3118. 'gtest_tests': 'foo_tests',
  3119. },
  3120. },
  3121. },
  3122. },
  3123. ]
  3124. """
  3125. FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL = """\
  3126. [
  3127. {
  3128. 'project': 'chromium',
  3129. 'bucket': 'ci',
  3130. 'name': 'chromium.test',
  3131. 'machines': {
  3132. 'Fake Tester': {
  3133. 'os_type': 'linux',
  3134. 'mixins': ['builder_mixin'],
  3135. 'swarming': {},
  3136. 'test_suites': {
  3137. 'gtest_tests': 'foo_tests',
  3138. },
  3139. },
  3140. },
  3141. },
  3142. ]
  3143. """
  3144. FOO_GTESTS_DIMENSION_SETS_MIXIN_WATERFALL = """\
  3145. [
  3146. {
  3147. 'project': 'chromium',
  3148. 'bucket': 'ci',
  3149. 'name': 'chromium.test',
  3150. 'machines': {
  3151. 'Fake Tester': {
  3152. 'mixins': [
  3153. 'dimension_set_mixin_1',
  3154. 'dimension_set_mixin_2',
  3155. 'duplicate_dimension_set_mixin_1',
  3156. 'dimension_mixin',
  3157. ],
  3158. 'swarming': {},
  3159. 'test_suites': {
  3160. 'gtest_tests': 'foo_tests',
  3161. },
  3162. },
  3163. },
  3164. },
  3165. ]
  3166. """
  3167. FOO_GTESTS_WATERFALL_MIXIN_BUILDER_REMOVE_MIXIN_WATERFALL = """\
  3168. [
  3169. {
  3170. 'mixins': ['waterfall_mixin'],
  3171. 'project': 'chromium',
  3172. 'bucket': 'ci',
  3173. 'name': 'chromium.test',
  3174. 'machines': {
  3175. 'Fake Tester': {
  3176. 'remove_mixins': ['waterfall_mixin'],
  3177. 'swarming': {},
  3178. 'test_suites': {
  3179. 'gtest_tests': 'foo_tests',
  3180. },
  3181. },
  3182. },
  3183. },
  3184. ]
  3185. """
  3186. FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
  3187. [
  3188. {
  3189. 'project': 'chromium',
  3190. 'bucket': 'ci',
  3191. 'name': 'chromium.test',
  3192. 'machines': {
  3193. 'Fake Tester': {
  3194. 'mixins': ['random_mixin'],
  3195. 'swarming': {},
  3196. 'test_suites': {
  3197. 'gtest_tests': 'foo_tests',
  3198. },
  3199. },
  3200. },
  3201. },
  3202. ]
  3203. """
  3204. FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
  3205. [
  3206. {
  3207. 'project': 'chromium',
  3208. 'bucket': 'ci',
  3209. 'name': 'chromium.test',
  3210. 'machines': {
  3211. 'Fake Tester': {
  3212. 'mixins': ['dimension_mixin'],
  3213. 'swarming': {},
  3214. 'test_suites': {
  3215. 'gtest_tests': 'foo_tests',
  3216. },
  3217. },
  3218. },
  3219. },
  3220. ]
  3221. """
  3222. FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
  3223. [
  3224. {
  3225. 'project': 'chromium',
  3226. 'bucket': 'ci',
  3227. 'name': 'chromium.test',
  3228. 'machines': {
  3229. 'Fake Tester': {
  3230. 'mixins': ['dimension_mixin'],
  3231. 'os_type': 'win',
  3232. 'browser_config': 'release',
  3233. 'test_suites': {
  3234. 'gpu_telemetry_tests': 'foo_tests',
  3235. },
  3236. },
  3237. },
  3238. },
  3239. ]
  3240. """
  3241. # Swarming mixins must be a list, a single string is not allowed.
  3242. FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
  3243. [
  3244. {
  3245. 'project': 'chromium',
  3246. 'bucket': 'ci',
  3247. 'name': 'chromium.test',
  3248. 'machines': {
  3249. 'Fake Tester': {
  3250. 'mixins': 'dimension_mixin',
  3251. 'swarming': {},
  3252. 'test_suites': {
  3253. 'gtest_tests': 'foo_tests',
  3254. },
  3255. },
  3256. },
  3257. },
  3258. ]
  3259. """
  3260. FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
  3261. [
  3262. {
  3263. 'project': 'chromium',
  3264. 'bucket': 'ci',
  3265. 'name': 'chromium.test',
  3266. 'machines': {
  3267. 'Fake Tester': {
  3268. 'mixins': ['nonexistant'],
  3269. 'swarming': {},
  3270. 'test_suites': {
  3271. 'gtest_tests': 'foo_tests',
  3272. },
  3273. },
  3274. },
  3275. },
  3276. ]
  3277. """
  3278. FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
  3279. [
  3280. {
  3281. 'project': 'chromium',
  3282. 'bucket': 'ci',
  3283. 'name': 'chromium.test',
  3284. 'mixins': ['waterfall_mixin'],
  3285. 'machines': {
  3286. 'Fake Tester': {
  3287. 'swarming': {},
  3288. 'test_suites': {
  3289. 'gtest_tests': 'foo_tests',
  3290. },
  3291. },
  3292. },
  3293. },
  3294. ]
  3295. """
  3296. FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
  3297. [
  3298. {
  3299. 'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
  3300. 'project': 'chromium',
  3301. 'bucket': 'ci',
  3302. 'name': 'chromium.test',
  3303. 'machines': {
  3304. 'Fake Tester': {
  3305. 'swarming': {
  3306. 'dimension_sets': [
  3307. {
  3308. 'kvm': '1',
  3309. },
  3310. ],
  3311. },
  3312. 'test_suites': {
  3313. 'gtest_tests': 'foo_tests',
  3314. },
  3315. },
  3316. },
  3317. },
  3318. ]
  3319. """
  3320. FOO_TEST_SUITE_WITH_MIXIN = """\
  3321. {
  3322. 'basic_suites': {
  3323. 'foo_tests': {
  3324. 'foo_test': {
  3325. 'swarming': {
  3326. 'dimension_sets': [
  3327. {
  3328. 'integrity': 'high',
  3329. }
  3330. ],
  3331. 'expiration': 120,
  3332. },
  3333. 'mixins': ['test_mixin'],
  3334. },
  3335. },
  3336. },
  3337. }
  3338. """
  3339. # These mixins are invalid; if passed to check_input_file_consistency, they will
  3340. # fail. These are used for output file consistency checks.
  3341. SWARMING_MIXINS = """\
  3342. {
  3343. 'builder_mixin': {
  3344. 'swarming': {
  3345. 'value': 'builder',
  3346. },
  3347. },
  3348. 'dimension_mixin': {
  3349. 'swarming': {
  3350. 'dimensions': {
  3351. 'iama': 'mixin',
  3352. },
  3353. },
  3354. },
  3355. 'random_mixin': {
  3356. 'value': 'random',
  3357. },
  3358. 'test_mixin': {
  3359. 'swarming': {
  3360. 'value': 'test',
  3361. },
  3362. },
  3363. 'waterfall_mixin': {
  3364. 'swarming': {
  3365. 'value': 'waterfall',
  3366. },
  3367. },
  3368. }
  3369. """
  3370. SWARMING_MIXINS_APPEND = """\
  3371. {
  3372. 'builder_mixin': {
  3373. '$mixin_append': {
  3374. 'args': [ '--mixin-argument' ],
  3375. 'linux_args': [ '--linux-mixin-argument' ],
  3376. },
  3377. },
  3378. }
  3379. """
  3380. SWARMING_MIXINS_APPEND_NOT_LIST = """\
  3381. {
  3382. 'builder_mixin': {
  3383. '$mixin_append': {
  3384. 'args': 'I am not a list',
  3385. },
  3386. },
  3387. }
  3388. """
  3389. SWARMING_MIXINS_APPEND_TO_SWARMING = """\
  3390. {
  3391. 'builder_mixin': {
  3392. '$mixin_append': {
  3393. 'swarming': [ 'swarming!' ],
  3394. },
  3395. },
  3396. }
  3397. """
  3398. SWARMING_MIXINS_APPEND_NAMED_CACHES = """\
  3399. {
  3400. 'builder_mixin': {
  3401. '$mixin_append': {
  3402. 'swarming': {
  3403. 'named_caches': [
  3404. {
  3405. 'name': 'cache',
  3406. 'file': 'cache_file',
  3407. },
  3408. ]
  3409. },
  3410. },
  3411. },
  3412. }
  3413. """
  3414. SWARMING_MIXINS_APPEND_OTHER_KEYS_WITH_NAMED_CACHES = """\
  3415. {
  3416. 'builder_mixin': {
  3417. '$mixin_append': {
  3418. 'swarming': {
  3419. 'named_caches': [
  3420. {
  3421. 'name': 'cache',
  3422. 'file': 'cache_file',
  3423. },
  3424. ],
  3425. 'other_key': 'some value',
  3426. },
  3427. },
  3428. },
  3429. }
  3430. """
  3431. SWARMING_MIXINS_DIMENSION_SETS = """\
  3432. {
  3433. 'dimension_set_mixin_1': {
  3434. 'swarming': {
  3435. 'dimension_sets': [
  3436. {
  3437. 'value': 'ds1',
  3438. },
  3439. ],
  3440. },
  3441. },
  3442. 'dimension_set_mixin_2': {
  3443. 'swarming': {
  3444. 'dimension_sets': [
  3445. {
  3446. 'value': 'ds2',
  3447. },
  3448. ],
  3449. },
  3450. },
  3451. 'duplicate_dimension_set_mixin_1': {
  3452. 'swarming': {
  3453. 'dimension_sets': [
  3454. {
  3455. 'value': 'ds1',
  3456. },
  3457. ],
  3458. },
  3459. },
  3460. 'dimension_mixin': {
  3461. 'swarming': {
  3462. 'dimensions': {
  3463. 'other_value': 'dimension_mixin',
  3464. },
  3465. },
  3466. },
  3467. }
  3468. """
  3469. SWARMING_MIXINS_DUPLICATED = """\
  3470. {
  3471. 'builder_mixin': {
  3472. 'value': 'builder',
  3473. },
  3474. 'builder_mixin': {
  3475. 'value': 'builder',
  3476. },
  3477. }
  3478. """
  3479. SWARMING_MIXINS_UNSORTED = """\
  3480. {
  3481. 'b_mixin': {
  3482. 'b': 'b',
  3483. },
  3484. 'a_mixin': {
  3485. 'a': 'a',
  3486. },
  3487. 'c_mixin': {
  3488. 'c': 'c',
  3489. },
  3490. }
  3491. """
  3492. SWARMING_MIXINS_SORTED = """\
  3493. {
  3494. 'a_mixin': {
  3495. 'a': 'a',
  3496. },
  3497. 'b_mixin': {
  3498. 'b': 'b',
  3499. },
  3500. 'c_mixin': {
  3501. 'c': 'c',
  3502. },
  3503. }
  3504. """
  3505. WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT = """\
  3506. {
  3507. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3508. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3509. "Fake Tester": {
  3510. "gtest_tests": [
  3511. {
  3512. "merge": {
  3513. "args": [],
  3514. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3515. },
  3516. "swarming": {
  3517. "can_use_on_swarming_builders": true,
  3518. "dimension_sets": [
  3519. {
  3520. "other_value": "dimension_mixin",
  3521. "value": "ds1"
  3522. },
  3523. {
  3524. "other_value": "dimension_mixin",
  3525. "value": "ds2"
  3526. }
  3527. ]
  3528. },
  3529. "test": "foo_test"
  3530. }
  3531. ]
  3532. }
  3533. }
  3534. """
  3535. WATERFALL_MIXIN_WATERFALL_OUTPUT = """\
  3536. {
  3537. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3538. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3539. "Fake Tester": {
  3540. "gtest_tests": [
  3541. {
  3542. "merge": {
  3543. "args": [],
  3544. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3545. },
  3546. "swarming": {
  3547. "can_use_on_swarming_builders": true,
  3548. "dimension_sets": [
  3549. {
  3550. "integrity": "high"
  3551. }
  3552. ],
  3553. "expiration": 120,
  3554. "value": "waterfall"
  3555. },
  3556. "test": "foo_test"
  3557. }
  3558. ]
  3559. }
  3560. }
  3561. """
  3562. WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT = """\
  3563. {
  3564. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3565. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3566. "Fake Tester": {
  3567. "gtest_tests": [
  3568. {
  3569. "merge": {
  3570. "args": [],
  3571. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3572. },
  3573. "swarming": {
  3574. "can_use_on_swarming_builders": true,
  3575. "dimension_sets": [
  3576. {
  3577. "integrity": "high"
  3578. }
  3579. ],
  3580. "expiration": 120
  3581. },
  3582. "test": "foo_test"
  3583. }
  3584. ]
  3585. }
  3586. }
  3587. """
  3588. WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT = """\
  3589. {
  3590. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3591. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3592. "Fake Tester": {
  3593. "gtest_tests": [
  3594. {
  3595. "merge": {
  3596. "args": [],
  3597. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3598. },
  3599. "swarming": {
  3600. "can_use_on_swarming_builders": true,
  3601. "dimension_sets": [
  3602. {
  3603. "integrity": "high"
  3604. }
  3605. ],
  3606. "expiration": 120,
  3607. "value": "exception"
  3608. },
  3609. "test": "foo_test"
  3610. }
  3611. ]
  3612. }
  3613. }
  3614. """
  3615. BUILDER_MIXIN_WATERFALL_OUTPUT = """\
  3616. {
  3617. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3618. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3619. "Fake Tester": {
  3620. "gtest_tests": [
  3621. {
  3622. "merge": {
  3623. "args": [],
  3624. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3625. },
  3626. "swarming": {
  3627. "can_use_on_swarming_builders": true,
  3628. "dimension_sets": [
  3629. {
  3630. "integrity": "high"
  3631. }
  3632. ],
  3633. "expiration": 120,
  3634. "value": "builder"
  3635. },
  3636. "test": "foo_test"
  3637. }
  3638. ]
  3639. }
  3640. }
  3641. """
  3642. BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT = """\
  3643. {
  3644. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3645. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3646. "Fake Tester": {
  3647. "gtest_tests": [
  3648. {
  3649. "merge": {
  3650. "args": [],
  3651. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3652. },
  3653. "swarming": {
  3654. "can_use_on_swarming_builders": true,
  3655. "dimension_sets": [
  3656. {
  3657. "integrity": "high"
  3658. }
  3659. ],
  3660. "expiration": 120
  3661. },
  3662. "test": "foo_test",
  3663. "value": "random"
  3664. }
  3665. ]
  3666. }
  3667. }
  3668. """
  3669. BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT = """\
  3670. {
  3671. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3672. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3673. "Fake Tester": {
  3674. "gtest_tests": [
  3675. {
  3676. "args": [
  3677. "--c_arg",
  3678. "--mixin-argument"
  3679. ],
  3680. "merge": {
  3681. "args": [],
  3682. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3683. },
  3684. "swarming": {
  3685. "can_use_on_swarming_builders": true
  3686. },
  3687. "test": "foo_test"
  3688. }
  3689. ]
  3690. }
  3691. }
  3692. """
  3693. BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT = """\
  3694. {
  3695. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3696. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3697. "Fake Tester": {
  3698. "gtest_tests": [
  3699. {
  3700. "args": [
  3701. "--c_arg",
  3702. "--mixin-argument",
  3703. "--linux-mixin-argument"
  3704. ],
  3705. "merge": {
  3706. "args": [],
  3707. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3708. },
  3709. "swarming": {
  3710. "can_use_on_swarming_builders": true
  3711. },
  3712. "test": "foo_test"
  3713. }
  3714. ]
  3715. }
  3716. }
  3717. """
  3718. BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT = """\
  3719. {
  3720. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3721. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3722. "Fake Tester": {
  3723. "gtest_tests": [
  3724. {
  3725. "merge": {
  3726. "args": [],
  3727. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3728. },
  3729. "swarming": {
  3730. "can_use_on_swarming_builders": true,
  3731. "named_caches": [
  3732. {
  3733. "file": "cache_in_test_file",
  3734. "name": "cache_in_test"
  3735. },
  3736. {
  3737. "file": "cache_file",
  3738. "name": "cache"
  3739. }
  3740. ]
  3741. },
  3742. "test": "foo_test"
  3743. }
  3744. ]
  3745. }
  3746. }
  3747. """
  3748. TEST_MIXIN_WATERFALL_OUTPUT = """\
  3749. {
  3750. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3751. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3752. "Fake Tester": {
  3753. "gtest_tests": [
  3754. {
  3755. "merge": {
  3756. "args": [],
  3757. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3758. },
  3759. "swarming": {
  3760. "can_use_on_swarming_builders": true,
  3761. "dimension_sets": [
  3762. {
  3763. "integrity": "high",
  3764. "kvm": "1"
  3765. }
  3766. ],
  3767. "expiration": 120,
  3768. "value": "test"
  3769. },
  3770. "test": "foo_test"
  3771. }
  3772. ]
  3773. }
  3774. }
  3775. """
  3776. DIMENSIONS_MIXIN_WATERFALL_OUTPUT = """\
  3777. {
  3778. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  3779. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  3780. "Fake Tester": {
  3781. "gtest_tests": [
  3782. {
  3783. "merge": {
  3784. "args": [],
  3785. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  3786. },
  3787. "swarming": {
  3788. "can_use_on_swarming_builders": true,
  3789. "dimension_sets": [
  3790. {
  3791. "iama": "mixin",
  3792. "integrity": "high"
  3793. }
  3794. ],
  3795. "expiration": 120,
  3796. "value": "test"
  3797. },
  3798. "test": "foo_test"
  3799. }
  3800. ]
  3801. }
  3802. }
  3803. """
  3804. class MixinTests(TestCase):
  3805. """Tests for the mixins feature."""
  3806. def test_mixins_must_be_sorted(self):
  3807. fbb = FakeBBGen(self.args,
  3808. FOO_GTESTS_SORTING_MIXINS_WATERFALL,
  3809. FOO_TEST_SUITE,
  3810. LUCI_MILO_CFG,
  3811. mixins=SWARMING_MIXINS_SORTED)
  3812. fbb.check_input_file_consistency(verbose=True)
  3813. self.assertFalse(fbb.printed_lines)
  3814. fbb = FakeBBGen(self.args,
  3815. FOO_GTESTS_SORTING_MIXINS_WATERFALL,
  3816. FOO_TEST_SUITE,
  3817. LUCI_MILO_CFG,
  3818. mixins=SWARMING_MIXINS_UNSORTED)
  3819. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3820. fbb.check_input_file_consistency(verbose=True)
  3821. joined_lines = '\n'.join(fbb.printed_lines)
  3822. self.assertRegex(joined_lines, '.*\+ ._mixin.*')
  3823. self.assertRegex(joined_lines, '.*\- ._mixin.*')
  3824. fbb.printed_lines = []
  3825. self.assertFalse(fbb.printed_lines)
  3826. def test_waterfall(self):
  3827. fbb = FakeBBGen(self.args,
  3828. FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
  3829. FOO_TEST_SUITE,
  3830. LUCI_MILO_CFG,
  3831. mixins=SWARMING_MIXINS)
  3832. self.create_testing_buildbot_json_file('chromium.test.json',
  3833. WATERFALL_MIXIN_WATERFALL_OUTPUT)
  3834. self.create_testing_buildbot_json_file('chromium.ci.json',
  3835. WATERFALL_MIXIN_WATERFALL_OUTPUT)
  3836. fbb.check_output_file_consistency(verbose=True)
  3837. self.assertFalse(fbb.printed_lines)
  3838. def test_waterfall_exception_overrides(self):
  3839. fbb = FakeBBGen(self.args,
  3840. FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
  3841. FOO_TEST_SUITE,
  3842. LUCI_MILO_CFG,
  3843. exceptions=SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
  3844. mixins=SWARMING_MIXINS)
  3845. self.create_testing_buildbot_json_file(
  3846. 'chromium.test.json', WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT)
  3847. self.create_testing_buildbot_json_file(
  3848. 'chromium.ci.json', WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT)
  3849. fbb.check_output_file_consistency(verbose=True)
  3850. self.assertFalse(fbb.printed_lines)
  3851. def test_builder(self):
  3852. fbb = FakeBBGen(self.args,
  3853. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  3854. FOO_TEST_SUITE,
  3855. LUCI_MILO_CFG,
  3856. mixins=SWARMING_MIXINS)
  3857. self.create_testing_buildbot_json_file('chromium.test.json',
  3858. BUILDER_MIXIN_WATERFALL_OUTPUT)
  3859. self.create_testing_buildbot_json_file('chromium.ci.json',
  3860. BUILDER_MIXIN_WATERFALL_OUTPUT)
  3861. fbb.check_output_file_consistency(verbose=True)
  3862. self.assertFalse(fbb.printed_lines)
  3863. def test_builder_non_swarming(self):
  3864. fbb = FakeBBGen(self.args,
  3865. FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
  3866. FOO_TEST_SUITE,
  3867. LUCI_MILO_CFG,
  3868. mixins=SWARMING_MIXINS)
  3869. self.create_testing_buildbot_json_file(
  3870. 'chromium.test.json', BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
  3871. self.create_testing_buildbot_json_file(
  3872. 'chromium.ci.json', BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
  3873. fbb.check_output_file_consistency(verbose=True)
  3874. self.assertFalse(fbb.printed_lines)
  3875. def test_test_suite(self):
  3876. fbb = FakeBBGen(self.args,
  3877. FOO_GTESTS_WATERFALL,
  3878. FOO_TEST_SUITE_WITH_MIXIN,
  3879. LUCI_MILO_CFG,
  3880. mixins=SWARMING_MIXINS)
  3881. self.create_testing_buildbot_json_file('chromium.test.json',
  3882. TEST_MIXIN_WATERFALL_OUTPUT)
  3883. self.create_testing_buildbot_json_file('chromium.ci.json',
  3884. TEST_MIXIN_WATERFALL_OUTPUT)
  3885. fbb.check_output_file_consistency(verbose=True)
  3886. self.assertFalse(fbb.printed_lines)
  3887. def test_dimension(self):
  3888. fbb = FakeBBGen(self.args,
  3889. FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
  3890. FOO_TEST_SUITE_WITH_MIXIN,
  3891. LUCI_MILO_CFG,
  3892. mixins=SWARMING_MIXINS)
  3893. self.create_testing_buildbot_json_file('chromium.test.json',
  3894. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3895. self.create_testing_buildbot_json_file('chromium.ci.json',
  3896. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3897. fbb.check_output_file_consistency(verbose=True)
  3898. self.assertFalse(fbb.printed_lines)
  3899. def test_dimension_gpu(self):
  3900. fbb = FakeBBGen(self.args,
  3901. FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
  3902. FOO_TEST_SUITE_WITH_MIXIN,
  3903. LUCI_MILO_CFG,
  3904. mixins=SWARMING_MIXINS,
  3905. gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
  3906. self.create_testing_buildbot_json_file('chromium.test.json',
  3907. GPU_DIMENSIONS_WATERFALL_OUTPUT)
  3908. self.create_testing_buildbot_json_file('chromium.ci.json',
  3909. GPU_DIMENSIONS_WATERFALL_OUTPUT)
  3910. fbb.check_output_file_consistency(verbose=True)
  3911. self.assertFalse(fbb.printed_lines)
  3912. def test_unreferenced(self):
  3913. fbb = FakeBBGen(self.args,
  3914. FOO_GTESTS_WATERFALL,
  3915. FOO_TEST_SUITE_WITH_MIXIN,
  3916. LUCI_MILO_CFG,
  3917. mixins=SWARMING_MIXINS)
  3918. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  3919. '.*mixins are unreferenced.*'):
  3920. fbb.check_input_file_consistency(verbose=True)
  3921. self.assertFalse(fbb.printed_lines)
  3922. def test_unused(self):
  3923. fbb = FakeBBGen(self.args,
  3924. FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
  3925. FOO_TEST_SUITE_WITH_MIXIN,
  3926. LUCI_MILO_CFG,
  3927. mixins=SWARMING_MIXINS)
  3928. self.create_testing_buildbot_json_file('chromium.test.json',
  3929. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3930. self.create_testing_buildbot_json_file('chromium.ci.json',
  3931. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3932. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3933. fbb.check_output_file_consistency(verbose=True)
  3934. self.assertFalse(fbb.printed_lines)
  3935. def test_list(self):
  3936. fbb = FakeBBGen(self.args,
  3937. FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
  3938. FOO_TEST_SUITE_WITH_MIXIN,
  3939. LUCI_MILO_CFG,
  3940. mixins=SWARMING_MIXINS)
  3941. self.create_testing_buildbot_json_file('chromium.test.json',
  3942. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3943. self.create_testing_buildbot_json_file('chromium.ci.json',
  3944. DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
  3945. with self.assertRaises(generate_buildbot_json.BBGenErr):
  3946. fbb.check_output_file_consistency(verbose=True)
  3947. self.assertFalse(fbb.printed_lines)
  3948. def test_no_duplicate_keys(self):
  3949. fbb = FakeBBGen(self.args,
  3950. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  3951. FOO_TEST_SUITE,
  3952. LUCI_MILO_CFG,
  3953. mixins=SWARMING_MIXINS_DUPLICATED)
  3954. with self.assertRaisesRegex(
  3955. generate_buildbot_json.BBGenErr,
  3956. 'The following files have invalid keys: mixins.pyl'):
  3957. fbb.check_input_file_consistency(verbose=True)
  3958. joined_lines = '\n'.join(fbb.printed_lines)
  3959. self.assertRegex(joined_lines, '.*\- builder_mixin')
  3960. fbb.printed_lines = []
  3961. self.assertFalse(fbb.printed_lines)
  3962. def test_no_duplicate_keys_basic_test_suite(self):
  3963. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE_NOT_SORTED,
  3964. LUCI_MILO_CFG)
  3965. with self.assertRaisesRegex(
  3966. generate_buildbot_json.BBGenErr,
  3967. 'The following files have invalid keys: test_suites.pyl'):
  3968. fbb.check_input_file_consistency(verbose=True)
  3969. joined_lines = '\n'.join(fbb.printed_lines)
  3970. self.assertRegex(joined_lines, '.*\- a_test')
  3971. self.assertRegex(joined_lines, '.*\+ a_test')
  3972. fbb.printed_lines = []
  3973. self.assertFalse(fbb.printed_lines)
  3974. def test_type_assert_printing_help(self):
  3975. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, TEST_SUITES_SYNTAX_ERROR,
  3976. LUCI_MILO_CFG)
  3977. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  3978. 'Invalid \.pyl file \'test_suites.pyl\'.*'):
  3979. fbb.check_input_file_consistency(verbose=True)
  3980. self.assertEqual(fbb.printed_lines, [
  3981. '== test_suites.pyl ==',
  3982. '<snip>',
  3983. '1 {',
  3984. "2 'basic_suites': {",
  3985. '--------------------------------------------------------------------'
  3986. '------------',
  3987. '3 3: {',
  3988. '-------^------------------------------------------------------------'
  3989. '------------',
  3990. "4 'suite_c': {},",
  3991. '5 },',
  3992. '<snip>',
  3993. ])
  3994. def test_mixin_append_args(self):
  3995. fbb = FakeBBGen(self.args,
  3996. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  3997. FOO_TEST_SUITE_WITH_ARGS,
  3998. LUCI_MILO_CFG,
  3999. mixins=SWARMING_MIXINS_APPEND)
  4000. self.create_testing_buildbot_json_file(
  4001. 'chromium.test.json', BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT)
  4002. self.create_testing_buildbot_json_file(
  4003. 'chromium.ci.json', BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT)
  4004. fbb.check_output_file_consistency(verbose=True)
  4005. self.assertFalse(fbb.printed_lines)
  4006. def test_mixin_append_linux_args(self):
  4007. fbb = FakeBBGen(self.args,
  4008. FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL,
  4009. FOO_TEST_SUITE_WITH_ARGS,
  4010. LUCI_MILO_CFG,
  4011. mixins=SWARMING_MIXINS_APPEND)
  4012. self.create_testing_buildbot_json_file(
  4013. 'chromium.test.json', BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT)
  4014. self.create_testing_buildbot_json_file(
  4015. 'chromium.ci.json', BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT)
  4016. fbb.check_output_file_consistency(verbose=True)
  4017. self.assertFalse(fbb.printed_lines)
  4018. def test_mixin_append_swarming_named_caches(self):
  4019. fbb = FakeBBGen(self.args,
  4020. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  4021. FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES,
  4022. LUCI_MILO_CFG,
  4023. mixins=SWARMING_MIXINS_APPEND_NAMED_CACHES)
  4024. self.create_testing_buildbot_json_file(
  4025. 'chromium.test.json',
  4026. BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT)
  4027. self.create_testing_buildbot_json_file(
  4028. 'chromium.ci.json', BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT)
  4029. fbb.check_output_file_consistency(verbose=True)
  4030. self.assertFalse(fbb.printed_lines)
  4031. def test_mixin_append_swarming_error(self):
  4032. fbb = FakeBBGen(self.args,
  4033. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  4034. FOO_TEST_SUITE_WITH_ARGS,
  4035. LUCI_MILO_CFG,
  4036. mixins=SWARMING_MIXINS_APPEND_OTHER_KEYS_WITH_NAMED_CACHES)
  4037. with self.assertRaisesRegex(
  4038. generate_buildbot_json.BBGenErr,
  4039. 'Only named_caches is supported under swarming key in '
  4040. '\$mixin_append, but there are: \[\'named_caches\', \'other_key\'\]'):
  4041. fbb.check_output_file_consistency(verbose=True)
  4042. self.assertFalse(fbb.printed_lines)
  4043. def test_mixin_append_mixin_field_not_list(self):
  4044. fbb = FakeBBGen(self.args,
  4045. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  4046. FOO_TEST_SUITE_WITH_ARGS,
  4047. LUCI_MILO_CFG,
  4048. mixins=SWARMING_MIXINS_APPEND_NOT_LIST)
  4049. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  4050. 'Key "args" in \$mixin_append must be a list.'):
  4051. fbb.check_output_file_consistency(verbose=True)
  4052. self.assertFalse(fbb.printed_lines)
  4053. def test_mixin_append_test_field_not_list(self):
  4054. fbb = FakeBBGen(self.args,
  4055. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  4056. FOO_TEST_SUITE,
  4057. LUCI_MILO_CFG,
  4058. mixins=SWARMING_MIXINS_APPEND_TO_SWARMING)
  4059. with self.assertRaisesRegex(
  4060. generate_buildbot_json.BBGenErr,
  4061. 'Cannot apply \$mixin_append to non-list "swarming".'):
  4062. fbb.check_output_file_consistency(verbose=True)
  4063. self.assertFalse(fbb.printed_lines)
  4064. def test_remove_mixin_builder_remove_waterfall(self):
  4065. fbb = FakeBBGen(self.args,
  4066. FOO_GTESTS_WATERFALL_MIXIN_BUILDER_REMOVE_MIXIN_WATERFALL,
  4067. FOO_TEST_SUITE,
  4068. LUCI_MILO_CFG,
  4069. mixins=SWARMING_MIXINS)
  4070. self.create_testing_buildbot_json_file(
  4071. 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4072. self.create_testing_buildbot_json_file(
  4073. 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4074. fbb.check_output_file_consistency(verbose=True)
  4075. self.assertFalse(fbb.printed_lines)
  4076. def test_remove_mixin_test_remove_waterfall(self):
  4077. fbb = FakeBBGen(self.args,
  4078. FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
  4079. FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN,
  4080. LUCI_MILO_CFG,
  4081. mixins=SWARMING_MIXINS)
  4082. self.create_testing_buildbot_json_file(
  4083. 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4084. self.create_testing_buildbot_json_file(
  4085. 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4086. fbb.check_output_file_consistency(verbose=True)
  4087. self.assertFalse(fbb.printed_lines)
  4088. def test_remove_mixin_test_remove_builder(self):
  4089. fbb = FakeBBGen(self.args,
  4090. FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
  4091. FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN,
  4092. LUCI_MILO_CFG,
  4093. mixins=SWARMING_MIXINS)
  4094. self.create_testing_buildbot_json_file(
  4095. 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4096. self.create_testing_buildbot_json_file(
  4097. 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
  4098. fbb.check_output_file_consistency(verbose=True)
  4099. self.assertFalse(fbb.printed_lines)
  4100. def test_dimension_sets_application(self):
  4101. fbb = FakeBBGen(self.args,
  4102. FOO_GTESTS_DIMENSION_SETS_MIXIN_WATERFALL,
  4103. FOO_TEST_SUITE_NO_DIMENSIONS,
  4104. LUCI_MILO_CFG,
  4105. mixins=SWARMING_MIXINS_DIMENSION_SETS)
  4106. self.create_testing_buildbot_json_file(
  4107. 'chromium.test.json', WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT)
  4108. self.create_testing_buildbot_json_file(
  4109. 'chromium.ci.json', WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT)
  4110. fbb.check_output_file_consistency(verbose=True)
  4111. self.assertFalse(fbb.printed_lines)
  4112. TEST_SUITE_WITH_PARAMS = """\
  4113. {
  4114. 'basic_suites': {
  4115. 'bar_tests': {
  4116. 'bar_test': {
  4117. 'args': ['--no-xvfb'],
  4118. 'swarming': {
  4119. 'dimension_sets': [
  4120. {
  4121. 'device_os': 'NMF26U'
  4122. }
  4123. ],
  4124. },
  4125. 'should_retry_with_patch': False,
  4126. 'name': 'bar_test'
  4127. },
  4128. 'bar_test_test': {
  4129. 'swarming': {
  4130. 'dimension_sets': [
  4131. {
  4132. 'kvm': '1'
  4133. }
  4134. ],
  4135. 'hard_timeout': 1000
  4136. },
  4137. 'should_retry_with_patch': True
  4138. }
  4139. },
  4140. 'foo_tests': {
  4141. 'foo_test_empty': {},
  4142. 'foo_test': {
  4143. 'args': [
  4144. '--jobs=1',
  4145. '--verbose'
  4146. ],
  4147. 'swarming': {
  4148. 'dimension_sets': [
  4149. {
  4150. 'device_os': 'MMB29Q'
  4151. }
  4152. ],
  4153. 'hard_timeout': 1800
  4154. }
  4155. },
  4156. 'foo_test_test': {
  4157. 'swarming': {
  4158. },
  4159. 'name': 'pls'
  4160. },
  4161. },
  4162. },
  4163. 'compound_suites': {
  4164. 'composition_tests': [
  4165. 'foo_tests',
  4166. 'bar_tests',
  4167. ],
  4168. },
  4169. }
  4170. """
  4171. TEST_QUERY_BOTS_OUTPUT = {
  4172. "Fake Android M Tester": {
  4173. "gtest_tests": [
  4174. {
  4175. "test": "foo_test",
  4176. "merge": {
  4177. "args": [],
  4178. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4179. },
  4180. "swarming": {
  4181. "can_use_on_swarming_builders": False
  4182. }
  4183. }
  4184. ]
  4185. },
  4186. "Fake Android L Tester": {
  4187. "gtest_tests": [
  4188. {
  4189. "test": "foo_test",
  4190. "args": [
  4191. "--gs-results-bucket=chromium-result-details",
  4192. "--recover-devices"
  4193. ],
  4194. "merge": {
  4195. "args": [],
  4196. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4197. },
  4198. "swarming": {
  4199. "cipd_packages": [
  4200. {
  4201. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4202. "location": "bin",
  4203. "revision":
  4204. "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4205. }
  4206. ],
  4207. "dimension_sets":[
  4208. {
  4209. "device_os": "LMY41U",
  4210. "device_os_type": "user",
  4211. "device_type": "hammerhead",
  4212. 'os': 'Android'
  4213. }
  4214. ],
  4215. "can_use_on_swarming_builders": True
  4216. }
  4217. }
  4218. ]
  4219. },
  4220. "Fake Android K Tester": {
  4221. "additional_compile_targets": ["bar_test"],
  4222. "gtest_tests": [
  4223. {
  4224. "test": "foo_test",
  4225. "args": [
  4226. "--gs-results-bucket=chromium-result-details",
  4227. "--recover-devices"
  4228. ],
  4229. "merge": {
  4230. "args": [],
  4231. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4232. },
  4233. "swarming": {
  4234. "cipd_packages": [
  4235. {
  4236. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4237. "location": "bin",
  4238. "revision":
  4239. "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4240. }
  4241. ],
  4242. "dimension_sets": [
  4243. {
  4244. "device_os": "KTU84P",
  4245. "device_os_type": "userdebug",
  4246. "device_type": "hammerhead",
  4247. "os": "Android",
  4248. }
  4249. ],
  4250. "can_use_on_swarming_builders": True,
  4251. "output_links": [
  4252. {
  4253. "link": ["https://luci-logdog.appspot.com/v/?s",
  4254. "=android%2Fswarming%2Flogcats%2F",
  4255. "${TASK_ID}%2F%2B%2Funified_logcats"],
  4256. "name": "shard #${SHARD_INDEX} logcats"
  4257. }
  4258. ]
  4259. }
  4260. }
  4261. ]
  4262. },
  4263. "Android Builder": {
  4264. "additional_compile_targets": ["bar_test"]
  4265. }
  4266. }
  4267. TEST_QUERY_BOTS_TESTS_OUTPUT = {
  4268. "Fake Android M Tester": [
  4269. {
  4270. "merge": {
  4271. "args": [],
  4272. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4273. },
  4274. "test": "foo_test",
  4275. "swarming": {
  4276. "can_use_on_swarming_builders": False
  4277. }
  4278. }
  4279. ],
  4280. "Fake Android L Tester": [
  4281. {
  4282. "test": "foo_test",
  4283. "args": [
  4284. "--gs-results-bucket=chromium-result-details",
  4285. "--recover-devices"
  4286. ],
  4287. "merge": {
  4288. "args": [],
  4289. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4290. },
  4291. "swarming": {
  4292. "cipd_packages": [
  4293. {
  4294. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4295. "location": "bin",
  4296. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4297. }
  4298. ],
  4299. "dimension_sets": [
  4300. {
  4301. "device_os": "LMY41U",
  4302. "device_os_type": "user",
  4303. "device_type": "hammerhead",
  4304. "os": "Android"
  4305. }
  4306. ],
  4307. "can_use_on_swarming_builders": True
  4308. }
  4309. }
  4310. ],
  4311. "Android Builder": [],
  4312. "Fake Android K Tester": [
  4313. {
  4314. "test": "foo_test",
  4315. "args": [
  4316. "--gs-results-bucket=chromium-result-details",
  4317. "--recover-devices"
  4318. ],
  4319. "merge": {
  4320. "args": [],
  4321. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4322. },
  4323. "swarming": {
  4324. "cipd_packages": [
  4325. {
  4326. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4327. "location": "bin",
  4328. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4329. }
  4330. ],
  4331. "dimension_sets": [
  4332. {
  4333. "device_os": "KTU84P",
  4334. "device_os_type": "userdebug",
  4335. "device_type": "hammerhead",
  4336. "os": "Android"
  4337. }
  4338. ],
  4339. "can_use_on_swarming_builders": True,
  4340. "output_links": [
  4341. {
  4342. "link": [
  4343. "https://luci-logdog.appspot.com/v/?s",
  4344. "=android%2Fswarming%2Flogcats%2F",
  4345. "${TASK_ID}%2F%2B%2Funified_logcats"
  4346. ],
  4347. "name": "shard #${SHARD_INDEX} logcats"
  4348. }
  4349. ]
  4350. }
  4351. }
  4352. ]
  4353. }
  4354. TEST_QUERY_BOT_OUTPUT = {
  4355. "additional_compile_targets": ["bar_test"],
  4356. "gtest_tests": [
  4357. {
  4358. "test": "foo_test",
  4359. "args": [
  4360. "--gs-results-bucket=chromium-result-details",
  4361. "--recover-devices"
  4362. ],
  4363. "merge": {
  4364. "args": [],
  4365. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4366. },
  4367. "swarming": {
  4368. "cipd_packages": [
  4369. {
  4370. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4371. "location": "bin",
  4372. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4373. }
  4374. ],
  4375. "dimension_sets": [
  4376. {
  4377. "device_os": "KTU84P",
  4378. "device_os_type": "userdebug",
  4379. "device_type": "hammerhead",
  4380. "os": "Android"
  4381. }
  4382. ],
  4383. "can_use_on_swarming_builders": True,
  4384. "output_links": [
  4385. {
  4386. "link": ["https://luci-logdog.appspot.com/v/?s",
  4387. "=android%2Fswarming%2Flogcats%2F",
  4388. "${TASK_ID}%2F%2B%2Funified_logcats"
  4389. ],
  4390. "name": "shard #${SHARD_INDEX} logcats"
  4391. }
  4392. ]
  4393. }
  4394. }
  4395. ]
  4396. }
  4397. TEST_QUERY_BOT_TESTS_OUTPUT = [
  4398. {
  4399. "test": "foo_test",
  4400. "args": [
  4401. "--gs-results-bucket=chromium-result-details",
  4402. "--recover-devices"
  4403. ],
  4404. "merge": {
  4405. "args": [],
  4406. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4407. },
  4408. "swarming": {
  4409. "cipd_packages": [
  4410. {
  4411. "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
  4412. "location": "bin",
  4413. "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
  4414. }
  4415. ],
  4416. "dimension_sets": [
  4417. {
  4418. "device_os": "LMY41U",
  4419. "device_os_type": "user",
  4420. "device_type": "hammerhead",
  4421. "os": "Android"
  4422. }
  4423. ],
  4424. "can_use_on_swarming_builders": True
  4425. }
  4426. }
  4427. ]
  4428. TEST_QUERY_TESTS_OUTPUT = {
  4429. "bar_test": {},
  4430. "foo_test": {}
  4431. }
  4432. TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT = ["foo_test"]
  4433. TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT = ["bar_test"]
  4434. TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT = ["bar_test_test"]
  4435. TEST_QUERY_TESTS_PARAMS_OUTPUT = ['bar_test_test']
  4436. TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT = ['bar_test']
  4437. TEST_QUERY_TEST_OUTPUT = {}
  4438. TEST_QUERY_TEST_BOTS_OUTPUT = [
  4439. "Fake Android K Tester",
  4440. "Fake Android L Tester",
  4441. "Fake Android M Tester",
  4442. ]
  4443. TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT = ['Fake Tester']
  4444. TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT = []
  4445. class QueryTests(TestCase):
  4446. """Tests for the query feature."""
  4447. def test_query_bots(self):
  4448. self.override_args(query='bots',
  4449. check=False,
  4450. pyl_files_dir=None,
  4451. json=None,
  4452. waterfall_filters=[])
  4453. fbb = FakeBBGen(self.args,
  4454. ANDROID_WATERFALL,
  4455. GOOD_COMPOSITION_TEST_SUITES,
  4456. LUCI_MILO_CFG,
  4457. mixins=SWARMING_MIXINS_SORTED)
  4458. fbb.query(fbb.args)
  4459. query_json = json.loads("".join(fbb.printed_lines))
  4460. self.assertEqual(query_json, TEST_QUERY_BOTS_OUTPUT)
  4461. def test_query_bots_invalid(self):
  4462. self.override_args(query='bots/blah/blah',
  4463. check=False,
  4464. pyl_files_dir=None,
  4465. json=None,
  4466. waterfall_filters=[])
  4467. fbb = FakeBBGen(self.args,
  4468. ANDROID_WATERFALL,
  4469. GOOD_COMPOSITION_TEST_SUITES,
  4470. LUCI_MILO_CFG,
  4471. mixins=SWARMING_MIXINS_SORTED)
  4472. with self.assertRaises(SystemExit) as cm:
  4473. fbb.query(fbb.args)
  4474. self.assertEqual(cm.exception.code, 1)
  4475. self.assertTrue(fbb.printed_lines)
  4476. def test_query_bots_json(self):
  4477. self.override_args(query='bots',
  4478. check=False,
  4479. pyl_files_dir=None,
  4480. json='result.json',
  4481. waterfall_filters=[])
  4482. fbb = FakeBBGen(self.args,
  4483. ANDROID_WATERFALL,
  4484. GOOD_COMPOSITION_TEST_SUITES,
  4485. LUCI_MILO_CFG,
  4486. mixins=SWARMING_MIXINS_SORTED)
  4487. fbb.query(fbb.args)
  4488. self.assertFalse(fbb.printed_lines)
  4489. def test_query_bots_tests(self):
  4490. self.override_args(query='bots/tests',
  4491. check=False,
  4492. pyl_files_dir=None,
  4493. json=None,
  4494. waterfall_filters=[])
  4495. fbb = FakeBBGen(self.args,
  4496. ANDROID_WATERFALL,
  4497. GOOD_COMPOSITION_TEST_SUITES,
  4498. LUCI_MILO_CFG,
  4499. mixins=SWARMING_MIXINS_SORTED)
  4500. fbb.query(fbb.args)
  4501. query_json = json.loads("".join(fbb.printed_lines))
  4502. self.assertEqual(query_json, TEST_QUERY_BOTS_TESTS_OUTPUT)
  4503. def test_query_invalid_bots_tests(self):
  4504. self.override_args(query='bots/tdfjdk',
  4505. check=False,
  4506. pyl_files_dir=None,
  4507. json=None,
  4508. waterfall_filters=[])
  4509. fbb = FakeBBGen(self.args,
  4510. ANDROID_WATERFALL,
  4511. GOOD_COMPOSITION_TEST_SUITES,
  4512. LUCI_MILO_CFG,
  4513. mixins=SWARMING_MIXINS_SORTED)
  4514. with self.assertRaises(SystemExit) as cm:
  4515. fbb.query(fbb.args)
  4516. self.assertEqual(cm.exception.code, 1)
  4517. self.assertTrue(fbb.printed_lines)
  4518. def test_query_bot(self):
  4519. self.override_args(query='bot/Fake Android K Tester',
  4520. check=False,
  4521. pyl_files_dir=None,
  4522. json=None,
  4523. waterfall_filters=[])
  4524. fbb = FakeBBGen(self.args,
  4525. ANDROID_WATERFALL,
  4526. GOOD_COMPOSITION_TEST_SUITES,
  4527. LUCI_MILO_CFG,
  4528. mixins=SWARMING_MIXINS_SORTED)
  4529. fbb.query(fbb.args)
  4530. query_json = json.loads("".join(fbb.printed_lines))
  4531. self.maxDiff = None # pragma pylint: disable=attribute-defined-outside-init
  4532. self.assertEqual(query_json, TEST_QUERY_BOT_OUTPUT)
  4533. def test_query_bot_invalid_id(self):
  4534. self.override_args(query='bot/bot1',
  4535. check=False,
  4536. pyl_files_dir=None,
  4537. json=None,
  4538. waterfall_filters=[])
  4539. fbb = FakeBBGen(self.args,
  4540. ANDROID_WATERFALL,
  4541. GOOD_COMPOSITION_TEST_SUITES,
  4542. LUCI_MILO_CFG,
  4543. mixins=SWARMING_MIXINS_SORTED)
  4544. with self.assertRaises(SystemExit) as cm:
  4545. fbb.query(fbb.args)
  4546. self.assertEqual(cm.exception.code, 1)
  4547. self.assertTrue(fbb.printed_lines)
  4548. def test_query_bot_invalid_query_too_many(self):
  4549. self.override_args(query='bot/Fake Android K Tester/blah/blah',
  4550. check=False,
  4551. pyl_files_dir=None,
  4552. json=None,
  4553. waterfall_filters=[])
  4554. fbb = FakeBBGen(self.args,
  4555. ANDROID_WATERFALL,
  4556. GOOD_COMPOSITION_TEST_SUITES,
  4557. LUCI_MILO_CFG,
  4558. mixins=SWARMING_MIXINS_SORTED)
  4559. with self.assertRaises(SystemExit) as cm:
  4560. fbb.query(fbb.args)
  4561. self.assertEqual(cm.exception.code, 1)
  4562. self.assertTrue(fbb.printed_lines)
  4563. def test_query_bot_invalid_query_no_tests(self):
  4564. self.override_args(query='bot/Fake Android K Tester/blahs',
  4565. check=False,
  4566. pyl_files_dir=None,
  4567. json=None,
  4568. waterfall_filters=[])
  4569. fbb = FakeBBGen(self.args,
  4570. ANDROID_WATERFALL,
  4571. GOOD_COMPOSITION_TEST_SUITES,
  4572. LUCI_MILO_CFG,
  4573. mixins=SWARMING_MIXINS_SORTED)
  4574. with self.assertRaises(SystemExit) as cm:
  4575. fbb.query(fbb.args)
  4576. self.assertEqual(cm.exception.code, 1)
  4577. self.assertTrue(fbb.printed_lines)
  4578. def test_query_bot_tests(self):
  4579. self.override_args(query='bot/Fake Android L Tester/tests',
  4580. check=False,
  4581. pyl_files_dir=None,
  4582. json=None,
  4583. waterfall_filters=[])
  4584. fbb = FakeBBGen(self.args,
  4585. ANDROID_WATERFALL,
  4586. GOOD_COMPOSITION_TEST_SUITES,
  4587. LUCI_MILO_CFG,
  4588. mixins=SWARMING_MIXINS_SORTED)
  4589. fbb.query(fbb.args)
  4590. query_json = json.loads("".join(fbb.printed_lines))
  4591. self.assertEqual(query_json, TEST_QUERY_BOT_TESTS_OUTPUT)
  4592. def test_query_tests(self):
  4593. self.override_args(query='tests',
  4594. check=False,
  4595. pyl_files_dir=None,
  4596. json=None,
  4597. waterfall_filters=[])
  4598. fbb = FakeBBGen(self.args,
  4599. ANDROID_WATERFALL,
  4600. GOOD_COMPOSITION_TEST_SUITES,
  4601. LUCI_MILO_CFG,
  4602. mixins=SWARMING_MIXINS_SORTED)
  4603. fbb.query(fbb.args)
  4604. query_json = json.loads("".join(fbb.printed_lines))
  4605. self.assertEqual(query_json, TEST_QUERY_TESTS_OUTPUT)
  4606. def test_query_tests_invalid(self):
  4607. self.override_args(query='tests/blah/blah',
  4608. check=False,
  4609. pyl_files_dir=None,
  4610. json=None,
  4611. waterfall_filters=[])
  4612. fbb = FakeBBGen(self.args,
  4613. ANDROID_WATERFALL,
  4614. GOOD_COMPOSITION_TEST_SUITES,
  4615. LUCI_MILO_CFG,
  4616. mixins=SWARMING_MIXINS_SORTED)
  4617. with self.assertRaises(SystemExit) as cm:
  4618. fbb.query(fbb.args)
  4619. self.assertEqual(cm.exception.code, 1)
  4620. self.assertTrue(fbb.printed_lines)
  4621. def test_query_tests_multiple_params(self):
  4622. self.override_args(query='tests/--jobs=1&--verbose',
  4623. check=False,
  4624. pyl_files_dir=None,
  4625. json=None,
  4626. waterfall_filters=[])
  4627. fbb = FakeBBGen(self.args,
  4628. ANDROID_WATERFALL,
  4629. TEST_SUITE_WITH_PARAMS,
  4630. LUCI_MILO_CFG,
  4631. mixins=SWARMING_MIXINS_SORTED)
  4632. fbb.query(fbb.args)
  4633. query_json = json.loads("".join(fbb.printed_lines))
  4634. self.assertEqual(query_json, TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT)
  4635. def test_query_tests_invalid_params(self):
  4636. self.override_args(query='tests/device_os?',
  4637. check=False,
  4638. pyl_files_dir=None,
  4639. json=None,
  4640. waterfall_filters=[])
  4641. fbb = FakeBBGen(self.args,
  4642. ANDROID_WATERFALL,
  4643. TEST_SUITE_WITH_PARAMS,
  4644. LUCI_MILO_CFG,
  4645. mixins=SWARMING_MIXINS_SORTED)
  4646. with self.assertRaises(SystemExit) as cm:
  4647. fbb.query(fbb.args)
  4648. self.assertEqual(cm.exception.code, 1)
  4649. self.assertTrue(fbb.printed_lines)
  4650. def test_query_tests_dimension_params(self):
  4651. self.override_args(query='tests/device_os:NMF26U',
  4652. check=False,
  4653. pyl_files_dir=None,
  4654. json=None,
  4655. waterfall_filters=[])
  4656. fbb = FakeBBGen(self.args,
  4657. ANDROID_WATERFALL,
  4658. TEST_SUITE_WITH_PARAMS,
  4659. LUCI_MILO_CFG,
  4660. mixins=SWARMING_MIXINS_SORTED)
  4661. fbb.query(fbb.args)
  4662. query_json = json.loads("".join(fbb.printed_lines))
  4663. self.assertEqual(query_json, TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT)
  4664. def test_query_tests_swarming_params(self):
  4665. self.override_args(query='tests/hard_timeout:1000',
  4666. check=False,
  4667. pyl_files_dir=None,
  4668. json=None,
  4669. waterfall_filters=[])
  4670. fbb = FakeBBGen(self.args,
  4671. ANDROID_WATERFALL,
  4672. TEST_SUITE_WITH_PARAMS,
  4673. LUCI_MILO_CFG,
  4674. mixins=SWARMING_MIXINS_SORTED)
  4675. fbb.query(fbb.args)
  4676. query_json = json.loads("".join(fbb.printed_lines))
  4677. self.assertEqual(query_json, TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT)
  4678. def test_query_tests_params(self):
  4679. self.override_args(query='tests/should_retry_with_patch:true',
  4680. check=False,
  4681. pyl_files_dir=None,
  4682. json=None,
  4683. waterfall_filters=[])
  4684. fbb = FakeBBGen(self.args,
  4685. ANDROID_WATERFALL,
  4686. TEST_SUITE_WITH_PARAMS,
  4687. LUCI_MILO_CFG,
  4688. mixins=SWARMING_MIXINS_SORTED)
  4689. fbb.query(fbb.args)
  4690. query_json = json.loads("".join(fbb.printed_lines))
  4691. self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_OUTPUT)
  4692. def test_query_tests_params_false(self):
  4693. self.override_args(query='tests/should_retry_with_patch:false',
  4694. check=False,
  4695. pyl_files_dir=None,
  4696. json=None,
  4697. waterfall_filters=[])
  4698. fbb = FakeBBGen(self.args,
  4699. ANDROID_WATERFALL,
  4700. TEST_SUITE_WITH_PARAMS,
  4701. LUCI_MILO_CFG,
  4702. mixins=SWARMING_MIXINS_SORTED)
  4703. fbb.query(fbb.args)
  4704. query_json = json.loads("".join(fbb.printed_lines))
  4705. self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT)
  4706. def test_query_test(self):
  4707. self.override_args(query='test/foo_test',
  4708. check=False,
  4709. pyl_files_dir=None,
  4710. json=None,
  4711. waterfall_filters=[])
  4712. fbb = FakeBBGen(self.args,
  4713. ANDROID_WATERFALL,
  4714. GOOD_COMPOSITION_TEST_SUITES,
  4715. LUCI_MILO_CFG,
  4716. mixins=SWARMING_MIXINS_SORTED)
  4717. fbb.query(fbb.args)
  4718. query_json = json.loads("".join(fbb.printed_lines))
  4719. self.assertEqual(query_json, TEST_QUERY_TEST_OUTPUT)
  4720. def test_query_test_invalid_id(self):
  4721. self.override_args(query='test/foo_foo',
  4722. check=False,
  4723. pyl_files_dir=None,
  4724. json=None,
  4725. waterfall_filters=[])
  4726. fbb = FakeBBGen(self.args,
  4727. ANDROID_WATERFALL,
  4728. GOOD_COMPOSITION_TEST_SUITES,
  4729. LUCI_MILO_CFG,
  4730. mixins=SWARMING_MIXINS_SORTED)
  4731. with self.assertRaises(SystemExit) as cm:
  4732. fbb.query(fbb.args)
  4733. self.assertEqual(cm.exception.code, 1)
  4734. self.assertTrue(fbb.printed_lines)
  4735. def test_query_test_invalid_length(self):
  4736. self.override_args(query='test/foo_tests/foo/foo',
  4737. check=False,
  4738. pyl_files_dir=None,
  4739. json=None,
  4740. waterfall_filters=[])
  4741. fbb = FakeBBGen(self.args,
  4742. ANDROID_WATERFALL,
  4743. GOOD_COMPOSITION_TEST_SUITES,
  4744. LUCI_MILO_CFG,
  4745. mixins=SWARMING_MIXINS_SORTED)
  4746. with self.assertRaises(SystemExit) as cm:
  4747. fbb.query(fbb.args)
  4748. self.assertEqual(cm.exception.code, 1)
  4749. self.assertTrue(fbb.printed_lines)
  4750. def test_query_test_bots(self):
  4751. self.override_args(query='test/foo_test/bots',
  4752. check=False,
  4753. pyl_files_dir=None,
  4754. json=None,
  4755. waterfall_filters=[])
  4756. fbb = FakeBBGen(self.args,
  4757. ANDROID_WATERFALL,
  4758. GOOD_COMPOSITION_TEST_SUITES,
  4759. LUCI_MILO_CFG,
  4760. mixins=SWARMING_MIXINS_SORTED)
  4761. fbb.query(fbb.args)
  4762. query_json = json.loads("".join(fbb.printed_lines))
  4763. self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_OUTPUT)
  4764. def test_query_test_bots_isolated_scripts(self):
  4765. self.override_args(query='test/foo_test/bots',
  4766. check=False,
  4767. pyl_files_dir=None,
  4768. json=None,
  4769. waterfall_filters=[])
  4770. fbb = FakeBBGen(self.args,
  4771. FOO_ISOLATED_SCRIPTS_WATERFALL,
  4772. GOOD_COMPOSITION_TEST_SUITES,
  4773. LUCI_MILO_CFG,
  4774. mixins=SWARMING_MIXINS_SORTED)
  4775. fbb.query(fbb.args)
  4776. query_json = json.loads("".join(fbb.printed_lines))
  4777. self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT)
  4778. def test_query_test_bots_invalid(self):
  4779. self.override_args(query='test/foo_tests/foo',
  4780. check=False,
  4781. pyl_files_dir=None,
  4782. json=None,
  4783. waterfall_filters=[])
  4784. fbb = FakeBBGen(self.args,
  4785. ANDROID_WATERFALL,
  4786. GOOD_COMPOSITION_TEST_SUITES,
  4787. LUCI_MILO_CFG,
  4788. mixins=SWARMING_MIXINS_SORTED)
  4789. with self.assertRaises(SystemExit) as cm:
  4790. fbb.query(fbb.args)
  4791. self.assertEqual(cm.exception.code, 1)
  4792. self.assertTrue(fbb.printed_lines)
  4793. def test_query_test_bots_no_bots(self):
  4794. self.override_args(query='test/bar_tests/bots',
  4795. check=False,
  4796. pyl_files_dir=None,
  4797. json=None,
  4798. waterfall_filters=[])
  4799. fbb = FakeBBGen(self.args,
  4800. ANDROID_WATERFALL,
  4801. GOOD_COMPOSITION_TEST_SUITES,
  4802. LUCI_MILO_CFG,
  4803. mixins=SWARMING_MIXINS_SORTED)
  4804. fbb.query(fbb.args)
  4805. query_json = json.loads("".join(fbb.printed_lines))
  4806. self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT)
  4807. def test_query_invalid(self):
  4808. self.override_args(query='foo',
  4809. check=False,
  4810. pyl_files_dir=None,
  4811. json=None,
  4812. waterfall_filters=[])
  4813. fbb = FakeBBGen(self.args,
  4814. ANDROID_WATERFALL,
  4815. GOOD_COMPOSITION_TEST_SUITES,
  4816. LUCI_MILO_CFG,
  4817. mixins=SWARMING_MIXINS_SORTED)
  4818. with self.assertRaises(SystemExit) as cm:
  4819. fbb.query(fbb.args)
  4820. self.assertEqual(cm.exception.code, 1)
  4821. self.assertTrue(fbb.printed_lines)
  4822. FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES = """\
  4823. {
  4824. 'basic_suites': {
  4825. 'foo_tests': {
  4826. 'foo_test': {
  4827. 'args': [
  4828. '--enable-features',
  4829. 'Foo,Bar',
  4830. ],
  4831. },
  4832. },
  4833. },
  4834. }
  4835. """
  4836. FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE = """\
  4837. {
  4838. 'foo_test': {
  4839. 'replacements': {
  4840. 'Fake Tester': {
  4841. 'args': {
  4842. '--c_arg': None,
  4843. },
  4844. },
  4845. },
  4846. },
  4847. }
  4848. """
  4849. FOO_TEST_REPLACEMENTS_REMOVE_VALUE = """\
  4850. {
  4851. 'foo_test': {
  4852. 'replacements': {
  4853. 'Fake Tester': {
  4854. 'args': {
  4855. '--enable-features': None,
  4856. },
  4857. },
  4858. },
  4859. },
  4860. }
  4861. """
  4862. FOO_TEST_REPLACEMENTS_REPLACE_VALUE = """\
  4863. {
  4864. 'foo_test': {
  4865. 'replacements': {
  4866. 'Fake Tester': {
  4867. 'args': {
  4868. '--enable-features': 'Bar,Baz',
  4869. },
  4870. },
  4871. },
  4872. },
  4873. }
  4874. """
  4875. FOO_TEST_REPLACEMENTS_INVALID_KEY = """\
  4876. {
  4877. 'foo_test': {
  4878. 'replacements': {
  4879. 'Fake Tester': {
  4880. 'invalid': {
  4881. '--enable-features': 'Bar,Baz',
  4882. },
  4883. },
  4884. },
  4885. },
  4886. }
  4887. """
  4888. REPLACEMENTS_REMOVE_OUTPUT = """\
  4889. {
  4890. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  4891. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  4892. "Fake Tester": {
  4893. "gtest_tests": [
  4894. {
  4895. "args": [],
  4896. "merge": {
  4897. "args": [],
  4898. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4899. },
  4900. "swarming": {
  4901. "can_use_on_swarming_builders": true,
  4902. "dimension_sets": [
  4903. {
  4904. "kvm": "1"
  4905. }
  4906. ]
  4907. },
  4908. "test": "foo_test"
  4909. }
  4910. ]
  4911. }
  4912. }
  4913. """
  4914. REPLACEMENTS_VALUE_OUTPUT = """\
  4915. {
  4916. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  4917. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  4918. "Fake Tester": {
  4919. "gtest_tests": [
  4920. {
  4921. "args": [
  4922. "--enable-features=Bar,Baz"
  4923. ],
  4924. "merge": {
  4925. "args": [],
  4926. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4927. },
  4928. "swarming": {
  4929. "can_use_on_swarming_builders": true,
  4930. "dimension_sets": [
  4931. {
  4932. "kvm": "1"
  4933. }
  4934. ]
  4935. },
  4936. "test": "foo_test"
  4937. }
  4938. ]
  4939. }
  4940. }
  4941. """
  4942. REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT = """\
  4943. {
  4944. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  4945. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  4946. "Fake Tester": {
  4947. "gtest_tests": [
  4948. {
  4949. "args": [
  4950. "--enable-features",
  4951. "Bar,Baz"
  4952. ],
  4953. "merge": {
  4954. "args": [],
  4955. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  4956. },
  4957. "swarming": {
  4958. "can_use_on_swarming_builders": true,
  4959. "dimension_sets": [
  4960. {
  4961. "kvm": "1"
  4962. }
  4963. ]
  4964. },
  4965. "test": "foo_test"
  4966. }
  4967. ]
  4968. }
  4969. }
  4970. """
  4971. class ReplacementTests(TestCase):
  4972. """Tests for the arg replacement feature."""
  4973. def test_replacement_valid_remove_no_value(self):
  4974. fbb = FakeBBGen(self.args,
  4975. FOO_GTESTS_WATERFALL,
  4976. FOO_TEST_SUITE_WITH_ARGS,
  4977. LUCI_MILO_CFG,
  4978. exceptions=FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE)
  4979. self.create_testing_buildbot_json_file('chromium.test.json',
  4980. REPLACEMENTS_REMOVE_OUTPUT)
  4981. self.create_testing_buildbot_json_file('chromium.ci.json',
  4982. REPLACEMENTS_REMOVE_OUTPUT)
  4983. fbb.check_output_file_consistency(verbose=True)
  4984. self.assertFalse(fbb.printed_lines)
  4985. def test_replacement_valid_remove_value(self):
  4986. fbb = FakeBBGen(self.args,
  4987. FOO_GTESTS_WATERFALL,
  4988. FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
  4989. LUCI_MILO_CFG,
  4990. exceptions=FOO_TEST_REPLACEMENTS_REMOVE_VALUE)
  4991. self.create_testing_buildbot_json_file('chromium.test.json',
  4992. REPLACEMENTS_REMOVE_OUTPUT)
  4993. self.create_testing_buildbot_json_file('chromium.ci.json',
  4994. REPLACEMENTS_REMOVE_OUTPUT)
  4995. fbb.check_output_file_consistency(verbose=True)
  4996. self.assertFalse(fbb.printed_lines)
  4997. def test_replacement_valid_replace_value(self):
  4998. fbb = FakeBBGen(self.args,
  4999. FOO_GTESTS_WATERFALL,
  5000. FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
  5001. LUCI_MILO_CFG,
  5002. exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
  5003. self.create_testing_buildbot_json_file('chromium.test.json',
  5004. REPLACEMENTS_VALUE_OUTPUT)
  5005. self.create_testing_buildbot_json_file('chromium.ci.json',
  5006. REPLACEMENTS_VALUE_OUTPUT)
  5007. fbb.check_output_file_consistency(verbose=True)
  5008. self.assertFalse(fbb.printed_lines)
  5009. def test_replacement_valid_replace_value_separate_entries(self):
  5010. fbb = FakeBBGen(self.args,
  5011. FOO_GTESTS_WATERFALL,
  5012. FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES,
  5013. LUCI_MILO_CFG,
  5014. exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
  5015. self.create_testing_buildbot_json_file(
  5016. 'chromium.test.json', REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT)
  5017. self.create_testing_buildbot_json_file(
  5018. 'chromium.ci.json', REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT)
  5019. fbb.check_output_file_consistency(verbose=True)
  5020. self.assertFalse(fbb.printed_lines)
  5021. def test_replacement_invalid_key_not_valid(self):
  5022. fbb = FakeBBGen(self.args,
  5023. FOO_GTESTS_WATERFALL,
  5024. FOO_TEST_SUITE,
  5025. LUCI_MILO_CFG,
  5026. exceptions=FOO_TEST_REPLACEMENTS_INVALID_KEY)
  5027. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  5028. 'Given replacement key *'):
  5029. fbb.check_output_file_consistency(verbose=True)
  5030. def test_replacement_invalid_key_not_found(self):
  5031. fbb = FakeBBGen(self.args,
  5032. FOO_GTESTS_WATERFALL,
  5033. FOO_TEST_SUITE_WITH_ARGS,
  5034. LUCI_MILO_CFG,
  5035. exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
  5036. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  5037. 'Could not find *'):
  5038. fbb.check_output_file_consistency(verbose=True)
  5039. FOO_TEST_SUITE_WITH_MAGIC_ARGS = """\
  5040. {
  5041. 'basic_suites': {
  5042. 'foo_tests': {
  5043. 'foo_test': {
  5044. 'args': [
  5045. '$$MAGIC_SUBSTITUTION_TestOnlySubstitution',
  5046. ],
  5047. },
  5048. },
  5049. },
  5050. }
  5051. """
  5052. FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS = """\
  5053. {
  5054. 'basic_suites': {
  5055. 'foo_tests': {
  5056. 'foo_test': {
  5057. 'args': [
  5058. '$$MAGIC_SUBSTITUTION_NotARealSubstitution',
  5059. ],
  5060. },
  5061. },
  5062. },
  5063. }
  5064. """
  5065. MAGIC_SUBSTITUTIONS_OUTPUT = """\
  5066. {
  5067. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5068. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5069. "Fake Tester": {
  5070. "gtest_tests": [
  5071. {
  5072. "args": [
  5073. "--magic-substitution-success"
  5074. ],
  5075. "merge": {
  5076. "args": [],
  5077. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5078. },
  5079. "swarming": {
  5080. "can_use_on_swarming_builders": true,
  5081. "dimension_sets": [
  5082. {
  5083. "kvm": "1"
  5084. }
  5085. ]
  5086. },
  5087. "test": "foo_test"
  5088. }
  5089. ]
  5090. }
  5091. }
  5092. """
  5093. class MagicSubstitutionTests(TestCase):
  5094. """Tests for the magic substitution feature."""
  5095. def test_valid_function(self):
  5096. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
  5097. FOO_TEST_SUITE_WITH_MAGIC_ARGS, LUCI_MILO_CFG)
  5098. self.create_testing_buildbot_json_file('chromium.test.json',
  5099. MAGIC_SUBSTITUTIONS_OUTPUT)
  5100. self.create_testing_buildbot_json_file('chromium.ci.json',
  5101. MAGIC_SUBSTITUTIONS_OUTPUT)
  5102. fbb.check_output_file_consistency(verbose=True)
  5103. self.assertFalse(fbb.printed_lines)
  5104. def test_invalid_function(self):
  5105. fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
  5106. FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS, LUCI_MILO_CFG)
  5107. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  5108. 'Magic substitution function *'):
  5109. fbb.check_output_file_consistency(verbose=True)
  5110. # Matrix compound composition test suites
  5111. MATRIX_COMPOUND_EMPTY = """\
  5112. {
  5113. 'basic_suites': {
  5114. 'bar_tests': {
  5115. 'bar_test': {},
  5116. },
  5117. 'foo_tests': {
  5118. 'foo_test': {},
  5119. },
  5120. },
  5121. 'matrix_compound_suites': {
  5122. 'matrix_tests': {
  5123. 'foo_tests': {},
  5124. 'bar_tests': {},
  5125. },
  5126. },
  5127. }
  5128. """
  5129. MATRIX_COMPOUND_MISSING_IDENTIFIER = """\
  5130. {
  5131. 'basic_suites': {
  5132. 'foo_tests': {
  5133. 'foo_test': {},
  5134. },
  5135. },
  5136. 'matrix_compound_suites': {
  5137. 'matrix_tests': {
  5138. 'foo_tests': {
  5139. 'variants': [
  5140. {
  5141. 'swarming': {
  5142. 'dimension_sets': [
  5143. {
  5144. 'foo': 'bar',
  5145. },
  5146. ],
  5147. },
  5148. },
  5149. ],
  5150. },
  5151. },
  5152. },
  5153. }
  5154. """
  5155. MATRIX_COMPOUND_EMPTY_IDENTIFIER = """\
  5156. {
  5157. 'basic_suites': {
  5158. 'foo_tests': {
  5159. 'foo_test': {},
  5160. },
  5161. },
  5162. 'matrix_compound_suites': {
  5163. 'matrix_tests': {
  5164. 'foo_tests': {
  5165. 'variants': [
  5166. {
  5167. 'identifier': '',
  5168. 'swarming': {
  5169. 'dimension_sets': [
  5170. {
  5171. 'foo': 'empty identifier not allowed',
  5172. },
  5173. ],
  5174. },
  5175. },
  5176. ],
  5177. },
  5178. },
  5179. },
  5180. }
  5181. """
  5182. MATRIX_COMPOUND_TRAILING_IDENTIFIER = """\
  5183. {
  5184. 'basic_suites': {
  5185. 'foo_tests': {
  5186. 'foo_test': {},
  5187. },
  5188. },
  5189. 'matrix_compound_suites': {
  5190. 'matrix_tests': {
  5191. 'foo_tests': {
  5192. 'variants': [
  5193. {
  5194. 'identifier': ' ',
  5195. 'swarming': {
  5196. 'dimension_sets': [
  5197. {
  5198. 'foo': 'strip to empty not allowed',
  5199. },
  5200. ],
  5201. },
  5202. },
  5203. ],
  5204. },
  5205. 'foo_tests': {
  5206. 'variants': [
  5207. {
  5208. 'identifier': 'id ',
  5209. 'swarming': {
  5210. 'dimension_sets': [
  5211. {
  5212. 'foo': 'trailing whitespace not allowed',
  5213. },
  5214. ],
  5215. },
  5216. },
  5217. ],
  5218. },
  5219. },
  5220. },
  5221. }
  5222. """
  5223. MATRIX_MISMATCHED_SWARMING_LENGTH = """\
  5224. {
  5225. 'basic_suites': {
  5226. 'foo_tests': {
  5227. 'foo_test': {
  5228. 'swarming': {
  5229. 'dimension_sets': [
  5230. {
  5231. 'hello': 'world',
  5232. }
  5233. ],
  5234. },
  5235. },
  5236. },
  5237. },
  5238. 'matrix_compound_suites': {
  5239. 'matrix_tests': {
  5240. 'foo_tests': {
  5241. 'variants': [
  5242. {
  5243. 'identifier': 'test',
  5244. 'swarming': {
  5245. 'dimension_sets': [
  5246. {
  5247. 'foo': 'bar',
  5248. },
  5249. {
  5250. 'bar': 'foo',
  5251. }
  5252. ],
  5253. },
  5254. },
  5255. ],
  5256. },
  5257. },
  5258. },
  5259. }
  5260. """
  5261. MATRIX_REF_NONEXISTENT = """\
  5262. {
  5263. 'basic_suites': {
  5264. 'foo_tests': {
  5265. 'foo_test': {},
  5266. },
  5267. },
  5268. 'matrix_compound_suites': {
  5269. 'matrix_tests': {
  5270. 'bar_test': {},
  5271. },
  5272. },
  5273. }
  5274. """
  5275. MATRIX_COMPOUND_REF_COMPOSITION = """\
  5276. {
  5277. 'basic_suites': {
  5278. 'foo_tests': {
  5279. 'foo_test': {},
  5280. },
  5281. },
  5282. 'compound_suites': {
  5283. 'sample_composition': {
  5284. 'foo_tests': {},
  5285. },
  5286. },
  5287. 'matrix_compound_suites': {
  5288. 'matrix_tests': {
  5289. 'sample_composition': {},
  5290. },
  5291. },
  5292. }
  5293. """
  5294. MATRIX_COMPOSITION_REF_MATRIX = """\
  5295. {
  5296. 'basic_suites': {
  5297. 'foo_tests': {
  5298. 'foo_test': {},
  5299. },
  5300. },
  5301. 'matrix_compound_suites': {
  5302. 'a_test': {
  5303. 'foo_tests': {},
  5304. },
  5305. 'matrix_tests': {
  5306. 'a_test': {},
  5307. },
  5308. },
  5309. }
  5310. """
  5311. MATRIX_COMPOUND_VARIANTS_MIXINS_MERGE = """\
  5312. {
  5313. 'basic_suites': {
  5314. 'foo_tests': {
  5315. 'set': {
  5316. 'mixins': [ 'test_mixin' ],
  5317. },
  5318. },
  5319. },
  5320. 'matrix_compound_suites': {
  5321. 'matrix_tests': {
  5322. 'foo_tests': {
  5323. 'variants': [
  5324. {
  5325. 'mixins': [ 'dimension_mixin' ],
  5326. },
  5327. ],
  5328. },
  5329. },
  5330. },
  5331. }
  5332. """
  5333. MATRIX_COMPOUND_VARIANTS_MIXINS = """\
  5334. {
  5335. 'basic_suites': {
  5336. 'foo_tests': {
  5337. 'set': {
  5338. 'mixins': [ 'test_mixin' ],
  5339. },
  5340. },
  5341. },
  5342. 'matrix_compound_suites': {
  5343. 'matrix_tests': {
  5344. 'foo_tests': {
  5345. 'variants': [
  5346. {
  5347. 'mixins': [
  5348. 'dimension_mixin',
  5349. 'waterfall_mixin',
  5350. 'builder_mixin',
  5351. 'random_mixin'
  5352. ],
  5353. },
  5354. ],
  5355. },
  5356. },
  5357. },
  5358. }
  5359. """
  5360. MATRIX_COMPOUND_VARIANTS_MIXINS_REMOVE = """\
  5361. {
  5362. 'basic_suites': {
  5363. 'foo_tests': {
  5364. 'set': {
  5365. 'remove_mixins': ['builder_mixin'],
  5366. },
  5367. },
  5368. },
  5369. 'matrix_compound_suites': {
  5370. 'matrix_tests': {
  5371. 'foo_tests': {
  5372. 'variants': [
  5373. {
  5374. 'mixins': [ 'builder_mixin' ],
  5375. }
  5376. ],
  5377. },
  5378. },
  5379. },
  5380. }
  5381. """
  5382. MATRIX_COMPOUND_CONFLICTING_TEST_SUITES = """\
  5383. {
  5384. 'basic_suites': {
  5385. 'bar_tests': {
  5386. 'baz_tests': {
  5387. 'args': [
  5388. '--bar',
  5389. ],
  5390. }
  5391. },
  5392. 'foo_tests': {
  5393. 'baz_tests': {
  5394. 'args': [
  5395. '--foo',
  5396. ],
  5397. }
  5398. },
  5399. },
  5400. 'matrix_compound_suites': {
  5401. 'matrix_tests': {
  5402. 'bar_tests': {
  5403. 'variants': [
  5404. {
  5405. 'identifier': 'bar',
  5406. }
  5407. ],
  5408. },
  5409. 'foo_tests': {
  5410. 'variants': [
  5411. {
  5412. 'identifier': 'foo'
  5413. }
  5414. ]
  5415. }
  5416. },
  5417. },
  5418. }
  5419. """
  5420. MATRIX_COMPOUND_TARGETS_ARGS = """\
  5421. {
  5422. 'basic_suites': {
  5423. 'foo_tests': {
  5424. 'args_test': {
  5425. 'args': [
  5426. '--iam'
  5427. ],
  5428. },
  5429. }
  5430. },
  5431. 'matrix_compound_suites': {
  5432. 'matrix_tests': {
  5433. 'foo_tests': {
  5434. 'variants': [
  5435. {
  5436. 'identifier': 'args',
  5437. 'args': [
  5438. '--anarg',
  5439. ],
  5440. },
  5441. {
  5442. 'identifier': 'swarming',
  5443. 'swarming': {
  5444. 'a': 'b',
  5445. 'dimension_sets': [
  5446. {
  5447. 'hello': 'world',
  5448. }
  5449. ]
  5450. }
  5451. },
  5452. {
  5453. 'identifier': 'mixins',
  5454. 'mixins': [ 'dimension_mixin' ],
  5455. }
  5456. ],
  5457. },
  5458. },
  5459. },
  5460. }
  5461. """
  5462. MATRIX_COMPOUND_TARGETS_MIXINS = """\
  5463. {
  5464. 'basic_suites': {
  5465. 'foo_tests': {
  5466. 'mixins_test': {
  5467. 'mixins': [ 'test_mixin' ],
  5468. },
  5469. }
  5470. },
  5471. 'matrix_compound_suites': {
  5472. 'matrix_tests': {
  5473. 'foo_tests': {
  5474. 'mixins': [ 'random_mixin' ],
  5475. 'variants': [
  5476. {
  5477. 'identifier': 'args',
  5478. 'args': [
  5479. '--anarg',
  5480. ],
  5481. },
  5482. {
  5483. 'identifier': 'swarming',
  5484. 'swarming': {
  5485. 'a': 'b',
  5486. 'dimension_sets': [
  5487. {
  5488. 'hello': 'world',
  5489. }
  5490. ]
  5491. }
  5492. },
  5493. {
  5494. 'identifier': 'mixins',
  5495. 'mixins': [ 'dimension_mixin' ],
  5496. }
  5497. ],
  5498. },
  5499. },
  5500. },
  5501. }
  5502. """
  5503. MATRIX_COMPOUND_TARGETS_SWARMING = """\
  5504. {
  5505. 'basic_suites': {
  5506. 'foo_tests': {
  5507. 'swarming_test': {
  5508. 'swarming': {
  5509. 'foo': 'bar',
  5510. 'dimension_sets': [
  5511. {
  5512. 'foo': 'bar',
  5513. },
  5514. ],
  5515. },
  5516. },
  5517. }
  5518. },
  5519. 'matrix_compound_suites': {
  5520. 'matrix_tests': {
  5521. 'foo_tests': {
  5522. 'variants': [
  5523. {
  5524. 'identifier': 'args',
  5525. 'args': [
  5526. '--anarg',
  5527. ],
  5528. },
  5529. {
  5530. 'identifier': 'swarming',
  5531. 'swarming': {
  5532. 'a': 'b',
  5533. 'dimension_sets': [
  5534. {
  5535. 'hello': 'world',
  5536. }
  5537. ]
  5538. }
  5539. },
  5540. {
  5541. 'identifier': 'mixins',
  5542. 'mixins': [ 'dimension_mixin' ],
  5543. }
  5544. ],
  5545. },
  5546. },
  5547. },
  5548. }
  5549. """
  5550. MATRIX_COMPOUND_VARIANTS_REF = """\
  5551. {
  5552. 'basic_suites': {
  5553. 'foo_tests': {
  5554. 'swarming_test': {},
  5555. }
  5556. },
  5557. 'matrix_compound_suites': {
  5558. 'matrix_tests': {
  5559. 'foo_tests': {
  5560. 'variants': [
  5561. 'a_variant'
  5562. ],
  5563. },
  5564. },
  5565. },
  5566. }
  5567. """
  5568. MATRIX_COMPOUND_TEST_WITH_TEST_KEY = """\
  5569. {
  5570. 'basic_suites': {
  5571. 'foo_tests': {
  5572. 'swarming_test': {
  5573. 'test': 'foo_test_apk'
  5574. },
  5575. }
  5576. },
  5577. 'matrix_compound_suites': {
  5578. 'matrix_tests': {
  5579. 'foo_tests': {
  5580. 'variants': [
  5581. 'a_variant',
  5582. ],
  5583. },
  5584. },
  5585. },
  5586. }
  5587. """
  5588. MATRIX_COMPOUND_MIXED_VARIANTS_REF = """\
  5589. {
  5590. 'basic_suites': {
  5591. 'foo_tests': {
  5592. 'swarming_test': {},
  5593. }
  5594. },
  5595. 'matrix_compound_suites': {
  5596. 'matrix_tests': {
  5597. 'foo_tests': {
  5598. 'variants': [
  5599. 'a_variant',
  5600. {
  5601. 'args': [
  5602. 'a',
  5603. 'b'
  5604. ],
  5605. 'identifier': 'ab',
  5606. }
  5607. ],
  5608. },
  5609. },
  5610. },
  5611. }
  5612. """
  5613. VARIANTS_FILE = """\
  5614. {
  5615. 'a_variant': {
  5616. 'args': [
  5617. '--platform',
  5618. 'device',
  5619. '--version',
  5620. '1'
  5621. ],
  5622. 'identifier': 'a_variant'
  5623. }
  5624. }
  5625. """
  5626. MULTI_VARIANTS_FILE = """\
  5627. {
  5628. 'a_variant': {
  5629. 'args': [
  5630. '--platform',
  5631. 'device',
  5632. '--version',
  5633. '1'
  5634. ],
  5635. 'identifier': 'a_variant'
  5636. },
  5637. 'b_variant': {
  5638. 'args': [
  5639. '--platform',
  5640. 'sim',
  5641. '--version',
  5642. '2'
  5643. ],
  5644. 'identifier': 'b_variant'
  5645. }
  5646. }
  5647. """
  5648. # # Dictionary composition test suite outputs
  5649. MATRIX_COMPOUND_OUTPUT = """\
  5650. {
  5651. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5652. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5653. "Fake Tester": {
  5654. "gtest_tests": [
  5655. {
  5656. "merge": {
  5657. "args": [],
  5658. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5659. },
  5660. "swarming": {
  5661. "can_use_on_swarming_builders": true
  5662. },
  5663. "test": "bar_test"
  5664. },
  5665. {
  5666. "merge": {
  5667. "args": [],
  5668. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5669. },
  5670. "swarming": {
  5671. "can_use_on_swarming_builders": true
  5672. },
  5673. "test": "foo_test"
  5674. }
  5675. ]
  5676. }
  5677. }
  5678. """
  5679. MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT = """\
  5680. {
  5681. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5682. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5683. "Fake Tester": {
  5684. "gtest_tests": [
  5685. {
  5686. "args": [
  5687. "--platform",
  5688. "device",
  5689. "--version",
  5690. "1"
  5691. ],
  5692. "merge": {
  5693. "args": [],
  5694. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5695. },
  5696. "name": "swarming_test a_variant",
  5697. "swarming": {
  5698. "can_use_on_swarming_builders": true
  5699. },
  5700. "test": "foo_test_apk",
  5701. "variant_id": "a_variant"
  5702. }
  5703. ]
  5704. }
  5705. }
  5706. """
  5707. MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS = """\
  5708. {
  5709. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5710. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5711. "Fake Tester": {
  5712. "gtest_tests": [
  5713. {
  5714. "args": [
  5715. "--iam",
  5716. "--anarg"
  5717. ],
  5718. "merge": {
  5719. "args": [],
  5720. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5721. },
  5722. "name": "args_test args",
  5723. "swarming": {
  5724. "can_use_on_swarming_builders": true
  5725. },
  5726. "test": "args_test",
  5727. "variant_id": "args"
  5728. },
  5729. {
  5730. "args": [
  5731. "--iam"
  5732. ],
  5733. "merge": {
  5734. "args": [],
  5735. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5736. },
  5737. "name": "args_test mixins",
  5738. "swarming": {
  5739. "can_use_on_swarming_builders": true,
  5740. "dimension_sets": [
  5741. {
  5742. "iama": "mixin"
  5743. }
  5744. ]
  5745. },
  5746. "test": "args_test",
  5747. "variant_id": "mixins"
  5748. },
  5749. {
  5750. "args": [
  5751. "--iam"
  5752. ],
  5753. "merge": {
  5754. "args": [],
  5755. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5756. },
  5757. "name": "args_test swarming",
  5758. "swarming": {
  5759. "a": "b",
  5760. "can_use_on_swarming_builders": true,
  5761. "dimension_sets": [
  5762. {
  5763. "hello": "world"
  5764. }
  5765. ]
  5766. },
  5767. "test": "args_test",
  5768. "variant_id": "swarming"
  5769. }
  5770. ]
  5771. }
  5772. }
  5773. """
  5774. MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS = """\
  5775. {
  5776. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5777. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5778. "Fake Tester": {
  5779. "gtest_tests": [
  5780. {
  5781. "args": [
  5782. "--anarg"
  5783. ],
  5784. "merge": {
  5785. "args": [],
  5786. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5787. },
  5788. "name": "mixins_test args",
  5789. "swarming": {
  5790. "can_use_on_swarming_builders": true,
  5791. "value": "test"
  5792. },
  5793. "test": "mixins_test",
  5794. "value": "random",
  5795. "variant_id": "args"
  5796. },
  5797. {
  5798. "args": [],
  5799. "merge": {
  5800. "args": [],
  5801. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5802. },
  5803. "name": "mixins_test mixins",
  5804. "swarming": {
  5805. "can_use_on_swarming_builders": true,
  5806. "dimension_sets": [
  5807. {
  5808. "iama": "mixin"
  5809. }
  5810. ],
  5811. "value": "test"
  5812. },
  5813. "test": "mixins_test",
  5814. "value": "random",
  5815. "variant_id": "mixins"
  5816. },
  5817. {
  5818. "args": [],
  5819. "merge": {
  5820. "args": [],
  5821. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5822. },
  5823. "name": "mixins_test swarming",
  5824. "swarming": {
  5825. "a": "b",
  5826. "can_use_on_swarming_builders": true,
  5827. "dimension_sets": [
  5828. {
  5829. "hello": "world"
  5830. }
  5831. ],
  5832. "value": "test"
  5833. },
  5834. "test": "mixins_test",
  5835. "value": "random",
  5836. "variant_id": "swarming"
  5837. }
  5838. ]
  5839. }
  5840. }
  5841. """
  5842. MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING = """\
  5843. {
  5844. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5845. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5846. "Fake Tester": {
  5847. "gtest_tests": [
  5848. {
  5849. "args": [
  5850. "--anarg"
  5851. ],
  5852. "merge": {
  5853. "args": [],
  5854. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5855. },
  5856. "name": "swarming_test args",
  5857. "swarming": {
  5858. "can_use_on_swarming_builders": true,
  5859. "dimension_sets": [
  5860. {
  5861. "foo": "bar"
  5862. }
  5863. ],
  5864. "foo": "bar"
  5865. },
  5866. "test": "swarming_test",
  5867. "variant_id": "args"
  5868. },
  5869. {
  5870. "args": [],
  5871. "merge": {
  5872. "args": [],
  5873. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5874. },
  5875. "name": "swarming_test mixins",
  5876. "swarming": {
  5877. "can_use_on_swarming_builders": true,
  5878. "dimension_sets": [
  5879. {
  5880. "foo": "bar",
  5881. "iama": "mixin"
  5882. }
  5883. ],
  5884. "foo": "bar"
  5885. },
  5886. "test": "swarming_test",
  5887. "variant_id": "mixins"
  5888. },
  5889. {
  5890. "args": [],
  5891. "merge": {
  5892. "args": [],
  5893. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5894. },
  5895. "name": "swarming_test swarming",
  5896. "swarming": {
  5897. "a": "b",
  5898. "can_use_on_swarming_builders": true,
  5899. "dimension_sets": [
  5900. {
  5901. "foo": "bar",
  5902. "hello": "world"
  5903. }
  5904. ],
  5905. "foo": "bar"
  5906. },
  5907. "test": "swarming_test",
  5908. "variant_id": "swarming"
  5909. }
  5910. ]
  5911. }
  5912. }
  5913. """
  5914. MATRIX_COMPOUND_VARIANTS_REF_OUTPUT = """\
  5915. {
  5916. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  5917. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  5918. "Fake Tester": {
  5919. "gtest_tests": [
  5920. {
  5921. "args": [
  5922. "--platform",
  5923. "device",
  5924. "--version",
  5925. "1"
  5926. ],
  5927. "merge": {
  5928. "args": [],
  5929. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  5930. },
  5931. "name": "swarming_test a_variant",
  5932. "swarming": {
  5933. "can_use_on_swarming_builders": true
  5934. },
  5935. "test": "swarming_test",
  5936. "variant_id": "a_variant"
  5937. }
  5938. ]
  5939. }
  5940. }
  5941. """
  5942. EMPTY_SKYLAB_TEST_EXCEPTIONS = """\
  5943. {
  5944. 'tast.foo OCTOPUS_TOT': {
  5945. 'remove_from': [
  5946. 'Fake Tester',
  5947. ]
  5948. },
  5949. 'tast.foo OCTOPUS_TOT-1': {
  5950. 'remove_from': [
  5951. 'Fake Tester',
  5952. ]
  5953. }
  5954. }
  5955. """
  5956. MATRIX_SKYLAB_WATERFALL = """\
  5957. [
  5958. {
  5959. 'project': 'chromium',
  5960. 'bucket': 'ci',
  5961. 'name': 'chromium.test',
  5962. 'machines': {
  5963. 'Fake Tester': {
  5964. 'test_suites': {
  5965. 'skylab_tests': 'cros_skylab_basic_x86',
  5966. },
  5967. },
  5968. },
  5969. },
  5970. ]
  5971. """
  5972. MATRIX_COMPOUND_SKYLAB_REF = """\
  5973. {
  5974. 'basic_suites': {
  5975. 'cros_skylab_basic': {
  5976. 'tast.basic': {
  5977. 'suite': 'tast.basic',
  5978. 'timeout': 3600,
  5979. },
  5980. 'tast.foo': {
  5981. 'suite': 'tast.foo',
  5982. 'timeout': 3600,
  5983. },
  5984. },
  5985. },
  5986. 'compound_suites': {},
  5987. 'matrix_compound_suites': {
  5988. 'cros_skylab_basic_x86': {
  5989. 'cros_skylab_basic': {
  5990. 'variants': [
  5991. {
  5992. 'skylab': {
  5993. 'cros_board': 'octopus',
  5994. 'cros_chrome_version': '89.0.3234.0',
  5995. 'cros_img': 'octopus-release/R89-13655.0.0',
  5996. },
  5997. 'enabled': True,
  5998. 'identifier': 'OCTOPUS_TOT',
  5999. },
  6000. {
  6001. 'skylab': {
  6002. 'cros_board': 'octopus',
  6003. 'cros_chrome_version': '88.0.2324.0',
  6004. 'cros_img': 'octopus-release/R88-13597.23.0',
  6005. },
  6006. 'enabled': True,
  6007. 'identifier': 'OCTOPUS_TOT-1',
  6008. },
  6009. ]
  6010. },
  6011. },
  6012. },
  6013. }
  6014. """
  6015. ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF = """\
  6016. {
  6017. 'basic_suites': {
  6018. 'cros_skylab_basic': {
  6019. 'tast.basic': {
  6020. 'suite': 'tast.basic',
  6021. 'timeout': 3600,
  6022. },
  6023. 'tast.foo': {
  6024. 'suite': 'tast.foo',
  6025. 'timeout': 3600,
  6026. },
  6027. },
  6028. },
  6029. 'compound_suites': {},
  6030. 'matrix_compound_suites': {
  6031. 'cros_skylab_basic_x86': {
  6032. 'cros_skylab_basic': {
  6033. 'variants': [
  6034. {
  6035. 'skylab': {
  6036. 'cros_board': 'octopus',
  6037. 'cros_chrome_version': '89.0.3234.0',
  6038. 'cros_img': 'octopus-release/R89-13655.0.0',
  6039. },
  6040. 'enabled': True,
  6041. 'identifier': 'OCTOPUS_TOT',
  6042. },
  6043. {
  6044. 'skylab': {
  6045. 'cros_board': 'octopus',
  6046. 'cros_chrome_version': '88.0.2324.0',
  6047. 'cros_img': 'octopus-release/R88-13597.23.0',
  6048. },
  6049. 'enabled': False,
  6050. 'identifier': 'OCTOPUS_TOT-1',
  6051. },
  6052. ]
  6053. },
  6054. },
  6055. },
  6056. }
  6057. """
  6058. VARIATION_SKYLAB_OUTPUT = """\
  6059. {
  6060. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  6061. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  6062. "Fake Tester": {
  6063. "skylab_tests": [
  6064. {
  6065. "args": [],
  6066. "cros_board": "octopus",
  6067. "cros_img": "octopus-release/R89-13655.0.0",
  6068. "name": "tast.basic OCTOPUS_TOT",
  6069. "suite": "tast.basic",
  6070. "swarming": {},
  6071. "test": "tast.basic",
  6072. "timeout": 3600,
  6073. "variant_id": "OCTOPUS_TOT"
  6074. },
  6075. {
  6076. "args": [],
  6077. "cros_board": "octopus",
  6078. "cros_img": "octopus-release/R88-13597.23.0",
  6079. "name": "tast.basic OCTOPUS_TOT-1",
  6080. "suite": "tast.basic",
  6081. "swarming": {},
  6082. "test": "tast.basic",
  6083. "timeout": 3600,
  6084. "variant_id": "OCTOPUS_TOT-1"
  6085. }
  6086. ]
  6087. }
  6088. }
  6089. """
  6090. ENABLED_AND_DISABLED_VARIATION_SKYLAB_OUTPUT = """\
  6091. {
  6092. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  6093. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  6094. "Fake Tester": {
  6095. "skylab_tests": [
  6096. {
  6097. "args": [],
  6098. "cros_board": "octopus",
  6099. "cros_img": "octopus-release/R89-13655.0.0",
  6100. "name": "tast.basic OCTOPUS_TOT",
  6101. "suite": "tast.basic",
  6102. "swarming": {},
  6103. "test": "tast.basic",
  6104. "timeout": 3600,
  6105. "variant_id": "OCTOPUS_TOT"
  6106. }
  6107. ]
  6108. }
  6109. }
  6110. """
  6111. class MatrixCompositionTests(TestCase):
  6112. def test_good_structure_no_configs(self):
  6113. """
  6114. Tests matrix compound test suite structure with no configs,
  6115. no conflicts and no bad references
  6116. """
  6117. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6118. MATRIX_COMPOUND_EMPTY, LUCI_MILO_CFG)
  6119. self.create_testing_buildbot_json_file('chromium.test.json',
  6120. MATRIX_COMPOUND_OUTPUT)
  6121. self.create_testing_buildbot_json_file('chromium.ci.json',
  6122. MATRIX_COMPOUND_OUTPUT)
  6123. fbb.check_output_file_consistency(verbose=True)
  6124. self.assertFalse(fbb.printed_lines)
  6125. def test_missing_identifier(self):
  6126. """
  6127. Variant is missing an identifier
  6128. """
  6129. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6130. MATRIX_COMPOUND_MISSING_IDENTIFIER, LUCI_MILO_CFG)
  6131. with self.assertRaisesRegex(
  6132. generate_buildbot_json.BBGenErr,
  6133. 'Missing required identifier field in matrix compound suite*'):
  6134. fbb.check_output_file_consistency(verbose=True)
  6135. def test_empty_identifier(self):
  6136. """
  6137. Variant identifier is empty.
  6138. """
  6139. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6140. MATRIX_COMPOUND_EMPTY_IDENTIFIER, LUCI_MILO_CFG)
  6141. with self.assertRaisesRegex(
  6142. generate_buildbot_json.BBGenErr,
  6143. 'Identifier field can not be "" in matrix compound suite*'):
  6144. fbb.check_output_file_consistency(verbose=True)
  6145. def test_trailing_identifier(self):
  6146. """
  6147. Variant identifier has trailing whitespace.
  6148. """
  6149. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6150. MATRIX_COMPOUND_TRAILING_IDENTIFIER, LUCI_MILO_CFG)
  6151. with self.assertRaisesRegex(
  6152. generate_buildbot_json.BBGenErr,
  6153. 'Identifier field can not have leading and trailing whitespace in'
  6154. ' matrix compound suite*'):
  6155. fbb.check_output_file_consistency(verbose=True)
  6156. def test_mismatched_swarming_length(self):
  6157. """
  6158. Swarming dimension set length mismatch test. Composition set > basic set
  6159. """
  6160. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6161. MATRIX_MISMATCHED_SWARMING_LENGTH, LUCI_MILO_CFG)
  6162. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6163. 'Error merging lists by key *'):
  6164. fbb.check_output_file_consistency(verbose=True)
  6165. def test_noexistent_ref(self):
  6166. """
  6167. Test referencing a non-existent basic test suite
  6168. """
  6169. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6170. MATRIX_REF_NONEXISTENT, LUCI_MILO_CFG)
  6171. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6172. 'Unable to find reference to *'):
  6173. fbb.check_output_file_consistency(verbose=True)
  6174. def test_ref_to_composition(self):
  6175. """
  6176. Test referencing another composition test suite
  6177. """
  6178. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6179. MATRIX_COMPOUND_REF_COMPOSITION, LUCI_MILO_CFG)
  6180. with self.assertRaisesRegex(
  6181. generate_buildbot_json.BBGenErr,
  6182. 'matrix_compound_suites may not refer to other *'):
  6183. fbb.check_output_file_consistency(verbose=True)
  6184. def test_ref_to_matrix(self):
  6185. """
  6186. Test referencing another matrix test suite
  6187. """
  6188. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6189. MATRIX_COMPOSITION_REF_MATRIX, LUCI_MILO_CFG)
  6190. with self.assertRaisesRegex(
  6191. generate_buildbot_json.BBGenErr,
  6192. 'matrix_compound_suites may not refer to other *'):
  6193. fbb.check_output_file_consistency(verbose=True)
  6194. def test_conflicting_names(self):
  6195. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6196. MATRIX_COMPOUND_CONFLICTING_TEST_SUITES, LUCI_MILO_CFG)
  6197. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6198. 'Conflicting test definitions.*'):
  6199. fbb.check_input_file_consistency(verbose=True)
  6200. self.assertFalse(fbb.printed_lines)
  6201. def test_variants_swarming_dict_merge_args(self):
  6202. """
  6203. Test targets with swarming dictionary defined by both basic and matrix
  6204. """
  6205. fbb = FakeBBGen(self.args,
  6206. MATRIX_GTEST_SUITE_WATERFALL,
  6207. MATRIX_COMPOUND_TARGETS_ARGS,
  6208. LUCI_MILO_CFG,
  6209. mixins=SWARMING_MIXINS)
  6210. self.create_testing_buildbot_json_file(
  6211. 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS)
  6212. self.create_testing_buildbot_json_file(
  6213. 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS)
  6214. fbb.check_output_file_consistency(verbose=True)
  6215. self.assertFalse(fbb.printed_lines)
  6216. def test_variants_swarming_dict_merge_mixins(self):
  6217. """
  6218. Test targets with swarming dictionary defined by both basic and matrix
  6219. """
  6220. fbb = FakeBBGen(self.args,
  6221. MATRIX_GTEST_SUITE_WATERFALL,
  6222. MATRIX_COMPOUND_TARGETS_MIXINS,
  6223. LUCI_MILO_CFG,
  6224. mixins=SWARMING_MIXINS)
  6225. self.create_testing_buildbot_json_file(
  6226. 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS)
  6227. self.create_testing_buildbot_json_file(
  6228. 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS)
  6229. fbb.check_output_file_consistency(verbose=True)
  6230. self.assertFalse(fbb.printed_lines)
  6231. def test_variants_swarming_dict_swarming(self):
  6232. """
  6233. Test targets with swarming dictionary defined by both basic and matrix
  6234. """
  6235. fbb = FakeBBGen(self.args,
  6236. MATRIX_GTEST_SUITE_WATERFALL,
  6237. MATRIX_COMPOUND_TARGETS_SWARMING,
  6238. LUCI_MILO_CFG,
  6239. mixins=SWARMING_MIXINS)
  6240. self.create_testing_buildbot_json_file(
  6241. 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING)
  6242. self.create_testing_buildbot_json_file(
  6243. 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING)
  6244. fbb.check_output_file_consistency(verbose=True)
  6245. self.assertFalse(fbb.printed_lines)
  6246. def test_variant_test_suite_with_test_key(self):
  6247. """
  6248. Test targets in matrix compound test suites with variants
  6249. """
  6250. fbb = FakeBBGen(self.args,
  6251. MATRIX_GTEST_SUITE_WATERFALL,
  6252. MATRIX_COMPOUND_TEST_WITH_TEST_KEY,
  6253. LUCI_MILO_CFG,
  6254. variants=VARIANTS_FILE)
  6255. self.create_testing_buildbot_json_file(
  6256. 'chromium.test.json', MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT)
  6257. self.create_testing_buildbot_json_file(
  6258. 'chromium.ci.json', MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT)
  6259. fbb.check_output_file_consistency(verbose=True)
  6260. self.assertFalse(fbb.printed_lines)
  6261. def test_variants_pyl_ref(self):
  6262. """Test targets with variants string ref"""
  6263. fbb = FakeBBGen(self.args,
  6264. MATRIX_GTEST_SUITE_WATERFALL,
  6265. MATRIX_COMPOUND_VARIANTS_REF,
  6266. LUCI_MILO_CFG,
  6267. variants=VARIANTS_FILE)
  6268. self.create_testing_buildbot_json_file('chromium.test.json',
  6269. MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
  6270. self.create_testing_buildbot_json_file('chromium.ci.json',
  6271. MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
  6272. fbb.check_output_file_consistency(verbose=True)
  6273. self.assertFalse(fbb.printed_lines)
  6274. def test_variants_pyl_no_ref(self):
  6275. """Test targets with variants string ref, not defined in variants.pyl"""
  6276. fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
  6277. MATRIX_COMPOUND_VARIANTS_REF, LUCI_MILO_CFG)
  6278. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6279. 'Missing variant definition for *'):
  6280. fbb.check_output_file_consistency(verbose=True)
  6281. def test_variants_pyl_all_unreferenced(self):
  6282. """Test targets with variants in variants.pyl, unreferenced in tests"""
  6283. fbb = FakeBBGen(self.args,
  6284. MATRIX_GTEST_SUITE_WATERFALL,
  6285. MATRIX_COMPOUND_MIXED_VARIANTS_REF,
  6286. LUCI_MILO_CFG,
  6287. variants=MULTI_VARIANTS_FILE)
  6288. # self.create_testing_buildbot_json_file(
  6289. # 'chromium.test.json', MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
  6290. with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6291. 'The following variants were unreferenced *'):
  6292. fbb.check_input_file_consistency(verbose=True)
  6293. def test_good_skylab_matrix_with_variants(self):
  6294. fbb = FakeBBGen(self.args,
  6295. MATRIX_SKYLAB_WATERFALL,
  6296. MATRIX_COMPOUND_SKYLAB_REF,
  6297. LUCI_MILO_CFG,
  6298. exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS)
  6299. self.create_testing_buildbot_json_file('chromium.test.json',
  6300. VARIATION_SKYLAB_OUTPUT)
  6301. fbb.check_input_file_consistency(verbose=True)
  6302. fbb.check_output_file_consistency(verbose=True)
  6303. self.assertFalse(fbb.printed_lines)
  6304. def test_enabled_and_disabled_skylab_matrix_with_variants(self):
  6305. """Test with disabled variants"""
  6306. fbb = FakeBBGen(self.args,
  6307. MATRIX_SKYLAB_WATERFALL,
  6308. ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF,
  6309. LUCI_MILO_CFG,
  6310. exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS)
  6311. # some skylab test variant is disabled; the corresponding skylab tests
  6312. # is not generated.
  6313. self.create_testing_buildbot_json_file(
  6314. 'chromium.test.json', ENABLED_AND_DISABLED_VARIATION_SKYLAB_OUTPUT)
  6315. fbb.check_input_file_consistency(verbose=True)
  6316. fbb.check_output_file_consistency(verbose=True)
  6317. self.assertFalse(fbb.printed_lines)
  6318. MAC_TEST_SUITE = """\
  6319. {
  6320. 'basic_suites': {
  6321. 'foo_tests': {
  6322. 'foo_test': {
  6323. },
  6324. },
  6325. },
  6326. }
  6327. """
  6328. MAC_GTESTS_WATERFALL = """\
  6329. [
  6330. {
  6331. 'project': 'chromium',
  6332. 'bucket': 'ci',
  6333. 'name': 'chromium.test',
  6334. 'machines': {
  6335. 'Mac': {
  6336. 'swarming': {
  6337. 'can_use_on_swarming_builders': True,
  6338. },
  6339. 'test_suites': {
  6340. 'gtest_tests': 'foo_tests',
  6341. },
  6342. },
  6343. },
  6344. },
  6345. ]
  6346. """
  6347. MAC_GTEST_WATERFALL_OUTPUT = """\
  6348. {
  6349. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  6350. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  6351. "Mac": {
  6352. "gtest_tests": [
  6353. {
  6354. "merge": {
  6355. "args": [],
  6356. "script": "//testing/merge_scripts/standard_gtest_merge.py"
  6357. },
  6358. "swarming": {
  6359. "can_use_on_swarming_builders": true
  6360. },
  6361. "test": "foo_test"
  6362. }
  6363. ]
  6364. }
  6365. }
  6366. """
  6367. MAC_ISOLATED_SCRIPTS_WATERFALL = """\
  6368. [
  6369. {
  6370. 'project': 'chromium',
  6371. 'bucket': 'ci',
  6372. 'name': 'chromium.test',
  6373. 'machines': {
  6374. 'Mac': {
  6375. 'swarming': {
  6376. 'dimension_sets': [
  6377. {
  6378. 'os': 'Mac',
  6379. },
  6380. ],
  6381. },
  6382. 'test_suites': {
  6383. 'isolated_scripts': 'foo_tests',
  6384. },
  6385. },
  6386. },
  6387. },
  6388. ]
  6389. """
  6390. MAC_ISOLATED_SCRIPTS_WATERFALL_OUTPUT = """\
  6391. {
  6392. "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
  6393. "AAAAA2 See generate_buildbot_json.py to make changes": {},
  6394. "Mac": {
  6395. "isolated_scripts": [
  6396. {
  6397. "isolate_name": "foo_test",
  6398. "merge": {
  6399. "args": [],
  6400. "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
  6401. },
  6402. "name": "foo_test",
  6403. "swarming": {
  6404. "can_use_on_swarming_builders": true,
  6405. "dimension_sets": [
  6406. {
  6407. "os": "Mac"
  6408. }
  6409. ]
  6410. }
  6411. }
  6412. ]
  6413. }
  6414. }
  6415. """
  6416. MAC_LUCI_MILO_CFG = """\
  6417. consoles {
  6418. builders {
  6419. name: "buildbucket/luci.chromium.ci/Mac"
  6420. }
  6421. }
  6422. """
  6423. class SwarmingTests(TestCase):
  6424. def test_mac_builder_with_no_cpu_dimension_in_gtest_fails(self):
  6425. fbb = FakeBBGen(self.args, MAC_GTESTS_WATERFALL, MAC_TEST_SUITE,
  6426. MAC_LUCI_MILO_CFG)
  6427. self.create_testing_buildbot_json_file('chromium.test.json',
  6428. MAC_GTEST_WATERFALL_OUTPUT)
  6429. fbb.check_input_file_consistency(verbose=True)
  6430. self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6431. 'os and cpu',
  6432. fbb.check_output_file_consistency,
  6433. verbose=True)
  6434. self.assertFalse(fbb.printed_lines)
  6435. def test_mac_builder_with_no_cpu_dimension_in_isolated_script_fails(self):
  6436. fbb = FakeBBGen(self.args, MAC_ISOLATED_SCRIPTS_WATERFALL, MAC_TEST_SUITE,
  6437. MAC_LUCI_MILO_CFG)
  6438. self.create_testing_buildbot_json_file(
  6439. 'chromium.test.json', MAC_ISOLATED_SCRIPTS_WATERFALL_OUTPUT)
  6440. fbb.check_input_file_consistency(verbose=True)
  6441. self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
  6442. 'os and cpu',
  6443. fbb.check_output_file_consistency,
  6444. verbose=True)
  6445. self.assertFalse(fbb.printed_lines)
  6446. if __name__ == '__main__':
  6447. unittest.main()