12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830 |
- #!/usr/bin/env vpython3
- # Copyright 2017 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Tests for generate_buildbot_json.py."""
- import argparse
- import contextlib
- import json
- import os
- import unittest
- import generate_buildbot_json
- from pyfakefs import fake_filesystem_unittest
- # pylint: disable=super-with-arguments
- EMPTY_PYL_FILE = """\
- {
- }
- """
- # Use this value to refer to the directory containing this code
- # The tests use a fake filesystem and python filesystem calls are monkey-patched
- # to use the fake filesystem, which affects abspath
- THIS_DIR = os.path.dirname(os.path.abspath(__file__))
- class TestCase(fake_filesystem_unittest.TestCase):
- def setUp(self):
- self.setUpPyfakefs()
- self.fs.cwd = THIS_DIR
- self.args = generate_buildbot_json.BBJSONGenerator.parse_args([])
- def override_args(self, **kwargs):
- for k, v in kwargs.items():
- setattr(self.args, k, v)
- def create_testing_buildbot_json_file(self, path, contents):
- return self.fs.create_file(os.path.join(THIS_DIR, path), contents=contents)
- @contextlib.contextmanager
- def dump_on_failure(fbb, dump=True):
- try:
- yield
- except:
- if dump:
- for l in fbb.printed_lines:
- print(l)
- raise
- class FakeBBGen(generate_buildbot_json.BBJSONGenerator):
- def __init__(self,
- args,
- waterfalls,
- test_suites,
- luci_milo_cfg,
- project_pyl='{"validate_source_side_specs_have_builder": True}',
- exceptions=EMPTY_PYL_FILE,
- mixins=EMPTY_PYL_FILE,
- gn_isolate_map=EMPTY_PYL_FILE,
- variants=EMPTY_PYL_FILE):
- super(FakeBBGen, self).__init__(args)
- pyl_files_dir = args.pyl_files_dir or THIS_DIR
- infra_config_dir = args.infra_config_dir
- files = {
- (pyl_files_dir, 'waterfalls.pyl'): waterfalls,
- (pyl_files_dir, 'test_suites.pyl'): test_suites,
- (pyl_files_dir, 'test_suite_exceptions.pyl'): exceptions,
- (pyl_files_dir, 'mixins.pyl'): mixins,
- (pyl_files_dir, 'gn_isolate_map.pyl'): gn_isolate_map,
- (pyl_files_dir, 'gn_isolate_map2.pyl'): GPU_TELEMETRY_GN_ISOLATE_MAP,
- (pyl_files_dir, 'variants.pyl'): variants,
- (infra_config_dir, 'generated/project.pyl'): project_pyl,
- (infra_config_dir, 'generated/luci/luci-milo.cfg'): luci_milo_cfg,
- (infra_config_dir, 'generated/luci/luci-milo-dev.cfg'): '',
- }
- for (d, filename), content in files.items():
- if content is None:
- continue
- path = os.path.join(d, filename)
- parent = os.path.abspath(os.path.dirname(path))
- if not os.path.exists(parent):
- os.makedirs(parent)
- with open(path, 'w') as f:
- f.write(content)
- self.printed_lines = []
- def print_line(self, line):
- self.printed_lines.append(line)
- # pragma pylint: disable=arguments-differ
- def check_output_file_consistency(self, verbose=False, dump=True):
- with dump_on_failure(self, dump=verbose and dump):
- super(FakeBBGen, self).check_output_file_consistency(verbose)
- # pragma pylint: enable=arguments-differ
- FOO_GTESTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1',
- },
- ],
- },
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- 'args': [
- '--enable-features=Baz',
- ],
- },
- },
- },
- ]
- """
- FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- "device_type": "foo_device",
- },
- ],
- },
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- 'os_type': 'chromeos',
- },
- },
- },
- ]
- """
- FOO_LINUX_GTESTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'linux',
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- COMPOSITION_GTEST_SUITE_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'composition_tests',
- },
- 'args': [
- '--this-is-an-argument',
- ],
- },
- },
- },
- ]
- """
- FOO_ISOLATED_SCRIPTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'isolated_scripts': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'android',
- 'test_suites': {
- 'isolated_scripts': 'composition_tests',
- },
- 'use_android_presentation': True,
- },
- },
- },
- ]
- """
- FOO_SCRIPT_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'scripts': 'foo_scripts',
- },
- },
- },
- },
- ]
- """
- FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'forbid_script_tests': True,
- 'test_suites': {
- 'scripts': 'foo_scripts',
- },
- },
- },
- },
- ]
- """
- FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'forbid_script_tests': True,
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'scripts': 'foo_scripts',
- },
- },
- },
- },
- ]
- """
- FOO_JUNIT_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'junit_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'win',
- 'browser_config': 'release',
- 'swarming': {
- 'dimension_sets': [
- {
- 'gpu': '10de:1cb3',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'android',
- 'browser_config': 'android-chromium',
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_type': 'bullhead',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'android',
- 'browser_config': 'not-a-real-browser',
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_type': 'bullhead',
- },
- ],
- },
- 'test_suites': {
- 'android_webview_gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'fuchsia',
- 'browser_config': 'fuchsia-chrome',
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'fuchsia',
- 'browser_config': 'not-a-real-browser',
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1',
- },
- ],
- },
- 'test_suites': {
- 'cast_streaming_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- NVIDIA_GPU_TELEMETRY_TEST_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'win',
- 'browser_config': 'release',
- 'swarming': {
- 'dimension_sets': [
- {
- 'gpu': '10de:1cb3-26.21.14.3102',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- INTEL_GPU_TELEMETRY_TEST_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'win',
- 'browser_config': 'release',
- 'swarming': {
- 'dimension_sets': [
- {
- 'gpu': '8086:5912-24.20.100.6286',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'win',
- 'browser_config': 'release',
- 'swarming': {
- 'dimension_sets': [
- {
- 'gpu': '8086:3e92-24.20.100.6286',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'composition_tests',
- },
- },
- },
- },
- ]
- """
- GPU_TELEMETRY_TEST_VARIANTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'win',
- 'browser_config': 'release',
- 'swarming': {
- 'dimension_sets': [
- {
- 'gpu': '8086:3e92-24.20.100.6286',
- },
- ],
- },
- 'test_suites': {
- 'gpu_telemetry_tests': 'matrix_tests',
- },
- },
- },
- },
- ]
- """
- UNKNOWN_TEST_SUITE_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'baz_tests',
- },
- },
- },
- },
- ]
- """
- UNKNOWN_TEST_SUITE_TYPE_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- 'foo_test_type': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- ANDROID_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Android Builder': {
- 'additional_compile_targets': [
- 'bar_test',
- ],
- },
- 'Fake Android K Tester': {
- 'additional_compile_targets': [
- 'bar_test',
- ],
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_os': 'KTU84P',
- 'device_type': 'hammerhead',
- 'os': 'Android',
- },
- ],
- },
- 'os_type': 'android',
- 'skip_merge_script': True,
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- 'Fake Android L Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_os': 'LMY41U',
- 'device_os_type': 'user',
- 'device_type': 'hammerhead',
- 'os': 'Android',
- },
- ],
- },
- 'os_type': 'android',
- 'skip_merge_script': True,
- 'skip_output_links': True,
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- 'Fake Android M Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_os': 'MMB29Q',
- 'device_type': 'bullhead',
- 'os': 'Android',
- },
- ],
- },
- 'os_type': 'android',
- 'use_swarming': False,
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- UNKNOWN_BOT_GTESTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Unknown Bot': {
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- MATRIX_GTEST_SUITE_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'matrix_tests',
- },
- },
- },
- },
- ]
- """
- MATRIX_GTEST_SUITE_WATERFALL_MIXINS = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['dimension_mixin'],
- 'test_suites': {
- 'gtest_tests': 'matrix_tests',
- },
- },
- },
- },
- ]
- """
- FOO_TEST_SUITE = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': 'high',
- }
- ],
- 'expiration': 120,
- },
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_NO_DIMENSIONS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_NOT_SORTED = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- 'a_test': {},
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'args': [
- '--c_arg',
- ],
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'swarming': {
- 'named_caches': [
- {
- 'name': 'cache_in_test',
- 'file': 'cache_in_test_file',
- },
- ],
- },
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_LINUX_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'linux_args': [
- '--no-xvfb',
- ],
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'args': [
- '--enable-features=Foo,Bar',
- ],
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'remove_mixins': ['waterfall_mixin'],
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': 'high',
- }
- ],
- 'expiration': 120,
- },
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'remove_mixins': ['builder_mixin'],
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': 'high',
- }
- ],
- 'expiration': 120,
- },
- },
- },
- },
- }
- """
- FOO_SCRIPT_SUITE = """\
- {
- 'basic_suites': {
- 'foo_scripts': {
- 'foo_test': {
- 'script': 'foo.py',
- },
- 'bar_test': {
- 'script': 'bar.py',
- },
- },
- },
- }
- """
- GOOD_COMPOSITION_TEST_SUITES = """\
- {
- 'basic_suites': {
- 'bar_tests': {
- 'bar_test': {},
- },
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'compound_suites': {
- 'composition_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- BAD_COMPOSITION_TEST_SUITES = """\
- {
- 'basic_suites': {
- 'bar_tests': {},
- 'foo_tests': {},
- },
- 'compound_suites': {
- 'buggy_composition_tests': [
- 'bar_tests',
- ],
- 'composition_tests': [
- 'foo_tests',
- 'buggy_composition_tests',
- ],
- },
- }
- """
- CONFLICTING_COMPOSITION_TEST_SUITES = """\
- {
- 'basic_suites': {
- 'bar_tests': {
- 'baz_tests': {
- 'args': [
- '--bar',
- ],
- }
- },
- 'foo_tests': {
- 'baz_tests': {
- 'args': [
- '--foo',
- ],
- }
- },
- },
- 'compound_suites': {
- 'foobar_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- DUPLICATES_COMPOSITION_TEST_SUITES = """\
- {
- 'basic_suites': {
- 'bar_tests': {},
- 'buggy_composition_tests': {},
- 'foo_tests': {},
- },
- 'compound_suites': {
- 'bar_tests': [
- 'foo_tests',
- ],
- 'composition_tests': [
- 'foo_tests',
- 'buggy_composition_tests',
- ],
- },
- }
- """
- SCRIPT_SUITE = """\
- {
- 'basic_suites': {
- 'foo_scripts': {
- 'foo_test': {
- 'script': 'foo.py',
- },
- },
- },
- }
- """
- UNREFED_TEST_SUITE = """\
- {
- 'basic_suites': {
- 'bar_tests': {},
- 'foo_tests': {},
- },
- }
- """
- REUSING_TEST_WITH_DIFFERENT_NAME = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- 'variation_test': {
- 'args': [
- '--variation',
- ],
- 'test': 'foo_test',
- },
- },
- },
- }
- """
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo': {},
- },
- 'bar_tests': {
- 'bar_test': {},
- },
- },
- 'compound_suites': {
- 'composition_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- COMPOSITION_SUITE_WITH_GPU_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo': {
- 'args': [
- '--gpu-vendor-id',
- '${gpu_vendor_id}',
- '--gpu-device-id',
- '${gpu_device_id}',
- ],
- },
- },
- 'bar_tests': {
- 'bar_test': {},
- },
- },
- 'compound_suites': {
- 'composition_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- COMPOSITION_SUITE_WITH_PIXEL_AND_FILTER = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'pixel': {
- 'args': [
- '--git-revision=aaaa',
- '--test-filter=bar',
- ],
- },
- },
- 'bar_tests': {
- 'bar_test': {},
- },
- },
- 'compound_suites': {
- 'composition_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- GTEST_AS_ISOLATED_SCRIPT_SUITE = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'script': 'foo.py',
- 'use_isolated_scripts_api': True,
- },
- },
- },
- }
- """
- SCRIPT_WITH_ARGS_EXCEPTIONS = """\
- {
- 'foo_test': {
- 'modifications': {
- 'Fake Tester': {
- 'args': ['--fake-arg'],
- },
- },
- },
- }
- """
- SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS = """\
- {
- 'foo_test': {
- 'modifications': {
- 'Fake Tester': {
- 'swarming': {
- 'value': 'exception',
- },
- },
- },
- },
- }
- """
- NO_BAR_TEST_EXCEPTIONS = """\
- {
- 'bar_test': {
- 'remove_from': [
- 'Fake Tester',
- ]
- }
- }
- """
- EMPTY_BAR_TEST_EXCEPTIONS = """\
- {
- 'bar_test': {
- }
- }
- """
- EXCEPTIONS_SORTED = """\
- {
- 'suite_c': {
- 'modifications': {
- 'Fake Tester': {
- 'foo': 'bar',
- },
- },
- },
- 'suite_d': {
- 'modifications': {
- 'Fake Tester': {
- 'foo': 'baz',
- },
- },
- },
- }
- """
- EXCEPTIONS_UNSORTED = """\
- {
- 'suite_d': {
- 'modifications': {
- 'Fake Tester': {
- 'foo': 'baz',
- },
- },
- },
- 'suite_c': {
- 'modifications': {
- 'Fake Tester': {
- 'foo': 'bar',
- },
- },
- },
- }
- """
- EXCEPTIONS_PER_TEST_UNSORTED = """\
- {
- 'suite_d': {
- 'modifications': {
- 'Other Tester': {
- 'foo': 'baz',
- },
- 'Fake Tester': {
- 'foo': 'baz',
- },
- },
- },
- }
- """
- EXCEPTIONS_DUPS_REMOVE_FROM = """\
- {
- 'suite_d': {
- 'remove_from': [
- 'Fake Tester',
- 'Fake Tester',
- ],
- 'modifications': {
- 'Fake Tester': {
- 'foo': 'baz',
- },
- },
- },
- }
- """
- FOO_TEST_MODIFICATIONS = """\
- {
- 'foo_test': {
- 'modifications': {
- 'Fake Tester': {
- 'args': [
- '--bar',
- ],
- 'swarming': {
- 'hard_timeout': 600,
- },
- },
- },
- }
- }
- """
- FOO_TEST_EXPLICIT_NONE_EXCEPTIONS = """\
- {
- 'foo_test': {
- 'modifications': {
- 'Fake Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': None,
- },
- ],
- },
- },
- },
- },
- }
- """
- NONEXISTENT_REMOVAL = """\
- {
- 'foo_test': {
- 'remove_from': [
- 'Nonexistent Tester',
- ]
- }
- }
- """
- NONEXISTENT_MODIFICATION = """\
- {
- 'foo_test': {
- 'modifications': {
- 'Nonexistent Tester': {
- 'args': [],
- },
- },
- }
- }
- """
- COMPOSITION_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "bar_test"
- },
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--this-is-an-argument"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "bar_test"
- },
- {
- "args": [
- "--this-is-an-argument"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- VARIATION_GTEST_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test",
- "test_id_prefix": "ninja://chrome/test:foo_test/"
- },
- {
- "args": [
- "--variation"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "variation_test",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test",
- "test_id_prefix": "ninja://chrome/test:foo_test/"
- }
- ]
- }
- }
- """
- FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "script": "foo.py",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test",
- "test_id_prefix": "ninja://chrome/test:foo_test/",
- "use_isolated_scripts_api": true
- }
- ]
- }
- }
- """
- COMPOSITION_WATERFALL_FILTERED_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- MERGED_ARGS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--c_arg",
- "--bar"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ],
- "hard_timeout": 600
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- LINUX_ARGS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--no-xvfb"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- MERGED_ENABLE_FEATURES_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--enable-features=Foo,Bar,Baz"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- MODIFIED_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--bar"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high",
- "kvm": "1"
- }
- ],
- "expiration": 120,
- "hard_timeout": 600
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- EXPLICIT_NONE_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- ISOLATED_SCRIPT_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "isolate_name": "foo_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_test",
- "swarming": {
- "can_use_on_swarming_builders": true
- }
- }
- ]
- }
- }
- """
- ISOLATED_SCRIPT_OUTPUT_ANDROID = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "--gs-results-bucket=chromium-result-details"
- ],
- "isolate_name": "foo_test",
- "merge": {
- "args": [
- "--bucket",
- "chromium-result-details",
- "--test-name",
- "foo_test"
- ],
- "script": \
- "//build/android/pylib/results/presentation/test_results_presentation.py"
- },
- "name": "foo_test",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": \
- "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "output_links": [
- {
- "link": [
- "https://luci-logdog.appspot.com/v/?s",
- "=android%2Fswarming%2Flogcats%2F",
- "${TASK_ID}%2F%2B%2Funified_logcats"
- ],
- "name": "shard #${SHARD_INDEX} logcats"
- }
- ]
- }
- }
- ]
- }
- }
- """
- SCRIPT_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "scripts": [
- {
- "name": "foo_test",
- "script": "foo.py"
- }
- ]
- }
- }
- """
- SCRIPT_WITH_ARGS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "scripts": [
- {
- "args": [
- "--fake-arg"
- ],
- "name": "foo_test",
- "script": "foo.py"
- }
- ]
- }
- }
- """
- JUNIT_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "junit_tests": [
- {
- "name": "foo_test",
- "test": "foo_test"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "10de:1cb3"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_OUTPUT_ANDROID = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=android-chromium",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test_android_chrome",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_type": "bullhead"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_android_chrome/"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=android-webview-instrumentation",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test_android_webview",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_type": "bullhead"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_android_webview/"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=fuchsia-chrome",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test_fuchsia",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_fuchsia/"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=cast-streaming-shell",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test_fuchsia",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test_fuchsia/"
- }
- ]
- }
- }
- """
- NVIDIA_GPU_TELEMETRY_TEST_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
- "--gpu-vendor-id",
- "10de",
- "--gpu-device-id",
- "1cb3"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "10de:1cb3-26.21.14.3102"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
- }
- ]
- }
- }
- """
- INTEL_GPU_TELEMETRY_TEST_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
- "--gpu-vendor-id",
- "8086",
- "--gpu-device-id",
- "5912"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:5912-24.20.100.6286"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
- }
- ]
- }
- }
- """
- INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
- "--gpu-vendor-id",
- "8086",
- "--gpu-device-id",
- "3e92"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_tests",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:3e92-24.20.100.6286"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
- }
- ]
- }
- }
- """
- GPU_TELEMETRY_TEST_VARIANTS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "swarming_test",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
- "--platform",
- "device",
- "--version",
- "1"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "swarming_test a_variant",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:3e92-24.20.100.6286"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/",
- "variant_id": "a_variant"
- },
- {
- "args": [
- "swarming_test",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc",
- "a",
- "b"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "swarming_test ab",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "gpu": "8086:3e92-24.20.100.6286"
- }
- ],
- "idempotent": false
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/",
- "variant_id": "ab"
- }
- ]
- }
- }
- """
- ANDROID_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Android Builder": {
- "additional_compile_targets": [
- "bar_test"
- ]
- },
- "Fake Android K Tester": {
- "additional_compile_targets": [
- "bar_test"
- ],
- "gtest_tests": [
- {
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": \
- "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "KTU84P",
- "device_os_type": "userdebug",
- "device_type": "hammerhead",
- "integrity": "high",
- "os": "Android"
- }
- ],
- "expiration": 120,
- "output_links": [
- {
- "link": [
- "https://luci-logdog.appspot.com/v/?s",
- "=android%2Fswarming%2Flogcats%2F",
- "${TASK_ID}%2F%2B%2Funified_logcats"
- ],
- "name": "shard #${SHARD_INDEX} logcats"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- },
- "Fake Android L Tester": {
- "gtest_tests": [
- {
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": \
- "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "LMY41U",
- "device_os_type": "user",
- "device_type": "hammerhead",
- "integrity": "high",
- "os": "Android"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test"
- }
- ]
- },
- "Fake Android M Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": false
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- CHROMEOS_TRIGGER_SCRIPT_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "device_type": "foo_device",
- "integrity": "high"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test",
- "trigger_script": {
- "script": "//testing/trigger_scripts/chromeos_device_trigger.py"
- }
- }
- ]
- }
- }
- """
- GPU_DIMENSIONS_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "isolated_scripts": [
- {
- "args": [
- "foo_test",
- "--show-stdout",
- "--browser=release",
- "--passthrough",
- "-v",
- "--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
- ],
- "isolate_name": "telemetry_gpu_integration_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_test",
- "should_retry_with_patch": false,
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "iama": "mixin",
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "idempotent": false,
- "value": "test"
- },
- "test_id_prefix": "ninja://chrome/test:telemetry_gpu_integration_test/"
- }
- ]
- }
- }
- """
- LUCI_MILO_CFG = """\
- consoles {
- builders {
- name: "buildbucket/luci.chromium.ci/Fake Tester"
- }
- }
- """
- LUCI_MILO_CFG_WATERFALL_SORTING = """\
- consoles {
- builders {
- name: "buildbucket/luci.chromium.ci/Fake Tester"
- name: "buildbucket/luci.chromium.ci/Really Fake Tester"
- }
- }
- """
- TEST_SUITE_SORTING_WATERFALL = """
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- ]
- """
- TEST_SUITE_SORTED_WATERFALL = """
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.zz.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- ]
- """
- TEST_SUITE_UNSORTED_WATERFALL_1 = """
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.zz.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- ]
- """
- TEST_SUITE_UNSORTED_WATERFALL_2 = """
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.zz.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- 'Really Fake Tester': {
- 'test_suites': {
- 'gtest_tests': 'suite_a',
- 'scripts': 'suite_b',
- },
- },
- },
- },
- ]
- """
- # Note that the suites in basic_suites would be sorted after the suites in
- # compound_suites. This is valid though, because each set of suites is sorted
- # separately.
- # suite_c is a 'gtest_tests' test
- # suite_d is a 'scripts' test
- TEST_SUITE_SORTED = """\
- {
- 'basic_suites': {
- 'suite_c': {
- 'suite_c': {},
- },
- 'suite_d': {
- 'script': {
- 'script': 'suite_d.py',
- }
- },
- },
- 'compound_suites': {
- 'suite_a': [
- 'suite_c',
- ],
- 'suite_b': [
- 'suite_d',
- ],
- },
- }
- """
- TEST_SUITE_UNSORTED_1 = """\
- {
- 'basic_suites': {
- 'suite_d': {
- 'a': 'b',
- },
- 'suite_c': {
- 'a': 'b',
- },
- },
- 'compound_suites': {
- 'suite_a': [
- 'suite_c',
- ],
- 'suite_b': [
- 'suite_d',
- ],
- },
- }
- """
- TEST_SUITE_UNSORTED_2 = """\
- {
- 'basic_suites': {
- 'suite_c': {
- 'a': 'b',
- },
- 'suite_d': {
- 'a': 'b',
- },
- },
- 'compound_suites': {
- 'suite_b': [
- 'suite_c',
- ],
- 'suite_a': [
- 'suite_d',
- ],
- },
- }
- """
- TEST_SUITE_UNSORTED_3 = """\
- {
- 'basic_suites': {
- 'suite_d': {
- 'a': 'b',
- },
- 'suite_c': {
- 'a': 'b',
- },
- },
- 'compound_suites': {
- 'suite_b': [
- 'suite_c',
- ],
- 'suite_a': [
- 'suite_d',
- ],
- },
- }
- """
- TEST_SUITES_SYNTAX_ERROR = """\
- {
- 'basic_suites': {
- 3: {
- 'suite_c': {},
- },
- },
- 'compound_suites': {},
- }
- """
- GN_ISOLATE_MAP="""\
- {
- 'foo_test': {
- 'label': '//chrome/test:foo_test',
- 'type': 'windowed_test_launcher',
- }
- }
- """
- GPU_TELEMETRY_GN_ISOLATE_MAP="""\
- {
- 'telemetry_gpu_integration_test': {
- 'label': '//chrome/test:telemetry_gpu_integration_test',
- 'type': 'script',
- }
- }
- """
- GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID = """\
- {
- 'telemetry_gpu_integration_test_android_chrome': {
- 'label': '//chrome/test:telemetry_gpu_integration_test_android_chrome',
- 'type': 'script',
- }
- }
- """
- GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW = """\
- {
- 'telemetry_gpu_integration_test_android_webview': {
- 'label': '//chrome/test:telemetry_gpu_integration_test_android_webview',
- 'type': 'script',
- }
- }
- """
- GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA = """\
- {
- 'telemetry_gpu_integration_test_fuchsia': {
- 'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
- 'type': 'script',
- }
- }
- """
- GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING = """\
- {
- 'telemetry_gpu_integration_test_fuchsia': {
- 'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
- 'type': 'script',
- }
- }
- """
- GN_ISOLATE_MAP_KEY_LABEL_MISMATCH="""\
- {
- 'foo_test': {
- 'label': '//chrome/test:foo_test_tmp',
- 'type': 'windowed_test_launcher',
- }
- }
- """
- GN_ISOLATE_MAP_USING_IMPLICIT_NAME="""\
- {
- 'foo_test': {
- 'label': '//chrome/foo_test',
- 'type': 'windowed_test_launcher',
- }
- }
- """
- class UnitTest(TestCase):
- def test_base_generator(self):
- # Only needed for complete code coverage.
- self.assertRaises(NotImplementedError,
- generate_buildbot_json.BaseGenerator(None).generate,
- None, None, None, None)
- self.assertRaises(NotImplementedError,
- generate_buildbot_json.BaseGenerator(None).sort,
- None)
- def test_good_test_suites_are_ok(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE,
- LUCI_MILO_CFG)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_good_composition_test_suites_are_ok(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_bad_composition_test_suites_are_caught(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
- BAD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'compound_suites may not refer to.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_composition_test_suites_no_conflicts(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
- CONFLICTING_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Conflicting test definitions.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_composition_test_suites_no_duplicate_names(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
- DUPLICATES_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- '.*may not duplicate basic test suite.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_unknown_test_suites_are_caught(self):
- fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_WATERFALL, FOO_TEST_SUITE,
- LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Test suite baz_tests from machine.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_unknown_test_suite_types_are_caught(self):
- fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_TYPE_WATERFALL,
- FOO_TEST_SUITE, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Unknown test suite type foo_test_type.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_unrefed_test_suite_caught(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, UNREFED_TEST_SUITE,
- LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- '.*unreferenced.*bar_tests.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_good_waterfall_output(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- COMPOSITION_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_reusing_gtest_targets(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- REUSING_TEST_WITH_DIFFERENT_NAME,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- VARIATION_GTEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_load_multiple_isolate_map_files_with_duplicates(self):
- self.args.isolate_map_files = ['gn_isolate_map.pyl']
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- REUSING_TEST_WITH_DIFFERENT_NAME,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Duplicate targets in isolate map files.*'):
- fbb.load_configuration_files()
- def test_load_multiple_isolate_map_files_without_duplicates(self):
- self.args.isolate_map_files = ['gn_isolate_map2.pyl']
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- REUSING_TEST_WITH_DIFFERENT_NAME,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP)
- fbb.load_configuration_files()
- isolate_dict = {}
- isolate_map_1 = fbb.load_pyl_file('gn_isolate_map.pyl')
- isolate_map_2 = fbb.load_pyl_file('gn_isolate_map2.pyl')
- isolate_dict.update(isolate_map_1)
- isolate_dict.update(isolate_map_2)
- self.assertEqual(isolate_dict, fbb.gn_isolate_map)
- def test_gn_isolate_map_with_label_mismatch(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP_KEY_LABEL_MISMATCH)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'key name.*foo_test.*label.*'
- 'foo_test_tmp.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gn_isolate_map_using_implicit_gn_name(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP_USING_IMPLICIT_NAME)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Malformed.*//chrome/foo_test.*for key.*'
- 'foo_test.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_noop_exception_does_nothing(self):
- fbb = FakeBBGen(self.args,
- COMPOSITION_GTEST_SUITE_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=EMPTY_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- COMPOSITION_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_test_arg_merges(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_MODIFICATIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MERGED_ARGS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_enable_features_arg_merges(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL,
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MERGED_ENABLE_FEATURES_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_linux_args(self):
- fbb = FakeBBGen(self.args, FOO_LINUX_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_LINUX_ARGS, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- LINUX_ARGS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_test_filtering(self):
- fbb = FakeBBGen(self.args,
- COMPOSITION_GTEST_SUITE_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', COMPOSITION_WATERFALL_FILTERED_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_test_modifications(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_MODIFICATIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MODIFIED_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json', MODIFIED_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_test_with_explicit_none(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_EXPLICIT_NONE_EXCEPTIONS,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- EXPLICIT_NONE_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- EXPLICIT_NONE_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_isolated_script_tests(self):
- fbb = FakeBBGen(self.args,
- FOO_ISOLATED_SCRIPTS_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- ISOLATED_SCRIPT_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- ISOLATED_SCRIPT_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_isolated_script_tests(self):
- fbb = FakeBBGen(self.args,
- FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- ISOLATED_SCRIPT_OUTPUT_ANDROID)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- ISOLATED_SCRIPT_OUTPUT_ANDROID)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_script_with_args(self):
- fbb = FakeBBGen(self.args,
- FOO_SCRIPT_WATERFALL,
- SCRIPT_SUITE,
- LUCI_MILO_CFG,
- exceptions=SCRIPT_WITH_ARGS_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- SCRIPT_WITH_ARGS_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- SCRIPT_WITH_ARGS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_script(self):
- fbb = FakeBBGen(self.args,
- FOO_SCRIPT_WATERFALL,
- FOO_SCRIPT_SUITE,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json', SCRIPT_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json', SCRIPT_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_script_machine_forbids_scripts(self):
- fbb = FakeBBGen(self.args,
- FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS,
- FOO_SCRIPT_SUITE,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Attempted to generate a script test on tester.*'):
- fbb.check_output_file_consistency(verbose=True)
- def test_script_waterfall_forbids_scripts(self):
- fbb = FakeBBGen(self.args,
- FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS,
- FOO_SCRIPT_SUITE,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Attempted to generate a script test on tester.*'):
- fbb.check_output_file_consistency(verbose=True)
- def test_junit_tests(self):
- fbb = FakeBBGen(self.args,
- FOO_JUNIT_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json', JUNIT_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json', JUNIT_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL,
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_TELEMETRY_TEST_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- GPU_TELEMETRY_TEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests_pixel_with_filter(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL,
- COMPOSITION_SUITE_WITH_PIXEL_AND_FILTER,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_TELEMETRY_TEST_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- GPU_TELEMETRY_TEST_OUTPUT)
- with self.assertRaises(RuntimeError):
- fbb.check_output_file_consistency(verbose=True)
- def test_gpu_telemetry_tests_android(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID,
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_TELEMETRY_TEST_OUTPUT_ANDROID)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- GPU_TELEMETRY_TEST_OUTPUT_ANDROID)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests_android_webview(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW,
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', GPU_TELEMETRY_TEST_OUTPUT_ANDROID_WEBVIEW)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests_fuchsia(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA,
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- GPU_TELEMETRY_TEST_OUTPUT_FUCHSIA)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests_cast_streaming(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING,
- COMPOSITION_SUITE_WITH_NAME_NOT_ENDING_IN_TEST,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', GPU_TELEMETRY_TEST_OUTPUT_CAST_STREAMING)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nvidia_gpu_telemetry_tests(self):
- fbb = FakeBBGen(self.args,
- NVIDIA_GPU_TELEMETRY_TEST_WATERFALL,
- COMPOSITION_SUITE_WITH_GPU_ARGS,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- NVIDIA_GPU_TELEMETRY_TEST_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- NVIDIA_GPU_TELEMETRY_TEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_intel_gpu_telemetry_tests(self):
- fbb = FakeBBGen(self.args,
- INTEL_GPU_TELEMETRY_TEST_WATERFALL,
- COMPOSITION_SUITE_WITH_GPU_ARGS,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- INTEL_GPU_TELEMETRY_TEST_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- INTEL_GPU_TELEMETRY_TEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gpu_telemetry_tests_with_variants(self):
- fbb = FakeBBGen(self.args,
- GPU_TELEMETRY_TEST_VARIANTS_WATERFALL,
- MATRIX_COMPOUND_MIXED_VARIANTS_REF,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP,
- variants=MULTI_VARIANTS_FILE)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_TELEMETRY_TEST_VARIANTS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_intel_uhd_gpu_telemetry_tests(self):
- fbb = FakeBBGen(self.args,
- INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL,
- COMPOSITION_SUITE_WITH_GPU_ARGS,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- INTEL_UHD_GPU_TELEMETRY_TEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_gtest_as_isolated_Script(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- GTEST_AS_ISOLATED_SCRIPT_SUITE,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', FOO_WATERFALL_GTEST_ISOLATED_SCRIPT_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_ungenerated_output_files_are_caught(self):
- fbb = FakeBBGen(self.args,
- COMPOSITION_GTEST_SUITE_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- exceptions=NO_BAR_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', '\n' + COMPOSITION_WATERFALL_FILTERED_OUTPUT)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_output_file_consistency(verbose=True, dump=False)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(
- joined_lines, 'File chromium.test.json did not have the following'
- ' expected contents:.*')
- self.assertRegex(joined_lines, '.*--- expected.*')
- self.assertRegex(joined_lines, '.*\+\+\+ current.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_android_output_options(self):
- fbb = FakeBBGen(self.args, ANDROID_WATERFALL, FOO_TEST_SUITE, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- ANDROID_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- ANDROID_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nonexistent_removal_raises(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=NONEXISTENT_REMOVAL)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'The following nonexistent machines.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nonexistent_modification_raises(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=NONEXISTENT_MODIFICATION)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'The following nonexistent machines.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_waterfall_args(self):
- fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', COMPOSITION_WATERFALL_WITH_ARGS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_chromeos_trigger_script_output(self):
- fbb = FakeBBGen(self.args, FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL,
- FOO_TEST_SUITE, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- CHROMEOS_TRIGGER_SCRIPT_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- CHROMEOS_TRIGGER_SCRIPT_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_relative_pyl_file_dir(self):
- self.override_args(pyl_files_dir='relative/path/', waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- REUSING_TEST_WITH_DIFFERENT_NAME,
- LUCI_MILO_CFG,
- gn_isolate_map=GN_ISOLATE_MAP)
- fbb.check_input_file_consistency(verbose=True)
- self.create_testing_buildbot_json_file('relative/path/chromium.test.json',
- VARIATION_GTEST_OUTPUT)
- self.create_testing_buildbot_json_file('relative/path/chromium.ci.json',
- VARIATION_GTEST_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nonexistent_bot_raises(self):
- fbb = FakeBBGen(self.args, UNKNOWN_BOT_GTESTS_WATERFALL, FOO_TEST_SUITE,
- LUCI_MILO_CFG)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nonexistent_bot_raises_when_no_project_pyl_exists(self):
- fbb = FakeBBGen(self.args,
- UNKNOWN_BOT_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- project_pyl=None)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_nonexistent_bot_does_not_raise_when_validation_disabled(self):
- fbb = FakeBBGen(
- self.args,
- UNKNOWN_BOT_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- project_pyl='{"validate_source_side_specs_have_builder": False}')
- fbb.check_input_file_consistency(verbose=True)
- def test_waterfalls_must_be_sorted(self):
- fbb = FakeBBGen(self.args, TEST_SUITE_SORTED_WATERFALL, TEST_SUITE_SORTED,
- LUCI_MILO_CFG_WATERFALL_SORTING)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_1,
- TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'The following files have invalid keys: waterfalls.pyl'):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = '\n'.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+ chromium\..*test.*')
- self.assertRegex(joined_lines, '.*\- chromium\..*test.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_2,
- TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'The following files have invalid keys: waterfalls.pyl'):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+.*Fake Tester.*')
- self.assertRegex(joined_lines, '.*\-.*Fake Tester.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_test_suite_exceptions_must_be_sorted(self):
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_SORTED)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_DUPS_REMOVE_FROM)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\- Fake Tester.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_test_suite_exceptions_no_dups_remove_from(self):
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_SORTED)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_PER_TEST_UNSORTED)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+ Fake Tester.*')
- self.assertRegex(joined_lines, '.*\- Fake Tester.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_test_suite_exceptions_per_test_must_be_sorted(self):
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_SORTED)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args,
- TEST_SUITE_SORTING_WATERFALL,
- TEST_SUITE_SORTED,
- LUCI_MILO_CFG,
- exceptions=EXCEPTIONS_UNSORTED)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+ suite_.*')
- self.assertRegex(joined_lines, '.*\- suite_.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_test_suites_must_be_sorted(self):
- fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, TEST_SUITE_SORTED,
- LUCI_MILO_CFG)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- for unsorted in (
- TEST_SUITE_UNSORTED_1,
- TEST_SUITE_UNSORTED_2,
- TEST_SUITE_UNSORTED_3,
- ):
- fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, unsorted,
- LUCI_MILO_CFG)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+ suite_.*')
- self.assertRegex(joined_lines, '.*\- suite_.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
- [
- {
- 'mixins': ['waterfall_mixin'],
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['builder_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'os_type': 'linux',
- 'mixins': ['builder_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_DIMENSION_SETS_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': [
- 'dimension_set_mixin_1',
- 'dimension_set_mixin_2',
- 'duplicate_dimension_set_mixin_1',
- 'dimension_mixin',
- ],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_WATERFALL_MIXIN_BUILDER_REMOVE_MIXIN_WATERFALL = """\
- [
- {
- 'mixins': ['waterfall_mixin'],
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'remove_mixins': ['waterfall_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['random_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['dimension_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['dimension_mixin'],
- 'os_type': 'win',
- 'browser_config': 'release',
- 'test_suites': {
- 'gpu_telemetry_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- # Swarming mixins must be a list, a single string is not allowed.
- FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': 'dimension_mixin',
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['nonexistant'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'mixins': ['waterfall_mixin'],
- 'machines': {
- 'Fake Tester': {
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
- [
- {
- 'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1',
- },
- ],
- },
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- FOO_TEST_SUITE_WITH_MIXIN = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': 'high',
- }
- ],
- 'expiration': 120,
- },
- 'mixins': ['test_mixin'],
- },
- },
- },
- }
- """
- # These mixins are invalid; if passed to check_input_file_consistency, they will
- # fail. These are used for output file consistency checks.
- SWARMING_MIXINS = """\
- {
- 'builder_mixin': {
- 'swarming': {
- 'value': 'builder',
- },
- },
- 'dimension_mixin': {
- 'swarming': {
- 'dimensions': {
- 'iama': 'mixin',
- },
- },
- },
- 'random_mixin': {
- 'value': 'random',
- },
- 'test_mixin': {
- 'swarming': {
- 'value': 'test',
- },
- },
- 'waterfall_mixin': {
- 'swarming': {
- 'value': 'waterfall',
- },
- },
- }
- """
- SWARMING_MIXINS_APPEND = """\
- {
- 'builder_mixin': {
- '$mixin_append': {
- 'args': [ '--mixin-argument' ],
- 'linux_args': [ '--linux-mixin-argument' ],
- },
- },
- }
- """
- SWARMING_MIXINS_APPEND_NOT_LIST = """\
- {
- 'builder_mixin': {
- '$mixin_append': {
- 'args': 'I am not a list',
- },
- },
- }
- """
- SWARMING_MIXINS_APPEND_TO_SWARMING = """\
- {
- 'builder_mixin': {
- '$mixin_append': {
- 'swarming': [ 'swarming!' ],
- },
- },
- }
- """
- SWARMING_MIXINS_APPEND_NAMED_CACHES = """\
- {
- 'builder_mixin': {
- '$mixin_append': {
- 'swarming': {
- 'named_caches': [
- {
- 'name': 'cache',
- 'file': 'cache_file',
- },
- ]
- },
- },
- },
- }
- """
- SWARMING_MIXINS_APPEND_OTHER_KEYS_WITH_NAMED_CACHES = """\
- {
- 'builder_mixin': {
- '$mixin_append': {
- 'swarming': {
- 'named_caches': [
- {
- 'name': 'cache',
- 'file': 'cache_file',
- },
- ],
- 'other_key': 'some value',
- },
- },
- },
- }
- """
- SWARMING_MIXINS_DIMENSION_SETS = """\
- {
- 'dimension_set_mixin_1': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'value': 'ds1',
- },
- ],
- },
- },
- 'dimension_set_mixin_2': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'value': 'ds2',
- },
- ],
- },
- },
- 'duplicate_dimension_set_mixin_1': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'value': 'ds1',
- },
- ],
- },
- },
- 'dimension_mixin': {
- 'swarming': {
- 'dimensions': {
- 'other_value': 'dimension_mixin',
- },
- },
- },
- }
- """
- SWARMING_MIXINS_DUPLICATED = """\
- {
- 'builder_mixin': {
- 'value': 'builder',
- },
- 'builder_mixin': {
- 'value': 'builder',
- },
- }
- """
- SWARMING_MIXINS_UNSORTED = """\
- {
- 'b_mixin': {
- 'b': 'b',
- },
- 'a_mixin': {
- 'a': 'a',
- },
- 'c_mixin': {
- 'c': 'c',
- },
- }
- """
- SWARMING_MIXINS_SORTED = """\
- {
- 'a_mixin': {
- 'a': 'a',
- },
- 'b_mixin': {
- 'b': 'b',
- },
- 'c_mixin': {
- 'c': 'c',
- },
- }
- """
- WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "other_value": "dimension_mixin",
- "value": "ds1"
- },
- {
- "other_value": "dimension_mixin",
- "value": "ds2"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- WATERFALL_MIXIN_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "waterfall"
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "exception"
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- BUILDER_MIXIN_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "builder"
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test",
- "value": "random"
- }
- ]
- }
- }
- """
- BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--c_arg",
- "--mixin-argument"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--c_arg",
- "--mixin-argument",
- "--linux-mixin-argument"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "named_caches": [
- {
- "file": "cache_in_test_file",
- "name": "cache_in_test"
- },
- {
- "file": "cache_file",
- "name": "cache"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- TEST_MIXIN_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high",
- "kvm": "1"
- }
- ],
- "expiration": 120,
- "value": "test"
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "iama": "mixin",
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "test"
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- class MixinTests(TestCase):
- """Tests for the mixins feature."""
- def test_mixins_must_be_sorted(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_SORTING_MIXINS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_SORTING_MIXINS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_UNSORTED)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = '\n'.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\+ ._mixin.*')
- self.assertRegex(joined_lines, '.*\- ._mixin.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_waterfall(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- WATERFALL_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- WATERFALL_MIXIN_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_waterfall_exception_overrides(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_builder(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- BUILDER_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- BUILDER_MIXIN_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_builder_non_swarming(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_test_suite(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- TEST_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- TEST_MIXIN_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_dimension(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_dimension_gpu(self):
- fbb = FakeBBGen(self.args,
- FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS,
- gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
- self.create_testing_buildbot_json_file('chromium.test.json',
- GPU_DIMENSIONS_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- GPU_DIMENSIONS_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_unreferenced(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- '.*mixins are unreferenced.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_unused(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_list(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- DIMENSIONS_MIXIN_WATERFALL_OUTPUT)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_no_duplicate_keys(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_DUPLICATED)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'The following files have invalid keys: mixins.pyl'):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = '\n'.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\- builder_mixin')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_no_duplicate_keys_basic_test_suite(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE_NOT_SORTED,
- LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'The following files have invalid keys: test_suites.pyl'):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = '\n'.join(fbb.printed_lines)
- self.assertRegex(joined_lines, '.*\- a_test')
- self.assertRegex(joined_lines, '.*\+ a_test')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
- def test_type_assert_printing_help(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, TEST_SUITES_SYNTAX_ERROR,
- LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Invalid \.pyl file \'test_suites.pyl\'.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertEqual(fbb.printed_lines, [
- '== test_suites.pyl ==',
- '<snip>',
- '1 {',
- "2 'basic_suites': {",
- '--------------------------------------------------------------------'
- '------------',
- '3 3: {',
- '-------^------------------------------------------------------------'
- '------------',
- "4 'suite_c': {},",
- '5 },',
- '<snip>',
- ])
- def test_mixin_append_args(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', BUILDER_MIXIN_APPEND_ARGS_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mixin_append_linux_args(self):
- fbb = FakeBBGen(self.args,
- FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', BUILDER_MIXIN_APPEND_ARGS_LINUX_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mixin_append_swarming_named_caches(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND_NAMED_CACHES)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json',
- BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', BUILDER_MIXIN_APPEND_NAMED_CACHES_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mixin_append_swarming_error(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND_OTHER_KEYS_WITH_NAMED_CACHES)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Only named_caches is supported under swarming key in '
- '\$mixin_append, but there are: \[\'named_caches\', \'other_key\'\]'):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mixin_append_mixin_field_not_list(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND_NOT_LIST)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Key "args" in \$mixin_append must be a list.'):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mixin_append_test_field_not_list(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_APPEND_TO_SWARMING)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Cannot apply \$mixin_append to non-list "swarming".'):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_remove_mixin_builder_remove_waterfall(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL_MIXIN_BUILDER_REMOVE_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_remove_mixin_test_remove_waterfall(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_remove_mixin_test_remove_builder(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', WATERFALL_MIXIN_REMOVE_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_dimension_sets_application(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_DIMENSION_SETS_MIXIN_WATERFALL,
- FOO_TEST_SUITE_NO_DIMENSIONS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_DIMENSION_SETS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', WATERFALL_DIMENSION_SETS_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- TEST_SUITE_WITH_PARAMS = """\
- {
- 'basic_suites': {
- 'bar_tests': {
- 'bar_test': {
- 'args': ['--no-xvfb'],
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_os': 'NMF26U'
- }
- ],
- },
- 'should_retry_with_patch': False,
- 'name': 'bar_test'
- },
- 'bar_test_test': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1'
- }
- ],
- 'hard_timeout': 1000
- },
- 'should_retry_with_patch': True
- }
- },
- 'foo_tests': {
- 'foo_test_empty': {},
- 'foo_test': {
- 'args': [
- '--jobs=1',
- '--verbose'
- ],
- 'swarming': {
- 'dimension_sets': [
- {
- 'device_os': 'MMB29Q'
- }
- ],
- 'hard_timeout': 1800
- }
- },
- 'foo_test_test': {
- 'swarming': {
- },
- 'name': 'pls'
- },
- },
- },
- 'compound_suites': {
- 'composition_tests': [
- 'foo_tests',
- 'bar_tests',
- ],
- },
- }
- """
- TEST_QUERY_BOTS_OUTPUT = {
- "Fake Android M Tester": {
- "gtest_tests": [
- {
- "test": "foo_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": False
- }
- }
- ]
- },
- "Fake Android L Tester": {
- "gtest_tests": [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision":
- "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets":[
- {
- "device_os": "LMY41U",
- "device_os_type": "user",
- "device_type": "hammerhead",
- 'os': 'Android'
- }
- ],
- "can_use_on_swarming_builders": True
- }
- }
- ]
- },
- "Fake Android K Tester": {
- "additional_compile_targets": ["bar_test"],
- "gtest_tests": [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision":
- "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "KTU84P",
- "device_os_type": "userdebug",
- "device_type": "hammerhead",
- "os": "Android",
- }
- ],
- "can_use_on_swarming_builders": True,
- "output_links": [
- {
- "link": ["https://luci-logdog.appspot.com/v/?s",
- "=android%2Fswarming%2Flogcats%2F",
- "${TASK_ID}%2F%2B%2Funified_logcats"],
- "name": "shard #${SHARD_INDEX} logcats"
- }
- ]
- }
- }
- ]
- },
- "Android Builder": {
- "additional_compile_targets": ["bar_test"]
- }
- }
- TEST_QUERY_BOTS_TESTS_OUTPUT = {
- "Fake Android M Tester": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "test": "foo_test",
- "swarming": {
- "can_use_on_swarming_builders": False
- }
- }
- ],
- "Fake Android L Tester": [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "LMY41U",
- "device_os_type": "user",
- "device_type": "hammerhead",
- "os": "Android"
- }
- ],
- "can_use_on_swarming_builders": True
- }
- }
- ],
- "Android Builder": [],
- "Fake Android K Tester": [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "KTU84P",
- "device_os_type": "userdebug",
- "device_type": "hammerhead",
- "os": "Android"
- }
- ],
- "can_use_on_swarming_builders": True,
- "output_links": [
- {
- "link": [
- "https://luci-logdog.appspot.com/v/?s",
- "=android%2Fswarming%2Flogcats%2F",
- "${TASK_ID}%2F%2B%2Funified_logcats"
- ],
- "name": "shard #${SHARD_INDEX} logcats"
- }
- ]
- }
- }
- ]
- }
- TEST_QUERY_BOT_OUTPUT = {
- "additional_compile_targets": ["bar_test"],
- "gtest_tests": [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "KTU84P",
- "device_os_type": "userdebug",
- "device_type": "hammerhead",
- "os": "Android"
- }
- ],
- "can_use_on_swarming_builders": True,
- "output_links": [
- {
- "link": ["https://luci-logdog.appspot.com/v/?s",
- "=android%2Fswarming%2Flogcats%2F",
- "${TASK_ID}%2F%2B%2Funified_logcats"
- ],
- "name": "shard #${SHARD_INDEX} logcats"
- }
- ]
- }
- }
- ]
- }
- TEST_QUERY_BOT_TESTS_OUTPUT = [
- {
- "test": "foo_test",
- "args": [
- "--gs-results-bucket=chromium-result-details",
- "--recover-devices"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/luci/logdog/butler/${platform}",
- "location": "bin",
- "revision": "git_revision:ff387eadf445b24c935f1cf7d6ddd279f8a6b04c"
- }
- ],
- "dimension_sets": [
- {
- "device_os": "LMY41U",
- "device_os_type": "user",
- "device_type": "hammerhead",
- "os": "Android"
- }
- ],
- "can_use_on_swarming_builders": True
- }
- }
- ]
- TEST_QUERY_TESTS_OUTPUT = {
- "bar_test": {},
- "foo_test": {}
- }
- TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT = ["foo_test"]
- TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT = ["bar_test"]
- TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT = ["bar_test_test"]
- TEST_QUERY_TESTS_PARAMS_OUTPUT = ['bar_test_test']
- TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT = ['bar_test']
- TEST_QUERY_TEST_OUTPUT = {}
- TEST_QUERY_TEST_BOTS_OUTPUT = [
- "Fake Android K Tester",
- "Fake Android L Tester",
- "Fake Android M Tester",
- ]
- TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT = ['Fake Tester']
- TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT = []
- class QueryTests(TestCase):
- """Tests for the query feature."""
- def test_query_bots(self):
- self.override_args(query='bots',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_BOTS_OUTPUT)
- def test_query_bots_invalid(self):
- self.override_args(query='bots/blah/blah',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_bots_json(self):
- self.override_args(query='bots',
- check=False,
- pyl_files_dir=None,
- json='result.json',
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- self.assertFalse(fbb.printed_lines)
- def test_query_bots_tests(self):
- self.override_args(query='bots/tests',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_BOTS_TESTS_OUTPUT)
- def test_query_invalid_bots_tests(self):
- self.override_args(query='bots/tdfjdk',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_bot(self):
- self.override_args(query='bot/Fake Android K Tester',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.maxDiff = None # pragma pylint: disable=attribute-defined-outside-init
- self.assertEqual(query_json, TEST_QUERY_BOT_OUTPUT)
- def test_query_bot_invalid_id(self):
- self.override_args(query='bot/bot1',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_bot_invalid_query_too_many(self):
- self.override_args(query='bot/Fake Android K Tester/blah/blah',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_bot_invalid_query_no_tests(self):
- self.override_args(query='bot/Fake Android K Tester/blahs',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_bot_tests(self):
- self.override_args(query='bot/Fake Android L Tester/tests',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_BOT_TESTS_OUTPUT)
- def test_query_tests(self):
- self.override_args(query='tests',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_OUTPUT)
- def test_query_tests_invalid(self):
- self.override_args(query='tests/blah/blah',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_tests_multiple_params(self):
- self.override_args(query='tests/--jobs=1&--verbose',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT)
- def test_query_tests_invalid_params(self):
- self.override_args(query='tests/device_os?',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_tests_dimension_params(self):
- self.override_args(query='tests/device_os:NMF26U',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT)
- def test_query_tests_swarming_params(self):
- self.override_args(query='tests/hard_timeout:1000',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT)
- def test_query_tests_params(self):
- self.override_args(query='tests/should_retry_with_patch:true',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_OUTPUT)
- def test_query_tests_params_false(self):
- self.override_args(query='tests/should_retry_with_patch:false',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- TEST_SUITE_WITH_PARAMS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT)
- def test_query_test(self):
- self.override_args(query='test/foo_test',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TEST_OUTPUT)
- def test_query_test_invalid_id(self):
- self.override_args(query='test/foo_foo',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_test_invalid_length(self):
- self.override_args(query='test/foo_tests/foo/foo',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_test_bots(self):
- self.override_args(query='test/foo_test/bots',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_OUTPUT)
- def test_query_test_bots_isolated_scripts(self):
- self.override_args(query='test/foo_test/bots',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- FOO_ISOLATED_SCRIPTS_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT)
- def test_query_test_bots_invalid(self):
- self.override_args(query='test/foo_tests/foo',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- def test_query_test_bots_no_bots(self):
- self.override_args(query='test/bar_tests/bots',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- fbb.query(fbb.args)
- query_json = json.loads("".join(fbb.printed_lines))
- self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT)
- def test_query_invalid(self):
- self.override_args(query='foo',
- check=False,
- pyl_files_dir=None,
- json=None,
- waterfall_filters=[])
- fbb = FakeBBGen(self.args,
- ANDROID_WATERFALL,
- GOOD_COMPOSITION_TEST_SUITES,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS_SORTED)
- with self.assertRaises(SystemExit) as cm:
- fbb.query(fbb.args)
- self.assertEqual(cm.exception.code, 1)
- self.assertTrue(fbb.printed_lines)
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'args': [
- '--enable-features',
- 'Foo,Bar',
- ],
- },
- },
- },
- }
- """
- FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE = """\
- {
- 'foo_test': {
- 'replacements': {
- 'Fake Tester': {
- 'args': {
- '--c_arg': None,
- },
- },
- },
- },
- }
- """
- FOO_TEST_REPLACEMENTS_REMOVE_VALUE = """\
- {
- 'foo_test': {
- 'replacements': {
- 'Fake Tester': {
- 'args': {
- '--enable-features': None,
- },
- },
- },
- },
- }
- """
- FOO_TEST_REPLACEMENTS_REPLACE_VALUE = """\
- {
- 'foo_test': {
- 'replacements': {
- 'Fake Tester': {
- 'args': {
- '--enable-features': 'Bar,Baz',
- },
- },
- },
- },
- }
- """
- FOO_TEST_REPLACEMENTS_INVALID_KEY = """\
- {
- 'foo_test': {
- 'replacements': {
- 'Fake Tester': {
- 'invalid': {
- '--enable-features': 'Bar,Baz',
- },
- },
- },
- },
- }
- """
- REPLACEMENTS_REMOVE_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- REPLACEMENTS_VALUE_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--enable-features=Bar,Baz"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--enable-features",
- "Bar,Baz"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- class ReplacementTests(TestCase):
- """Tests for the arg replacement feature."""
- def test_replacement_valid_remove_no_value(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE)
- self.create_testing_buildbot_json_file('chromium.test.json',
- REPLACEMENTS_REMOVE_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- REPLACEMENTS_REMOVE_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_replacement_valid_remove_value(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_REMOVE_VALUE)
- self.create_testing_buildbot_json_file('chromium.test.json',
- REPLACEMENTS_REMOVE_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- REPLACEMENTS_REMOVE_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_replacement_valid_replace_value(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
- self.create_testing_buildbot_json_file('chromium.test.json',
- REPLACEMENTS_VALUE_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- REPLACEMENTS_VALUE_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_replacement_valid_replace_value_separate_entries(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', REPLACEMENTS_VALUE_SEPARATE_ENTRIES_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_replacement_invalid_key_not_valid(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_INVALID_KEY)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Given replacement key *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_replacement_invalid_key_not_found(self):
- fbb = FakeBBGen(self.args,
- FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_ARGS,
- LUCI_MILO_CFG,
- exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Could not find *'):
- fbb.check_output_file_consistency(verbose=True)
- FOO_TEST_SUITE_WITH_MAGIC_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'args': [
- '$$MAGIC_SUBSTITUTION_TestOnlySubstitution',
- ],
- },
- },
- },
- }
- """
- FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'args': [
- '$$MAGIC_SUBSTITUTION_NotARealSubstitution',
- ],
- },
- },
- },
- }
- """
- MAGIC_SUBSTITUTIONS_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--magic-substitution-success"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "kvm": "1"
- }
- ]
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- class MagicSubstitutionTests(TestCase):
- """Tests for the magic substitution feature."""
- def test_valid_function(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_MAGIC_ARGS, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MAGIC_SUBSTITUTIONS_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- MAGIC_SUBSTITUTIONS_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_invalid_function(self):
- fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Magic substitution function *'):
- fbb.check_output_file_consistency(verbose=True)
- # Matrix compound composition test suites
- MATRIX_COMPOUND_EMPTY = """\
- {
- 'basic_suites': {
- 'bar_tests': {
- 'bar_test': {},
- },
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {},
- 'bar_tests': {},
- },
- },
- }
- """
- MATRIX_COMPOUND_MISSING_IDENTIFIER = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'swarming': {
- 'dimension_sets': [
- {
- 'foo': 'bar',
- },
- ],
- },
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_EMPTY_IDENTIFIER = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': '',
- 'swarming': {
- 'dimension_sets': [
- {
- 'foo': 'empty identifier not allowed',
- },
- ],
- },
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_TRAILING_IDENTIFIER = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': ' ',
- 'swarming': {
- 'dimension_sets': [
- {
- 'foo': 'strip to empty not allowed',
- },
- ],
- },
- },
- ],
- },
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': 'id ',
- 'swarming': {
- 'dimension_sets': [
- {
- 'foo': 'trailing whitespace not allowed',
- },
- ],
- },
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_MISMATCHED_SWARMING_LENGTH = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'hello': 'world',
- }
- ],
- },
- },
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': 'test',
- 'swarming': {
- 'dimension_sets': [
- {
- 'foo': 'bar',
- },
- {
- 'bar': 'foo',
- }
- ],
- },
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_REF_NONEXISTENT = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'bar_test': {},
- },
- },
- }
- """
- MATRIX_COMPOUND_REF_COMPOSITION = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'compound_suites': {
- 'sample_composition': {
- 'foo_tests': {},
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'sample_composition': {},
- },
- },
- }
- """
- MATRIX_COMPOSITION_REF_MATRIX = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {},
- },
- },
- 'matrix_compound_suites': {
- 'a_test': {
- 'foo_tests': {},
- },
- 'matrix_tests': {
- 'a_test': {},
- },
- },
- }
- """
- MATRIX_COMPOUND_VARIANTS_MIXINS_MERGE = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'set': {
- 'mixins': [ 'test_mixin' ],
- },
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'mixins': [ 'dimension_mixin' ],
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_VARIANTS_MIXINS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'set': {
- 'mixins': [ 'test_mixin' ],
- },
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'mixins': [
- 'dimension_mixin',
- 'waterfall_mixin',
- 'builder_mixin',
- 'random_mixin'
- ],
- },
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_VARIANTS_MIXINS_REMOVE = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'set': {
- 'remove_mixins': ['builder_mixin'],
- },
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'mixins': [ 'builder_mixin' ],
- }
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_CONFLICTING_TEST_SUITES = """\
- {
- 'basic_suites': {
- 'bar_tests': {
- 'baz_tests': {
- 'args': [
- '--bar',
- ],
- }
- },
- 'foo_tests': {
- 'baz_tests': {
- 'args': [
- '--foo',
- ],
- }
- },
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'bar_tests': {
- 'variants': [
- {
- 'identifier': 'bar',
- }
- ],
- },
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': 'foo'
- }
- ]
- }
- },
- },
- }
- """
- MATRIX_COMPOUND_TARGETS_ARGS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'args_test': {
- 'args': [
- '--iam'
- ],
- },
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': 'args',
- 'args': [
- '--anarg',
- ],
- },
- {
- 'identifier': 'swarming',
- 'swarming': {
- 'a': 'b',
- 'dimension_sets': [
- {
- 'hello': 'world',
- }
- ]
- }
- },
- {
- 'identifier': 'mixins',
- 'mixins': [ 'dimension_mixin' ],
- }
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_TARGETS_MIXINS = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'mixins_test': {
- 'mixins': [ 'test_mixin' ],
- },
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'mixins': [ 'random_mixin' ],
- 'variants': [
- {
- 'identifier': 'args',
- 'args': [
- '--anarg',
- ],
- },
- {
- 'identifier': 'swarming',
- 'swarming': {
- 'a': 'b',
- 'dimension_sets': [
- {
- 'hello': 'world',
- }
- ]
- }
- },
- {
- 'identifier': 'mixins',
- 'mixins': [ 'dimension_mixin' ],
- }
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_TARGETS_SWARMING = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'swarming_test': {
- 'swarming': {
- 'foo': 'bar',
- 'dimension_sets': [
- {
- 'foo': 'bar',
- },
- ],
- },
- },
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- {
- 'identifier': 'args',
- 'args': [
- '--anarg',
- ],
- },
- {
- 'identifier': 'swarming',
- 'swarming': {
- 'a': 'b',
- 'dimension_sets': [
- {
- 'hello': 'world',
- }
- ]
- }
- },
- {
- 'identifier': 'mixins',
- 'mixins': [ 'dimension_mixin' ],
- }
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_VARIANTS_REF = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'swarming_test': {},
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- 'a_variant'
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_TEST_WITH_TEST_KEY = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'swarming_test': {
- 'test': 'foo_test_apk'
- },
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- 'a_variant',
- ],
- },
- },
- },
- }
- """
- MATRIX_COMPOUND_MIXED_VARIANTS_REF = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'swarming_test': {},
- }
- },
- 'matrix_compound_suites': {
- 'matrix_tests': {
- 'foo_tests': {
- 'variants': [
- 'a_variant',
- {
- 'args': [
- 'a',
- 'b'
- ],
- 'identifier': 'ab',
- }
- ],
- },
- },
- },
- }
- """
- VARIANTS_FILE = """\
- {
- 'a_variant': {
- 'args': [
- '--platform',
- 'device',
- '--version',
- '1'
- ],
- 'identifier': 'a_variant'
- }
- }
- """
- MULTI_VARIANTS_FILE = """\
- {
- 'a_variant': {
- 'args': [
- '--platform',
- 'device',
- '--version',
- '1'
- ],
- 'identifier': 'a_variant'
- },
- 'b_variant': {
- 'args': [
- '--platform',
- 'sim',
- '--version',
- '2'
- ],
- 'identifier': 'b_variant'
- }
- }
- """
- # # Dictionary composition test suite outputs
- MATRIX_COMPOUND_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "bar_test"
- },
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--platform",
- "device",
- "--version",
- "1"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "swarming_test a_variant",
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test_apk",
- "variant_id": "a_variant"
- }
- ]
- }
- }
- """
- MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--iam",
- "--anarg"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "args_test args",
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "args_test",
- "variant_id": "args"
- },
- {
- "args": [
- "--iam"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "args_test mixins",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "iama": "mixin"
- }
- ]
- },
- "test": "args_test",
- "variant_id": "mixins"
- },
- {
- "args": [
- "--iam"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "args_test swarming",
- "swarming": {
- "a": "b",
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "hello": "world"
- }
- ]
- },
- "test": "args_test",
- "variant_id": "swarming"
- }
- ]
- }
- }
- """
- MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--anarg"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "mixins_test args",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "value": "test"
- },
- "test": "mixins_test",
- "value": "random",
- "variant_id": "args"
- },
- {
- "args": [],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "mixins_test mixins",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "iama": "mixin"
- }
- ],
- "value": "test"
- },
- "test": "mixins_test",
- "value": "random",
- "variant_id": "mixins"
- },
- {
- "args": [],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "mixins_test swarming",
- "swarming": {
- "a": "b",
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "hello": "world"
- }
- ],
- "value": "test"
- },
- "test": "mixins_test",
- "value": "random",
- "variant_id": "swarming"
- }
- ]
- }
- }
- """
- MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--anarg"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "swarming_test args",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "foo": "bar"
- }
- ],
- "foo": "bar"
- },
- "test": "swarming_test",
- "variant_id": "args"
- },
- {
- "args": [],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "swarming_test mixins",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "foo": "bar",
- "iama": "mixin"
- }
- ],
- "foo": "bar"
- },
- "test": "swarming_test",
- "variant_id": "mixins"
- },
- {
- "args": [],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "swarming_test swarming",
- "swarming": {
- "a": "b",
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "foo": "bar",
- "hello": "world"
- }
- ],
- "foo": "bar"
- },
- "test": "swarming_test",
- "variant_id": "swarming"
- }
- ]
- }
- }
- """
- MATRIX_COMPOUND_VARIANTS_REF_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "args": [
- "--platform",
- "device",
- "--version",
- "1"
- ],
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "name": "swarming_test a_variant",
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "swarming_test",
- "variant_id": "a_variant"
- }
- ]
- }
- }
- """
- EMPTY_SKYLAB_TEST_EXCEPTIONS = """\
- {
- 'tast.foo OCTOPUS_TOT': {
- 'remove_from': [
- 'Fake Tester',
- ]
- },
- 'tast.foo OCTOPUS_TOT-1': {
- 'remove_from': [
- 'Fake Tester',
- ]
- }
- }
- """
- MATRIX_SKYLAB_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'test_suites': {
- 'skylab_tests': 'cros_skylab_basic_x86',
- },
- },
- },
- },
- ]
- """
- MATRIX_COMPOUND_SKYLAB_REF = """\
- {
- 'basic_suites': {
- 'cros_skylab_basic': {
- 'tast.basic': {
- 'suite': 'tast.basic',
- 'timeout': 3600,
- },
- 'tast.foo': {
- 'suite': 'tast.foo',
- 'timeout': 3600,
- },
- },
- },
- 'compound_suites': {},
- 'matrix_compound_suites': {
- 'cros_skylab_basic_x86': {
- 'cros_skylab_basic': {
- 'variants': [
- {
- 'skylab': {
- 'cros_board': 'octopus',
- 'cros_chrome_version': '89.0.3234.0',
- 'cros_img': 'octopus-release/R89-13655.0.0',
- },
- 'enabled': True,
- 'identifier': 'OCTOPUS_TOT',
- },
- {
- 'skylab': {
- 'cros_board': 'octopus',
- 'cros_chrome_version': '88.0.2324.0',
- 'cros_img': 'octopus-release/R88-13597.23.0',
- },
- 'enabled': True,
- 'identifier': 'OCTOPUS_TOT-1',
- },
- ]
- },
- },
- },
- }
- """
- ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF = """\
- {
- 'basic_suites': {
- 'cros_skylab_basic': {
- 'tast.basic': {
- 'suite': 'tast.basic',
- 'timeout': 3600,
- },
- 'tast.foo': {
- 'suite': 'tast.foo',
- 'timeout': 3600,
- },
- },
- },
- 'compound_suites': {},
- 'matrix_compound_suites': {
- 'cros_skylab_basic_x86': {
- 'cros_skylab_basic': {
- 'variants': [
- {
- 'skylab': {
- 'cros_board': 'octopus',
- 'cros_chrome_version': '89.0.3234.0',
- 'cros_img': 'octopus-release/R89-13655.0.0',
- },
- 'enabled': True,
- 'identifier': 'OCTOPUS_TOT',
- },
- {
- 'skylab': {
- 'cros_board': 'octopus',
- 'cros_chrome_version': '88.0.2324.0',
- 'cros_img': 'octopus-release/R88-13597.23.0',
- },
- 'enabled': False,
- 'identifier': 'OCTOPUS_TOT-1',
- },
- ]
- },
- },
- },
- }
- """
- VARIATION_SKYLAB_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "skylab_tests": [
- {
- "args": [],
- "cros_board": "octopus",
- "cros_img": "octopus-release/R89-13655.0.0",
- "name": "tast.basic OCTOPUS_TOT",
- "suite": "tast.basic",
- "swarming": {},
- "test": "tast.basic",
- "timeout": 3600,
- "variant_id": "OCTOPUS_TOT"
- },
- {
- "args": [],
- "cros_board": "octopus",
- "cros_img": "octopus-release/R88-13597.23.0",
- "name": "tast.basic OCTOPUS_TOT-1",
- "suite": "tast.basic",
- "swarming": {},
- "test": "tast.basic",
- "timeout": 3600,
- "variant_id": "OCTOPUS_TOT-1"
- }
- ]
- }
- }
- """
- ENABLED_AND_DISABLED_VARIATION_SKYLAB_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "skylab_tests": [
- {
- "args": [],
- "cros_board": "octopus",
- "cros_img": "octopus-release/R89-13655.0.0",
- "name": "tast.basic OCTOPUS_TOT",
- "suite": "tast.basic",
- "swarming": {},
- "test": "tast.basic",
- "timeout": 3600,
- "variant_id": "OCTOPUS_TOT"
- }
- ]
- }
- }
- """
- class MatrixCompositionTests(TestCase):
- def test_good_structure_no_configs(self):
- """
- Tests matrix compound test suite structure with no configs,
- no conflicts and no bad references
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_EMPTY, LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MATRIX_COMPOUND_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- MATRIX_COMPOUND_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_missing_identifier(self):
- """
- Variant is missing an identifier
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_MISSING_IDENTIFIER, LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Missing required identifier field in matrix compound suite*'):
- fbb.check_output_file_consistency(verbose=True)
- def test_empty_identifier(self):
- """
- Variant identifier is empty.
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_EMPTY_IDENTIFIER, LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Identifier field can not be "" in matrix compound suite*'):
- fbb.check_output_file_consistency(verbose=True)
- def test_trailing_identifier(self):
- """
- Variant identifier has trailing whitespace.
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_TRAILING_IDENTIFIER, LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'Identifier field can not have leading and trailing whitespace in'
- ' matrix compound suite*'):
- fbb.check_output_file_consistency(verbose=True)
- def test_mismatched_swarming_length(self):
- """
- Swarming dimension set length mismatch test. Composition set > basic set
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_MISMATCHED_SWARMING_LENGTH, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Error merging lists by key *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_noexistent_ref(self):
- """
- Test referencing a non-existent basic test suite
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_REF_NONEXISTENT, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Unable to find reference to *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_ref_to_composition(self):
- """
- Test referencing another composition test suite
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_REF_COMPOSITION, LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'matrix_compound_suites may not refer to other *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_ref_to_matrix(self):
- """
- Test referencing another matrix test suite
- """
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOSITION_REF_MATRIX, LUCI_MILO_CFG)
- with self.assertRaisesRegex(
- generate_buildbot_json.BBGenErr,
- 'matrix_compound_suites may not refer to other *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_conflicting_names(self):
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_CONFLICTING_TEST_SUITES, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Conflicting test definitions.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variants_swarming_dict_merge_args(self):
- """
- Test targets with swarming dictionary defined by both basic and matrix
- """
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_TARGETS_ARGS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_ARGS)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variants_swarming_dict_merge_mixins(self):
- """
- Test targets with swarming dictionary defined by both basic and matrix
- """
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_TARGETS_MIXINS,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_MIXINS)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variants_swarming_dict_swarming(self):
- """
- Test targets with swarming dictionary defined by both basic and matrix
- """
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_TARGETS_SWARMING,
- LUCI_MILO_CFG,
- mixins=SWARMING_MIXINS)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', MATRIX_TARGET_DICT_MERGE_OUTPUT_SWARMING)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variant_test_suite_with_test_key(self):
- """
- Test targets in matrix compound test suites with variants
- """
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_TEST_WITH_TEST_KEY,
- LUCI_MILO_CFG,
- variants=VARIANTS_FILE)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT)
- self.create_testing_buildbot_json_file(
- 'chromium.ci.json', MATRIX_COMPOUND_TEST_SUITE_WITH_TEST_KEY_DICT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variants_pyl_ref(self):
- """Test targets with variants string ref"""
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_VARIANTS_REF,
- LUCI_MILO_CFG,
- variants=VARIANTS_FILE)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
- self.create_testing_buildbot_json_file('chromium.ci.json',
- MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_variants_pyl_no_ref(self):
- """Test targets with variants string ref, not defined in variants.pyl"""
- fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_VARIANTS_REF, LUCI_MILO_CFG)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'Missing variant definition for *'):
- fbb.check_output_file_consistency(verbose=True)
- def test_variants_pyl_all_unreferenced(self):
- """Test targets with variants in variants.pyl, unreferenced in tests"""
- fbb = FakeBBGen(self.args,
- MATRIX_GTEST_SUITE_WATERFALL,
- MATRIX_COMPOUND_MIXED_VARIANTS_REF,
- LUCI_MILO_CFG,
- variants=MULTI_VARIANTS_FILE)
- # self.create_testing_buildbot_json_file(
- # 'chromium.test.json', MATRIX_COMPOUND_VARIANTS_REF_OUTPUT)
- with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'The following variants were unreferenced *'):
- fbb.check_input_file_consistency(verbose=True)
- def test_good_skylab_matrix_with_variants(self):
- fbb = FakeBBGen(self.args,
- MATRIX_SKYLAB_WATERFALL,
- MATRIX_COMPOUND_SKYLAB_REF,
- LUCI_MILO_CFG,
- exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS)
- self.create_testing_buildbot_json_file('chromium.test.json',
- VARIATION_SKYLAB_OUTPUT)
- fbb.check_input_file_consistency(verbose=True)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_enabled_and_disabled_skylab_matrix_with_variants(self):
- """Test with disabled variants"""
- fbb = FakeBBGen(self.args,
- MATRIX_SKYLAB_WATERFALL,
- ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF,
- LUCI_MILO_CFG,
- exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS)
- # some skylab test variant is disabled; the corresponding skylab tests
- # is not generated.
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', ENABLED_AND_DISABLED_VARIATION_SKYLAB_OUTPUT)
- fbb.check_input_file_consistency(verbose=True)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
- MAC_TEST_SUITE = """\
- {
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- },
- },
- },
- }
- """
- MAC_GTESTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Mac': {
- 'swarming': {
- 'can_use_on_swarming_builders': True,
- },
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- MAC_GTEST_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Mac": {
- "gtest_tests": [
- {
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_gtest_merge.py"
- },
- "swarming": {
- "can_use_on_swarming_builders": true
- },
- "test": "foo_test"
- }
- ]
- }
- }
- """
- MAC_ISOLATED_SCRIPTS_WATERFALL = """\
- [
- {
- 'project': 'chromium',
- 'bucket': 'ci',
- 'name': 'chromium.test',
- 'machines': {
- 'Mac': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'os': 'Mac',
- },
- ],
- },
- 'test_suites': {
- 'isolated_scripts': 'foo_tests',
- },
- },
- },
- },
- ]
- """
- MAC_ISOLATED_SCRIPTS_WATERFALL_OUTPUT = """\
- {
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Mac": {
- "isolated_scripts": [
- {
- "isolate_name": "foo_test",
- "merge": {
- "args": [],
- "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
- },
- "name": "foo_test",
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "os": "Mac"
- }
- ]
- }
- }
- ]
- }
- }
- """
- MAC_LUCI_MILO_CFG = """\
- consoles {
- builders {
- name: "buildbucket/luci.chromium.ci/Mac"
- }
- }
- """
- class SwarmingTests(TestCase):
- def test_mac_builder_with_no_cpu_dimension_in_gtest_fails(self):
- fbb = FakeBBGen(self.args, MAC_GTESTS_WATERFALL, MAC_TEST_SUITE,
- MAC_LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file('chromium.test.json',
- MAC_GTEST_WATERFALL_OUTPUT)
- fbb.check_input_file_consistency(verbose=True)
- self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'os and cpu',
- fbb.check_output_file_consistency,
- verbose=True)
- self.assertFalse(fbb.printed_lines)
- def test_mac_builder_with_no_cpu_dimension_in_isolated_script_fails(self):
- fbb = FakeBBGen(self.args, MAC_ISOLATED_SCRIPTS_WATERFALL, MAC_TEST_SUITE,
- MAC_LUCI_MILO_CFG)
- self.create_testing_buildbot_json_file(
- 'chromium.test.json', MAC_ISOLATED_SCRIPTS_WATERFALL_OUTPUT)
- fbb.check_input_file_consistency(verbose=True)
- self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
- 'os and cpu',
- fbb.check_output_file_consistency,
- verbose=True)
- self.assertFalse(fbb.printed_lines)
- if __name__ == '__main__':
- unittest.main()
|