mb.py 85 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267
  1. #!/usr/bin/env python3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """MB - the Meta-Build wrapper around GN.
  6. MB is a wrapper script for GN that can be used to generate build files
  7. for sets of canned configurations and analyze them.
  8. """
  9. from __future__ import absolute_import
  10. from __future__ import print_function
  11. import argparse
  12. import ast
  13. import collections
  14. import errno
  15. import json
  16. import os
  17. import pipes
  18. import platform
  19. import re
  20. import shutil
  21. import sys
  22. import subprocess
  23. import tempfile
  24. import traceback
  25. import zipfile
  26. if sys.version_info.major == 2:
  27. from urllib2 import urlopen
  28. else:
  29. from urllib.request import urlopen
  30. CHROMIUM_SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(
  31. os.path.abspath(__file__))))
  32. sys.path = [os.path.join(CHROMIUM_SRC_DIR, 'build')] + sys.path
  33. sys.path.insert(0, os.path.join(
  34. os.path.dirname(os.path.abspath(__file__)), '..'))
  35. import gn_helpers
  36. from mb.lib import validation
  37. def DefaultVals():
  38. """Default mixin values"""
  39. return {
  40. 'args_file': '',
  41. 'gn_args': '',
  42. }
  43. def PruneVirtualEnv():
  44. # Set by VirtualEnv, no need to keep it.
  45. os.environ.pop('VIRTUAL_ENV', None)
  46. # Set by VPython, if scripts want it back they have to set it explicitly.
  47. os.environ.pop('PYTHONNOUSERSITE', None)
  48. # Look for "activate_this.py" in this path, which is installed by VirtualEnv.
  49. # This mechanism is used by vpython as well to sanitize VirtualEnvs from
  50. # $PATH.
  51. os.environ['PATH'] = os.pathsep.join([
  52. p for p in os.environ.get('PATH', '').split(os.pathsep)
  53. if not os.path.isfile(os.path.join(p, 'activate_this.py'))
  54. ])
  55. def main(args):
  56. # Prune all evidence of VPython/VirtualEnv out of the environment. This means
  57. # that we 'unwrap' vpython VirtualEnv path/env manipulation. Invocations of
  58. # `python` from GN should never inherit the gn.py's own VirtualEnv. This also
  59. # helps to ensure that generated ninja files do not reference python.exe from
  60. # the VirtualEnv generated from depot_tools' own .vpython file (or lack
  61. # thereof), but instead reference the default python from the PATH.
  62. PruneVirtualEnv()
  63. mbw = MetaBuildWrapper()
  64. return mbw.Main(args)
  65. class MetaBuildWrapper:
  66. def __init__(self):
  67. self.chromium_src_dir = CHROMIUM_SRC_DIR
  68. self.default_config = os.path.join(self.chromium_src_dir, 'tools', 'mb',
  69. 'mb_config.pyl')
  70. self.default_isolate_map = os.path.join(self.chromium_src_dir, 'testing',
  71. 'buildbot', 'gn_isolate_map.pyl')
  72. self.executable = sys.executable
  73. self.platform = sys.platform
  74. self.sep = os.sep
  75. self.args = argparse.Namespace()
  76. self.configs = {}
  77. self.public_artifact_builders = None
  78. self.builder_groups = {}
  79. self.mixins = {}
  80. self.isolate_exe = 'isolate.exe' if self.platform.startswith(
  81. 'win') else 'isolate'
  82. self.use_luci_auth = False
  83. self.rts_out_dir = self.PathJoin('gen', 'rts')
  84. self.banned_from_rts = set()
  85. def PostArgsInit(self):
  86. self.use_luci_auth = getattr(self.args, 'luci_auth', False)
  87. if 'config_file' in self.args and self.args.config_file is None:
  88. self.args.config_file = self.default_config
  89. if 'expectations_dir' in self.args and self.args.expectations_dir is None:
  90. self.args.expectations_dir = os.path.join(
  91. os.path.dirname(self.args.config_file), 'mb_config_expectations')
  92. banned_from_rts_map = json.loads(
  93. self.ReadFile(
  94. self.PathJoin(self.chromium_src_dir, 'tools', 'mb',
  95. 'rts_banned_suites.json')))
  96. self.banned_from_rts.update(banned_from_rts_map.get('*', set()))
  97. if getattr(self.args, 'builder', None):
  98. self.banned_from_rts.update(
  99. banned_from_rts_map.get(self.args.builder, set()))
  100. def Main(self, args):
  101. self.ParseArgs(args)
  102. self.PostArgsInit()
  103. try:
  104. ret = self.args.func()
  105. if ret != 0:
  106. self.DumpInputFiles()
  107. return ret
  108. except KeyboardInterrupt:
  109. self.Print('interrupted, exiting')
  110. return 130
  111. except Exception:
  112. self.DumpInputFiles()
  113. s = traceback.format_exc()
  114. for l in s.splitlines():
  115. self.Print(l)
  116. return 1
  117. def ParseArgs(self, argv):
  118. def AddCommonOptions(subp):
  119. group = subp.add_mutually_exclusive_group()
  120. group.add_argument(
  121. '-m', '--builder-group',
  122. help='builder group name to look up config from')
  123. subp.add_argument('-b', '--builder',
  124. help='builder name to look up config from')
  125. subp.add_argument('-c', '--config',
  126. help='configuration to analyze')
  127. subp.add_argument('--phase',
  128. help='optional phase name (used when builders '
  129. 'do multiple compiles with different '
  130. 'arguments in a single build)')
  131. subp.add_argument('-i', '--isolate-map-file', metavar='PATH',
  132. help='path to isolate map file '
  133. '(default is %(default)s)',
  134. default=[],
  135. action='append',
  136. dest='isolate_map_files')
  137. subp.add_argument('-n', '--dryrun', action='store_true',
  138. help='Do a dry run (i.e., do nothing, just print '
  139. 'the commands that will run)')
  140. subp.add_argument('-v', '--verbose', action='store_true',
  141. help='verbose logging')
  142. subp.add_argument('--root', help='Path to GN source root')
  143. subp.add_argument('--dotfile', help='Path to GN dotfile')
  144. AddExpansionOptions(subp)
  145. def AddExpansionOptions(subp):
  146. # These are the args needed to expand a config file into the full
  147. # parsed dicts of GN args.
  148. subp.add_argument('-f',
  149. '--config-file',
  150. metavar='PATH',
  151. help=('path to config file '
  152. '(default is mb_config.pyl'))
  153. subp.add_argument('-g', '--goma-dir', help='path to goma directory')
  154. subp.add_argument('--android-version-code',
  155. help='Sets GN arg android_default_version_code')
  156. subp.add_argument('--android-version-name',
  157. help='Sets GN arg android_default_version_name')
  158. subp.add_argument('--rts',
  159. default=None,
  160. help='which regression test selection model to use'
  161. ' For more info about RTS, please see'
  162. ' //docs/testing/regression-test-selection.md')
  163. subp.add_argument('--use-rts',
  164. action='store_true',
  165. default=False,
  166. help='Deprecated argument for enabling RTS')
  167. # TODO(crbug.com/1060857): Remove this once swarming task templates
  168. # support command prefixes.
  169. luci_auth_group = subp.add_mutually_exclusive_group()
  170. luci_auth_group.add_argument(
  171. '--luci-auth',
  172. action='store_true',
  173. help='Run isolated commands under `luci-auth context`.')
  174. luci_auth_group.add_argument(
  175. '--no-luci-auth',
  176. action='store_false',
  177. dest='luci_auth',
  178. help='Do not run isolated commands under `luci-auth context`.')
  179. parser = argparse.ArgumentParser(
  180. prog='mb', description='mb (meta-build) is a python wrapper around GN. '
  181. 'See the user guide in '
  182. '//tools/mb/docs/user_guide.md for detailed usage '
  183. 'instructions.')
  184. subps = parser.add_subparsers()
  185. subp = subps.add_parser('analyze',
  186. description='Analyze whether changes to a set of '
  187. 'files will cause a set of binaries to '
  188. 'be rebuilt.')
  189. AddCommonOptions(subp)
  190. subp.add_argument('path',
  191. help='path build was generated into.')
  192. subp.add_argument('input_path',
  193. help='path to a file containing the input arguments '
  194. 'as a JSON object.')
  195. subp.add_argument('output_path',
  196. help='path to a file containing the output arguments '
  197. 'as a JSON object.')
  198. subp.add_argument('--json-output',
  199. help='Write errors to json.output')
  200. subp.set_defaults(func=self.CmdAnalyze)
  201. subp = subps.add_parser('export',
  202. description='Print out the expanded configuration '
  203. 'for each builder as a JSON object.')
  204. AddExpansionOptions(subp)
  205. subp.set_defaults(func=self.CmdExport)
  206. subp = subps.add_parser('get-swarming-command',
  207. description='Get the command needed to run the '
  208. 'binary under swarming')
  209. AddCommonOptions(subp)
  210. subp.add_argument('--no-build',
  211. dest='build',
  212. default=True,
  213. action='store_false',
  214. help='Do not build, just isolate')
  215. subp.add_argument('--as-list',
  216. action='store_true',
  217. help='return the command line as a JSON-formatted '
  218. 'list of strings instead of single string')
  219. subp.add_argument('path',
  220. help=('path to generate build into (or use).'
  221. ' This can be either a regular path or a '
  222. 'GN-style source-relative path like '
  223. '//out/Default.'))
  224. subp.add_argument('target', help='ninja target to build and run')
  225. subp.set_defaults(func=self.CmdGetSwarmingCommand)
  226. subp = subps.add_parser('train',
  227. description='Writes the expanded configuration '
  228. 'for each builder as JSON files to a configured '
  229. 'directory.')
  230. subp.add_argument('-f',
  231. '--config-file',
  232. metavar='PATH',
  233. help='path to config file (default is mb_config.pyl')
  234. subp.add_argument('--expectations-dir',
  235. metavar='PATH',
  236. help='path to dir containing expectation files')
  237. subp.add_argument('-n',
  238. '--dryrun',
  239. action='store_true',
  240. help='Do a dry run (i.e., do nothing, just print '
  241. 'the commands that will run)')
  242. subp.add_argument('-v',
  243. '--verbose',
  244. action='store_true',
  245. help='verbose logging')
  246. subp.set_defaults(func=self.CmdTrain)
  247. subp = subps.add_parser('gen',
  248. description='Generate a new set of build files.')
  249. AddCommonOptions(subp)
  250. subp.add_argument('--swarming-targets-file',
  251. help='generates runtime dependencies for targets listed '
  252. 'in file as .isolate and .isolated.gen.json files. '
  253. 'Targets should be listed by name, separated by '
  254. 'newline.')
  255. subp.add_argument('--json-output',
  256. help='Write errors to json.output')
  257. subp.add_argument('--rts-target-change-recall',
  258. type=float,
  259. help='how much safety is needed when selecting tests. '
  260. '0.0 is the lowest and 1.0 is the highest')
  261. subp.add_argument('path',
  262. help='path to generate build into')
  263. subp.set_defaults(func=self.CmdGen)
  264. subp = subps.add_parser('isolate-everything',
  265. description='Generates a .isolate for all targets. '
  266. 'Requires that mb.py gen has already '
  267. 'been run.')
  268. AddCommonOptions(subp)
  269. subp.set_defaults(func=self.CmdIsolateEverything)
  270. subp.add_argument('path',
  271. help='path build was generated into')
  272. subp = subps.add_parser('isolate',
  273. description='Generate the .isolate files for a '
  274. 'given binary.')
  275. AddCommonOptions(subp)
  276. subp.add_argument('--no-build', dest='build', default=True,
  277. action='store_false',
  278. help='Do not build, just isolate')
  279. subp.add_argument('-j', '--jobs', type=int,
  280. help='Number of jobs to pass to ninja')
  281. subp.add_argument('path',
  282. help='path build was generated into')
  283. subp.add_argument('target',
  284. help='ninja target to generate the isolate for')
  285. subp.set_defaults(func=self.CmdIsolate)
  286. subp = subps.add_parser('lookup',
  287. description='Look up the command for a given '
  288. 'config or builder.')
  289. AddCommonOptions(subp)
  290. subp.add_argument('--quiet', default=False, action='store_true',
  291. help='Print out just the arguments, '
  292. 'do not emulate the output of the gen subcommand.')
  293. subp.add_argument('--recursive', default=False, action='store_true',
  294. help='Lookup arguments from imported files, '
  295. 'implies --quiet')
  296. subp.set_defaults(func=self.CmdLookup)
  297. subp = subps.add_parser('try',
  298. description='Try your change on a remote builder')
  299. AddCommonOptions(subp)
  300. subp.add_argument('target',
  301. help='ninja target to build and run')
  302. subp.add_argument('--force', default=False, action='store_true',
  303. help='Force the job to run. Ignores local checkout state;'
  304. ' by default, the tool doesn\'t trigger jobs if there are'
  305. ' local changes which are not present on Gerrit.')
  306. subp.set_defaults(func=self.CmdTry)
  307. subp = subps.add_parser(
  308. 'run', formatter_class=argparse.RawDescriptionHelpFormatter)
  309. subp.description = (
  310. 'Build, isolate, and run the given binary with the command line\n'
  311. 'listed in the isolate. You may pass extra arguments after the\n'
  312. 'target; use "--" if the extra arguments need to include switches.\n'
  313. '\n'
  314. 'Examples:\n'
  315. '\n'
  316. ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n'
  317. ' //out/Default content_browsertests\n'
  318. '\n'
  319. ' % tools/mb/mb.py run out/Default content_browsertests\n'
  320. '\n'
  321. ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n'
  322. ' --test-launcher-retry-limit=0'
  323. '\n'
  324. )
  325. AddCommonOptions(subp)
  326. subp.add_argument('-j', '--jobs', type=int,
  327. help='Number of jobs to pass to ninja')
  328. subp.add_argument('--no-build', dest='build', default=True,
  329. action='store_false',
  330. help='Do not build, just isolate and run')
  331. subp.add_argument('path',
  332. help=('path to generate build into (or use).'
  333. ' This can be either a regular path or a '
  334. 'GN-style source-relative path like '
  335. '//out/Default.'))
  336. subp.add_argument('-s', '--swarmed', action='store_true',
  337. help='Run under swarming with the default dimensions')
  338. subp.add_argument('-d', '--dimension', default=[], action='append', nargs=2,
  339. dest='dimensions', metavar='FOO bar',
  340. help='dimension to filter on')
  341. subp.add_argument('--tags', default=[], action='append', metavar='FOO:BAR',
  342. help='Tags to assign to the swarming task')
  343. subp.add_argument('--no-default-dimensions', action='store_false',
  344. dest='default_dimensions', default=True,
  345. help='Do not automatically add dimensions to the task')
  346. subp.add_argument('target',
  347. help='ninja target to build and run')
  348. subp.add_argument('extra_args', nargs='*',
  349. help=('extra args to pass to the isolate to run. Use '
  350. '"--" as the first arg if you need to pass '
  351. 'switches'))
  352. subp.set_defaults(func=self.CmdRun)
  353. subp = subps.add_parser('validate',
  354. description='Validate the config file.')
  355. AddExpansionOptions(subp)
  356. subp.add_argument('--expectations-dir',
  357. metavar='PATH',
  358. help='path to dir containing expectation files')
  359. subp.add_argument('--skip-dcheck-check',
  360. help='Skip check for dcheck_always_on.',
  361. action='store_true')
  362. subp.set_defaults(func=self.CmdValidate)
  363. subp = subps.add_parser('zip',
  364. description='Generate a .zip containing the files '
  365. 'needed for a given binary.')
  366. AddCommonOptions(subp)
  367. subp.add_argument('--no-build', dest='build', default=True,
  368. action='store_false',
  369. help='Do not build, just isolate')
  370. subp.add_argument('-j', '--jobs', type=int,
  371. help='Number of jobs to pass to ninja')
  372. subp.add_argument('path',
  373. help='path build was generated into')
  374. subp.add_argument('target',
  375. help='ninja target to generate the isolate for')
  376. subp.add_argument('zip_path',
  377. help='path to zip file to create')
  378. subp.set_defaults(func=self.CmdZip)
  379. subp = subps.add_parser('help',
  380. help='Get help on a subcommand.')
  381. subp.add_argument(nargs='?', action='store', dest='subcommand',
  382. help='The command to get help for.')
  383. subp.set_defaults(func=self.CmdHelp)
  384. self.args = parser.parse_args(argv)
  385. def DumpInputFiles(self):
  386. def DumpContentsOfFilePassedTo(arg_name, path):
  387. if path and self.Exists(path):
  388. self.Print("\n# To recreate the file passed to %s:" % arg_name)
  389. self.Print("%% cat > %s <<EOF" % path)
  390. contents = self.ReadFile(path)
  391. self.Print(contents)
  392. self.Print("EOF\n%\n")
  393. if getattr(self.args, 'input_path', None):
  394. DumpContentsOfFilePassedTo(
  395. 'argv[0] (input_path)', self.args.input_path)
  396. if getattr(self.args, 'swarming_targets_file', None):
  397. DumpContentsOfFilePassedTo(
  398. '--swarming-targets-file', self.args.swarming_targets_file)
  399. def CmdAnalyze(self):
  400. vals = self.Lookup()
  401. return self.RunGNAnalyze(vals)
  402. def CmdExport(self):
  403. obj = self._ToJsonish()
  404. s = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '))
  405. self.Print(s)
  406. return 0
  407. def CmdTrain(self):
  408. expectations_dir = self.args.expectations_dir
  409. if not self.Exists(expectations_dir):
  410. self.Print('Expectations dir (%s) does not exist.' % expectations_dir)
  411. return 1
  412. # Removing every expectation file then immediately re-generating them will
  413. # clear out deleted groups.
  414. for f in self.ListDir(expectations_dir):
  415. self.RemoveFile(os.path.join(expectations_dir, f))
  416. obj = self._ToJsonish()
  417. for builder_group, builder in sorted(obj.items()):
  418. expectation_file = os.path.join(expectations_dir, builder_group + '.json')
  419. json_s = json.dumps(builder,
  420. indent=2,
  421. sort_keys=True,
  422. separators=(',', ': '))
  423. self.WriteFile(expectation_file, json_s)
  424. return 0
  425. def RtsSelect(self):
  426. if self.args.rts == 'rts-ml-chromium':
  427. model_dir = self.PathJoin(self.chromium_src_dir, 'testing', 'rts',
  428. self.args.rts, self._CipdPlatform())
  429. exe = self.PathJoin(model_dir, self.args.rts)
  430. else:
  431. model_dir = self.PathJoin(self.chromium_src_dir, 'testing', 'rts',
  432. self._CipdPlatform())
  433. exe = self.PathJoin(model_dir, self.args.rts)
  434. if self.platform == 'win32':
  435. exe += '.exe'
  436. args = [
  437. exe,
  438. 'select',
  439. '-gen-inverse',
  440. '-model-dir', model_dir, \
  441. '-out', self.PathJoin(self.ToAbsPath(self.args.path), self.rts_out_dir),
  442. '-checkout', self.chromium_src_dir,
  443. ]
  444. if self.args.rts_target_change_recall:
  445. if (self.args.rts_target_change_recall < 0
  446. or self.args.rts_target_change_recall > 1):
  447. self.WriteFailureAndRaise(
  448. 'rts-target-change-recall must be between (0 and 1]', None)
  449. args += ['-target-change-recall', str(self.args.rts_target_change_recall)]
  450. ret, _, err = self.Run(args, force_verbose=True)
  451. if ret != 0:
  452. self.WriteFailureAndRaise(err, None)
  453. def CmdGen(self):
  454. if self.args.rts:
  455. self.RtsSelect()
  456. vals = self.Lookup()
  457. return self.RunGNGen(vals)
  458. def CmdGetSwarmingCommand(self):
  459. vals = self.GetConfig()
  460. command, _ = self.GetSwarmingCommand(self.args.target, vals)
  461. if self.args.as_list:
  462. self.Print(json.dumps(command))
  463. else:
  464. self.Print(' '.join(command))
  465. return 0
  466. def CmdIsolateEverything(self):
  467. vals = self.Lookup()
  468. return self.RunGNGenAllIsolates(vals)
  469. def CmdHelp(self):
  470. if self.args.subcommand:
  471. self.ParseArgs([self.args.subcommand, '--help'])
  472. else:
  473. self.ParseArgs(['--help'])
  474. def CmdIsolate(self):
  475. vals = self.GetConfig()
  476. if not vals:
  477. return 1
  478. if self.args.build:
  479. ret = self.Build(self.args.target)
  480. if ret != 0:
  481. return ret
  482. return self.RunGNIsolate(vals)
  483. def CmdLookup(self):
  484. vals = self.Lookup()
  485. _, gn_args = self.GNArgs(vals, expand_imports=self.args.recursive)
  486. if self.args.quiet or self.args.recursive:
  487. self.Print(gn_args, end='')
  488. else:
  489. cmd = self.GNCmd('gen', '_path_')
  490. self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
  491. self.PrintCmd(cmd)
  492. return 0
  493. def CmdTry(self):
  494. ninja_target = self.args.target
  495. if ninja_target.startswith('//'):
  496. self.Print("Expected a ninja target like base_unittests, got %s" % (
  497. ninja_target))
  498. return 1
  499. _, out, _ = self.Run(['git', 'cl', 'diff', '--stat'], force_verbose=False)
  500. if out:
  501. self.Print("Your checkout appears to local changes which are not uploaded"
  502. " to Gerrit. Changes must be committed and uploaded to Gerrit"
  503. " to be tested using this tool.")
  504. if not self.args.force:
  505. return 1
  506. json_path = self.PathJoin(self.chromium_src_dir, 'out.json')
  507. try:
  508. ret, out, err = self.Run(
  509. ['git', 'cl', 'issue', '--json=out.json'], force_verbose=False)
  510. if ret != 0:
  511. self.Print(
  512. "Unable to fetch current issue. Output and error:\n%s\n%s" % (
  513. out, err
  514. ))
  515. return ret
  516. with open(json_path) as f:
  517. issue_data = json.load(f)
  518. finally:
  519. if self.Exists(json_path):
  520. os.unlink(json_path)
  521. if not issue_data['issue']:
  522. self.Print("Missing issue data. Upload your CL to Gerrit and try again.")
  523. return 1
  524. class LedException(Exception):
  525. pass
  526. def run_cmd(previous_res, cmd):
  527. if self.args.verbose:
  528. self.Print(('| ' if previous_res else '') + ' '.join(cmd))
  529. res, out, err = self.Call(cmd, input=previous_res)
  530. if res != 0:
  531. self.Print("Err while running '%s'. Output:\n%s\nstderr:\n%s" % (
  532. ' '.join(cmd), out, err))
  533. raise LedException()
  534. return out
  535. try:
  536. result = LedResult(None, run_cmd).then(
  537. # TODO(martiniss): maybe don't always assume the bucket?
  538. 'led', 'get-builder', 'luci.chromium.try:%s' % self.args.builder).then(
  539. 'led', 'edit', '-r', 'chromium_trybot_experimental',
  540. '-p', 'tests=["%s"]' % ninja_target).then(
  541. 'led', 'edit-system', '--tag=purpose:user-debug-mb-try').then(
  542. 'led', 'edit-cr-cl', issue_data['issue_url']).then(
  543. 'led', 'launch').result
  544. except LedException:
  545. self.Print("If this is an unexpected error message, please file a bug"
  546. " with https://goto.google.com/mb-try-bug")
  547. raise
  548. swarming_data = json.loads(result)['swarming']
  549. self.Print("Launched task at https://%s/task?id=%s" % (
  550. swarming_data['host_name'], swarming_data['task_id']))
  551. def CmdRun(self):
  552. vals = self.GetConfig()
  553. if not vals:
  554. return 1
  555. if self.args.build:
  556. self.Print('')
  557. ret = self.Build(self.args.target)
  558. if ret:
  559. return ret
  560. self.Print('')
  561. ret = self.RunGNIsolate(vals)
  562. if ret:
  563. return ret
  564. self.Print('')
  565. if self.args.swarmed:
  566. cmd, _ = self.GetSwarmingCommand(self.args.target, vals)
  567. return self._RunUnderSwarming(self.args.path, self.args.target, cmd)
  568. return self._RunLocallyIsolated(self.args.path, self.args.target)
  569. def CmdZip(self):
  570. ret = self.CmdIsolate()
  571. if ret:
  572. return ret
  573. zip_dir = None
  574. try:
  575. zip_dir = self.TempDir()
  576. remap_cmd = [
  577. self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
  578. self.isolate_exe), 'remap', '-i',
  579. self.PathJoin(self.args.path, self.args.target + '.isolate'),
  580. '-outdir', zip_dir
  581. ]
  582. ret, _, _ = self.Run(remap_cmd)
  583. if ret:
  584. return ret
  585. zip_path = self.args.zip_path
  586. with zipfile.ZipFile(
  587. zip_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as fp:
  588. for root, _, files in os.walk(zip_dir):
  589. for filename in files:
  590. path = self.PathJoin(root, filename)
  591. fp.write(path, self.RelPath(path, zip_dir))
  592. return 0
  593. finally:
  594. if zip_dir:
  595. self.RemoveDirectory(zip_dir)
  596. def _RunUnderSwarming(self, build_dir, target, isolate_cmd):
  597. cas_instance = 'chromium-swarm'
  598. swarming_server = 'chromium-swarm.appspot.com'
  599. # TODO(dpranke): Look up the information for the target in
  600. # the //testing/buildbot.json file, if possible, so that we
  601. # can determine the isolate target, command line, and additional
  602. # swarming parameters, if possible.
  603. #
  604. # TODO(dpranke): Also, add support for sharding and merging results.
  605. dimensions = []
  606. for k, v in self._DefaultDimensions() + self.args.dimensions:
  607. dimensions += ['-d', '%s=%s' % (k, v)]
  608. archive_json_path = self.ToSrcRelPath(
  609. '%s/%s.archive.json' % (build_dir, target))
  610. cmd = [
  611. self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
  612. self.isolate_exe),
  613. 'archive',
  614. '-i',
  615. self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
  616. '-cas-instance',
  617. cas_instance,
  618. '-dump-json',
  619. archive_json_path,
  620. ]
  621. # Talking to the isolateserver may fail because we're not logged in.
  622. # We trap the command explicitly and rewrite the error output so that
  623. # the error message is actually correct for a Chromium check out.
  624. self.PrintCmd(cmd)
  625. ret, out, _ = self.Run(cmd, force_verbose=False)
  626. if ret:
  627. self.Print(' -> returned %d' % ret)
  628. if out:
  629. self.Print(out, end='')
  630. return ret
  631. try:
  632. archive_hashes = json.loads(self.ReadFile(archive_json_path))
  633. except Exception:
  634. self.Print(
  635. 'Failed to read JSON file "%s"' % archive_json_path, file=sys.stderr)
  636. return 1
  637. try:
  638. cas_digest = archive_hashes[target]
  639. except Exception:
  640. self.Print(
  641. 'Cannot find hash for "%s" in "%s", file content: %s' %
  642. (target, archive_json_path, archive_hashes),
  643. file=sys.stderr)
  644. return 1
  645. tags = ['-tag=%s' % tag for tag in self.args.tags]
  646. try:
  647. json_dir = self.TempDir()
  648. json_file = self.PathJoin(json_dir, 'task.json')
  649. cmd = [
  650. self.PathJoin('tools', 'luci-go', 'swarming'),
  651. 'trigger',
  652. '-digest',
  653. cas_digest,
  654. '-server',
  655. swarming_server,
  656. '-tag=purpose:user-debug-mb',
  657. '-relative-cwd',
  658. self.ToSrcRelPath(build_dir),
  659. '-dump-json',
  660. json_file,
  661. ] + tags + dimensions + ['--'] + list(isolate_cmd)
  662. if self.args.extra_args:
  663. cmd += self.args.extra_args
  664. self.Print('')
  665. ret, _, _ = self.Run(cmd, force_verbose=True, capture_output=False)
  666. if ret:
  667. return ret
  668. task_json = self.ReadFile(json_file)
  669. task_id = json.loads(task_json)["tasks"][0]['task_id']
  670. collect_output = self.PathJoin(json_dir, 'collect_output.json')
  671. cmd = [
  672. self.PathJoin('tools', 'luci-go', 'swarming'),
  673. 'collect',
  674. '-server',
  675. swarming_server,
  676. '-task-output-stdout=console',
  677. '-task-summary-json',
  678. collect_output,
  679. task_id,
  680. ]
  681. ret, _, _ = self.Run(cmd, force_verbose=True, capture_output=False)
  682. if ret != 0:
  683. return ret
  684. collect_json = json.loads(self.ReadFile(collect_output))
  685. # The exit_code field might not be included if the task was successful.
  686. ret = int(
  687. collect_json.get(task_id, {}).get('results', {}).get('exit_code', 0))
  688. finally:
  689. if json_dir:
  690. self.RemoveDirectory(json_dir)
  691. return ret
  692. def _RunLocallyIsolated(self, build_dir, target):
  693. cmd = [
  694. self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
  695. self.isolate_exe),
  696. 'run',
  697. '-i',
  698. self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
  699. ]
  700. if self.args.extra_args:
  701. cmd += ['--'] + self.args.extra_args
  702. ret, _, _ = self.Run(cmd, force_verbose=True, capture_output=False)
  703. return ret
  704. def _DefaultDimensions(self):
  705. if not self.args.default_dimensions:
  706. return []
  707. # This code is naive and just picks reasonable defaults per platform.
  708. if self.platform == 'darwin':
  709. os_dim = ('os', 'Mac-10.13')
  710. elif self.platform.startswith('linux'):
  711. os_dim = ('os', 'Ubuntu-16.04')
  712. elif self.platform == 'win32':
  713. os_dim = ('os', 'Windows-10')
  714. else:
  715. raise MBErr('unrecognized platform string "%s"' % self.platform)
  716. return [('pool', 'chromium.tests'),
  717. ('cpu', 'x86-64'),
  718. os_dim]
  719. def _ToJsonish(self):
  720. """Dumps the config file into a json-friendly expanded dict.
  721. Returns:
  722. A dict with builder group -> builder -> all GN args mapping.
  723. """
  724. self.ReadConfigFile(self.args.config_file)
  725. obj = {}
  726. for builder_group, builders in self.builder_groups.items():
  727. obj[builder_group] = {}
  728. for builder in builders:
  729. config = self.builder_groups[builder_group][builder]
  730. if not config:
  731. continue
  732. def flatten(config):
  733. flattened_config = FlattenConfig(self.configs, self.mixins, config)
  734. if flattened_config['gn_args'] == 'error':
  735. return None
  736. args = {'gn_args': gn_helpers.FromGNArgs(flattened_config['gn_args'])}
  737. if flattened_config.get('args_file'):
  738. args['args_file'] = flattened_config['args_file']
  739. return args
  740. if isinstance(config, dict):
  741. # This is a 'phased' builder. Each key in the config is a different
  742. # phase of the builder.
  743. args = {}
  744. for k, v in config.items():
  745. flattened = flatten(v)
  746. if flattened is None:
  747. continue
  748. args[k] = flattened
  749. elif config.startswith('//'):
  750. args = config
  751. else:
  752. args = flatten(config)
  753. if args is None:
  754. continue
  755. obj[builder_group][builder] = args
  756. return obj
  757. def CmdValidate(self, print_ok=True):
  758. errs = []
  759. self.ReadConfigFile(self.args.config_file)
  760. # Build a list of all of the configs referenced by builders.
  761. all_configs = validation.GetAllConfigs(self.builder_groups)
  762. # Check that every referenced args file or config actually exists.
  763. for config, loc in all_configs.items():
  764. if config.startswith('//'):
  765. if not self.Exists(self.ToAbsPath(config)):
  766. errs.append('Unknown args file "%s" referenced from "%s".' %
  767. (config, loc))
  768. elif not config in self.configs:
  769. errs.append('Unknown config "%s" referenced from "%s".' %
  770. (config, loc))
  771. # Check that every config and mixin is referenced.
  772. validation.CheckAllConfigsAndMixinsReferenced(errs, all_configs,
  773. self.configs, self.mixins)
  774. if self.args.config_file == self.default_config:
  775. validation.EnsureNoProprietaryMixins(errs, self.builder_groups,
  776. self.configs, self.mixins)
  777. validation.CheckDuplicateConfigs(errs, self.configs, self.mixins,
  778. self.builder_groups, FlattenConfig)
  779. if not self.args.skip_dcheck_check:
  780. self._ValidateEach(errs, validation.CheckDebugDCheckOrOfficial)
  781. if errs:
  782. raise MBErr(('mb config file %s has problems:\n ' %
  783. self.args.config_file) + '\n '.join(errs))
  784. expectations_dir = self.args.expectations_dir
  785. # TODO(crbug.com/1117577): Force all versions of mb_config.pyl to have
  786. # expectations. For now, just ignore those that don't have them.
  787. if self.Exists(expectations_dir):
  788. jsonish_blob = self._ToJsonish()
  789. if not validation.CheckExpectations(self, jsonish_blob, expectations_dir):
  790. raise MBErr("Expectations out of date. Please run 'mb.py train'.")
  791. validation.CheckKeyOrdering(errs, self.builder_groups, self.configs,
  792. self.mixins)
  793. if errs:
  794. raise MBErr('mb config file not sorted:\n' + '\n'.join(errs))
  795. if print_ok:
  796. self.Print('mb config file %s looks ok.' % self.args.config_file)
  797. return 0
  798. def _ValidateEach(self, errs, validate):
  799. """Checks a validate function against every builder config.
  800. This loops over all the builders in the config file, invoking the
  801. validate function against the full set of GN args. Any errors found
  802. should be appended to the errs list passed in; the validation
  803. function signature is
  804. validate(errs:list, gn_args:dict, builder_group:str, builder:str,
  805. phase:(str|None))"""
  806. for builder_group, builders in self.builder_groups.items():
  807. for builder, config in builders.items():
  808. if isinstance(config, dict):
  809. for phase, phase_config in config.items():
  810. vals = FlattenConfig(self.configs, self.mixins, phase_config)
  811. if vals['gn_args'] == 'error':
  812. continue
  813. try:
  814. parsed_gn_args, _ = self.GNArgs(vals, expand_imports=True)
  815. except IOError:
  816. # The builder must use an args file that was not checked out or
  817. # generated, so we should just ignore it.
  818. parsed_gn_args, _ = self.GNArgs(vals, expand_imports=False)
  819. validate(errs, parsed_gn_args, builder_group, builder, phase)
  820. else:
  821. vals = FlattenConfig(self.configs, self.mixins, config)
  822. if vals['gn_args'] == 'error':
  823. continue
  824. try:
  825. parsed_gn_args, _ = self.GNArgs(vals, expand_imports=True)
  826. except IOError:
  827. # The builder must use an args file that was not checked out or
  828. # generated, so we should just ignore it.
  829. parsed_gn_args, _ = self.GNArgs(vals, expand_imports=False)
  830. validate(errs, parsed_gn_args, builder_group, builder, phase=None)
  831. def GetConfig(self):
  832. build_dir = self.args.path
  833. vals = DefaultVals()
  834. if self.args.builder or self.args.builder_group or self.args.config:
  835. vals = self.Lookup()
  836. # Re-run gn gen in order to ensure the config is consistent with the
  837. # build dir.
  838. self.RunGNGen(vals)
  839. return vals
  840. toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
  841. 'toolchain.ninja')
  842. if not self.Exists(toolchain_path):
  843. self.Print('Must either specify a path to an existing GN build dir '
  844. 'or pass in a -m/-b pair or a -c flag to specify the '
  845. 'configuration')
  846. return {}
  847. vals['gn_args'] = self.GNArgsFromDir(build_dir)
  848. return vals
  849. def GNArgsFromDir(self, build_dir):
  850. args_contents = ""
  851. gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')
  852. if self.Exists(gn_args_path):
  853. args_contents = self.ReadFile(gn_args_path)
  854. # Handle any .gni file imports, e.g. the ones used by CrOS. This should
  855. # be automatically handled by gn_helpers.FromGNArgs (via its call to
  856. # gn_helpers.GNValueParser.ReplaceImports), but that currently breaks
  857. # mb_unittest since it mocks out file reads itself instead of using
  858. # pyfakefs. This results in gn_helpers trying to read a non-existent file.
  859. # The implementation of ReplaceImports here can be removed once the
  860. # unittests use pyfakefs.
  861. def ReplaceImports(input_contents):
  862. output_contents = ''
  863. for l in input_contents.splitlines(True):
  864. if not l.strip().startswith('#') and 'import(' in l:
  865. import_file = l.split('"', 2)[1]
  866. import_file = self.ToAbsPath(import_file)
  867. imported_contents = self.ReadFile(import_file)
  868. output_contents += ReplaceImports(imported_contents) + '\n'
  869. else:
  870. output_contents += l
  871. return output_contents
  872. args_contents = ReplaceImports(args_contents)
  873. args_dict = gn_helpers.FromGNArgs(args_contents)
  874. # Re-add the quotes around strings so they show up as they would in the
  875. # args.gn file.
  876. for k, v in args_dict.items():
  877. if isinstance(v, str):
  878. args_dict[k] = '"%s"' % v
  879. return ' '.join(['%s=%s' % (k, v) for (k, v) in args_dict.items()])
  880. def Lookup(self):
  881. self.ReadConfigFile(self.args.config_file)
  882. try:
  883. config = self.ConfigFromArgs()
  884. except MBErr as e:
  885. # TODO(crbug.com/912681) While iOS bots are migrated to use the
  886. # Chromium recipe, we want to ensure that we're checking MB's
  887. # configurations first before going to iOS.
  888. # This is to be removed once the migration is complete.
  889. vals = self.ReadIOSBotConfig()
  890. if not vals:
  891. raise e
  892. return vals
  893. # TODO(crbug.com/912681) Some iOS bots have a definition, with ios_error
  894. # as an indicator that it's incorrect. We utilize this to check the
  895. # iOS JSON instead, and error out if there exists no definition at all.
  896. # This is to be removed once the migration is complete.
  897. if config == 'ios_error':
  898. vals = self.ReadIOSBotConfig()
  899. if not vals:
  900. raise MBErr('No iOS definition was found. Please ensure there is a '
  901. 'definition for the given iOS bot under '
  902. 'mb_config.pyl or a JSON file definition under '
  903. '//ios/build/bots.')
  904. return vals
  905. if config.startswith('//'):
  906. if not self.Exists(self.ToAbsPath(config)):
  907. raise MBErr('args file "%s" not found' % config)
  908. vals = DefaultVals()
  909. vals['args_file'] = config
  910. else:
  911. if not config in self.configs:
  912. raise MBErr(
  913. 'Config "%s" not found in %s' % (config, self.args.config_file))
  914. vals = FlattenConfig(self.configs, self.mixins, config)
  915. return vals
  916. def ReadIOSBotConfig(self):
  917. if not self.args.builder_group or not self.args.builder:
  918. return {}
  919. path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots',
  920. self.args.builder_group, self.args.builder + '.json')
  921. if not self.Exists(path):
  922. return {}
  923. contents = json.loads(self.ReadFile(path))
  924. gn_args = ' '.join(contents.get('gn_args', []))
  925. vals = DefaultVals()
  926. vals['gn_args'] = gn_args
  927. return vals
  928. def ReadConfigFile(self, config_file):
  929. if not self.Exists(config_file):
  930. raise MBErr('config file not found at %s' % config_file)
  931. try:
  932. contents = ast.literal_eval(self.ReadFile(config_file))
  933. except SyntaxError as e:
  934. raise MBErr('Failed to parse config file "%s": %s' %
  935. (config_file, e)) from e
  936. self.configs = contents['configs']
  937. self.mixins = contents['mixins']
  938. self.builder_groups = contents.get('builder_groups')
  939. self.public_artifact_builders = contents.get('public_artifact_builders')
  940. def ReadIsolateMap(self):
  941. if not self.args.isolate_map_files:
  942. self.args.isolate_map_files = [self.default_isolate_map]
  943. for f in self.args.isolate_map_files:
  944. if not self.Exists(f):
  945. raise MBErr('isolate map file not found at %s' % f)
  946. isolate_maps = {}
  947. for isolate_map in self.args.isolate_map_files:
  948. try:
  949. isolate_map = ast.literal_eval(self.ReadFile(isolate_map))
  950. duplicates = set(isolate_map).intersection(isolate_maps)
  951. if duplicates:
  952. raise MBErr(
  953. 'Duplicate targets in isolate map files: %s.' %
  954. ', '.join(duplicates))
  955. isolate_maps.update(isolate_map)
  956. except SyntaxError as e:
  957. raise MBErr('Failed to parse isolate map file "%s": %s' %
  958. (isolate_map, e)) from e
  959. return isolate_maps
  960. def ConfigFromArgs(self):
  961. if self.args.config:
  962. if self.args.builder_group or self.args.builder:
  963. raise MBErr('Can not specific both -c/--config and --builder-group '
  964. 'or -b/--builder')
  965. return self.args.config
  966. if not self.args.builder_group or not self.args.builder:
  967. raise MBErr('Must specify either -c/--config or '
  968. '(--builder-group and -b/--builder)')
  969. if not self.args.builder_group in self.builder_groups:
  970. raise MBErr('Builder group name "%s" not found in "%s"' %
  971. (self.args.builder_group, self.args.config_file))
  972. if not self.args.builder in self.builder_groups[self.args.builder_group]:
  973. raise MBErr('Builder name "%s" not found under groups[%s] in "%s"' %
  974. (self.args.builder, self.args.builder_group,
  975. self.args.config_file))
  976. config = self.builder_groups[self.args.builder_group][self.args.builder]
  977. if isinstance(config, dict):
  978. if self.args.phase is None:
  979. raise MBErr('Must specify a build --phase for %s on %s' %
  980. (self.args.builder, self.args.builder_group))
  981. phase = str(self.args.phase)
  982. if phase not in config:
  983. raise MBErr('Phase %s doesn\'t exist for %s on %s' %
  984. (phase, self.args.builder, self.args.builder_group))
  985. return config[phase]
  986. if self.args.phase is not None:
  987. raise MBErr('Must not specify a build --phase for %s on %s' %
  988. (self.args.builder, self.args.builder_group))
  989. return config
  990. def RunGNGen(self, vals, compute_inputs_for_analyze=False, check=True):
  991. build_dir = self.args.path
  992. if check:
  993. cmd = self.GNCmd('gen', build_dir, '--check')
  994. else:
  995. cmd = self.GNCmd('gen', build_dir)
  996. _, gn_args = self.GNArgs(vals)
  997. if compute_inputs_for_analyze:
  998. gn_args += ' compute_inputs_for_analyze=true'
  999. # Since GN hasn't run yet, the build directory may not even exist.
  1000. self.MaybeMakeDirectory(self.ToAbsPath(build_dir))
  1001. gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
  1002. self.WriteFile(gn_args_path, gn_args, force_verbose=True)
  1003. if getattr(self.args, 'swarming_targets_file', None):
  1004. # We need GN to generate the list of runtime dependencies for
  1005. # the compile targets listed (one per line) in the file so
  1006. # we can run them via swarming. We use gn_isolate_map.pyl to convert
  1007. # the compile targets to the matching GN labels.
  1008. path = self.args.swarming_targets_file
  1009. if not self.Exists(path):
  1010. self.WriteFailureAndRaise('"%s" does not exist' % path,
  1011. output_path=None)
  1012. contents = self.ReadFile(path)
  1013. isolate_targets = set(contents.splitlines())
  1014. isolate_map = self.ReadIsolateMap()
  1015. self.RemovePossiblyStaleRuntimeDepsFiles(vals, isolate_targets,
  1016. isolate_map, build_dir)
  1017. err, labels = self.MapTargetsToLabels(isolate_map, isolate_targets)
  1018. if err:
  1019. raise MBErr(err)
  1020. gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
  1021. self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n')
  1022. cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
  1023. ret, output, _ = self.Run(cmd)
  1024. if ret != 0:
  1025. if self.args.json_output:
  1026. # write errors to json.output
  1027. self.WriteJSON({'output': output}, self.args.json_output)
  1028. # If `gn gen` failed, we should exit early rather than trying to
  1029. # generate isolates. Run() will have already logged any error output.
  1030. self.Print('GN gen failed: %d' % ret)
  1031. return ret
  1032. if getattr(self.args, 'swarming_targets_file', None):
  1033. ret = self.GenerateIsolates(vals, isolate_targets, isolate_map, build_dir)
  1034. return ret
  1035. def RunGNGenAllIsolates(self, vals):
  1036. """
  1037. This command generates all .isolate files.
  1038. This command assumes that "mb.py gen" has already been run, as it relies on
  1039. "gn ls" to fetch all gn targets. If uses that output, combined with the
  1040. isolate_map, to determine all isolates that can be generated for the current
  1041. gn configuration.
  1042. """
  1043. build_dir = self.args.path
  1044. ret, output, _ = self.Run(self.GNCmd('ls', build_dir),
  1045. force_verbose=False)
  1046. if ret != 0:
  1047. # If `gn ls` failed, we should exit early rather than trying to
  1048. # generate isolates.
  1049. self.Print('GN ls failed: %d' % ret)
  1050. return ret
  1051. # Create a reverse map from isolate label to isolate dict.
  1052. isolate_map = self.ReadIsolateMap()
  1053. isolate_dict_map = {}
  1054. for key, isolate_dict in isolate_map.items():
  1055. isolate_dict_map[isolate_dict['label']] = isolate_dict
  1056. isolate_dict_map[isolate_dict['label']]['isolate_key'] = key
  1057. runtime_deps = []
  1058. isolate_targets = []
  1059. # For every GN target, look up the isolate dict.
  1060. for line in output.splitlines():
  1061. target = line.strip()
  1062. if target in isolate_dict_map:
  1063. if isolate_dict_map[target]['type'] == 'additional_compile_target':
  1064. # By definition, additional_compile_targets are not tests, so we
  1065. # shouldn't generate isolates for them.
  1066. continue
  1067. isolate_targets.append(isolate_dict_map[target]['isolate_key'])
  1068. runtime_deps.append(target)
  1069. self.RemovePossiblyStaleRuntimeDepsFiles(vals, isolate_targets,
  1070. isolate_map, build_dir)
  1071. gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
  1072. self.WriteFile(gn_runtime_deps_path, '\n'.join(runtime_deps) + '\n')
  1073. cmd = self.GNCmd('gen', build_dir)
  1074. cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
  1075. self.Run(cmd)
  1076. return self.GenerateIsolates(vals, isolate_targets, isolate_map, build_dir)
  1077. def RemovePossiblyStaleRuntimeDepsFiles(self, vals, targets, isolate_map,
  1078. build_dir):
  1079. # TODO(crbug.com/932700): Because `gn gen --runtime-deps-list-file`
  1080. # puts the runtime_deps file in different locations based on the actual
  1081. # type of a target, we may end up with multiple possible runtime_deps
  1082. # files in a given build directory, where some of the entries might be
  1083. # stale (since we might be reusing an existing build directory).
  1084. #
  1085. # We need to be able to get the right one reliably; you might think
  1086. # we can just pick the newest file, but because GN won't update timestamps
  1087. # if the contents of the files change, an older runtime_deps
  1088. # file might actually be the one we should use over a newer one (see
  1089. # crbug.com/932387 for a more complete explanation and example).
  1090. #
  1091. # In order to avoid this, we need to delete any possible runtime_deps
  1092. # files *prior* to running GN. As long as the files aren't actually
  1093. # needed during the build, this hopefully will not cause unnecessary
  1094. # build work, and so it should be safe.
  1095. #
  1096. # Ultimately, we should just make sure we get the runtime_deps files
  1097. # in predictable locations so we don't have this issue at all, and
  1098. # that's what crbug.com/932700 is for.
  1099. possible_rpaths = self.PossibleRuntimeDepsPaths(vals, targets, isolate_map)
  1100. for rpaths in possible_rpaths.values():
  1101. for rpath in rpaths:
  1102. path = self.ToAbsPath(build_dir, rpath)
  1103. if self.Exists(path):
  1104. self.RemoveFile(path)
  1105. def _FilterOutUnneededSkylabDeps(self, deps):
  1106. """Filter out the runtime dependencies not used by Skylab.
  1107. Skylab is CrOS infra facilities for us to run hardware tests. These files
  1108. may appear in the test target's runtime_deps but unnecessary for our tests
  1109. to execute in a CrOS device.
  1110. """
  1111. file_ignore_list = [
  1112. re.compile(r'.*build/android.*'),
  1113. re.compile(r'.*build/chromeos.*'),
  1114. re.compile(r'.*build/cros_cache.*'),
  1115. # The following matches anything under //testing/ that isn't under
  1116. # //testing/buildbot/filters/.
  1117. re.compile(r'.*testing/(?!buildbot/filters).*'),
  1118. re.compile(r'.*third_party/chromite.*'),
  1119. # No test target should rely on files in [output_dir]/gen.
  1120. re.compile(r'^gen/.*'),
  1121. ]
  1122. return [f for f in deps if not any(r.match(f) for r in file_ignore_list)]
  1123. def _DedupDependencies(self, deps):
  1124. """Remove the deps already contained by other paths."""
  1125. def _add(root, path):
  1126. cur = path.popleft()
  1127. # Only continue the recursion if the path has child nodes
  1128. # AND the current node is not ended by other existing paths.
  1129. if path and root.get(cur) != {}:
  1130. return _add(root.setdefault(cur, {}), path)
  1131. # Cut this path, because child nodes are already included.
  1132. root[cur] = {}
  1133. return root
  1134. def _list(root, prefix, res):
  1135. for k, v in root.items():
  1136. if v == {}:
  1137. res.append('%s/%s' % (prefix, k))
  1138. continue
  1139. _list(v, '%s/%s' % (prefix, k), res)
  1140. return res
  1141. root = {}
  1142. for d in deps:
  1143. q = collections.deque(d.rstrip('/').split('/'))
  1144. _add(root, q)
  1145. return [p.lstrip('/') for p in _list(root, '', [])]
  1146. def GenerateIsolates(self, vals, ninja_targets, isolate_map, build_dir):
  1147. """
  1148. Generates isolates for a list of ninja targets.
  1149. Ninja targets are transformed to GN targets via isolate_map.
  1150. This function assumes that a previous invocation of "mb.py gen" has
  1151. generated runtime deps for all targets.
  1152. """
  1153. possible_rpaths = self.PossibleRuntimeDepsPaths(vals, ninja_targets,
  1154. isolate_map)
  1155. for target, rpaths in possible_rpaths.items():
  1156. # TODO(crbug.com/932700): We don't know where each .runtime_deps
  1157. # file might be, but assuming we called
  1158. # RemovePossiblyStaleRuntimeDepsFiles prior to calling `gn gen`,
  1159. # there should only be one file.
  1160. found_one = False
  1161. path_to_use = None
  1162. for r in rpaths:
  1163. path = self.ToAbsPath(build_dir, r)
  1164. if self.Exists(path):
  1165. if found_one:
  1166. raise MBErr('Found more than one of %s' % ', '.join(rpaths))
  1167. path_to_use = path
  1168. found_one = True
  1169. if not found_one:
  1170. raise MBErr('Did not find any of %s' % ', '.join(rpaths))
  1171. command, extra_files = self.GetSwarmingCommand(target, vals)
  1172. runtime_deps = self.ReadFile(path_to_use).splitlines()
  1173. runtime_deps = self._DedupDependencies(runtime_deps)
  1174. if 'is_skylab=true' in vals['gn_args']:
  1175. runtime_deps = self._FilterOutUnneededSkylabDeps(runtime_deps)
  1176. canonical_target = target.replace(':','_').replace('/','_')
  1177. ret = self.WriteIsolateFiles(build_dir, command, canonical_target,
  1178. runtime_deps, vals, extra_files)
  1179. if ret != 0:
  1180. return ret
  1181. return 0
  1182. def AddFilterFileArg(self, target, build_dir, command, inverted=False):
  1183. filter_file = ('%s_inverted' % target if inverted else target) + '.filter'
  1184. filter_file_path = self.PathJoin(self.rts_out_dir, filter_file)
  1185. abs_filter_file_path = self.ToAbsPath(build_dir, filter_file_path)
  1186. filter_exists = self.Exists(abs_filter_file_path)
  1187. if filter_exists:
  1188. command.append('--test-launcher-filter-file=%s' % filter_file_path)
  1189. self.Print('added RTS filter file to command: %s' % filter_file)
  1190. return filter_exists
  1191. def PossibleRuntimeDepsPaths(self, vals, ninja_targets, isolate_map):
  1192. """Returns a map of targets to possible .runtime_deps paths.
  1193. Each ninja target maps on to a GN label, but depending on the type
  1194. of the GN target, `gn gen --runtime-deps-list-file` will write
  1195. the .runtime_deps files into different locations. Unfortunately, in
  1196. some cases we don't actually know which of multiple locations will
  1197. actually be used, so we return all plausible candidates.
  1198. The paths that are returned are relative to the build directory.
  1199. """
  1200. android = 'target_os="android"' in vals['gn_args']
  1201. ios = 'target_os="ios"' in vals['gn_args']
  1202. fuchsia = 'target_os="fuchsia"' in vals['gn_args']
  1203. win = self.platform == 'win32' or 'target_os="win"' in vals['gn_args']
  1204. possible_runtime_deps_rpaths = {}
  1205. for target in ninja_targets:
  1206. target_type = isolate_map[target]['type']
  1207. label = isolate_map[target]['label']
  1208. stamp_runtime_deps = 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')
  1209. # TODO(https://crbug.com/876065): 'official_tests' use
  1210. # type='additional_compile_target' to isolate tests. This is not the
  1211. # intended use for 'additional_compile_target'.
  1212. if (target_type == 'additional_compile_target' and
  1213. target != 'official_tests'):
  1214. # By definition, additional_compile_targets are not tests, so we
  1215. # shouldn't generate isolates for them.
  1216. raise MBErr('Cannot generate isolate for %s since it is an '
  1217. 'additional_compile_target.' % target)
  1218. if fuchsia or ios or target_type == 'generated_script':
  1219. # iOS and Fuchsia targets end up as groups.
  1220. # generated_script targets are always actions.
  1221. rpaths = [stamp_runtime_deps]
  1222. elif android:
  1223. # Android targets may be either android_apk or executable. The former
  1224. # will result in runtime_deps associated with the stamp file, while the
  1225. # latter will result in runtime_deps associated with the executable.
  1226. label = isolate_map[target]['label']
  1227. rpaths = [
  1228. target + '.runtime_deps',
  1229. stamp_runtime_deps]
  1230. elif (target_type == 'script'
  1231. or isolate_map[target].get('label_type') == 'group'):
  1232. # For script targets, the build target is usually a group,
  1233. # for which gn generates the runtime_deps next to the stamp file
  1234. # for the label, which lives under the obj/ directory, but it may
  1235. # also be an executable.
  1236. label = isolate_map[target]['label']
  1237. rpaths = [stamp_runtime_deps]
  1238. if win:
  1239. rpaths += [ target + '.exe.runtime_deps' ]
  1240. else:
  1241. rpaths += [ target + '.runtime_deps' ]
  1242. elif win:
  1243. rpaths = [target + '.exe.runtime_deps']
  1244. else:
  1245. rpaths = [target + '.runtime_deps']
  1246. possible_runtime_deps_rpaths[target] = rpaths
  1247. return possible_runtime_deps_rpaths
  1248. def RunGNIsolate(self, vals):
  1249. target = self.args.target
  1250. isolate_map = self.ReadIsolateMap()
  1251. err, labels = self.MapTargetsToLabels(isolate_map, [target])
  1252. if err:
  1253. raise MBErr(err)
  1254. label = labels[0]
  1255. build_dir = self.args.path
  1256. command, extra_files = self.GetSwarmingCommand(target, vals)
  1257. # Any warning for an unused arg will get interleaved into the cmd's
  1258. # stdout. When that happens, the isolate step below will fail with an
  1259. # obscure error when it tries processing the lines of the warning. Fail
  1260. # quickly in that case to avoid confusion
  1261. cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps',
  1262. '--fail-on-unused-args')
  1263. ret, out, _ = self.Call(cmd)
  1264. if ret != 0:
  1265. if out:
  1266. self.Print(out)
  1267. return ret
  1268. runtime_deps = out.splitlines()
  1269. ret = self.WriteIsolateFiles(build_dir, command, target, runtime_deps, vals,
  1270. extra_files)
  1271. if ret != 0:
  1272. return ret
  1273. ret, _, _ = self.Run([
  1274. self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
  1275. self.isolate_exe),
  1276. 'check',
  1277. '-i',
  1278. self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
  1279. ],
  1280. capture_output=False)
  1281. return ret
  1282. def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, vals,
  1283. extra_files):
  1284. isolate_path = self.ToAbsPath(build_dir, target + '.isolate')
  1285. files = sorted(set(runtime_deps + extra_files))
  1286. # Complain if any file is a directory that's inside the build directory,
  1287. # since that makes incremental builds incorrect. See
  1288. # https://crbug.com/912946
  1289. is_android = 'target_os="android"' in vals['gn_args']
  1290. is_cros = ('target_os="chromeos"' in vals['gn_args']
  1291. or 'is_chromeos_device=true' in vals['gn_args'])
  1292. is_msan = 'is_msan=true' in vals['gn_args']
  1293. is_ios = 'target_os="ios"' in vals['gn_args']
  1294. # pylint: disable=consider-using-ternary
  1295. is_mac = ((self.platform == 'darwin' and not is_ios)
  1296. or 'target_os="mac"' in vals['gn_args'])
  1297. err = ''
  1298. for f in files:
  1299. # Skip a few configs that need extra cleanup for now.
  1300. # TODO(https://crbug.com/912946): Fix everything on all platforms and
  1301. # enable check everywhere.
  1302. if is_android:
  1303. break
  1304. # iOS has generated directories in gn data items.
  1305. # Skipping for iOS instead of listing all apps.
  1306. if is_ios:
  1307. break
  1308. # Skip a few existing violations that need to be cleaned up. Each of
  1309. # these will lead to incorrect incremental builds if their directory
  1310. # contents change. Do not add to this list, except for mac bundles until
  1311. # crbug.com/1000667 is fixed.
  1312. # TODO(https://crbug.com/912946): Remove this if statement.
  1313. if ((is_msan and f == 'instrumented_libraries_prebuilt/')
  1314. or f == 'mr_extension/' or # https://crbug.com/997947
  1315. f.startswith('nacl_test_data/') or
  1316. f.startswith('ppapi_nacl_tests_libs/') or
  1317. (is_cros and f in ( # https://crbug.com/1002509
  1318. 'chromevox_test_data/',
  1319. 'gen/ui/file_manager/file_manager/',
  1320. 'resources/chromeos/',
  1321. 'resources/chromeos/accessibility/accessibility_common/',
  1322. 'resources/chromeos/accessibility/chromevox/',
  1323. 'resources/chromeos/accessibility/select_to_speak/',
  1324. 'test_data/chrome/browser/resources/chromeos/accessibility/'
  1325. 'accessibility_common/',
  1326. 'test_data/chrome/browser/resources/chromeos/accessibility/'
  1327. 'chromevox/',
  1328. 'test_data/chrome/browser/resources/chromeos/accessibility/'
  1329. 'select_to_speak/',
  1330. )) or (is_mac and f in ( # https://crbug.com/1000667
  1331. 'Chromium Framework.framework/',
  1332. 'Chromium Helper.app/',
  1333. 'Chromium.app/',
  1334. 'ChromiumUpdater.app/',
  1335. 'ChromiumUpdater_test.app/',
  1336. 'Content Shell.app/',
  1337. 'Google Chrome Framework.framework/',
  1338. 'Google Chrome Helper (Alerts).app/',
  1339. 'Google Chrome Helper (GPU).app/',
  1340. 'Google Chrome Helper (Plugin).app/',
  1341. 'Google Chrome Helper (Renderer).app/',
  1342. 'Google Chrome Helper.app/',
  1343. 'Google Chrome.app/',
  1344. 'GoogleUpdater.app/',
  1345. 'GoogleUpdater_test.app/',
  1346. 'UpdaterTestApp Framework.framework/',
  1347. 'UpdaterTestApp.app/',
  1348. 'blink_deprecated_test_plugin.plugin/',
  1349. 'blink_test_plugin.plugin/',
  1350. 'corb_test_plugin.plugin/',
  1351. 'obj/tools/grit/brotli_mac_asan_workaround/',
  1352. 'ppapi_tests.plugin/',
  1353. 'ui_unittests Framework.framework/',
  1354. ))):
  1355. continue
  1356. # This runs before the build, so we can't use isdir(f). But
  1357. # isolate.py luckily requires data directories to end with '/', so we
  1358. # can check for that.
  1359. if not f.startswith('../../') and f.endswith('/'):
  1360. # Don't use self.PathJoin() -- all involved paths consistently use
  1361. # forward slashes, so don't add one single backslash on Windows.
  1362. err += '\n' + build_dir + '/' + f
  1363. if err:
  1364. self.Print('error: gn `data` items may not list generated directories; '
  1365. 'list files in directory instead for:' + err)
  1366. return 1
  1367. isolate = {
  1368. 'variables': {
  1369. 'command': command,
  1370. 'files': files,
  1371. }
  1372. }
  1373. # For more info about RTS, please see
  1374. # //docs/testing/regression-test-selection.md
  1375. if self.args.rts:
  1376. if target in self.banned_from_rts:
  1377. self.Print('%s is banned for RTS on this builder' % target)
  1378. isolate['variables']['command'] = command
  1379. else:
  1380. inverted_command = command.copy()
  1381. self.AddFilterFileArg(target, build_dir, command, inverted=False)
  1382. isolate['variables']['command'] = command
  1383. inverted_filter_exists = self.AddFilterFileArg(target,
  1384. build_dir,
  1385. inverted_command,
  1386. inverted=True)
  1387. if inverted_filter_exists:
  1388. isolate['variables']['inverted_command'] = inverted_command
  1389. self.WriteFile(isolate_path, json.dumps(isolate, sort_keys=True) + '\n')
  1390. self.WriteJSON(
  1391. {
  1392. 'args': [
  1393. '--isolate',
  1394. self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
  1395. ],
  1396. 'dir': self.chromium_src_dir,
  1397. 'version': 1,
  1398. },
  1399. isolate_path + 'd.gen.json',
  1400. )
  1401. return 0
  1402. def MapTargetsToLabels(self, isolate_map, targets):
  1403. labels = []
  1404. err = ''
  1405. for target in targets:
  1406. if target == 'all':
  1407. labels.append(target)
  1408. elif target.startswith('//'):
  1409. labels.append(target)
  1410. else:
  1411. if target in isolate_map:
  1412. if isolate_map[target]['type'] == 'unknown':
  1413. err += ('test target "%s" type is unknown\n' % target)
  1414. else:
  1415. labels.append(isolate_map[target]['label'])
  1416. else:
  1417. err += ('target "%s" not found in '
  1418. '//testing/buildbot/gn_isolate_map.pyl\n' % target)
  1419. return err, labels
  1420. def GNCmd(self, subcommand, path, *args):
  1421. if self.platform.startswith('linux'):
  1422. subdir, exe = 'linux64', 'gn'
  1423. elif self.platform == 'darwin':
  1424. subdir, exe = 'mac', 'gn'
  1425. elif self.platform == 'aix6':
  1426. subdir, exe = 'aix', 'gn'
  1427. else:
  1428. subdir, exe = 'win', 'gn.exe'
  1429. gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe)
  1430. cmd = [gn_path, subcommand]
  1431. if self.args.root:
  1432. cmd += ['--root=' + self.args.root]
  1433. if self.args.dotfile:
  1434. cmd += ['--dotfile=' + self.args.dotfile]
  1435. return cmd + [path] + list(args)
  1436. def GNArgs(self, vals, expand_imports=False):
  1437. """Returns the gn args from vals as a Python dict and a text string.
  1438. If expand_imports is true, any import() lines will be read in and
  1439. valuese them will be included."""
  1440. gn_args = vals['gn_args']
  1441. if self.args.goma_dir:
  1442. gn_args += ' goma_dir="%s"' % self.args.goma_dir
  1443. android_version_code = self.args.android_version_code
  1444. if android_version_code:
  1445. gn_args += ' android_default_version_code="%s"' % android_version_code
  1446. android_version_name = self.args.android_version_name
  1447. if android_version_name:
  1448. gn_args += ' android_default_version_name="%s"' % android_version_name
  1449. if self.args.rts:
  1450. gn_args += ' use_rts=true'
  1451. args_gn_lines = []
  1452. parsed_gn_args = {}
  1453. args_file = vals.get('args_file', None)
  1454. if args_file:
  1455. if expand_imports:
  1456. content = self.ReadFile(self.ToAbsPath(args_file))
  1457. parsed_gn_args = gn_helpers.FromGNArgs(content)
  1458. else:
  1459. args_gn_lines.append('import("%s")' % args_file)
  1460. # Canonicalize the arg string into a sorted, newline-separated list
  1461. # of key-value pairs, and de-dup the keys if need be so that only
  1462. # the last instance of each arg is listed.
  1463. parsed_gn_args.update(gn_helpers.FromGNArgs(gn_args))
  1464. args_gn_lines.append(gn_helpers.ToGNString(parsed_gn_args))
  1465. return parsed_gn_args, '\n'.join(args_gn_lines)
  1466. def GetSwarmingCommand(self, target, vals):
  1467. isolate_map = self.ReadIsolateMap()
  1468. is_android = 'target_os="android"' in vals['gn_args']
  1469. is_fuchsia = 'target_os="fuchsia"' in vals['gn_args']
  1470. is_cros = ('target_os="chromeos"' in vals['gn_args']
  1471. or 'is_chromeos_device=true' in vals['gn_args'])
  1472. is_cros_device = 'is_chromeos_device=true' in vals['gn_args']
  1473. is_ios = 'target_os="ios"' in vals['gn_args']
  1474. # pylint: disable=consider-using-ternary
  1475. is_mac = ((self.platform == 'darwin' and not is_ios)
  1476. or 'target_os="mac"' in vals['gn_args'])
  1477. is_win = self.platform == 'win32' or 'target_os="win"' in vals['gn_args']
  1478. is_lacros = 'chromeos_is_browser_only=true' in vals['gn_args']
  1479. test_type = isolate_map[target]['type']
  1480. if self.use_luci_auth:
  1481. cmdline = ['luci-auth.exe' if is_win else 'luci-auth', 'context', '--']
  1482. else:
  1483. cmdline = []
  1484. if test_type == 'generated_script' or is_ios or is_lacros:
  1485. assert 'script' not in isolate_map[target], (
  1486. 'generated_scripts can no longer customize the script path')
  1487. if is_win:
  1488. default_script = 'bin\\run_{}.bat'.format(target)
  1489. else:
  1490. default_script = 'bin/run_{}'.format(target)
  1491. script = isolate_map[target].get('script', default_script)
  1492. # TODO(crbug.com/816629): remove any use of 'args' from
  1493. # generated_scripts.
  1494. cmdline += [script] + isolate_map[target].get('args', [])
  1495. return cmdline, []
  1496. # TODO(crbug.com/816629): Convert all targets to generated_scripts
  1497. # and delete the rest of this function.
  1498. # This should be true if tests with type='windowed_test_launcher' are
  1499. # expected to run using xvfb. For example, Linux Desktop, X11 CrOS and
  1500. # Ozone CrOS builds on Linux (xvfb is not used on CrOS HW or VMs). Note
  1501. # that one Ozone build can be used to run different backends. Currently,
  1502. # tests are executed for the headless and X11 backends and both can run
  1503. # under Xvfb on Linux.
  1504. use_xvfb = (self.platform.startswith('linux') and not is_android
  1505. and not is_fuchsia and not is_cros_device)
  1506. asan = 'is_asan=true' in vals['gn_args']
  1507. msan = 'is_msan=true' in vals['gn_args']
  1508. tsan = 'is_tsan=true' in vals['gn_args']
  1509. cfi_diag = 'use_cfi_diag=true' in vals['gn_args']
  1510. clang_coverage = 'use_clang_coverage=true' in vals['gn_args']
  1511. java_coverage = 'use_jacoco_coverage=true' in vals['gn_args']
  1512. javascript_coverage = 'use_javascript_coverage=true' in vals['gn_args']
  1513. executable = isolate_map[target].get('executable', target)
  1514. executable_suffix = isolate_map[target].get(
  1515. 'executable_suffix', '.exe' if is_win else '')
  1516. vpython_exe = 'vpython3'
  1517. extra_files = [
  1518. '../../.vpython3',
  1519. '../../testing/test_env.py',
  1520. ]
  1521. if is_android and test_type != 'script':
  1522. if asan:
  1523. cmdline += [os.path.join('bin', 'run_with_asan'), '--']
  1524. cmdline += [
  1525. vpython_exe, '../../build/android/test_wrapper/logdog_wrapper.py',
  1526. '--target', target, '--logdog-bin-cmd',
  1527. '../../.task_template_packages/logdog_butler'
  1528. ]
  1529. if test_type != 'junit_test':
  1530. cmdline += ['--store-tombstones']
  1531. if clang_coverage or java_coverage:
  1532. cmdline += ['--coverage-dir', '${ISOLATED_OUTDIR}']
  1533. elif is_fuchsia and test_type != 'script':
  1534. # On Fuchsia, the generated bin/run_* test scripts are used both in
  1535. # infrastructure and by developers. test_env.py is intended to establish a
  1536. # predictable environment for automated testing. In particular, it adds
  1537. # CHROME_HEADLESS=1 to the environment for child processes. This variable
  1538. # is a signal to both test and production code that it is running in the
  1539. # context of automated an testing environment, and should not be present
  1540. # for normal developer workflows.
  1541. cmdline += [
  1542. vpython_exe,
  1543. '../../testing/test_env.py',
  1544. os.path.join('bin', 'run_%s' % target),
  1545. '--test-launcher-bot-mode',
  1546. '--logs-dir=${ISOLATED_OUTDIR}',
  1547. ]
  1548. elif is_cros_device and test_type != 'script':
  1549. cmdline += [
  1550. os.path.join('bin', 'run_%s' % target),
  1551. '--logs-dir=${ISOLATED_OUTDIR}',
  1552. ]
  1553. elif use_xvfb and test_type == 'windowed_test_launcher':
  1554. extra_files.append('../../testing/xvfb.py')
  1555. cmdline += [
  1556. vpython_exe,
  1557. '../../testing/xvfb.py',
  1558. './' + str(executable) + executable_suffix,
  1559. '--test-launcher-bot-mode',
  1560. '--asan=%d' % asan,
  1561. '--lsan=%d' % asan, # Enable lsan when asan is enabled.
  1562. '--msan=%d' % msan,
  1563. '--tsan=%d' % tsan,
  1564. '--cfi-diag=%d' % cfi_diag,
  1565. ]
  1566. if javascript_coverage:
  1567. cmdline += ['--devtools-code-coverage=${ISOLATED_OUTDIR}']
  1568. elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
  1569. cmdline += [
  1570. vpython_exe,
  1571. '../../testing/test_env.py',
  1572. './' + str(executable) + executable_suffix,
  1573. '--test-launcher-bot-mode',
  1574. '--asan=%d' % asan,
  1575. # Enable lsan when asan is enabled except on Windows where LSAN isn't
  1576. # supported.
  1577. # TODO(https://crbug.com/948939): Enable on Mac once things pass.
  1578. # TODO(https://crbug.com/974478): Enable on ChromeOS once things pass.
  1579. '--lsan=%d' % (asan and not is_mac and not is_win and not is_cros),
  1580. '--msan=%d' % msan,
  1581. '--tsan=%d' % tsan,
  1582. '--cfi-diag=%d' % cfi_diag,
  1583. ]
  1584. elif test_type == 'script':
  1585. # If we're testing a CrOS simplechrome build, assume we need to prepare a
  1586. # DUT for testing. So prepend the command to run with the test wrapper.
  1587. if is_cros_device:
  1588. cmdline += [
  1589. os.path.join('bin', 'cros_test_wrapper'),
  1590. '--logs-dir=${ISOLATED_OUTDIR}',
  1591. '--',
  1592. ]
  1593. if is_android:
  1594. extra_files.append('../../build/android/test_wrapper/logdog_wrapper.py')
  1595. cmdline += [
  1596. vpython_exe,
  1597. '../../testing/test_env.py',
  1598. '../../build/android/test_wrapper/logdog_wrapper.py',
  1599. '--script',
  1600. '../../' + self.ToSrcRelPath(isolate_map[target]['script']),
  1601. '--logdog-bin-cmd',
  1602. '../../.task_template_packages/logdog_butler',
  1603. ]
  1604. else:
  1605. cmdline += [
  1606. vpython_exe, '../../testing/test_env.py',
  1607. '../../' + self.ToSrcRelPath(isolate_map[target]['script'])
  1608. ]
  1609. elif test_type == 'additional_compile_target':
  1610. cmdline = [
  1611. './' + str(target) + executable_suffix,
  1612. ]
  1613. else:
  1614. self.WriteFailureAndRaise('No command line for %s found (test type %s).'
  1615. % (target, test_type), output_path=None)
  1616. cmdline += isolate_map[target].get('args', [])
  1617. return cmdline, extra_files
  1618. def ToAbsPath(self, build_path, *comps):
  1619. return self.PathJoin(self.chromium_src_dir,
  1620. self.ToSrcRelPath(build_path),
  1621. *comps)
  1622. def ToSrcRelPath(self, path):
  1623. """Returns a relative path from the top of the repo."""
  1624. if path.startswith('//'):
  1625. return path[2:].replace('/', self.sep)
  1626. return self.RelPath(path, self.chromium_src_dir)
  1627. def RunGNAnalyze(self, vals):
  1628. # Analyze runs before 'gn gen' now, so we need to run gn gen
  1629. # in order to ensure that we have a build directory.
  1630. ret = self.RunGNGen(vals, compute_inputs_for_analyze=True, check=False)
  1631. if ret != 0:
  1632. return ret
  1633. build_path = self.args.path
  1634. input_path = self.args.input_path
  1635. gn_input_path = input_path + '.gn'
  1636. output_path = self.args.output_path
  1637. gn_output_path = output_path + '.gn'
  1638. inp = self.ReadInputJSON(['files', 'test_targets',
  1639. 'additional_compile_targets'])
  1640. if self.args.verbose:
  1641. self.Print()
  1642. self.Print('analyze input:')
  1643. self.PrintJSON(inp)
  1644. self.Print()
  1645. # This shouldn't normally happen, but could due to unusual race conditions,
  1646. # like a try job that gets scheduled before a patch lands but runs after
  1647. # the patch has landed.
  1648. if not inp['files']:
  1649. self.Print('Warning: No files modified in patch, bailing out early.')
  1650. self.WriteJSON({
  1651. 'status': 'No dependency',
  1652. 'compile_targets': [],
  1653. 'test_targets': [],
  1654. }, output_path)
  1655. return 0
  1656. gn_inp = {}
  1657. gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')]
  1658. isolate_map = self.ReadIsolateMap()
  1659. err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels(
  1660. isolate_map, inp['additional_compile_targets'])
  1661. if err:
  1662. raise MBErr(err)
  1663. err, gn_inp['test_targets'] = self.MapTargetsToLabels(
  1664. isolate_map, inp['test_targets'])
  1665. if err:
  1666. raise MBErr(err)
  1667. labels_to_targets = {}
  1668. for i, label in enumerate(gn_inp['test_targets']):
  1669. labels_to_targets[label] = inp['test_targets'][i]
  1670. try:
  1671. self.WriteJSON(gn_inp, gn_input_path)
  1672. cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path)
  1673. ret, output, _ = self.Run(cmd, force_verbose=True)
  1674. if ret != 0:
  1675. if self.args.json_output:
  1676. # write errors to json.output
  1677. self.WriteJSON({'output': output}, self.args.json_output)
  1678. return ret
  1679. gn_outp_str = self.ReadFile(gn_output_path)
  1680. try:
  1681. gn_outp = json.loads(gn_outp_str)
  1682. except Exception as e:
  1683. self.Print("Failed to parse the JSON string GN returned: %s\n%s"
  1684. % (repr(gn_outp_str), str(e)))
  1685. raise
  1686. outp = {}
  1687. if 'status' in gn_outp:
  1688. outp['status'] = gn_outp['status']
  1689. if 'error' in gn_outp:
  1690. outp['error'] = gn_outp['error']
  1691. if 'invalid_targets' in gn_outp:
  1692. outp['invalid_targets'] = gn_outp['invalid_targets']
  1693. if 'compile_targets' in gn_outp:
  1694. all_input_compile_targets = sorted(
  1695. set(inp['test_targets'] + inp['additional_compile_targets']))
  1696. # If we're building 'all', we can throw away the rest of the targets
  1697. # since they're redundant.
  1698. if 'all' in gn_outp['compile_targets']:
  1699. outp['compile_targets'] = ['all']
  1700. else:
  1701. outp['compile_targets'] = gn_outp['compile_targets']
  1702. # crbug.com/736215: When GN returns targets back, for targets in
  1703. # the default toolchain, GN will have generated a phony ninja
  1704. # target matching the label, and so we can safely (and easily)
  1705. # transform any GN label into the matching ninja target. For
  1706. # targets in other toolchains, though, GN doesn't generate the
  1707. # phony targets, and we don't know how to turn the labels into
  1708. # compile targets. In this case, we also conservatively give up
  1709. # and build everything. Probably the right thing to do here is
  1710. # to have GN return the compile targets directly.
  1711. if any("(" in target for target in outp['compile_targets']):
  1712. self.Print('WARNING: targets with non-default toolchains were '
  1713. 'found, building everything instead.')
  1714. outp['compile_targets'] = all_input_compile_targets
  1715. else:
  1716. outp['compile_targets'] = [
  1717. label.replace('//', '') for label in outp['compile_targets']]
  1718. # Windows has a maximum command line length of 8k; even Linux
  1719. # maxes out at 128k; if analyze returns a *really long* list of
  1720. # targets, we just give up and conservatively build everything instead.
  1721. # Probably the right thing here is for ninja to support response
  1722. # files as input on the command line
  1723. # (see https://github.com/ninja-build/ninja/issues/1355).
  1724. # Android targets use a lot of templates and often exceed 7kb.
  1725. # https://crbug.com/946266
  1726. max_cmd_length_kb = 64 if platform.system() == 'Linux' else 7
  1727. if len(' '.join(outp['compile_targets'])) > max_cmd_length_kb * 1024:
  1728. self.Print('WARNING: Too many compile targets were affected.')
  1729. self.Print('WARNING: Building everything instead to avoid '
  1730. 'command-line length issues.')
  1731. outp['compile_targets'] = all_input_compile_targets
  1732. if 'test_targets' in gn_outp:
  1733. outp['test_targets'] = [
  1734. labels_to_targets[label] for label in gn_outp['test_targets']]
  1735. if self.args.verbose:
  1736. self.Print()
  1737. self.Print('analyze output:')
  1738. self.PrintJSON(outp)
  1739. self.Print()
  1740. self.WriteJSON(outp, output_path)
  1741. finally:
  1742. if self.Exists(gn_input_path):
  1743. self.RemoveFile(gn_input_path)
  1744. if self.Exists(gn_output_path):
  1745. self.RemoveFile(gn_output_path)
  1746. return 0
  1747. def ReadInputJSON(self, required_keys):
  1748. path = self.args.input_path
  1749. output_path = self.args.output_path
  1750. if not self.Exists(path):
  1751. self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
  1752. try:
  1753. inp = json.loads(self.ReadFile(path))
  1754. except Exception as e:
  1755. self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
  1756. (path, e), output_path)
  1757. for k in required_keys:
  1758. if not k in inp:
  1759. self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
  1760. output_path)
  1761. return inp
  1762. def WriteFailureAndRaise(self, msg, output_path):
  1763. if output_path:
  1764. self.WriteJSON({'error': msg}, output_path, force_verbose=True)
  1765. raise MBErr(msg)
  1766. def WriteJSON(self, obj, path, force_verbose=False):
  1767. try:
  1768. self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
  1769. force_verbose=force_verbose)
  1770. except Exception as e:
  1771. raise MBErr('Error %s writing to the output path "%s"' % (e, path)) from e
  1772. def PrintCmd(self, cmd):
  1773. if self.platform == 'win32':
  1774. shell_quoter = QuoteForCmd
  1775. else:
  1776. shell_quoter = pipes.quote
  1777. if cmd[0] == self.executable:
  1778. cmd = ['python'] + cmd[1:]
  1779. self.Print(*[shell_quoter(arg) for arg in cmd])
  1780. def PrintJSON(self, obj):
  1781. self.Print(json.dumps(obj, indent=2, sort_keys=True))
  1782. def Build(self, target):
  1783. build_dir = self.ToSrcRelPath(self.args.path)
  1784. if self.platform == 'win32':
  1785. # On Windows use the batch script since there is no exe
  1786. ninja_cmd = ['autoninja.bat', '-C', build_dir]
  1787. else:
  1788. ninja_cmd = ['autoninja', '-C', build_dir]
  1789. if self.args.jobs:
  1790. ninja_cmd.extend(['-j', '%d' % self.args.jobs])
  1791. ninja_cmd.append(target)
  1792. ret, _, _ = self.Run(ninja_cmd, capture_output=False)
  1793. return ret
  1794. def Run(self, cmd, env=None, force_verbose=True, capture_output=True):
  1795. # This function largely exists so it can be overridden for testing.
  1796. if self.args.dryrun or self.args.verbose or force_verbose:
  1797. self.PrintCmd(cmd)
  1798. if self.args.dryrun:
  1799. return 0, '', ''
  1800. ret, out, err = self.Call(cmd, env=env, capture_output=capture_output)
  1801. if self.args.verbose or force_verbose:
  1802. if ret != 0:
  1803. self.Print(' -> returned %d' % ret)
  1804. if out:
  1805. # This is the error seen on the logs
  1806. self.Print(out, end='')
  1807. if err:
  1808. self.Print(err, end='', file=sys.stderr)
  1809. return ret, out, err
  1810. # Call has argument input to match subprocess.run
  1811. def Call(
  1812. self,
  1813. cmd,
  1814. env=None,
  1815. capture_output=True,
  1816. input='',
  1817. ): # pylint: disable=redefined-builtin
  1818. # We are returning the exit code, we don't want an exception thrown
  1819. # for non-zero exit code
  1820. # pylint: disable=subprocess-run-check
  1821. p = subprocess.run(cmd,
  1822. shell=False,
  1823. capture_output=capture_output,
  1824. cwd=self.chromium_src_dir,
  1825. env=env,
  1826. text=True,
  1827. input=input)
  1828. return p.returncode, p.stdout, p.stderr
  1829. def _CipdPlatform(self):
  1830. """Returns current CIPD platform, e.g. linux-amd64.
  1831. Assumes AMD64.
  1832. """
  1833. if self.platform == 'win32':
  1834. return 'windows-amd64'
  1835. if self.platform == 'darwin':
  1836. return 'mac-amd64'
  1837. return 'linux-amd64'
  1838. def ExpandUser(self, path):
  1839. # This function largely exists so it can be overridden for testing.
  1840. return os.path.expanduser(path)
  1841. def Exists(self, path):
  1842. # This function largely exists so it can be overridden for testing.
  1843. return os.path.exists(path)
  1844. def Fetch(self, url):
  1845. # This function largely exists so it can be overridden for testing.
  1846. f = urlopen(url)
  1847. contents = f.read()
  1848. f.close()
  1849. return contents
  1850. def ListDir(self, path):
  1851. # This function largely exists so it can be overridden for testing.
  1852. return os.listdir(path)
  1853. def MaybeMakeDirectory(self, path):
  1854. try:
  1855. os.makedirs(path)
  1856. except OSError as e:
  1857. if e.errno != errno.EEXIST:
  1858. raise
  1859. def PathJoin(self, *comps):
  1860. # This function largely exists so it can be overriden for testing.
  1861. return os.path.join(*comps)
  1862. def Print(self, *args, **kwargs):
  1863. # This function largely exists so it can be overridden for testing.
  1864. print(*args, **kwargs)
  1865. if kwargs.get('stream', sys.stdout) == sys.stdout:
  1866. sys.stdout.flush()
  1867. def ReadFile(self, path):
  1868. # This function largely exists so it can be overriden for testing.
  1869. with open(path) as fp:
  1870. return fp.read()
  1871. def RelPath(self, path, start='.'):
  1872. # This function largely exists so it can be overriden for testing.
  1873. return os.path.relpath(path, start)
  1874. def RemoveFile(self, path):
  1875. # This function largely exists so it can be overriden for testing.
  1876. os.remove(path)
  1877. def RemoveDirectory(self, abs_path):
  1878. if self.platform == 'win32':
  1879. # In other places in chromium, we often have to retry this command
  1880. # because we're worried about other processes still holding on to
  1881. # file handles, but when MB is invoked, it will be early enough in the
  1882. # build that their should be no other processes to interfere. We
  1883. # can change this if need be.
  1884. self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
  1885. else:
  1886. shutil.rmtree(abs_path, ignore_errors=True)
  1887. def TempDir(self):
  1888. # This function largely exists so it can be overriden for testing.
  1889. return tempfile.mkdtemp(prefix='mb_')
  1890. def TempFile(self, mode='w'):
  1891. # This function largely exists so it can be overriden for testing.
  1892. return tempfile.NamedTemporaryFile(mode=mode, delete=False)
  1893. def WriteFile(self, path, contents, force_verbose=False):
  1894. # This function largely exists so it can be overriden for testing.
  1895. if self.args.dryrun or self.args.verbose or force_verbose:
  1896. self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
  1897. with open(path, 'w') as fp:
  1898. return fp.write(contents)
  1899. class LedResult:
  1900. """Holds the result of a led operation. Can be chained using |then|."""
  1901. def __init__(self, result, run_cmd):
  1902. self._result = result
  1903. self._run_cmd = run_cmd
  1904. @property
  1905. def result(self):
  1906. """The mutable result data of the previous led call as decoded JSON."""
  1907. return self._result
  1908. def then(self, *cmd):
  1909. """Invoke led, passing it the current `result` data as input.
  1910. Returns another LedResult object with the output of the command.
  1911. """
  1912. return self.__class__(
  1913. self._run_cmd(self._result, cmd), self._run_cmd)
  1914. def FlattenConfig(config_pool, mixin_pool, config):
  1915. mixins = config_pool[config]
  1916. vals = DefaultVals()
  1917. visited = []
  1918. FlattenMixins(mixin_pool, mixins, vals, visited)
  1919. return vals
  1920. def FlattenMixins(mixin_pool, mixins_to_flatten, vals, visited):
  1921. for m in mixins_to_flatten:
  1922. if m not in mixin_pool:
  1923. raise MBErr('Unknown mixin "%s"' % m)
  1924. visited.append(m)
  1925. mixin_vals = mixin_pool[m]
  1926. if 'args_file' in mixin_vals:
  1927. if vals['args_file']:
  1928. raise MBErr('args_file specified multiple times in mixins '
  1929. 'for mixin %s' % m)
  1930. vals['args_file'] = mixin_vals['args_file']
  1931. if 'gn_args' in mixin_vals:
  1932. if vals['gn_args']:
  1933. vals['gn_args'] += ' ' + mixin_vals['gn_args']
  1934. else:
  1935. vals['gn_args'] = mixin_vals['gn_args']
  1936. if 'mixins' in mixin_vals:
  1937. FlattenMixins(mixin_pool, mixin_vals['mixins'], vals, visited)
  1938. return vals
  1939. class MBErr(Exception):
  1940. pass
  1941. # See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful
  1942. # details of this next section, which handles escaping command lines
  1943. # so that they can be copied and pasted into a cmd window.
  1944. UNSAFE_FOR_SET = set('^<>&|')
  1945. UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%'))
  1946. ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"'))
  1947. def QuoteForSet(arg):
  1948. if any(a in UNSAFE_FOR_SET for a in arg):
  1949. arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg)
  1950. return arg
  1951. def QuoteForCmd(arg):
  1952. # First, escape the arg so that CommandLineToArgvW will parse it properly.
  1953. if arg == '' or ' ' in arg or '"' in arg:
  1954. quote_re = re.compile(r'(\\*)"')
  1955. arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
  1956. # Then check to see if the arg contains any metacharacters other than
  1957. # double quotes; if it does, quote everything (including the double
  1958. # quotes) for safety.
  1959. if any(a in UNSAFE_FOR_CMD for a in arg):
  1960. arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
  1961. return arg
  1962. if __name__ == '__main__':
  1963. sys.exit(main(sys.argv[1:]))