run-swarmed.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. #!/usr/bin/env python3
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Runs a gtest-based test on Swarming, optionally many times, collecting the
  6. output of the runs into a directory. Useful for flake checking, and faster than
  7. using trybots by avoiding repeated bot_update, compile, archive, etc. and
  8. allowing greater parallelism.
  9. To use, run in a new shell (it blocks until all Swarming jobs complete):
  10. tools/run-swarmed.py out/rel base_unittests
  11. The logs of the runs will be stored in results/ (or specify a results directory
  12. with --results=some_dir). You can then do something like `grep -L SUCCESS
  13. results/*` to find the tests that failed or otherwise process the log files.
  14. See //docs/workflow/debugging-with-swarming.md for more details.
  15. """
  16. import argparse
  17. import hashlib
  18. import json
  19. import multiprocessing.dummy
  20. import os
  21. import shutil
  22. import subprocess
  23. import sys
  24. import traceback
  25. CHROMIUM_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
  26. BUILD_DIR = os.path.join(CHROMIUM_ROOT, 'build')
  27. if BUILD_DIR not in sys.path:
  28. sys.path.insert(0, BUILD_DIR)
  29. import gn_helpers
  30. INTERNAL_ERROR_EXIT_CODE = -1000
  31. DEFAULT_ANDROID_DEVICE_TYPE = "walleye"
  32. def _Spawn(args):
  33. """Triggers a swarming job. The arguments passed are:
  34. - The index of the job;
  35. - The command line arguments object;
  36. - The digest of test files.
  37. The return value is passed to a collect-style map() and consists of:
  38. - The index of the job;
  39. - The json file created by triggering and used to collect results;
  40. - The command line arguments object.
  41. """
  42. try:
  43. return _DoSpawn(args)
  44. except Exception as e:
  45. traceback.print_exc()
  46. return None
  47. def _DoSpawn(args):
  48. index, args, cas_digest, swarming_command = args
  49. runner_args = []
  50. json_file = os.path.join(args.results, '%d.json' % index)
  51. trigger_args = [
  52. 'tools/luci-go/swarming',
  53. 'trigger',
  54. '-S',
  55. 'https://chromium-swarm.appspot.com',
  56. '-digest',
  57. cas_digest,
  58. '-dump-json',
  59. json_file,
  60. '-tag=purpose:user-debug-run-swarmed',
  61. ]
  62. if args.target_os == 'fuchsia':
  63. trigger_args += [
  64. '-d',
  65. 'kvm=1',
  66. '-d',
  67. 'gpu=none',
  68. ]
  69. elif args.target_os == 'android':
  70. if args.arch == 'x86':
  71. # No x86 Android devices are available in swarming. So assume we want to
  72. # run on emulators when building for x86 on Android.
  73. args.swarming_os = 'Linux'
  74. args.pool = 'chromium.tests.avd'
  75. # generic_android28 == Android P emulator. See //tools/android/avd/proto/
  76. # for other options.
  77. runner_args.append(
  78. '--avd-config=../../tools/android/avd/proto/generic_android28.textpb')
  79. elif args.device_type is None and args.device_os is None:
  80. # The aliases for device type are stored here:
  81. # luci/appengine/swarming/ui2/modules/alias.js
  82. # for example 'blueline' = 'Pixel 3'
  83. trigger_args += ['-d', 'device_type=' + DEFAULT_ANDROID_DEVICE_TYPE]
  84. elif args.arch != 'detect':
  85. trigger_args += [
  86. '-d',
  87. 'cpu=' + args.arch,
  88. ]
  89. if args.device_type:
  90. trigger_args += ['-d', 'device_type=' + args.device_type]
  91. if args.device_os:
  92. trigger_args += ['-d', 'device_os=' + args.device_os]
  93. if not args.no_test_flags:
  94. # These flags are recognized by our test runners, but do not work
  95. # when running custom scripts.
  96. runner_args += [
  97. '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json'
  98. ]
  99. if 'junit' not in args.target_name:
  100. runner_args += ['--system-log-file=${ISOLATED_OUTDIR}/system_log']
  101. if args.gtest_filter:
  102. runner_args.append('--gtest_filter=' + args.gtest_filter)
  103. if args.gtest_repeat:
  104. runner_args.append('--gtest_repeat=' + args.gtest_repeat)
  105. if args.test_launcher_shard_index and args.test_launcher_total_shards:
  106. runner_args.append('--test-launcher-shard-index=' +
  107. args.test_launcher_shard_index)
  108. runner_args.append('--test-launcher-total-shards=' +
  109. args.test_launcher_total_shards)
  110. elif args.target_os == 'fuchsia':
  111. filter_file = \
  112. 'testing/buildbot/filters/fuchsia.' + args.target_name + '.filter'
  113. if os.path.isfile(filter_file):
  114. runner_args.append('--test-launcher-filter-file=../../' + filter_file)
  115. runner_args.extend(args.runner_args)
  116. trigger_args.extend(['-d', 'os=' + args.swarming_os])
  117. trigger_args.extend(['-d', 'pool=' + args.pool])
  118. trigger_args.extend(['--relative-cwd', args.out_dir, '--'])
  119. trigger_args.extend(swarming_command)
  120. trigger_args.extend(runner_args)
  121. with open(os.devnull, 'w') as nul:
  122. subprocess.check_call(trigger_args, stdout=nul)
  123. return (index, json_file, args)
  124. def _Collect(spawn_result):
  125. if spawn_result is None:
  126. return 1
  127. index, json_file, args = spawn_result
  128. with open(json_file) as f:
  129. task_json = json.load(f)
  130. task_ids = [task['task_id'] for task in task_json['tasks']]
  131. for t in task_ids:
  132. print('Task {}: https://chromium-swarm.appspot.com/task?id={}'.format(
  133. index, t))
  134. p = subprocess.Popen(
  135. [
  136. 'tools/luci-go/swarming',
  137. 'collect',
  138. '-S',
  139. 'https://chromium-swarm.appspot.com',
  140. '--task-output-stdout=console',
  141. ] + task_ids,
  142. stdout=subprocess.PIPE,
  143. stderr=subprocess.STDOUT)
  144. stdout = p.communicate()[0]
  145. if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
  146. exit_code = INTERNAL_ERROR_EXIT_CODE
  147. file_suffix = '.INTERNAL_ERROR'
  148. else:
  149. exit_code = p.returncode
  150. file_suffix = '' if exit_code == 0 else '.FAILED'
  151. filename = '%d%s.stdout.txt' % (index, file_suffix)
  152. with open(os.path.join(args.results, filename), 'wb') as f:
  153. f.write(stdout)
  154. return exit_code
  155. def main():
  156. parser = argparse.ArgumentParser()
  157. parser.add_argument('--swarming-os', help='OS specifier for Swarming.')
  158. parser.add_argument('--target-os', default='detect', help='gn target_os')
  159. parser.add_argument('--arch', '-a', default='detect',
  160. help='CPU architecture of the test binary.')
  161. parser.add_argument('--build',
  162. dest='build',
  163. action='store_true',
  164. help='Build before isolating.')
  165. parser.add_argument('--no-build',
  166. dest='build',
  167. action='store_false',
  168. help='Do not build, just isolate (default).')
  169. parser.add_argument('--isolate-map-file', '-i',
  170. help='path to isolate map file if not using default')
  171. parser.add_argument('--copies', '-n', type=int, default=1,
  172. help='Number of copies to spawn.')
  173. parser.add_argument(
  174. '--device-os', help='Run tests on the given version of Android.')
  175. parser.add_argument(
  176. '--device-type',
  177. help='device_type specifier for Swarming'
  178. ' from https://chromium-swarm.appspot.com/botlist .')
  179. parser.add_argument('--pool',
  180. default='chromium.tests',
  181. help='Use the given swarming pool.')
  182. parser.add_argument('--results', '-r', default='results',
  183. help='Directory in which to store results.')
  184. parser.add_argument(
  185. '--gtest_filter',
  186. help='Deprecated. Pass as test runner arg instead, like \'-- '
  187. '--gtest_filter="*#testFoo"\'')
  188. parser.add_argument(
  189. '--gtest_repeat',
  190. help='Deprecated. Pass as test runner arg instead, like \'-- '
  191. '--gtest_repeat=99\'')
  192. parser.add_argument(
  193. '--test-launcher-shard-index',
  194. help='Shard index to run. Use with --test-launcher-total-shards.')
  195. parser.add_argument('--test-launcher-total-shards',
  196. help='Number of shards to split the test into. Use with'
  197. ' --test-launcher-shard-index.')
  198. parser.add_argument('--no-test-flags', action='store_true',
  199. help='Do not add --test-launcher-summary-output and '
  200. '--system-log-file flags to the comment.')
  201. parser.add_argument('out_dir', type=str, help='Build directory.')
  202. parser.add_argument('target_name', type=str, help='Name of target to run.')
  203. parser.add_argument(
  204. 'runner_args',
  205. nargs='*',
  206. type=str,
  207. help='Arguments to pass to the test runner, e.g. gtest_filter and '
  208. 'gtest_repeat.')
  209. args = parser.parse_intermixed_args()
  210. with open(os.path.join(args.out_dir, 'args.gn')) as f:
  211. gn_args = gn_helpers.FromGNArgs(f.read())
  212. if args.target_os == 'detect':
  213. if 'target_os' in gn_args:
  214. args.target_os = gn_args['target_os'].strip('"')
  215. else:
  216. args.target_os = {
  217. 'darwin': 'mac',
  218. 'linux': 'linux',
  219. 'win32': 'win'
  220. }[sys.platform]
  221. if args.swarming_os is None:
  222. args.swarming_os = {
  223. 'mac': 'Mac',
  224. 'win': 'Windows',
  225. 'linux': 'Linux',
  226. 'android': 'Android',
  227. 'fuchsia': 'Linux'
  228. }[args.target_os]
  229. if args.target_os == 'win' and args.target_name.endswith('.exe'):
  230. # The machinery expects not to have a '.exe' suffix.
  231. args.target_name = os.path.splitext(args.target_name)[0]
  232. # Determine the CPU architecture of the test binary, if not specified.
  233. if args.arch == 'detect':
  234. if args.target_os not in ('android', 'mac', 'win'):
  235. executable_info = subprocess.check_output(
  236. ['file', os.path.join(args.out_dir, args.target_name)], text=True)
  237. if 'ARM aarch64' in executable_info:
  238. args.arch = 'arm64',
  239. else:
  240. args.arch = 'x86-64'
  241. elif args.target_os == 'android':
  242. args.arch = gn_args.get('target_cpu', 'detect')
  243. # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
  244. # once mb.py is in python3
  245. mb_cmd = ['tools/mb/mb', 'isolate']
  246. if not args.build:
  247. mb_cmd.append('--no-build')
  248. if args.isolate_map_file:
  249. mb_cmd += ['--isolate-map-file', args.isolate_map_file]
  250. mb_cmd += ['//' + args.out_dir, args.target_name]
  251. subprocess.check_call(mb_cmd, shell=os.name == 'nt')
  252. print('If you get authentication errors, follow:')
  253. print(
  254. ' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating'
  255. )
  256. print('Uploading to isolate server, this can take a while...')
  257. isolate = os.path.join(args.out_dir, args.target_name + '.isolate')
  258. archive_json = os.path.join(args.out_dir, args.target_name + '.archive.json')
  259. subprocess.check_output([
  260. 'tools/luci-go/isolate', 'archive', '-cas-instance', 'chromium-swarm',
  261. '-isolate', isolate, '-dump-json', archive_json
  262. ])
  263. with open(archive_json) as f:
  264. cas_digest = json.load(f).get(args.target_name)
  265. # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
  266. # once mb.py is in python3
  267. mb_cmd = ['tools/mb/mb', 'get-swarming-command', '--as-list']
  268. if not args.build:
  269. mb_cmd.append('--no-build')
  270. if args.isolate_map_file:
  271. mb_cmd += ['--isolate-map-file', args.isolate_map_file]
  272. mb_cmd += ['//' + args.out_dir, args.target_name]
  273. mb_output = subprocess.check_output(mb_cmd, shell=os.name == 'nt')
  274. swarming_cmd = json.loads(mb_output)
  275. if os.path.isdir(args.results):
  276. shutil.rmtree(args.results)
  277. os.makedirs(args.results)
  278. try:
  279. print('Triggering %d tasks...' % args.copies)
  280. # Use dummy since threadpools give better exception messages
  281. # than process pools do, and threads work fine for what we're doing.
  282. pool = multiprocessing.dummy.Pool()
  283. spawn_args = [(i, args, cas_digest, swarming_cmd)
  284. for i in range(args.copies)]
  285. spawn_results = pool.imap_unordered(_Spawn, spawn_args)
  286. exit_codes = []
  287. collect_results = pool.imap_unordered(_Collect, spawn_results)
  288. for result in collect_results:
  289. exit_codes.append(result)
  290. successes = sum(1 for x in exit_codes if x == 0)
  291. errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
  292. failures = len(exit_codes) - successes - errors
  293. clear_to_eol = '\033[K'
  294. print(
  295. '\r[%d/%d] collected: '
  296. '%d successes, %d failures, %d bot errors...%s' %
  297. (len(exit_codes), args.copies, successes, failures, errors,
  298. clear_to_eol),
  299. end=' ')
  300. sys.stdout.flush()
  301. print()
  302. print('Results logs collected into', os.path.abspath(args.results) + '.')
  303. finally:
  304. pool.close()
  305. pool.join()
  306. return 0
  307. if __name__ == '__main__':
  308. sys.exit(main())