123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354 |
- #!/usr/bin/env python3
- # Copyright 2017 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Runs a gtest-based test on Swarming, optionally many times, collecting the
- output of the runs into a directory. Useful for flake checking, and faster than
- using trybots by avoiding repeated bot_update, compile, archive, etc. and
- allowing greater parallelism.
- To use, run in a new shell (it blocks until all Swarming jobs complete):
- tools/run-swarmed.py out/rel base_unittests
- The logs of the runs will be stored in results/ (or specify a results directory
- with --results=some_dir). You can then do something like `grep -L SUCCESS
- results/*` to find the tests that failed or otherwise process the log files.
- See //docs/workflow/debugging-with-swarming.md for more details.
- """
- import argparse
- import hashlib
- import json
- import multiprocessing.dummy
- import os
- import shutil
- import subprocess
- import sys
- import traceback
- CHROMIUM_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
- BUILD_DIR = os.path.join(CHROMIUM_ROOT, 'build')
- if BUILD_DIR not in sys.path:
- sys.path.insert(0, BUILD_DIR)
- import gn_helpers
- INTERNAL_ERROR_EXIT_CODE = -1000
- DEFAULT_ANDROID_DEVICE_TYPE = "walleye"
- def _Spawn(args):
- """Triggers a swarming job. The arguments passed are:
- - The index of the job;
- - The command line arguments object;
- - The digest of test files.
- The return value is passed to a collect-style map() and consists of:
- - The index of the job;
- - The json file created by triggering and used to collect results;
- - The command line arguments object.
- """
- try:
- return _DoSpawn(args)
- except Exception as e:
- traceback.print_exc()
- return None
- def _DoSpawn(args):
- index, args, cas_digest, swarming_command = args
- runner_args = []
- json_file = os.path.join(args.results, '%d.json' % index)
- trigger_args = [
- 'tools/luci-go/swarming',
- 'trigger',
- '-S',
- 'https://chromium-swarm.appspot.com',
- '-digest',
- cas_digest,
- '-dump-json',
- json_file,
- '-tag=purpose:user-debug-run-swarmed',
- ]
- if args.target_os == 'fuchsia':
- trigger_args += [
- '-d',
- 'kvm=1',
- '-d',
- 'gpu=none',
- ]
- elif args.target_os == 'android':
- if args.arch == 'x86':
- # No x86 Android devices are available in swarming. So assume we want to
- # run on emulators when building for x86 on Android.
- args.swarming_os = 'Linux'
- args.pool = 'chromium.tests.avd'
- # generic_android28 == Android P emulator. See //tools/android/avd/proto/
- # for other options.
- runner_args.append(
- '--avd-config=../../tools/android/avd/proto/generic_android28.textpb')
- elif args.device_type is None and args.device_os is None:
- # The aliases for device type are stored here:
- # luci/appengine/swarming/ui2/modules/alias.js
- # for example 'blueline' = 'Pixel 3'
- trigger_args += ['-d', 'device_type=' + DEFAULT_ANDROID_DEVICE_TYPE]
- elif args.arch != 'detect':
- trigger_args += [
- '-d',
- 'cpu=' + args.arch,
- ]
- if args.device_type:
- trigger_args += ['-d', 'device_type=' + args.device_type]
- if args.device_os:
- trigger_args += ['-d', 'device_os=' + args.device_os]
- if not args.no_test_flags:
- # These flags are recognized by our test runners, but do not work
- # when running custom scripts.
- runner_args += [
- '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json'
- ]
- if 'junit' not in args.target_name:
- runner_args += ['--system-log-file=${ISOLATED_OUTDIR}/system_log']
- if args.gtest_filter:
- runner_args.append('--gtest_filter=' + args.gtest_filter)
- if args.gtest_repeat:
- runner_args.append('--gtest_repeat=' + args.gtest_repeat)
- if args.test_launcher_shard_index and args.test_launcher_total_shards:
- runner_args.append('--test-launcher-shard-index=' +
- args.test_launcher_shard_index)
- runner_args.append('--test-launcher-total-shards=' +
- args.test_launcher_total_shards)
- elif args.target_os == 'fuchsia':
- filter_file = \
- 'testing/buildbot/filters/fuchsia.' + args.target_name + '.filter'
- if os.path.isfile(filter_file):
- runner_args.append('--test-launcher-filter-file=../../' + filter_file)
- runner_args.extend(args.runner_args)
- trigger_args.extend(['-d', 'os=' + args.swarming_os])
- trigger_args.extend(['-d', 'pool=' + args.pool])
- trigger_args.extend(['--relative-cwd', args.out_dir, '--'])
- trigger_args.extend(swarming_command)
- trigger_args.extend(runner_args)
- with open(os.devnull, 'w') as nul:
- subprocess.check_call(trigger_args, stdout=nul)
- return (index, json_file, args)
- def _Collect(spawn_result):
- if spawn_result is None:
- return 1
- index, json_file, args = spawn_result
- with open(json_file) as f:
- task_json = json.load(f)
- task_ids = [task['task_id'] for task in task_json['tasks']]
- for t in task_ids:
- print('Task {}: https://chromium-swarm.appspot.com/task?id={}'.format(
- index, t))
- p = subprocess.Popen(
- [
- 'tools/luci-go/swarming',
- 'collect',
- '-S',
- 'https://chromium-swarm.appspot.com',
- '--task-output-stdout=console',
- ] + task_ids,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- stdout = p.communicate()[0]
- if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
- exit_code = INTERNAL_ERROR_EXIT_CODE
- file_suffix = '.INTERNAL_ERROR'
- else:
- exit_code = p.returncode
- file_suffix = '' if exit_code == 0 else '.FAILED'
- filename = '%d%s.stdout.txt' % (index, file_suffix)
- with open(os.path.join(args.results, filename), 'wb') as f:
- f.write(stdout)
- return exit_code
- def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--swarming-os', help='OS specifier for Swarming.')
- parser.add_argument('--target-os', default='detect', help='gn target_os')
- parser.add_argument('--arch', '-a', default='detect',
- help='CPU architecture of the test binary.')
- parser.add_argument('--build',
- dest='build',
- action='store_true',
- help='Build before isolating.')
- parser.add_argument('--no-build',
- dest='build',
- action='store_false',
- help='Do not build, just isolate (default).')
- parser.add_argument('--isolate-map-file', '-i',
- help='path to isolate map file if not using default')
- parser.add_argument('--copies', '-n', type=int, default=1,
- help='Number of copies to spawn.')
- parser.add_argument(
- '--device-os', help='Run tests on the given version of Android.')
- parser.add_argument(
- '--device-type',
- help='device_type specifier for Swarming'
- ' from https://chromium-swarm.appspot.com/botlist .')
- parser.add_argument('--pool',
- default='chromium.tests',
- help='Use the given swarming pool.')
- parser.add_argument('--results', '-r', default='results',
- help='Directory in which to store results.')
- parser.add_argument(
- '--gtest_filter',
- help='Deprecated. Pass as test runner arg instead, like \'-- '
- '--gtest_filter="*#testFoo"\'')
- parser.add_argument(
- '--gtest_repeat',
- help='Deprecated. Pass as test runner arg instead, like \'-- '
- '--gtest_repeat=99\'')
- parser.add_argument(
- '--test-launcher-shard-index',
- help='Shard index to run. Use with --test-launcher-total-shards.')
- parser.add_argument('--test-launcher-total-shards',
- help='Number of shards to split the test into. Use with'
- ' --test-launcher-shard-index.')
- parser.add_argument('--no-test-flags', action='store_true',
- help='Do not add --test-launcher-summary-output and '
- '--system-log-file flags to the comment.')
- parser.add_argument('out_dir', type=str, help='Build directory.')
- parser.add_argument('target_name', type=str, help='Name of target to run.')
- parser.add_argument(
- 'runner_args',
- nargs='*',
- type=str,
- help='Arguments to pass to the test runner, e.g. gtest_filter and '
- 'gtest_repeat.')
- args = parser.parse_intermixed_args()
- with open(os.path.join(args.out_dir, 'args.gn')) as f:
- gn_args = gn_helpers.FromGNArgs(f.read())
- if args.target_os == 'detect':
- if 'target_os' in gn_args:
- args.target_os = gn_args['target_os'].strip('"')
- else:
- args.target_os = {
- 'darwin': 'mac',
- 'linux': 'linux',
- 'win32': 'win'
- }[sys.platform]
- if args.swarming_os is None:
- args.swarming_os = {
- 'mac': 'Mac',
- 'win': 'Windows',
- 'linux': 'Linux',
- 'android': 'Android',
- 'fuchsia': 'Linux'
- }[args.target_os]
- if args.target_os == 'win' and args.target_name.endswith('.exe'):
- # The machinery expects not to have a '.exe' suffix.
- args.target_name = os.path.splitext(args.target_name)[0]
- # Determine the CPU architecture of the test binary, if not specified.
- if args.arch == 'detect':
- if args.target_os not in ('android', 'mac', 'win'):
- executable_info = subprocess.check_output(
- ['file', os.path.join(args.out_dir, args.target_name)], text=True)
- if 'ARM aarch64' in executable_info:
- args.arch = 'arm64',
- else:
- args.arch = 'x86-64'
- elif args.target_os == 'android':
- args.arch = gn_args.get('target_cpu', 'detect')
- # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
- # once mb.py is in python3
- mb_cmd = ['tools/mb/mb', 'isolate']
- if not args.build:
- mb_cmd.append('--no-build')
- if args.isolate_map_file:
- mb_cmd += ['--isolate-map-file', args.isolate_map_file]
- mb_cmd += ['//' + args.out_dir, args.target_name]
- subprocess.check_call(mb_cmd, shell=os.name == 'nt')
- print('If you get authentication errors, follow:')
- print(
- ' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating'
- )
- print('Uploading to isolate server, this can take a while...')
- isolate = os.path.join(args.out_dir, args.target_name + '.isolate')
- archive_json = os.path.join(args.out_dir, args.target_name + '.archive.json')
- subprocess.check_output([
- 'tools/luci-go/isolate', 'archive', '-cas-instance', 'chromium-swarm',
- '-isolate', isolate, '-dump-json', archive_json
- ])
- with open(archive_json) as f:
- cas_digest = json.load(f).get(args.target_name)
- # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
- # once mb.py is in python3
- mb_cmd = ['tools/mb/mb', 'get-swarming-command', '--as-list']
- if not args.build:
- mb_cmd.append('--no-build')
- if args.isolate_map_file:
- mb_cmd += ['--isolate-map-file', args.isolate_map_file]
- mb_cmd += ['//' + args.out_dir, args.target_name]
- mb_output = subprocess.check_output(mb_cmd, shell=os.name == 'nt')
- swarming_cmd = json.loads(mb_output)
- if os.path.isdir(args.results):
- shutil.rmtree(args.results)
- os.makedirs(args.results)
- try:
- print('Triggering %d tasks...' % args.copies)
- # Use dummy since threadpools give better exception messages
- # than process pools do, and threads work fine for what we're doing.
- pool = multiprocessing.dummy.Pool()
- spawn_args = [(i, args, cas_digest, swarming_cmd)
- for i in range(args.copies)]
- spawn_results = pool.imap_unordered(_Spawn, spawn_args)
- exit_codes = []
- collect_results = pool.imap_unordered(_Collect, spawn_results)
- for result in collect_results:
- exit_codes.append(result)
- successes = sum(1 for x in exit_codes if x == 0)
- errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
- failures = len(exit_codes) - successes - errors
- clear_to_eol = '\033[K'
- print(
- '\r[%d/%d] collected: '
- '%d successes, %d failures, %d bot errors...%s' %
- (len(exit_codes), args.copies, successes, failures, errors,
- clear_to_eol),
- end=' ')
- sys.stdout.flush()
- print()
- print('Results logs collected into', os.path.abspath(args.results) + '.')
- finally:
- pool.close()
- pool.join()
- return 0
- if __name__ == '__main__':
- sys.exit(main())
|