test_runner.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2018 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. import argparse
  7. import collections
  8. import json
  9. import logging
  10. import os
  11. import re
  12. import shutil
  13. import signal
  14. import socket
  15. import sys
  16. import tempfile
  17. import six
  18. # The following non-std imports are fetched via vpython. See the list at
  19. # //.vpython
  20. import dateutil.parser # pylint: disable=import-error
  21. import jsonlines # pylint: disable=import-error
  22. import psutil # pylint: disable=import-error
  23. CHROMIUM_SRC_PATH = os.path.abspath(
  24. os.path.join(os.path.dirname(__file__), '..', '..'))
  25. # Use the android test-runner's gtest results support library for generating
  26. # output json ourselves.
  27. sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'android'))
  28. from pylib.base import base_test_result # pylint: disable=import-error
  29. from pylib.results import json_results # pylint: disable=import-error
  30. sys.path.insert(0, os.path.join(CHROMIUM_SRC_PATH, 'build', 'util'))
  31. from lib.results import result_sink # pylint: disable=import-error
  32. assert not six.PY2, 'Py2 not supported for this file.'
  33. import subprocess # pylint: disable=import-error,wrong-import-order
  34. DEFAULT_CROS_CACHE = os.path.abspath(
  35. os.path.join(CHROMIUM_SRC_PATH, 'build', 'cros_cache'))
  36. CHROMITE_PATH = os.path.abspath(
  37. os.path.join(CHROMIUM_SRC_PATH, 'third_party', 'chromite'))
  38. CROS_RUN_TEST_PATH = os.path.abspath(
  39. os.path.join(CHROMITE_PATH, 'bin', 'cros_run_test'))
  40. LACROS_LAUNCHER_SCRIPT_PATH = os.path.abspath(
  41. os.path.join(CHROMIUM_SRC_PATH, 'build', 'lacros',
  42. 'mojo_connection_lacros_launcher.py'))
  43. # This is a special hostname that resolves to a different DUT in the lab
  44. # depending on which lab machine you're on.
  45. LAB_DUT_HOSTNAME = 'variable_chromeos_device_hostname'
  46. SYSTEM_LOG_LOCATIONS = [
  47. '/home/chronos/crash/',
  48. '/var/log/chrome/',
  49. '/var/log/messages',
  50. '/var/log/ui/',
  51. ]
  52. TAST_DEBUG_DOC = 'https://bit.ly/2LgvIXz'
  53. class TestFormatError(Exception):
  54. pass
  55. class RemoteTest:
  56. # This is a basic shell script that can be appended to in order to invoke the
  57. # test on the device.
  58. BASIC_SHELL_SCRIPT = [
  59. '#!/bin/sh',
  60. # /home and /tmp are mounted with "noexec" in the device, but some of our
  61. # tools and tests use those dirs as a workspace (eg: vpython downloads
  62. # python binaries to ~/.vpython-root and /tmp/vpython_bootstrap).
  63. # /usr/local/tmp doesn't have this restriction, so change the location of
  64. # the home and temp dirs for the duration of the test.
  65. 'export HOME=/usr/local/tmp',
  66. 'export TMPDIR=/usr/local/tmp',
  67. ]
  68. def __init__(self, args, unknown_args):
  69. self._additional_args = unknown_args
  70. self._path_to_outdir = args.path_to_outdir
  71. self._test_launcher_summary_output = args.test_launcher_summary_output
  72. self._logs_dir = args.logs_dir
  73. self._use_vm = args.use_vm
  74. self._rdb_client = result_sink.TryInitClient()
  75. self._retries = 0
  76. self._timeout = None
  77. self._test_launcher_shard_index = args.test_launcher_shard_index
  78. self._test_launcher_total_shards = args.test_launcher_total_shards
  79. # The location on disk of a shell script that can be optionally used to
  80. # invoke the test on the device. If it's not set, we assume self._test_cmd
  81. # contains the test invocation.
  82. self._on_device_script = None
  83. self._test_cmd = [
  84. CROS_RUN_TEST_PATH,
  85. '--board',
  86. args.board,
  87. '--cache-dir',
  88. args.cros_cache,
  89. ]
  90. if args.use_vm:
  91. self._test_cmd += [
  92. '--start',
  93. # Don't persist any filesystem changes after the VM shutsdown.
  94. '--copy-on-write',
  95. ]
  96. else:
  97. self._test_cmd += [
  98. '--device', args.device if args.device else LAB_DUT_HOSTNAME
  99. ]
  100. if args.logs_dir:
  101. for log in SYSTEM_LOG_LOCATIONS:
  102. self._test_cmd += ['--results-src', log]
  103. self._test_cmd += [
  104. '--results-dest-dir',
  105. os.path.join(args.logs_dir, 'system_logs')
  106. ]
  107. if args.flash:
  108. self._test_cmd += ['--flash']
  109. if args.public_image:
  110. self._test_cmd += ['--public-image']
  111. self._test_env = setup_env()
  112. @property
  113. def suite_name(self):
  114. raise NotImplementedError('Child classes need to define suite name.')
  115. @property
  116. def test_cmd(self):
  117. return self._test_cmd
  118. def write_test_script_to_disk(self, script_contents):
  119. # Since we're using an on_device_script to invoke the test, we'll need to
  120. # set cwd.
  121. self._test_cmd += [
  122. '--remote-cmd',
  123. '--cwd',
  124. os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH),
  125. ]
  126. logging.info('Running the following command on the device:')
  127. logging.info('\n%s', '\n'.join(script_contents))
  128. fd, tmp_path = tempfile.mkstemp(suffix='.sh', dir=self._path_to_outdir)
  129. os.fchmod(fd, 0o755)
  130. with os.fdopen(fd, 'w') as f:
  131. f.write('\n'.join(script_contents) + '\n')
  132. return tmp_path
  133. def run_test(self):
  134. # Traps SIGTERM and kills all child processes of cros_run_test when it's
  135. # caught. This will allow us to capture logs from the device if a test hangs
  136. # and gets timeout-killed by swarming. See also:
  137. # https://chromium.googlesource.com/infra/luci/luci-py/+/main/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance
  138. test_proc = None
  139. def _kill_child_procs(trapped_signal, _):
  140. logging.warning('Received signal %d. Killing child processes of test.',
  141. trapped_signal)
  142. if not test_proc or not test_proc.pid:
  143. # This shouldn't happen?
  144. logging.error('Test process not running.')
  145. return
  146. for child in psutil.Process(test_proc.pid).children():
  147. logging.warning('Killing process %s', child)
  148. child.kill()
  149. signal.signal(signal.SIGTERM, _kill_child_procs)
  150. for i in range(self._retries + 1):
  151. logging.info('########################################')
  152. logging.info('Test attempt #%d', i)
  153. logging.info('########################################')
  154. test_proc = subprocess.Popen(
  155. self._test_cmd,
  156. stdout=sys.stdout,
  157. stderr=sys.stderr,
  158. env=self._test_env)
  159. try:
  160. test_proc.wait(timeout=self._timeout)
  161. except subprocess.TimeoutExpired: # pylint: disable=no-member
  162. logging.error('Test timed out. Sending SIGTERM.')
  163. # SIGTERM the proc and wait 10s for it to close.
  164. test_proc.terminate()
  165. try:
  166. test_proc.wait(timeout=10)
  167. except subprocess.TimeoutExpired: # pylint: disable=no-member
  168. # If it hasn't closed in 10s, SIGKILL it.
  169. logging.error('Test did not exit in time. Sending SIGKILL.')
  170. test_proc.kill()
  171. test_proc.wait()
  172. logging.info('Test exitted with %d.', test_proc.returncode)
  173. if test_proc.returncode == 0:
  174. break
  175. self.post_run(test_proc.returncode)
  176. # Allow post_run to override test proc return code. (Useful when the host
  177. # side Tast bin returns 0 even for failed tests.)
  178. return test_proc.returncode
  179. def post_run(self, return_code):
  180. if self._on_device_script:
  181. os.remove(self._on_device_script)
  182. # Create a simple json results file for a test run. The results will contain
  183. # only one test (suite_name), and will either be a PASS or FAIL depending on
  184. # return_code.
  185. if self._test_launcher_summary_output:
  186. result = (
  187. base_test_result.ResultType.FAIL
  188. if return_code else base_test_result.ResultType.PASS)
  189. suite_result = base_test_result.BaseTestResult(self.suite_name, result)
  190. run_results = base_test_result.TestRunResults()
  191. run_results.AddResult(suite_result)
  192. with open(self._test_launcher_summary_output, 'w') as f:
  193. json.dump(json_results.GenerateResultsDict([run_results]), f)
  194. if self._rdb_client:
  195. self._rdb_client.Post(self.suite_name, result, None, None, None)
  196. @staticmethod
  197. def get_artifacts(path):
  198. """Crawls a given directory for file artifacts to attach to a test.
  199. Args:
  200. path: Path to a directory to search for artifacts.
  201. Returns:
  202. A dict mapping name of the artifact to its absolute filepath.
  203. """
  204. artifacts = {}
  205. for dirpath, _, filenames in os.walk(path):
  206. for f in filenames:
  207. artifact_path = os.path.join(dirpath, f)
  208. artifact_id = os.path.relpath(artifact_path, path)
  209. # Some artifacts will have non-Latin characters in the filename, eg:
  210. # 'ui_tree_Chinese Pinyin-你好.txt'. ResultDB's API rejects such
  211. # characters as an artifact ID, so force the file name down into ascii.
  212. # For more info, see:
  213. # https://source.chromium.org/chromium/infra/infra/+/main:go/src/go.chromium.org/luci/resultdb/proto/v1/artifact.proto;drc=3bff13b8037ca76ec19f9810033d914af7ec67cb;l=46
  214. artifact_id = artifact_id.encode('ascii', 'replace').decode()
  215. artifact_id = artifact_id.replace('\\', '?')
  216. artifacts[artifact_id] = {
  217. 'filePath': artifact_path,
  218. }
  219. return artifacts
  220. class TastTest(RemoteTest):
  221. def __init__(self, args, unknown_args):
  222. super().__init__(args, unknown_args)
  223. self._suite_name = args.suite_name
  224. self._tast_vars = args.tast_vars
  225. self._tests = args.tests
  226. # The CQ passes in '--gtest_filter' when specifying tests to skip. Store it
  227. # here and parse it later to integrate it into Tast executions.
  228. self._gtest_style_filter = args.gtest_filter
  229. self._attr_expr = args.attr_expr
  230. self._should_strip = args.strip_chrome
  231. self._deploy_lacros = args.deploy_lacros
  232. if self._deploy_lacros and self._should_strip:
  233. raise TestFormatError(
  234. '--strip-chrome is only applicable to ash-chrome because '
  235. 'lacros-chrome deployment uses --nostrip by default, so it cannot '
  236. 'be specificed with --deploy-lacros.')
  237. if not self._logs_dir:
  238. # The host-side Tast bin returns 0 when tests fail, so we need to capture
  239. # and parse its json results to reliably determine if tests fail.
  240. raise TestFormatError(
  241. 'When using the host-side Tast bin, "--logs-dir" must be passed in '
  242. 'order to parse its results.')
  243. # If the first test filter is negative, it should be safe to assume all of
  244. # them are, so just test the first filter.
  245. if self._gtest_style_filter and self._gtest_style_filter[0] == '-':
  246. raise TestFormatError('Negative test filters not supported for Tast.')
  247. @property
  248. def suite_name(self):
  249. return self._suite_name
  250. def build_test_command(self):
  251. unsupported_args = [
  252. '--test-launcher-retry-limit',
  253. '--test-launcher-batch-limit',
  254. '--gtest_repeat',
  255. ]
  256. for unsupported_arg in unsupported_args:
  257. if any(arg.startswith(unsupported_arg) for arg in self._additional_args):
  258. logging.info(
  259. '%s not supported for Tast tests. The arg will be ignored.',
  260. unsupported_arg)
  261. self._additional_args = [
  262. arg for arg in self._additional_args
  263. if not arg.startswith(unsupported_arg)
  264. ]
  265. # Lacros deployment mounts itself by default.
  266. self._test_cmd.extend([
  267. '--deploy-lacros', '--lacros-launcher-script',
  268. LACROS_LAUNCHER_SCRIPT_PATH
  269. ] if self._deploy_lacros else ['--deploy', '--mount'])
  270. self._test_cmd += [
  271. '--build-dir',
  272. os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH)
  273. ] + self._additional_args
  274. # Capture tast's results in the logs dir as well.
  275. if self._logs_dir:
  276. self._test_cmd += [
  277. '--results-dir',
  278. self._logs_dir,
  279. ]
  280. self._test_cmd += [
  281. '--tast-total-shards=%d' % self._test_launcher_total_shards,
  282. '--tast-shard-index=%d' % self._test_launcher_shard_index,
  283. ]
  284. # If we're using a test filter, replace the contents of the Tast
  285. # conditional with a long list of "name:test" expressions, one for each
  286. # test in the filter.
  287. if self._gtest_style_filter:
  288. if self._attr_expr or self._tests:
  289. logging.warning(
  290. 'Presence of --gtest_filter will cause the specified Tast expr'
  291. ' or test list to be ignored.')
  292. names = []
  293. for test in self._gtest_style_filter.split(':'):
  294. names.append('"name:%s"' % test)
  295. self._attr_expr = '(' + ' || '.join(names) + ')'
  296. if self._attr_expr:
  297. # Don't use pipes.quote() here. Something funky happens with the arg
  298. # as it gets passed down from cros_run_test to tast. (Tast picks up the
  299. # escaping single quotes and complains that the attribute expression
  300. # "must be within parentheses".)
  301. self._test_cmd.append('--tast=%s' % self._attr_expr)
  302. else:
  303. self._test_cmd.append('--tast')
  304. self._test_cmd.extend(self._tests)
  305. for v in self._tast_vars or []:
  306. self._test_cmd.extend(['--tast-var', v])
  307. # Mounting ash-chrome gives it enough disk space to not need stripping,
  308. # but only for one not instrumented with code coverage.
  309. # Lacros uses --nostrip by default, so there is no need to specify.
  310. if not self._deploy_lacros and not self._should_strip:
  311. self._test_cmd.append('--nostrip')
  312. def post_run(self, return_code):
  313. tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl')
  314. if not os.path.exists(tast_results_path):
  315. logging.error(
  316. 'Tast results not found at %s. Falling back to generic result '
  317. 'reporting.', tast_results_path)
  318. return super().post_run(return_code)
  319. # See the link below for the format of the results:
  320. # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult
  321. with jsonlines.open(tast_results_path) as reader:
  322. tast_results = collections.deque(reader)
  323. suite_results = base_test_result.TestRunResults()
  324. for test in tast_results:
  325. errors = test['errors']
  326. start, end = test['start'], test['end']
  327. # Use dateutil to parse the timestamps since datetime can't handle
  328. # nanosecond precision.
  329. duration = dateutil.parser.parse(end) - dateutil.parser.parse(start)
  330. # If the duration is negative, Tast has likely reported an incorrect
  331. # duration. See https://issuetracker.google.com/issues/187973541. Round
  332. # up to 0 in that case to avoid confusing RDB.
  333. duration_ms = max(duration.total_seconds() * 1000, 0)
  334. if bool(test['skipReason']):
  335. result = base_test_result.ResultType.SKIP
  336. elif errors:
  337. result = base_test_result.ResultType.FAIL
  338. else:
  339. result = base_test_result.ResultType.PASS
  340. primary_error_message = None
  341. error_log = ''
  342. if errors:
  343. # See the link below for the format of these errors:
  344. # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go
  345. primary_error_message = errors[0]['reason']
  346. for err in errors:
  347. error_log += err['stack'] + '\n'
  348. debug_link = ("If you're unsure why this test failed, consult the steps "
  349. 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC)
  350. base_result = base_test_result.BaseTestResult(
  351. test['name'], result, duration=duration_ms, log=error_log)
  352. suite_results.AddResult(base_result)
  353. self._maybe_handle_perf_results(test['name'])
  354. if self._rdb_client:
  355. # Walk the contents of the test's "outDir" and atttach any file found
  356. # inside as an RDB 'artifact'. (This could include system logs, screen
  357. # shots, etc.)
  358. artifacts = self.get_artifacts(test['outDir'])
  359. self._rdb_client.Post(
  360. test['name'],
  361. result,
  362. duration_ms,
  363. error_log,
  364. None,
  365. artifacts=artifacts,
  366. failure_reason=primary_error_message,
  367. html_artifact=debug_link)
  368. if self._rdb_client and self._logs_dir:
  369. # Attach artifacts from the device that don't apply to a single test.
  370. artifacts = self.get_artifacts(
  371. os.path.join(self._logs_dir, 'system_logs'))
  372. artifacts.update(
  373. self.get_artifacts(os.path.join(self._logs_dir, 'crashes')))
  374. self._rdb_client.ReportInvocationLevelArtifacts(artifacts)
  375. if self._test_launcher_summary_output:
  376. with open(self._test_launcher_summary_output, 'w') as f:
  377. json.dump(json_results.GenerateResultsDict([suite_results]), f)
  378. if not suite_results.DidRunPass():
  379. return 1
  380. if return_code:
  381. logging.warning(
  382. 'No failed tests found, but exit code of %d was returned from '
  383. 'cros_run_test.', return_code)
  384. return return_code
  385. return 0
  386. def _maybe_handle_perf_results(self, test_name):
  387. """Prepares any perf results from |test_name| for process_perf_results.
  388. - process_perf_results looks for top level directories containing a
  389. perf_results.json file and a test_results.json file. The directory names
  390. are used as the benchmark names.
  391. - If a perf_results.json or results-chart.json file exists in the
  392. |test_name| results directory, a top level directory is created and the
  393. perf results file is copied to perf_results.json.
  394. - A trivial test_results.json file is also created to indicate that the test
  395. succeeded (this function would not be called otherwise).
  396. - When process_perf_results is run, it will find the expected files in the
  397. named directory and upload the benchmark results.
  398. """
  399. perf_results = os.path.join(self._logs_dir, 'tests', test_name,
  400. 'perf_results.json')
  401. # TODO(stevenjb): Remove check for crosbolt results-chart.json file.
  402. if not os.path.exists(perf_results):
  403. perf_results = os.path.join(self._logs_dir, 'tests', test_name,
  404. 'results-chart.json')
  405. if os.path.exists(perf_results):
  406. benchmark_dir = os.path.join(self._logs_dir, test_name)
  407. if not os.path.isdir(benchmark_dir):
  408. os.makedirs(benchmark_dir)
  409. shutil.copyfile(perf_results,
  410. os.path.join(benchmark_dir, 'perf_results.json'))
  411. # process_perf_results.py expects a test_results.json file.
  412. test_results = {'valid': True, 'failures': []}
  413. with open(os.path.join(benchmark_dir, 'test_results.json'), 'w') as out:
  414. json.dump(test_results, out)
  415. class GTestTest(RemoteTest):
  416. # The following list corresponds to paths that should not be copied over to
  417. # the device during tests. In other words, these files are only ever used on
  418. # the host.
  419. _FILE_IGNORELIST = [
  420. re.compile(r'.*build/android.*'),
  421. re.compile(r'.*build/chromeos.*'),
  422. re.compile(r'.*build/cros_cache.*'),
  423. # The following matches anything under //testing/ that isn't under
  424. # //testing/buildbot/filters/.
  425. re.compile(r'.*testing/(?!buildbot/filters).*'),
  426. re.compile(r'.*third_party/chromite.*'),
  427. ]
  428. def __init__(self, args, unknown_args):
  429. super().__init__(args, unknown_args)
  430. self._test_exe = args.test_exe
  431. self._runtime_deps_path = args.runtime_deps_path
  432. self._vpython_dir = args.vpython_dir
  433. self._on_device_script = None
  434. self._env_vars = args.env_var
  435. self._stop_ui = args.stop_ui
  436. self._trace_dir = args.trace_dir
  437. @property
  438. def suite_name(self):
  439. return self._test_exe
  440. def build_test_command(self):
  441. # To keep things easy for us, ensure both types of output locations are
  442. # the same.
  443. if self._test_launcher_summary_output and self._logs_dir:
  444. json_out_dir = os.path.dirname(self._test_launcher_summary_output) or '.'
  445. if os.path.abspath(json_out_dir) != os.path.abspath(self._logs_dir):
  446. raise TestFormatError(
  447. '--test-launcher-summary-output and --logs-dir must point to '
  448. 'the same directory.')
  449. if self._test_launcher_summary_output:
  450. result_dir, result_file = os.path.split(
  451. self._test_launcher_summary_output)
  452. # If args.test_launcher_summary_output is a file in cwd, result_dir will
  453. # be an empty string, so replace it with '.' when this is the case so
  454. # cros_run_test can correctly handle it.
  455. if not result_dir:
  456. result_dir = '.'
  457. device_result_file = '/tmp/%s' % result_file
  458. self._test_cmd += [
  459. '--results-src',
  460. device_result_file,
  461. '--results-dest-dir',
  462. result_dir,
  463. ]
  464. if self._trace_dir and self._logs_dir:
  465. trace_path = os.path.dirname(self._trace_dir) or '.'
  466. if os.path.abspath(trace_path) != os.path.abspath(self._logs_dir):
  467. raise TestFormatError(
  468. '--trace-dir and --logs-dir must point to the same directory.')
  469. if self._trace_dir:
  470. trace_path, trace_dirname = os.path.split(self._trace_dir)
  471. device_trace_dir = '/tmp/%s' % trace_dirname
  472. self._test_cmd += [
  473. '--results-src',
  474. device_trace_dir,
  475. '--results-dest-dir',
  476. trace_path,
  477. ]
  478. # Build the shell script that will be used on the device to invoke the test.
  479. # Stored here as a list of lines.
  480. device_test_script_contents = self.BASIC_SHELL_SCRIPT[:]
  481. for var_name, var_val in self._env_vars:
  482. device_test_script_contents += ['export %s=%s' % (var_name, var_val)]
  483. if self._vpython_dir:
  484. vpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
  485. 'vpython3')
  486. cpython_path = os.path.join(self._path_to_outdir, self._vpython_dir,
  487. 'bin', 'python3')
  488. if not os.path.exists(vpython_path) or not os.path.exists(cpython_path):
  489. raise TestFormatError(
  490. '--vpython-dir must point to a dir with both '
  491. 'infra/3pp/tools/cpython3 and infra/tools/luci/vpython installed.')
  492. vpython_spec_path = os.path.relpath(
  493. os.path.join(CHROMIUM_SRC_PATH, '.vpython3'), self._path_to_outdir)
  494. # Initialize the vpython cache. This can take 10-20s, and some tests
  495. # can't afford to wait that long on the first invocation.
  496. device_test_script_contents.extend([
  497. 'export PATH=$PWD/%s:$PWD/%s/bin/:$PATH' %
  498. (self._vpython_dir, self._vpython_dir),
  499. 'vpython3 -vpython-spec %s -vpython-tool install' %
  500. (vpython_spec_path),
  501. ])
  502. test_invocation = ('LD_LIBRARY_PATH=./ ./%s --test-launcher-shard-index=%d '
  503. '--test-launcher-total-shards=%d' %
  504. (self._test_exe, self._test_launcher_shard_index,
  505. self._test_launcher_total_shards))
  506. if self._test_launcher_summary_output:
  507. test_invocation += ' --test-launcher-summary-output=%s' % (
  508. device_result_file)
  509. if self._trace_dir:
  510. device_test_script_contents.extend([
  511. 'rm -rf %s' % device_trace_dir,
  512. 'su chronos -c -- "mkdir -p %s"' % device_trace_dir,
  513. ])
  514. test_invocation += ' --trace-dir=%s' % device_trace_dir
  515. if self._additional_args:
  516. test_invocation += ' %s' % ' '.join(self._additional_args)
  517. if self._stop_ui:
  518. device_test_script_contents += [
  519. 'stop ui',
  520. ]
  521. # The UI service on the device owns the chronos user session, so shutting
  522. # it down as chronos kills the entire execution of the test. So we'll have
  523. # to run as root up until the test invocation.
  524. test_invocation = 'su chronos -c -- "%s"' % test_invocation
  525. # And we'll need to chown everything since cros_run_test's "--as-chronos"
  526. # option normally does that for us.
  527. device_test_script_contents.append('chown -R chronos: ../..')
  528. else:
  529. self._test_cmd += [
  530. # Some tests fail as root, so run as the less privileged user
  531. # 'chronos'.
  532. '--as-chronos',
  533. ]
  534. device_test_script_contents.append(test_invocation)
  535. self._on_device_script = self.write_test_script_to_disk(
  536. device_test_script_contents)
  537. runtime_files = [os.path.relpath(self._on_device_script)]
  538. runtime_files += self._read_runtime_files()
  539. if self._vpython_dir:
  540. # --vpython-dir is relative to the out dir, but --files expects paths
  541. # relative to src dir, so fix the path up a bit.
  542. runtime_files.append(
  543. os.path.relpath(
  544. os.path.abspath(
  545. os.path.join(self._path_to_outdir, self._vpython_dir)),
  546. CHROMIUM_SRC_PATH))
  547. for f in runtime_files:
  548. self._test_cmd.extend(['--files', f])
  549. self._test_cmd += [
  550. '--',
  551. './' + os.path.relpath(self._on_device_script, self._path_to_outdir)
  552. ]
  553. def _read_runtime_files(self):
  554. if not self._runtime_deps_path:
  555. return []
  556. abs_runtime_deps_path = os.path.abspath(
  557. os.path.join(self._path_to_outdir, self._runtime_deps_path))
  558. with open(abs_runtime_deps_path) as runtime_deps_file:
  559. files = [l.strip() for l in runtime_deps_file if l]
  560. rel_file_paths = []
  561. for f in files:
  562. rel_file_path = os.path.relpath(
  563. os.path.abspath(os.path.join(self._path_to_outdir, f)))
  564. if not any(regex.match(rel_file_path) for regex in self._FILE_IGNORELIST):
  565. rel_file_paths.append(rel_file_path)
  566. return rel_file_paths
  567. def post_run(self, _):
  568. if self._on_device_script:
  569. os.remove(self._on_device_script)
  570. if self._test_launcher_summary_output and self._rdb_client:
  571. logging.error('Native ResultDB integration is not supported for GTests. '
  572. 'Upload results via result_adapter instead. '
  573. 'See crbug.com/1330441.')
  574. def device_test(args, unknown_args):
  575. # cros_run_test has trouble with relative paths that go up directories,
  576. # so cd to src/, which should be the root of all data deps.
  577. os.chdir(CHROMIUM_SRC_PATH)
  578. # TODO: Remove the above when depot_tool's pylint is updated to include the
  579. # fix to https://github.com/PyCQA/pylint/issues/710.
  580. if args.test_type == 'tast':
  581. test = TastTest(args, unknown_args)
  582. else:
  583. test = GTestTest(args, unknown_args)
  584. test.build_test_command()
  585. logging.info('Running the following command on the device:')
  586. logging.info(' '.join(test.test_cmd))
  587. return test.run_test()
  588. def host_cmd(args, cmd_args):
  589. if not cmd_args:
  590. raise TestFormatError('Must specify command to run on the host.')
  591. if args.deploy_chrome and not args.path_to_outdir:
  592. raise TestFormatError(
  593. '--path-to-outdir must be specified if --deploy-chrome is passed.')
  594. cros_run_test_cmd = [
  595. CROS_RUN_TEST_PATH,
  596. '--board',
  597. args.board,
  598. '--cache-dir',
  599. os.path.join(CHROMIUM_SRC_PATH, args.cros_cache),
  600. ]
  601. if args.use_vm:
  602. cros_run_test_cmd += [
  603. '--start',
  604. # Don't persist any filesystem changes after the VM shutsdown.
  605. '--copy-on-write',
  606. ]
  607. else:
  608. cros_run_test_cmd += [
  609. '--device', args.device if args.device else LAB_DUT_HOSTNAME
  610. ]
  611. if args.verbose:
  612. cros_run_test_cmd.append('--debug')
  613. if args.flash:
  614. cros_run_test_cmd.append('--flash')
  615. if args.public_image:
  616. cros_run_test_cmd += ['--public-image']
  617. if args.logs_dir:
  618. for log in SYSTEM_LOG_LOCATIONS:
  619. cros_run_test_cmd += ['--results-src', log]
  620. cros_run_test_cmd += [
  621. '--results-dest-dir',
  622. os.path.join(args.logs_dir, 'system_logs')
  623. ]
  624. test_env = setup_env()
  625. if args.deploy_chrome or args.deploy_lacros:
  626. if args.deploy_lacros:
  627. cros_run_test_cmd.extend([
  628. '--deploy-lacros', '--lacros-launcher-script',
  629. LACROS_LAUNCHER_SCRIPT_PATH
  630. ])
  631. else:
  632. # Mounting ash-chrome gives it enough disk space to not need stripping
  633. # most of the time.
  634. cros_run_test_cmd.extend(['--deploy', '--mount'])
  635. if not args.strip_chrome:
  636. cros_run_test_cmd.append('--nostrip')
  637. cros_run_test_cmd += [
  638. '--build-dir',
  639. os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir)
  640. ]
  641. cros_run_test_cmd += [
  642. '--host-cmd',
  643. '--',
  644. ] + cmd_args
  645. logging.info('Running the following command:')
  646. logging.info(' '.join(cros_run_test_cmd))
  647. return subprocess.call(
  648. cros_run_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env)
  649. def setup_env():
  650. """Returns a copy of the current env with some needed vars added."""
  651. env = os.environ.copy()
  652. # Some chromite scripts expect chromite/bin to be on PATH.
  653. env['PATH'] = env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin')
  654. # deploy_chrome needs a set of GN args used to build chrome to determine if
  655. # certain libraries need to be pushed to the device. It looks for the args via
  656. # an env var. To trigger the default deploying behavior, give it a dummy set
  657. # of args.
  658. # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd
  659. # line args.
  660. if not env.get('GN_ARGS'):
  661. env['GN_ARGS'] = 'enable_nacl = true'
  662. if not env.get('USE'):
  663. env['USE'] = 'highdpi'
  664. return env
  665. def add_common_args(*parsers):
  666. for parser in parsers:
  667. parser.add_argument('--verbose', '-v', action='store_true')
  668. parser.add_argument(
  669. '--board', type=str, required=True, help='Type of CrOS device.')
  670. parser.add_argument(
  671. '--cros-cache',
  672. type=str,
  673. default=DEFAULT_CROS_CACHE,
  674. help='Path to cros cache.')
  675. parser.add_argument(
  676. '--path-to-outdir',
  677. type=str,
  678. required=True,
  679. help='Path to output directory, all of whose contents will be '
  680. 'deployed to the device.')
  681. parser.add_argument(
  682. '--runtime-deps-path',
  683. type=str,
  684. help='Runtime data dependency file from GN.')
  685. parser.add_argument(
  686. '--vpython-dir',
  687. type=str,
  688. help='Location on host of a directory containing a vpython binary to '
  689. 'deploy to the device before the test starts. The location of '
  690. 'this dir will be added onto PATH in the device. WARNING: The '
  691. 'arch of the device might not match the arch of the host, so '
  692. 'avoid using "${platform}" when downloading vpython via CIPD.')
  693. parser.add_argument(
  694. '--logs-dir',
  695. type=str,
  696. dest='logs_dir',
  697. help='Will copy everything under /var/log/ from the device after the '
  698. 'test into the specified dir.')
  699. # Shard args are parsed here since we might also specify them via env vars.
  700. parser.add_argument(
  701. '--test-launcher-shard-index',
  702. type=int,
  703. default=os.environ.get('GTEST_SHARD_INDEX', 0),
  704. help='Index of the external shard to run.')
  705. parser.add_argument(
  706. '--test-launcher-total-shards',
  707. type=int,
  708. default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
  709. help='Total number of external shards.')
  710. parser.add_argument(
  711. '--flash',
  712. action='store_true',
  713. help='Will flash the device to the current SDK version before running '
  714. 'the test.')
  715. parser.add_argument(
  716. '--public-image',
  717. action='store_true',
  718. help='Will flash a public "full" image to the device.')
  719. vm_or_device_group = parser.add_mutually_exclusive_group()
  720. vm_or_device_group.add_argument(
  721. '--use-vm',
  722. action='store_true',
  723. help='Will run the test in the VM instead of a device.')
  724. vm_or_device_group.add_argument(
  725. '--device',
  726. type=str,
  727. help='Hostname (or IP) of device to run the test on. This arg is not '
  728. 'required if --use-vm is set.')
  729. def main():
  730. parser = argparse.ArgumentParser()
  731. subparsers = parser.add_subparsers(dest='test_type')
  732. # Host-side test args.
  733. host_cmd_parser = subparsers.add_parser(
  734. 'host-cmd',
  735. help='Runs a host-side test. Pass the host-side command to run after '
  736. '"--". If --use-vm is passed, hostname and port for the device '
  737. 'will be 127.0.0.1:9222.')
  738. host_cmd_parser.set_defaults(func=host_cmd)
  739. host_cmd_parser.add_argument(
  740. '--deploy-chrome',
  741. action='store_true',
  742. help='Will deploy a locally built ash-chrome binary to the device before '
  743. 'running the host-cmd.')
  744. host_cmd_parser.add_argument(
  745. '--deploy-lacros',
  746. action='store_true',
  747. help='Deploy a lacros-chrome instead of ash-chrome.')
  748. host_cmd_parser.add_argument(
  749. '--strip-chrome',
  750. action='store_true',
  751. help='Strips symbols from ash-chrome before deploying to the device.')
  752. gtest_parser = subparsers.add_parser(
  753. 'gtest', help='Runs a device-side gtest.')
  754. gtest_parser.set_defaults(func=device_test)
  755. gtest_parser.add_argument(
  756. '--test-exe',
  757. type=str,
  758. required=True,
  759. help='Path to test executable to run inside the device.')
  760. # GTest args. Some are passed down to the test binary in the device. Others
  761. # are parsed here since they might need tweaking or special handling.
  762. gtest_parser.add_argument(
  763. '--test-launcher-summary-output',
  764. type=str,
  765. help='When set, will pass the same option down to the test and retrieve '
  766. 'its result file at the specified location.')
  767. gtest_parser.add_argument(
  768. '--stop-ui',
  769. action='store_true',
  770. help='Will stop the UI service in the device before running the test.')
  771. gtest_parser.add_argument(
  772. '--trace-dir',
  773. type=str,
  774. help='When set, will pass down to the test to generate the trace and '
  775. 'retrieve the trace files to the specified location.')
  776. gtest_parser.add_argument(
  777. '--env-var',
  778. nargs=2,
  779. action='append',
  780. default=[],
  781. help='Env var to set on the device for the duration of the test. '
  782. 'Expected format is "--env-var SOME_VAR_NAME some_var_value". Specify '
  783. 'multiple times for more than one var.')
  784. # Tast test args.
  785. # pylint: disable=line-too-long
  786. tast_test_parser = subparsers.add_parser(
  787. 'tast',
  788. help='Runs a device-side set of Tast tests. For more details, see: '
  789. 'https://chromium.googlesource.com/chromiumos/platform/tast/+/main/docs/running_tests.md'
  790. )
  791. tast_test_parser.set_defaults(func=device_test)
  792. tast_test_parser.add_argument(
  793. '--suite-name',
  794. type=str,
  795. required=True,
  796. help='Name to apply to the set of Tast tests to run. This has no effect '
  797. 'on what is executed, but is used mainly for test results reporting '
  798. 'and tracking (eg: flakiness dashboard).')
  799. tast_test_parser.add_argument(
  800. '--test-launcher-summary-output',
  801. type=str,
  802. help='Generates a simple GTest-style JSON result file for the test run.')
  803. tast_test_parser.add_argument(
  804. '--attr-expr',
  805. type=str,
  806. help='A boolean expression whose matching tests will run '
  807. '(eg: ("dep:chrome")).')
  808. tast_test_parser.add_argument(
  809. '--strip-chrome',
  810. action='store_true',
  811. help='Strips symbols from ash-chrome before deploying to the device.')
  812. tast_test_parser.add_argument(
  813. '--deploy-lacros',
  814. action='store_true',
  815. help='Deploy a lacros-chrome instead of ash-chrome.')
  816. tast_test_parser.add_argument(
  817. '--tast-var',
  818. action='append',
  819. dest='tast_vars',
  820. help='Runtime variables for Tast tests, and the format are expected to '
  821. 'be "key=value" pairs.')
  822. tast_test_parser.add_argument(
  823. '--test',
  824. '-t',
  825. action='append',
  826. dest='tests',
  827. help='A Tast test to run in the device (eg: "login.Chrome").')
  828. tast_test_parser.add_argument(
  829. '--gtest_filter',
  830. type=str,
  831. help="Similar to GTest's arg of the same name, this will filter out the "
  832. "specified tests from the Tast run. However, due to the nature of Tast's "
  833. 'cmd-line API, this will overwrite the value(s) of "--test" above.')
  834. add_common_args(gtest_parser, tast_test_parser, host_cmd_parser)
  835. args = sys.argv[1:]
  836. unknown_args = []
  837. # If a '--' is present in the args, treat everything to the right of it as
  838. # args to the test and everything to the left as args to this test runner.
  839. # Otherwise treat all known args as args to this test runner and all unknown
  840. # args as test args.
  841. if '--' in args:
  842. unknown_args = args[args.index('--') + 1:]
  843. args = args[0:args.index('--')]
  844. if unknown_args:
  845. args = parser.parse_args(args=args)
  846. else:
  847. args, unknown_args = parser.parse_known_args()
  848. logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
  849. if not args.use_vm and not args.device:
  850. logging.warning(
  851. 'The test runner is now assuming running in the lab environment, if '
  852. 'this is unintentional, please re-invoke the test runner with the '
  853. '"--use-vm" arg if using a VM, otherwise use the "--device=<DUT>" arg '
  854. 'to specify a DUT.')
  855. # If we're not running on a VM, but haven't specified a hostname, assume
  856. # we're on a lab bot and are trying to run a test on a lab DUT. See if the
  857. # magic lab DUT hostname resolves to anything. (It will in the lab and will
  858. # not on dev machines.)
  859. try:
  860. socket.getaddrinfo(LAB_DUT_HOSTNAME, None)
  861. except socket.gaierror:
  862. logging.error('The default lab DUT hostname of %s is unreachable.',
  863. LAB_DUT_HOSTNAME)
  864. return 1
  865. if args.flash and args.public_image:
  866. # The flashing tools depend on being unauthenticated with GS when flashing
  867. # public images, so make sure the env var GS uses to locate its creds is
  868. # unset in that case.
  869. os.environ.pop('BOTO_CONFIG', None)
  870. return args.func(args, unknown_args)
  871. if __name__ == '__main__':
  872. sys.exit(main())