123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635 |
- # Copyright 2020 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Test apps for running tests using xcodebuild."""
- import os
- import plistlib
- import struct
- import subprocess
- import time
- import shard_util
- import test_runner
- import test_runner_errors
- import xcode_util
- OUTPUT_DISABLED_TESTS_TEST_ARG = '--write-compiled-tests-json-to-writable-path'
- def get_gtest_filter(included, excluded):
- """Returns the GTest filter to filter the given test cases.
- If only included or excluded is provided, uses GTest filter inclusion or
- exclusion syntax for the given list. If both are provided, uses included list
- minus any tests in excluded list as tests to be included.
- Args:
- included: List of test cases to be included.
- excluded: List of test cases to be excluded.
- Returns:
- A string which can be supplied to --gtest_filter.
- """
- assert included or excluded, 'One of included or excluded list should exist.'
- if included and excluded:
- included = list(set(included) - set(excluded))
- excluded = []
- # A colon-separated list of tests cases.
- # e.g. a:b:c matches a, b, c.
- # e.g. -a:b:c matches everything except a, b, c.
- test_filter = ':'.join(test for test in sorted(included + excluded))
- # This means all tests in |included| are in |excluded|.
- if not test_filter:
- return '-*'
- return '-%s' % test_filter if excluded else test_filter
- def get_bundle_id(app_path):
- """Get bundle identifier for app.
- Args:
- app_path: (str) A path to app.
- """
- return subprocess.check_output([
- '/usr/libexec/PlistBuddy',
- '-c',
- 'Print:CFBundleIdentifier',
- os.path.join(app_path, 'Info.plist'),
- ]).decode("utf-8").rstrip()
- def is_running_rosetta():
- """Returns whether Python is being translated by Rosetta.
- Returns:
- True if the Python interpreter is being run as an x86_64 binary on an arm64
- macOS machine. False if it is running as an arm64 binary, or if it is
- running on an Intel machine.
- """
- translated = subprocess.check_output(
- ['sysctl', '-i', '-b', 'sysctl.proc_translated'])
- # "sysctl -b" is expected to return a 4-byte integer response. 1 means the
- # current process is running under Rosetta, 0 means it is not. On x86_64
- # machines, this variable does not exist at all, so "-i" is used to return a
- # 0-byte response instead of throwing an error.
- if len(translated) != 4:
- return False
- return struct.unpack('i', translated)[0] > 0
- class GTestsApp(object):
- """Gtests app to run.
- Stores data about egtests:
- test_app: full path to an app.
- """
- def __init__(self, test_app, **kwargs):
- """Initialize Egtests.
- Args:
- test_app: (str) full path to egtests app.
- (Following are potential args in **kwargs)
- included_tests: (list) Specific tests to run
- E.g.
- [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
- excluded_tests: (list) Specific tests not to run
- E.g.
- [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
- test_args: List of strings to pass as arguments to the test when
- launching.
- env_vars: List of environment variables to pass to the test itself.
- release: (bool) Whether the app is release build.
- repeat_count: (int) Number of times to run each test case.
- inserted_libs: List of libraries to insert when running the test.
- Raises:
- AppNotFoundError: If the given app does not exist
- """
- if not os.path.exists(test_app):
- raise test_runner.AppNotFoundError(test_app)
- self.test_app_path = test_app
- self.project_path = os.path.dirname(self.test_app_path)
- self.test_args = kwargs.get('test_args') or []
- self.env_vars = {}
- for env_var in kwargs.get('env_vars') or []:
- env_var = env_var.split('=', 1)
- self.env_vars[env_var[0]] = None if len(env_var) == 1 else env_var[1]
- # Keep the initial included tests since creating target. Do not modify.
- self.initial_included_tests = kwargs.get('included_tests') or []
- # This may be modified between test launches.
- self.included_tests = kwargs.get('included_tests') or []
- # This may be modified between test launches.
- self.excluded_tests = kwargs.get('excluded_tests') or []
- self.disabled_tests = []
- self.module_name = os.path.splitext(os.path.basename(test_app))[0]
- self.release = kwargs.get('release')
- self.repeat_count = kwargs.get('repeat_count') or 1
- self.host_app_path = kwargs.get('host_app_path')
- self.inserted_libs = kwargs.get('inserted_libs') or []
- def remove_gtest_sharding_env_vars(self):
- """Removes sharding related env vars from self.env_vars."""
- for env_var_key in ['GTEST_SHARD_INDEX', 'GTEST_TOTAL_SHARDS']:
- self.env_vars.pop(env_var_key, None)
- def fill_xctest_run(self, out_dir):
- """Fills xctestrun file by egtests.
- Args:
- out_dir: (str) A path where xctestrun will store.
- Returns:
- A path to xctestrun file.
- """
- folder = os.path.abspath(os.path.join(out_dir, os.pardir))
- if not os.path.exists(folder):
- os.makedirs(folder)
- xctestrun = os.path.join(folder, 'run_%d.xctestrun' % int(time.time()))
- if not os.path.exists(xctestrun):
- with open(xctestrun, 'w'):
- pass
- # Creates a dict with data about egtests to run - fill all required fields:
- # egtests_module, egtest_app_path, egtests_xctest_path and
- # filtered tests if filter is specified.
- # Write data in temp xctest run file.
- plistlib.writePlist(self.fill_xctestrun_node(), xctestrun)
- return xctestrun
- @staticmethod
- def _replace_multiple_slashes(name):
- """Replace slashes with dots (.) except at the end."""
- count = name.count('/')
- if count == 0:
- return name
- return name.replace('/', '.', count - 1)
- def fill_xctestrun_node(self):
- """Fills only required nodes for egtests in xctestrun file.
- Returns:
- A node with filled required fields about egtests.
- """
- module = self.module_name + '_module'
- # If --run-with-custom-webkit is passed as a test arg, set up
- # DYLD_FRAMEWORK_PATH and DYLD_LIBRARY_PATH to load the custom webkit
- # modules.
- dyld_path = self.project_path
- if '--run-with-custom-webkit' in self.test_args:
- if self.host_app_path:
- webkit_path = os.path.join(self.host_app_path, 'WebKitFrameworks')
- else:
- webkit_path = os.path.join(self.test_app_path, 'WebKitFrameworks')
- dyld_path = dyld_path + ':' + webkit_path
- module_data = {
- 'TestBundlePath': self.test_app_path,
- 'TestHostPath': self.test_app_path,
- 'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
- 'TestingEnvironmentVariables': {
- 'DYLD_LIBRARY_PATH':
- '%s:__PLATFORMS__/iPhoneSimulator.platform/Developer/Library' %
- dyld_path,
- 'DYLD_FRAMEWORK_PATH':
- '%s:__PLATFORMS__/iPhoneSimulator.platform/'
- 'Developer/Library/Frameworks' % dyld_path,
- }
- }
- if self.inserted_libs:
- module_data['TestingEnvironmentVariables'][
- 'DYLD_INSERT_LIBRARIES'] = ':'.join(self.inserted_libs)
- xctestrun_data = {module: module_data}
- gtest_filter = []
- if self.included_tests or self.excluded_tests:
- gtest_filter = get_gtest_filter(self.included_tests, self.excluded_tests)
- # Removed previous gtest-filter if exists.
- self.test_args = [el for el in self.test_args
- if not el.startswith('--gtest_filter=')]
- self.test_args.append('--gtest_filter=%s' % gtest_filter)
- if self.repeat_count > 1:
- self.test_args.append('--gtest_repeat=%s' % self.repeat_count)
- if self.env_vars:
- xctestrun_data[module].update({'EnvironmentVariables': self.env_vars})
- if self.test_args:
- xctestrun_data[module].update({'CommandLineArguments': self.test_args})
- if self.excluded_tests:
- xctestrun_data[module].update({
- 'SkipTestIdentifiers': [
- self._replace_multiple_slashes(x) for x in self.excluded_tests
- ]
- })
- if self.included_tests:
- xctestrun_data[module].update({
- 'OnlyTestIdentifiers': [
- self._replace_multiple_slashes(x) for x in self.included_tests
- ]
- })
- return xctestrun_data
- def command(self, out_dir, destination, shards):
- """Returns the command that launches tests using xcodebuild.
- Format of command:
- xcodebuild test-without-building -xctestrun file.xctestrun \
- -parallel-testing-enabled YES -parallel-testing-worker-count %d% \
- [-destination "destination"] -resultBundlePath %output_path%
- Args:
- out_dir: (str) An output directory.
- destination: (str) A destination of running simulator.
- shards: (int) A number of shards.
- Returns:
- A list of strings forming the command to launch the test.
- """
- cmd = []
- if is_running_rosetta():
- cmd.extend(['arch', '-arch', 'arm64'])
- cmd.extend([
- 'xcodebuild', 'test-without-building', '-xctestrun',
- self.fill_xctest_run(out_dir), '-destination', destination,
- '-resultBundlePath', out_dir
- ])
- if shards > 1:
- cmd.extend([
- '-parallel-testing-enabled', 'YES', '-parallel-testing-worker-count',
- str(shards)
- ])
- return cmd
- def get_all_tests(self):
- """Gets all tests to run in this object."""
- # Method names that starts with test* and also are in *TestCase classes
- # but they are not test-methods.
- # TODO(crbug.com/982435): Rename not test methods with test-suffix.
- non_test_prefixes = [
- 'ChromeTestCase/testServer', 'FindInPageTestCase/testURL',
- 'setUpForTestCase'
- ]
- # TODO(crbug.com/1123681): Move all_tests to class var. Set all_tests,
- # disabled_tests values in initialization to avoid multiple calls to otool.
- all_tests = []
- # Only store the tests when there is the test arg.
- store_disabled_tests = OUTPUT_DISABLED_TESTS_TEST_ARG in self.test_args
- self.disabled_tests = []
- for test_class, test_method in shard_util.fetch_test_names(
- self.test_app_path,
- self.host_app_path,
- self.release,
- enabled_tests_only=False):
- test_name = '%s/%s' % (test_class, test_method)
- if any(test_name.startswith(prefix) for prefix in non_test_prefixes):
- continue
- # |self.initial_included_tests| contains the tests to execute, which
- # may be a subset of all tests b/c of the iOS test sharding logic in
- # run.py. Filter by |self.initial_included_tests| if specified.
- # |self.initial_included_tests| might store test class or full name.
- included = self.initial_included_tests
- if not included or test_name in included or test_class in included:
- if test_method.startswith('test'):
- all_tests.append(test_name)
- elif store_disabled_tests:
- self.disabled_tests.append(test_name)
- return all_tests
- class EgtestsApp(GTestsApp):
- """Egtests to run.
- Stores data about egtests:
- egtests_app: full path to egtests app.
- project_path: root project folder.
- module_name: egtests module name.
- included_tests: List of tests to run.
- excluded_tests: List of tests not to run.
- """
- def __init__(self, egtests_app, **kwargs):
- """Initialize Egtests.
- Args:
- egtests_app: (str) full path to egtests app.
- (Following are potential args in **kwargs)
- included_tests: (list) Specific tests to run
- E.g.
- [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
- excluded_tests: (list) Specific tests not to run
- E.g.
- [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
- test_args: List of strings to pass as arguments to the test when
- launching.
- env_vars: List of environment variables to pass to the test itself.
- host_app_path: (str) full path to host app.
- inserted_libs: List of libraries to insert when running the test.
- repeat_count: (int) Number of times to run each test case.
- Raises:
- AppNotFoundError: If the given app does not exist
- """
- inserted_libs = list(kwargs.get('inserted_libs') or [])
- inserted_libs.append('__PLATFORMS__/iPhoneSimulator.platform/Developer/'
- 'usr/lib/libXCTestBundleInject.dylib')
- kwargs['inserted_libs'] = inserted_libs
- super(EgtestsApp, self).__init__(egtests_app, **kwargs)
- def _xctest_path(self):
- """Gets xctest-file from egtests/PlugIns folder.
- Returns:
- A path for xctest in the format of /PlugIns/file.xctest
- Raises:
- PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
- XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
- """
- plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
- if not os.path.exists(plugins_dir):
- raise test_runner.PlugInsNotFoundError(plugins_dir)
- plugin_xctest = None
- if os.path.exists(plugins_dir):
- for plugin in os.listdir(plugins_dir):
- if plugin.endswith('.xctest'):
- plugin_xctest = os.path.join(plugins_dir, plugin)
- if not plugin_xctest:
- raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
- return plugin_xctest.replace(self.test_app_path, '')
- def command(self, out_dir, destination, shards):
- """Returns the command that launches tests for EG Tests.
- See details in parent class method docstring. This method appends the
- command line switch if test repeat is required.
- """
- cmd = super(EgtestsApp, self).command(out_dir, destination, shards)
- if self.repeat_count > 1:
- if xcode_util.using_xcode_13_or_higher():
- cmd += ['-test-iterations', str(self.repeat_count)]
- else:
- raise test_runner_errors.XcodeUnsupportedFeatureError(
- 'Test repeat is only supported in Xcode 13 or higher!')
- return cmd
- def fill_xctestrun_node(self):
- """Fills only required nodes for egtests in xctestrun file.
- Returns:
- A node with filled required fields about egtests.
- """
- xctestrun_data = super(EgtestsApp, self).fill_xctestrun_node()
- module_data = xctestrun_data[self.module_name + '_module']
- module_data['TestBundlePath'] = '__TESTHOST__%s' % self._xctest_path()
- module_data['TestingEnvironmentVariables'][
- 'XCInjectBundleInto'] = '__TESTHOST__/%s' % self.module_name
- if self.host_app_path:
- # Module data specific to EG2 tests
- module_data['IsUITestBundle'] = True
- module_data['IsXCTRunnerHostedTestBundle'] = True
- module_data['UITargetAppPath'] = '%s' % self.host_app_path
- module_data['UITargetAppBundleIdentifier'] = get_bundle_id(
- self.host_app_path)
- # Special handling for Xcode10.2
- dependent_products = [
- module_data['UITargetAppPath'],
- module_data['TestBundlePath'],
- module_data['TestHostPath']
- ]
- module_data['DependentProductPaths'] = dependent_products
- # Module data specific to EG1 tests
- else:
- module_data['IsAppHostedTestBundle'] = True
- return xctestrun_data
- class DeviceXCTestUnitTestsApp(GTestsApp):
- """XCTest hosted unit tests to run on devices.
- This is for the XCTest framework hosted unit tests running on devices.
- Stores data about tests:
- tests_app: full path to tests app.
- project_path: root project folder.
- module_name: egtests module name.
- included_tests: List of tests to run.
- excluded_tests: List of tests not to run.
- """
- def __init__(self, tests_app, **kwargs):
- """Initialize the class.
- Args:
- tests_app: (str) full path to tests app.
- (Following are potential args in **kwargs)
- included_tests: (list) Specific tests to run
- E.g.
- [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
- excluded_tests: (list) Specific tests not to run
- E.g.
- [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
- test_args: List of strings to pass as arguments to the test when
- launching. Test arg to run as XCTest based unit test will be appended.
- env_vars: List of environment variables to pass to the test itself.
- repeat_count: (int) Number of times to run each test case.
- Raises:
- AppNotFoundError: If the given app does not exist
- """
- test_args = list(kwargs.get('test_args') or [])
- test_args.append('--enable-run-ios-unittests-with-xctest')
- kwargs['test_args'] = test_args
- super(DeviceXCTestUnitTestsApp, self).__init__(tests_app, **kwargs)
- # TODO(crbug.com/1077277): Refactor class structure and remove duplicate code.
- def _xctest_path(self):
- """Gets xctest-file from egtests/PlugIns folder.
- Returns:
- A path for xctest in the format of /PlugIns/file.xctest
- Raises:
- PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
- XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
- """
- plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
- if not os.path.exists(plugins_dir):
- raise test_runner.PlugInsNotFoundError(plugins_dir)
- plugin_xctest = None
- if os.path.exists(plugins_dir):
- for plugin in os.listdir(plugins_dir):
- if plugin.endswith('.xctest'):
- plugin_xctest = os.path.join(plugins_dir, plugin)
- if not plugin_xctest:
- raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
- return plugin_xctest.replace(self.test_app_path, '')
- def fill_xctestrun_node(self):
- """Fills only required nodes for XCTest hosted unit tests in xctestrun file.
- Returns:
- A node with filled required fields about tests.
- """
- xctestrun_data = {
- 'TestTargetName': {
- 'IsAppHostedTestBundle': True,
- 'TestBundlePath': '__TESTHOST__%s' % self._xctest_path(),
- 'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
- 'TestHostPath': '%s' % self.test_app_path,
- 'TestingEnvironmentVariables': {
- 'DYLD_INSERT_LIBRARIES':
- '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib',
- 'DYLD_LIBRARY_PATH':
- '__PLATFORMS__/iPhoneOS.platform/Developer/Library',
- 'DYLD_FRAMEWORK_PATH':
- '__PLATFORMS__/iPhoneOS.platform/Developer/'
- 'Library/Frameworks',
- 'XCInjectBundleInto':
- '__TESTHOST__/%s' % self.module_name
- }
- }
- }
- if self.env_vars:
- xctestrun_data['TestTargetName'].update(
- {'EnvironmentVariables': self.env_vars})
- if self.included_tests or self.excluded_tests:
- gtest_filter = get_gtest_filter(self.included_tests, self.excluded_tests)
- # Removed previous gtest-filter if exists.
- self.test_args = [
- el for el in self.test_args if not el.startswith('--gtest_filter=')
- ]
- self.test_args.append('--gtest_filter=%s' % gtest_filter)
- if self.repeat_count > 1:
- self.test_args.append('--gtest_repeat=%s' % self.repeat_count)
- self.test_args.append('--gmock_verbose=error')
- xctestrun_data['TestTargetName'].update(
- {'CommandLineArguments': self.test_args})
- return xctestrun_data
- class SimulatorXCTestUnitTestsApp(GTestsApp):
- """XCTest hosted unit tests to run on simulators.
- This is for the XCTest framework hosted unit tests running on simulators.
- Stores data about tests:
- tests_app: full path to tests app.
- project_path: root project folder.
- module_name: egtests module name.
- included_tests: List of tests to run.
- excluded_tests: List of tests not to run.
- """
- def __init__(self, tests_app, **kwargs):
- """Initialize the class.
- Args:
- tests_app: (str) full path to tests app.
- (Following are potential args in **kwargs)
- included_tests: (list) Specific tests to run
- E.g.
- [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
- excluded_tests: (list) Specific tests not to run
- E.g.
- [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
- test_args: List of strings to pass as arguments to the test when
- launching. Test arg to run as XCTest based unit test will be appended.
- env_vars: List of environment variables to pass to the test itself.
- repeat_count: (int) Number of times to run each test case.
- Raises:
- AppNotFoundError: If the given app does not exist
- """
- test_args = list(kwargs.get('test_args') or [])
- test_args.append('--enable-run-ios-unittests-with-xctest')
- kwargs['test_args'] = test_args
- super(SimulatorXCTestUnitTestsApp, self).__init__(tests_app, **kwargs)
- # TODO(crbug.com/1077277): Refactor class structure and remove duplicate code.
- def _xctest_path(self):
- """Gets xctest-file from egtests/PlugIns folder.
- Returns:
- A path for xctest in the format of /PlugIns/file.xctest
- Raises:
- PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
- XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
- """
- plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
- if not os.path.exists(plugins_dir):
- raise test_runner.PlugInsNotFoundError(plugins_dir)
- plugin_xctest = None
- if os.path.exists(plugins_dir):
- for plugin in os.listdir(plugins_dir):
- if plugin.endswith('.xctest'):
- plugin_xctest = os.path.join(plugins_dir, plugin)
- if not plugin_xctest:
- raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
- return plugin_xctest.replace(self.test_app_path, '')
- def fill_xctestrun_node(self):
- """Fills only required nodes for XCTest hosted unit tests in xctestrun file.
- Returns:
- A node with filled required fields about tests.
- """
- xctestrun_data = {
- 'TestTargetName': {
- 'IsAppHostedTestBundle': True,
- 'TestBundlePath': '__TESTHOST__%s' % self._xctest_path(),
- 'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
- 'TestHostPath': '%s' % self.test_app_path,
- 'TestingEnvironmentVariables': {
- 'DYLD_INSERT_LIBRARIES':
- '__PLATFORMS__/iPhoneSimulator.platform/Developer/usr/lib/'
- 'libXCTestBundleInject.dylib',
- 'DYLD_LIBRARY_PATH':
- '__PLATFORMS__/iPhoneSimulator.platform/Developer/Library',
- 'DYLD_FRAMEWORK_PATH':
- '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
- 'Library/Frameworks',
- 'XCInjectBundleInto':
- '__TESTHOST__/%s' % self.module_name
- }
- }
- }
- if self.env_vars:
- xctestrun_data['TestTargetName'].update(
- {'EnvironmentVariables': self.env_vars})
- if self.included_tests or self.excluded_tests:
- gtest_filter = get_gtest_filter(self.included_tests, self.excluded_tests)
- # Removed previous gtest-filter if exists.
- self.test_args = [
- el for el in self.test_args if not el.startswith('--gtest_filter=')
- ]
- self.test_args.append('--gtest_filter=%s' % gtest_filter)
- if self.repeat_count > 1:
- self.test_args.append('--gtest_repeat=%s' % self.repeat_count)
- self.test_args.append('--gmock_verbose=error')
- xctestrun_data['TestTargetName'].update(
- {'CommandLineArguments': self.test_args})
- return xctestrun_data
|