123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313 |
- #!/usr/bin/env vpython3
- # Copyright 2016 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Unittests for test_runner.py."""
- import collections
- import logging
- import mock
- import os
- import tempfile
- import unittest
- import iossim_util
- import result_sink_util
- import test_apps
- from test_result_util import ResultCollection, TestResult, TestStatus
- import test_runner
- class TestCase(unittest.TestCase):
- """Test case which supports installing mocks. Uninstalls on tear down."""
- def __init__(self, *args, **kwargs):
- """Initializes a new instance of this class."""
- super(TestCase, self).__init__(*args, **kwargs)
- # Maps object to a dict which maps names of mocked members to their
- # original values.
- self._mocks = collections.OrderedDict()
- def mock(self, obj, member, mock):
- """Installs mock in place of the named member of the given obj.
- Args:
- obj: Any object.
- member: String naming the attribute of the object to mock.
- mock: The mock to install.
- """
- self._mocks.setdefault(obj, collections.OrderedDict()).setdefault(
- member, getattr(obj, member))
- setattr(obj, member, mock)
- def unmock(self, obj, member):
- """Uninstalls the mock from the named member of given obj.
- Args:
- obj: An obj who's member has been mocked
- member: String naming the attribute of the object to unmock
- """
- if self._mocks[obj][member]:
- setattr(obj, member, self._mocks[obj][member])
- def tearDown(self, *args, **kwargs):
- """Uninstalls mocks."""
- super(TestCase, self).tearDown(*args, **kwargs)
- for obj in self._mocks:
- for member, original_value in self._mocks[obj].items():
- setattr(obj, member, original_value)
- class SimulatorTestRunnerTest(TestCase):
- """Tests for test_runner.SimulatorTestRunner."""
- def setUp(self):
- super(SimulatorTestRunnerTest, self).setUp()
- self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID')
- self.mock(result_sink_util.ResultSinkClient,
- 'post', lambda *args, **kwargs: None)
- self.mock(test_runner, 'get_current_xcode_info', lambda: {
- 'version': 'test version', 'build': 'test build', 'path': 'test/path'})
- self.mock(test_apps, 'get_bundle_id', lambda _: 'fake-bundle-id')
- self.mock(test_apps.SimulatorXCTestUnitTestsApp,
- '_xctest_path', lambda _: 'fake-path')
- self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '')
- self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
- self.mock(os.path, 'exists', lambda _: True)
- self.mock(test_runner.TestRunner, 'set_sigterm_handler',
- lambda self, handler: 0)
- self.mock(os, 'listdir', lambda _: [])
- def test_app_not_found(self):
- """Ensures AppNotFoundError is raised."""
- self.mock(os.path, 'exists', lambda p: not p.endswith('fake-app'))
- with self.assertRaises(test_runner.AppNotFoundError):
- test_runner.SimulatorTestRunner(
- 'fake-app',
- 'fake-iossim',
- 'platform',
- 'os',
- 'out-dir',
- )
- def test_iossim_not_found(self):
- """Ensures SimulatorNotFoundError is raised."""
- self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim'))
- with self.assertRaises(test_runner.SimulatorNotFoundError):
- test_runner.SimulatorTestRunner(
- 'fake-app',
- 'fake-iossim',
- 'iPhone X',
- '11.4',
- 'out-dir',
- )
- def test_init(self):
- """Ensures instance is created."""
- tr = test_runner.SimulatorTestRunner(
- 'fake-app',
- 'fake-iossim',
- 'iPhone X',
- '11.4',
- 'out-dir',
- )
- self.assertTrue(tr)
- @mock.patch('test_runner.SimulatorTestRunner.tear_down')
- @mock.patch('test_runner.SimulatorTestRunner.set_up')
- @mock.patch('test_runner.TestRunner._run')
- def test_startup_crash(self, mock_run, _1, _2):
- """Ensures test is relaunched once on startup crash."""
- result = ResultCollection()
- result.crashed = True
- mock_run.return_value = result
- tr = test_runner.SimulatorTestRunner(
- 'fake-app',
- 'fake-iossim',
- 'iPhone X',
- '11.4',
- 'out-dir',
- xctest=True,
- )
- with self.assertRaises(test_runner.AppLaunchError):
- tr.launch()
- self.assertEquals(len(mock_run.mock_calls), 2)
- def test_relaunch(self):
- """Ensures test is relaunched on test crash until tests complete."""
- def set_up(self):
- return
- @staticmethod
- def _run(cmd, shards=None):
- if not any('retry_after_crash' in cmd_arg for cmd_arg in cmd):
- # First run, has no test filter supplied. Mock a crash.
- result = ResultCollection(
- test_results=[TestResult('crash', TestStatus.CRASH)])
- result.crashed = True
- result.add_test_result(TestResult('pass', TestStatus.PASS))
- result.add_test_result(
- TestResult('fail', TestStatus.FAIL, test_log='some logs'))
- return result
- else:
- return ResultCollection(
- test_results=[TestResult('crash', TestStatus.PASS)])
- def tear_down(self):
- return
- self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
- self.mock(test_runner.TestRunner, '_run', _run)
- self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
- tr = test_runner.SimulatorTestRunner(
- 'fake-app',
- 'fake-iossim',
- 'iPhone X',
- '11.4',
- 'out-dir',
- )
- tr.launch()
- self.assertTrue(tr.logs)
- @mock.patch('test_runner.SimulatorTestRunner.tear_down')
- @mock.patch('test_runner.SimulatorTestRunner.set_up')
- @mock.patch('test_runner.TestRunner._run')
- def test_failed_test_retry(self, mock_run, _1, _2):
- test1_fail_result = TestResult('test1', TestStatus.FAIL)
- test2_fail_result = TestResult('test2', TestStatus.FAIL)
- test1_pass_result = TestResult('test1', TestStatus.PASS)
- test2_pass_result = TestResult('test2', TestStatus.PASS)
- result1 = ResultCollection(
- test_results=[test1_fail_result, test2_fail_result])
- retry_result1 = ResultCollection(test_results=[test1_pass_result])
- retry_result2 = ResultCollection(test_results=[test2_pass_result])
- mock_run.side_effect = [result1, retry_result1, retry_result2]
- tr = test_runner.SimulatorTestRunner(
- 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
- tr.launch()
- self.assertEquals(len(mock_run.mock_calls), 3)
- self.assertTrue(tr.logs)
- @mock.patch('test_runner.SimulatorTestRunner.tear_down')
- @mock.patch('test_runner.SimulatorTestRunner.set_up')
- @mock.patch('test_runner.TestRunner._run')
- def test_crashed_if_crash_in_final_crash_retry(self, mock_run, _1, _2):
- test1_crash_result = TestResult('test1', TestStatus.CRASH)
- test2_crash_result = TestResult('test2', TestStatus.CRASH)
- test3_pass_result = TestResult('test3', TestStatus.PASS)
- test1_pass_result = TestResult('test1', TestStatus.PASS)
- test2_pass_result = TestResult('test2', TestStatus.PASS)
- initial_result = ResultCollection(test_results=[test1_crash_result])
- initial_result.crashed = True
- crash_retry1_result = ResultCollection(test_results=[test2_crash_result])
- crash_retry1_result.crashed = True
- crash_retry2_result = ResultCollection(test_results=[test3_pass_result])
- crash_retry2_result.crashed = True
- test_retry1_result = ResultCollection(test_results=[test1_pass_result])
- test_retry2_result = ResultCollection(test_results=[test2_pass_result])
- mock_run.side_effect = [
- initial_result, crash_retry1_result, crash_retry2_result,
- test_retry1_result, test_retry2_result
- ]
- tr = test_runner.SimulatorTestRunner(
- 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
- tr.launch()
- self.assertEquals(len(mock_run.mock_calls), 5)
- self.assertTrue(tr.test_results['interrupted'])
- self.assertIn('test suite crash', tr.logs)
- self.assertTrue(tr.logs)
- @mock.patch('test_runner.SimulatorTestRunner.tear_down')
- @mock.patch('test_runner.SimulatorTestRunner.set_up')
- @mock.patch('test_runner.TestRunner._run')
- def test_not_crashed_if_no_crash_in_final_crash_retry(self, mock_run, _1, _2):
- test1_crash_result = TestResult('test1', TestStatus.CRASH)
- test2_crash_result = TestResult('test2', TestStatus.CRASH)
- test3_pass_result = TestResult('test3', TestStatus.PASS)
- test1_pass_result = TestResult('test1', TestStatus.PASS)
- test2_pass_result = TestResult('test2', TestStatus.PASS)
- initial_result = ResultCollection(test_results=[test1_crash_result])
- initial_result.crashed = True
- crash_retry1_result = ResultCollection(test_results=[test2_crash_result])
- crash_retry1_result.crashed = True
- crash_retry2_result = ResultCollection(test_results=[test3_pass_result])
- test_retry1_result = ResultCollection(test_results=[test1_pass_result])
- test_retry2_result = ResultCollection(test_results=[test2_pass_result])
- mock_run.side_effect = [
- initial_result, crash_retry1_result, crash_retry2_result,
- test_retry1_result, test_retry2_result
- ]
- tr = test_runner.SimulatorTestRunner(
- 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
- tr.launch()
- self.assertEquals(len(mock_run.mock_calls), 5)
- self.assertFalse(tr.test_results['interrupted'])
- self.assertTrue(tr.logs)
- @mock.patch('test_runner.SimulatorTestRunner.tear_down')
- @mock.patch('test_runner.SimulatorTestRunner.set_up')
- @mock.patch('test_runner.TestRunner._run')
- def test_not_crashed_if_crashed_in_failed_test_retry(self, mock_run, _1, _2):
- test1_fail_result = TestResult('test1', TestStatus.FAIL)
- initial_result = ResultCollection(test_results=[test1_fail_result])
- test1_retry1_result = ResultCollection(test_results=[test1_fail_result])
- test1_retry2_result = ResultCollection(test_results=[test1_fail_result])
- test1_retry3_result = ResultCollection()
- test1_retry3_result.crashed = True
- mock_run.side_effect = [
- initial_result, test1_retry1_result, test1_retry2_result,
- test1_retry3_result
- ]
- tr = test_runner.SimulatorTestRunner(
- 'fake-app', 'fake-iossim', 'iPhone X', '11.4', 'out-dir', retries=3)
- tr.launch()
- self.assertEquals(len(mock_run.mock_calls), 4)
- self.assertFalse(tr.test_results['interrupted'])
- self.assertEquals(tr.test_results['tests']['test1']['actual'],
- 'FAIL FAIL FAIL SKIP')
- self.assertTrue(tr.logs)
- class DeviceTestRunnerTest(TestCase):
- def setUp(self):
- super(DeviceTestRunnerTest, self).setUp()
- def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
- return True
- self.mock(result_sink_util.ResultSinkClient,
- 'post', lambda *args, **kwargs: None)
- self.mock(test_runner, 'get_current_xcode_info', lambda: {
- 'version': 'test version', 'build': 'test build', 'path': 'test/path'})
- self.mock(test_runner, 'install_xcode', install_xcode)
- self.mock(test_runner.subprocess,
- 'check_output', lambda _: b'fake-bundle-id')
- self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
- self.mock(os.path, 'exists', lambda _: True)
- self.mock(os, 'listdir', lambda _: [])
- self.mock(tempfile, 'mkstemp', lambda: '/tmp/tmp_file')
- self.tr = test_runner.DeviceTestRunner(
- 'fake-app',
- 'xcode-version',
- 'xcode-build',
- 'out-dir',
- )
- self.tr.xctestrun_data = {'TestTargetName': {}}
- if __name__ == '__main__':
- logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
- level=logging.DEBUG, datefmt='%I:%M:%S')
- unittest.main()
|