run_cts_test.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. #!/usr/bin/env vpython3
  2. # Copyright 2019 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. from __future__ import absolute_import
  6. import logging
  7. import os
  8. import tempfile
  9. import sys
  10. import unittest
  11. import mock # pylint: disable=import-error
  12. import run_cts
  13. sys.path.append(os.path.join(
  14. os.path.dirname(__file__), os.pardir, os.pardir, 'build', 'android'))
  15. # pylint: disable=wrong-import-position,import-error
  16. import devil_chromium # pylint: disable=unused-import
  17. from devil.android.ndk import abis
  18. from devil.android.sdk import version_codes
  19. class _RunCtsTest(unittest.TestCase):
  20. """Unittests for the run_cts module.
  21. """
  22. _EXCLUDED_TEST = 'bad#test'
  23. # This conforms to schema of the test_run entry in
  24. # cts_config/webview_cts_gcs_path.json
  25. _CTS_RUN = {'apk': 'module.apk', 'excludes': [{'match': _EXCLUDED_TEST}]}
  26. @staticmethod
  27. def _getArgsMock(**kwargs):
  28. args = {
  29. 'test_filter_files': None,
  30. 'test_filter': None,
  31. 'isolated_script_test_filter': None,
  32. 'skip_expected_failures': False
  33. }
  34. args.update(kwargs)
  35. return mock.Mock(**args)
  36. def _getSkipString(self):
  37. return self._EXCLUDED_TEST.replace('#', '.')
  38. def testDetermineArch_arm64(self):
  39. logging_mock = mock.Mock()
  40. logging.info = logging_mock
  41. device = mock.Mock(product_cpu_abi=abis.ARM_64)
  42. self.assertEqual(run_cts.DetermineArch(device), 'arm64')
  43. # We should log a message to explain how we auto-determined the arch. We
  44. # don't assert the message itself, since that's rather strict.
  45. logging_mock.assert_called()
  46. def testDetermineArch_unsupported(self):
  47. device = mock.Mock(product_cpu_abi='madeup-abi')
  48. with self.assertRaises(Exception) as _:
  49. run_cts.DetermineArch(device)
  50. def testDetermineCtsRelease_marshmallow(self):
  51. logging_mock = mock.Mock()
  52. logging.info = logging_mock
  53. device = mock.Mock(build_version_sdk=version_codes.MARSHMALLOW)
  54. self.assertEqual(run_cts.DetermineCtsRelease(device), 'M')
  55. # We should log a message to explain how we auto-determined the CTS release.
  56. # We don't assert the message itself, since that's rather strict.
  57. logging_mock.assert_called()
  58. def testDetermineCtsRelease_tooLow(self):
  59. device = mock.Mock(build_version_sdk=version_codes.LOLLIPOP_MR1)
  60. with self.assertRaises(Exception) as cm:
  61. run_cts.DetermineCtsRelease(device)
  62. message = str(cm.exception)
  63. self.assertIn("We don't support running CTS tests on platforms less than",
  64. message)
  65. def testDetermineCtsRelease_tooHigh(self):
  66. device = mock.Mock(build_version_sdk=version_codes.OREO)
  67. # Mock this out with a couple version codes to check that the logic is
  68. # correct, without making assumptions about what version_codes we may
  69. # support in the future.
  70. mock_sdk_platform_dict = {
  71. version_codes.MARSHMALLOW: 'min fake release',
  72. version_codes.NOUGAT: 'max fake release',
  73. }
  74. run_cts.SDK_PLATFORM_DICT = mock_sdk_platform_dict
  75. with self.assertRaises(Exception) as cm:
  76. run_cts.DetermineCtsRelease(device)
  77. message = str(cm.exception)
  78. self.assertIn('--cts-release max fake release', message,
  79. msg='Should recommend the highest supported CTS release')
  80. def testNoFilter_SkipExpectedFailures(self):
  81. mock_args = self._getArgsMock(skip_expected_failures=True)
  82. skip = self._getSkipString()
  83. self.assertEqual([run_cts.TEST_FILTER_OPT + '=-' + skip],
  84. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  85. def testNoFilter_ExcludedMatches(self):
  86. mock_args = self._getArgsMock(skip_expected_failures=False)
  87. self.assertEqual([],
  88. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  89. def testFilter_CombinesExcludedMatches(self):
  90. mock_args = self._getArgsMock(test_filter='good#test',
  91. skip_expected_failures=False)
  92. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test'],
  93. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  94. def testFilter_CombinesAll(self):
  95. mock_args = self._getArgsMock(test_filter='good#test',
  96. skip_expected_failures=True)
  97. skip = self._getSkipString()
  98. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test-' + skip],
  99. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  100. def testFilter_ForMultipleTests(self):
  101. mock_args = self._getArgsMock(test_filter='good#t1:good#t2',
  102. skip_expected_failures=True)
  103. skip = self._getSkipString()
  104. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.t1:good.t2-' + skip],
  105. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  106. def testFilter_IncludesForArchitecture(self):
  107. mock_args = self._getArgsMock()
  108. cts_run = {
  109. 'apk':
  110. 'module.apk',
  111. 'includes': [{
  112. 'match': 'good#test1',
  113. 'arch': 'x86'
  114. }, {
  115. 'match': 'good#test2'
  116. }, {
  117. 'match': 'exclude#test4',
  118. 'arch': 'arm64'
  119. }]
  120. }
  121. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test1:good.test2'],
  122. run_cts.GetTestRunFilterArg(mock_args, cts_run,
  123. arch='x86'))
  124. def testFilter_ExcludesForArchitecture(self):
  125. mock_args = self._getArgsMock(skip_expected_failures=True)
  126. cts_run = {
  127. 'apk':
  128. 'module.apk',
  129. 'excludes': [{
  130. 'match': 'good#test1',
  131. 'arch': 'x86'
  132. }, {
  133. 'match': 'good#test2'
  134. }, {
  135. 'match': 'exclude#test4',
  136. 'arch': 'arm64'
  137. }]
  138. }
  139. self.assertEqual([run_cts.TEST_FILTER_OPT + '=-good.test1:good.test2'],
  140. run_cts.GetTestRunFilterArg(mock_args, cts_run,
  141. arch='x86'))
  142. def testFilter_IncludesForMode(self):
  143. mock_args = self._getArgsMock()
  144. cts_run = {
  145. 'apk':
  146. 'module.apk',
  147. 'includes': [{
  148. 'match': 'good#test1',
  149. 'mode': 'instant'
  150. }, {
  151. 'match': 'good#test2'
  152. }, {
  153. 'match': 'exclude#test4',
  154. 'mode': 'full'
  155. }]
  156. }
  157. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test1:good.test2'],
  158. run_cts.GetTestRunFilterArg(mock_args,
  159. cts_run,
  160. test_app_mode='instant'))
  161. def testFilter_ExcludesForMode(self):
  162. mock_args = self._getArgsMock(skip_expected_failures=True)
  163. cts_run = {
  164. 'apk':
  165. 'module.apk',
  166. 'excludes': [{
  167. 'match': 'good#test1',
  168. 'mode': 'instant'
  169. }, {
  170. 'match': 'good#test2'
  171. }, {
  172. 'match': 'exclude#test4',
  173. 'mode': 'full'
  174. }]
  175. }
  176. self.assertEqual([run_cts.TEST_FILTER_OPT + '=-good.test1:good.test2'],
  177. run_cts.GetTestRunFilterArg(mock_args,
  178. cts_run,
  179. test_app_mode='instant'))
  180. def testIsolatedFilter_CombinesExcludedMatches(self):
  181. mock_args = self._getArgsMock(isolated_script_test_filter='good#test',
  182. skip_expected_failures=False)
  183. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test'],
  184. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  185. def testIsolatedFilter_CombinesAll(self):
  186. mock_args = self._getArgsMock(isolated_script_test_filter='good#test',
  187. skip_expected_failures=True)
  188. skip = self._getSkipString()
  189. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test-' + skip],
  190. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  191. def testIsolatedFilter_ForMultipleTests(self):
  192. # Isolated test filters use :: to separate matches
  193. mock_args = self._getArgsMock(
  194. isolated_script_test_filter='good#t1::good#t2',
  195. skip_expected_failures=True)
  196. skip = self._getSkipString()
  197. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.t1:good.t2-' + skip],
  198. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  199. @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name "
  200. "doesn't work in Windows.")
  201. def testFilterFile_CombinesExcludedMatches(self):
  202. with tempfile.NamedTemporaryFile(prefix='cts_run_test') as filter_file:
  203. filter_file.write('suite.goodtest'.encode())
  204. filter_file.seek(0)
  205. mock_args = self._getArgsMock(test_filter_files=[filter_file.name],
  206. skip_expected_failures=False)
  207. self.assertEqual([run_cts.TEST_FILTER_OPT + '=suite.goodtest'],
  208. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  209. @unittest.skipIf(os.name == "nt", "Opening NamedTemporaryFile by name "
  210. "doesn't work in Windows.")
  211. def testFilterFile_CombinesAll(self):
  212. with tempfile.NamedTemporaryFile(prefix='cts_run_test') as filter_file:
  213. filter_file.write('suite.goodtest'.encode())
  214. filter_file.seek(0)
  215. mock_args = self._getArgsMock(test_filter_files=[filter_file.name],
  216. skip_expected_failures=True)
  217. skip = self._getSkipString()
  218. self.assertEqual([run_cts.TEST_FILTER_OPT + '=suite.goodtest-' + skip],
  219. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  220. def testNegative_Filter(self):
  221. mock_args = self._getArgsMock(test_filter='-good#t1:good#t2',
  222. skip_expected_failures=True)
  223. skip = self._getSkipString()
  224. self.assertEqual([run_cts.TEST_FILTER_OPT + '=-good.t1:good.t2:' + skip],
  225. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  226. def testNegative_IsolatedFilter(self):
  227. mock_args = self._getArgsMock(
  228. isolated_script_test_filter='-good#t1::good#t2',
  229. skip_expected_failures=True)
  230. skip = self._getSkipString()
  231. self.assertEqual([run_cts.TEST_FILTER_OPT + '=-good.t1:good.t2:' + skip],
  232. run_cts.GetTestRunFilterArg(mock_args, self._CTS_RUN))
  233. def testFilter_OverridesInclusion(self):
  234. mock_args = self._getArgsMock(test_filter='good#test1',
  235. skip_expected_failures=False)
  236. cts_run = {'apk': 'module.apk', 'includes': [{'match': 'good#test2'}]}
  237. self.assertEqual([run_cts.TEST_FILTER_OPT + '=good.test1'],
  238. run_cts.GetTestRunFilterArg(mock_args, cts_run))
  239. if __name__ == '__main__':
  240. unittest.main()