shard_util_test.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. #!/usr/bin/env vpython3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import collections
  6. from mock import patch
  7. import os
  8. import unittest
  9. import shard_util
  10. DEBUG_APP_OTOOL_OUTPUT = '\n'.join([
  11. 'Meta Class', 'name 0x1064b8438 CacheTestCase',
  12. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  13. 'imp 0x1075e6887 -[CacheTestCase testA]', 'types 0x1064cc3e1',
  14. 'imp 0x1075e6887 -[CacheTestCase testB]',
  15. 'imp 0x1075e6887 -[CacheTestCase testc]', 'name 0x1064b8438 TabUITestCase',
  16. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  17. 'imp 0x1075e6887 -[TabUITestCase testD]', 'types 0x1064cc3e1 v16@0:8',
  18. 'imp 0x1075e6887 -[TabUITestCase testE]',
  19. 'name 0x1064b8438 KeyboardTestCase',
  20. 'imp 0x1075e6887 -[KeyboardTestCase testF]',
  21. 'name 0x1064b8438 PasswordsTestCase',
  22. 'imp 0x1075e6887 -[PasswordsTestCase testG]',
  23. 'name 0x1064b8438 ToolBarTestCase',
  24. 'imp 0x1075e6887 -[ToolBarTestCase testH]',
  25. 'imp 0x1075e6887 -[ToolBarTestCase DISABLED_testI]',
  26. 'imp 0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0'
  27. ]).encode('utf-8')
  28. # Debug app otool output format in Xcode 11.4 toolchain.
  29. DEBUG_APP_OTOOL_OUTPUT_114 = '\n'.join([
  30. 'Meta Class', 'name 0x1064b8438 CacheTestCase',
  31. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  32. ' imp 0x1075e6887 -[CacheTestCase testA]', ' types 0x1064cc3e1',
  33. ' imp 0x1075e6887 -[CacheTestCase testB]',
  34. ' imp 0x1075e6887 -[CacheTestCase testc]',
  35. ' name 0x1064b8438 TabUITestCase',
  36. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  37. ' imp 0x1075e6887 -[TabUITestCase testD]',
  38. ' types 0x1064cc3e1 v16@0:8',
  39. ' imp 0x1075e6887 -[TabUITestCase testE]',
  40. ' name 0x1064b8438 KeyboardTestCase',
  41. ' imp 0x1075e6887 -[KeyboardTestCase testF]',
  42. ' name 0x1064b8438 PasswordsTestCase',
  43. ' imp 0x1075e6887 -[PasswordsTestCase testG]',
  44. ' name 0x1064b8438 ToolBarTestCase',
  45. ' imp 0x1075e6887 -[ToolBarTestCase testH]',
  46. ' imp 0x1075e6887 -[ToolBarTestCase DISABLED_testI]',
  47. ' imp 0x1075e6887 -[ToolBarTestCase FLAKY_testJ]', 'version 0'
  48. ]).encode('utf-8')
  49. RELEASE_APP_OTOOL_OUTPUT = '\n'.join([
  50. 'Meta Class', 'name 0x1064b8438 CacheTestCase',
  51. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  52. 'name 0x1075e6887 testA', 'types 0x1064cc3e1', 'name 0x1075e6887 testB',
  53. 'name 0x1075e6887 testc', 'baseProtocols 0x0',
  54. 'name 0x1064b8438 CacheTestCase', 'unrelated line', 'Meta Class',
  55. 'name 0x1064b8438 TabUITestCase', 'no test methods in this case',
  56. 'name 0x1064b8438 TabUITestCase', 'unrelated line',
  57. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  58. 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testD',
  59. 'types 0x1064cc3e1 v16@0:8', 'name 0x1075e6887 testE',
  60. 'name 0x1075e6887 testF', 'baseProtocols 0x0',
  61. 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testUnrelatedG',
  62. 'unrelated line', 'name 0x1064b8438 ChromeTestCase',
  63. 'name 0x1064b8438 setUp', 'name 0x1064b8438 testPort',
  64. 'name 0x5345ac561 testSomeUnrelatedUtil', 'baseProtocols 0x0',
  65. 'name 0x1064b8438 ChromeTestCase', 'unrelated line',
  66. 'name 0x1064b8438 invalidTestCase', 'name 0x1075e6887 testG',
  67. 'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase',
  68. 'name 0x1075e6887 testG', 'name 0x1075e6887 testH',
  69. 'name 0x1075e6887 DISABLED_testI', 'name 0x1075e6887 FLAKY_testJ',
  70. 'name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0'
  71. ]).encode('utf-8')
  72. RELEASE_APP_OTOOL_OUTPUT_CLASS_NOT_IN_PAIRS = '\n'.join([
  73. 'Meta Class', 'name 0x1064b8438 CacheTestCase',
  74. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  75. 'name 0x1075e6887 testA', 'types 0x1064cc3e1', 'name 0x1075e6887 testB',
  76. 'name 0x1075e6887 testc', 'baseProtocols 0x0',
  77. 'name 0x1064b8438 CacheTestCase', 'unrelated line', 'Meta Class',
  78. 'name 0x1064b8438 TabUITestCase', 'no test methods in this case',
  79. 'name 0x1064b8438 TabUITestCase', 'unrelated line',
  80. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  81. 'name 0x1064b8438 KeyboardTest', 'name 0x1075e6887 testD',
  82. 'types 0x1064cc3e1 v16@0:8', 'name 0x1075e6887 testE',
  83. 'name 0x1075e6887 testF', 'baseProtocols 0x0',
  84. 'name 0x1075e6887 testUnrelatedG', 'unrelated line',
  85. 'name 0x1064b8438 ChromeTestCase', 'name 0x1064b8438 setUp',
  86. 'name 0x1064b8438 testPort', 'name 0x5345ac561 testSomeUnrelatedUtil',
  87. 'baseProtocols 0x0', 'name 0x1064b8438 ChromeTestCase', 'unrelated line',
  88. 'name 0x1064b8438 invalidTestCase', 'name 0x1075e6887 testG',
  89. 'baseProtocols 0x0', 'name 0x1064b8438 ToolBarTestCase',
  90. 'name 0x1075e6887 testG', 'name 0x1075e6887 testH',
  91. 'name 0x1075e6887 DISABLED_testI', 'name 0x1075e6887 FLAKY_testJ',
  92. 'name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0'
  93. ]).encode('utf-8')
  94. # Release app otool output format in Xcode 11.4 toolchain.
  95. RELEASE_APP_OTOOL_OUTPUT_114 = '\n'.join([
  96. 'Meta Class', ' name 0x1064b8438 CacheTestCase',
  97. 'baseMethods 0x1068586d8 (struct method_list_t *)',
  98. ' name 0x1075e6887 testA', ' types 0x1064cc3e1',
  99. ' name 0x1075e6887 testB', ' name 0x1075e6887 testc',
  100. 'baseProtocols 0x0', ' name 0x1064b8438 CacheTestCase',
  101. 'unrelated line', 'Meta Class', ' name 0x1064b8438 TabUITestCase',
  102. 'no test methods in this case', ' name 0x1064b8438 TabUITestCase',
  103. 'unrelated line', 'baseMethods 0x1068586d8 (struct method_list_t *)',
  104. ' name 0x1064b8438 KeyboardTest', ' name 0x1075e6887 testD',
  105. ' types 0x1064cc3e1 v16@0:8', ' name 0x1075e6887 testE',
  106. ' name 0x1075e6887 testF', 'baseProtocols 0x0',
  107. ' name 0x1064b8438 KeyboardTest',
  108. ' name 0x1075e6887 testUnrelatedG', 'unrelated line',
  109. ' name 0x1064b8438 ChromeTestCase', ' name 0x1064b8438 setUp',
  110. ' name 0x1064b8438 testPort',
  111. ' name 0x5345ac561 testSomeUnrelatedUtil', 'baseProtocols 0x0',
  112. ' name 0x1064b8438 ChromeTestCase', 'unrelated line',
  113. ' name 0x1064b8438 invalidTestCase', ' name 0x1075e6887 testG',
  114. 'baseProtocols 0x0', ' name 0x1064b8438 ToolBarTestCase',
  115. ' name 0x1075e6887 testG', ' name 0x1075e6887 testH',
  116. ' name 0x1075e6887 DISABLED_testI',
  117. ' name 0x1075e6887 FLAKY_testJ',
  118. ' name 0x1064b8438 ToolBarTestCase', 'baseProtocols 0x0', 'version 0'
  119. ]).encode('utf-8')
  120. class TestShardUtil(unittest.TestCase):
  121. """Test cases for shard_util.py"""
  122. @patch('shard_util.os.path.abspath')
  123. def test_determine_path_non_eg2(self, mock_abspath):
  124. mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py'
  125. app = 'some_ios_test.app'
  126. actual_path = shard_util.determine_app_path(app)
  127. expected_path = os.path.join('/b/s/w/ir', 'out/Debug', app, 'some_ios_test')
  128. self.assertEqual(actual_path, expected_path)
  129. @patch('shard_util.os.path.abspath')
  130. def test_determine_path_eg2(self, mock_abspath):
  131. mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py'
  132. app = 'some_ios_test-Runner.app'
  133. host = 'some_host.app'
  134. actual_path = shard_util.determine_app_path(app, host)
  135. expected_path = os.path.join('/b/s/w/ir', 'out/Debug', app, 'PlugIns',
  136. 'some_ios_test.xctest', 'some_ios_test')
  137. self.assertEqual(actual_path, expected_path)
  138. @patch('shard_util.os.path.abspath')
  139. def test_determine_path_eg2_release(self, mock_abspath):
  140. mock_abspath.return_value = '/b/s/w/ir/ios/build/bots/scripts/share_util.py'
  141. app = 'some_ios_test-Runner.app'
  142. host = 'some_host.app'
  143. actual_path = shard_util.determine_app_path(app, host, True)
  144. expected_path = os.path.join('/b/s/w/ir', 'out/Release', app, 'PlugIns',
  145. 'some_ios_test.xctest', 'some_ios_test')
  146. self.assertEqual(actual_path, expected_path)
  147. def test_fetch_test_names_debug(self):
  148. """Ensures that the debug output is formatted correctly"""
  149. resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT)
  150. self.assertEqual(len(resp), 10)
  151. expected_test_names = [
  152. ('CacheTestCase', 'testA'),
  153. ('CacheTestCase', 'testB'),
  154. ('CacheTestCase', 'testc'),
  155. ('TabUITestCase', 'testD'),
  156. ('TabUITestCase', 'testE'),
  157. ('KeyboardTestCase', 'testF'),
  158. ('PasswordsTestCase', 'testG'),
  159. ('ToolBarTestCase', 'testH'),
  160. ('ToolBarTestCase', 'DISABLED_testI'),
  161. ('ToolBarTestCase', 'FLAKY_testJ'),
  162. ]
  163. for test_name in expected_test_names:
  164. self.assertTrue(test_name in resp)
  165. test_cases = [test_case for (test_case, _) in resp]
  166. # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1,
  167. # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3})
  168. counts = collections.Counter(test_cases).most_common()
  169. name, _ = counts[0]
  170. # CacheTestCase and ToolBarTestCase each have 3 entries.
  171. # In case of ties, most_common() returns the first encountered at index 0.
  172. self.assertEqual(name, 'CacheTestCase')
  173. def test_fetch_test_counts_release(self):
  174. """Ensures that the release output is formatted correctly"""
  175. resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT)
  176. self.assertEqual(len(resp), 10)
  177. expected_test_names = [
  178. ('CacheTestCase', 'testA'),
  179. ('CacheTestCase', 'testB'),
  180. ('CacheTestCase', 'testc'),
  181. ('KeyboardTest', 'testD'),
  182. ('KeyboardTest', 'testE'),
  183. ('KeyboardTest', 'testF'),
  184. ('ToolBarTestCase', 'testG'),
  185. ('ToolBarTestCase', 'testH'),
  186. ('ToolBarTestCase', 'DISABLED_testI'),
  187. ('ToolBarTestCase', 'FLAKY_testJ'),
  188. ]
  189. for test_name in expected_test_names:
  190. self.assertTrue(test_name in resp)
  191. test_cases = [test_case for (test_case, _) in resp]
  192. # ({'KeyboardTest': 3, 'CacheTestCase': 3,
  193. # 'ToolBarTestCase': 4})
  194. counts = collections.Counter(test_cases).most_common()
  195. name, _ = counts[0]
  196. self.assertEqual(name, 'ToolBarTestCase')
  197. def test_fetch_test_error_release(self):
  198. """Ensures that unexpected release output raises error."""
  199. with self.assertRaises(shard_util.ShardingError) as context:
  200. shard_util.fetch_test_names_for_release(
  201. RELEASE_APP_OTOOL_OUTPUT_CLASS_NOT_IN_PAIRS)
  202. expected_message = (
  203. 'Incorrect otool output in which a test class name doesn\'t appear in '
  204. 'group of 2. Test class: KeyboardTest')
  205. self.assertTrue(expected_message in str(context.exception))
  206. def test_fetch_test_names_debug_114(self):
  207. """Test the debug output from otool in Xcode 11.4"""
  208. resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT_114)
  209. self.assertEqual(len(resp), 10)
  210. expected_test_names = [
  211. ('CacheTestCase', 'testA'),
  212. ('CacheTestCase', 'testB'),
  213. ('CacheTestCase', 'testc'),
  214. ('TabUITestCase', 'testD'),
  215. ('TabUITestCase', 'testE'),
  216. ('KeyboardTestCase', 'testF'),
  217. ('PasswordsTestCase', 'testG'),
  218. ('ToolBarTestCase', 'testH'),
  219. ('ToolBarTestCase', 'DISABLED_testI'),
  220. ('ToolBarTestCase', 'FLAKY_testJ'),
  221. ]
  222. for test_name in expected_test_names:
  223. self.assertTrue(test_name in resp)
  224. test_cases = [test_case for (test_case, _) in resp]
  225. # ({'CacheTestCase': 3, 'TabUITestCase': 2, 'PasswordsTestCase': 1,
  226. # 'KeyboardTestCase': 1, 'ToolBarTestCase': 3})
  227. counts = collections.Counter(test_cases).most_common()
  228. name, _ = counts[0]
  229. # CacheTestCase and ToolBarTestCase each have 3 entries.
  230. # In case of ties, most_common() returns the first encountered at index 0.
  231. self.assertEqual(name, 'CacheTestCase')
  232. def test_fetch_test_counts_release_114(self):
  233. """Test the release output from otool in Xcode 11.4"""
  234. resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT_114)
  235. self.assertEqual(len(resp), 10)
  236. expected_test_names = [
  237. ('CacheTestCase', 'testA'),
  238. ('CacheTestCase', 'testB'),
  239. ('CacheTestCase', 'testc'),
  240. ('KeyboardTest', 'testD'),
  241. ('KeyboardTest', 'testE'),
  242. ('KeyboardTest', 'testF'),
  243. ('ToolBarTestCase', 'testG'),
  244. ('ToolBarTestCase', 'testH'),
  245. ('ToolBarTestCase', 'DISABLED_testI'),
  246. ('ToolBarTestCase', 'FLAKY_testJ'),
  247. ]
  248. for test_name in expected_test_names:
  249. self.assertTrue(test_name in resp)
  250. test_cases = [test_case for (test_case, _) in resp]
  251. # ({'KeyboardTest': 3, 'CacheTestCase': 3,
  252. # 'ToolBarTestCase': 4})
  253. counts = collections.Counter(test_cases).most_common()
  254. name, _ = counts[0]
  255. self.assertEqual(name, 'ToolBarTestCase')
  256. def test_balance_into_sublists_debug(self):
  257. """Ensure the balancing algorithm works"""
  258. resp = shard_util.fetch_test_names_for_debug(DEBUG_APP_OTOOL_OUTPUT)
  259. test_cases = [test_case for (test_case, _) in resp]
  260. test_counts = collections.Counter(test_cases)
  261. sublists_1 = shard_util.balance_into_sublists(test_counts, 1)
  262. self.assertEqual(len(sublists_1), 1)
  263. self.assertEqual(len(sublists_1[0]), 5)
  264. sublists_3 = shard_util.balance_into_sublists(test_counts, 3)
  265. self.assertEqual(len(sublists_3), 3)
  266. # CacheTestCase has 3,
  267. # TabUITestCase has 2, ToolBarTestCase has 4
  268. # PasswordsTestCase has 1, KeyboardTestCase has 1
  269. # They will be balanced into:
  270. # [[ToolBarTestCase], [CacheTestCase, PasswordsTestCase],
  271. # [TabUITestCase, KeyboardTestCase]]
  272. self.assertEqual(
  273. sorted([len(sublists_3[0]),
  274. len(sublists_3[1]),
  275. len(sublists_3[2])]), [1, 2, 2])
  276. def test_balance_into_sublists_release(self):
  277. """Ensure the balancing algorithm works"""
  278. resp = shard_util.fetch_test_names_for_release(RELEASE_APP_OTOOL_OUTPUT)
  279. test_cases = [test_case for (test_case, _) in resp]
  280. test_counts = collections.Counter(test_cases)
  281. sublists_3 = shard_util.balance_into_sublists(test_counts, 3)
  282. self.assertEqual(len(sublists_3), 3)
  283. # KeyboardTest has 3
  284. # CacheTestCase has 3
  285. # ToolbarTest Case has 4
  286. # They will be balanced as one in each shard.
  287. self.assertEqual(len(sublists_3[0]), 1)
  288. self.assertEqual(len(sublists_3[1]), 1)
  289. self.assertEqual(len(sublists_3[2]), 1)
  290. if __name__ == '__main__':
  291. unittest.main()