update_cts.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
  1. #!/usr/bin/env vpython3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Update CTS Tests to a new version."""
  6. from __future__ import print_function
  7. import argparse
  8. import logging
  9. import os
  10. import re
  11. import shutil
  12. import sys
  13. import tempfile
  14. import zipfile
  15. sys.path.append(
  16. os.path.join(
  17. os.path.dirname(__file__), os.pardir, os.pardir, 'third_party',
  18. 'catapult', 'devil'))
  19. # pylint: disable=wrong-import-position,import-error
  20. from devil.utils import cmd_helper
  21. from devil.utils import logging_common
  22. import cts_utils
  23. def _query_git_for_cts_tags():
  24. cts_git_url = 'https://android.googlesource.com/platform/cts/'
  25. tags = cmd_helper.GetCmdOutput(['git', 'ls-remote', '--tags',
  26. cts_git_url]).splitlines()
  27. print('[Updating CTS versions] Retrieved the CTS git tags')
  28. return tags
  29. class PathError(IOError):
  30. def __init__(self, path, err_desc):
  31. # pylint: disable=super-with-arguments
  32. # Since this is currently executed with python2, we cannot go with python3.
  33. super(PathError, self).__init__('"%s": %s' % (path, err_desc))
  34. class MissingDirError(PathError):
  35. """An expected directory is missing, usually indicates a step was missed
  36. during the CTS update process. Try to perform the missing step.
  37. """
  38. def __init__(self, path):
  39. # pylint: disable=super-with-arguments
  40. # Since this is currently executed with python2, we cannot go with python3.
  41. super(MissingDirError, self).__init__(path, 'directory is missing.')
  42. class DirExistsError(PathError):
  43. """A directory is already present, usually indicates a step was repeated
  44. in the same working directory. Try to delete the reported directory.
  45. """
  46. def __init__(self, path):
  47. # pylint: disable=super-with-arguments
  48. # Since this is currently executed with python2, we cannot go with python3.
  49. super(DirExistsError, self).__init__(path, 'directory already exists.')
  50. class MissingFileError(PathError):
  51. """Files are missing during CIPD staging, ensure that all files were
  52. downloaded and that CIPD download worked properly.
  53. """
  54. def __init__(self, path):
  55. # pylint: disable=super-with-arguments
  56. # Since this is currently executed with python2, we cannot go with python3.
  57. super(MissingFileError, self).__init__(path, 'file is missing.')
  58. class InconsistentFilesException(Exception):
  59. """Test files in CTS config and cipd yaml have gotten out of sync."""
  60. class UncommittedChangeException(Exception):
  61. """Files are about to be modified but previously uncommitted changes exist."""
  62. def __init__(self, path):
  63. # pylint: disable=super-with-arguments
  64. # Since this is currently executed with python2, we cannot go with python3.
  65. super(UncommittedChangeException, self).__init__(
  66. path, 'has uncommitted changes.')
  67. class UpdateCTS:
  68. """Updates CTS archive to a new version.
  69. Prereqs:
  70. - Update the tools/cts_config/webview_cts_gcs_path.json file with origin,
  71. and filenames for each platform. See:
  72. https://source.android.com/compatibility/cts/downloads for the latest
  73. versions.
  74. Performs the following tasks to simplify the CTS test update process:
  75. - Read the desired CTS versions from
  76. tools/cts_config/webview_cts_gcs_path.json file.
  77. - Download CTS test zip files from Android's public repository.
  78. - Filter only the WebView CTS test apks into smaller zip files.
  79. - Update the CTS CIPD package with the filtered zip files.
  80. - Update DEPS and testing/buildbot/test_suites.pyl with updated CTS CIPD
  81. package version.
  82. - Regenerate the buildbot json files.
  83. After these steps are completed, the user can commit and upload
  84. the CL to Chromium Gerrit.
  85. """
  86. def __init__(self, work_dir, repo_root):
  87. """Construct UpdateCTS instance.
  88. Args:
  89. work_dir: Directory used to download and stage cipd updates
  90. repo_root: Repository root (e.g. /path/to/chromium/src) to base
  91. all configuration files
  92. """
  93. self._work_dir = os.path.abspath(work_dir)
  94. self._download_dir = os.path.join(self._work_dir, 'downloaded')
  95. self._filter_dir = os.path.join(self._work_dir, 'filtered')
  96. self._cipd_dir = os.path.join(self._work_dir, 'cipd')
  97. self._stage_dir = os.path.join(self._work_dir, 'staged')
  98. self._version_file = os.path.join(self._work_dir, 'cipd_version.txt')
  99. self._repo_root = os.path.abspath(repo_root)
  100. helper = cts_utils.ChromiumRepoHelper(self._repo_root)
  101. self._repo_helper = helper
  102. self._cts_config_path = helper.rebase(cts_utils.TOOLS_DIR,
  103. cts_utils.CONFIG_FILE)
  104. self._CTSConfig = cts_utils.CTSConfig(self._cts_config_path)
  105. self._CIPDYaml = cts_utils.CTSCIPDYaml(
  106. helper.rebase(cts_utils.TOOLS_DIR, cts_utils.CIPD_FILE))
  107. @property
  108. def download_dir(self):
  109. """Full directory path where full test zips are to be downloaded to."""
  110. return self._download_dir
  111. def _check_for_latest_cts_versions(self, cts_tags):
  112. """Query for the latest cts versions per platform
  113. We can retrieve the newest CTS versions by searching through the git tags
  114. for each CTS version and looking for the latest
  115. """
  116. prefixes = [(platform, self._CTSConfig.get_git_tag_prefix(platform))
  117. for platform in self._CTSConfig.iter_platforms()]
  118. release_versions = dict()
  119. tag_prefix_regexes = {
  120. # Do a forward lookup for the tag prefix plus an '_r'
  121. # Eg: 'android-cts-7.0_r2'
  122. # Then retrieve the digits after this
  123. tag_prefix: re.compile('(?<=/%s_r)\\d*' % re.escape(tag_prefix))
  124. for _, tag_prefix in prefixes
  125. }
  126. for tag in cts_tags:
  127. for platform, prefix in prefixes:
  128. matches = tag_prefix_regexes[prefix].search(tag)
  129. if matches:
  130. version = int(matches.group(0))
  131. if release_versions.get(platform, -1) < version:
  132. release_versions[platform] = version
  133. print('[Updating CTS versions] Retrieved the latest CTS versions')
  134. return release_versions
  135. def _update_cts_config_file_download_origins(self, release_versions):
  136. """ Update the CTS release version for each architecture
  137. and then save the config json
  138. """
  139. for platform, arch in self._CTSConfig.iter_platform_archs():
  140. self._CTSConfig.set_release_version(platform, arch,
  141. release_versions[platform])
  142. self._CTSConfig.save()
  143. print('[Updating CTS versions] Updated cts config')
  144. def update_cts_download_origins_cmd(self):
  145. """Performs the cts download origins update command"""
  146. tags = _query_git_for_cts_tags()
  147. release_versions = self._check_for_latest_cts_versions(tags)
  148. self._update_cts_config_file_download_origins(release_versions)
  149. def download_cts_cmd(self, platforms=None):
  150. """Performs the download sub-command."""
  151. if platforms is None:
  152. platforms = self._CTSConfig.get_platforms()
  153. print('Downloading CTS tests for %d platforms, could take a few'
  154. ' minutes ...' % len(platforms))
  155. self.download_cts(platforms)
  156. def create_cipd_cmd(self):
  157. """Performs the create-cipd sub-command."""
  158. print('Updating WebView CTS package in CIPD.')
  159. self.filter_downloaded_cts()
  160. self.download_cipd()
  161. self.stage_cipd_update()
  162. self.commit_staged_cipd()
  163. def update_repository_cmd(self):
  164. """Performs the update-checkout sub-command."""
  165. print('Updating current checkout with changes.')
  166. self.update_repository()
  167. def download_cts(self, platforms=None):
  168. """Download full test zip files to <work_dir>/downloaded/.
  169. It is an error to call this if work_dir already contains downloaded/.
  170. Args:
  171. platforms: List of platforms (e.g. ['O', 'P']), defaults to all
  172. Raises:
  173. DirExistsError: If downloaded/ already exists in work_dir.
  174. """
  175. if platforms is None:
  176. platforms = self._CTSConfig.get_platforms()
  177. if os.path.exists(self._download_dir):
  178. raise DirExistsError(self._download_dir)
  179. threads = []
  180. for p, a in self._CTSConfig.iter_platform_archs():
  181. if p not in platforms:
  182. continue
  183. origin = self._CTSConfig.get_origin(p, a)
  184. destination = os.path.join(self._download_dir,
  185. self._CTSConfig.get_origin_zip(p, a))
  186. logging.info('Starting download from %s to %s.', origin, destination)
  187. threads.append((origin, cts_utils.download(origin, destination)))
  188. for t in threads:
  189. t[1].join()
  190. logging.info('Finished download from %s', t[0])
  191. def filter_downloaded_cts(self):
  192. """Filter files from downloaded/ to filtered/ to contain only WebView apks.
  193. It is an error to call this if downloaded/ doesn't exist or if filtered/
  194. already exists.
  195. Raises:
  196. DirExistsError: If filtered/ already exists in work_dir.
  197. MissingDirError: If downloaded/ does not exist in work_dir.
  198. """
  199. if os.path.exists(self._filter_dir):
  200. raise DirExistsError(self._filter_dir)
  201. if not os.path.isdir(self._download_dir):
  202. raise MissingDirError(self._download_dir)
  203. os.makedirs(self._filter_dir)
  204. with cts_utils.chdir(self._download_dir):
  205. downloads = os.listdir('.')
  206. for download in downloads:
  207. logging.info('Filtering %s to %s/', download, self._filter_dir)
  208. cts_utils.filter_cts_file(self._CTSConfig, download, self._filter_dir)
  209. def download_cipd(self):
  210. """Download cts archive of the version found in DEPS to cipd/ directory.
  211. It is an error to call this if cipd/ already exists under work_cir.
  212. Raises:
  213. DirExistsError: If cipd/ already exists in work_dir.
  214. """
  215. if os.path.exists(self._cipd_dir):
  216. raise DirExistsError(self._cipd_dir)
  217. version = self._repo_helper.get_cipd_dependency_rev()
  218. logging.info('Download current CIPD version %s to %s/', version,
  219. self._cipd_dir)
  220. cts_utils.cipd_download(self._CIPDYaml, version, self._cipd_dir)
  221. def stage_cipd_update(self):
  222. """Stage CIPD package for update by combining CIPD and filtered CTS files.
  223. It is an error to call this if filtered/ and cipd/ do not already exist
  224. under work_dir, or if staged already exists under work_dir.
  225. Raises:
  226. DirExistsError: If staged/ already exists in work_dir.
  227. MissingDirError: If filtered/ or cipd/ does not exist in work_dir.
  228. """
  229. if not os.path.isdir(self._filter_dir):
  230. raise MissingDirError(self._filter_dir)
  231. if not os.path.isdir(self._cipd_dir):
  232. raise MissingDirError(self._cipd_dir)
  233. if os.path.isdir(self._stage_dir):
  234. raise DirExistsError(self._stage_dir)
  235. os.makedirs(self._stage_dir)
  236. filtered = os.listdir(self._filter_dir)
  237. self._CIPDYaml.clear_files()
  238. for p, a in self._CTSConfig.iter_platform_archs():
  239. origin_base = self._CTSConfig.get_origin_zip(p, a)
  240. cipd_zip = self._CTSConfig.get_cipd_zip(p, a)
  241. dest_path = os.path.join(self._stage_dir, cipd_zip)
  242. if not os.path.isdir(os.path.dirname(dest_path)):
  243. os.makedirs(os.path.dirname(dest_path))
  244. self._CIPDYaml.append_file(cipd_zip)
  245. if origin_base in filtered:
  246. logging.info('Staging downloaded and filtered version of %s to %s.',
  247. origin_base, dest_path)
  248. cmd_helper.RunCmd(
  249. ['cp', os.path.join(self._filter_dir, origin_base), dest_path])
  250. else:
  251. logging.info('Staging reused %s to %s/',
  252. os.path.join(self._cipd_dir, cipd_zip), dest_path)
  253. cmd_helper.RunCmd(
  254. ['cp', os.path.join(self._cipd_dir, cipd_zip), dest_path])
  255. self._CIPDYaml.write(
  256. os.path.join(self._stage_dir, self._CIPDYaml.get_file_basename()))
  257. def commit_staged_cipd(self):
  258. """Upload the staged CIPD files to CIPD.
  259. Raises:
  260. MissingDirError: If staged/ does not exist in work_dir.
  261. InconsistentFilesException: If errors are detected in staged config files.
  262. MissingFileExcepition: If files are missing from CTS zip files.
  263. """
  264. if not os.path.isdir(self._stage_dir):
  265. raise MissingDirError(self._stage_dir)
  266. staged_yaml_path = os.path.join(self._stage_dir,
  267. self._CIPDYaml.get_file_basename())
  268. staged_yaml = cts_utils.CTSCIPDYaml(file_path=staged_yaml_path)
  269. staged_yaml_files = staged_yaml.get_files()
  270. if cts_utils.CTS_DEP_PACKAGE != staged_yaml.get_package():
  271. raise InconsistentFilesException('Bad CTS package name in staged yaml '
  272. '{}: {} '.format(
  273. staged_yaml_path,
  274. staged_yaml.get_package()))
  275. for p, a in self._CTSConfig.iter_platform_archs():
  276. cipd_zip = self._CTSConfig.get_cipd_zip(p, a)
  277. cipd_zip_path = os.path.join(self._stage_dir, cipd_zip)
  278. if not os.path.exists(cipd_zip_path):
  279. raise MissingFileError(cipd_zip_path)
  280. with zipfile.ZipFile(cipd_zip_path) as zf:
  281. cipd_zip_contents = zf.namelist()
  282. missing_apks = set(self._CTSConfig.get_apks(p)) - set(cipd_zip_contents)
  283. if missing_apks:
  284. raise MissingFileError('%s in %s' % (str(missing_apks), cipd_zip_path))
  285. if cipd_zip not in staged_yaml_files:
  286. raise InconsistentFilesException(cipd_zip +
  287. ' missing from staged cipd.yaml file')
  288. logging.info('Updating CIPD CTS version using %s', staged_yaml_path)
  289. new_cipd_version = cts_utils.update_cipd_package(staged_yaml_path)
  290. with open(self._version_file, 'w') as vf:
  291. logging.info('Saving new CIPD version %s to %s', new_cipd_version,
  292. vf.name)
  293. vf.write(new_cipd_version)
  294. def update_repository(self):
  295. """Update chromium checkout with changes for this update.
  296. After this is called, git add -u && git commit && git cl upload
  297. will still be needed to generate the CL.
  298. Raises:
  299. MissingFileError: If CIPD has not yet been staged or updated.
  300. UncommittedChangeException: If repo files have uncommitted changes.
  301. InconsistentFilesException: If errors are detected in staged config files.
  302. """
  303. if not os.path.exists(self._version_file):
  304. raise MissingFileError(self._version_file)
  305. staged_yaml_path = os.path.join(self._stage_dir,
  306. self._CIPDYaml.get_file_basename())
  307. if not os.path.exists(staged_yaml_path):
  308. raise MissingFileError(staged_yaml_path)
  309. with open(self._version_file) as vf:
  310. new_cipd_version = vf.read()
  311. logging.info('Read in new CIPD version %s from %s', new_cipd_version,
  312. vf.name)
  313. repo_cipd_yaml = self._CIPDYaml.get_file_path()
  314. for f in self._repo_helper.cipd_referrers + [repo_cipd_yaml]:
  315. git_status = self._repo_helper.git_status(f)
  316. if git_status:
  317. raise UncommittedChangeException(f)
  318. repo_cipd_package = self._repo_helper.cts_cipd_package
  319. staged_yaml = cts_utils.CTSCIPDYaml(file_path=staged_yaml_path)
  320. if repo_cipd_package != staged_yaml.get_package():
  321. raise InconsistentFilesException(
  322. 'Inconsistent CTS package name, {} in {}, but {} in {}'.format(
  323. repo_cipd_package, cts_utils.DEPS_FILE, staged_yaml.get_package(),
  324. staged_yaml.get_file_path()))
  325. logging.info('Updating files that reference %s under %s.',
  326. cts_utils.CTS_DEP_PACKAGE, self._repo_root)
  327. self._repo_helper.update_cts_cipd_rev(new_cipd_version)
  328. logging.info('Regenerate buildbot json files under %s.', self._repo_root)
  329. self._repo_helper.update_testing_json()
  330. logging.info('Copy staged %s to %s.', staged_yaml_path, repo_cipd_yaml)
  331. cmd_helper.RunCmd(['cp', staged_yaml_path, repo_cipd_yaml])
  332. logging.info('Ensure CIPD CTS package at %s to the new version %s',
  333. repo_cipd_yaml, new_cipd_version)
  334. cts_utils.cipd_ensure(self._CIPDYaml.get_package(), new_cipd_version,
  335. os.path.dirname(repo_cipd_yaml))
  336. DESC = """Updates the WebView CTS tests to a new version.
  337. See https://source.android.com/compatibility/cts/downloads for the latest
  338. versions.
  339. Please create a new branch, then edit the
  340. {}
  341. file with updated origin and file name before running this script.
  342. After performing all steps, perform git add then commit.""".format(
  343. os.path.join(cts_utils.TOOLS_DIR, cts_utils.CONFIG_FILE))
  344. ALL_CMD = 'all-steps'
  345. UPDATE_CONFIG = 'update-config'
  346. DOWNLOAD_CMD = 'download'
  347. CIPD_UPDATE_CMD = 'create-cipd'
  348. CHECKOUT_UPDATE_CMD = 'update-checkout'
  349. def add_dessert_arg(parser):
  350. """Add --dessert argument to a parser.
  351. Args:
  352. parser: The parser object to add to
  353. """
  354. parser.add_argument(
  355. '--dessert',
  356. '-d',
  357. action='append',
  358. help='Android dessert letter(s) for which to perform CTS update.')
  359. def add_workdir_arg(parser, is_required):
  360. """Add --work-dir argument to a parser.
  361. Args:
  362. parser: The parser object to add to
  363. is_required: Is this a required argument
  364. """
  365. parser.add_argument(
  366. '--workdir',
  367. '-w',
  368. required=is_required,
  369. help='Use this directory for'
  370. ' intermediate files.')
  371. def main():
  372. parser = argparse.ArgumentParser(
  373. description=DESC, formatter_class=argparse.RawTextHelpFormatter)
  374. logging_common.AddLoggingArguments(parser)
  375. subparsers = parser.add_subparsers(dest='cmd')
  376. all_subparser = subparsers.add_parser(
  377. ALL_CMD,
  378. help='Performs all other sub-commands, in the correct order. This is'
  379. ' usually what you want.')
  380. add_dessert_arg(all_subparser)
  381. add_workdir_arg(all_subparser, False)
  382. update_config_subparser = subparsers.add_parser(
  383. UPDATE_CONFIG,
  384. help='Update the CTS config to the newest release versions.')
  385. add_workdir_arg(update_config_subparser, False)
  386. download_subparser = subparsers.add_parser(
  387. DOWNLOAD_CMD,
  388. help='Only downloads files to workdir for later use by other'
  389. ' sub-commands.')
  390. add_dessert_arg(download_subparser)
  391. add_workdir_arg(download_subparser, True)
  392. cipd_subparser = subparsers.add_parser(
  393. CIPD_UPDATE_CMD,
  394. help='Create a new CIPD package version for CTS tests. This requires'
  395. ' that {} was completed in the same workdir.'.format(DOWNLOAD_CMD))
  396. add_workdir_arg(cipd_subparser, True)
  397. checkout_subparser = subparsers.add_parser(
  398. CHECKOUT_UPDATE_CMD,
  399. help='Updates files in the current git branch. This requires that {} was'
  400. ' completed in the same workdir.'.format(CIPD_UPDATE_CMD))
  401. add_workdir_arg(checkout_subparser, True)
  402. args = parser.parse_args()
  403. logging_common.InitializeLogging(args)
  404. temp_workdir = None
  405. if args.workdir is None:
  406. temp_workdir = tempfile.mkdtemp()
  407. workdir = temp_workdir
  408. else:
  409. workdir = args.workdir
  410. if not os.path.isdir(workdir):
  411. raise ValueError(
  412. '--workdir {} should already be a directory.'.format(workdir))
  413. if not os.access(workdir, os.W_OK | os.X_OK):
  414. raise ValueError('--workdir {} is not writable.'.format(workdir))
  415. try:
  416. cts_updater = UpdateCTS(work_dir=workdir, repo_root=cts_utils.SRC_DIR)
  417. if args.cmd == UPDATE_CONFIG:
  418. cts_updater.update_cts_download_origins_cmd()
  419. elif args.cmd == DOWNLOAD_CMD:
  420. cts_updater.download_cts_cmd(platforms=args.dessert)
  421. elif args.cmd == CIPD_UPDATE_CMD:
  422. cts_updater.create_cipd_cmd()
  423. elif args.cmd == CHECKOUT_UPDATE_CMD:
  424. cts_updater.update_repository_cmd()
  425. elif args.cmd == ALL_CMD:
  426. cts_updater.update_cts_download_origins_cmd()
  427. cts_updater.download_cts_cmd()
  428. cts_updater.create_cipd_cmd()
  429. cts_updater.update_repository_cmd()
  430. finally:
  431. if temp_workdir is not None:
  432. logging.info('Removing temporary workdir %s', temp_workdir)
  433. shutil.rmtree(temp_workdir)
  434. if __name__ == '__main__':
  435. main()