cts_utils.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491
  1. # Copyright 2020 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Stage the Chromium checkout to update CTS test version."""
  5. import contextlib
  6. import json
  7. import operator
  8. import os
  9. import re
  10. import sys
  11. import tempfile
  12. import threading
  13. try:
  14. # Workaround for py2/3 compatibility.
  15. # TODO(pbirk): remove once py2 support is no longer needed.
  16. import urllib.request as urllib_request
  17. except ImportError:
  18. import urllib as urllib_request
  19. import zipfile
  20. sys.path.append(
  21. os.path.join(
  22. os.path.dirname(__file__), os.pardir, os.pardir, 'third_party',
  23. 'catapult', 'devil'))
  24. # pylint: disable=wrong-import-position,import-error
  25. from devil.utils import cmd_helper
  26. sys.path.append(
  27. os.path.join(
  28. os.path.dirname(__file__), os.pardir, os.pardir, 'third_party',
  29. 'catapult', 'common', 'py_utils'))
  30. # pylint: disable=wrong-import-position,import-error
  31. from py_utils import tempfile_ext
  32. SRC_DIR = os.path.abspath(
  33. os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
  34. TOOLS_DIR = os.path.join('android_webview', 'tools')
  35. CONFIG_FILE = os.path.join('cts_config', 'webview_cts_gcs_path.json')
  36. CONFIG_PATH = os.path.join(SRC_DIR, TOOLS_DIR, CONFIG_FILE)
  37. CIPD_FILE = os.path.join('cts_archive', 'cipd.yaml')
  38. CIPD_PATH = os.path.join(SRC_DIR, TOOLS_DIR, CIPD_FILE)
  39. DEPS_FILE = 'DEPS'
  40. TEST_SUITES_FILE = os.path.join('testing', 'buildbot', 'test_suites.pyl')
  41. CTS_DEP_NAME = 'src/android_webview/tools/cts_archive'
  42. CTS_DEP_PACKAGE = 'chromium/android_webview/tools/cts_archive'
  43. CIPD_REFERRERS = [DEPS_FILE, TEST_SUITES_FILE]
  44. _GENERATE_BUILDBOT_JSON = os.path.join('testing', 'buildbot',
  45. 'generate_buildbot_json.py')
  46. _ENSURE_FORMAT = """$ParanoidMode CheckIntegrity
  47. @Subdir cipd
  48. {} {}"""
  49. _ENSURE_SUBDIR = 'cipd'
  50. _RE_COMMENT_OR_BLANK = re.compile(r'^ *(#.*)?$')
  51. class CTSConfig:
  52. """Represents a CTS config file."""
  53. def __init__(self, file_path=CONFIG_PATH):
  54. """Constructs a representation of the CTS config file.
  55. Only read operations are provided by this object. Users should edit the
  56. file manually for any modifications.
  57. Args:
  58. file_path: Path to file.
  59. """
  60. self._path = os.path.abspath(file_path)
  61. with open(self._path) as f:
  62. self._config = json.load(f)
  63. def save(self):
  64. with open(self._path, 'w') as file:
  65. json.dump(self._config, file, indent=2)
  66. file.write("\n")
  67. def get_platforms(self):
  68. return sorted(self._config.keys())
  69. def get_archs(self, platform):
  70. return sorted(self._config[platform]['arch'].keys())
  71. def get_git_tag_prefix(self, platform):
  72. return self._config[platform]['git']['tag_prefix']
  73. def iter_platforms(self):
  74. for p in self.get_platforms():
  75. yield p
  76. def iter_platform_archs(self):
  77. for p in self.get_platforms():
  78. for a in self.get_archs(p):
  79. yield p, a
  80. def get_cipd_zip(self, platform, arch):
  81. return self._config[platform]['arch'][arch]['filename']
  82. def get_origin(self, platform, arch):
  83. return self._config[platform]['arch'][arch]['_origin']
  84. def get_origin_zip(self, platform, arch):
  85. return os.path.basename(self.get_origin(platform, arch))
  86. def get_apks(self, platform):
  87. return sorted([r['apk'] for r in self._config[platform]['test_runs']])
  88. def get_additional_apks(self, platform):
  89. return [
  90. apk['apk'] for r in self._config[platform]['test_runs']
  91. for apk in r.get('additional_apks', [])
  92. ]
  93. def set_release_version(self, platform, arch, release):
  94. pattern = re.compile(r'(?<=_r)\d*')
  95. def update_release_version(field):
  96. return pattern.sub(str(release),
  97. self._config[platform]['arch'][arch][field])
  98. self._config[platform]['arch'][arch] = {
  99. 'filename': update_release_version('filename'),
  100. '_origin': update_release_version('_origin'),
  101. 'unzip_dir': update_release_version('unzip_dir'),
  102. }
  103. class CTSCIPDYaml:
  104. """Represents a CTS CIPD yaml file."""
  105. RE_PACKAGE = r'^package:\s*(\S+)\s*$'
  106. RE_DESC = r'^description:\s*(.+)$'
  107. RE_DATA = r'^data:\s*$'
  108. RE_FILE = r'^\s+-\s+file:\s*(.+)$'
  109. # TODO(crbug.com/1049432): Replace with yaml parser
  110. @classmethod
  111. def parse(cls, lines):
  112. result = {}
  113. for line in lines:
  114. if len(line) == 0 or line[0] == '#':
  115. continue
  116. package_match = re.match(cls.RE_PACKAGE, line)
  117. if package_match:
  118. result['package'] = package_match.group(1)
  119. continue
  120. desc_match = re.match(cls.RE_DESC, line)
  121. if desc_match:
  122. result['description'] = desc_match.group(1)
  123. continue
  124. if re.match(cls.RE_DATA, line):
  125. result['data'] = []
  126. if 'data' in result:
  127. file_match = re.match(cls.RE_FILE, line)
  128. if file_match:
  129. result['data'].append({'file': file_match.group(1)})
  130. return result
  131. def __init__(self, file_path=CIPD_PATH):
  132. """Constructs a representation of CTS CIPD yaml file.
  133. Note the file won't be modified unless write is called
  134. with its path.
  135. Args:
  136. file_path: Path to file.
  137. """
  138. self._path = os.path.abspath(file_path)
  139. self._header = []
  140. # Read header comments
  141. with open(self._path) as f:
  142. for l in f.readlines():
  143. if re.match(_RE_COMMENT_OR_BLANK, l):
  144. self._header.append(l)
  145. else:
  146. break
  147. # Read yaml data
  148. with open(self._path) as f:
  149. self._yaml = CTSCIPDYaml.parse(f.readlines())
  150. def get_file_path(self):
  151. """Get full file path of yaml file that this was constructed from."""
  152. return self._path
  153. def get_file_basename(self):
  154. """Get base file name that this was constructed from."""
  155. return os.path.basename(self._path)
  156. def get_package(self):
  157. """Get package name."""
  158. return self._yaml['package']
  159. def clear_files(self):
  160. """Clears all files in file (only in local memory, does not modify file)."""
  161. self._yaml['data'] = []
  162. def append_file(self, file_name):
  163. """Add file_name to list of files."""
  164. self._yaml['data'].append({'file': str(file_name)})
  165. def remove_file(self, file_name):
  166. """Remove file_name from list of files."""
  167. old_file_names = self.get_files()
  168. new_file_names = [name for name in old_file_names if name != file_name]
  169. self._yaml['data'] = [{'file': name} for name in new_file_names]
  170. def get_files(self):
  171. """Get list of files in yaml file."""
  172. return [e['file'] for e in self._yaml['data']]
  173. def write(self, file_path):
  174. """(Over)write file_path with the cipd.yaml representation."""
  175. dir_name = os.path.dirname(file_path)
  176. if not os.path.isdir(dir_name):
  177. os.makedirs(dir_name)
  178. with open(file_path, 'w') as f:
  179. f.writelines(self._get_yamls())
  180. def _get_yamls(self):
  181. """Return the cipd.yaml file contents of this object."""
  182. output = []
  183. output += self._header
  184. output.append('package: {}\n'.format(self._yaml['package']))
  185. output.append('description: {}\n'.format(self._yaml['description']))
  186. output.append('data:\n')
  187. for d in sorted(self._yaml['data'], key=operator.itemgetter('file')):
  188. output.append(' - file: {}\n'.format(d.get('file')))
  189. return output
  190. def cipd_ensure(package, version, root_dir):
  191. """Ensures CIPD package is installed at root_dir.
  192. Args:
  193. package: CIPD name of package
  194. version: Package version
  195. root_dir: Directory to install package into
  196. """
  197. def _createEnsureFile(package, version, file_path):
  198. with open(file_path, 'w') as f:
  199. f.write(_ENSURE_FORMAT.format(package, version))
  200. def _ensure(root, ensure_file):
  201. ret = cmd_helper.RunCmd(
  202. ['cipd', 'ensure', '-root', root, '-ensure-file', ensure_file])
  203. if ret:
  204. raise IOError('Error while running cipd ensure: ' + ret)
  205. with tempfile.NamedTemporaryFile() as f:
  206. _createEnsureFile(package, version, f.name)
  207. _ensure(root_dir, f.name)
  208. def cipd_download(cipd, version, download_dir):
  209. """Downloads CIPD package files.
  210. This is different from cipd ensure in that actual files will exist at
  211. download_dir instead of symlinks.
  212. Args:
  213. cipd: CTSCIPDYaml object
  214. version: Version of package
  215. download_dir: Destination directory
  216. """
  217. package = cipd.get_package()
  218. download_dir_abs = os.path.abspath(download_dir)
  219. if not os.path.isdir(download_dir_abs):
  220. os.makedirs(download_dir_abs)
  221. with tempfile_ext.NamedTemporaryDirectory() as workDir, chdir(workDir):
  222. cipd_ensure(package, version, '.')
  223. for file_name in cipd.get_files():
  224. src_path = os.path.join(_ENSURE_SUBDIR, file_name)
  225. dest_path = os.path.join(download_dir_abs, file_name)
  226. dest_dir = os.path.dirname(dest_path)
  227. if not os.path.isdir(dest_dir):
  228. os.makedirs(dest_dir)
  229. ret = cmd_helper.RunCmd(['cp', '--reflink=never', src_path, dest_path])
  230. if ret:
  231. raise IOError('Error file copy from ' + file_name + ' to ' + dest_path)
  232. def filter_cts_file(cts_config, cts_zip_file, dest_dir):
  233. """Filters out non-webview test apks from downloaded CTS zip file.
  234. Args:
  235. cts_config: CTSConfig object
  236. cts_zip_file: Path to downloaded CTS zip, retaining the original filename
  237. dest_dir: Destination directory to filter to, filename will be unchanged
  238. """
  239. for p in cts_config.get_platforms():
  240. for a in cts_config.get_archs(p):
  241. o = cts_config.get_origin(p, a)
  242. base_name = os.path.basename(o)
  243. if base_name == os.path.basename(cts_zip_file):
  244. filterzip(cts_zip_file,
  245. cts_config.get_apks(p) + cts_config.get_additional_apks(p),
  246. os.path.join(dest_dir, base_name))
  247. return
  248. raise ValueError('Could not find platform and arch for: ' + cts_zip_file)
  249. class ChromiumRepoHelper:
  250. """Performs operations on Chromium checkout."""
  251. def __init__(self, root_dir=SRC_DIR):
  252. self._root_dir = os.path.abspath(root_dir)
  253. self._cipd_referrers = [
  254. os.path.join(self._root_dir, p) for p in CIPD_REFERRERS
  255. ]
  256. @property
  257. def cipd_referrers(self):
  258. return self._cipd_referrers
  259. @property
  260. def cts_cipd_package(self):
  261. return CTS_DEP_PACKAGE
  262. def get_cipd_dependency_rev(self):
  263. """Return CTS CIPD revision in the checkout's DEPS file."""
  264. deps_file = os.path.join(self._root_dir, DEPS_FILE)
  265. # Use the gclient command instead of gclient_eval since the latter is not
  266. # intended for direct use outside of depot_tools. The .bat file extension
  267. # must be explicitly specified when shell=False.
  268. gclient = 'gclient.bat' if os.name == 'nt' else 'gclient'
  269. cmd = [
  270. gclient, 'getdep', '--revision',
  271. '%s:%s' % (CTS_DEP_NAME, CTS_DEP_PACKAGE), '--deps-file', deps_file
  272. ]
  273. env = os.environ
  274. # Disable auto-update of depot tools since update_depot_tools may not be
  275. # available (for example, on the presubmit bot), and it's probably best not
  276. # to perform surprise updates anyways.
  277. env.update({'DEPOT_TOOLS_UPDATE': '0'})
  278. status, output, err = cmd_helper.GetCmdStatusOutputAndError(cmd, env=env)
  279. if status != 0:
  280. raise Exception('Command "%s" failed: %s' % (' '.join(cmd), err))
  281. return output.strip()
  282. def update_cts_cipd_rev(self, new_version):
  283. """Update references to CTS CIPD revision in checkout.
  284. Args:
  285. new_version: New version to use
  286. """
  287. old_version = self.get_cipd_dependency_rev()
  288. for path in self.cipd_referrers:
  289. replace_cipd_revision(path, old_version, new_version)
  290. def git_status(self, path):
  291. """Returns canonical git status of file.
  292. Args:
  293. path: Path to file.
  294. Returns:
  295. Output of git status --porcelain.
  296. """
  297. with chdir(self._root_dir):
  298. output = cmd_helper.GetCmdOutput(['git', 'status', '--porcelain', path])
  299. return output
  300. def update_testing_json(self):
  301. """Performs generate_buildbot_json.py.
  302. Raises:
  303. IOError: If generation failed.
  304. """
  305. with chdir(self._root_dir):
  306. ret = cmd_helper.RunCmd(['python', _GENERATE_BUILDBOT_JSON])
  307. if ret:
  308. raise IOError('Error while generating_buildbot_json.py')
  309. def rebase(self, *rel_path_parts):
  310. """Construct absolute path from parts relative to root_dir.
  311. Args:
  312. rel_path_parts: Parts of the root relative path.
  313. Returns:
  314. The absolute path.
  315. """
  316. return os.path.join(self._root_dir, *rel_path_parts)
  317. def replace_cipd_revision(file_path, old_revision, new_revision):
  318. """Replaces cipd revision strings in file.
  319. Args:
  320. file_path: Path to file.
  321. old_revision: Old cipd revision to be replaced.
  322. new_revision: New cipd revision to use as replacement.
  323. Returns:
  324. Number of replaced occurrences.
  325. Raises:
  326. IOError: If no occurrences were found.
  327. """
  328. with open(file_path) as f:
  329. contents = f.read()
  330. num = contents.count(old_revision)
  331. if not num:
  332. raise IOError('Did not find old CIPD revision {} in {}'.format(
  333. old_revision, file_path))
  334. newcontents = contents.replace(old_revision, new_revision)
  335. with open(file_path, 'w') as f:
  336. f.write(newcontents)
  337. return num
  338. @contextlib.contextmanager
  339. def chdir(dirPath):
  340. """Context manager that changes working directory."""
  341. cwd = os.getcwd()
  342. os.chdir(dirPath)
  343. try:
  344. yield
  345. finally:
  346. os.chdir(cwd)
  347. def filterzip(inputPath, pathList, outputPath):
  348. """Copy a subset of files from input archive into output archive.
  349. Args:
  350. inputPath: Input archive path
  351. pathList: List of file names from input archive to copy
  352. outputPath: Output archive path
  353. """
  354. with zipfile.ZipFile(os.path.abspath(inputPath), 'r') as inputZip,\
  355. zipfile.ZipFile(os.path.abspath(outputPath), 'w') as outputZip,\
  356. tempfile_ext.NamedTemporaryDirectory() as workDir,\
  357. chdir(workDir):
  358. for p in pathList:
  359. inputZip.extract(p)
  360. outputZip.write(p)
  361. def download(url, destination):
  362. """Asynchronously download url to path specified by destination.
  363. Args:
  364. url: Url location of file.
  365. destination: Path where file should be saved to.
  366. If destination parent directories do not exist, they will be created.
  367. Returns the download thread which can then be joined by the caller to
  368. wait for download completion.
  369. """
  370. dest_dir = os.path.dirname(destination)
  371. if not os.path.isdir(dest_dir):
  372. os.makedirs(dest_dir)
  373. t = threading.Thread(target=urllib_request.urlretrieve,
  374. args=(url, destination))
  375. t.start()
  376. return t
  377. def update_cipd_package(cipd_yaml_path):
  378. """Updates the CIPD package specified by cipd_yaml_path.
  379. Args:
  380. cipd_yaml_path: Path of cipd yaml specification file
  381. """
  382. cipd_yaml_path_abs = os.path.abspath(cipd_yaml_path)
  383. with chdir(os.path.dirname(cipd_yaml_path_abs)),\
  384. tempfile.NamedTemporaryFile() as jsonOut:
  385. ret = cmd_helper.RunCmd([
  386. 'cipd', 'create', '-pkg-def', cipd_yaml_path_abs, '-json-output',
  387. jsonOut.name
  388. ])
  389. if ret:
  390. raise IOError('Error during cipd create.')
  391. return json.load(jsonOut)['result']['instance_id']