fetch_all_androidx.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. #!/usr/bin/env python3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """A script to generate build.gradle from template and run fetch_all.py
  6. More specifically, to generate build.gradle:
  7. - It downloads the BUILD_INFO file for the latest androidx snapshot from
  8. https://androidx.dev/snapshots/builds
  9. - It replaces {{androidx_repository_url}} with the URL for the latest snapshot
  10. - For each dependency, it looks up the version in the BUILD_INFO file and
  11. substitutes the version into {{androidx_dependency_version}}.
  12. """
  13. import argparse
  14. import contextlib
  15. import json
  16. import logging
  17. import os
  18. import re
  19. import shutil
  20. import stat
  21. import subprocess
  22. import tempfile
  23. import urllib
  24. from urllib import request
  25. _ANDROIDX_PATH = os.path.normpath(os.path.join(__file__, '..'))
  26. _FETCH_ALL_PATH = os.path.normpath(
  27. os.path.join(_ANDROIDX_PATH, '..', 'android_deps', 'fetch_all.py'))
  28. # URL to BUILD_INFO in latest androidx snapshot.
  29. _ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL = 'https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO'
  30. # Snapshot repository URL with {{version}} placeholder.
  31. _SNAPSHOT_REPOSITORY_URL = 'https://androidx.dev/snapshots/builds/{{version}}/artifacts/repository'
  32. # When androidx roller is breaking, and a fix is not immenent, use this to pin a
  33. # broken library to an old known-working version.
  34. # * The first element of each tuple is the path to the artifact of the latest
  35. # version of the library. It could change if the version is rev'ed in a new
  36. # snapshot.
  37. # * The second element is a URL to replace the file with. Find the URL for older
  38. # versions of libraries by looking in the BUILD_INFO for the older version
  39. # (e.g.: https://androidx.dev/snapshots/builds/8545498/artifacts/BUILD_INFO)
  40. _OVERRIDES = [
  41. # Example:
  42. #('androidx_core_core/core-1.9.0-SNAPSHOT.aar',
  43. # 'https://androidx.dev/snapshots/builds/8545498/artifacts/repository/'
  44. # 'androidx/core/core/1.8.0-SNAPSHOT/core-1.8.0-20220505.122105-1.aar'),
  45. # Context: https://crbug.com/1349920 and https://crbug.com/1349521
  46. ('androidx_appcompat_appcompat/appcompat-1.6.0-SNAPSHOT.aar',
  47. 'https://androidx.dev/snapshots/builds/8811104/artifacts/repository/'
  48. 'androidx/appcompat/appcompat/1.5.0-SNAPSHOT/'
  49. 'appcompat-1.5.0-20220708.124951-1.aar'),
  50. ('androidx_appcompat_appcompat_resources/'
  51. 'appcompat-resources-1.6.0-SNAPSHOT.aar',
  52. 'https://androidx.dev/snapshots/builds/8811104/artifacts/repository/'
  53. 'androidx/appcompat/appcompat-resources/1.5.0-SNAPSHOT/'
  54. 'appcompat-resources-1.5.0-20220708.124951-1.aar'),
  55. ]
  56. def _build_snapshot_repository_url(version):
  57. return _SNAPSHOT_REPOSITORY_URL.replace('{{version}}', version)
  58. def _delete_readonly_files(paths):
  59. for path in paths:
  60. if os.path.exists(path):
  61. os.chmod(path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IWUSR)
  62. os.remove(path)
  63. def _parse_dir_list(dir_list):
  64. """Computes 'library_group:library_name'->library_version mapping.
  65. Args:
  66. dir_list: List of androidx library directories.
  67. """
  68. dependency_version_map = dict()
  69. for dir_entry in dir_list:
  70. stripped_dir = dir_entry.strip()
  71. if not stripped_dir.startswith('repository/androidx/'):
  72. continue
  73. dir_components = stripped_dir.split('/')
  74. # Expected format:
  75. # "repository/androidx/library_group/library_name/library_version/pom_or_jar"
  76. if len(dir_components) < 6:
  77. continue
  78. dependency_package = 'androidx.' + '.'.join(dir_components[2:-3])
  79. dependency_module = '{}:{}'.format(dependency_package,
  80. dir_components[-3])
  81. if dependency_module not in dependency_version_map:
  82. dependency_version_map[dependency_module] = dir_components[-2]
  83. return dependency_version_map
  84. def _compute_replacement(dependency_version_map, androidx_repository_url,
  85. line):
  86. """Computes output line for build.gradle from build.gradle.template line.
  87. Replaces {{android_repository_url}}, {{androidx_dependency_version}} and
  88. {{version_overrides}}.
  89. Args:
  90. dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
  91. androidx_repository_url: URL of the maven repository.
  92. line: Input line from the build.gradle.template.
  93. """
  94. line = line.replace('{{androidx_repository_url}}', androidx_repository_url)
  95. if line.strip() == '{{version_overrides}}':
  96. lines = ['versionOverrideMap = [:]']
  97. for dependency, version in sorted(dependency_version_map.items()):
  98. lines.append(f"versionOverrideMap['{dependency}'] = '{version}'")
  99. return '\n'.join(lines)
  100. match = re.search(r'\'(\S+):{{androidx_dependency_version}}\'', line)
  101. if not match:
  102. return line
  103. dependency = match.group(1)
  104. version = dependency_version_map.get(dependency)
  105. if not version:
  106. raise Exception(f'Version for {dependency} not found.')
  107. return line.replace('{{androidx_dependency_version}}', version)
  108. @contextlib.contextmanager
  109. def _build_dir():
  110. dirname = tempfile.mkdtemp()
  111. try:
  112. yield dirname
  113. finally:
  114. shutil.rmtree(dirname)
  115. def _download_and_parse_build_info():
  116. """Downloads and parses BUILD_INFO file."""
  117. with _build_dir() as build_dir:
  118. androidx_build_info_response = request.urlopen(
  119. _ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL)
  120. androidx_build_info_url = androidx_build_info_response.geturl()
  121. logging.info('URL for the latest build info: %s',
  122. androidx_build_info_url)
  123. androidx_build_info_path = os.path.join(build_dir, 'BUILD_INFO')
  124. with open(androidx_build_info_path, 'w') as f:
  125. f.write(androidx_build_info_response.read().decode('utf-8'))
  126. # Strip '/repository' from pattern.
  127. resolved_snapshot_repository_url_pattern = (
  128. _build_snapshot_repository_url('([0-9]*)').rsplit('/', 1)[0])
  129. version = re.match(resolved_snapshot_repository_url_pattern,
  130. androidx_build_info_url).group(1)
  131. with open(androidx_build_info_path, 'r') as f:
  132. build_info_dict = json.loads(f.read())
  133. dir_list = build_info_dict['target']['dir_list']
  134. return dir_list, version
  135. def _create_local_dir_list(repo_path):
  136. repo_path = repo_path.rstrip('/')
  137. prefix_len = len(repo_path) + 1
  138. ret = []
  139. for dirpath, _, filenames in os.walk(repo_path):
  140. for name in filenames:
  141. ret.append(os.path.join('repository', dirpath[prefix_len:], name))
  142. return ret
  143. def _process_build_gradle(dependency_version_map, androidx_repository_url):
  144. """Generates build.gradle from template.
  145. Args:
  146. dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
  147. androidx_repository_url: URL of the maven repository.
  148. """
  149. build_gradle_template_path = os.path.join(_ANDROIDX_PATH,
  150. 'build.gradle.template')
  151. build_gradle_out_path = os.path.join(_ANDROIDX_PATH, 'build.gradle')
  152. # |build_gradle_out_path| is not deleted after script has finished running. The file is in
  153. # .gitignore and thus will be excluded from uploaded CLs.
  154. with open(build_gradle_template_path, 'r') as template_f, \
  155. open(build_gradle_out_path, 'w') as out:
  156. for template_line in template_f:
  157. replacement = _compute_replacement(dependency_version_map,
  158. androidx_repository_url,
  159. template_line)
  160. out.write(replacement)
  161. def _write_cipd_yaml(libs_dir, version, cipd_yaml_path):
  162. """Writes cipd.yaml file at the passed-in path."""
  163. lib_dirs = os.listdir(libs_dir)
  164. if not lib_dirs:
  165. raise Exception('No generated libraries in {}'.format(libs_dir))
  166. data_files = [
  167. 'BUILD.gn', 'VERSION.txt', 'additional_readme_paths.json',
  168. 'build.gradle'
  169. ]
  170. for lib_dir in lib_dirs:
  171. abs_lib_dir = os.path.join(libs_dir, lib_dir)
  172. androidx_rel_lib_dir = os.path.relpath(abs_lib_dir, _ANDROIDX_PATH)
  173. if not os.path.isdir(abs_lib_dir):
  174. continue
  175. lib_files = os.listdir(abs_lib_dir)
  176. if not 'cipd.yaml' in lib_files:
  177. continue
  178. for lib_file in lib_files:
  179. if lib_file == 'cipd.yaml' or lib_file == 'OWNERS':
  180. continue
  181. data_files.append(os.path.join(androidx_rel_lib_dir, lib_file))
  182. contents = [
  183. '# Copyright 2021 The Chromium Authors. All rights reserved.',
  184. '# Use of this source code is governed by a BSD-style license that can be',
  185. '# found in the LICENSE file.',
  186. '# version: ' + version,
  187. 'package: chromium/third_party/androidx',
  188. 'description: androidx',
  189. 'data:',
  190. ]
  191. contents.extend('- file: ' + f for f in data_files)
  192. with open(cipd_yaml_path, 'w') as out:
  193. out.write('\n'.join(contents))
  194. def main():
  195. parser = argparse.ArgumentParser(description=__doc__)
  196. parser.add_argument('-v',
  197. '--verbose',
  198. dest='verbose_count',
  199. default=0,
  200. action='count',
  201. help='Verbose level (multiple times for more)')
  202. parser.add_argument('--local-repo',
  203. help='Path to a locally androidx maven repo to use '
  204. 'instead of fetching the latest.')
  205. args = parser.parse_args()
  206. logging.basicConfig(
  207. level=logging.WARNING - 10 * args.verbose_count,
  208. format='%(levelname).1s %(relativeCreated)6d %(message)s')
  209. libs_dir = os.path.join(_ANDROIDX_PATH, 'libs')
  210. if os.path.exists(libs_dir):
  211. shutil.rmtree(libs_dir)
  212. # Files uploaded to cipd are read-only. Delete them because they will be
  213. # re-generated.
  214. _delete_readonly_files([
  215. os.path.join(_ANDROIDX_PATH, 'BUILD.gn'),
  216. os.path.join(_ANDROIDX_PATH, 'VERSION.txt'),
  217. os.path.join(_ANDROIDX_PATH, 'additional_readme_paths.json'),
  218. os.path.join(_ANDROIDX_PATH, 'build.gradle'),
  219. ])
  220. if args.local_repo:
  221. version = 'local'
  222. dir_list = _create_local_dir_list(args.local_repo)
  223. androidx_snapshot_repository_url = ('file://' +
  224. os.path.abspath(args.local_repo))
  225. else:
  226. dir_list, version = _download_and_parse_build_info()
  227. androidx_snapshot_repository_url = _build_snapshot_repository_url(
  228. version)
  229. dependency_version_map = _parse_dir_list(dir_list)
  230. _process_build_gradle(dependency_version_map,
  231. androidx_snapshot_repository_url)
  232. fetch_all_cmd = [
  233. _FETCH_ALL_PATH, '--android-deps-dir', _ANDROIDX_PATH,
  234. '--ignore-vulnerabilities'
  235. ]
  236. for subpath, url in _OVERRIDES:
  237. fetch_all_cmd += ['--override-artifact', f'{subpath}:{url}']
  238. subprocess.run(fetch_all_cmd, check=True)
  239. if not args.local_repo:
  240. # Prepend '0' to version to avoid conflicts with previous version format.
  241. version = 'cr-0' + version
  242. version_txt_path = os.path.join(_ANDROIDX_PATH, 'VERSION.txt')
  243. with open(version_txt_path, 'w') as f:
  244. f.write(version)
  245. yaml_path = os.path.join(_ANDROIDX_PATH, 'cipd.yaml')
  246. _write_cipd_yaml(libs_dir, version, yaml_path)
  247. if __name__ == '__main__':
  248. main()