123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304 |
- #!/usr/bin/env python3
- # Copyright 2020 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """A script to generate build.gradle from template and run fetch_all.py
- More specifically, to generate build.gradle:
- - It downloads the BUILD_INFO file for the latest androidx snapshot from
- https://androidx.dev/snapshots/builds
- - It replaces {{androidx_repository_url}} with the URL for the latest snapshot
- - For each dependency, it looks up the version in the BUILD_INFO file and
- substitutes the version into {{androidx_dependency_version}}.
- """
- import argparse
- import contextlib
- import json
- import logging
- import os
- import re
- import shutil
- import stat
- import subprocess
- import tempfile
- import urllib
- from urllib import request
- _ANDROIDX_PATH = os.path.normpath(os.path.join(__file__, '..'))
- _FETCH_ALL_PATH = os.path.normpath(
- os.path.join(_ANDROIDX_PATH, '..', 'android_deps', 'fetch_all.py'))
- # URL to BUILD_INFO in latest androidx snapshot.
- _ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL = 'https://androidx.dev/snapshots/latest/artifacts/BUILD_INFO'
- # Snapshot repository URL with {{version}} placeholder.
- _SNAPSHOT_REPOSITORY_URL = 'https://androidx.dev/snapshots/builds/{{version}}/artifacts/repository'
- # When androidx roller is breaking, and a fix is not immenent, use this to pin a
- # broken library to an old known-working version.
- # * The first element of each tuple is the path to the artifact of the latest
- # version of the library. It could change if the version is rev'ed in a new
- # snapshot.
- # * The second element is a URL to replace the file with. Find the URL for older
- # versions of libraries by looking in the BUILD_INFO for the older version
- # (e.g.: https://androidx.dev/snapshots/builds/8545498/artifacts/BUILD_INFO)
- _OVERRIDES = [
- # Example:
- #('androidx_core_core/core-1.9.0-SNAPSHOT.aar',
- # 'https://androidx.dev/snapshots/builds/8545498/artifacts/repository/'
- # 'androidx/core/core/1.8.0-SNAPSHOT/core-1.8.0-20220505.122105-1.aar'),
- # Context: https://crbug.com/1349920 and https://crbug.com/1349521
- ('androidx_appcompat_appcompat/appcompat-1.6.0-SNAPSHOT.aar',
- 'https://androidx.dev/snapshots/builds/8811104/artifacts/repository/'
- 'androidx/appcompat/appcompat/1.5.0-SNAPSHOT/'
- 'appcompat-1.5.0-20220708.124951-1.aar'),
- ('androidx_appcompat_appcompat_resources/'
- 'appcompat-resources-1.6.0-SNAPSHOT.aar',
- 'https://androidx.dev/snapshots/builds/8811104/artifacts/repository/'
- 'androidx/appcompat/appcompat-resources/1.5.0-SNAPSHOT/'
- 'appcompat-resources-1.5.0-20220708.124951-1.aar'),
- ]
- def _build_snapshot_repository_url(version):
- return _SNAPSHOT_REPOSITORY_URL.replace('{{version}}', version)
- def _delete_readonly_files(paths):
- for path in paths:
- if os.path.exists(path):
- os.chmod(path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IWUSR)
- os.remove(path)
- def _parse_dir_list(dir_list):
- """Computes 'library_group:library_name'->library_version mapping.
- Args:
- dir_list: List of androidx library directories.
- """
- dependency_version_map = dict()
- for dir_entry in dir_list:
- stripped_dir = dir_entry.strip()
- if not stripped_dir.startswith('repository/androidx/'):
- continue
- dir_components = stripped_dir.split('/')
- # Expected format:
- # "repository/androidx/library_group/library_name/library_version/pom_or_jar"
- if len(dir_components) < 6:
- continue
- dependency_package = 'androidx.' + '.'.join(dir_components[2:-3])
- dependency_module = '{}:{}'.format(dependency_package,
- dir_components[-3])
- if dependency_module not in dependency_version_map:
- dependency_version_map[dependency_module] = dir_components[-2]
- return dependency_version_map
- def _compute_replacement(dependency_version_map, androidx_repository_url,
- line):
- """Computes output line for build.gradle from build.gradle.template line.
- Replaces {{android_repository_url}}, {{androidx_dependency_version}} and
- {{version_overrides}}.
- Args:
- dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
- androidx_repository_url: URL of the maven repository.
- line: Input line from the build.gradle.template.
- """
- line = line.replace('{{androidx_repository_url}}', androidx_repository_url)
- if line.strip() == '{{version_overrides}}':
- lines = ['versionOverrideMap = [:]']
- for dependency, version in sorted(dependency_version_map.items()):
- lines.append(f"versionOverrideMap['{dependency}'] = '{version}'")
- return '\n'.join(lines)
- match = re.search(r'\'(\S+):{{androidx_dependency_version}}\'', line)
- if not match:
- return line
- dependency = match.group(1)
- version = dependency_version_map.get(dependency)
- if not version:
- raise Exception(f'Version for {dependency} not found.')
- return line.replace('{{androidx_dependency_version}}', version)
- @contextlib.contextmanager
- def _build_dir():
- dirname = tempfile.mkdtemp()
- try:
- yield dirname
- finally:
- shutil.rmtree(dirname)
- def _download_and_parse_build_info():
- """Downloads and parses BUILD_INFO file."""
- with _build_dir() as build_dir:
- androidx_build_info_response = request.urlopen(
- _ANDROIDX_LATEST_SNAPSHOT_BUILD_INFO_URL)
- androidx_build_info_url = androidx_build_info_response.geturl()
- logging.info('URL for the latest build info: %s',
- androidx_build_info_url)
- androidx_build_info_path = os.path.join(build_dir, 'BUILD_INFO')
- with open(androidx_build_info_path, 'w') as f:
- f.write(androidx_build_info_response.read().decode('utf-8'))
- # Strip '/repository' from pattern.
- resolved_snapshot_repository_url_pattern = (
- _build_snapshot_repository_url('([0-9]*)').rsplit('/', 1)[0])
- version = re.match(resolved_snapshot_repository_url_pattern,
- androidx_build_info_url).group(1)
- with open(androidx_build_info_path, 'r') as f:
- build_info_dict = json.loads(f.read())
- dir_list = build_info_dict['target']['dir_list']
- return dir_list, version
- def _create_local_dir_list(repo_path):
- repo_path = repo_path.rstrip('/')
- prefix_len = len(repo_path) + 1
- ret = []
- for dirpath, _, filenames in os.walk(repo_path):
- for name in filenames:
- ret.append(os.path.join('repository', dirpath[prefix_len:], name))
- return ret
- def _process_build_gradle(dependency_version_map, androidx_repository_url):
- """Generates build.gradle from template.
- Args:
- dependency_version_map: An "dependency_group:dependency_name"->dependency_version mapping.
- androidx_repository_url: URL of the maven repository.
- """
- build_gradle_template_path = os.path.join(_ANDROIDX_PATH,
- 'build.gradle.template')
- build_gradle_out_path = os.path.join(_ANDROIDX_PATH, 'build.gradle')
- # |build_gradle_out_path| is not deleted after script has finished running. The file is in
- # .gitignore and thus will be excluded from uploaded CLs.
- with open(build_gradle_template_path, 'r') as template_f, \
- open(build_gradle_out_path, 'w') as out:
- for template_line in template_f:
- replacement = _compute_replacement(dependency_version_map,
- androidx_repository_url,
- template_line)
- out.write(replacement)
- def _write_cipd_yaml(libs_dir, version, cipd_yaml_path):
- """Writes cipd.yaml file at the passed-in path."""
- lib_dirs = os.listdir(libs_dir)
- if not lib_dirs:
- raise Exception('No generated libraries in {}'.format(libs_dir))
- data_files = [
- 'BUILD.gn', 'VERSION.txt', 'additional_readme_paths.json',
- 'build.gradle'
- ]
- for lib_dir in lib_dirs:
- abs_lib_dir = os.path.join(libs_dir, lib_dir)
- androidx_rel_lib_dir = os.path.relpath(abs_lib_dir, _ANDROIDX_PATH)
- if not os.path.isdir(abs_lib_dir):
- continue
- lib_files = os.listdir(abs_lib_dir)
- if not 'cipd.yaml' in lib_files:
- continue
- for lib_file in lib_files:
- if lib_file == 'cipd.yaml' or lib_file == 'OWNERS':
- continue
- data_files.append(os.path.join(androidx_rel_lib_dir, lib_file))
- contents = [
- '# Copyright 2021 The Chromium Authors. All rights reserved.',
- '# Use of this source code is governed by a BSD-style license that can be',
- '# found in the LICENSE file.',
- '# version: ' + version,
- 'package: chromium/third_party/androidx',
- 'description: androidx',
- 'data:',
- ]
- contents.extend('- file: ' + f for f in data_files)
- with open(cipd_yaml_path, 'w') as out:
- out.write('\n'.join(contents))
- def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('-v',
- '--verbose',
- dest='verbose_count',
- default=0,
- action='count',
- help='Verbose level (multiple times for more)')
- parser.add_argument('--local-repo',
- help='Path to a locally androidx maven repo to use '
- 'instead of fetching the latest.')
- args = parser.parse_args()
- logging.basicConfig(
- level=logging.WARNING - 10 * args.verbose_count,
- format='%(levelname).1s %(relativeCreated)6d %(message)s')
- libs_dir = os.path.join(_ANDROIDX_PATH, 'libs')
- if os.path.exists(libs_dir):
- shutil.rmtree(libs_dir)
- # Files uploaded to cipd are read-only. Delete them because they will be
- # re-generated.
- _delete_readonly_files([
- os.path.join(_ANDROIDX_PATH, 'BUILD.gn'),
- os.path.join(_ANDROIDX_PATH, 'VERSION.txt'),
- os.path.join(_ANDROIDX_PATH, 'additional_readme_paths.json'),
- os.path.join(_ANDROIDX_PATH, 'build.gradle'),
- ])
- if args.local_repo:
- version = 'local'
- dir_list = _create_local_dir_list(args.local_repo)
- androidx_snapshot_repository_url = ('file://' +
- os.path.abspath(args.local_repo))
- else:
- dir_list, version = _download_and_parse_build_info()
- androidx_snapshot_repository_url = _build_snapshot_repository_url(
- version)
- dependency_version_map = _parse_dir_list(dir_list)
- _process_build_gradle(dependency_version_map,
- androidx_snapshot_repository_url)
- fetch_all_cmd = [
- _FETCH_ALL_PATH, '--android-deps-dir', _ANDROIDX_PATH,
- '--ignore-vulnerabilities'
- ]
- for subpath, url in _OVERRIDES:
- fetch_all_cmd += ['--override-artifact', f'{subpath}:{url}']
- subprocess.run(fetch_all_cmd, check=True)
- if not args.local_repo:
- # Prepend '0' to version to avoid conflicts with previous version format.
- version = 'cr-0' + version
- version_txt_path = os.path.join(_ANDROIDX_PATH, 'VERSION.txt')
- with open(version_txt_path, 'w') as f:
- f.write(version)
- yaml_path = os.path.join(_ANDROIDX_PATH, 'cipd.yaml')
- _write_cipd_yaml(libs_dir, version, yaml_path)
- if __name__ == '__main__':
- main()
|