123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648 |
- #!/usr/bin/env python3
- # Copyright 2018 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """A script used to manage Google Maven dependencies for Chromium.
- For each dependency in `build.gradle`:
- - Download the library
- - Download the LICENSE
- - Generate a README.chromium file
- - Generate a GN target in BUILD.gn
- - Generate .info files for AAR libraries
- - Generate a 'deps' entry in DEPS.
- """
- import argparse
- import collections
- import concurrent.futures
- import contextlib
- import fnmatch
- import logging
- import tempfile
- import textwrap
- import os
- import re
- import shutil
- import subprocess
- import urllib.request
- import zipfile
- # Assume this script is stored under third_party/android_deps/
- _CHROMIUM_SRC = os.path.normpath(os.path.join(__file__, '..', '..', '..'))
- # Default android_deps directory.
- _PRIMARY_ANDROID_DEPS_DIR = os.path.join(_CHROMIUM_SRC, 'third_party',
- 'android_deps')
- # Path to additional_readme_paths.json relative to custom 'android_deps' directory.
- _ADDITIONAL_README_PATHS = 'additional_readme_paths.json'
- # Path to BUILD.gn file from custom 'android_deps' directory.
- _BUILD_GN = 'BUILD.gn'
- # Path to build.gradle file relative to custom 'android_deps' directory.
- _BUILD_GRADLE = 'build.gradle'
- # Location of the android_deps libs directory relative to custom 'android_deps' directory.
- _LIBS_DIR = 'libs'
- _GN_PATH = os.path.join(_CHROMIUM_SRC, 'third_party', 'depot_tools', 'gn')
- _GRADLEW = os.path.join(_CHROMIUM_SRC, 'third_party', 'gradle_wrapper',
- 'gradlew')
- # Git-controlled files needed by, but not updated by this tool.
- # Relative to _PRIMARY_ANDROID_DEPS_DIR.
- _PRIMARY_ANDROID_DEPS_FILES = [
- 'buildSrc',
- 'licenses',
- 'settings.gradle.template',
- 'vulnerability_supressions.xml',
- ]
- # Git-controlled files needed by and updated by this tool.
- # Relative to args.android_deps_dir.
- _CUSTOM_ANDROID_DEPS_FILES = [
- os.path.join('..', '..', 'DEPS'),
- _BUILD_GN,
- _ADDITIONAL_README_PATHS,
- 'subprojects.txt',
- ]
- # Dictionary mapping long info file names to shorter ones to avoid paths being
- # over 200 chars. This must match the dictionary in BuildConfigGenerator.groovy.
- _REDUCED_ID_LENGTH_MAP = {
- 'com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework':
- 'com_google_android_accessibility_test_framework',
- }
- # If this file exists in an aar file then it is appended to LICENSE
- _THIRD_PARTY_LICENSE_FILENAME = 'third_party_licenses.txt'
- # Path to the aar.py script used to generate .info files.
- _AAR_PY = os.path.join(_CHROMIUM_SRC, 'build', 'android', 'gyp', 'aar.py')
- @contextlib.contextmanager
- def BuildDir(dirname=None):
- """Helper function used to manage a build directory.
- Args:
- dirname: Optional build directory path. If not provided, a temporary
- directory will be created and cleaned up on exit.
- Returns:
- A python context manager modelling a directory path. The manager
- removes the directory if necessary on exit.
- """
- delete = False
- if not dirname:
- dirname = tempfile.mkdtemp()
- delete = True
- try:
- yield dirname
- finally:
- if delete:
- shutil.rmtree(dirname)
- def RaiseCommandException(args, returncode, output, error):
- """Raise an exception whose message describing a command failure.
- Args:
- args: shell command-line (as passed to subprocess.call())
- returncode: status code.
- error: standard error output.
- Raises:
- a new Exception.
- """
- message = 'Command failed with status {}: {}\n'.format(returncode, args)
- if output:
- message += 'Output:-----------------------------------------\n{}\n' \
- '------------------------------------------------\n'.format(output)
- if error:
- message += 'Error message: ---------------------------------\n{}\n' \
- '------------------------------------------------\n'.format(error)
- raise Exception(message)
- def RunCommand(args, print_stdout=False, cwd=None):
- """Run a new shell command.
- This function runs without printing anything.
- Args:
- args: A string or a list of strings for the shell command.
- Raises:
- On failure, raise an Exception that contains the command's arguments,
- return status, and standard output + error merged in a single message.
- """
- logging.debug('Run %s', args)
- stdout = None if print_stdout else subprocess.PIPE
- p = subprocess.Popen(args, stdout=stdout, cwd=cwd)
- pout, _ = p.communicate()
- if p.returncode != 0:
- RaiseCommandException(args, p.returncode, None, pout)
- def RunCommandAndGetOutput(args):
- """Run a new shell command. Return its output. Exception on failure.
- This function runs without printing anything.
- Args:
- args: A string or a list of strings for the shell command.
- Returns:
- The command's output.
- Raises:
- On failure, raise an Exception that contains the command's arguments,
- return status, and standard output, and standard error as separate
- messages.
- """
- logging.debug('Run %s', args)
- p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- pout, perr = p.communicate()
- if p.returncode != 0:
- RaiseCommandException(args, p.returncode, pout, perr)
- return pout
- def MakeDirectory(dir_path):
- """Make directory |dir_path| recursively if necessary."""
- if dir_path != '' and not os.path.isdir(dir_path):
- logging.debug('mkdir [%s]', dir_path)
- os.makedirs(dir_path)
- def DeleteDirectory(dir_path):
- """Recursively delete a directory if it exists."""
- if os.path.exists(dir_path):
- logging.debug('rmdir [%s]', dir_path)
- shutil.rmtree(dir_path)
- def Copy(src_dir, src_paths, dst_dir, dst_paths, src_path_must_exist=True):
- """Copies |src_paths| in |src_dir| to |dst_paths| in |dst_dir|.
- Args:
- src_dir: Directory containing |src_paths|.
- src_paths: Files to copy.
- dst_dir: Directory containing |dst_paths|.
- dst_paths: Copy destinations.
- src_paths_must_exist: If False, do not throw error if the file for one of
- |src_paths| does not exist.
- """
- assert len(src_paths) == len(dst_paths)
- missing_files = []
- for src_path, dst_path in zip(src_paths, dst_paths):
- abs_src_path = os.path.join(src_dir, src_path)
- abs_dst_path = os.path.join(dst_dir, dst_path)
- if os.path.exists(abs_src_path):
- CopyFileOrDirectory(abs_src_path, abs_dst_path)
- elif src_path_must_exist:
- missing_files.append(src_path)
- if missing_files:
- raise Exception('Missing files from {}: {}'.format(
- src_dir, missing_files))
- def CopyFileOrDirectory(src_path, dst_path, ignore_extension=None):
- """Copy file or directory |src_path| into |dst_path| exactly.
- Args:
- src_path: Source path.
- dst_path: Destination path.
- ignore_extension: File extension of files not to copy, starting with '.'. If None, all files
- are copied.
- """
- assert not ignore_extension or ignore_extension[0] == '.'
- src_path = os.path.normpath(src_path)
- dst_path = os.path.normpath(dst_path)
- logging.debug('copy [%s -> %s]', src_path, dst_path)
- MakeDirectory(os.path.dirname(dst_path))
- if os.path.isdir(src_path):
- # Copy directory recursively.
- DeleteDirectory(dst_path)
- ignore = None
- if ignore_extension:
- ignore = shutil.ignore_patterns('*' + ignore_extension)
- shutil.copytree(src_path, dst_path, ignore=ignore)
- elif not ignore_extension or not re.match('.*\.' + ignore_extension[1:],
- src_path):
- shutil.copy(src_path, dst_path)
- def ReadFile(file_path):
- """Read a file, return its content."""
- with open(file_path) as f:
- return f.read()
- def ReadFileAsLines(file_path):
- """Read a file as a series of lines."""
- with open(file_path) as f:
- return f.readlines()
- def WriteFile(file_path, file_data):
- """Write a file."""
- if isinstance(file_data, str):
- file_data = file_data.encode('utf8')
- MakeDirectory(os.path.dirname(file_path))
- with open(file_path, 'wb') as f:
- f.write(file_data)
- def FindInDirectory(directory, filename_filter):
- """Find all files in a directory that matches a given filename filter."""
- files = []
- for root, _dirnames, filenames in os.walk(directory):
- matched_files = fnmatch.filter(filenames, filename_filter)
- files.extend((os.path.join(root, f) for f in matched_files))
- return files
- # Named tuple describing a CIPD package.
- # - path: Path to cipd.yaml file.
- # - name: cipd package name.
- # - tag: cipd tag.
- CipdPackageInfo = collections.namedtuple('CipdPackageInfo',
- ['path', 'name', 'tag'])
- # Regular expressions used to extract useful info from cipd.yaml files
- # generated by Gradle. See BuildConfigGenerator.groovy:makeCipdYaml()
- _RE_CIPD_CREATE = re.compile('cipd create --pkg-def cipd.yaml -tag (\S*)')
- _RE_CIPD_PACKAGE = re.compile('package: (\S*)')
- def _ParseSubprojects(subproject_path):
- """Parses listing of subproject build.gradle files. Returns list of paths."""
- if not os.path.exists(subproject_path):
- return None
- subprojects = []
- for subproject in open(subproject_path):
- subproject = subproject.strip()
- if subproject and not subproject.startswith('#'):
- subprojects.append(subproject)
- return subprojects
- def _GenerateSettingsGradle(subproject_dirs, settings_template_path,
- settings_out_path):
- """Generates settings file by replacing "{{subproject_dirs}}" string in template.
- Args:
- subproject_dirs: List of subproject directories to substitute into template.
- settings_template_path: Path of template file to substitute into.
- settings_out_path: Path of output settings.gradle file.
- """
- with open(settings_template_path) as f:
- template_content = f.read()
- subproject_dirs_str = ''
- if subproject_dirs:
- subproject_dirs_str = '\'' + '\',\''.join(subproject_dirs) + '\''
- template_content = template_content.replace('{{subproject_dirs}}',
- subproject_dirs_str)
- with open(settings_out_path, 'w') as f:
- f.write(template_content)
- def _BuildGradleCmd(build_android_deps_dir, task):
- return [
- _GRADLEW, '-p', build_android_deps_dir, '--stacktrace',
- '--warning-mode', 'all', task
- ]
- def _CheckVulnerabilities(build_android_deps_dir, report_dst):
- logging.warning('Running Gradle dependencyCheckAnalyze. This may take a '
- 'few minutes the first time.')
- # Separate command from main gradle command so that we can provide specific
- # diagnostics in case of failure of this step.
- gradle_cmd = _BuildGradleCmd(build_android_deps_dir,
- 'dependencyCheckAnalyze')
- report_src = os.path.join(build_android_deps_dir, 'build', 'reports')
- if os.path.exists(report_dst):
- shutil.rmtree(report_dst)
- try:
- logging.info('CMD: %s', ' '.join(gradle_cmd))
- subprocess.run(gradle_cmd, check=True)
- except subprocess.CalledProcessError:
- report_path = os.path.join(report_dst, 'dependency-check-report.html')
- logging.error(
- textwrap.dedent("""
- =============================================================================
- A package has a known vulnerability. It may not be in a package or packages
- which you just added, but you need to resolve the problem before proceeding.
- If you can't easily fix it by rolling the package to a fixed version now,
- please file a crbug of type= Bug-Security providing all relevant information,
- and then rerun this command with --ignore-vulnerabilities.
- The html version of the report is avialable at: {}
- =============================================================================
- """.format(report_path)))
- raise
- finally:
- if os.path.exists(report_src):
- CopyFileOrDirectory(report_src, report_dst)
- def _ReduceNameLength(path_str):
- """Returns a shorter path string if needed.
- Args:
- path_str: A String representing the path.
- Returns:
- A String (possibly shortened) of that path.
- """
- return _REDUCED_ID_LENGTH_MAP.get(path_str, path_str)
- def GetCipdPackageInfo(cipd_yaml_path):
- """Returns the CIPD package name corresponding to a given cipd.yaml file.
- Args:
- cipd_yaml_path: Path of input cipd.yaml file.
- Returns:
- A (package_name, package_tag) tuple.
- Raises:
- Exception if the file could not be read.
- """
- package_name = None
- package_tag = None
- for line in ReadFileAsLines(cipd_yaml_path):
- m = _RE_CIPD_PACKAGE.match(line)
- if m:
- package_name = m.group(1)
- m = _RE_CIPD_CREATE.search(line)
- if m:
- package_tag = m.group(1)
- if not package_name or not package_tag:
- raise Exception('Invalid cipd.yaml format: ' + cipd_yaml_path)
- return (package_name, package_tag)
- def ParseDeps(root_dir, libs_dir):
- """Parse an android_deps/libs and retrieve package information.
- Args:
- root_dir: Path to a root Chromium or build directory.
- Returns:
- A directory mapping package names to tuples of
- (cipd_yaml_file, package_name, package_tag), where |cipd_yaml_file|
- is the path to the cipd.yaml file, related to |libs_dir|,
- and |package_name| and |package_tag| are the extracted from it.
- """
- result = {}
- root_dir = os.path.abspath(root_dir)
- libs_dir = os.path.abspath(os.path.join(root_dir, libs_dir))
- for cipd_file in FindInDirectory(libs_dir, 'cipd.yaml'):
- pkg_name, pkg_tag = GetCipdPackageInfo(cipd_file)
- cipd_path = os.path.dirname(cipd_file)
- cipd_path = cipd_path[len(root_dir) + 1:]
- result[pkg_name] = CipdPackageInfo(cipd_path, pkg_name, pkg_tag)
- return result
- def PrintPackageList(packages, list_name):
- """Print a list of packages to standard output.
- Args:
- packages: list of package names.
- list_name: a simple word describing the package list (e.g. 'new')
- """
- print(' {} {} packages:'.format(len(packages), list_name))
- print('\n'.join(' - ' + p for p in packages))
- def _DownloadOverrides(overrides, build_libs_dir):
- for spec in overrides:
- subpath, url = spec.split(':', 1)
- target_path = os.path.join(build_libs_dir, subpath)
- if not os.path.isfile(target_path):
- found_files = 'Found instead:\n' + '\n'.join(
- FindInDirectory(os.path.dirname(target_path), '*'))
- raise Exception(
- f'Override path does not exist: {target_path}\n{found_files}')
- logging.info('Fetching override for %s', target_path)
- with urllib.request.urlopen(url) as response:
- with open(target_path, 'wb') as f:
- shutil.copyfileobj(response, f)
- def _CreateAarInfos(aar_files):
- jobs = []
- for aar_file in aar_files:
- aar_dirname = os.path.dirname(aar_file)
- aar_info_name = _ReduceNameLength(
- os.path.basename(aar_dirname)) + '.info'
- aar_info_path = os.path.join(aar_dirname, aar_info_name)
- logging.debug('- %s', aar_info_name)
- cmd = [_AAR_PY, 'list', aar_file, '--output', aar_info_path]
- if aar_info_name == 'com_google_android_material_material.info':
- # Keep in sync with copy in BuildConfigGenerator.groovy.
- resource_exclusion_glbos = [
- 'res/layout*/*calendar*',
- 'res/layout*/*chip_input*',
- 'res/layout*/*clock*',
- 'res/layout*/*picker*',
- 'res/layout*/*time*',
- ]
- cmd += [
- '--resource-exclusion-globs',
- repr(resource_exclusion_glbos).replace("'", '"')
- ]
- proc = subprocess.Popen(cmd)
- jobs.append((cmd, proc))
- for cmd, proc in jobs:
- if proc.wait():
- raise Exception('Command Failed: {}\n'.format(' '.join(cmd)))
- def main():
- parser = argparse.ArgumentParser(
- description=__doc__,
- formatter_class=argparse.RawDescriptionHelpFormatter)
- parser.add_argument(
- '--android-deps-dir',
- help='Path to directory containing build.gradle from chromium-dir.',
- default=_PRIMARY_ANDROID_DEPS_DIR)
- parser.add_argument(
- '--build-dir',
- help='Path to build directory (default is temporary directory).')
- parser.add_argument('--ignore-licenses',
- help='Ignores licenses for these deps.',
- action='store_true')
- parser.add_argument('--ignore-vulnerabilities',
- help='Ignores vulnerabilities for these deps.',
- action='store_true')
- parser.add_argument('--override-artifact',
- action='append',
- help='lib_subpath:url of .aar / .jar to override.')
- parser.add_argument('-v',
- '--verbose',
- dest='verbose_count',
- default=0,
- action='count',
- help='Verbose level (multiple times for more)')
- args = parser.parse_args()
- logging.basicConfig(
- level=logging.WARNING - 10 * args.verbose_count,
- format='%(levelname).1s %(relativeCreated)6d %(message)s')
- debug = args.verbose_count >= 2
- if not os.path.isfile(os.path.join(args.android_deps_dir, _BUILD_GRADLE)):
- raise Exception('--android-deps-dir {} does not contain {}.'.format(
- args.android_deps_dir, _BUILD_GRADLE))
- is_primary_android_deps = args.android_deps_dir == _PRIMARY_ANDROID_DEPS_DIR
- android_deps_subdir = os.path.relpath(args.android_deps_dir, _CHROMIUM_SRC)
- with BuildDir(args.build_dir) as build_dir:
- build_android_deps_dir = os.path.join(build_dir, android_deps_subdir)
- logging.info('Using build directory: %s', build_dir)
- Copy(_PRIMARY_ANDROID_DEPS_DIR, _PRIMARY_ANDROID_DEPS_FILES,
- build_android_deps_dir, _PRIMARY_ANDROID_DEPS_FILES)
- Copy(args.android_deps_dir, [_BUILD_GRADLE], build_android_deps_dir,
- [_BUILD_GRADLE])
- Copy(args.android_deps_dir,
- _CUSTOM_ANDROID_DEPS_FILES,
- build_android_deps_dir,
- _CUSTOM_ANDROID_DEPS_FILES,
- src_path_must_exist=is_primary_android_deps)
- subprojects = _ParseSubprojects(
- os.path.join(args.android_deps_dir, 'subprojects.txt'))
- subproject_dirs = []
- if subprojects:
- for (index, subproject) in enumerate(subprojects):
- subproject_dir = 'subproject{}'.format(index)
- Copy(args.android_deps_dir, [subproject],
- build_android_deps_dir,
- [os.path.join(subproject_dir, 'build.gradle')])
- subproject_dirs.append(subproject_dir)
- _GenerateSettingsGradle(
- subproject_dirs,
- os.path.join(_PRIMARY_ANDROID_DEPS_DIR,
- 'settings.gradle.template'),
- os.path.join(build_android_deps_dir, 'settings.gradle'))
- if not args.ignore_vulnerabilities:
- report_dst = os.path.join(args.android_deps_dir,
- 'vulnerability_reports')
- _CheckVulnerabilities(build_android_deps_dir, report_dst)
- logging.info('Running Gradle.')
- # This gradle command generates the new DEPS and BUILD.gn files, it can
- # also handle special cases.
- # Edit BuildConfigGenerator.groovy#addSpecialTreatment for such cases.
- gradle_cmd = _BuildGradleCmd(build_android_deps_dir, 'setupRepository')
- if debug:
- gradle_cmd.append('--debug')
- if args.ignore_licenses:
- gradle_cmd.append('-PskipLicenses=true')
- subprocess.run(gradle_cmd, check=True)
- logging.info('# Reformat %s.',
- os.path.join(args.android_deps_dir, _BUILD_GN))
- gn_args = [
- os.path.relpath(_GN_PATH, _CHROMIUM_SRC), 'format',
- os.path.join(os.path.relpath(build_android_deps_dir, _CHROMIUM_SRC),
- _BUILD_GN)
- ]
- RunCommand(gn_args, print_stdout=debug, cwd=_CHROMIUM_SRC)
- build_libs_dir = os.path.join(build_android_deps_dir, _LIBS_DIR)
- if args.override_artifact:
- _DownloadOverrides(args.override_artifact, build_libs_dir)
- aar_files = FindInDirectory(build_libs_dir, '*.aar')
- logging.info('# Generate Android .aar info files.')
- _CreateAarInfos(aar_files)
- if not args.ignore_licenses:
- logging.info('# Looking for nested license files.')
- for aar_file in aar_files:
- # Play Services .aar files have embedded licenses.
- with zipfile.ZipFile(aar_file) as z:
- if _THIRD_PARTY_LICENSE_FILENAME in z.namelist():
- aar_dirname = os.path.dirname(aar_file)
- license_path = os.path.join(aar_dirname, 'LICENSE')
- # Make sure to append as we don't want to lose the
- # existing license.
- with open(license_path, 'ab') as f:
- f.write(z.read(_THIRD_PARTY_LICENSE_FILENAME))
- logging.info('# Compare CIPD packages.')
- existing_packages = ParseDeps(args.android_deps_dir, _LIBS_DIR)
- build_packages = ParseDeps(build_android_deps_dir, _LIBS_DIR)
- deleted_packages = []
- updated_packages = []
- for pkg in sorted(existing_packages):
- if pkg not in build_packages:
- deleted_packages.append(pkg)
- else:
- existing_info = existing_packages[pkg]
- build_info = build_packages[pkg]
- if existing_info.tag != build_info.tag:
- updated_packages.append(pkg)
- new_packages = sorted(set(build_packages) - set(existing_packages))
- # Copy updated DEPS and BUILD.gn to build directory.
- Copy(build_android_deps_dir,
- _CUSTOM_ANDROID_DEPS_FILES,
- args.android_deps_dir,
- _CUSTOM_ANDROID_DEPS_FILES,
- src_path_must_exist=is_primary_android_deps)
- # Delete obsolete or updated package directories.
- for pkg in existing_packages.values():
- pkg_path = os.path.join(args.android_deps_dir, pkg.path)
- DeleteDirectory(pkg_path)
- # Copy new and updated packages from build directory.
- for pkg in build_packages.values():
- pkg_path = pkg.path
- dst_pkg_path = os.path.join(args.android_deps_dir, pkg_path)
- src_pkg_path = os.path.join(build_android_deps_dir, pkg_path)
- CopyFileOrDirectory(src_pkg_path,
- dst_pkg_path,
- ignore_extension=".tmp")
- # Useful for printing timestamp.
- logging.info('All Done.')
- if new_packages:
- PrintPackageList(new_packages, 'new')
- if updated_packages:
- PrintPackageList(updated_packages, 'updated')
- if deleted_packages:
- PrintPackageList(deleted_packages, 'deleted')
- if __name__ == "__main__":
- main()
|