fetch_all_autorolled.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. #!/usr/bin/env python3
  2. # Copyright 2021 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """A script to generate build.gradle from template and run fetch_all.py
  6. More specifically, to generate build.gradle:
  7. - It downloads the gradle metadata for guava and parses the file to determine
  8. the latest guava version.
  9. - It replaces {{guava_version}} in build.gralde.template with the latest guava
  10. version.
  11. """
  12. import os
  13. import re
  14. import shutil
  15. import subprocess
  16. import tempfile
  17. import urllib
  18. from urllib import request
  19. from xml.etree import ElementTree
  20. _AUTOROLLED_PATH = os.path.normpath(os.path.join(__file__, '..'))
  21. _GUAVA_MAVEN_METADATA_XML_URL = 'https://repo.maven.apache.org/maven2/com/google/guava/guava/maven-metadata.xml'
  22. _FETCH_ALL_PATH = os.path.normpath(
  23. os.path.join(_AUTOROLLED_PATH, '..', 'android_deps', 'fetch_all.py'))
  24. def _download_and_compute_latest_guava_version():
  25. """Downloads gradle metadata for guava. Returns latest guava version."""
  26. metadata_xml_response = request.urlopen(_GUAVA_MAVEN_METADATA_XML_URL)
  27. xml_tree = ElementTree.fromstring(metadata_xml_response.read())
  28. latest_version = xml_tree.find('versioning/latest')
  29. match = re.match(r'([0-9]*\.[0-9]*).*', latest_version.text)
  30. version = match.group(1)
  31. jre_version = version + '-jre'
  32. android_version = version + '-android'
  33. version_jre_path = 'versioning/versions/version[.=\'' + jre_version + '\']'
  34. version_android_path = 'versioning/versions/version[.=\'' + android_version + '\']'
  35. if xml_tree.find(version_jre_path) == None or xml_tree.find(version_android_path) == None:
  36. raise Exception('{} or {} version not found in {}.'.format(jre_version, android_version, _GUAVA_MAVEN_METADATA_XML_URL))
  37. return version
  38. def _process_build_gradle(guava_version):
  39. """Generates build.gradle from template.
  40. Args:
  41. guava_version: Latest guava version.
  42. """
  43. template_path = os.path.join(_AUTOROLLED_PATH, 'build.gradle.template')
  44. with open(template_path) as f:
  45. template_content = f.read()
  46. out_path = os.path.join(_AUTOROLLED_PATH, 'build.gradle')
  47. with open(out_path, 'w') as f:
  48. out = template_content.replace('{{guava_version}}', guava_version)
  49. f.write(out)
  50. def _extract_files_from_yaml(yaml_path):
  51. """Extracts '- file' file listings from yaml file."""
  52. out = None
  53. with open(yaml_path, 'r') as f:
  54. for line in f.readlines():
  55. line = line.rstrip('\n')
  56. if line == 'data:':
  57. out = []
  58. continue
  59. if out is not None:
  60. if not line.startswith('- file:'):
  61. raise Exception(
  62. '{} has unsupported attributes. Only \'- file\' is supported'
  63. .format(yaml_path))
  64. out.append(line.rsplit(' ', 1)[1])
  65. if not out:
  66. raise Exception('{} does not have \'data\' section.'.format(yaml_path))
  67. return out
  68. def _write_cipd_yaml(libs_dir, cipd_yaml_path):
  69. """Writes cipd.yaml file at the passed-in path."""
  70. lib_dirs = os.listdir(libs_dir)
  71. if not lib_dirs:
  72. raise Exception('No generated libraries in {}'.format(libs_dir))
  73. data_files = ['BUILD.gn', 'additional_readme_paths.json']
  74. for lib_dir in lib_dirs:
  75. abs_lib_dir = os.path.join(libs_dir, lib_dir)
  76. autorolled_rel_lib_dir = os.path.relpath(abs_lib_dir, _AUTOROLLED_PATH)
  77. if not os.path.isdir(abs_lib_dir):
  78. continue
  79. lib_files = os.listdir(abs_lib_dir)
  80. if not 'cipd.yaml' in lib_files:
  81. continue
  82. if not 'README.chromium' in lib_files:
  83. raise Exception('README.chromium not in {}'.format(abs_lib_dir))
  84. if not 'LICENSE' in lib_files:
  85. raise Exception('LICENSE not in {}'.format(abs_lib_dir))
  86. data_files.append(os.path.join(autorolled_rel_lib_dir,
  87. 'README.chromium'))
  88. data_files.append(os.path.join(autorolled_rel_lib_dir, 'LICENSE'))
  89. _rel_extracted_files = _extract_files_from_yaml(
  90. os.path.join(abs_lib_dir, 'cipd.yaml'))
  91. data_files.extend(
  92. os.path.join(autorolled_rel_lib_dir, f)
  93. for f in _rel_extracted_files)
  94. contents = [
  95. '# Copyright 2021 The Chromium Authors. All rights reserved.',
  96. '# Use of this source code is governed by a BSD-style license that can be',
  97. '# found in the LICENSE file.',
  98. 'package: chromium/third_party/android_deps_autorolled', 'description: android_deps_autorolled',
  99. 'data:'
  100. ]
  101. contents.extend('- file: ' + f for f in data_files)
  102. with open(cipd_yaml_path, 'w') as out:
  103. out.write('\n'.join(contents))
  104. def main():
  105. libs_dir = os.path.join(_AUTOROLLED_PATH, 'libs')
  106. # Let recipe delete contents of lib directory because it has API to retry
  107. # directory deletion if the first deletion attempt does not work.
  108. if os.path.exists(libs_dir) and os.listdir(libs_dir):
  109. raise Exception('Recipe did not empty \'libs\' directory.')
  110. guava_version = _download_and_compute_latest_guava_version()
  111. _process_build_gradle(guava_version)
  112. fetch_all_cmd = [
  113. _FETCH_ALL_PATH, '--android-deps-dir', _AUTOROLLED_PATH,
  114. '--ignore-vulnerabilities'
  115. ]
  116. subprocess.run(fetch_all_cmd, check=True)
  117. _write_cipd_yaml(libs_dir, os.path.join(_AUTOROLLED_PATH, 'cipd.yaml'))
  118. if __name__ == '__main__':
  119. main()