update_sdk.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514
  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2017 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. r"""This script downloads / packages & uploads Android SDK packages.
  7. It could be run when we need to update sdk packages to latest version.
  8. It has 2 usages:
  9. 1) download: downloading a new version of the SDK via sdkmanager
  10. 2) package: wrapping SDK directory into CIPD-compatible packages and
  11. uploading the new packages via CIPD to server.
  12. Providing '--dry-run' option to show what packages to be
  13. created and uploaded without actually doing either.
  14. Both downloading and uploading allows to either specify a package, or
  15. deal with default packages (build-tools, platform-tools, platforms and
  16. tools).
  17. Example usage:
  18. 1) updating default packages:
  19. $ update_sdk.py download
  20. (optional) $ update_sdk.py package --dry-run
  21. $ update_sdk.py package
  22. 2) updating a specified package:
  23. $ update_sdk.py download -p "build-tools;27.0.3"
  24. (optional) $ update_sdk.py package --dry-run -p build-tools \
  25. --version 27.0.3
  26. $ update_sdk.py package -p build-tools --version 27.0.3
  27. Note that `package` could update the package argument to the checkout
  28. version in .gn file //build/config/android/config.gni. If having git
  29. changes, please prepare to upload a CL that updates the SDK version.
  30. """
  31. from __future__ import print_function
  32. import argparse
  33. import os
  34. import re
  35. import shutil
  36. import subprocess
  37. import sys
  38. import tempfile
  39. _SRC_ROOT = os.path.realpath(
  40. os.path.join(os.path.dirname(__file__), '..', '..', '..'))
  41. _SRC_DEPS_PATH = os.path.join(_SRC_ROOT, 'DEPS')
  42. _SDK_PUBLIC_ROOT = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
  43. 'public')
  44. _SDKMANAGER_PATH = os.path.join(_SRC_ROOT, 'third_party', 'android_sdk',
  45. 'public', 'tools', 'bin', 'sdkmanager')
  46. _ANDROID_CONFIG_GNI_PATH = os.path.join(_SRC_ROOT, 'build', 'config', 'android',
  47. 'config.gni')
  48. _TOOLS_LIB_PATH = os.path.join(_SDK_PUBLIC_ROOT, 'tools', 'lib')
  49. _DEFAULT_DOWNLOAD_PACKAGES = [
  50. 'build-tools', 'platform-tools', 'platforms', 'tools'
  51. ]
  52. # TODO(shenghuazhang): Search package versions from available packages through
  53. # the sdkmanager, instead of hardcoding the package names w/ version.
  54. # TODO(yliuyliu): we might not need the latest version if unstable,
  55. # will double-check this later.
  56. _DEFAULT_PACKAGES_DICT = {
  57. 'build-tools': 'build-tools;27.0.3',
  58. 'platforms': 'platforms;android-28',
  59. 'sources': 'sources;android-28',
  60. }
  61. _GN_ARGUMENTS_TO_UPDATE = {
  62. 'build-tools': 'default_android_sdk_build_tools_version',
  63. 'tools': 'android_sdk_tools_version_suffix',
  64. 'platforms': 'default_android_sdk_version',
  65. }
  66. _COMMON_JAR_SUFFIX_PATTERN = re.compile(
  67. r'^common' # file name begins with 'common'
  68. r'(-[\d\.]+(-dev)?)' # group of suffix e.g.'-26.0.0-dev', '-25.3.2'
  69. r'\.jar$' # ends with .jar
  70. )
  71. def _DownloadSdk(arguments):
  72. """Download sdk package from sdkmanager.
  73. If package isn't provided, update build-tools, platform-tools, platforms,
  74. and tools.
  75. Args:
  76. arguments: The arguments parsed from argparser.
  77. """
  78. for pkg in arguments.package:
  79. # If package is not a sdk-style path, try to match a default path to it.
  80. if pkg in _DEFAULT_PACKAGES_DICT:
  81. print('Coercing %s to %s' % (pkg, _DEFAULT_PACKAGES_DICT[pkg]))
  82. pkg = _DEFAULT_PACKAGES_DICT[pkg]
  83. download_sdk_cmd = [
  84. _SDKMANAGER_PATH, '--install',
  85. '--sdk_root=%s' % arguments.sdk_root, pkg
  86. ]
  87. if arguments.verbose:
  88. download_sdk_cmd.append('--verbose')
  89. subprocess.check_call(download_sdk_cmd)
  90. def _FindPackageVersion(package, sdk_root):
  91. """Find sdk package version.
  92. Two options for package version:
  93. 1) Use the version in name if package name contains ';version'
  94. 2) For simple name package, search its version from 'Installed packages'
  95. via `sdkmanager --list`
  96. Args:
  97. package: The Android SDK package.
  98. sdk_root: The Android SDK root path.
  99. Returns:
  100. The version of package.
  101. Raises:
  102. Exception: cannot find the version of package.
  103. """
  104. sdkmanager_list_cmd = [
  105. _SDKMANAGER_PATH,
  106. '--list',
  107. '--sdk_root=%s' % sdk_root,
  108. ]
  109. if package in _DEFAULT_PACKAGES_DICT:
  110. # Get the version after ';' from package name
  111. package = _DEFAULT_PACKAGES_DICT[package]
  112. return package.split(';')[1]
  113. else:
  114. # Get the package version via `sdkmanager --list`. The logic is:
  115. # Check through 'Installed packages' which is at the first section of
  116. # `sdkmanager --list` output, example:
  117. # Installed packages:=====================] 100% Computing updates...
  118. # Path | Version | Description
  119. # ------- | ------- | -------
  120. # build-tools;27.0.3 | 27.0.3 | Android SDK Build-Tools 27.0.3
  121. # emulator | 26.0.3 | Android Emulator
  122. # platforms;android-27 | 1 | Android SDK Platform 27
  123. # tools | 26.1.1 | Android SDK Tools
  124. #
  125. # Available Packages:
  126. # ....
  127. # When found a line containing the package path, grap its version between
  128. # the first and second '|'. Since the 'Installed packages' list section ends
  129. # by the first new line, the check loop should be ended when reaches a '\n'.
  130. output = subprocess.check_output(sdkmanager_list_cmd)
  131. for line in output.splitlines():
  132. if ' ' + package + ' ' in line:
  133. # if found package path, catch its version which in the first '|...|'
  134. return line.split('|')[1].strip()
  135. if line == '\n': # Reaches the end of 'Installed packages' list
  136. break
  137. raise Exception('Cannot find the version of package %s' % package)
  138. def _ReplaceVersionInFile(file_path, pattern, version, dry_run=False):
  139. """Replace the version of sdk package argument in file.
  140. Check whether the version in file is the same as the new version first.
  141. Replace the version if not dry run.
  142. Args:
  143. file_path: Path to the file to update the version of sdk package argument.
  144. pattern: Pattern for the sdk package argument. Must capture at least one
  145. group that the first group is the argument line excluding version.
  146. version: The new version of the package.
  147. dry_run: Bool. To show what packages would be created and packages, without
  148. actually doing either.
  149. """
  150. with tempfile.NamedTemporaryFile() as temp_file:
  151. with open(file_path) as f:
  152. for line in f:
  153. new_line = re.sub(pattern, r'\g<1>\g<2>%s\g<3>\n' % version, line)
  154. if new_line != line:
  155. print(' Note: file "%s" argument ' % file_path +
  156. '"%s" would be updated to "%s".' % (line.strip(), version))
  157. temp_file.write(new_line)
  158. if not dry_run:
  159. temp_file.flush()
  160. shutil.move(temp_file.name, file_path)
  161. temp_file.delete = False
  162. def GetCipdPackagePath(pkg_yaml_file):
  163. """Find CIPD package path in .yaml file.
  164. There should one line in .yaml file, e.g.:
  165. "package: chrome_internal/third_party/android_sdk/internal/q/add-ons" or
  166. "package: chromium/third_party/android_sdk/public/platforms"
  167. Args:
  168. pkg_yaml_file: The yaml file to find CIPD package path.
  169. Returns:
  170. The CIPD package path in yaml file.
  171. """
  172. cipd_package_path = ''
  173. with open(pkg_yaml_file) as f:
  174. pattern = re.compile(
  175. # Match the argument with "package: "
  176. r'(^\s*package:\s*)'
  177. # The CIPD package path we want
  178. r'([\w\/-]+)'
  179. # End of string
  180. r'(\s*?$)')
  181. for line in f:
  182. found = re.match(pattern, line)
  183. if found:
  184. cipd_package_path = found.group(2)
  185. break
  186. return cipd_package_path
  187. def UploadSdkPackage(sdk_root, dry_run, service_url, package, yaml_file,
  188. verbose):
  189. """Build and upload a package instance file to CIPD.
  190. This would also update gn and ensure files to the package version as
  191. uploading to CIPD.
  192. Args:
  193. sdk_root: Root of the sdk packages.
  194. dry_run: Bool. To show what packages would be created and packages, without
  195. actually doing either.
  196. service_url: The url of the CIPD service.
  197. package: The package to be uploaded to CIPD.
  198. yaml_file: Path to the yaml file that defines what to put into the package.
  199. Default as //third_party/android_sdk/public/cipd_*.yaml
  200. verbose: Enable more logging.
  201. Returns:
  202. New instance ID when CIPD package created.
  203. Raises:
  204. IOError: cannot find .yaml file, CIPD package path or instance ID for
  205. package.
  206. CalledProcessError: cipd command failed to create package.
  207. """
  208. pkg_yaml_file = yaml_file or os.path.join(sdk_root, 'cipd_%s.yaml' % package)
  209. if not os.path.exists(pkg_yaml_file):
  210. raise IOError('Cannot find .yaml file for package %s' % package)
  211. cipd_package_path = GetCipdPackagePath(pkg_yaml_file)
  212. if not cipd_package_path:
  213. raise IOError('Cannot find CIPD package path in %s' % pkg_yaml_file)
  214. if dry_run:
  215. print('This `package` command (without -n/--dry-run) would create and ' +
  216. 'upload the package %s to CIPD.' % package)
  217. else:
  218. upload_sdk_cmd = [
  219. 'cipd', 'create', '-pkg-def', pkg_yaml_file, '-service-url', service_url
  220. ]
  221. if verbose:
  222. upload_sdk_cmd.extend(['-log-level', 'debug'])
  223. output = subprocess.check_output(upload_sdk_cmd)
  224. # Need to match pattern to find new instance ID.
  225. # e.g.: chromium/third_party/android_sdk/public/platforms:\
  226. # Kg2t9p0YnQk8bldUv4VA3o156uPXLUfIFAmVZ-Gm5ewC
  227. pattern = re.compile(
  228. # Match the argument with "Instance: %s:" for cipd_package_path
  229. (r'(^\s*Instance: %s:)' % cipd_package_path) +
  230. # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
  231. r'([-\w.]+)'
  232. # End of string
  233. r'(\s*?$)')
  234. for line in output.splitlines():
  235. found = re.match(pattern, line)
  236. if found:
  237. # Return new instance ID.
  238. return found.group(2)
  239. # Raises error if instance ID not found.
  240. raise IOError('Cannot find instance ID by creating package %s' % package)
  241. def UpdateInstanceId(package,
  242. deps_path,
  243. dry_run,
  244. new_instance_id,
  245. release_version=None):
  246. """Find the sdk pkg version in DEPS and modify it as cipd uploading version.
  247. TODO(shenghuazhang): use DEPS edition operations after issue crbug.com/760633
  248. fixed.
  249. DEPS file hooks sdk package with version with suffix -crX, e.g. '26.0.2-cr1'.
  250. If pkg_version is the base number of the existing version in DEPS, e.g.
  251. '26.0.2', return '26.0.2-cr2' as the version uploading to CIPD. If not the
  252. base number, return ${pkg_version}-cr0.
  253. Args:
  254. package: The name of the package.
  255. deps_path: Path to deps file which gclient hooks sdk pkg w/ versions.
  256. dry_run: Bool. To show what packages would be created and packages, without
  257. actually doing either.
  258. new_instance_id: New instance ID after CIPD package created.
  259. release_version: Android sdk release version e.g. 'o_mr1', 'p'.
  260. """
  261. var_package = package
  262. if release_version:
  263. var_package = release_version + '_' + var_package
  264. package_var_pattern = re.compile(
  265. # Match the argument with "'android_sdk_*_version': '" with whitespaces.
  266. r'(^\s*\'android_sdk_%s_version\'\s*:\s*\')' % var_package +
  267. # instance ID e.g. DLK621q5_Bga5EsOr7cp6bHWWxFKx6UHLu_Ix_m3AckC.
  268. r'([-\w.]+)'
  269. # End of string
  270. r'(\',?$)')
  271. with tempfile.NamedTemporaryFile() as temp_file:
  272. with open(deps_path) as f:
  273. for line in f:
  274. new_line = line
  275. found = re.match(package_var_pattern, line)
  276. if found:
  277. instance_id = found.group(2)
  278. new_line = re.sub(package_var_pattern,
  279. r'\g<1>%s\g<3>' % new_instance_id, line)
  280. print(
  281. ' Note: deps file "%s" argument ' % deps_path +
  282. '"%s" would be updated to "%s".' % (instance_id, new_instance_id))
  283. temp_file.write(new_line)
  284. if not dry_run:
  285. temp_file.flush()
  286. shutil.move(temp_file.name, deps_path)
  287. temp_file.delete = False
  288. def ChangeVersionInGNI(package, arg_version, gn_args_dict, gni_file_path,
  289. dry_run):
  290. """Change the sdk package version in config.gni file."""
  291. if package in gn_args_dict:
  292. version_config_name = gn_args_dict.get(package)
  293. # Regex to parse the line of sdk package version gn argument, e.g.
  294. # ' default_android_sdk_version = "27"'. Capture a group for the line
  295. # excluding the version.
  296. gn_arg_pattern = re.compile(
  297. # Match the argument with '=' and whitespaces. Capture a group for it.
  298. r'(^\s*%s\s*=\s*)' % version_config_name +
  299. # Optional quote.
  300. r'("?)' +
  301. # Version number. E.g. 27, 27.0.3, -26.0.0-dev
  302. r'(?:[-\w\s.]+)' +
  303. # Optional quote.
  304. r'("?)' +
  305. # End of string
  306. r'$')
  307. _ReplaceVersionInFile(gni_file_path, gn_arg_pattern, arg_version, dry_run)
  308. def GetToolsSuffix(tools_lib_path):
  309. """Get the gn config of package 'tools' suffix.
  310. Check jar file name of 'common*.jar' in tools/lib, which could be
  311. 'common.jar', common-<version>-dev.jar' or 'common-<version>.jar'.
  312. If suffix exists, return the suffix.
  313. Args:
  314. tools_lib_path: The path of tools/lib.
  315. Returns:
  316. The suffix of tools package.
  317. """
  318. tools_lib_jars_list = os.listdir(tools_lib_path)
  319. for file_name in tools_lib_jars_list:
  320. found = re.match(_COMMON_JAR_SUFFIX_PATTERN, file_name)
  321. if found:
  322. return found.group(1)
  323. def _GetArgVersion(pkg_version, package):
  324. """Get the argument version.
  325. Args:
  326. pkg_version: The package version.
  327. package: The package name.
  328. Returns:
  329. The argument version.
  330. """
  331. # Remove all chars except for digits and dots in version
  332. arg_version = re.sub(r'[^\d\.]', '', pkg_version)
  333. if package == 'tools':
  334. suffix = GetToolsSuffix(_TOOLS_LIB_PATH)
  335. if suffix:
  336. arg_version = suffix
  337. else:
  338. arg_version = '-%s' % arg_version
  339. return arg_version
  340. def _UploadSdkPackage(arguments):
  341. """Upload SDK packages to CIPD.
  342. Args:
  343. arguments: The arguments parsed by argparser.
  344. Raises:
  345. IOError: Don't use --version/--yaml-file for default packages.
  346. """
  347. packages = arguments.package
  348. if not packages:
  349. packages = _DEFAULT_DOWNLOAD_PACKAGES
  350. if arguments.version or arguments.yaml_file:
  351. raise IOError("Don't use --version/--yaml-file for default packages.")
  352. for package in packages:
  353. pkg_version = arguments.version
  354. if not pkg_version:
  355. pkg_version = _FindPackageVersion(package, arguments.sdk_root)
  356. # Upload SDK package to CIPD, and update the package instance ID hooking
  357. # in DEPS file.
  358. new_instance_id = UploadSdkPackage(
  359. os.path.join(arguments.sdk_root, '..'), arguments.dry_run,
  360. arguments.service_url, package, arguments.yaml_file, arguments.verbose)
  361. UpdateInstanceId(package, _SRC_DEPS_PATH, arguments.dry_run,
  362. new_instance_id)
  363. if package in _GN_ARGUMENTS_TO_UPDATE:
  364. # Update the package version config in gn file
  365. arg_version = _GetArgVersion(pkg_version, package)
  366. ChangeVersionInGNI(package, arg_version, _GN_ARGUMENTS_TO_UPDATE,
  367. _ANDROID_CONFIG_GNI_PATH, arguments.dry_run)
  368. def main():
  369. parser = argparse.ArgumentParser(
  370. description='A script to download Android SDK packages '
  371. 'via sdkmanager and upload to CIPD.')
  372. subparsers = parser.add_subparsers(title='commands')
  373. download_parser = subparsers.add_parser(
  374. 'download',
  375. help='Download sdk package to the latest version from sdkmanager.')
  376. download_parser.set_defaults(func=_DownloadSdk)
  377. download_parser.add_argument(
  378. '-p',
  379. '--package',
  380. nargs=1,
  381. default=_DEFAULT_DOWNLOAD_PACKAGES,
  382. help='The package of the SDK needs to be installed/updated. '
  383. 'Note that package name should be a sdk-style path e.g. '
  384. '"platforms;android-27" or "platform-tools". If package '
  385. 'is not specified, update "build-tools;27.0.3", "tools" '
  386. '"platform-tools" and "platforms;android-27" by default.')
  387. download_parser.add_argument(
  388. '--sdk-root', help='base path to the Android SDK root')
  389. download_parser.add_argument(
  390. '-v', '--verbose', action='store_true', help='print debug information')
  391. package_parser = subparsers.add_parser(
  392. 'package', help='Create and upload package instance file to CIPD.')
  393. package_parser.set_defaults(func=_UploadSdkPackage)
  394. package_parser.add_argument(
  395. '-n',
  396. '--dry-run',
  397. action='store_true',
  398. help='Dry run won\'t trigger creating instances or uploading packages. '
  399. 'It shows what packages would be created and uploaded to CIPD. '
  400. 'It also shows the possible updates of sdk version on files.')
  401. package_parser.add_argument(
  402. '-p',
  403. '--package',
  404. nargs=1,
  405. help='The package to be uploaded to CIPD. Note that package '
  406. 'name is a simple path e.g. "platforms" or "build-tools" '
  407. 'which matches package name on CIPD service. Default by '
  408. 'build-tools, platform-tools, platforms and tools')
  409. package_parser.add_argument(
  410. '--version',
  411. help='Version of the uploading package instance through CIPD.')
  412. package_parser.add_argument(
  413. '--yaml-file',
  414. help='Path to *.yaml file that defines what to put into the package.'
  415. 'Default as //third_party/android_sdk/public/cipd_<package>.yaml')
  416. package_parser.add_argument(
  417. '--service-url',
  418. help='The url of the CIPD service.',
  419. default='https://chrome-infra-packages.appspot.com')
  420. package_parser.add_argument(
  421. '--sdk-root', help='base path to the Android SDK root')
  422. package_parser.add_argument(
  423. '-v', '--verbose', action='store_true', help='print debug information')
  424. args = parser.parse_args()
  425. if not args.sdk_root:
  426. args.sdk_root = _SDK_PUBLIC_ROOT
  427. args.func(args)
  428. if __name__ == '__main__':
  429. sys.exit(main())