update_images.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. #!/usr/bin/env python3
  2. # Copyright 2020 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Updates the Fuchsia images to the given revision. Should be used in a
  6. 'hooks_os' entry so that it only runs when .gclient's target_os includes
  7. 'fuchsia'."""
  8. import argparse
  9. import itertools
  10. import logging
  11. import os
  12. import re
  13. import shutil
  14. import subprocess
  15. import sys
  16. import tarfile
  17. from common import GetHostOsFromPlatform, GetHostArchFromPlatform, \
  18. DIR_SOURCE_ROOT, IMAGES_ROOT
  19. sys.path.append(os.path.join(DIR_SOURCE_ROOT, 'build'))
  20. import find_depot_tools
  21. IMAGE_SIGNATURE_FILE = '.hash'
  22. def DownloadAndUnpackFromCloudStorage(url, output_dir):
  23. """Fetches a tarball from GCS and uncompresses it to |output_dir|."""
  24. # Pass the compressed stream directly to 'tarfile'; don't bother writing it
  25. # to disk first.
  26. cmd = [
  27. os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'), 'cp', url,
  28. '-'
  29. ]
  30. logging.debug('Running "%s"', ' '.join(cmd))
  31. task = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
  32. try:
  33. tarfile.open(mode='r|gz', fileobj=task.stdout).extractall(path=output_dir)
  34. except tarfile.ReadError as exc:
  35. task.wait()
  36. stderr = task.stderr.read()
  37. raise subprocess.CalledProcessError(
  38. task.returncode, cmd,
  39. "Failed to read a tarfile from gsutil.py.\n{}".format(
  40. stderr if stderr else "")) from exc
  41. task.wait()
  42. if task.returncode:
  43. raise subprocess.CalledProcessError(task.returncode, cmd,
  44. task.stderr.read())
  45. # TODO(crbug.com/1138433): Investigate whether we can deprecate
  46. # use of sdk_bucket.txt.
  47. def GetOverrideCloudStorageBucket():
  48. """Read bucket entry from sdk_bucket.txt"""
  49. return ReadFile('sdk-bucket.txt').strip()
  50. def MakeCleanDirectory(directory_name):
  51. if (os.path.exists(directory_name)):
  52. shutil.rmtree(directory_name)
  53. os.mkdir(directory_name)
  54. def ReadFile(filename):
  55. """Read a file in this directory."""
  56. with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f:
  57. return f.read()
  58. def StrExpansion():
  59. return lambda str_value: str_value
  60. def VarLookup(local_scope):
  61. return lambda var_name: local_scope['vars'][var_name]
  62. def GetImageHashList(bucket):
  63. """Read filename entries from sdk-hash-files.list (one per line), substitute
  64. {platform} in each entry if present, and read from each filename."""
  65. assert (GetHostOsFromPlatform() == 'linux')
  66. filenames = [
  67. line.strip() for line in ReadFile('sdk-hash-files.list').replace(
  68. '{platform}', 'linux_internal').splitlines()
  69. ]
  70. image_hashes = [ReadFile(filename).strip() for filename in filenames]
  71. return image_hashes
  72. def ParseDepsDict(deps_content):
  73. local_scope = {}
  74. global_scope = {
  75. 'Str': StrExpansion(),
  76. 'Var': VarLookup(local_scope),
  77. 'deps_os': {},
  78. }
  79. exec(deps_content, global_scope, local_scope)
  80. return local_scope
  81. def ParseDepsFile(filename):
  82. with open(filename, 'rb') as f:
  83. deps_content = f.read()
  84. return ParseDepsDict(deps_content)
  85. def GetImageHash(bucket):
  86. """Gets the hash identifier of the newest generation of images."""
  87. if bucket == 'fuchsia-sdk':
  88. hashes = GetImageHashList(bucket)
  89. return max(hashes)
  90. deps_file = os.path.join(DIR_SOURCE_ROOT, 'DEPS')
  91. return ParseDepsFile(deps_file)['vars']['fuchsia_version'].split(':')[1]
  92. def GetImageSignature(image_hash, boot_images):
  93. return 'gn:{image_hash}:{boot_images}:'.format(image_hash=image_hash,
  94. boot_images=boot_images)
  95. def GetAllImages(boot_image_names):
  96. if not boot_image_names:
  97. return
  98. all_device_types = ['generic', 'qemu']
  99. all_archs = ['x64', 'arm64']
  100. images_to_download = set()
  101. for boot_image in boot_image_names.split(','):
  102. components = boot_image.split('.')
  103. if len(components) != 2:
  104. continue
  105. device_type, arch = components
  106. device_images = all_device_types if device_type == '*' else [device_type]
  107. arch_images = all_archs if arch == '*' else [arch]
  108. images_to_download.update(itertools.product(device_images, arch_images))
  109. return images_to_download
  110. def DownloadBootImages(bucket, image_hash, boot_image_names, image_root_dir):
  111. images_to_download = GetAllImages(boot_image_names)
  112. for image_to_download in images_to_download:
  113. device_type = image_to_download[0]
  114. arch = image_to_download[1]
  115. image_output_dir = os.path.join(image_root_dir, arch, device_type)
  116. if os.path.exists(image_output_dir):
  117. continue
  118. logging.info('Downloading Fuchsia boot images for %s.%s...' %
  119. (device_type, arch))
  120. # Legacy images use different naming conventions. See fxbug.dev/85552.
  121. legacy_delimiter_device_types = ['qemu', 'generic']
  122. if bucket == 'fuchsia-sdk' or \
  123. device_type not in legacy_delimiter_device_types:
  124. type_arch_connector = '.'
  125. else:
  126. type_arch_connector = '-'
  127. images_tarball_url = 'gs://{bucket}/development/{image_hash}/images/'\
  128. '{device_type}{type_arch_connector}{arch}.tgz'.format(
  129. bucket=bucket, image_hash=image_hash, device_type=device_type,
  130. type_arch_connector=type_arch_connector, arch=arch)
  131. try:
  132. DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir)
  133. except subprocess.CalledProcessError as e:
  134. logging.exception('Failed to download image %s from URL: %s',
  135. image_to_download, images_tarball_url)
  136. def main():
  137. parser = argparse.ArgumentParser()
  138. parser.add_argument('--verbose',
  139. '-v',
  140. action='store_true',
  141. help='Enable debug-level logging.')
  142. parser.add_argument(
  143. '--boot-images',
  144. type=str,
  145. required=True,
  146. help='List of boot images to download, represented as a comma separated '
  147. 'list. Wildcards are allowed. ')
  148. parser.add_argument(
  149. '--default-bucket',
  150. type=str,
  151. default='fuchsia',
  152. help='The Google Cloud Storage bucket in which the Fuchsia images are '
  153. 'stored. Entry in sdk-bucket.txt will override this flag.')
  154. parser.add_argument(
  155. '--image-root-dir',
  156. default=IMAGES_ROOT,
  157. help='Specify the root directory of the downloaded images. Optional')
  158. args = parser.parse_args()
  159. logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
  160. # If no boot images need to be downloaded, exit.
  161. if not args.boot_images:
  162. return 0
  163. # Check whether there's Fuchsia support for this platform.
  164. GetHostOsFromPlatform()
  165. # Use the bucket in sdk-bucket.txt if an entry exists.
  166. # Otherwise use the default bucket.
  167. bucket = GetOverrideCloudStorageBucket() or args.default_bucket
  168. image_hash = GetImageHash(bucket)
  169. if not image_hash:
  170. return 1
  171. signature_filename = os.path.join(args.image_root_dir, IMAGE_SIGNATURE_FILE)
  172. current_signature = (open(signature_filename, 'r').read().strip()
  173. if os.path.exists(signature_filename) else '')
  174. new_signature = GetImageSignature(image_hash, args.boot_images)
  175. if current_signature != new_signature:
  176. logging.info('Downloading Fuchsia images %s...' % image_hash)
  177. MakeCleanDirectory(args.image_root_dir)
  178. try:
  179. DownloadBootImages(bucket, image_hash, args.boot_images,
  180. args.image_root_dir)
  181. with open(signature_filename, 'w') as f:
  182. f.write(new_signature)
  183. except subprocess.CalledProcessError as e:
  184. logging.error(("command '%s' failed with status %d.%s"), " ".join(e.cmd),
  185. e.returncode, " Details: " + e.output if e.output else "")
  186. return 0
  187. if __name__ == '__main__':
  188. sys.exit(main())