pkg_repo.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. # Copyright 2019 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. import common
  5. import json
  6. import logging
  7. import os
  8. import shutil
  9. import subprocess
  10. import tempfile
  11. import time
  12. # Maximum amount of time to block while waiting for "pm serve" to come up.
  13. _PM_SERVE_LISTEN_TIMEOUT_SECS = 10
  14. # Amount of time to sleep in between busywaits for "pm serve"'s port file.
  15. _PM_SERVE_POLL_INTERVAL = 0.1
  16. _MANAGED_REPO_NAME = 'chromium-test-package-server'
  17. _HOSTS = ['fuchsia.com', 'chrome.com', 'chromium.org']
  18. class PkgRepo(object):
  19. """Abstract interface for a repository used to serve packages to devices."""
  20. def __init__(self):
  21. pass
  22. def PublishPackage(self, package_path):
  23. pm_tool = common.GetHostToolPathFromPlatform('pm')
  24. # Flags for `pm publish`:
  25. # https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/src/sys/pkg/bin/pm/cmd/pm/publish/publish.go
  26. # https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/src/sys/pkg/bin/pm/repo/config.go
  27. # -a: Publish archived package
  28. # -f <path>: Path to packages
  29. # -r <path>: Path to repository
  30. # -vt: Repo versioning based on time rather than monotonic version number
  31. # increase
  32. # -v: Verbose output
  33. subprocess.check_call([
  34. pm_tool, 'publish', '-a', '-f', package_path, '-r',
  35. self.GetPath(), '-vt', '-v'
  36. ], stderr=subprocess.STDOUT)
  37. def GetPath(self):
  38. pass
  39. class ManagedPkgRepo(PkgRepo):
  40. """Creates and serves packages from an ephemeral repository."""
  41. def __init__(self, target):
  42. super(ManagedPkgRepo, self).__init__()
  43. self._with_count = 0
  44. self._target = target
  45. self._pkg_root = tempfile.mkdtemp()
  46. pm_tool = common.GetHostToolPathFromPlatform('pm')
  47. subprocess.check_call([pm_tool, 'newrepo', '-repo', self._pkg_root])
  48. logging.debug('Creating and serving temporary package root: {}.'.format(
  49. self._pkg_root))
  50. with tempfile.NamedTemporaryFile() as pm_port_file:
  51. # Flags for `pm serve`:
  52. # https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/src/sys/pkg/bin/pm/cmd/pm/serve/serve.go
  53. self._pm_serve_task = subprocess.Popen([
  54. pm_tool, 'serve',
  55. '-d', os.path.join(self._pkg_root, 'repository'),
  56. '-c', '2', # Use config.json format v2, the default for pkgctl.
  57. '-q', # Don't log transfer activity.
  58. '-l', ':0', # Bind to ephemeral port.
  59. '-f', pm_port_file.name # Publish port number to |pm_port_file|.
  60. ]) # yapf: disable
  61. # Busywait until 'pm serve' starts the server and publishes its port to
  62. # a temporary file.
  63. timeout = time.time() + _PM_SERVE_LISTEN_TIMEOUT_SECS
  64. serve_port = None
  65. while not serve_port:
  66. if time.time() > timeout:
  67. raise Exception(
  68. 'Timeout waiting for \'pm serve\' to publish its port.')
  69. with open(pm_port_file.name, 'r', encoding='utf8') as serve_port_file:
  70. serve_port = serve_port_file.read()
  71. time.sleep(_PM_SERVE_POLL_INTERVAL)
  72. serve_port = int(serve_port)
  73. logging.debug('pm serve is active on port {}.'.format(serve_port))
  74. remote_port = common.ConnectPortForwardingTask(target, serve_port, 0)
  75. self._RegisterPkgRepository(self._pkg_root, remote_port)
  76. def __enter__(self):
  77. self._with_count += 1
  78. return self
  79. def __exit__(self, type, value, tb):
  80. # Allows the repository to delete itself when it leaves the scope of a
  81. # 'with' block.
  82. self._with_count -= 1
  83. if self._with_count > 0:
  84. return
  85. self._UnregisterPkgRepository()
  86. self._pm_serve_task.kill()
  87. self._pm_serve_task = None
  88. logging.info('Cleaning up package root: ' + self._pkg_root)
  89. shutil.rmtree(self._pkg_root)
  90. self._pkg_root = None
  91. def GetPath(self):
  92. return self._pkg_root
  93. def _RegisterPkgRepository(self, tuf_repo, remote_port):
  94. """Configures a device to use a local TUF repository as an installation
  95. source for packages.
  96. |tuf_repo|: The host filesystem path to the TUF repository.
  97. |remote_port|: The reverse-forwarded port used to connect to instance of
  98. `pm serve` that is serving the contents of |tuf_repo|."""
  99. # Extract the public signing key for inclusion in the config file.
  100. root_keys = []
  101. root_json_path = os.path.join(tuf_repo, 'repository', 'root.json')
  102. root_json = json.load(open(root_json_path, 'r'))
  103. for root_key_id in root_json['signed']['roles']['root']['keyids']:
  104. root_keys.append({
  105. 'type':
  106. root_json['signed']['keys'][root_key_id]['keytype'],
  107. 'value':
  108. root_json['signed']['keys'][root_key_id]['keyval']['public']
  109. })
  110. # "pm serve" can automatically generate a "config.json" file at query time,
  111. # but the file is unusable because it specifies URLs with port
  112. # numbers that are unreachable from across the port forwarding boundary.
  113. # So instead, we generate our own config file with the forwarded port
  114. # numbers instead.
  115. config_file = open(os.path.join(tuf_repo, 'repository', 'repo_config.json'),
  116. 'w')
  117. json.dump(
  118. {
  119. 'repo_url':
  120. 'fuchsia-pkg://{}'.format(_MANAGED_REPO_NAME),
  121. 'root_keys':
  122. root_keys,
  123. 'mirrors': [{
  124. "mirror_url": 'http://127.0.0.1:{}'.format(remote_port),
  125. "subscribe": True
  126. }],
  127. 'root_threshold':
  128. 1,
  129. 'root_version':
  130. 1
  131. }, config_file)
  132. config_file.close()
  133. # Register the repo.
  134. return_code = self._target.RunCommand([
  135. ('pkgctl repo rm fuchsia-pkg://{}; ' +
  136. 'pkgctl repo add url http://127.0.0.1:{}/repo_config.json; ').format(
  137. _MANAGED_REPO_NAME, remote_port)
  138. ])
  139. if return_code != 0:
  140. raise Exception(
  141. 'Error code {} when running pkgctl repo add.'.format(return_code))
  142. self._AddHostReplacementRule(_MANAGED_REPO_NAME)
  143. def _UnregisterPkgRepository(self):
  144. """Unregisters the package repository."""
  145. logging.debug('Unregistering package repository.')
  146. self._target.RunCommand(
  147. ['pkgctl', 'repo', 'rm', 'fuchsia-pkg://{}'.format(_MANAGED_REPO_NAME)])
  148. # Re-enable 'devhost' repo if it's present. This is useful for devices that
  149. # were booted with 'fx serve'.
  150. self._AddHostReplacementRule('devhost', silent=True)
  151. def _AddHostReplacementRule(self, host_replacement, silent=False):
  152. rule = json.dumps({
  153. 'version':
  154. '1',
  155. 'content': [{
  156. 'host_match': host,
  157. 'host_replacement': host_replacement,
  158. 'path_prefix_match': '/',
  159. 'path_prefix_replacement': '/'
  160. } for host in _HOSTS]
  161. })
  162. return_code = self._target.RunCommand(
  163. ['pkgctl', 'rule', 'replace', 'json', "'{}'".format(rule)], silent)
  164. if not silent and return_code != 0:
  165. raise Exception(
  166. 'Error code {} when running pkgctl rule replace with {}'.format(
  167. return_code, rule))
  168. class ExternalPkgRepo(PkgRepo):
  169. """Publishes packages to a package repository located and served externally
  170. (ie. located under a Fuchsia build directory and served by "fx serve"."""
  171. def __init__(self, pkg_root, symbol_root):
  172. super(PkgRepo, self).__init__()
  173. self._pkg_root = pkg_root
  174. self._symbol_root = symbol_root
  175. logging.info('Using existing package root: {}'.format(pkg_root))
  176. logging.info('ATTENTION: This will not start a package server. ' +
  177. 'Please run "fx serve" manually.')
  178. def GetPath(self):
  179. return self._pkg_root
  180. def PublishPackage(self, package_path):
  181. super(ExternalPkgRepo, self).PublishPackage(package_path)
  182. self._InstallSymbols(os.path.join(os.path.dirname(package_path), 'ids.txt'))
  183. def __enter__(self):
  184. return self
  185. def __exit__(self, type, value, tb):
  186. pass
  187. def _InstallSymbols(self, package_path):
  188. """Installs debug symbols for a packageinto the GDB-standard symbol
  189. directory located at |self.symbol_root|."""
  190. ids_txt_path = os.path.join(os.path.dirname(package_path), 'ids.txt')
  191. for entry in open(ids_txt_path, 'r'):
  192. build_id, binary_relpath = entry.strip().split(' ')
  193. binary_abspath = os.path.abspath(
  194. os.path.join(os.path.dirname(ids_txt_path), binary_relpath))
  195. symbol_dir = os.path.join(self._symbol_root, build_id[:2])
  196. symbol_file = os.path.join(symbol_dir, build_id[2:] + '.debug')
  197. if not os.path.exists(symbol_dir):
  198. os.makedirs(symbol_dir)
  199. if os.path.islink(symbol_file) or os.path.exists(symbol_file):
  200. # Clobber the existing entry to ensure that the symlink's target is
  201. # up to date.
  202. os.unlink(symbol_file)
  203. os.symlink(os.path.relpath(binary_abspath, symbol_dir), symbol_file)