recipes.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. #!/bin/sh
  2. # Copyright 2019 The LUCI Authors. All rights reserved.
  3. # Use of this source code is governed under the Apache License, Version 2.0
  4. # that can be found in the LICENSE file.
  5. # We want to run python in unbuffered mode; however shebangs on linux grab the
  6. # entire rest of the shebang line as a single argument, leading to errors like:
  7. #
  8. # /usr/bin/env: 'python -u': No such file or directory
  9. #
  10. # This little shell hack is a triple-quoted noop in python, but in sh it
  11. # evaluates to re-exec'ing this script in unbuffered mode.
  12. # pylint: disable=pointless-string-statement
  13. ''''exec python -u -- "$0" ${1+"$@"} # '''
  14. # vi: syntax=python
  15. """Bootstrap script to clone and forward to the recipe engine tool.
  16. *******************
  17. ** DO NOT MODIFY **
  18. *******************
  19. This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/recipes.py.
  20. To fix bugs, fix in the googlesource repo then run the autoroller.
  21. """
  22. # pylint: disable=wrong-import-position
  23. import argparse
  24. import json
  25. import logging
  26. import os
  27. import subprocess
  28. import sys
  29. import urlparse
  30. from collections import namedtuple
  31. # The dependency entry for the recipe_engine in the client repo's recipes.cfg
  32. #
  33. # url (str) - the url to the engine repo we want to use.
  34. # revision (str) - the git revision for the engine to get.
  35. # branch (str) - the branch to fetch for the engine as an absolute ref (e.g.
  36. # refs/heads/master)
  37. EngineDep = namedtuple('EngineDep', 'url revision branch')
  38. class MalformedRecipesCfg(Exception):
  39. def __init__(self, msg, path):
  40. full_message = 'malformed recipes.cfg: %s: %r' % (msg, path)
  41. super(MalformedRecipesCfg, self).__init__(full_message)
  42. def parse(repo_root, recipes_cfg_path):
  43. """Parse is a lightweight a recipes.cfg file parser.
  44. Args:
  45. repo_root (str) - native path to the root of the repo we're trying to run
  46. recipes for.
  47. recipes_cfg_path (str) - native path to the recipes.cfg file to process.
  48. Returns (as tuple):
  49. engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the
  50. current repo IS the recipe_engine.
  51. recipes_path (str) - native path to where the recipes live inside of the
  52. current repo (i.e. the folder containing `recipes/` and/or
  53. `recipe_modules`)
  54. """
  55. with open(recipes_cfg_path, 'rU') as fh:
  56. pb = json.load(fh)
  57. try:
  58. if pb['api_version'] != 2:
  59. raise MalformedRecipesCfg('unknown version %d' % pb['api_version'],
  60. recipes_cfg_path)
  61. # If we're running ./recipes.py from the recipe_engine repo itself, then
  62. # return None to signal that there's no EngineDep.
  63. repo_name = pb.get('repo_name')
  64. if not repo_name:
  65. repo_name = pb['project_id']
  66. if repo_name == 'recipe_engine':
  67. return None, pb.get('recipes_path', '')
  68. engine = pb['deps']['recipe_engine']
  69. if 'url' not in engine:
  70. raise MalformedRecipesCfg(
  71. 'Required field "url" in dependency "recipe_engine" not found',
  72. recipes_cfg_path)
  73. engine.setdefault('revision', '')
  74. engine.setdefault('branch', 'refs/heads/master')
  75. recipes_path = pb.get('recipes_path', '')
  76. # TODO(iannucci): only support absolute refs
  77. if not engine['branch'].startswith('refs/'):
  78. engine['branch'] = 'refs/heads/' + engine['branch']
  79. recipes_path = os.path.join(repo_root,
  80. recipes_path.replace('/', os.path.sep))
  81. return EngineDep(**engine), recipes_path
  82. except KeyError as ex:
  83. raise MalformedRecipesCfg(ex.message, recipes_cfg_path)
  84. _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else ''
  85. GIT = 'git' + _BAT
  86. VPYTHON = 'vpython' + _BAT
  87. CIPD = 'cipd' + _BAT
  88. REQUIRED_BINARIES = {GIT, VPYTHON, CIPD}
  89. def _is_executable(path):
  90. return os.path.isfile(path) and os.access(path, os.X_OK)
  91. # TODO: Use shutil.which once we switch to Python3.
  92. def _is_on_path(basename):
  93. for path in os.environ['PATH'].split(os.pathsep):
  94. full_path = os.path.join(path, basename)
  95. if _is_executable(full_path):
  96. return True
  97. return False
  98. def _subprocess_call(argv, **kwargs):
  99. logging.info('Running %r', argv)
  100. return subprocess.call(argv, **kwargs)
  101. def _git_check_call(argv, **kwargs):
  102. argv = [GIT] + argv
  103. logging.info('Running %r', argv)
  104. subprocess.check_call(argv, **kwargs)
  105. def _git_output(argv, **kwargs):
  106. argv = [GIT] + argv
  107. logging.info('Running %r', argv)
  108. return subprocess.check_output(argv, **kwargs)
  109. def parse_args(argv):
  110. """This extracts a subset of the arguments that this bootstrap script cares
  111. about. Currently this consists of:
  112. * an override for the recipe engine in the form of `-O recipe_engine=/path`
  113. * the --package option.
  114. """
  115. PREFIX = 'recipe_engine='
  116. p = argparse.ArgumentParser(add_help=False)
  117. p.add_argument('-O', '--project-override', action='append')
  118. p.add_argument('--package', type=os.path.abspath)
  119. args, _ = p.parse_known_args(argv)
  120. for override in args.project_override or ():
  121. if override.startswith(PREFIX):
  122. return override[len(PREFIX):], args.package
  123. return None, args.package
  124. def checkout_engine(engine_path, repo_root, recipes_cfg_path):
  125. dep, recipes_path = parse(repo_root, recipes_cfg_path)
  126. if dep is None:
  127. # we're running from the engine repo already!
  128. return os.path.join(repo_root, recipes_path)
  129. url = dep.url
  130. if not engine_path and url.startswith('file://'):
  131. engine_path = urlparse.urlparse(url).path
  132. if not engine_path:
  133. revision = dep.revision
  134. branch = dep.branch
  135. # Ensure that we have the recipe engine cloned.
  136. engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine')
  137. with open(os.devnull, 'w') as NUL:
  138. # Note: this logic mirrors the logic in recipe_engine/fetch.py
  139. _git_check_call(['init', engine_path], stdout=NUL)
  140. try:
  141. _git_check_call(['rev-parse', '--verify',
  142. '%s^{commit}' % revision],
  143. cwd=engine_path,
  144. stdout=NUL,
  145. stderr=NUL)
  146. except subprocess.CalledProcessError:
  147. _git_check_call(['fetch', url, branch],
  148. cwd=engine_path,
  149. stdout=NUL,
  150. stderr=NUL)
  151. try:
  152. _git_check_call(['diff', '--quiet', revision], cwd=engine_path)
  153. except subprocess.CalledProcessError:
  154. _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path)
  155. # If the engine has refactored/moved modules we need to clean all .pyc files
  156. # or things will get squirrely.
  157. _git_check_call(['clean', '-qxf'], cwd=engine_path)
  158. return engine_path
  159. def main():
  160. for required_binary in REQUIRED_BINARIES:
  161. if not _is_on_path(required_binary):
  162. return 'Required binary is not found on PATH: %s' % required_binary
  163. if '--verbose' in sys.argv:
  164. logging.getLogger().setLevel(logging.INFO)
  165. args = sys.argv[1:]
  166. engine_override, recipes_cfg_path = parse_args(args)
  167. if recipes_cfg_path:
  168. # calculate repo_root from recipes_cfg_path
  169. repo_root = os.path.dirname(
  170. os.path.dirname(os.path.dirname(recipes_cfg_path)))
  171. else:
  172. # find repo_root with git and calculate recipes_cfg_path
  173. repo_root = (
  174. _git_output(['rev-parse', '--show-toplevel'],
  175. cwd=os.path.abspath(os.path.dirname(__file__))).strip())
  176. repo_root = os.path.abspath(repo_root)
  177. recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
  178. args = ['--package', recipes_cfg_path] + args
  179. engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path)
  180. try:
  181. return _subprocess_call(
  182. [VPYTHON, '-u',
  183. os.path.join(engine_path, 'recipe_engine', 'main.py')] + args)
  184. except KeyboardInterrupt:
  185. return 1
  186. if __name__ == '__main__':
  187. sys.exit(main())