123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242 |
- #!/bin/sh
- # Copyright 2019 The LUCI Authors. All rights reserved.
- # Use of this source code is governed under the Apache License, Version 2.0
- # that can be found in the LICENSE file.
- # We want to run python in unbuffered mode; however shebangs on linux grab the
- # entire rest of the shebang line as a single argument, leading to errors like:
- #
- # /usr/bin/env: 'python -u': No such file or directory
- #
- # This little shell hack is a triple-quoted noop in python, but in sh it
- # evaluates to re-exec'ing this script in unbuffered mode.
- # pylint: disable=pointless-string-statement
- ''''exec python -u -- "$0" ${1+"$@"} # '''
- # vi: syntax=python
- """Bootstrap script to clone and forward to the recipe engine tool.
- *******************
- ** DO NOT MODIFY **
- *******************
- This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/recipes.py.
- To fix bugs, fix in the googlesource repo then run the autoroller.
- """
- # pylint: disable=wrong-import-position
- import argparse
- import json
- import logging
- import os
- import subprocess
- import sys
- import urlparse
- from collections import namedtuple
- # The dependency entry for the recipe_engine in the client repo's recipes.cfg
- #
- # url (str) - the url to the engine repo we want to use.
- # revision (str) - the git revision for the engine to get.
- # branch (str) - the branch to fetch for the engine as an absolute ref (e.g.
- # refs/heads/master)
- EngineDep = namedtuple('EngineDep', 'url revision branch')
- class MalformedRecipesCfg(Exception):
- def __init__(self, msg, path):
- full_message = 'malformed recipes.cfg: %s: %r' % (msg, path)
- super(MalformedRecipesCfg, self).__init__(full_message)
- def parse(repo_root, recipes_cfg_path):
- """Parse is a lightweight a recipes.cfg file parser.
- Args:
- repo_root (str) - native path to the root of the repo we're trying to run
- recipes for.
- recipes_cfg_path (str) - native path to the recipes.cfg file to process.
- Returns (as tuple):
- engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the
- current repo IS the recipe_engine.
- recipes_path (str) - native path to where the recipes live inside of the
- current repo (i.e. the folder containing `recipes/` and/or
- `recipe_modules`)
- """
- with open(recipes_cfg_path, 'rU') as fh:
- pb = json.load(fh)
- try:
- if pb['api_version'] != 2:
- raise MalformedRecipesCfg('unknown version %d' % pb['api_version'],
- recipes_cfg_path)
- # If we're running ./recipes.py from the recipe_engine repo itself, then
- # return None to signal that there's no EngineDep.
- repo_name = pb.get('repo_name')
- if not repo_name:
- repo_name = pb['project_id']
- if repo_name == 'recipe_engine':
- return None, pb.get('recipes_path', '')
- engine = pb['deps']['recipe_engine']
- if 'url' not in engine:
- raise MalformedRecipesCfg(
- 'Required field "url" in dependency "recipe_engine" not found',
- recipes_cfg_path)
- engine.setdefault('revision', '')
- engine.setdefault('branch', 'refs/heads/master')
- recipes_path = pb.get('recipes_path', '')
- # TODO(iannucci): only support absolute refs
- if not engine['branch'].startswith('refs/'):
- engine['branch'] = 'refs/heads/' + engine['branch']
- recipes_path = os.path.join(repo_root,
- recipes_path.replace('/', os.path.sep))
- return EngineDep(**engine), recipes_path
- except KeyError as ex:
- raise MalformedRecipesCfg(ex.message, recipes_cfg_path)
- _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else ''
- GIT = 'git' + _BAT
- VPYTHON = 'vpython' + _BAT
- CIPD = 'cipd' + _BAT
- REQUIRED_BINARIES = {GIT, VPYTHON, CIPD}
- def _is_executable(path):
- return os.path.isfile(path) and os.access(path, os.X_OK)
- # TODO: Use shutil.which once we switch to Python3.
- def _is_on_path(basename):
- for path in os.environ['PATH'].split(os.pathsep):
- full_path = os.path.join(path, basename)
- if _is_executable(full_path):
- return True
- return False
- def _subprocess_call(argv, **kwargs):
- logging.info('Running %r', argv)
- return subprocess.call(argv, **kwargs)
- def _git_check_call(argv, **kwargs):
- argv = [GIT] + argv
- logging.info('Running %r', argv)
- subprocess.check_call(argv, **kwargs)
- def _git_output(argv, **kwargs):
- argv = [GIT] + argv
- logging.info('Running %r', argv)
- return subprocess.check_output(argv, **kwargs)
- def parse_args(argv):
- """This extracts a subset of the arguments that this bootstrap script cares
- about. Currently this consists of:
- * an override for the recipe engine in the form of `-O recipe_engine=/path`
- * the --package option.
- """
- PREFIX = 'recipe_engine='
- p = argparse.ArgumentParser(add_help=False)
- p.add_argument('-O', '--project-override', action='append')
- p.add_argument('--package', type=os.path.abspath)
- args, _ = p.parse_known_args(argv)
- for override in args.project_override or ():
- if override.startswith(PREFIX):
- return override[len(PREFIX):], args.package
- return None, args.package
- def checkout_engine(engine_path, repo_root, recipes_cfg_path):
- dep, recipes_path = parse(repo_root, recipes_cfg_path)
- if dep is None:
- # we're running from the engine repo already!
- return os.path.join(repo_root, recipes_path)
- url = dep.url
- if not engine_path and url.startswith('file://'):
- engine_path = urlparse.urlparse(url).path
- if not engine_path:
- revision = dep.revision
- branch = dep.branch
- # Ensure that we have the recipe engine cloned.
- engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine')
- with open(os.devnull, 'w') as NUL:
- # Note: this logic mirrors the logic in recipe_engine/fetch.py
- _git_check_call(['init', engine_path], stdout=NUL)
- try:
- _git_check_call(['rev-parse', '--verify',
- '%s^{commit}' % revision],
- cwd=engine_path,
- stdout=NUL,
- stderr=NUL)
- except subprocess.CalledProcessError:
- _git_check_call(['fetch', url, branch],
- cwd=engine_path,
- stdout=NUL,
- stderr=NUL)
- try:
- _git_check_call(['diff', '--quiet', revision], cwd=engine_path)
- except subprocess.CalledProcessError:
- _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path)
- # If the engine has refactored/moved modules we need to clean all .pyc files
- # or things will get squirrely.
- _git_check_call(['clean', '-qxf'], cwd=engine_path)
- return engine_path
- def main():
- for required_binary in REQUIRED_BINARIES:
- if not _is_on_path(required_binary):
- return 'Required binary is not found on PATH: %s' % required_binary
- if '--verbose' in sys.argv:
- logging.getLogger().setLevel(logging.INFO)
- args = sys.argv[1:]
- engine_override, recipes_cfg_path = parse_args(args)
- if recipes_cfg_path:
- # calculate repo_root from recipes_cfg_path
- repo_root = os.path.dirname(
- os.path.dirname(os.path.dirname(recipes_cfg_path)))
- else:
- # find repo_root with git and calculate recipes_cfg_path
- repo_root = (
- _git_output(['rev-parse', '--show-toplevel'],
- cwd=os.path.abspath(os.path.dirname(__file__))).strip())
- repo_root = os.path.abspath(repo_root)
- recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
- args = ['--package', recipes_cfg_path] + args
- engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path)
- try:
- return _subprocess_call(
- [VPYTHON, '-u',
- os.path.join(engine_path, 'recipe_engine', 'main.py')] + args)
- except KeyboardInterrupt:
- return 1
- if __name__ == '__main__':
- sys.exit(main())
|