123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253 |
- #!/usr/bin/env python
- # Copyright 2014 Google Inc.
- #
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Parse a DEPS file and git checkout all of the dependencies.
- Args:
- An optional list of deps_os values.
- Environment Variables:
- GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in
- your default path.
- GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
- will use the file ../DEPS relative to this script's directory.
- GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
- Git Config:
- To disable syncing of a single repository:
- cd path/to/repository
- git config sync-deps.disable true
- To re-enable sync:
- cd path/to/repository
- git config --unset sync-deps.disable
- """
- import os
- import subprocess
- import sys
- import threading
- def git_executable():
- """Find the git executable.
- Returns:
- A string suitable for passing to subprocess functions, or None.
- """
- envgit = os.environ.get('GIT_EXECUTABLE')
- searchlist = ['git']
- if envgit:
- searchlist.insert(0, envgit)
- with open(os.devnull, 'w') as devnull:
- for git in searchlist:
- try:
- subprocess.call([git, '--version'], stdout=devnull)
- except (OSError,):
- continue
- return git
- return None
- DEFAULT_DEPS_PATH = os.path.normpath(
- os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
- def usage(deps_file_path = None):
- sys.stderr.write(
- 'Usage: run to grab dependencies, with optional platform support:\n')
- sys.stderr.write(' %s %s' % (sys.executable, __file__))
- if deps_file_path:
- parsed_deps = parse_file_to_dict(deps_file_path)
- if 'deps_os' in parsed_deps:
- for deps_os in parsed_deps['deps_os']:
- sys.stderr.write(' [%s]' % deps_os)
- sys.stderr.write('\n\n')
- sys.stderr.write(__doc__)
- def git_repository_sync_is_disabled(git, directory):
- try:
- disable = subprocess.check_output(
- [git, 'config', 'sync-deps.disable'], cwd=directory)
- return disable.lower().strip() in ['true', '1', 'yes', 'on']
- except subprocess.CalledProcessError:
- return False
- def is_git_toplevel(git, directory):
- """Return true iff the directory is the top level of a Git repository.
- Args:
- git (string) the git executable
- directory (string) the path into which the repository
- is expected to be checked out.
- """
- try:
- toplevel = subprocess.check_output(
- [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
- return os.path.realpath(directory) == os.path.realpath(toplevel)
- except subprocess.CalledProcessError:
- return False
- def status(directory, checkoutable):
- def truncate(s, length):
- return s if len(s) <= length else s[:(length - 3)] + '...'
- dlen = 36
- directory = truncate(directory, dlen)
- checkoutable = truncate(checkoutable, 40)
- sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
- def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
- """Checkout (and clone if needed) a Git repository.
- Args:
- git (string) the git executable
- repo (string) the location of the repository, suitable
- for passing to `git clone`.
- checkoutable (string) a tag, branch, or commit, suitable for
- passing to `git checkout`
- directory (string) the path into which the repository
- should be checked out.
- verbose (boolean)
- Raises an exception if any calls to git fail.
- """
- if not os.path.isdir(directory):
- subprocess.check_call(
- [git, 'clone', '--quiet', repo, directory])
- if not is_git_toplevel(git, directory):
- # if the directory exists, but isn't a git repo, you will modify
- # the parent repostory, which isn't what you want.
- sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
- return
- # Check to see if this repo is disabled. Quick return.
- if git_repository_sync_is_disabled(git, directory):
- sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
- return
- with open(os.devnull, 'w') as devnull:
- # If this fails, we will fetch before trying again. Don't spam user
- # with error infomation.
- if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
- cwd=directory, stderr=devnull):
- # if this succeeds, skip slow `git fetch`.
- if verbose:
- status(directory, checkoutable) # Success.
- return
- # If the repo has changed, always force use of the correct repo.
- # If origin already points to repo, this is a quick no-op.
- subprocess.check_call(
- [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
- subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
- subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
- if verbose:
- status(directory, checkoutable) # Success.
- def parse_file_to_dict(path):
- dictionary = {}
- execfile(path, dictionary)
- return dictionary
- def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
- """Grab dependencies, with optional platform support.
- Args:
- deps_file_path (string) Path to the DEPS file.
- command_line_os_requests (list of strings) Can be empty list.
- List of strings that should each be a key in the deps_os
- dictionary in the DEPS file.
- Raises git Exceptions.
- """
- git = git_executable()
- assert git
- deps_file_directory = os.path.dirname(deps_file_path)
- deps_file = parse_file_to_dict(deps_file_path)
- dependencies = deps_file['deps'].copy()
- os_specific_dependencies = deps_file.get('deps_os', dict())
- if 'all' in command_line_os_requests:
- for value in os_specific_dependencies.itervalues():
- dependencies.update(value)
- else:
- for os_name in command_line_os_requests:
- # Add OS-specific dependencies
- if os_name in os_specific_dependencies:
- dependencies.update(os_specific_dependencies[os_name])
- for directory in dependencies:
- for other_dir in dependencies:
- if directory.startswith(other_dir + '/'):
- raise Exception('%r is parent of %r' % (other_dir, directory))
- list_of_arg_lists = []
- for directory in sorted(dependencies):
- if not isinstance(dependencies[directory], basestring):
- if verbose:
- print 'Skipping "%s".' % directory
- continue
- if '@' in dependencies[directory]:
- repo, checkoutable = dependencies[directory].split('@', 1)
- else:
- raise Exception("please specify commit or tag")
- relative_directory = os.path.join(deps_file_directory, directory)
- list_of_arg_lists.append(
- (git, repo, checkoutable, relative_directory, verbose))
- multithread(git_checkout_to_directory, list_of_arg_lists)
- def multithread(function, list_of_arg_lists):
- # for args in list_of_arg_lists:
- # function(*args)
- # return
- threads = []
- for args in list_of_arg_lists:
- thread = threading.Thread(None, function, None, args)
- thread.start()
- threads.append(thread)
- for thread in threads:
- thread.join()
- def main(argv):
- deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
- verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
- if '--help' in argv or '-h' in argv:
- usage(deps_file_path)
- return 1
- git_sync_deps(deps_file_path, argv, verbose)
- subprocess.check_call(
- [sys.executable,
- os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
- return 0
- if __name__ == '__main__':
- exit(main(sys.argv[1:]))
|