git-sync-deps 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253
  1. #!/usr/bin/env python
  2. # Copyright 2014 Google Inc.
  3. #
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Parse a DEPS file and git checkout all of the dependencies.
  7. Args:
  8. An optional list of deps_os values.
  9. Environment Variables:
  10. GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in
  11. your default path.
  12. GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
  13. will use the file ../DEPS relative to this script's directory.
  14. GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
  15. Git Config:
  16. To disable syncing of a single repository:
  17. cd path/to/repository
  18. git config sync-deps.disable true
  19. To re-enable sync:
  20. cd path/to/repository
  21. git config --unset sync-deps.disable
  22. """
  23. import os
  24. import subprocess
  25. import sys
  26. import threading
  27. def git_executable():
  28. """Find the git executable.
  29. Returns:
  30. A string suitable for passing to subprocess functions, or None.
  31. """
  32. envgit = os.environ.get('GIT_EXECUTABLE')
  33. searchlist = ['git']
  34. if envgit:
  35. searchlist.insert(0, envgit)
  36. with open(os.devnull, 'w') as devnull:
  37. for git in searchlist:
  38. try:
  39. subprocess.call([git, '--version'], stdout=devnull)
  40. except (OSError,):
  41. continue
  42. return git
  43. return None
  44. DEFAULT_DEPS_PATH = os.path.normpath(
  45. os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
  46. def usage(deps_file_path = None):
  47. sys.stderr.write(
  48. 'Usage: run to grab dependencies, with optional platform support:\n')
  49. sys.stderr.write(' %s %s' % (sys.executable, __file__))
  50. if deps_file_path:
  51. parsed_deps = parse_file_to_dict(deps_file_path)
  52. if 'deps_os' in parsed_deps:
  53. for deps_os in parsed_deps['deps_os']:
  54. sys.stderr.write(' [%s]' % deps_os)
  55. sys.stderr.write('\n\n')
  56. sys.stderr.write(__doc__)
  57. def git_repository_sync_is_disabled(git, directory):
  58. try:
  59. disable = subprocess.check_output(
  60. [git, 'config', 'sync-deps.disable'], cwd=directory)
  61. return disable.lower().strip() in ['true', '1', 'yes', 'on']
  62. except subprocess.CalledProcessError:
  63. return False
  64. def is_git_toplevel(git, directory):
  65. """Return true iff the directory is the top level of a Git repository.
  66. Args:
  67. git (string) the git executable
  68. directory (string) the path into which the repository
  69. is expected to be checked out.
  70. """
  71. try:
  72. toplevel = subprocess.check_output(
  73. [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
  74. return os.path.realpath(directory) == os.path.realpath(toplevel)
  75. except subprocess.CalledProcessError:
  76. return False
  77. def status(directory, checkoutable):
  78. def truncate(s, length):
  79. return s if len(s) <= length else s[:(length - 3)] + '...'
  80. dlen = 36
  81. directory = truncate(directory, dlen)
  82. checkoutable = truncate(checkoutable, 40)
  83. sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
  84. def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
  85. """Checkout (and clone if needed) a Git repository.
  86. Args:
  87. git (string) the git executable
  88. repo (string) the location of the repository, suitable
  89. for passing to `git clone`.
  90. checkoutable (string) a tag, branch, or commit, suitable for
  91. passing to `git checkout`
  92. directory (string) the path into which the repository
  93. should be checked out.
  94. verbose (boolean)
  95. Raises an exception if any calls to git fail.
  96. """
  97. if not os.path.isdir(directory):
  98. subprocess.check_call(
  99. [git, 'clone', '--quiet', repo, directory])
  100. if not is_git_toplevel(git, directory):
  101. # if the directory exists, but isn't a git repo, you will modify
  102. # the parent repostory, which isn't what you want.
  103. sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
  104. return
  105. # Check to see if this repo is disabled. Quick return.
  106. if git_repository_sync_is_disabled(git, directory):
  107. sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
  108. return
  109. with open(os.devnull, 'w') as devnull:
  110. # If this fails, we will fetch before trying again. Don't spam user
  111. # with error infomation.
  112. if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
  113. cwd=directory, stderr=devnull):
  114. # if this succeeds, skip slow `git fetch`.
  115. if verbose:
  116. status(directory, checkoutable) # Success.
  117. return
  118. # If the repo has changed, always force use of the correct repo.
  119. # If origin already points to repo, this is a quick no-op.
  120. subprocess.check_call(
  121. [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
  122. subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
  123. subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
  124. if verbose:
  125. status(directory, checkoutable) # Success.
  126. def parse_file_to_dict(path):
  127. dictionary = {}
  128. execfile(path, dictionary)
  129. return dictionary
  130. def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
  131. """Grab dependencies, with optional platform support.
  132. Args:
  133. deps_file_path (string) Path to the DEPS file.
  134. command_line_os_requests (list of strings) Can be empty list.
  135. List of strings that should each be a key in the deps_os
  136. dictionary in the DEPS file.
  137. Raises git Exceptions.
  138. """
  139. git = git_executable()
  140. assert git
  141. deps_file_directory = os.path.dirname(deps_file_path)
  142. deps_file = parse_file_to_dict(deps_file_path)
  143. dependencies = deps_file['deps'].copy()
  144. os_specific_dependencies = deps_file.get('deps_os', dict())
  145. if 'all' in command_line_os_requests:
  146. for value in os_specific_dependencies.itervalues():
  147. dependencies.update(value)
  148. else:
  149. for os_name in command_line_os_requests:
  150. # Add OS-specific dependencies
  151. if os_name in os_specific_dependencies:
  152. dependencies.update(os_specific_dependencies[os_name])
  153. for directory in dependencies:
  154. for other_dir in dependencies:
  155. if directory.startswith(other_dir + '/'):
  156. raise Exception('%r is parent of %r' % (other_dir, directory))
  157. list_of_arg_lists = []
  158. for directory in sorted(dependencies):
  159. if not isinstance(dependencies[directory], basestring):
  160. if verbose:
  161. print 'Skipping "%s".' % directory
  162. continue
  163. if '@' in dependencies[directory]:
  164. repo, checkoutable = dependencies[directory].split('@', 1)
  165. else:
  166. raise Exception("please specify commit or tag")
  167. relative_directory = os.path.join(deps_file_directory, directory)
  168. list_of_arg_lists.append(
  169. (git, repo, checkoutable, relative_directory, verbose))
  170. multithread(git_checkout_to_directory, list_of_arg_lists)
  171. def multithread(function, list_of_arg_lists):
  172. # for args in list_of_arg_lists:
  173. # function(*args)
  174. # return
  175. threads = []
  176. for args in list_of_arg_lists:
  177. thread = threading.Thread(None, function, None, args)
  178. thread.start()
  179. threads.append(thread)
  180. for thread in threads:
  181. thread.join()
  182. def main(argv):
  183. deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
  184. verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
  185. if '--help' in argv or '-h' in argv:
  186. usage(deps_file_path)
  187. return 1
  188. git_sync_deps(deps_file_path, argv, verbose)
  189. subprocess.check_call(
  190. [sys.executable,
  191. os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
  192. return 0
  193. if __name__ == '__main__':
  194. exit(main(sys.argv[1:]))