git-make-shallow 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169
  1. #!/usr/bin/env python3
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-only
  4. #
  5. """git-make-shallow: make the current git repository shallow
  6. Remove the history of the specified revisions, then optionally filter the
  7. available refs to those specified.
  8. """
  9. import argparse
  10. import collections
  11. import errno
  12. import itertools
  13. import os
  14. import subprocess
  15. import sys
  16. version = 1.0
  17. def main():
  18. if sys.version_info < (3, 4, 0):
  19. sys.exit('Python 3.4 or greater is required')
  20. git_dir = check_output(['git', 'rev-parse', '--git-dir']).rstrip()
  21. shallow_file = os.path.join(git_dir, 'shallow')
  22. if os.path.exists(shallow_file):
  23. try:
  24. check_output(['git', 'fetch', '--unshallow'])
  25. except subprocess.CalledProcessError:
  26. try:
  27. os.unlink(shallow_file)
  28. except OSError as exc:
  29. if exc.errno != errno.ENOENT:
  30. raise
  31. args = process_args()
  32. revs = check_output(['git', 'rev-list'] + args.revisions).splitlines()
  33. make_shallow(shallow_file, args.revisions, args.refs)
  34. ref_revs = check_output(['git', 'rev-list'] + args.refs).splitlines()
  35. remaining_history = set(revs) & set(ref_revs)
  36. for rev in remaining_history:
  37. if check_output(['git', 'rev-parse', '{}^@'.format(rev)]):
  38. sys.exit('Error: %s was not made shallow' % rev)
  39. filter_refs(args.refs)
  40. if args.shrink:
  41. shrink_repo(git_dir)
  42. subprocess.check_call(['git', 'fsck', '--unreachable'])
  43. def process_args():
  44. # TODO: add argument to automatically keep local-only refs, since they
  45. # can't be easily restored with a git fetch.
  46. parser = argparse.ArgumentParser(description='Remove the history of the specified revisions, then optionally filter the available refs to those specified.')
  47. parser.add_argument('--ref', '-r', metavar='REF', action='append', dest='refs', help='remove all but the specified refs (cumulative)')
  48. parser.add_argument('--shrink', '-s', action='store_true', help='shrink the git repository by repacking and pruning')
  49. parser.add_argument('revisions', metavar='REVISION', nargs='+', help='a git revision/commit')
  50. if len(sys.argv) < 2:
  51. parser.print_help()
  52. sys.exit(2)
  53. args = parser.parse_args()
  54. if args.refs:
  55. args.refs = check_output(['git', 'rev-parse', '--symbolic-full-name'] + args.refs).splitlines()
  56. else:
  57. args.refs = get_all_refs(lambda r, t, tt: t == 'commit' or tt == 'commit')
  58. args.refs = list(filter(lambda r: not r.endswith('/HEAD'), args.refs))
  59. args.revisions = check_output(['git', 'rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines()
  60. return args
  61. def check_output(cmd, input=None):
  62. return subprocess.check_output(cmd, universal_newlines=True, input=input)
  63. def make_shallow(shallow_file, revisions, refs):
  64. """Remove the history of the specified revisions."""
  65. for rev in follow_history_intersections(revisions, refs):
  66. print("Processing %s" % rev)
  67. with open(shallow_file, 'a') as f:
  68. f.write(rev + '\n')
  69. def get_all_refs(ref_filter=None):
  70. """Return all the existing refs in this repository, optionally filtering the refs."""
  71. ref_output = check_output(['git', 'for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)'])
  72. ref_split = [tuple(iter_extend(l.rsplit('\t'), 3)) for l in ref_output.splitlines()]
  73. if ref_filter:
  74. ref_split = (e for e in ref_split if ref_filter(*e))
  75. refs = [r[0] for r in ref_split]
  76. return refs
  77. def iter_extend(iterable, length, obj=None):
  78. """Ensure that iterable is the specified length by extending with obj."""
  79. return itertools.islice(itertools.chain(iterable, itertools.repeat(obj)), length)
  80. def filter_refs(refs):
  81. """Remove all but the specified refs from the git repository."""
  82. all_refs = get_all_refs()
  83. to_remove = set(all_refs) - set(refs)
  84. if to_remove:
  85. check_output(['xargs', '-0', '-n', '1', 'git', 'update-ref', '-d', '--no-deref'],
  86. input=''.join(l + '\0' for l in to_remove))
  87. def follow_history_intersections(revisions, refs):
  88. """Determine all the points where the history of the specified revisions intersects the specified refs."""
  89. queue = collections.deque(revisions)
  90. seen = set()
  91. for rev in iter_except(queue.popleft, IndexError):
  92. if rev in seen:
  93. continue
  94. parents = check_output(['git', 'rev-parse', '%s^@' % rev]).splitlines()
  95. yield rev
  96. seen.add(rev)
  97. if not parents:
  98. continue
  99. check_refs = check_output(['git', 'merge-base', '--independent'] + sorted(refs)).splitlines()
  100. for parent in parents:
  101. for ref in check_refs:
  102. print("Checking %s vs %s" % (parent, ref))
  103. try:
  104. merge_base = check_output(['git', 'merge-base', parent, ref]).rstrip()
  105. except subprocess.CalledProcessError:
  106. continue
  107. else:
  108. queue.append(merge_base)
  109. def iter_except(func, exception, start=None):
  110. """Yield a function repeatedly until it raises an exception."""
  111. try:
  112. if start is not None:
  113. yield start()
  114. while True:
  115. yield func()
  116. except exception:
  117. pass
  118. def shrink_repo(git_dir):
  119. """Shrink the newly shallow repository, removing the unreachable objects."""
  120. subprocess.check_call(['git', 'reflog', 'expire', '--expire-unreachable=now', '--all'])
  121. subprocess.check_call(['git', 'repack', '-ad'])
  122. try:
  123. os.unlink(os.path.join(git_dir, 'objects', 'info', 'alternates'))
  124. except OSError as exc:
  125. if exc.errno != errno.ENOENT:
  126. raise
  127. subprocess.check_call(['git', 'prune', '--expire', 'now'])
  128. if __name__ == '__main__':
  129. main()