gitutil.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. import command
  5. import re
  6. import os
  7. import series
  8. import subprocess
  9. import sys
  10. import terminal
  11. import checkpatch
  12. import settings
  13. import tools
  14. # True to use --no-decorate - we check this in Setup()
  15. use_no_decorate = True
  16. def LogCmd(commit_range, git_dir=None, oneline=False, reverse=False,
  17. count=None):
  18. """Create a command to perform a 'git log'
  19. Args:
  20. commit_range: Range expression to use for log, None for none
  21. git_dir: Path to git repository (None to use default)
  22. oneline: True to use --oneline, else False
  23. reverse: True to reverse the log (--reverse)
  24. count: Number of commits to list, or None for no limit
  25. Return:
  26. List containing command and arguments to run
  27. """
  28. cmd = ['git']
  29. if git_dir:
  30. cmd += ['--git-dir', git_dir]
  31. cmd += ['--no-pager', 'log', '--no-color']
  32. if oneline:
  33. cmd.append('--oneline')
  34. if use_no_decorate:
  35. cmd.append('--no-decorate')
  36. if reverse:
  37. cmd.append('--reverse')
  38. if count is not None:
  39. cmd.append('-n%d' % count)
  40. if commit_range:
  41. cmd.append(commit_range)
  42. # Add this in case we have a branch with the same name as a directory.
  43. # This avoids messages like this, for example:
  44. # fatal: ambiguous argument 'test': both revision and filename
  45. cmd.append('--')
  46. return cmd
  47. def CountCommitsToBranch():
  48. """Returns number of commits between HEAD and the tracking branch.
  49. This looks back to the tracking branch and works out the number of commits
  50. since then.
  51. Return:
  52. Number of patches that exist on top of the branch
  53. """
  54. pipe = [LogCmd('@{upstream}..', oneline=True),
  55. ['wc', '-l']]
  56. stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  57. patch_count = int(stdout)
  58. return patch_count
  59. def NameRevision(commit_hash):
  60. """Gets the revision name for a commit
  61. Args:
  62. commit_hash: Commit hash to look up
  63. Return:
  64. Name of revision, if any, else None
  65. """
  66. pipe = ['git', 'name-rev', commit_hash]
  67. stdout = command.RunPipe([pipe], capture=True, oneline=True).stdout
  68. # We expect a commit, a space, then a revision name
  69. name = stdout.split(' ')[1].strip()
  70. return name
  71. def GuessUpstream(git_dir, branch):
  72. """Tries to guess the upstream for a branch
  73. This lists out top commits on a branch and tries to find a suitable
  74. upstream. It does this by looking for the first commit where
  75. 'git name-rev' returns a plain branch name, with no ! or ^ modifiers.
  76. Args:
  77. git_dir: Git directory containing repo
  78. branch: Name of branch
  79. Returns:
  80. Tuple:
  81. Name of upstream branch (e.g. 'upstream/master') or None if none
  82. Warning/error message, or None if none
  83. """
  84. pipe = [LogCmd(branch, git_dir=git_dir, oneline=True, count=100)]
  85. result = command.RunPipe(pipe, capture=True, capture_stderr=True,
  86. raise_on_error=False)
  87. if result.return_code:
  88. return None, "Branch '%s' not found" % branch
  89. for line in result.stdout.splitlines()[1:]:
  90. commit_hash = line.split(' ')[0]
  91. name = NameRevision(commit_hash)
  92. if '~' not in name and '^' not in name:
  93. if name.startswith('remotes/'):
  94. name = name[8:]
  95. return name, "Guessing upstream as '%s'" % name
  96. return None, "Cannot find a suitable upstream for branch '%s'" % branch
  97. def GetUpstream(git_dir, branch):
  98. """Returns the name of the upstream for a branch
  99. Args:
  100. git_dir: Git directory containing repo
  101. branch: Name of branch
  102. Returns:
  103. Tuple:
  104. Name of upstream branch (e.g. 'upstream/master') or None if none
  105. Warning/error message, or None if none
  106. """
  107. try:
  108. remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  109. 'branch.%s.remote' % branch)
  110. merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  111. 'branch.%s.merge' % branch)
  112. except:
  113. upstream, msg = GuessUpstream(git_dir, branch)
  114. return upstream, msg
  115. if remote == '.':
  116. return merge, None
  117. elif remote and merge:
  118. leaf = merge.split('/')[-1]
  119. return '%s/%s' % (remote, leaf), None
  120. else:
  121. raise ValueError("Cannot determine upstream branch for branch "
  122. "'%s' remote='%s', merge='%s'" % (branch, remote, merge))
  123. def GetRangeInBranch(git_dir, branch, include_upstream=False):
  124. """Returns an expression for the commits in the given branch.
  125. Args:
  126. git_dir: Directory containing git repo
  127. branch: Name of branch
  128. Return:
  129. Expression in the form 'upstream..branch' which can be used to
  130. access the commits. If the branch does not exist, returns None.
  131. """
  132. upstream, msg = GetUpstream(git_dir, branch)
  133. if not upstream:
  134. return None, msg
  135. rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch)
  136. return rstr, msg
  137. def CountCommitsInRange(git_dir, range_expr):
  138. """Returns the number of commits in the given range.
  139. Args:
  140. git_dir: Directory containing git repo
  141. range_expr: Range to check
  142. Return:
  143. Number of patches that exist in the supplied range or None if none
  144. were found
  145. """
  146. pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True)]
  147. result = command.RunPipe(pipe, capture=True, capture_stderr=True,
  148. raise_on_error=False)
  149. if result.return_code:
  150. return None, "Range '%s' not found or is invalid" % range_expr
  151. patch_count = len(result.stdout.splitlines())
  152. return patch_count, None
  153. def CountCommitsInBranch(git_dir, branch, include_upstream=False):
  154. """Returns the number of commits in the given branch.
  155. Args:
  156. git_dir: Directory containing git repo
  157. branch: Name of branch
  158. Return:
  159. Number of patches that exist on top of the branch, or None if the
  160. branch does not exist.
  161. """
  162. range_expr, msg = GetRangeInBranch(git_dir, branch, include_upstream)
  163. if not range_expr:
  164. return None, msg
  165. return CountCommitsInRange(git_dir, range_expr)
  166. def CountCommits(commit_range):
  167. """Returns the number of commits in the given range.
  168. Args:
  169. commit_range: Range of commits to count (e.g. 'HEAD..base')
  170. Return:
  171. Number of patches that exist on top of the branch
  172. """
  173. pipe = [LogCmd(commit_range, oneline=True),
  174. ['wc', '-l']]
  175. stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  176. patch_count = int(stdout)
  177. return patch_count
  178. def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
  179. """Checkout the selected commit for this build
  180. Args:
  181. commit_hash: Commit hash to check out
  182. """
  183. pipe = ['git']
  184. if git_dir:
  185. pipe.extend(['--git-dir', git_dir])
  186. if work_tree:
  187. pipe.extend(['--work-tree', work_tree])
  188. pipe.append('checkout')
  189. if force:
  190. pipe.append('-f')
  191. pipe.append(commit_hash)
  192. result = command.RunPipe([pipe], capture=True, raise_on_error=False,
  193. capture_stderr=True)
  194. if result.return_code != 0:
  195. raise OSError('git checkout (%s): %s' % (pipe, result.stderr))
  196. def Clone(git_dir, output_dir):
  197. """Checkout the selected commit for this build
  198. Args:
  199. commit_hash: Commit hash to check out
  200. """
  201. pipe = ['git', 'clone', git_dir, '.']
  202. result = command.RunPipe([pipe], capture=True, cwd=output_dir,
  203. capture_stderr=True)
  204. if result.return_code != 0:
  205. raise OSError('git clone: %s' % result.stderr)
  206. def Fetch(git_dir=None, work_tree=None):
  207. """Fetch from the origin repo
  208. Args:
  209. commit_hash: Commit hash to check out
  210. """
  211. pipe = ['git']
  212. if git_dir:
  213. pipe.extend(['--git-dir', git_dir])
  214. if work_tree:
  215. pipe.extend(['--work-tree', work_tree])
  216. pipe.append('fetch')
  217. result = command.RunPipe([pipe], capture=True, capture_stderr=True)
  218. if result.return_code != 0:
  219. raise OSError('git fetch: %s' % result.stderr)
  220. def CreatePatches(start, count, series):
  221. """Create a series of patches from the top of the current branch.
  222. The patch files are written to the current directory using
  223. git format-patch.
  224. Args:
  225. start: Commit to start from: 0=HEAD, 1=next one, etc.
  226. count: number of commits to include
  227. Return:
  228. Filename of cover letter
  229. List of filenames of patch files
  230. """
  231. if series.get('version'):
  232. version = '%s ' % series['version']
  233. cmd = ['git', 'format-patch', '-M', '--signoff']
  234. if series.get('cover'):
  235. cmd.append('--cover-letter')
  236. prefix = series.GetPatchPrefix()
  237. if prefix:
  238. cmd += ['--subject-prefix=%s' % prefix]
  239. cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)]
  240. stdout = command.RunList(cmd)
  241. files = stdout.splitlines()
  242. # We have an extra file if there is a cover letter
  243. if series.get('cover'):
  244. return files[0], files[1:]
  245. else:
  246. return None, files
  247. def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True):
  248. """Build a list of email addresses based on an input list.
  249. Takes a list of email addresses and aliases, and turns this into a list
  250. of only email address, by resolving any aliases that are present.
  251. If the tag is given, then each email address is prepended with this
  252. tag and a space. If the tag starts with a minus sign (indicating a
  253. command line parameter) then the email address is quoted.
  254. Args:
  255. in_list: List of aliases/email addresses
  256. tag: Text to put before each address
  257. alias: Alias dictionary
  258. raise_on_error: True to raise an error when an alias fails to match,
  259. False to just print a message.
  260. Returns:
  261. List of email addresses
  262. >>> alias = {}
  263. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  264. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  265. >>> alias['mary'] = ['Mary Poppins <m.poppins@cloud.net>']
  266. >>> alias['boys'] = ['fred', ' john']
  267. >>> alias['all'] = ['fred ', 'john', ' mary ']
  268. >>> BuildEmailList(['john', 'mary'], None, alias)
  269. ['j.bloggs@napier.co.nz', 'Mary Poppins <m.poppins@cloud.net>']
  270. >>> BuildEmailList(['john', 'mary'], '--to', alias)
  271. ['--to "j.bloggs@napier.co.nz"', \
  272. '--to "Mary Poppins <m.poppins@cloud.net>"']
  273. >>> BuildEmailList(['john', 'mary'], 'Cc', alias)
  274. ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>']
  275. """
  276. quote = '"' if tag and tag[0] == '-' else ''
  277. raw = []
  278. for item in in_list:
  279. raw += LookupEmail(item, alias, raise_on_error=raise_on_error)
  280. result = []
  281. for item in raw:
  282. item = tools.FromUnicode(item)
  283. if not item in result:
  284. result.append(item)
  285. if tag:
  286. return ['%s %s%s%s' % (tag, quote, email, quote) for email in result]
  287. return result
  288. def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
  289. self_only=False, alias=None, in_reply_to=None, thread=False,
  290. smtp_server=None):
  291. """Email a patch series.
  292. Args:
  293. series: Series object containing destination info
  294. cover_fname: filename of cover letter
  295. args: list of filenames of patch files
  296. dry_run: Just return the command that would be run
  297. raise_on_error: True to raise an error when an alias fails to match,
  298. False to just print a message.
  299. cc_fname: Filename of Cc file for per-commit Cc
  300. self_only: True to just email to yourself as a test
  301. in_reply_to: If set we'll pass this to git as --in-reply-to.
  302. Should be a message ID that this is in reply to.
  303. thread: True to add --thread to git send-email (make
  304. all patches reply to cover-letter or first patch in series)
  305. smtp_server: SMTP server to use to send patches
  306. Returns:
  307. Git command that was/would be run
  308. # For the duration of this doctest pretend that we ran patman with ./patman
  309. >>> _old_argv0 = sys.argv[0]
  310. >>> sys.argv[0] = './patman'
  311. >>> alias = {}
  312. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  313. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  314. >>> alias['mary'] = ['m.poppins@cloud.net']
  315. >>> alias['boys'] = ['fred', ' john']
  316. >>> alias['all'] = ['fred ', 'john', ' mary ']
  317. >>> alias[os.getenv('USER')] = ['this-is-me@me.com']
  318. >>> series = series.Series()
  319. >>> series.to = ['fred']
  320. >>> series.cc = ['mary']
  321. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  322. False, alias)
  323. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  324. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
  325. >>> EmailPatches(series, None, ['p1'], True, True, 'cc-fname', False, \
  326. alias)
  327. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  328. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" p1'
  329. >>> series.cc = ['all']
  330. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  331. True, alias)
  332. 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \
  333. --cc-cmd cc-fname" cover p1 p2'
  334. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  335. False, alias)
  336. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  337. "f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \
  338. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
  339. # Restore argv[0] since we clobbered it.
  340. >>> sys.argv[0] = _old_argv0
  341. """
  342. to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error)
  343. if not to:
  344. git_config_to = command.Output('git', 'config', 'sendemail.to',
  345. raise_on_error=False)
  346. if not git_config_to:
  347. print("No recipient.\n"
  348. "Please add something like this to a commit\n"
  349. "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n"
  350. "Or do something like this\n"
  351. "git config sendemail.to u-boot@lists.denx.de")
  352. return
  353. cc = BuildEmailList(list(set(series.get('cc')) - set(series.get('to'))),
  354. '--cc', alias, raise_on_error)
  355. if self_only:
  356. to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error)
  357. cc = []
  358. cmd = ['git', 'send-email', '--annotate']
  359. if smtp_server:
  360. cmd.append('--smtp-server=%s' % smtp_server)
  361. if in_reply_to:
  362. cmd.append('--in-reply-to="%s"' % tools.FromUnicode(in_reply_to))
  363. if thread:
  364. cmd.append('--thread')
  365. cmd += to
  366. cmd += cc
  367. cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)]
  368. if cover_fname:
  369. cmd.append(cover_fname)
  370. cmd += args
  371. cmdstr = ' '.join(cmd)
  372. if not dry_run:
  373. os.system(cmdstr)
  374. return cmdstr
  375. def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0):
  376. """If an email address is an alias, look it up and return the full name
  377. TODO: Why not just use git's own alias feature?
  378. Args:
  379. lookup_name: Alias or email address to look up
  380. alias: Dictionary containing aliases (None to use settings default)
  381. raise_on_error: True to raise an error when an alias fails to match,
  382. False to just print a message.
  383. Returns:
  384. tuple:
  385. list containing a list of email addresses
  386. Raises:
  387. OSError if a recursive alias reference was found
  388. ValueError if an alias was not found
  389. >>> alias = {}
  390. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  391. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  392. >>> alias['mary'] = ['m.poppins@cloud.net']
  393. >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz']
  394. >>> alias['all'] = ['fred ', 'john', ' mary ']
  395. >>> alias['loop'] = ['other', 'john', ' mary ']
  396. >>> alias['other'] = ['loop', 'john', ' mary ']
  397. >>> LookupEmail('mary', alias)
  398. ['m.poppins@cloud.net']
  399. >>> LookupEmail('arthur.wellesley@howe.ro.uk', alias)
  400. ['arthur.wellesley@howe.ro.uk']
  401. >>> LookupEmail('boys', alias)
  402. ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz']
  403. >>> LookupEmail('all', alias)
  404. ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
  405. >>> LookupEmail('odd', alias)
  406. Traceback (most recent call last):
  407. ...
  408. ValueError: Alias 'odd' not found
  409. >>> LookupEmail('loop', alias)
  410. Traceback (most recent call last):
  411. ...
  412. OSError: Recursive email alias at 'other'
  413. >>> LookupEmail('odd', alias, raise_on_error=False)
  414. Alias 'odd' not found
  415. []
  416. >>> # In this case the loop part will effectively be ignored.
  417. >>> LookupEmail('loop', alias, raise_on_error=False)
  418. Recursive email alias at 'other'
  419. Recursive email alias at 'john'
  420. Recursive email alias at 'mary'
  421. ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
  422. """
  423. if not alias:
  424. alias = settings.alias
  425. lookup_name = lookup_name.strip()
  426. if '@' in lookup_name: # Perhaps a real email address
  427. return [lookup_name]
  428. lookup_name = lookup_name.lower()
  429. col = terminal.Color()
  430. out_list = []
  431. if level > 10:
  432. msg = "Recursive email alias at '%s'" % lookup_name
  433. if raise_on_error:
  434. raise OSError(msg)
  435. else:
  436. print(col.Color(col.RED, msg))
  437. return out_list
  438. if lookup_name:
  439. if not lookup_name in alias:
  440. msg = "Alias '%s' not found" % lookup_name
  441. if raise_on_error:
  442. raise ValueError(msg)
  443. else:
  444. print(col.Color(col.RED, msg))
  445. return out_list
  446. for item in alias[lookup_name]:
  447. todo = LookupEmail(item, alias, raise_on_error, level + 1)
  448. for new_item in todo:
  449. if not new_item in out_list:
  450. out_list.append(new_item)
  451. #print("No match for alias '%s'" % lookup_name)
  452. return out_list
  453. def GetTopLevel():
  454. """Return name of top-level directory for this git repo.
  455. Returns:
  456. Full path to git top-level directory
  457. This test makes sure that we are running tests in the right subdir
  458. >>> os.path.realpath(os.path.dirname(__file__)) == \
  459. os.path.join(GetTopLevel(), 'tools', 'patman')
  460. True
  461. """
  462. return command.OutputOneLine('git', 'rev-parse', '--show-toplevel')
  463. def GetAliasFile():
  464. """Gets the name of the git alias file.
  465. Returns:
  466. Filename of git alias file, or None if none
  467. """
  468. fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile',
  469. raise_on_error=False)
  470. if fname:
  471. fname = os.path.join(GetTopLevel(), fname.strip())
  472. return fname
  473. def GetDefaultUserName():
  474. """Gets the user.name from .gitconfig file.
  475. Returns:
  476. User name found in .gitconfig file, or None if none
  477. """
  478. uname = command.OutputOneLine('git', 'config', '--global', 'user.name')
  479. return uname
  480. def GetDefaultUserEmail():
  481. """Gets the user.email from the global .gitconfig file.
  482. Returns:
  483. User's email found in .gitconfig file, or None if none
  484. """
  485. uemail = command.OutputOneLine('git', 'config', '--global', 'user.email')
  486. return uemail
  487. def GetDefaultSubjectPrefix():
  488. """Gets the format.subjectprefix from local .git/config file.
  489. Returns:
  490. Subject prefix found in local .git/config file, or None if none
  491. """
  492. sub_prefix = command.OutputOneLine('git', 'config', 'format.subjectprefix',
  493. raise_on_error=False)
  494. return sub_prefix
  495. def Setup():
  496. """Set up git utils, by reading the alias files."""
  497. # Check for a git alias file also
  498. global use_no_decorate
  499. alias_fname = GetAliasFile()
  500. if alias_fname:
  501. settings.ReadGitAliases(alias_fname)
  502. cmd = LogCmd(None, count=0)
  503. use_no_decorate = (command.RunPipe([cmd], raise_on_error=False)
  504. .return_code == 0)
  505. def GetHead():
  506. """Get the hash of the current HEAD
  507. Returns:
  508. Hash of HEAD
  509. """
  510. return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H')
  511. if __name__ == "__main__":
  512. import doctest
  513. doctest.testmod()