gitutil.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. import re
  5. import os
  6. import subprocess
  7. import sys
  8. from patman import checkpatch
  9. from patman import command
  10. from patman import series
  11. from patman import settings
  12. from patman import terminal
  13. from patman import tools
  14. # True to use --no-decorate - we check this in Setup()
  15. use_no_decorate = True
  16. def LogCmd(commit_range, git_dir=None, oneline=False, reverse=False,
  17. count=None):
  18. """Create a command to perform a 'git log'
  19. Args:
  20. commit_range: Range expression to use for log, None for none
  21. git_dir: Path to git repository (None to use default)
  22. oneline: True to use --oneline, else False
  23. reverse: True to reverse the log (--reverse)
  24. count: Number of commits to list, or None for no limit
  25. Return:
  26. List containing command and arguments to run
  27. """
  28. cmd = ['git']
  29. if git_dir:
  30. cmd += ['--git-dir', git_dir]
  31. cmd += ['--no-pager', 'log', '--no-color']
  32. if oneline:
  33. cmd.append('--oneline')
  34. if use_no_decorate:
  35. cmd.append('--no-decorate')
  36. if reverse:
  37. cmd.append('--reverse')
  38. if count is not None:
  39. cmd.append('-n%d' % count)
  40. if commit_range:
  41. cmd.append(commit_range)
  42. # Add this in case we have a branch with the same name as a directory.
  43. # This avoids messages like this, for example:
  44. # fatal: ambiguous argument 'test': both revision and filename
  45. cmd.append('--')
  46. return cmd
  47. def CountCommitsToBranch():
  48. """Returns number of commits between HEAD and the tracking branch.
  49. This looks back to the tracking branch and works out the number of commits
  50. since then.
  51. Return:
  52. Number of patches that exist on top of the branch
  53. """
  54. pipe = [LogCmd('@{upstream}..', oneline=True),
  55. ['wc', '-l']]
  56. stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  57. patch_count = int(stdout)
  58. return patch_count
  59. def NameRevision(commit_hash):
  60. """Gets the revision name for a commit
  61. Args:
  62. commit_hash: Commit hash to look up
  63. Return:
  64. Name of revision, if any, else None
  65. """
  66. pipe = ['git', 'name-rev', commit_hash]
  67. stdout = command.RunPipe([pipe], capture=True, oneline=True).stdout
  68. # We expect a commit, a space, then a revision name
  69. name = stdout.split(' ')[1].strip()
  70. return name
  71. def GuessUpstream(git_dir, branch):
  72. """Tries to guess the upstream for a branch
  73. This lists out top commits on a branch and tries to find a suitable
  74. upstream. It does this by looking for the first commit where
  75. 'git name-rev' returns a plain branch name, with no ! or ^ modifiers.
  76. Args:
  77. git_dir: Git directory containing repo
  78. branch: Name of branch
  79. Returns:
  80. Tuple:
  81. Name of upstream branch (e.g. 'upstream/master') or None if none
  82. Warning/error message, or None if none
  83. """
  84. pipe = [LogCmd(branch, git_dir=git_dir, oneline=True, count=100)]
  85. result = command.RunPipe(pipe, capture=True, capture_stderr=True,
  86. raise_on_error=False)
  87. if result.return_code:
  88. return None, "Branch '%s' not found" % branch
  89. for line in result.stdout.splitlines()[1:]:
  90. commit_hash = line.split(' ')[0]
  91. name = NameRevision(commit_hash)
  92. if '~' not in name and '^' not in name:
  93. if name.startswith('remotes/'):
  94. name = name[8:]
  95. return name, "Guessing upstream as '%s'" % name
  96. return None, "Cannot find a suitable upstream for branch '%s'" % branch
  97. def GetUpstream(git_dir, branch):
  98. """Returns the name of the upstream for a branch
  99. Args:
  100. git_dir: Git directory containing repo
  101. branch: Name of branch
  102. Returns:
  103. Tuple:
  104. Name of upstream branch (e.g. 'upstream/master') or None if none
  105. Warning/error message, or None if none
  106. """
  107. try:
  108. remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  109. 'branch.%s.remote' % branch)
  110. merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config',
  111. 'branch.%s.merge' % branch)
  112. except:
  113. upstream, msg = GuessUpstream(git_dir, branch)
  114. return upstream, msg
  115. if remote == '.':
  116. return merge, None
  117. elif remote and merge:
  118. leaf = merge.split('/')[-1]
  119. return '%s/%s' % (remote, leaf), None
  120. else:
  121. raise ValueError("Cannot determine upstream branch for branch "
  122. "'%s' remote='%s', merge='%s'" % (branch, remote, merge))
  123. def GetRangeInBranch(git_dir, branch, include_upstream=False):
  124. """Returns an expression for the commits in the given branch.
  125. Args:
  126. git_dir: Directory containing git repo
  127. branch: Name of branch
  128. Return:
  129. Expression in the form 'upstream..branch' which can be used to
  130. access the commits. If the branch does not exist, returns None.
  131. """
  132. upstream, msg = GetUpstream(git_dir, branch)
  133. if not upstream:
  134. return None, msg
  135. rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch)
  136. return rstr, msg
  137. def CountCommitsInRange(git_dir, range_expr):
  138. """Returns the number of commits in the given range.
  139. Args:
  140. git_dir: Directory containing git repo
  141. range_expr: Range to check
  142. Return:
  143. Number of patches that exist in the supplied range or None if none
  144. were found
  145. """
  146. pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True)]
  147. result = command.RunPipe(pipe, capture=True, capture_stderr=True,
  148. raise_on_error=False)
  149. if result.return_code:
  150. return None, "Range '%s' not found or is invalid" % range_expr
  151. patch_count = len(result.stdout.splitlines())
  152. return patch_count, None
  153. def CountCommitsInBranch(git_dir, branch, include_upstream=False):
  154. """Returns the number of commits in the given branch.
  155. Args:
  156. git_dir: Directory containing git repo
  157. branch: Name of branch
  158. Return:
  159. Number of patches that exist on top of the branch, or None if the
  160. branch does not exist.
  161. """
  162. range_expr, msg = GetRangeInBranch(git_dir, branch, include_upstream)
  163. if not range_expr:
  164. return None, msg
  165. return CountCommitsInRange(git_dir, range_expr)
  166. def CountCommits(commit_range):
  167. """Returns the number of commits in the given range.
  168. Args:
  169. commit_range: Range of commits to count (e.g. 'HEAD..base')
  170. Return:
  171. Number of patches that exist on top of the branch
  172. """
  173. pipe = [LogCmd(commit_range, oneline=True),
  174. ['wc', '-l']]
  175. stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout
  176. patch_count = int(stdout)
  177. return patch_count
  178. def Checkout(commit_hash, git_dir=None, work_tree=None, force=False):
  179. """Checkout the selected commit for this build
  180. Args:
  181. commit_hash: Commit hash to check out
  182. """
  183. pipe = ['git']
  184. if git_dir:
  185. pipe.extend(['--git-dir', git_dir])
  186. if work_tree:
  187. pipe.extend(['--work-tree', work_tree])
  188. pipe.append('checkout')
  189. if force:
  190. pipe.append('-f')
  191. pipe.append(commit_hash)
  192. result = command.RunPipe([pipe], capture=True, raise_on_error=False,
  193. capture_stderr=True)
  194. if result.return_code != 0:
  195. raise OSError('git checkout (%s): %s' % (pipe, result.stderr))
  196. def Clone(git_dir, output_dir):
  197. """Checkout the selected commit for this build
  198. Args:
  199. commit_hash: Commit hash to check out
  200. """
  201. pipe = ['git', 'clone', git_dir, '.']
  202. result = command.RunPipe([pipe], capture=True, cwd=output_dir,
  203. capture_stderr=True)
  204. if result.return_code != 0:
  205. raise OSError('git clone: %s' % result.stderr)
  206. def Fetch(git_dir=None, work_tree=None):
  207. """Fetch from the origin repo
  208. Args:
  209. commit_hash: Commit hash to check out
  210. """
  211. pipe = ['git']
  212. if git_dir:
  213. pipe.extend(['--git-dir', git_dir])
  214. if work_tree:
  215. pipe.extend(['--work-tree', work_tree])
  216. pipe.append('fetch')
  217. result = command.RunPipe([pipe], capture=True, capture_stderr=True)
  218. if result.return_code != 0:
  219. raise OSError('git fetch: %s' % result.stderr)
  220. def CreatePatches(start, count, series):
  221. """Create a series of patches from the top of the current branch.
  222. The patch files are written to the current directory using
  223. git format-patch.
  224. Args:
  225. start: Commit to start from: 0=HEAD, 1=next one, etc.
  226. count: number of commits to include
  227. Return:
  228. Filename of cover letter
  229. List of filenames of patch files
  230. """
  231. if series.get('version'):
  232. version = '%s ' % series['version']
  233. cmd = ['git', 'format-patch', '-M', '--signoff']
  234. if series.get('cover'):
  235. cmd.append('--cover-letter')
  236. prefix = series.GetPatchPrefix()
  237. if prefix:
  238. cmd += ['--subject-prefix=%s' % prefix]
  239. cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)]
  240. stdout = command.RunList(cmd)
  241. files = stdout.splitlines()
  242. # We have an extra file if there is a cover letter
  243. if series.get('cover'):
  244. return files[0], files[1:]
  245. else:
  246. return None, files
  247. def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True):
  248. """Build a list of email addresses based on an input list.
  249. Takes a list of email addresses and aliases, and turns this into a list
  250. of only email address, by resolving any aliases that are present.
  251. If the tag is given, then each email address is prepended with this
  252. tag and a space. If the tag starts with a minus sign (indicating a
  253. command line parameter) then the email address is quoted.
  254. Args:
  255. in_list: List of aliases/email addresses
  256. tag: Text to put before each address
  257. alias: Alias dictionary
  258. raise_on_error: True to raise an error when an alias fails to match,
  259. False to just print a message.
  260. Returns:
  261. List of email addresses
  262. >>> alias = {}
  263. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  264. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  265. >>> alias['mary'] = ['Mary Poppins <m.poppins@cloud.net>']
  266. >>> alias['boys'] = ['fred', ' john']
  267. >>> alias['all'] = ['fred ', 'john', ' mary ']
  268. >>> BuildEmailList(['john', 'mary'], None, alias)
  269. ['j.bloggs@napier.co.nz', 'Mary Poppins <m.poppins@cloud.net>']
  270. >>> BuildEmailList(['john', 'mary'], '--to', alias)
  271. ['--to "j.bloggs@napier.co.nz"', \
  272. '--to "Mary Poppins <m.poppins@cloud.net>"']
  273. >>> BuildEmailList(['john', 'mary'], 'Cc', alias)
  274. ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>']
  275. """
  276. quote = '"' if tag and tag[0] == '-' else ''
  277. raw = []
  278. for item in in_list:
  279. raw += LookupEmail(item, alias, raise_on_error=raise_on_error)
  280. result = []
  281. for item in raw:
  282. item = tools.FromUnicode(item)
  283. if not item in result:
  284. result.append(item)
  285. if tag:
  286. return ['%s %s%s%s' % (tag, quote, email, quote) for email in result]
  287. return result
  288. def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
  289. self_only=False, alias=None, in_reply_to=None, thread=False,
  290. smtp_server=None):
  291. """Email a patch series.
  292. Args:
  293. series: Series object containing destination info
  294. cover_fname: filename of cover letter
  295. args: list of filenames of patch files
  296. dry_run: Just return the command that would be run
  297. raise_on_error: True to raise an error when an alias fails to match,
  298. False to just print a message.
  299. cc_fname: Filename of Cc file for per-commit Cc
  300. self_only: True to just email to yourself as a test
  301. in_reply_to: If set we'll pass this to git as --in-reply-to.
  302. Should be a message ID that this is in reply to.
  303. thread: True to add --thread to git send-email (make
  304. all patches reply to cover-letter or first patch in series)
  305. smtp_server: SMTP server to use to send patches
  306. Returns:
  307. Git command that was/would be run
  308. # For the duration of this doctest pretend that we ran patman with ./patman
  309. >>> _old_argv0 = sys.argv[0]
  310. >>> sys.argv[0] = './patman'
  311. >>> alias = {}
  312. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  313. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  314. >>> alias['mary'] = ['m.poppins@cloud.net']
  315. >>> alias['boys'] = ['fred', ' john']
  316. >>> alias['all'] = ['fred ', 'john', ' mary ']
  317. >>> alias[os.getenv('USER')] = ['this-is-me@me.com']
  318. >>> series = series.Series()
  319. >>> series.to = ['fred']
  320. >>> series.cc = ['mary']
  321. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  322. False, alias)
  323. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  324. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
  325. >>> EmailPatches(series, None, ['p1'], True, True, 'cc-fname', False, \
  326. alias)
  327. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  328. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" p1'
  329. >>> series.cc = ['all']
  330. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  331. True, alias)
  332. 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \
  333. --cc-cmd cc-fname" cover p1 p2'
  334. >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \
  335. False, alias)
  336. 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \
  337. "f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \
  338. "m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2'
  339. # Restore argv[0] since we clobbered it.
  340. >>> sys.argv[0] = _old_argv0
  341. """
  342. to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error)
  343. if not to:
  344. git_config_to = command.Output('git', 'config', 'sendemail.to',
  345. raise_on_error=False)
  346. if not git_config_to:
  347. print("No recipient.\n"
  348. "Please add something like this to a commit\n"
  349. "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n"
  350. "Or do something like this\n"
  351. "git config sendemail.to u-boot@lists.denx.de")
  352. return
  353. cc = BuildEmailList(list(set(series.get('cc')) - set(series.get('to'))),
  354. '--cc', alias, raise_on_error)
  355. if self_only:
  356. to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error)
  357. cc = []
  358. cmd = ['git', 'send-email', '--annotate']
  359. if smtp_server:
  360. cmd.append('--smtp-server=%s' % smtp_server)
  361. if in_reply_to:
  362. cmd.append('--in-reply-to="%s"' % tools.FromUnicode(in_reply_to))
  363. if thread:
  364. cmd.append('--thread')
  365. cmd += to
  366. cmd += cc
  367. cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)]
  368. if cover_fname:
  369. cmd.append(cover_fname)
  370. cmd += args
  371. cmdstr = ' '.join(cmd)
  372. if not dry_run:
  373. os.system(cmdstr)
  374. return cmdstr
  375. def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0):
  376. """If an email address is an alias, look it up and return the full name
  377. TODO: Why not just use git's own alias feature?
  378. Args:
  379. lookup_name: Alias or email address to look up
  380. alias: Dictionary containing aliases (None to use settings default)
  381. raise_on_error: True to raise an error when an alias fails to match,
  382. False to just print a message.
  383. Returns:
  384. tuple:
  385. list containing a list of email addresses
  386. Raises:
  387. OSError if a recursive alias reference was found
  388. ValueError if an alias was not found
  389. >>> alias = {}
  390. >>> alias['fred'] = ['f.bloggs@napier.co.nz']
  391. >>> alias['john'] = ['j.bloggs@napier.co.nz']
  392. >>> alias['mary'] = ['m.poppins@cloud.net']
  393. >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz']
  394. >>> alias['all'] = ['fred ', 'john', ' mary ']
  395. >>> alias['loop'] = ['other', 'john', ' mary ']
  396. >>> alias['other'] = ['loop', 'john', ' mary ']
  397. >>> LookupEmail('mary', alias)
  398. ['m.poppins@cloud.net']
  399. >>> LookupEmail('arthur.wellesley@howe.ro.uk', alias)
  400. ['arthur.wellesley@howe.ro.uk']
  401. >>> LookupEmail('boys', alias)
  402. ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz']
  403. >>> LookupEmail('all', alias)
  404. ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
  405. >>> LookupEmail('odd', alias)
  406. Traceback (most recent call last):
  407. ...
  408. ValueError: Alias 'odd' not found
  409. >>> LookupEmail('loop', alias)
  410. Traceback (most recent call last):
  411. ...
  412. OSError: Recursive email alias at 'other'
  413. >>> LookupEmail('odd', alias, raise_on_error=False)
  414. Alias 'odd' not found
  415. []
  416. >>> # In this case the loop part will effectively be ignored.
  417. >>> LookupEmail('loop', alias, raise_on_error=False)
  418. Recursive email alias at 'other'
  419. Recursive email alias at 'john'
  420. Recursive email alias at 'mary'
  421. ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net']
  422. """
  423. if not alias:
  424. alias = settings.alias
  425. lookup_name = lookup_name.strip()
  426. if '@' in lookup_name: # Perhaps a real email address
  427. return [lookup_name]
  428. lookup_name = lookup_name.lower()
  429. col = terminal.Color()
  430. out_list = []
  431. if level > 10:
  432. msg = "Recursive email alias at '%s'" % lookup_name
  433. if raise_on_error:
  434. raise OSError(msg)
  435. else:
  436. print(col.Color(col.RED, msg))
  437. return out_list
  438. if lookup_name:
  439. if not lookup_name in alias:
  440. msg = "Alias '%s' not found" % lookup_name
  441. if raise_on_error:
  442. raise ValueError(msg)
  443. else:
  444. print(col.Color(col.RED, msg))
  445. return out_list
  446. for item in alias[lookup_name]:
  447. todo = LookupEmail(item, alias, raise_on_error, level + 1)
  448. for new_item in todo:
  449. if not new_item in out_list:
  450. out_list.append(new_item)
  451. #print("No match for alias '%s'" % lookup_name)
  452. return out_list
  453. def GetTopLevel():
  454. """Return name of top-level directory for this git repo.
  455. Returns:
  456. Full path to git top-level directory
  457. This test makes sure that we are running tests in the right subdir
  458. >>> os.path.realpath(os.path.dirname(__file__)) == \
  459. os.path.join(GetTopLevel(), 'tools', 'patman')
  460. True
  461. """
  462. return command.OutputOneLine('git', 'rev-parse', '--show-toplevel')
  463. def GetAliasFile():
  464. """Gets the name of the git alias file.
  465. Returns:
  466. Filename of git alias file, or None if none
  467. """
  468. fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile',
  469. raise_on_error=False)
  470. if fname:
  471. fname = os.path.join(GetTopLevel(), fname.strip())
  472. return fname
  473. def GetDefaultUserName():
  474. """Gets the user.name from .gitconfig file.
  475. Returns:
  476. User name found in .gitconfig file, or None if none
  477. """
  478. uname = command.OutputOneLine('git', 'config', '--global', 'user.name')
  479. return uname
  480. def GetDefaultUserEmail():
  481. """Gets the user.email from the global .gitconfig file.
  482. Returns:
  483. User's email found in .gitconfig file, or None if none
  484. """
  485. uemail = command.OutputOneLine('git', 'config', '--global', 'user.email')
  486. return uemail
  487. def GetDefaultSubjectPrefix():
  488. """Gets the format.subjectprefix from local .git/config file.
  489. Returns:
  490. Subject prefix found in local .git/config file, or None if none
  491. """
  492. sub_prefix = command.OutputOneLine('git', 'config', 'format.subjectprefix',
  493. raise_on_error=False)
  494. return sub_prefix
  495. def Setup():
  496. """Set up git utils, by reading the alias files."""
  497. # Check for a git alias file also
  498. global use_no_decorate
  499. alias_fname = GetAliasFile()
  500. if alias_fname:
  501. settings.ReadGitAliases(alias_fname)
  502. cmd = LogCmd(None, count=0)
  503. use_no_decorate = (command.RunPipe([cmd], raise_on_error=False)
  504. .return_code == 0)
  505. def GetHead():
  506. """Get the hash of the current HEAD
  507. Returns:
  508. Hash of HEAD
  509. """
  510. return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H')
  511. if __name__ == "__main__":
  512. import doctest
  513. doctest.testmod()