123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778 |
- ## @file
- # Check a patch for various format issues
- #
- # Copyright (c) 2015 - 2021, Intel Corporation. All rights reserved.<BR>
- # Copyright (C) 2020, Red Hat, Inc.<BR>
- # Copyright (c) 2020, ARM Ltd. All rights reserved.<BR>
- #
- # SPDX-License-Identifier: BSD-2-Clause-Patent
- #
- from __future__ import print_function
- VersionNumber = '0.1'
- __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation All rights reserved."
- import email
- import argparse
- import os
- import re
- import subprocess
- import sys
- import email.header
- class Verbose:
- SILENT, ONELINE, NORMAL = range(3)
- level = NORMAL
- class EmailAddressCheck:
- """Checks an email address."""
- def __init__(self, email, description):
- self.ok = True
- if email is None:
- self.error('Email address is missing!')
- return
- if description is None:
- self.error('Email description is missing!')
- return
- self.description = "'" + description + "'"
- self.check_email_address(email)
- def error(self, *err):
- if self.ok and Verbose.level > Verbose.ONELINE:
- print('The ' + self.description + ' email address is not valid:')
- self.ok = False
- if Verbose.level < Verbose.NORMAL:
- return
- count = 0
- for line in err:
- prefix = (' *', ' ')[count > 0]
- print(prefix, line)
- count += 1
- email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
- re.MULTILINE|re.IGNORECASE)
- def check_email_address(self, email):
- email = email.strip()
- mo = self.email_re1.match(email)
- if mo is None:
- self.error("Email format is invalid: " + email.strip())
- return
- name = mo.group(1).strip()
- if name == '':
- self.error("Name is not provided with email address: " +
- email)
- else:
- quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
- if name.find(',') >= 0 and not quoted:
- self.error('Add quotes (") around name with a comma: ' +
- name)
- if mo.group(2) == '':
- self.error("There should be a space between the name and " +
- "email address: " + email)
- if mo.group(3).find(' ') >= 0:
- self.error("The email address cannot contain a space: " +
- mo.group(3))
- if ' via Groups.Io' in name and mo.group(3).endswith('@groups.io'):
- self.error("Email rewritten by lists DMARC / DKIM / SPF: " +
- email)
- class CommitMessageCheck:
- """Checks the contents of a git commit message."""
- def __init__(self, subject, message, author_email):
- self.ok = True
- if subject is None and message is None:
- self.error('Commit message is missing!')
- return
- MergifyMerge = False
- if "mergify[bot]@users.noreply.github.com" in author_email:
- if "Merge branch" in subject:
- MergifyMerge = True
- self.subject = subject
- self.msg = message
- print (subject)
- self.check_contributed_under()
- if not MergifyMerge:
- self.check_signed_off_by()
- self.check_misc_signatures()
- self.check_overall_format()
- self.report_message_result()
- url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
- def report_message_result(self):
- if Verbose.level < Verbose.NORMAL:
- return
- if self.ok:
- # All checks passed
- return_code = 0
- print('The commit message format passed all checks.')
- else:
- return_code = 1
- if not self.ok:
- print(self.url)
- def error(self, *err):
- if self.ok and Verbose.level > Verbose.ONELINE:
- print('The commit message format is not valid:')
- self.ok = False
- if Verbose.level < Verbose.NORMAL:
- return
- count = 0
- for line in err:
- prefix = (' *', ' ')[count > 0]
- print(prefix, line)
- count += 1
- # Find 'contributed-under:' at the start of a line ignoring case and
- # requires ':' to be present. Matches if there is white space before
- # the tag or between the tag and the ':'.
- contributed_under_re = \
- re.compile(r'^\s*contributed-under\s*:', re.MULTILINE|re.IGNORECASE)
- def check_contributed_under(self):
- match = self.contributed_under_re.search(self.msg)
- if match is not None:
- self.error('Contributed-under! (Note: this must be ' +
- 'removed by the code contributor!)')
- @staticmethod
- def make_signature_re(sig, re_input=False):
- if re_input:
- sub_re = sig
- else:
- sub_re = sig.replace('-', r'[-\s]+')
- re_str = (r'^(?P<tag>' + sub_re +
- r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
- try:
- return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
- except Exception:
- print("Tried to compile re:", re_str)
- raise
- sig_block_re = \
- re.compile(r'''^
- (?: (?P<tag>[^:]+) \s* : \s*
- (?P<value>\S.*?) )
- |
- (?: \[ (?P<updater>[^:]+) \s* : \s*
- (?P<note>.+?) \s* \] )
- \s* $''',
- re.VERBOSE | re.MULTILINE)
- def find_signatures(self, sig):
- if not sig.endswith('-by') and sig != 'Cc':
- sig += '-by'
- regex = self.make_signature_re(sig)
- sigs = regex.findall(self.msg)
- bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
- for s in bad_case_sigs:
- self.error("'" +s[0] + "' should be '" + sig + "'")
- for s in sigs:
- if s[1] != '':
- self.error('There should be no spaces between ' + sig +
- " and the ':'")
- if s[2] != ' ':
- self.error("There should be a space after '" + sig + ":'")
- EmailAddressCheck(s[3], sig)
- return sigs
- def check_signed_off_by(self):
- sob='Signed-off-by'
- if self.msg.find(sob) < 0:
- self.error('Missing Signed-off-by! (Note: this must be ' +
- 'added by the code contributor!)')
- return
- sobs = self.find_signatures('Signed-off')
- if len(sobs) == 0:
- self.error('Invalid Signed-off-by format!')
- return
- sig_types = (
- 'Reviewed',
- 'Reported',
- 'Tested',
- 'Suggested',
- 'Acked',
- 'Cc'
- )
- def check_misc_signatures(self):
- for sig in self.sig_types:
- self.find_signatures(sig)
- cve_re = re.compile('CVE-[0-9]{4}-[0-9]{5}[^0-9]')
- def check_overall_format(self):
- lines = self.msg.splitlines()
- if len(lines) >= 1 and lines[0].endswith('\r\n'):
- empty_line = '\r\n'
- else:
- empty_line = '\n'
- lines.insert(0, empty_line)
- lines.insert(0, self.subject + empty_line)
- count = len(lines)
- if count <= 0:
- self.error('Empty commit message!')
- return
- if count >= 1 and re.search(self.cve_re, lines[0]):
- #
- # If CVE-xxxx-xxxxx is present in subject line, then limit length of
- # subject line to 92 characters
- #
- if len(lines[0].rstrip()) >= 93:
- self.error(
- 'First line of commit message (subject line) is too long (%d >= 93).' %
- (len(lines[0].rstrip()))
- )
- else:
- #
- # If CVE-xxxx-xxxxx is not present in subject line, then limit
- # length of subject line to 75 characters
- #
- if len(lines[0].rstrip()) >= 76:
- self.error(
- 'First line of commit message (subject line) is too long (%d >= 76).' %
- (len(lines[0].rstrip()))
- )
- if count >= 1 and len(lines[0].strip()) == 0:
- self.error('First line of commit message (subject line) ' +
- 'is empty.')
- if count >= 2 and lines[1].strip() != '':
- self.error('Second line of commit message should be ' +
- 'empty.')
- for i in range(2, count):
- if (len(lines[i]) >= 76 and
- len(lines[i].split()) > 1 and
- not lines[i].startswith('git-svn-id:') and
- not lines[i].startswith('Reviewed-by') and
- not lines[i].startswith('Acked-by:') and
- not lines[i].startswith('Tested-by:') and
- not lines[i].startswith('Reported-by:') and
- not lines[i].startswith('Suggested-by:') and
- not lines[i].startswith('Signed-off-by:') and
- not lines[i].startswith('Cc:')):
- #
- # Print a warning if body line is longer than 75 characters
- #
- print(
- 'WARNING - Line %d of commit message is too long (%d >= 76).' %
- (i + 1, len(lines[i]))
- )
- print(lines[i])
- last_sig_line = None
- for i in range(count - 1, 0, -1):
- line = lines[i]
- mo = self.sig_block_re.match(line)
- if mo is None:
- if line.strip() == '':
- break
- elif last_sig_line is not None:
- err2 = 'Add empty line before "%s"?' % last_sig_line
- self.error('The line before the signature block ' +
- 'should be empty', err2)
- else:
- self.error('The signature block was not found')
- break
- last_sig_line = line.strip()
- (START, PRE_PATCH, PATCH) = range(3)
- class GitDiffCheck:
- """Checks the contents of a git diff."""
- def __init__(self, diff):
- self.ok = True
- self.format_ok = True
- self.lines = diff.splitlines(True)
- self.count = len(self.lines)
- self.line_num = 0
- self.state = START
- self.new_bin = []
- while self.line_num < self.count and self.format_ok:
- line_num = self.line_num
- self.run()
- assert(self.line_num > line_num)
- self.report_message_result()
- def report_message_result(self):
- if Verbose.level < Verbose.NORMAL:
- return
- if self.ok:
- print('The code passed all checks.')
- if self.new_bin:
- print('\nWARNING - The following binary files will be added ' +
- 'into the repository:')
- for binary in self.new_bin:
- print(' ' + binary)
- def run(self):
- line = self.lines[self.line_num]
- if self.state in (PRE_PATCH, PATCH):
- if line.startswith('diff --git'):
- self.state = START
- if self.state == PATCH:
- if line.startswith('@@ '):
- self.state = PRE_PATCH
- elif len(line) >= 1 and line[0] not in ' -+' and \
- not line.startswith('\r\n') and \
- not line.startswith(r'\ No newline ') and not self.binary:
- for line in self.lines[self.line_num + 1:]:
- if line.startswith('diff --git'):
- self.format_error('diff found after end of patch')
- break
- self.line_num = self.count
- return
- if self.state == START:
- if line.startswith('diff --git'):
- self.state = PRE_PATCH
- self.filename = line[13:].split(' ', 1)[0]
- self.is_newfile = False
- self.force_crlf = True
- self.force_notabs = True
- if self.filename.endswith('.sh') or \
- self.filename.startswith('BaseTools/BinWrappers/PosixLike/') or \
- self.filename.startswith('BaseTools/BinPipWrappers/PosixLike/') or \
- self.filename == 'BaseTools/BuildEnv':
- #
- # Do not enforce CR/LF line endings for linux shell scripts.
- # Some linux shell scripts don't end with the ".sh" extension,
- # they are identified by their path.
- #
- self.force_crlf = False
- if self.filename == '.gitmodules' or \
- self.filename == 'BaseTools/Conf/diff.order':
- #
- # .gitmodules and diff orderfiles are used internally by git
- # use tabs and LF line endings. Do not enforce no tabs and
- # do not enforce CR/LF line endings.
- #
- self.force_crlf = False
- self.force_notabs = False
- if os.path.basename(self.filename) == 'GNUmakefile' or \
- os.path.basename(self.filename) == 'Makefile':
- self.force_notabs = False
- elif len(line.rstrip()) != 0:
- self.format_error("didn't find diff command")
- self.line_num += 1
- elif self.state == PRE_PATCH:
- if line.startswith('@@ '):
- self.state = PATCH
- self.binary = False
- elif line.startswith('GIT binary patch') or \
- line.startswith('Binary files'):
- self.state = PATCH
- self.binary = True
- if self.is_newfile:
- self.new_bin.append(self.filename)
- elif line.startswith('new file mode 160000'):
- #
- # New submodule. Do not enforce CR/LF line endings
- #
- self.force_crlf = False
- else:
- ok = False
- self.is_newfile = self.newfile_prefix_re.match(line)
- for pfx in self.pre_patch_prefixes:
- if line.startswith(pfx):
- ok = True
- if not ok:
- self.format_error("didn't find diff hunk marker (@@)")
- self.line_num += 1
- elif self.state == PATCH:
- if self.binary:
- pass
- elif line.startswith('-'):
- pass
- elif line.startswith('+'):
- self.check_added_line(line[1:])
- elif line.startswith('\r\n'):
- pass
- elif line.startswith(r'\ No newline '):
- pass
- elif not line.startswith(' '):
- self.format_error("unexpected patch line")
- self.line_num += 1
- pre_patch_prefixes = (
- '--- ',
- '+++ ',
- 'index ',
- 'new file ',
- 'deleted file ',
- 'old mode ',
- 'new mode ',
- 'similarity index ',
- 'copy from ',
- 'copy to ',
- 'rename ',
- )
- line_endings = ('\r\n', '\n\r', '\n', '\r')
- newfile_prefix_re = \
- re.compile(r'''^
- index\ 0+\.\.
- ''',
- re.VERBOSE)
- def added_line_error(self, msg, line):
- lines = [ msg ]
- if self.filename is not None:
- lines.append('File: ' + self.filename)
- lines.append('Line: ' + line)
- self.error(*lines)
- old_debug_re = \
- re.compile(r'''
- DEBUG \s* \( \s* \( \s*
- (?: DEBUG_[A-Z_]+ \s* \| \s*)*
- EFI_D_ ([A-Z_]+)
- ''',
- re.VERBOSE)
- def check_added_line(self, line):
- eol = ''
- for an_eol in self.line_endings:
- if line.endswith(an_eol):
- eol = an_eol
- line = line[:-len(eol)]
- stripped = line.rstrip()
- if self.force_crlf and eol != '\r\n' and (line.find('Subproject commit') == -1):
- self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
- line)
- if self.force_notabs and '\t' in line:
- self.added_line_error('Tab character used', line)
- if len(stripped) < len(line):
- self.added_line_error('Trailing whitespace found', line)
- mo = self.old_debug_re.search(line)
- if mo is not None:
- self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
- 'but DEBUG_' + mo.group(1) +
- ' is now recommended', line)
- split_diff_re = re.compile(r'''
- (?P<cmd>
- ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
- )
- (?P<index>
- ^ index \s+ .+ $
- )
- ''',
- re.IGNORECASE | re.VERBOSE | re.MULTILINE)
- def format_error(self, err):
- self.format_ok = False
- err = 'Patch format error: ' + err
- err2 = 'Line: ' + self.lines[self.line_num].rstrip()
- self.error(err, err2)
- def error(self, *err):
- if self.ok and Verbose.level > Verbose.ONELINE:
- print('Code format is not valid:')
- self.ok = False
- if Verbose.level < Verbose.NORMAL:
- return
- count = 0
- for line in err:
- prefix = (' *', ' ')[count > 0]
- print(prefix, line)
- count += 1
- class CheckOnePatch:
- """Checks the contents of a git email formatted patch.
- Various checks are performed on both the commit message and the
- patch content.
- """
- def __init__(self, name, patch):
- self.patch = patch
- self.find_patch_pieces()
- email_check = EmailAddressCheck(self.author_email, 'Author')
- email_ok = email_check.ok
- msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg, self.author_email)
- msg_ok = msg_check.ok
- diff_ok = True
- if self.diff is not None:
- diff_check = GitDiffCheck(self.diff)
- diff_ok = diff_check.ok
- self.ok = email_ok and msg_ok and diff_ok
- if Verbose.level == Verbose.ONELINE:
- if self.ok:
- result = 'ok'
- else:
- result = list()
- if not msg_ok:
- result.append('commit message')
- if not diff_ok:
- result.append('diff content')
- result = 'bad ' + ' and '.join(result)
- print(name, result)
- git_diff_re = re.compile(r'''
- ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
- ''',
- re.IGNORECASE | re.VERBOSE | re.MULTILINE)
- stat_re = \
- re.compile(r'''
- (?P<commit_message> [\s\S\r\n]* )
- (?P<stat>
- ^ --- $ [\r\n]+
- (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
- $ [\r\n]+ )+
- [\s\S\r\n]+
- )
- ''',
- re.IGNORECASE | re.VERBOSE | re.MULTILINE)
- subject_prefix_re = \
- re.compile(r'''^
- \s* (\[
- [^\[\]]* # Allow all non-brackets
- \])* \s*
- ''',
- re.VERBOSE)
- def find_patch_pieces(self):
- if sys.version_info < (3, 0):
- patch = self.patch.encode('ascii', 'ignore')
- else:
- patch = self.patch
- self.commit_msg = None
- self.stat = None
- self.commit_subject = None
- self.commit_prefix = None
- self.diff = None
- if patch.startswith('diff --git'):
- self.diff = patch
- return
- pmail = email.message_from_string(patch)
- parts = list(pmail.walk())
- assert(len(parts) == 1)
- assert(parts[0].get_content_type() == 'text/plain')
- content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
- mo = self.git_diff_re.search(content)
- if mo is not None:
- self.diff = content[mo.start():]
- content = content[:mo.start()]
- mo = self.stat_re.search(content)
- if mo is None:
- self.commit_msg = content
- else:
- self.stat = mo.group('stat')
- self.commit_msg = mo.group('commit_message')
- #
- # Parse subject line from email header. The subject line may be
- # composed of multiple parts with different encodings. Decode and
- # combine all the parts to produce a single string with the contents of
- # the decoded subject line.
- #
- parts = email.header.decode_header(pmail.get('subject'))
- subject = ''
- for (part, encoding) in parts:
- if encoding:
- part = part.decode(encoding)
- else:
- try:
- part = part.decode()
- except:
- pass
- subject = subject + part
- self.commit_subject = subject.replace('\r\n', '')
- self.commit_subject = self.commit_subject.replace('\n', '')
- self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
- self.author_email = pmail['from']
- class CheckGitCommits:
- """Reads patches from git based on the specified git revision range.
- The patches are read from git, and then checked.
- """
- def __init__(self, rev_spec, max_count):
- commits = self.read_commit_list_from_git(rev_spec, max_count)
- if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
- commits = [ rev_spec ]
- self.ok = True
- blank_line = False
- for commit in commits:
- if Verbose.level > Verbose.ONELINE:
- if blank_line:
- print()
- else:
- blank_line = True
- print('Checking git commit:', commit)
- email = self.read_committer_email_address_from_git(commit)
- self.ok &= EmailAddressCheck(email, 'Committer').ok
- patch = self.read_patch_from_git(commit)
- self.ok &= CheckOnePatch(commit, patch).ok
- if not commits:
- print("Couldn't find commit matching: '{}'".format(rev_spec))
- def read_commit_list_from_git(self, rev_spec, max_count):
- # Run git to get the commit patch
- cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
- if max_count is not None:
- cmd.append('--max-count=' + str(max_count))
- cmd.append(rev_spec)
- out = self.run_git(*cmd)
- return out.split() if out else []
- def read_patch_from_git(self, commit):
- # Run git to get the commit patch
- return self.run_git('show', '--pretty=email', '--no-textconv',
- '--no-use-mailmap', commit)
- def read_committer_email_address_from_git(self, commit):
- # Run git to get the committer email
- return self.run_git('show', '--pretty=%cn <%ce>', '--no-patch',
- '--no-use-mailmap', commit)
- def run_git(self, *args):
- cmd = [ 'git' ]
- cmd += args
- p = subprocess.Popen(cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- Result = p.communicate()
- return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
- class CheckOnePatchFile:
- """Performs a patch check for a single file.
- stdin is used when the filename is '-'.
- """
- def __init__(self, patch_filename):
- if patch_filename == '-':
- patch = sys.stdin.read()
- patch_filename = 'stdin'
- else:
- f = open(patch_filename, 'rb')
- patch = f.read().decode('utf-8', 'ignore')
- f.close()
- if Verbose.level > Verbose.ONELINE:
- print('Checking patch file:', patch_filename)
- self.ok = CheckOnePatch(patch_filename, patch).ok
- class CheckOneArg:
- """Performs a patch check for a single command line argument.
- The argument will be handed off to a file or git-commit based
- checker.
- """
- def __init__(self, param, max_count=None):
- self.ok = True
- if param == '-' or os.path.exists(param):
- checker = CheckOnePatchFile(param)
- else:
- checker = CheckGitCommits(param, max_count)
- self.ok = checker.ok
- class PatchCheckApp:
- """Checks patches based on the command line arguments."""
- def __init__(self):
- self.parse_options()
- patches = self.args.patches
- if len(patches) == 0:
- patches = [ 'HEAD' ]
- self.ok = True
- self.count = None
- for patch in patches:
- self.process_one_arg(patch)
- if self.count is not None:
- self.process_one_arg('HEAD')
- if self.ok:
- self.retval = 0
- else:
- self.retval = -1
- def process_one_arg(self, arg):
- if len(arg) >= 2 and arg[0] == '-':
- try:
- self.count = int(arg[1:])
- return
- except ValueError:
- pass
- self.ok &= CheckOneArg(arg, self.count).ok
- self.count = None
- def parse_options(self):
- parser = argparse.ArgumentParser(description=__copyright__)
- parser.add_argument('--version', action='version',
- version='%(prog)s ' + VersionNumber)
- parser.add_argument('patches', nargs='*',
- help='[patch file | git rev list]')
- group = parser.add_mutually_exclusive_group()
- group.add_argument("--oneline",
- action="store_true",
- help="Print one result per line")
- group.add_argument("--silent",
- action="store_true",
- help="Print nothing")
- self.args = parser.parse_args()
- if self.args.oneline:
- Verbose.level = Verbose.ONELINE
- if self.args.silent:
- Verbose.level = Verbose.SILENT
- if __name__ == "__main__":
- sys.exit(PatchCheckApp().retval)
|