PatchCheck.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635
  1. ## @file
  2. # Check a patch for various format issues
  3. #
  4. # Copyright (c) 2015 - 2018, Intel Corporation. All rights reserved.<BR>
  5. #
  6. # SPDX-License-Identifier: BSD-2-Clause-Patent
  7. #
  8. from __future__ import print_function
  9. VersionNumber = '0.1'
  10. __copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation All rights reserved."
  11. import email
  12. import argparse
  13. import os
  14. import re
  15. import subprocess
  16. import sys
  17. class Verbose:
  18. SILENT, ONELINE, NORMAL = range(3)
  19. level = NORMAL
  20. class CommitMessageCheck:
  21. """Checks the contents of a git commit message."""
  22. def __init__(self, subject, message):
  23. self.ok = True
  24. if subject is None and message is None:
  25. self.error('Commit message is missing!')
  26. return
  27. self.subject = subject
  28. self.msg = message
  29. self.check_contributed_under()
  30. self.check_signed_off_by()
  31. self.check_misc_signatures()
  32. self.check_overall_format()
  33. self.report_message_result()
  34. url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
  35. def report_message_result(self):
  36. if Verbose.level < Verbose.NORMAL:
  37. return
  38. if self.ok:
  39. # All checks passed
  40. return_code = 0
  41. print('The commit message format passed all checks.')
  42. else:
  43. return_code = 1
  44. if not self.ok:
  45. print(self.url)
  46. def error(self, *err):
  47. if self.ok and Verbose.level > Verbose.ONELINE:
  48. print('The commit message format is not valid:')
  49. self.ok = False
  50. if Verbose.level < Verbose.NORMAL:
  51. return
  52. count = 0
  53. for line in err:
  54. prefix = (' *', ' ')[count > 0]
  55. print(prefix, line)
  56. count += 1
  57. def check_contributed_under(self):
  58. cu_msg='Contributed-under: TianoCore Contribution Agreement 1.1'
  59. if self.msg.find(cu_msg) < 0:
  60. # Allow 1.0 for now while EDK II community transitions to 1.1
  61. cu_msg='Contributed-under: TianoCore Contribution Agreement 1.0'
  62. if self.msg.find(cu_msg) < 0:
  63. self.error('Missing Contributed-under! (Note: this must be ' +
  64. 'added by the code contributor!)')
  65. @staticmethod
  66. def make_signature_re(sig, re_input=False):
  67. if re_input:
  68. sub_re = sig
  69. else:
  70. sub_re = sig.replace('-', r'[-\s]+')
  71. re_str = (r'^(?P<tag>' + sub_re +
  72. r')(\s*):(\s*)(?P<value>\S.*?)(?:\s*)$')
  73. try:
  74. return re.compile(re_str, re.MULTILINE|re.IGNORECASE)
  75. except Exception:
  76. print("Tried to compile re:", re_str)
  77. raise
  78. sig_block_re = \
  79. re.compile(r'''^
  80. (?: (?P<tag>[^:]+) \s* : \s*
  81. (?P<value>\S.*?) )
  82. |
  83. (?: \[ (?P<updater>[^:]+) \s* : \s*
  84. (?P<note>.+?) \s* \] )
  85. \s* $''',
  86. re.VERBOSE | re.MULTILINE)
  87. def find_signatures(self, sig):
  88. if not sig.endswith('-by') and sig != 'Cc':
  89. sig += '-by'
  90. regex = self.make_signature_re(sig)
  91. sigs = regex.findall(self.msg)
  92. bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
  93. for s in bad_case_sigs:
  94. self.error("'" +s[0] + "' should be '" + sig + "'")
  95. for s in sigs:
  96. if s[1] != '':
  97. self.error('There should be no spaces between ' + sig +
  98. " and the ':'")
  99. if s[2] != ' ':
  100. self.error("There should be a space after '" + sig + ":'")
  101. self.check_email_address(s[3])
  102. return sigs
  103. email_re1 = re.compile(r'(?:\s*)(.*?)(\s*)<(.+)>\s*$',
  104. re.MULTILINE|re.IGNORECASE)
  105. def check_email_address(self, email):
  106. email = email.strip()
  107. mo = self.email_re1.match(email)
  108. if mo is None:
  109. self.error("Email format is invalid: " + email.strip())
  110. return
  111. name = mo.group(1).strip()
  112. if name == '':
  113. self.error("Name is not provided with email address: " +
  114. email)
  115. else:
  116. quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
  117. if name.find(',') >= 0 and not quoted:
  118. self.error('Add quotes (") around name with a comma: ' +
  119. name)
  120. if mo.group(2) == '':
  121. self.error("There should be a space between the name and " +
  122. "email address: " + email)
  123. if mo.group(3).find(' ') >= 0:
  124. self.error("The email address cannot contain a space: " +
  125. mo.group(3))
  126. def check_signed_off_by(self):
  127. sob='Signed-off-by'
  128. if self.msg.find(sob) < 0:
  129. self.error('Missing Signed-off-by! (Note: this must be ' +
  130. 'added by the code contributor!)')
  131. return
  132. sobs = self.find_signatures('Signed-off')
  133. if len(sobs) == 0:
  134. self.error('Invalid Signed-off-by format!')
  135. return
  136. sig_types = (
  137. 'Reviewed',
  138. 'Reported',
  139. 'Tested',
  140. 'Suggested',
  141. 'Acked',
  142. 'Cc'
  143. )
  144. def check_misc_signatures(self):
  145. for sig in self.sig_types:
  146. self.find_signatures(sig)
  147. def check_overall_format(self):
  148. lines = self.msg.splitlines()
  149. if len(lines) >= 1 and lines[0].endswith('\r\n'):
  150. empty_line = '\r\n'
  151. else:
  152. empty_line = '\n'
  153. lines.insert(0, empty_line)
  154. lines.insert(0, self.subject + empty_line)
  155. count = len(lines)
  156. if count <= 0:
  157. self.error('Empty commit message!')
  158. return
  159. if count >= 1 and len(lines[0]) >= 72:
  160. self.error('First line of commit message (subject line) ' +
  161. 'is too long.')
  162. if count >= 1 and len(lines[0].strip()) == 0:
  163. self.error('First line of commit message (subject line) ' +
  164. 'is empty.')
  165. if count >= 2 and lines[1].strip() != '':
  166. self.error('Second line of commit message should be ' +
  167. 'empty.')
  168. for i in range(2, count):
  169. if (len(lines[i]) >= 76 and
  170. len(lines[i].split()) > 1 and
  171. not lines[i].startswith('git-svn-id:')):
  172. self.error('Line %d of commit message is too long.' % (i + 1))
  173. last_sig_line = None
  174. for i in range(count - 1, 0, -1):
  175. line = lines[i]
  176. mo = self.sig_block_re.match(line)
  177. if mo is None:
  178. if line.strip() == '':
  179. break
  180. elif last_sig_line is not None:
  181. err2 = 'Add empty line before "%s"?' % last_sig_line
  182. self.error('The line before the signature block ' +
  183. 'should be empty', err2)
  184. else:
  185. self.error('The signature block was not found')
  186. break
  187. last_sig_line = line.strip()
  188. (START, PRE_PATCH, PATCH) = range(3)
  189. class GitDiffCheck:
  190. """Checks the contents of a git diff."""
  191. def __init__(self, diff):
  192. self.ok = True
  193. self.format_ok = True
  194. self.lines = diff.splitlines(True)
  195. self.count = len(self.lines)
  196. self.line_num = 0
  197. self.state = START
  198. self.new_bin = []
  199. while self.line_num < self.count and self.format_ok:
  200. line_num = self.line_num
  201. self.run()
  202. assert(self.line_num > line_num)
  203. self.report_message_result()
  204. def report_message_result(self):
  205. if Verbose.level < Verbose.NORMAL:
  206. return
  207. if self.ok:
  208. print('The code passed all checks.')
  209. if self.new_bin:
  210. print('\nWARNING - The following binary files will be added ' +
  211. 'into the repository:')
  212. for binary in self.new_bin:
  213. print(' ' + binary)
  214. def run(self):
  215. line = self.lines[self.line_num]
  216. if self.state in (PRE_PATCH, PATCH):
  217. if line.startswith('diff --git'):
  218. self.state = START
  219. if self.state == PATCH:
  220. if line.startswith('@@ '):
  221. self.state = PRE_PATCH
  222. elif len(line) >= 1 and line[0] not in ' -+' and \
  223. not line.startswith(r'\ No newline ') and not self.binary:
  224. for line in self.lines[self.line_num + 1:]:
  225. if line.startswith('diff --git'):
  226. self.format_error('diff found after end of patch')
  227. break
  228. self.line_num = self.count
  229. return
  230. if self.state == START:
  231. if line.startswith('diff --git'):
  232. self.state = PRE_PATCH
  233. self.filename = line[13:].split(' ', 1)[0]
  234. self.is_newfile = False
  235. self.force_crlf = not self.filename.endswith('.sh')
  236. elif len(line.rstrip()) != 0:
  237. self.format_error("didn't find diff command")
  238. self.line_num += 1
  239. elif self.state == PRE_PATCH:
  240. if line.startswith('@@ '):
  241. self.state = PATCH
  242. self.binary = False
  243. elif line.startswith('GIT binary patch') or \
  244. line.startswith('Binary files'):
  245. self.state = PATCH
  246. self.binary = True
  247. if self.is_newfile:
  248. self.new_bin.append(self.filename)
  249. else:
  250. ok = False
  251. self.is_newfile = self.newfile_prefix_re.match(line)
  252. for pfx in self.pre_patch_prefixes:
  253. if line.startswith(pfx):
  254. ok = True
  255. if not ok:
  256. self.format_error("didn't find diff hunk marker (@@)")
  257. self.line_num += 1
  258. elif self.state == PATCH:
  259. if self.binary:
  260. pass
  261. elif line.startswith('-'):
  262. pass
  263. elif line.startswith('+'):
  264. self.check_added_line(line[1:])
  265. elif line.startswith(r'\ No newline '):
  266. pass
  267. elif not line.startswith(' '):
  268. self.format_error("unexpected patch line")
  269. self.line_num += 1
  270. pre_patch_prefixes = (
  271. '--- ',
  272. '+++ ',
  273. 'index ',
  274. 'new file ',
  275. 'deleted file ',
  276. 'old mode ',
  277. 'new mode ',
  278. 'similarity index ',
  279. 'rename ',
  280. )
  281. line_endings = ('\r\n', '\n\r', '\n', '\r')
  282. newfile_prefix_re = \
  283. re.compile(r'''^
  284. index\ 0+\.\.
  285. ''',
  286. re.VERBOSE)
  287. def added_line_error(self, msg, line):
  288. lines = [ msg ]
  289. if self.filename is not None:
  290. lines.append('File: ' + self.filename)
  291. lines.append('Line: ' + line)
  292. self.error(*lines)
  293. old_debug_re = \
  294. re.compile(r'''
  295. DEBUG \s* \( \s* \( \s*
  296. (?: DEBUG_[A-Z_]+ \s* \| \s*)*
  297. EFI_D_ ([A-Z_]+)
  298. ''',
  299. re.VERBOSE)
  300. def check_added_line(self, line):
  301. eol = ''
  302. for an_eol in self.line_endings:
  303. if line.endswith(an_eol):
  304. eol = an_eol
  305. line = line[:-len(eol)]
  306. stripped = line.rstrip()
  307. if self.force_crlf and eol != '\r\n':
  308. self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
  309. line)
  310. if '\t' in line:
  311. self.added_line_error('Tab character used', line)
  312. if len(stripped) < len(line):
  313. self.added_line_error('Trailing whitespace found', line)
  314. mo = self.old_debug_re.search(line)
  315. if mo is not None:
  316. self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
  317. 'but DEBUG_' + mo.group(1) +
  318. ' is now recommended', line)
  319. split_diff_re = re.compile(r'''
  320. (?P<cmd>
  321. ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
  322. )
  323. (?P<index>
  324. ^ index \s+ .+ $
  325. )
  326. ''',
  327. re.IGNORECASE | re.VERBOSE | re.MULTILINE)
  328. def format_error(self, err):
  329. self.format_ok = False
  330. err = 'Patch format error: ' + err
  331. err2 = 'Line: ' + self.lines[self.line_num].rstrip()
  332. self.error(err, err2)
  333. def error(self, *err):
  334. if self.ok and Verbose.level > Verbose.ONELINE:
  335. print('Code format is not valid:')
  336. self.ok = False
  337. if Verbose.level < Verbose.NORMAL:
  338. return
  339. count = 0
  340. for line in err:
  341. prefix = (' *', ' ')[count > 0]
  342. print(prefix, line)
  343. count += 1
  344. class CheckOnePatch:
  345. """Checks the contents of a git email formatted patch.
  346. Various checks are performed on both the commit message and the
  347. patch content.
  348. """
  349. def __init__(self, name, patch):
  350. self.patch = patch
  351. self.find_patch_pieces()
  352. msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
  353. msg_ok = msg_check.ok
  354. diff_ok = True
  355. if self.diff is not None:
  356. diff_check = GitDiffCheck(self.diff)
  357. diff_ok = diff_check.ok
  358. self.ok = msg_ok and diff_ok
  359. if Verbose.level == Verbose.ONELINE:
  360. if self.ok:
  361. result = 'ok'
  362. else:
  363. result = list()
  364. if not msg_ok:
  365. result.append('commit message')
  366. if not diff_ok:
  367. result.append('diff content')
  368. result = 'bad ' + ' and '.join(result)
  369. print(name, result)
  370. git_diff_re = re.compile(r'''
  371. ^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
  372. ''',
  373. re.IGNORECASE | re.VERBOSE | re.MULTILINE)
  374. stat_re = \
  375. re.compile(r'''
  376. (?P<commit_message> [\s\S\r\n]* )
  377. (?P<stat>
  378. ^ --- $ [\r\n]+
  379. (?: ^ \s+ .+ \s+ \| \s+ \d+ \s+ \+* \-*
  380. $ [\r\n]+ )+
  381. [\s\S\r\n]+
  382. )
  383. ''',
  384. re.IGNORECASE | re.VERBOSE | re.MULTILINE)
  385. subject_prefix_re = \
  386. re.compile(r'''^
  387. \s* (\[
  388. [^\[\]]* # Allow all non-brackets
  389. \])* \s*
  390. ''',
  391. re.VERBOSE)
  392. def find_patch_pieces(self):
  393. if sys.version_info < (3, 0):
  394. patch = self.patch.encode('ascii', 'ignore')
  395. else:
  396. patch = self.patch
  397. self.commit_msg = None
  398. self.stat = None
  399. self.commit_subject = None
  400. self.commit_prefix = None
  401. self.diff = None
  402. if patch.startswith('diff --git'):
  403. self.diff = patch
  404. return
  405. pmail = email.message_from_string(patch)
  406. parts = list(pmail.walk())
  407. assert(len(parts) == 1)
  408. assert(parts[0].get_content_type() == 'text/plain')
  409. content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
  410. mo = self.git_diff_re.search(content)
  411. if mo is not None:
  412. self.diff = content[mo.start():]
  413. content = content[:mo.start()]
  414. mo = self.stat_re.search(content)
  415. if mo is None:
  416. self.commit_msg = content
  417. else:
  418. self.stat = mo.group('stat')
  419. self.commit_msg = mo.group('commit_message')
  420. self.commit_subject = pmail['subject'].replace('\r\n', '')
  421. self.commit_subject = self.commit_subject.replace('\n', '')
  422. self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
  423. class CheckGitCommits:
  424. """Reads patches from git based on the specified git revision range.
  425. The patches are read from git, and then checked.
  426. """
  427. def __init__(self, rev_spec, max_count):
  428. commits = self.read_commit_list_from_git(rev_spec, max_count)
  429. if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
  430. commits = [ rev_spec ]
  431. self.ok = True
  432. blank_line = False
  433. for commit in commits:
  434. if Verbose.level > Verbose.ONELINE:
  435. if blank_line:
  436. print()
  437. else:
  438. blank_line = True
  439. print('Checking git commit:', commit)
  440. patch = self.read_patch_from_git(commit)
  441. self.ok &= CheckOnePatch(commit, patch).ok
  442. if not commits:
  443. print("Couldn't find commit matching: '{}'".format(rev_spec))
  444. def read_commit_list_from_git(self, rev_spec, max_count):
  445. # Run git to get the commit patch
  446. cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
  447. if max_count is not None:
  448. cmd.append('--max-count=' + str(max_count))
  449. cmd.append(rev_spec)
  450. out = self.run_git(*cmd)
  451. return out.split() if out else []
  452. def read_patch_from_git(self, commit):
  453. # Run git to get the commit patch
  454. return self.run_git('show', '--pretty=email', commit)
  455. def run_git(self, *args):
  456. cmd = [ 'git' ]
  457. cmd += args
  458. p = subprocess.Popen(cmd,
  459. stdout=subprocess.PIPE,
  460. stderr=subprocess.STDOUT)
  461. Result = p.communicate()
  462. return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
  463. class CheckOnePatchFile:
  464. """Performs a patch check for a single file.
  465. stdin is used when the filename is '-'.
  466. """
  467. def __init__(self, patch_filename):
  468. if patch_filename == '-':
  469. patch = sys.stdin.read()
  470. patch_filename = 'stdin'
  471. else:
  472. f = open(patch_filename, 'rb')
  473. patch = f.read().decode('utf-8', 'ignore')
  474. f.close()
  475. if Verbose.level > Verbose.ONELINE:
  476. print('Checking patch file:', patch_filename)
  477. self.ok = CheckOnePatch(patch_filename, patch).ok
  478. class CheckOneArg:
  479. """Performs a patch check for a single command line argument.
  480. The argument will be handed off to a file or git-commit based
  481. checker.
  482. """
  483. def __init__(self, param, max_count=None):
  484. self.ok = True
  485. if param == '-' or os.path.exists(param):
  486. checker = CheckOnePatchFile(param)
  487. else:
  488. checker = CheckGitCommits(param, max_count)
  489. self.ok = checker.ok
  490. class PatchCheckApp:
  491. """Checks patches based on the command line arguments."""
  492. def __init__(self):
  493. self.parse_options()
  494. patches = self.args.patches
  495. if len(patches) == 0:
  496. patches = [ 'HEAD' ]
  497. self.ok = True
  498. self.count = None
  499. for patch in patches:
  500. self.process_one_arg(patch)
  501. if self.count is not None:
  502. self.process_one_arg('HEAD')
  503. if self.ok:
  504. self.retval = 0
  505. else:
  506. self.retval = -1
  507. def process_one_arg(self, arg):
  508. if len(arg) >= 2 and arg[0] == '-':
  509. try:
  510. self.count = int(arg[1:])
  511. return
  512. except ValueError:
  513. pass
  514. self.ok &= CheckOneArg(arg, self.count).ok
  515. self.count = None
  516. def parse_options(self):
  517. parser = argparse.ArgumentParser(description=__copyright__)
  518. parser.add_argument('--version', action='version',
  519. version='%(prog)s ' + VersionNumber)
  520. parser.add_argument('patches', nargs='*',
  521. help='[patch file | git rev list]')
  522. group = parser.add_mutually_exclusive_group()
  523. group.add_argument("--oneline",
  524. action="store_true",
  525. help="Print one result per line")
  526. group.add_argument("--silent",
  527. action="store_true",
  528. help="Print nothing")
  529. self.args = parser.parse_args()
  530. if self.args.oneline:
  531. Verbose.level = Verbose.ONELINE
  532. if self.args.silent:
  533. Verbose.level = Verbose.SILENT
  534. if __name__ == "__main__":
  535. sys.exit(PatchCheckApp().retval)