patchstream.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. import datetime
  5. import math
  6. import os
  7. import re
  8. import shutil
  9. import tempfile
  10. from patman import command
  11. from patman import commit
  12. from patman import gitutil
  13. from patman.series import Series
  14. # Tags that we detect and remove
  15. re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
  16. '|Reviewed-on:|Commit-\w*:')
  17. # Lines which are allowed after a TEST= line
  18. re_allowed_after_test = re.compile('^Signed-off-by:')
  19. # Signoffs
  20. re_signoff = re.compile('^Signed-off-by: *(.*)')
  21. # Cover letter tag
  22. re_cover = re.compile('^Cover-([a-z-]*): *(.*)')
  23. # Patch series tag
  24. re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
  25. # Change-Id will be used to generate the Message-Id and then be stripped
  26. re_change_id = re.compile('^Change-Id: *(.*)')
  27. # Commit series tag
  28. re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
  29. # Commit tags that we want to collect and keep
  30. re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
  31. # The start of a new commit in the git log
  32. re_commit = re.compile('^commit ([0-9a-f]*)$')
  33. # We detect these since checkpatch doesn't always do it
  34. re_space_before_tab = re.compile('^[+].* \t')
  35. # Match indented lines for changes
  36. re_leading_whitespace = re.compile('^\s')
  37. # States we can be in - can we use range() and still have comments?
  38. STATE_MSG_HEADER = 0 # Still in the message header
  39. STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
  40. STATE_PATCH_HEADER = 2 # In patch header (after the subject)
  41. STATE_DIFFS = 3 # In the diff part (past --- line)
  42. class PatchStream:
  43. """Class for detecting/injecting tags in a patch or series of patches
  44. We support processing the output of 'git log' to read out the tags we
  45. are interested in. We can also process a patch file in order to remove
  46. unwanted tags or inject additional ones. These correspond to the two
  47. phases of processing.
  48. """
  49. def __init__(self, series, name=None, is_log=False):
  50. self.skip_blank = False # True to skip a single blank line
  51. self.found_test = False # Found a TEST= line
  52. self.lines_after_test = 0 # Number of lines found after TEST=
  53. self.warn = [] # List of warnings we have collected
  54. self.linenum = 1 # Output line number we are up to
  55. self.in_section = None # Name of start...END section we are in
  56. self.notes = [] # Series notes
  57. self.section = [] # The current section...END section
  58. self.series = series # Info about the patch series
  59. self.is_log = is_log # True if indent like git log
  60. self.in_change = None # Name of the change list we are in
  61. self.change_version = 0 # Non-zero if we are in a change list
  62. self.change_lines = [] # Lines of the current change
  63. self.blank_count = 0 # Number of blank lines stored up
  64. self.state = STATE_MSG_HEADER # What state are we in?
  65. self.signoff = [] # Contents of signoff line
  66. self.commit = None # Current commit
  67. def AddToSeries(self, line, name, value):
  68. """Add a new Series-xxx tag.
  69. When a Series-xxx tag is detected, we come here to record it, if we
  70. are scanning a 'git log'.
  71. Args:
  72. line: Source line containing tag (useful for debug/error messages)
  73. name: Tag name (part after 'Series-')
  74. value: Tag value (part after 'Series-xxx: ')
  75. """
  76. if name == 'notes':
  77. self.in_section = name
  78. self.skip_blank = False
  79. if self.is_log:
  80. self.series.AddTag(self.commit, line, name, value)
  81. def AddToCommit(self, line, name, value):
  82. """Add a new Commit-xxx tag.
  83. When a Commit-xxx tag is detected, we come here to record it.
  84. Args:
  85. line: Source line containing tag (useful for debug/error messages)
  86. name: Tag name (part after 'Commit-')
  87. value: Tag value (part after 'Commit-xxx: ')
  88. """
  89. if name == 'notes':
  90. self.in_section = 'commit-' + name
  91. self.skip_blank = False
  92. def CloseCommit(self):
  93. """Save the current commit into our commit list, and reset our state"""
  94. if self.commit and self.is_log:
  95. self.series.AddCommit(self.commit)
  96. self.commit = None
  97. # If 'END' is missing in a 'Cover-letter' section, and that section
  98. # happens to show up at the very end of the commit message, this is
  99. # the chance for us to fix it up.
  100. if self.in_section == 'cover' and self.is_log:
  101. self.series.cover = self.section
  102. self.in_section = None
  103. self.skip_blank = True
  104. self.section = []
  105. def ParseVersion(self, value, line):
  106. """Parse a version from a *-changes tag
  107. Args:
  108. value: Tag value (part after 'xxx-changes: '
  109. line: Source line containing tag
  110. Returns:
  111. The version as an integer
  112. """
  113. try:
  114. return int(value)
  115. except ValueError as str:
  116. raise ValueError("%s: Cannot decode version info '%s'" %
  117. (self.commit.hash, line))
  118. def FinalizeChange(self):
  119. """Finalize a (multi-line) change and add it to the series or commit"""
  120. if not self.change_lines:
  121. return
  122. change = '\n'.join(self.change_lines)
  123. if self.in_change == 'Series':
  124. self.series.AddChange(self.change_version, self.commit, change)
  125. elif self.in_change == 'Cover':
  126. self.series.AddChange(self.change_version, None, change)
  127. elif self.in_change == 'Commit':
  128. self.commit.AddChange(self.change_version, change)
  129. self.change_lines = []
  130. def ProcessLine(self, line):
  131. """Process a single line of a patch file or commit log
  132. This process a line and returns a list of lines to output. The list
  133. may be empty or may contain multiple output lines.
  134. This is where all the complicated logic is located. The class's
  135. state is used to move between different states and detect things
  136. properly.
  137. We can be in one of two modes:
  138. self.is_log == True: This is 'git log' mode, where most output is
  139. indented by 4 characters and we are scanning for tags
  140. self.is_log == False: This is 'patch' mode, where we already have
  141. all the tags, and are processing patches to remove junk we
  142. don't want, and add things we think are required.
  143. Args:
  144. line: text line to process
  145. Returns:
  146. list of output lines, or [] if nothing should be output
  147. """
  148. # Initially we have no output. Prepare the input line string
  149. out = []
  150. line = line.rstrip('\n')
  151. commit_match = re_commit.match(line) if self.is_log else None
  152. if self.is_log:
  153. if line[:4] == ' ':
  154. line = line[4:]
  155. # Handle state transition and skipping blank lines
  156. series_tag_match = re_series_tag.match(line)
  157. change_id_match = re_change_id.match(line)
  158. commit_tag_match = re_commit_tag.match(line)
  159. cover_match = re_cover.match(line)
  160. signoff_match = re_signoff.match(line)
  161. leading_whitespace_match = re_leading_whitespace.match(line)
  162. tag_match = None
  163. if self.state == STATE_PATCH_HEADER:
  164. tag_match = re_tag.match(line)
  165. is_blank = not line.strip()
  166. if is_blank:
  167. if (self.state == STATE_MSG_HEADER
  168. or self.state == STATE_PATCH_SUBJECT):
  169. self.state += 1
  170. # We don't have a subject in the text stream of patch files
  171. # It has its own line with a Subject: tag
  172. if not self.is_log and self.state == STATE_PATCH_SUBJECT:
  173. self.state += 1
  174. elif commit_match:
  175. self.state = STATE_MSG_HEADER
  176. # If a tag is detected, or a new commit starts
  177. if series_tag_match or commit_tag_match or change_id_match or \
  178. cover_match or signoff_match or self.state == STATE_MSG_HEADER:
  179. # but we are already in a section, this means 'END' is missing
  180. # for that section, fix it up.
  181. if self.in_section:
  182. self.warn.append("Missing 'END' in section '%s'" % self.in_section)
  183. if self.in_section == 'cover':
  184. self.series.cover = self.section
  185. elif self.in_section == 'notes':
  186. if self.is_log:
  187. self.series.notes += self.section
  188. elif self.in_section == 'commit-notes':
  189. if self.is_log:
  190. self.commit.notes += self.section
  191. else:
  192. self.warn.append("Unknown section '%s'" % self.in_section)
  193. self.in_section = None
  194. self.skip_blank = True
  195. self.section = []
  196. # but we are already in a change list, that means a blank line
  197. # is missing, fix it up.
  198. if self.in_change:
  199. self.warn.append("Missing 'blank line' in section '%s-changes'" % self.in_change)
  200. self.FinalizeChange()
  201. self.in_change = None
  202. self.change_version = 0
  203. # If we are in a section, keep collecting lines until we see END
  204. if self.in_section:
  205. if line == 'END':
  206. if self.in_section == 'cover':
  207. self.series.cover = self.section
  208. elif self.in_section == 'notes':
  209. if self.is_log:
  210. self.series.notes += self.section
  211. elif self.in_section == 'commit-notes':
  212. if self.is_log:
  213. self.commit.notes += self.section
  214. else:
  215. self.warn.append("Unknown section '%s'" % self.in_section)
  216. self.in_section = None
  217. self.skip_blank = True
  218. self.section = []
  219. else:
  220. self.section.append(line)
  221. # Detect the commit subject
  222. elif not is_blank and self.state == STATE_PATCH_SUBJECT:
  223. self.commit.subject = line
  224. # Detect the tags we want to remove, and skip blank lines
  225. elif re_remove.match(line) and not commit_tag_match:
  226. self.skip_blank = True
  227. # TEST= should be the last thing in the commit, so remove
  228. # everything after it
  229. if line.startswith('TEST='):
  230. self.found_test = True
  231. elif self.skip_blank and is_blank:
  232. self.skip_blank = False
  233. # Detect Cover-xxx tags
  234. elif cover_match:
  235. name = cover_match.group(1)
  236. value = cover_match.group(2)
  237. if name == 'letter':
  238. self.in_section = 'cover'
  239. self.skip_blank = False
  240. elif name == 'letter-cc':
  241. self.AddToSeries(line, 'cover-cc', value)
  242. elif name == 'changes':
  243. self.in_change = 'Cover'
  244. self.change_version = self.ParseVersion(value, line)
  245. # If we are in a change list, key collected lines until a blank one
  246. elif self.in_change:
  247. if is_blank:
  248. # Blank line ends this change list
  249. self.FinalizeChange()
  250. self.in_change = None
  251. self.change_version = 0
  252. elif line == '---':
  253. self.FinalizeChange()
  254. self.in_change = None
  255. self.change_version = 0
  256. out = self.ProcessLine(line)
  257. elif self.is_log:
  258. if not leading_whitespace_match:
  259. self.FinalizeChange()
  260. self.change_lines.append(line)
  261. self.skip_blank = False
  262. # Detect Series-xxx tags
  263. elif series_tag_match:
  264. name = series_tag_match.group(1)
  265. value = series_tag_match.group(2)
  266. if name == 'changes':
  267. # value is the version number: e.g. 1, or 2
  268. self.in_change = 'Series'
  269. self.change_version = self.ParseVersion(value, line)
  270. else:
  271. self.AddToSeries(line, name, value)
  272. self.skip_blank = True
  273. # Detect Change-Id tags
  274. elif change_id_match:
  275. value = change_id_match.group(1)
  276. if self.is_log:
  277. if self.commit.change_id:
  278. raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
  279. (self.commit.hash, self.commit.change_id, value))
  280. self.commit.change_id = value
  281. self.skip_blank = True
  282. # Detect Commit-xxx tags
  283. elif commit_tag_match:
  284. name = commit_tag_match.group(1)
  285. value = commit_tag_match.group(2)
  286. if name == 'notes':
  287. self.AddToCommit(line, name, value)
  288. self.skip_blank = True
  289. elif name == 'changes':
  290. self.in_change = 'Commit'
  291. self.change_version = self.ParseVersion(value, line)
  292. # Detect the start of a new commit
  293. elif commit_match:
  294. self.CloseCommit()
  295. self.commit = commit.Commit(commit_match.group(1))
  296. # Detect tags in the commit message
  297. elif tag_match:
  298. # Remove Tested-by self, since few will take much notice
  299. if (tag_match.group(1) == 'Tested-by' and
  300. tag_match.group(2).find(os.getenv('USER') + '@') != -1):
  301. self.warn.append("Ignoring %s" % line)
  302. elif tag_match.group(1) == 'Patch-cc':
  303. self.commit.AddCc(tag_match.group(2).split(','))
  304. else:
  305. out = [line]
  306. # Suppress duplicate signoffs
  307. elif signoff_match:
  308. if (self.is_log or not self.commit or
  309. self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
  310. out = [line]
  311. # Well that means this is an ordinary line
  312. else:
  313. # Look for space before tab
  314. m = re_space_before_tab.match(line)
  315. if m:
  316. self.warn.append('Line %d/%d has space before tab' %
  317. (self.linenum, m.start()))
  318. # OK, we have a valid non-blank line
  319. out = [line]
  320. self.linenum += 1
  321. self.skip_blank = False
  322. if self.state == STATE_DIFFS:
  323. pass
  324. # If this is the start of the diffs section, emit our tags and
  325. # change log
  326. elif line == '---':
  327. self.state = STATE_DIFFS
  328. # Output the tags (signoff first), then change list
  329. out = []
  330. log = self.series.MakeChangeLog(self.commit)
  331. out += [line]
  332. if self.commit:
  333. out += self.commit.notes
  334. out += [''] + log
  335. elif self.found_test:
  336. if not re_allowed_after_test.match(line):
  337. self.lines_after_test += 1
  338. return out
  339. def Finalize(self):
  340. """Close out processing of this patch stream"""
  341. self.FinalizeChange()
  342. self.CloseCommit()
  343. if self.lines_after_test:
  344. self.warn.append('Found %d lines after TEST=' %
  345. self.lines_after_test)
  346. def WriteMessageId(self, outfd):
  347. """Write the Message-Id into the output.
  348. This is based on the Change-Id in the original patch, the version,
  349. and the prefix.
  350. Args:
  351. outfd: Output stream file object
  352. """
  353. if not self.commit.change_id:
  354. return
  355. # If the count is -1 we're testing, so use a fixed time
  356. if self.commit.count == -1:
  357. time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
  358. else:
  359. time_now = datetime.datetime.now()
  360. # In theory there is email.utils.make_msgid() which would be nice
  361. # to use, but it already produces something way too long and thus
  362. # will produce ugly commit lines if someone throws this into
  363. # a "Link:" tag in the final commit. So (sigh) roll our own.
  364. # Start with the time; presumably we wouldn't send the same series
  365. # with the same Change-Id at the exact same second.
  366. parts = [time_now.strftime("%Y%m%d%H%M%S")]
  367. # These seem like they would be nice to include.
  368. if 'prefix' in self.series:
  369. parts.append(self.series['prefix'])
  370. if 'version' in self.series:
  371. parts.append("v%s" % self.series['version'])
  372. parts.append(str(self.commit.count + 1))
  373. # The Change-Id must be last, right before the @
  374. parts.append(self.commit.change_id)
  375. # Join parts together with "." and write it out.
  376. outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
  377. def ProcessStream(self, infd, outfd):
  378. """Copy a stream from infd to outfd, filtering out unwanting things.
  379. This is used to process patch files one at a time.
  380. Args:
  381. infd: Input stream file object
  382. outfd: Output stream file object
  383. """
  384. # Extract the filename from each diff, for nice warnings
  385. fname = None
  386. last_fname = None
  387. re_fname = re.compile('diff --git a/(.*) b/.*')
  388. self.WriteMessageId(outfd)
  389. while True:
  390. line = infd.readline()
  391. if not line:
  392. break
  393. out = self.ProcessLine(line)
  394. # Try to detect blank lines at EOF
  395. for line in out:
  396. match = re_fname.match(line)
  397. if match:
  398. last_fname = fname
  399. fname = match.group(1)
  400. if line == '+':
  401. self.blank_count += 1
  402. else:
  403. if self.blank_count and (line == '-- ' or match):
  404. self.warn.append("Found possible blank line(s) at "
  405. "end of file '%s'" % last_fname)
  406. outfd.write('+\n' * self.blank_count)
  407. outfd.write(line + '\n')
  408. self.blank_count = 0
  409. self.Finalize()
  410. def GetMetaDataForList(commit_range, git_dir=None, count=None,
  411. series = None, allow_overwrite=False):
  412. """Reads out patch series metadata from the commits
  413. This does a 'git log' on the relevant commits and pulls out the tags we
  414. are interested in.
  415. Args:
  416. commit_range: Range of commits to count (e.g. 'HEAD..base')
  417. git_dir: Path to git repositiory (None to use default)
  418. count: Number of commits to list, or None for no limit
  419. series: Series object to add information into. By default a new series
  420. is started.
  421. allow_overwrite: Allow tags to overwrite an existing tag
  422. Returns:
  423. A Series object containing information about the commits.
  424. """
  425. if not series:
  426. series = Series()
  427. series.allow_overwrite = allow_overwrite
  428. params = gitutil.LogCmd(commit_range, reverse=True, count=count,
  429. git_dir=git_dir)
  430. stdout = command.RunPipe([params], capture=True).stdout
  431. ps = PatchStream(series, is_log=True)
  432. for line in stdout.splitlines():
  433. ps.ProcessLine(line)
  434. ps.Finalize()
  435. return series
  436. def GetMetaData(start, count):
  437. """Reads out patch series metadata from the commits
  438. This does a 'git log' on the relevant commits and pulls out the tags we
  439. are interested in.
  440. Args:
  441. start: Commit to start from: 0=HEAD, 1=next one, etc.
  442. count: Number of commits to list
  443. """
  444. return GetMetaDataForList('HEAD~%d' % start, None, count)
  445. def GetMetaDataForTest(text):
  446. """Process metadata from a file containing a git log. Used for tests
  447. Args:
  448. text:
  449. """
  450. series = Series()
  451. ps = PatchStream(series, is_log=True)
  452. for line in text.splitlines():
  453. ps.ProcessLine(line)
  454. ps.Finalize()
  455. return series
  456. def FixPatch(backup_dir, fname, series, commit):
  457. """Fix up a patch file, by adding/removing as required.
  458. We remove our tags from the patch file, insert changes lists, etc.
  459. The patch file is processed in place, and overwritten.
  460. A backup file is put into backup_dir (if not None).
  461. Args:
  462. fname: Filename to patch file to process
  463. series: Series information about this patch set
  464. commit: Commit object for this patch file
  465. Return:
  466. A list of errors, or [] if all ok.
  467. """
  468. handle, tmpname = tempfile.mkstemp()
  469. outfd = os.fdopen(handle, 'w', encoding='utf-8')
  470. infd = open(fname, 'r', encoding='utf-8')
  471. ps = PatchStream(series)
  472. ps.commit = commit
  473. ps.ProcessStream(infd, outfd)
  474. infd.close()
  475. outfd.close()
  476. # Create a backup file if required
  477. if backup_dir:
  478. shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
  479. shutil.move(tmpname, fname)
  480. return ps.warn
  481. def FixPatches(series, fnames):
  482. """Fix up a list of patches identified by filenames
  483. The patch files are processed in place, and overwritten.
  484. Args:
  485. series: The series object
  486. fnames: List of patch files to process
  487. """
  488. # Current workflow creates patches, so we shouldn't need a backup
  489. backup_dir = None #tempfile.mkdtemp('clean-patch')
  490. count = 0
  491. for fname in fnames:
  492. commit = series.commits[count]
  493. commit.patch = fname
  494. commit.count = count
  495. result = FixPatch(backup_dir, fname, series, commit)
  496. if result:
  497. print('%d warnings for %s:' % (len(result), fname))
  498. for warn in result:
  499. print('\t', warn)
  500. print
  501. count += 1
  502. print('Cleaned %d patches' % count)
  503. def InsertCoverLetter(fname, series, count):
  504. """Inserts a cover letter with the required info into patch 0
  505. Args:
  506. fname: Input / output filename of the cover letter file
  507. series: Series object
  508. count: Number of patches in the series
  509. """
  510. fd = open(fname, 'r')
  511. lines = fd.readlines()
  512. fd.close()
  513. fd = open(fname, 'w')
  514. text = series.cover
  515. prefix = series.GetPatchPrefix()
  516. for line in lines:
  517. if line.startswith('Subject:'):
  518. # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
  519. zero_repeat = int(math.log10(count)) + 1
  520. zero = '0' * zero_repeat
  521. line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
  522. # Insert our cover letter
  523. elif line.startswith('*** BLURB HERE ***'):
  524. # First the blurb test
  525. line = '\n'.join(text[1:]) + '\n'
  526. if series.get('notes'):
  527. line += '\n'.join(series.notes) + '\n'
  528. # Now the change list
  529. out = series.MakeChangeLog(None)
  530. line += '\n' + '\n'.join(out)
  531. fd.write(line)
  532. fd.close()