patchstream.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. import datetime
  5. import math
  6. import os
  7. import re
  8. import shutil
  9. import tempfile
  10. from patman import command
  11. from patman import commit
  12. from patman import gitutil
  13. from patman.series import Series
  14. # Tags that we detect and remove
  15. re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Review URL:'
  16. '|Reviewed-on:|Commit-\w*:')
  17. # Lines which are allowed after a TEST= line
  18. re_allowed_after_test = re.compile('^Signed-off-by:')
  19. # Signoffs
  20. re_signoff = re.compile('^Signed-off-by: *(.*)')
  21. # The start of the cover letter
  22. re_cover = re.compile('^Cover-letter:')
  23. # A cover letter Cc
  24. re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
  25. # Patch series tag
  26. re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
  27. # Change-Id will be used to generate the Message-Id and then be stripped
  28. re_change_id = re.compile('^Change-Id: *(.*)')
  29. # Commit series tag
  30. re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
  31. # Commit tags that we want to collect and keep
  32. re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
  33. # The start of a new commit in the git log
  34. re_commit = re.compile('^commit ([0-9a-f]*)$')
  35. # We detect these since checkpatch doesn't always do it
  36. re_space_before_tab = re.compile('^[+].* \t')
  37. # States we can be in - can we use range() and still have comments?
  38. STATE_MSG_HEADER = 0 # Still in the message header
  39. STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
  40. STATE_PATCH_HEADER = 2 # In patch header (after the subject)
  41. STATE_DIFFS = 3 # In the diff part (past --- line)
  42. class PatchStream:
  43. """Class for detecting/injecting tags in a patch or series of patches
  44. We support processing the output of 'git log' to read out the tags we
  45. are interested in. We can also process a patch file in order to remove
  46. unwanted tags or inject additional ones. These correspond to the two
  47. phases of processing.
  48. """
  49. def __init__(self, series, name=None, is_log=False):
  50. self.skip_blank = False # True to skip a single blank line
  51. self.found_test = False # Found a TEST= line
  52. self.lines_after_test = 0 # MNumber of lines found after TEST=
  53. self.warn = [] # List of warnings we have collected
  54. self.linenum = 1 # Output line number we are up to
  55. self.in_section = None # Name of start...END section we are in
  56. self.notes = [] # Series notes
  57. self.section = [] # The current section...END section
  58. self.series = series # Info about the patch series
  59. self.is_log = is_log # True if indent like git log
  60. self.in_change = 0 # Non-zero if we are in a change list
  61. self.blank_count = 0 # Number of blank lines stored up
  62. self.state = STATE_MSG_HEADER # What state are we in?
  63. self.signoff = [] # Contents of signoff line
  64. self.commit = None # Current commit
  65. def AddToSeries(self, line, name, value):
  66. """Add a new Series-xxx tag.
  67. When a Series-xxx tag is detected, we come here to record it, if we
  68. are scanning a 'git log'.
  69. Args:
  70. line: Source line containing tag (useful for debug/error messages)
  71. name: Tag name (part after 'Series-')
  72. value: Tag value (part after 'Series-xxx: ')
  73. """
  74. if name == 'notes':
  75. self.in_section = name
  76. self.skip_blank = False
  77. if self.is_log:
  78. self.series.AddTag(self.commit, line, name, value)
  79. def AddToCommit(self, line, name, value):
  80. """Add a new Commit-xxx tag.
  81. When a Commit-xxx tag is detected, we come here to record it.
  82. Args:
  83. line: Source line containing tag (useful for debug/error messages)
  84. name: Tag name (part after 'Commit-')
  85. value: Tag value (part after 'Commit-xxx: ')
  86. """
  87. if name == 'notes':
  88. self.in_section = 'commit-' + name
  89. self.skip_blank = False
  90. def CloseCommit(self):
  91. """Save the current commit into our commit list, and reset our state"""
  92. if self.commit and self.is_log:
  93. self.series.AddCommit(self.commit)
  94. self.commit = None
  95. # If 'END' is missing in a 'Cover-letter' section, and that section
  96. # happens to show up at the very end of the commit message, this is
  97. # the chance for us to fix it up.
  98. if self.in_section == 'cover' and self.is_log:
  99. self.series.cover = self.section
  100. self.in_section = None
  101. self.skip_blank = True
  102. self.section = []
  103. def ProcessLine(self, line):
  104. """Process a single line of a patch file or commit log
  105. This process a line and returns a list of lines to output. The list
  106. may be empty or may contain multiple output lines.
  107. This is where all the complicated logic is located. The class's
  108. state is used to move between different states and detect things
  109. properly.
  110. We can be in one of two modes:
  111. self.is_log == True: This is 'git log' mode, where most output is
  112. indented by 4 characters and we are scanning for tags
  113. self.is_log == False: This is 'patch' mode, where we already have
  114. all the tags, and are processing patches to remove junk we
  115. don't want, and add things we think are required.
  116. Args:
  117. line: text line to process
  118. Returns:
  119. list of output lines, or [] if nothing should be output
  120. """
  121. # Initially we have no output. Prepare the input line string
  122. out = []
  123. line = line.rstrip('\n')
  124. commit_match = re_commit.match(line) if self.is_log else None
  125. if self.is_log:
  126. if line[:4] == ' ':
  127. line = line[4:]
  128. # Handle state transition and skipping blank lines
  129. series_tag_match = re_series_tag.match(line)
  130. change_id_match = re_change_id.match(line)
  131. commit_tag_match = re_commit_tag.match(line)
  132. cover_match = re_cover.match(line)
  133. cover_cc_match = re_cover_cc.match(line)
  134. signoff_match = re_signoff.match(line)
  135. tag_match = None
  136. if self.state == STATE_PATCH_HEADER:
  137. tag_match = re_tag.match(line)
  138. is_blank = not line.strip()
  139. if is_blank:
  140. if (self.state == STATE_MSG_HEADER
  141. or self.state == STATE_PATCH_SUBJECT):
  142. self.state += 1
  143. # We don't have a subject in the text stream of patch files
  144. # It has its own line with a Subject: tag
  145. if not self.is_log and self.state == STATE_PATCH_SUBJECT:
  146. self.state += 1
  147. elif commit_match:
  148. self.state = STATE_MSG_HEADER
  149. # If a tag is detected, or a new commit starts
  150. if series_tag_match or commit_tag_match or change_id_match or \
  151. cover_match or cover_cc_match or signoff_match or \
  152. self.state == STATE_MSG_HEADER:
  153. # but we are already in a section, this means 'END' is missing
  154. # for that section, fix it up.
  155. if self.in_section:
  156. self.warn.append("Missing 'END' in section '%s'" % self.in_section)
  157. if self.in_section == 'cover':
  158. self.series.cover = self.section
  159. elif self.in_section == 'notes':
  160. if self.is_log:
  161. self.series.notes += self.section
  162. elif self.in_section == 'commit-notes':
  163. if self.is_log:
  164. self.commit.notes += self.section
  165. else:
  166. self.warn.append("Unknown section '%s'" % self.in_section)
  167. self.in_section = None
  168. self.skip_blank = True
  169. self.section = []
  170. # but we are already in a change list, that means a blank line
  171. # is missing, fix it up.
  172. if self.in_change:
  173. self.warn.append("Missing 'blank line' in section 'Series-changes'")
  174. self.in_change = 0
  175. # If we are in a section, keep collecting lines until we see END
  176. if self.in_section:
  177. if line == 'END':
  178. if self.in_section == 'cover':
  179. self.series.cover = self.section
  180. elif self.in_section == 'notes':
  181. if self.is_log:
  182. self.series.notes += self.section
  183. elif self.in_section == 'commit-notes':
  184. if self.is_log:
  185. self.commit.notes += self.section
  186. else:
  187. self.warn.append("Unknown section '%s'" % self.in_section)
  188. self.in_section = None
  189. self.skip_blank = True
  190. self.section = []
  191. else:
  192. self.section.append(line)
  193. # Detect the commit subject
  194. elif not is_blank and self.state == STATE_PATCH_SUBJECT:
  195. self.commit.subject = line
  196. # Detect the tags we want to remove, and skip blank lines
  197. elif re_remove.match(line) and not commit_tag_match:
  198. self.skip_blank = True
  199. # TEST= should be the last thing in the commit, so remove
  200. # everything after it
  201. if line.startswith('TEST='):
  202. self.found_test = True
  203. elif self.skip_blank and is_blank:
  204. self.skip_blank = False
  205. # Detect the start of a cover letter section
  206. elif cover_match:
  207. self.in_section = 'cover'
  208. self.skip_blank = False
  209. elif cover_cc_match:
  210. value = cover_cc_match.group(1)
  211. self.AddToSeries(line, 'cover-cc', value)
  212. # If we are in a change list, key collected lines until a blank one
  213. elif self.in_change:
  214. if is_blank:
  215. # Blank line ends this change list
  216. self.in_change = 0
  217. elif line == '---':
  218. self.in_change = 0
  219. out = self.ProcessLine(line)
  220. else:
  221. if self.is_log:
  222. self.series.AddChange(self.in_change, self.commit, line)
  223. self.skip_blank = False
  224. # Detect Series-xxx tags
  225. elif series_tag_match:
  226. name = series_tag_match.group(1)
  227. value = series_tag_match.group(2)
  228. if name == 'changes':
  229. # value is the version number: e.g. 1, or 2
  230. try:
  231. value = int(value)
  232. except ValueError as str:
  233. raise ValueError("%s: Cannot decode version info '%s'" %
  234. (self.commit.hash, line))
  235. self.in_change = int(value)
  236. else:
  237. self.AddToSeries(line, name, value)
  238. self.skip_blank = True
  239. # Detect Change-Id tags
  240. elif change_id_match:
  241. value = change_id_match.group(1)
  242. if self.is_log:
  243. if self.commit.change_id:
  244. raise ValueError("%s: Two Change-Ids: '%s' vs. '%s'" %
  245. (self.commit.hash, self.commit.change_id, value))
  246. self.commit.change_id = value
  247. self.skip_blank = True
  248. # Detect Commit-xxx tags
  249. elif commit_tag_match:
  250. name = commit_tag_match.group(1)
  251. value = commit_tag_match.group(2)
  252. if name == 'notes':
  253. self.AddToCommit(line, name, value)
  254. self.skip_blank = True
  255. # Detect the start of a new commit
  256. elif commit_match:
  257. self.CloseCommit()
  258. self.commit = commit.Commit(commit_match.group(1))
  259. # Detect tags in the commit message
  260. elif tag_match:
  261. # Remove Tested-by self, since few will take much notice
  262. if (tag_match.group(1) == 'Tested-by' and
  263. tag_match.group(2).find(os.getenv('USER') + '@') != -1):
  264. self.warn.append("Ignoring %s" % line)
  265. elif tag_match.group(1) == 'Patch-cc':
  266. self.commit.AddCc(tag_match.group(2).split(','))
  267. else:
  268. out = [line]
  269. # Suppress duplicate signoffs
  270. elif signoff_match:
  271. if (self.is_log or not self.commit or
  272. self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
  273. out = [line]
  274. # Well that means this is an ordinary line
  275. else:
  276. # Look for space before tab
  277. m = re_space_before_tab.match(line)
  278. if m:
  279. self.warn.append('Line %d/%d has space before tab' %
  280. (self.linenum, m.start()))
  281. # OK, we have a valid non-blank line
  282. out = [line]
  283. self.linenum += 1
  284. self.skip_blank = False
  285. if self.state == STATE_DIFFS:
  286. pass
  287. # If this is the start of the diffs section, emit our tags and
  288. # change log
  289. elif line == '---':
  290. self.state = STATE_DIFFS
  291. # Output the tags (signeoff first), then change list
  292. out = []
  293. log = self.series.MakeChangeLog(self.commit)
  294. out += [line]
  295. if self.commit:
  296. out += self.commit.notes
  297. out += [''] + log
  298. elif self.found_test:
  299. if not re_allowed_after_test.match(line):
  300. self.lines_after_test += 1
  301. return out
  302. def Finalize(self):
  303. """Close out processing of this patch stream"""
  304. self.CloseCommit()
  305. if self.lines_after_test:
  306. self.warn.append('Found %d lines after TEST=' %
  307. self.lines_after_test)
  308. def WriteMessageId(self, outfd):
  309. """Write the Message-Id into the output.
  310. This is based on the Change-Id in the original patch, the version,
  311. and the prefix.
  312. Args:
  313. outfd: Output stream file object
  314. """
  315. if not self.commit.change_id:
  316. return
  317. # If the count is -1 we're testing, so use a fixed time
  318. if self.commit.count == -1:
  319. time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
  320. else:
  321. time_now = datetime.datetime.now()
  322. # In theory there is email.utils.make_msgid() which would be nice
  323. # to use, but it already produces something way too long and thus
  324. # will produce ugly commit lines if someone throws this into
  325. # a "Link:" tag in the final commit. So (sigh) roll our own.
  326. # Start with the time; presumably we wouldn't send the same series
  327. # with the same Change-Id at the exact same second.
  328. parts = [time_now.strftime("%Y%m%d%H%M%S")]
  329. # These seem like they would be nice to include.
  330. if 'prefix' in self.series:
  331. parts.append(self.series['prefix'])
  332. if 'version' in self.series:
  333. parts.append("v%s" % self.series['version'])
  334. parts.append(str(self.commit.count + 1))
  335. # The Change-Id must be last, right before the @
  336. parts.append(self.commit.change_id)
  337. # Join parts together with "." and write it out.
  338. outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
  339. def ProcessStream(self, infd, outfd):
  340. """Copy a stream from infd to outfd, filtering out unwanting things.
  341. This is used to process patch files one at a time.
  342. Args:
  343. infd: Input stream file object
  344. outfd: Output stream file object
  345. """
  346. # Extract the filename from each diff, for nice warnings
  347. fname = None
  348. last_fname = None
  349. re_fname = re.compile('diff --git a/(.*) b/.*')
  350. self.WriteMessageId(outfd)
  351. while True:
  352. line = infd.readline()
  353. if not line:
  354. break
  355. out = self.ProcessLine(line)
  356. # Try to detect blank lines at EOF
  357. for line in out:
  358. match = re_fname.match(line)
  359. if match:
  360. last_fname = fname
  361. fname = match.group(1)
  362. if line == '+':
  363. self.blank_count += 1
  364. else:
  365. if self.blank_count and (line == '-- ' or match):
  366. self.warn.append("Found possible blank line(s) at "
  367. "end of file '%s'" % last_fname)
  368. outfd.write('+\n' * self.blank_count)
  369. outfd.write(line + '\n')
  370. self.blank_count = 0
  371. self.Finalize()
  372. def GetMetaDataForList(commit_range, git_dir=None, count=None,
  373. series = None, allow_overwrite=False):
  374. """Reads out patch series metadata from the commits
  375. This does a 'git log' on the relevant commits and pulls out the tags we
  376. are interested in.
  377. Args:
  378. commit_range: Range of commits to count (e.g. 'HEAD..base')
  379. git_dir: Path to git repositiory (None to use default)
  380. count: Number of commits to list, or None for no limit
  381. series: Series object to add information into. By default a new series
  382. is started.
  383. allow_overwrite: Allow tags to overwrite an existing tag
  384. Returns:
  385. A Series object containing information about the commits.
  386. """
  387. if not series:
  388. series = Series()
  389. series.allow_overwrite = allow_overwrite
  390. params = gitutil.LogCmd(commit_range, reverse=True, count=count,
  391. git_dir=git_dir)
  392. stdout = command.RunPipe([params], capture=True).stdout
  393. ps = PatchStream(series, is_log=True)
  394. for line in stdout.splitlines():
  395. ps.ProcessLine(line)
  396. ps.Finalize()
  397. return series
  398. def GetMetaData(start, count):
  399. """Reads out patch series metadata from the commits
  400. This does a 'git log' on the relevant commits and pulls out the tags we
  401. are interested in.
  402. Args:
  403. start: Commit to start from: 0=HEAD, 1=next one, etc.
  404. count: Number of commits to list
  405. """
  406. return GetMetaDataForList('HEAD~%d' % start, None, count)
  407. def GetMetaDataForTest(text):
  408. """Process metadata from a file containing a git log. Used for tests
  409. Args:
  410. text:
  411. """
  412. series = Series()
  413. ps = PatchStream(series, is_log=True)
  414. for line in text.splitlines():
  415. ps.ProcessLine(line)
  416. ps.Finalize()
  417. return series
  418. def FixPatch(backup_dir, fname, series, commit):
  419. """Fix up a patch file, by adding/removing as required.
  420. We remove our tags from the patch file, insert changes lists, etc.
  421. The patch file is processed in place, and overwritten.
  422. A backup file is put into backup_dir (if not None).
  423. Args:
  424. fname: Filename to patch file to process
  425. series: Series information about this patch set
  426. commit: Commit object for this patch file
  427. Return:
  428. A list of errors, or [] if all ok.
  429. """
  430. handle, tmpname = tempfile.mkstemp()
  431. outfd = os.fdopen(handle, 'w', encoding='utf-8')
  432. infd = open(fname, 'r', encoding='utf-8')
  433. ps = PatchStream(series)
  434. ps.commit = commit
  435. ps.ProcessStream(infd, outfd)
  436. infd.close()
  437. outfd.close()
  438. # Create a backup file if required
  439. if backup_dir:
  440. shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
  441. shutil.move(tmpname, fname)
  442. return ps.warn
  443. def FixPatches(series, fnames):
  444. """Fix up a list of patches identified by filenames
  445. The patch files are processed in place, and overwritten.
  446. Args:
  447. series: The series object
  448. fnames: List of patch files to process
  449. """
  450. # Current workflow creates patches, so we shouldn't need a backup
  451. backup_dir = None #tempfile.mkdtemp('clean-patch')
  452. count = 0
  453. for fname in fnames:
  454. commit = series.commits[count]
  455. commit.patch = fname
  456. commit.count = count
  457. result = FixPatch(backup_dir, fname, series, commit)
  458. if result:
  459. print('%d warnings for %s:' % (len(result), fname))
  460. for warn in result:
  461. print('\t', warn)
  462. print
  463. count += 1
  464. print('Cleaned %d patches' % count)
  465. def InsertCoverLetter(fname, series, count):
  466. """Inserts a cover letter with the required info into patch 0
  467. Args:
  468. fname: Input / output filename of the cover letter file
  469. series: Series object
  470. count: Number of patches in the series
  471. """
  472. fd = open(fname, 'r')
  473. lines = fd.readlines()
  474. fd.close()
  475. fd = open(fname, 'w')
  476. text = series.cover
  477. prefix = series.GetPatchPrefix()
  478. for line in lines:
  479. if line.startswith('Subject:'):
  480. # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
  481. zero_repeat = int(math.log10(count)) + 1
  482. zero = '0' * zero_repeat
  483. line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
  484. # Insert our cover letter
  485. elif line.startswith('*** BLURB HERE ***'):
  486. # First the blurb test
  487. line = '\n'.join(text[1:]) + '\n'
  488. if series.get('notes'):
  489. line += '\n'.join(series.notes) + '\n'
  490. # Now the change list
  491. out = series.MakeChangeLog(None)
  492. line += '\n' + '\n'.join(out)
  493. fd.write(line)
  494. fd.close()