patchstream.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. # Copyright (c) 2011 The Chromium OS Authors.
  2. #
  3. # See file CREDITS for list of people who contributed to this
  4. # project.
  5. #
  6. # This program is free software; you can redistribute it and/or
  7. # modify it under the terms of the GNU General Public License as
  8. # published by the Free Software Foundation; either version 2 of
  9. # the License, or (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program; if not, write to the Free Software
  18. # Foundation, Inc., 59 Temple Place, Suite 330, Boston,
  19. # MA 02111-1307 USA
  20. #
  21. import os
  22. import re
  23. import shutil
  24. import tempfile
  25. import command
  26. import commit
  27. import gitutil
  28. from series import Series
  29. # Tags that we detect and remove
  30. re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
  31. '|Reviewed-on:|Commit-\w*:')
  32. # Lines which are allowed after a TEST= line
  33. re_allowed_after_test = re.compile('^Signed-off-by:')
  34. # Signoffs
  35. re_signoff = re.compile('^Signed-off-by:')
  36. # The start of the cover letter
  37. re_cover = re.compile('^Cover-letter:')
  38. # A cover letter Cc
  39. re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
  40. # Patch series tag
  41. re_series = re.compile('^Series-(\w*): *(.*)')
  42. # Commit tags that we want to collect and keep
  43. re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Cc): (.*)')
  44. # The start of a new commit in the git log
  45. re_commit = re.compile('^commit (.*)')
  46. # We detect these since checkpatch doesn't always do it
  47. re_space_before_tab = re.compile('^[+].* \t')
  48. # States we can be in - can we use range() and still have comments?
  49. STATE_MSG_HEADER = 0 # Still in the message header
  50. STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
  51. STATE_PATCH_HEADER = 2 # In patch header (after the subject)
  52. STATE_DIFFS = 3 # In the diff part (past --- line)
  53. class PatchStream:
  54. """Class for detecting/injecting tags in a patch or series of patches
  55. We support processing the output of 'git log' to read out the tags we
  56. are interested in. We can also process a patch file in order to remove
  57. unwanted tags or inject additional ones. These correspond to the two
  58. phases of processing.
  59. """
  60. def __init__(self, series, name=None, is_log=False):
  61. self.skip_blank = False # True to skip a single blank line
  62. self.found_test = False # Found a TEST= line
  63. self.lines_after_test = 0 # MNumber of lines found after TEST=
  64. self.warn = [] # List of warnings we have collected
  65. self.linenum = 1 # Output line number we are up to
  66. self.in_section = None # Name of start...END section we are in
  67. self.notes = [] # Series notes
  68. self.section = [] # The current section...END section
  69. self.series = series # Info about the patch series
  70. self.is_log = is_log # True if indent like git log
  71. self.in_change = 0 # Non-zero if we are in a change list
  72. self.blank_count = 0 # Number of blank lines stored up
  73. self.state = STATE_MSG_HEADER # What state are we in?
  74. self.tags = [] # Tags collected, like Tested-by...
  75. self.signoff = [] # Contents of signoff line
  76. self.commit = None # Current commit
  77. def AddToSeries(self, line, name, value):
  78. """Add a new Series-xxx tag.
  79. When a Series-xxx tag is detected, we come here to record it, if we
  80. are scanning a 'git log'.
  81. Args:
  82. line: Source line containing tag (useful for debug/error messages)
  83. name: Tag name (part after 'Series-')
  84. value: Tag value (part after 'Series-xxx: ')
  85. """
  86. if name == 'notes':
  87. self.in_section = name
  88. self.skip_blank = False
  89. if self.is_log:
  90. self.series.AddTag(self.commit, line, name, value)
  91. def CloseCommit(self):
  92. """Save the current commit into our commit list, and reset our state"""
  93. if self.commit and self.is_log:
  94. self.series.AddCommit(self.commit)
  95. self.commit = None
  96. def FormatTags(self, tags):
  97. out_list = []
  98. for tag in sorted(tags):
  99. if tag.startswith('Cc:'):
  100. tag_list = tag[4:].split(',')
  101. out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
  102. else:
  103. out_list.append(tag)
  104. return out_list
  105. def ProcessLine(self, line):
  106. """Process a single line of a patch file or commit log
  107. This process a line and returns a list of lines to output. The list
  108. may be empty or may contain multiple output lines.
  109. This is where all the complicated logic is located. The class's
  110. state is used to move between different states and detect things
  111. properly.
  112. We can be in one of two modes:
  113. self.is_log == True: This is 'git log' mode, where most output is
  114. indented by 4 characters and we are scanning for tags
  115. self.is_log == False: This is 'patch' mode, where we already have
  116. all the tags, and are processing patches to remove junk we
  117. don't want, and add things we think are required.
  118. Args:
  119. line: text line to process
  120. Returns:
  121. list of output lines, or [] if nothing should be output
  122. """
  123. # Initially we have no output. Prepare the input line string
  124. out = []
  125. line = line.rstrip('\n')
  126. if self.is_log:
  127. if line[:4] == ' ':
  128. line = line[4:]
  129. # Handle state transition and skipping blank lines
  130. series_match = re_series.match(line)
  131. commit_match = re_commit.match(line) if self.is_log else None
  132. cover_cc_match = re_cover_cc.match(line)
  133. tag_match = None
  134. if self.state == STATE_PATCH_HEADER:
  135. tag_match = re_tag.match(line)
  136. is_blank = not line.strip()
  137. if is_blank:
  138. if (self.state == STATE_MSG_HEADER
  139. or self.state == STATE_PATCH_SUBJECT):
  140. self.state += 1
  141. # We don't have a subject in the text stream of patch files
  142. # It has its own line with a Subject: tag
  143. if not self.is_log and self.state == STATE_PATCH_SUBJECT:
  144. self.state += 1
  145. elif commit_match:
  146. self.state = STATE_MSG_HEADER
  147. # If we are in a section, keep collecting lines until we see END
  148. if self.in_section:
  149. if line == 'END':
  150. if self.in_section == 'cover':
  151. self.series.cover = self.section
  152. elif self.in_section == 'notes':
  153. if self.is_log:
  154. self.series.notes += self.section
  155. else:
  156. self.warn.append("Unknown section '%s'" % self.in_section)
  157. self.in_section = None
  158. self.skip_blank = True
  159. self.section = []
  160. else:
  161. self.section.append(line)
  162. # Detect the commit subject
  163. elif not is_blank and self.state == STATE_PATCH_SUBJECT:
  164. self.commit.subject = line
  165. # Detect the tags we want to remove, and skip blank lines
  166. elif re_remove.match(line):
  167. self.skip_blank = True
  168. # TEST= should be the last thing in the commit, so remove
  169. # everything after it
  170. if line.startswith('TEST='):
  171. self.found_test = True
  172. elif self.skip_blank and is_blank:
  173. self.skip_blank = False
  174. # Detect the start of a cover letter section
  175. elif re_cover.match(line):
  176. self.in_section = 'cover'
  177. self.skip_blank = False
  178. elif cover_cc_match:
  179. value = cover_cc_match.group(1)
  180. self.AddToSeries(line, 'cover-cc', value)
  181. # If we are in a change list, key collected lines until a blank one
  182. elif self.in_change:
  183. if is_blank:
  184. # Blank line ends this change list
  185. self.in_change = 0
  186. elif line == '---' or re_signoff.match(line):
  187. self.in_change = 0
  188. out = self.ProcessLine(line)
  189. else:
  190. if self.is_log:
  191. self.series.AddChange(self.in_change, self.commit, line)
  192. self.skip_blank = False
  193. # Detect Series-xxx tags
  194. elif series_match:
  195. name = series_match.group(1)
  196. value = series_match.group(2)
  197. if name == 'changes':
  198. # value is the version number: e.g. 1, or 2
  199. try:
  200. value = int(value)
  201. except ValueError as str:
  202. raise ValueError("%s: Cannot decode version info '%s'" %
  203. (self.commit.hash, line))
  204. self.in_change = int(value)
  205. else:
  206. self.AddToSeries(line, name, value)
  207. self.skip_blank = True
  208. # Detect the start of a new commit
  209. elif commit_match:
  210. self.CloseCommit()
  211. # TODO: We should store the whole hash, and just display a subset
  212. self.commit = commit.Commit(commit_match.group(1)[:8])
  213. # Detect tags in the commit message
  214. elif tag_match:
  215. # Remove Tested-by self, since few will take much notice
  216. if (tag_match.group(1) == 'Tested-by' and
  217. tag_match.group(2).find(os.getenv('USER') + '@') != -1):
  218. self.warn.append("Ignoring %s" % line)
  219. elif tag_match.group(1) == 'Cc':
  220. self.commit.AddCc(tag_match.group(2).split(','))
  221. else:
  222. self.tags.append(line);
  223. # Well that means this is an ordinary line
  224. else:
  225. pos = 1
  226. # Look for ugly ASCII characters
  227. for ch in line:
  228. # TODO: Would be nicer to report source filename and line
  229. if ord(ch) > 0x80:
  230. self.warn.append("Line %d/%d ('%s') has funny ascii char" %
  231. (self.linenum, pos, line))
  232. pos += 1
  233. # Look for space before tab
  234. m = re_space_before_tab.match(line)
  235. if m:
  236. self.warn.append('Line %d/%d has space before tab' %
  237. (self.linenum, m.start()))
  238. # OK, we have a valid non-blank line
  239. out = [line]
  240. self.linenum += 1
  241. self.skip_blank = False
  242. if self.state == STATE_DIFFS:
  243. pass
  244. # If this is the start of the diffs section, emit our tags and
  245. # change log
  246. elif line == '---':
  247. self.state = STATE_DIFFS
  248. # Output the tags (signeoff first), then change list
  249. out = []
  250. log = self.series.MakeChangeLog(self.commit)
  251. out += self.FormatTags(self.tags)
  252. out += [line] + log
  253. elif self.found_test:
  254. if not re_allowed_after_test.match(line):
  255. self.lines_after_test += 1
  256. return out
  257. def Finalize(self):
  258. """Close out processing of this patch stream"""
  259. self.CloseCommit()
  260. if self.lines_after_test:
  261. self.warn.append('Found %d lines after TEST=' %
  262. self.lines_after_test)
  263. def ProcessStream(self, infd, outfd):
  264. """Copy a stream from infd to outfd, filtering out unwanting things.
  265. This is used to process patch files one at a time.
  266. Args:
  267. infd: Input stream file object
  268. outfd: Output stream file object
  269. """
  270. # Extract the filename from each diff, for nice warnings
  271. fname = None
  272. last_fname = None
  273. re_fname = re.compile('diff --git a/(.*) b/.*')
  274. while True:
  275. line = infd.readline()
  276. if not line:
  277. break
  278. out = self.ProcessLine(line)
  279. # Try to detect blank lines at EOF
  280. for line in out:
  281. match = re_fname.match(line)
  282. if match:
  283. last_fname = fname
  284. fname = match.group(1)
  285. if line == '+':
  286. self.blank_count += 1
  287. else:
  288. if self.blank_count and (line == '-- ' or match):
  289. self.warn.append("Found possible blank line(s) at "
  290. "end of file '%s'" % last_fname)
  291. outfd.write('+\n' * self.blank_count)
  292. outfd.write(line + '\n')
  293. self.blank_count = 0
  294. self.Finalize()
  295. def GetMetaDataForList(commit_range, git_dir=None, count=None,
  296. series = Series()):
  297. """Reads out patch series metadata from the commits
  298. This does a 'git log' on the relevant commits and pulls out the tags we
  299. are interested in.
  300. Args:
  301. commit_range: Range of commits to count (e.g. 'HEAD..base')
  302. git_dir: Path to git repositiory (None to use default)
  303. count: Number of commits to list, or None for no limit
  304. series: Series object to add information into. By default a new series
  305. is started.
  306. Returns:
  307. A Series object containing information about the commits.
  308. """
  309. params = ['git', 'log', '--no-color', '--reverse', commit_range]
  310. if count is not None:
  311. params[2:2] = ['-n%d' % count]
  312. if git_dir:
  313. params[1:1] = ['--git-dir', git_dir]
  314. pipe = [params]
  315. stdout = command.RunPipe(pipe, capture=True).stdout
  316. ps = PatchStream(series, is_log=True)
  317. for line in stdout.splitlines():
  318. ps.ProcessLine(line)
  319. ps.Finalize()
  320. return series
  321. def GetMetaData(start, count):
  322. """Reads out patch series metadata from the commits
  323. This does a 'git log' on the relevant commits and pulls out the tags we
  324. are interested in.
  325. Args:
  326. start: Commit to start from: 0=HEAD, 1=next one, etc.
  327. count: Number of commits to list
  328. """
  329. return GetMetaDataForList('HEAD~%d' % start, None, count)
  330. def FixPatch(backup_dir, fname, series, commit):
  331. """Fix up a patch file, by adding/removing as required.
  332. We remove our tags from the patch file, insert changes lists, etc.
  333. The patch file is processed in place, and overwritten.
  334. A backup file is put into backup_dir (if not None).
  335. Args:
  336. fname: Filename to patch file to process
  337. series: Series information about this patch set
  338. commit: Commit object for this patch file
  339. Return:
  340. A list of errors, or [] if all ok.
  341. """
  342. handle, tmpname = tempfile.mkstemp()
  343. outfd = os.fdopen(handle, 'w')
  344. infd = open(fname, 'r')
  345. ps = PatchStream(series)
  346. ps.commit = commit
  347. ps.ProcessStream(infd, outfd)
  348. infd.close()
  349. outfd.close()
  350. # Create a backup file if required
  351. if backup_dir:
  352. shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
  353. shutil.move(tmpname, fname)
  354. return ps.warn
  355. def FixPatches(series, fnames):
  356. """Fix up a list of patches identified by filenames
  357. The patch files are processed in place, and overwritten.
  358. Args:
  359. series: The series object
  360. fnames: List of patch files to process
  361. """
  362. # Current workflow creates patches, so we shouldn't need a backup
  363. backup_dir = None #tempfile.mkdtemp('clean-patch')
  364. count = 0
  365. for fname in fnames:
  366. commit = series.commits[count]
  367. commit.patch = fname
  368. result = FixPatch(backup_dir, fname, series, commit)
  369. if result:
  370. print '%d warnings for %s:' % (len(result), fname)
  371. for warn in result:
  372. print '\t', warn
  373. print
  374. count += 1
  375. print 'Cleaned %d patches' % count
  376. return series
  377. def InsertCoverLetter(fname, series, count):
  378. """Inserts a cover letter with the required info into patch 0
  379. Args:
  380. fname: Input / output filename of the cover letter file
  381. series: Series object
  382. count: Number of patches in the series
  383. """
  384. fd = open(fname, 'r')
  385. lines = fd.readlines()
  386. fd.close()
  387. fd = open(fname, 'w')
  388. text = series.cover
  389. prefix = series.GetPatchPrefix()
  390. for line in lines:
  391. if line.startswith('Subject:'):
  392. # TODO: if more than 10 patches this should save 00/xx, not 0/xx
  393. line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
  394. # Insert our cover letter
  395. elif line.startswith('*** BLURB HERE ***'):
  396. # First the blurb test
  397. line = '\n'.join(text[1:]) + '\n'
  398. if series.get('notes'):
  399. line += '\n'.join(series.notes) + '\n'
  400. # Now the change list
  401. out = series.MakeChangeLog(None)
  402. line += '\n' + '\n'.join(out)
  403. fd.write(line)
  404. fd.close()