patchstream.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. # SPDX-License-Identifier: GPL-2.0+
  2. # Copyright (c) 2011 The Chromium OS Authors.
  3. #
  4. """Handles parsing a stream of commits/emails from 'git log' or other source"""
  5. import collections
  6. import datetime
  7. import io
  8. import math
  9. import os
  10. import re
  11. import queue
  12. import shutil
  13. import tempfile
  14. from patman import command
  15. from patman import commit
  16. from patman import gitutil
  17. from patman.series import Series
  18. # Tags that we detect and remove
  19. RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:'
  20. r'|Reviewed-on:|Commit-\w*:')
  21. # Lines which are allowed after a TEST= line
  22. RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:')
  23. # Signoffs
  24. RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)')
  25. # Cover letter tag
  26. RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)')
  27. # Patch series tag
  28. RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)')
  29. # Change-Id will be used to generate the Message-Id and then be stripped
  30. RE_CHANGE_ID = re.compile('^Change-Id: *(.*)')
  31. # Commit series tag
  32. RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)')
  33. # Commit tags that we want to collect and keep
  34. RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
  35. # The start of a new commit in the git log
  36. RE_COMMIT = re.compile('^commit ([0-9a-f]*)$')
  37. # We detect these since checkpatch doesn't always do it
  38. RE_SPACE_BEFORE_TAB = re.compile('^[+].* \t')
  39. # Match indented lines for changes
  40. RE_LEADING_WHITESPACE = re.compile(r'^\s')
  41. # Detect a 'diff' line
  42. RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$')
  43. # Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch
  44. RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)')
  45. # Detect line with invalid TAG
  46. RE_INV_TAG = re.compile('^Serie-([a-z-]*): *(.*)')
  47. # States we can be in - can we use range() and still have comments?
  48. STATE_MSG_HEADER = 0 # Still in the message header
  49. STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
  50. STATE_PATCH_HEADER = 2 # In patch header (after the subject)
  51. STATE_DIFFS = 3 # In the diff part (past --- line)
  52. class PatchStream:
  53. """Class for detecting/injecting tags in a patch or series of patches
  54. We support processing the output of 'git log' to read out the tags we
  55. are interested in. We can also process a patch file in order to remove
  56. unwanted tags or inject additional ones. These correspond to the two
  57. phases of processing.
  58. """
  59. def __init__(self, series, is_log=False):
  60. self.skip_blank = False # True to skip a single blank line
  61. self.found_test = False # Found a TEST= line
  62. self.lines_after_test = 0 # Number of lines found after TEST=
  63. self.linenum = 1 # Output line number we are up to
  64. self.in_section = None # Name of start...END section we are in
  65. self.notes = [] # Series notes
  66. self.section = [] # The current section...END section
  67. self.series = series # Info about the patch series
  68. self.is_log = is_log # True if indent like git log
  69. self.in_change = None # Name of the change list we are in
  70. self.change_version = 0 # Non-zero if we are in a change list
  71. self.change_lines = [] # Lines of the current change
  72. self.blank_count = 0 # Number of blank lines stored up
  73. self.state = STATE_MSG_HEADER # What state are we in?
  74. self.commit = None # Current commit
  75. # List of unquoted test blocks, each a list of str lines
  76. self.snippets = []
  77. self.cur_diff = None # Last 'diff' line seen (str)
  78. self.cur_line = None # Last context (@@) line seen (str)
  79. self.recent_diff = None # 'diff' line for current snippet (str)
  80. self.recent_line = None # '@@' line for current snippet (str)
  81. self.recent_quoted = collections.deque([], 5)
  82. self.recent_unquoted = queue.Queue()
  83. self.was_quoted = None
  84. @staticmethod
  85. def process_text(text, is_comment=False):
  86. """Process some text through this class using a default Commit/Series
  87. Args:
  88. text (str): Text to parse
  89. is_comment (bool): True if this is a comment rather than a patch.
  90. If True, PatchStream doesn't expect a patch subject at the
  91. start, but jumps straight into the body
  92. Returns:
  93. PatchStream: object with results
  94. """
  95. pstrm = PatchStream(Series())
  96. pstrm.commit = commit.Commit(None)
  97. infd = io.StringIO(text)
  98. outfd = io.StringIO()
  99. if is_comment:
  100. pstrm.state = STATE_PATCH_HEADER
  101. pstrm.process_stream(infd, outfd)
  102. return pstrm
  103. def _add_warn(self, warn):
  104. """Add a new warning to report to the user about the current commit
  105. The new warning is added to the current commit if not already present.
  106. Args:
  107. warn (str): Warning to report
  108. Raises:
  109. ValueError: Warning is generated with no commit associated
  110. """
  111. if not self.commit:
  112. print('Warning outside commit: %s' % warn)
  113. elif warn not in self.commit.warn:
  114. self.commit.warn.append(warn)
  115. def _add_to_series(self, line, name, value):
  116. """Add a new Series-xxx tag.
  117. When a Series-xxx tag is detected, we come here to record it, if we
  118. are scanning a 'git log'.
  119. Args:
  120. line (str): Source line containing tag (useful for debug/error
  121. messages)
  122. name (str): Tag name (part after 'Series-')
  123. value (str): Tag value (part after 'Series-xxx: ')
  124. """
  125. if name == 'notes':
  126. self.in_section = name
  127. self.skip_blank = False
  128. if self.is_log:
  129. warn = self.series.AddTag(self.commit, line, name, value)
  130. if warn:
  131. self.commit.warn.append(warn)
  132. def _add_to_commit(self, name):
  133. """Add a new Commit-xxx tag.
  134. When a Commit-xxx tag is detected, we come here to record it.
  135. Args:
  136. name (str): Tag name (part after 'Commit-')
  137. """
  138. if name == 'notes':
  139. self.in_section = 'commit-' + name
  140. self.skip_blank = False
  141. def _add_commit_rtag(self, rtag_type, who):
  142. """Add a response tag to the current commit
  143. Args:
  144. rtag_type (str): rtag type (e.g. 'Reviewed-by')
  145. who (str): Person who gave that rtag, e.g.
  146. 'Fred Bloggs <fred@bloggs.org>'
  147. """
  148. self.commit.AddRtag(rtag_type, who)
  149. def _close_commit(self):
  150. """Save the current commit into our commit list, and reset our state"""
  151. if self.commit and self.is_log:
  152. self.series.AddCommit(self.commit)
  153. self.commit = None
  154. # If 'END' is missing in a 'Cover-letter' section, and that section
  155. # happens to show up at the very end of the commit message, this is
  156. # the chance for us to fix it up.
  157. if self.in_section == 'cover' and self.is_log:
  158. self.series.cover = self.section
  159. self.in_section = None
  160. self.skip_blank = True
  161. self.section = []
  162. self.cur_diff = None
  163. self.recent_diff = None
  164. self.recent_line = None
  165. def _parse_version(self, value, line):
  166. """Parse a version from a *-changes tag
  167. Args:
  168. value (str): Tag value (part after 'xxx-changes: '
  169. line (str): Source line containing tag
  170. Returns:
  171. int: The version as an integer
  172. Raises:
  173. ValueError: the value cannot be converted
  174. """
  175. try:
  176. return int(value)
  177. except ValueError:
  178. raise ValueError("%s: Cannot decode version info '%s'" %
  179. (self.commit.hash, line))
  180. def _finalise_change(self):
  181. """_finalise a (multi-line) change and add it to the series or commit"""
  182. if not self.change_lines:
  183. return
  184. change = '\n'.join(self.change_lines)
  185. if self.in_change == 'Series':
  186. self.series.AddChange(self.change_version, self.commit, change)
  187. elif self.in_change == 'Cover':
  188. self.series.AddChange(self.change_version, None, change)
  189. elif self.in_change == 'Commit':
  190. self.commit.AddChange(self.change_version, change)
  191. self.change_lines = []
  192. def _finalise_snippet(self):
  193. """Finish off a snippet and add it to the list
  194. This is called when we get to the end of a snippet, i.e. the we enter
  195. the next block of quoted text:
  196. This is a comment from someone.
  197. Something else
  198. > Now we have some code <----- end of snippet
  199. > more code
  200. Now a comment about the above code
  201. This adds the snippet to our list
  202. """
  203. quoted_lines = []
  204. while self.recent_quoted:
  205. quoted_lines.append(self.recent_quoted.popleft())
  206. unquoted_lines = []
  207. valid = False
  208. while not self.recent_unquoted.empty():
  209. text = self.recent_unquoted.get()
  210. if not (text.startswith('On ') and text.endswith('wrote:')):
  211. unquoted_lines.append(text)
  212. if text:
  213. valid = True
  214. if valid:
  215. lines = []
  216. if self.recent_diff:
  217. lines.append('> File: %s' % self.recent_diff)
  218. if self.recent_line:
  219. out = '> Line: %s / %s' % self.recent_line[:2]
  220. if self.recent_line[2]:
  221. out += ': %s' % self.recent_line[2]
  222. lines.append(out)
  223. lines += quoted_lines + unquoted_lines
  224. if lines:
  225. self.snippets.append(lines)
  226. def process_line(self, line):
  227. """Process a single line of a patch file or commit log
  228. This process a line and returns a list of lines to output. The list
  229. may be empty or may contain multiple output lines.
  230. This is where all the complicated logic is located. The class's
  231. state is used to move between different states and detect things
  232. properly.
  233. We can be in one of two modes:
  234. self.is_log == True: This is 'git log' mode, where most output is
  235. indented by 4 characters and we are scanning for tags
  236. self.is_log == False: This is 'patch' mode, where we already have
  237. all the tags, and are processing patches to remove junk we
  238. don't want, and add things we think are required.
  239. Args:
  240. line (str): text line to process
  241. Returns:
  242. list: list of output lines, or [] if nothing should be output
  243. Raises:
  244. ValueError: a fatal error occurred while parsing, e.g. an END
  245. without a starting tag, or two commits with two change IDs
  246. """
  247. # Initially we have no output. Prepare the input line string
  248. out = []
  249. line = line.rstrip('\n')
  250. commit_match = RE_COMMIT.match(line) if self.is_log else None
  251. if self.is_log:
  252. if line[:4] == ' ':
  253. line = line[4:]
  254. # Handle state transition and skipping blank lines
  255. series_tag_match = RE_SERIES_TAG.match(line)
  256. change_id_match = RE_CHANGE_ID.match(line)
  257. commit_tag_match = RE_COMMIT_TAG.match(line)
  258. cover_match = RE_COVER.match(line)
  259. signoff_match = RE_SIGNOFF.match(line)
  260. leading_whitespace_match = RE_LEADING_WHITESPACE.match(line)
  261. diff_match = RE_DIFF.match(line)
  262. line_match = RE_LINE.match(line)
  263. invalid_match = RE_INV_TAG.match(line)
  264. tag_match = None
  265. if self.state == STATE_PATCH_HEADER:
  266. tag_match = RE_TAG.match(line)
  267. is_blank = not line.strip()
  268. if is_blank:
  269. if (self.state == STATE_MSG_HEADER
  270. or self.state == STATE_PATCH_SUBJECT):
  271. self.state += 1
  272. # We don't have a subject in the text stream of patch files
  273. # It has its own line with a Subject: tag
  274. if not self.is_log and self.state == STATE_PATCH_SUBJECT:
  275. self.state += 1
  276. elif commit_match:
  277. self.state = STATE_MSG_HEADER
  278. # If a tag is detected, or a new commit starts
  279. if series_tag_match or commit_tag_match or change_id_match or \
  280. cover_match or signoff_match or self.state == STATE_MSG_HEADER:
  281. # but we are already in a section, this means 'END' is missing
  282. # for that section, fix it up.
  283. if self.in_section:
  284. self._add_warn("Missing 'END' in section '%s'" % self.in_section)
  285. if self.in_section == 'cover':
  286. self.series.cover = self.section
  287. elif self.in_section == 'notes':
  288. if self.is_log:
  289. self.series.notes += self.section
  290. elif self.in_section == 'commit-notes':
  291. if self.is_log:
  292. self.commit.notes += self.section
  293. else:
  294. # This should not happen
  295. raise ValueError("Unknown section '%s'" % self.in_section)
  296. self.in_section = None
  297. self.skip_blank = True
  298. self.section = []
  299. # but we are already in a change list, that means a blank line
  300. # is missing, fix it up.
  301. if self.in_change:
  302. self._add_warn("Missing 'blank line' in section '%s-changes'" %
  303. self.in_change)
  304. self._finalise_change()
  305. self.in_change = None
  306. self.change_version = 0
  307. # If we are in a section, keep collecting lines until we see END
  308. if self.in_section:
  309. if line == 'END':
  310. if self.in_section == 'cover':
  311. self.series.cover = self.section
  312. elif self.in_section == 'notes':
  313. if self.is_log:
  314. self.series.notes += self.section
  315. elif self.in_section == 'commit-notes':
  316. if self.is_log:
  317. self.commit.notes += self.section
  318. else:
  319. # This should not happen
  320. raise ValueError("Unknown section '%s'" % self.in_section)
  321. self.in_section = None
  322. self.skip_blank = True
  323. self.section = []
  324. else:
  325. self.section.append(line)
  326. # If we are not in a section, it is an unexpected END
  327. elif line == 'END':
  328. raise ValueError("'END' wihout section")
  329. # Detect the commit subject
  330. elif not is_blank and self.state == STATE_PATCH_SUBJECT:
  331. self.commit.subject = line
  332. # Detect the tags we want to remove, and skip blank lines
  333. elif RE_REMOVE.match(line) and not commit_tag_match:
  334. self.skip_blank = True
  335. # TEST= should be the last thing in the commit, so remove
  336. # everything after it
  337. if line.startswith('TEST='):
  338. self.found_test = True
  339. elif self.skip_blank and is_blank:
  340. self.skip_blank = False
  341. # Detect Cover-xxx tags
  342. elif cover_match:
  343. name = cover_match.group(1)
  344. value = cover_match.group(2)
  345. if name == 'letter':
  346. self.in_section = 'cover'
  347. self.skip_blank = False
  348. elif name == 'letter-cc':
  349. self._add_to_series(line, 'cover-cc', value)
  350. elif name == 'changes':
  351. self.in_change = 'Cover'
  352. self.change_version = self._parse_version(value, line)
  353. # If we are in a change list, key collected lines until a blank one
  354. elif self.in_change:
  355. if is_blank:
  356. # Blank line ends this change list
  357. self._finalise_change()
  358. self.in_change = None
  359. self.change_version = 0
  360. elif line == '---':
  361. self._finalise_change()
  362. self.in_change = None
  363. self.change_version = 0
  364. out = self.process_line(line)
  365. elif self.is_log:
  366. if not leading_whitespace_match:
  367. self._finalise_change()
  368. self.change_lines.append(line)
  369. self.skip_blank = False
  370. # Detect Series-xxx tags
  371. elif series_tag_match:
  372. name = series_tag_match.group(1)
  373. value = series_tag_match.group(2)
  374. if name == 'changes':
  375. # value is the version number: e.g. 1, or 2
  376. self.in_change = 'Series'
  377. self.change_version = self._parse_version(value, line)
  378. else:
  379. self._add_to_series(line, name, value)
  380. self.skip_blank = True
  381. # Detect Change-Id tags
  382. elif change_id_match:
  383. value = change_id_match.group(1)
  384. if self.is_log:
  385. if self.commit.change_id:
  386. raise ValueError(
  387. "%s: Two Change-Ids: '%s' vs. '%s'" %
  388. (self.commit.hash, self.commit.change_id, value))
  389. self.commit.change_id = value
  390. self.skip_blank = True
  391. # Detect Commit-xxx tags
  392. elif commit_tag_match:
  393. name = commit_tag_match.group(1)
  394. value = commit_tag_match.group(2)
  395. if name == 'notes':
  396. self._add_to_commit(name)
  397. self.skip_blank = True
  398. elif name == 'changes':
  399. self.in_change = 'Commit'
  400. self.change_version = self._parse_version(value, line)
  401. else:
  402. self._add_warn('Line %d: Ignoring Commit-%s' %
  403. (self.linenum, name))
  404. # Detect invalid tags
  405. elif invalid_match:
  406. raise ValueError("Line %d: Invalid tag = '%s'" %
  407. (self.linenum, line))
  408. # Detect the start of a new commit
  409. elif commit_match:
  410. self._close_commit()
  411. self.commit = commit.Commit(commit_match.group(1))
  412. # Detect tags in the commit message
  413. elif tag_match:
  414. rtag_type, who = tag_match.groups()
  415. self._add_commit_rtag(rtag_type, who)
  416. # Remove Tested-by self, since few will take much notice
  417. if (rtag_type == 'Tested-by' and
  418. who.find(os.getenv('USER') + '@') != -1):
  419. self._add_warn("Ignoring '%s'" % line)
  420. elif rtag_type == 'Patch-cc':
  421. self.commit.AddCc(who.split(','))
  422. else:
  423. out = [line]
  424. # Suppress duplicate signoffs
  425. elif signoff_match:
  426. if (self.is_log or not self.commit or
  427. self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
  428. out = [line]
  429. # Well that means this is an ordinary line
  430. else:
  431. # Look for space before tab
  432. mat = RE_SPACE_BEFORE_TAB.match(line)
  433. if mat:
  434. self._add_warn('Line %d/%d has space before tab' %
  435. (self.linenum, mat.start()))
  436. # OK, we have a valid non-blank line
  437. out = [line]
  438. self.linenum += 1
  439. self.skip_blank = False
  440. if diff_match:
  441. self.cur_diff = diff_match.group(1)
  442. # If this is quoted, keep recent lines
  443. if not diff_match and self.linenum > 1 and line:
  444. if line.startswith('>'):
  445. if not self.was_quoted:
  446. self._finalise_snippet()
  447. self.recent_line = None
  448. if not line_match:
  449. self.recent_quoted.append(line)
  450. self.was_quoted = True
  451. self.recent_diff = self.cur_diff
  452. else:
  453. self.recent_unquoted.put(line)
  454. self.was_quoted = False
  455. if line_match:
  456. self.recent_line = line_match.groups()
  457. if self.state == STATE_DIFFS:
  458. pass
  459. # If this is the start of the diffs section, emit our tags and
  460. # change log
  461. elif line == '---':
  462. self.state = STATE_DIFFS
  463. # Output the tags (signoff first), then change list
  464. out = []
  465. log = self.series.MakeChangeLog(self.commit)
  466. out += [line]
  467. if self.commit:
  468. out += self.commit.notes
  469. out += [''] + log
  470. elif self.found_test:
  471. if not RE_ALLOWED_AFTER_TEST.match(line):
  472. self.lines_after_test += 1
  473. return out
  474. def finalise(self):
  475. """Close out processing of this patch stream"""
  476. self._finalise_snippet()
  477. self._finalise_change()
  478. self._close_commit()
  479. if self.lines_after_test:
  480. self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
  481. def _write_message_id(self, outfd):
  482. """Write the Message-Id into the output.
  483. This is based on the Change-Id in the original patch, the version,
  484. and the prefix.
  485. Args:
  486. outfd (io.IOBase): Output stream file object
  487. """
  488. if not self.commit.change_id:
  489. return
  490. # If the count is -1 we're testing, so use a fixed time
  491. if self.commit.count == -1:
  492. time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
  493. else:
  494. time_now = datetime.datetime.now()
  495. # In theory there is email.utils.make_msgid() which would be nice
  496. # to use, but it already produces something way too long and thus
  497. # will produce ugly commit lines if someone throws this into
  498. # a "Link:" tag in the final commit. So (sigh) roll our own.
  499. # Start with the time; presumably we wouldn't send the same series
  500. # with the same Change-Id at the exact same second.
  501. parts = [time_now.strftime("%Y%m%d%H%M%S")]
  502. # These seem like they would be nice to include.
  503. if 'prefix' in self.series:
  504. parts.append(self.series['prefix'])
  505. if 'version' in self.series:
  506. parts.append("v%s" % self.series['version'])
  507. parts.append(str(self.commit.count + 1))
  508. # The Change-Id must be last, right before the @
  509. parts.append(self.commit.change_id)
  510. # Join parts together with "." and write it out.
  511. outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
  512. def process_stream(self, infd, outfd):
  513. """Copy a stream from infd to outfd, filtering out unwanting things.
  514. This is used to process patch files one at a time.
  515. Args:
  516. infd (io.IOBase): Input stream file object
  517. outfd (io.IOBase): Output stream file object
  518. """
  519. # Extract the filename from each diff, for nice warnings
  520. fname = None
  521. last_fname = None
  522. re_fname = re.compile('diff --git a/(.*) b/.*')
  523. self._write_message_id(outfd)
  524. while True:
  525. line = infd.readline()
  526. if not line:
  527. break
  528. out = self.process_line(line)
  529. # Try to detect blank lines at EOF
  530. for line in out:
  531. match = re_fname.match(line)
  532. if match:
  533. last_fname = fname
  534. fname = match.group(1)
  535. if line == '+':
  536. self.blank_count += 1
  537. else:
  538. if self.blank_count and (line == '-- ' or match):
  539. self._add_warn("Found possible blank line(s) at end of file '%s'" %
  540. last_fname)
  541. outfd.write('+\n' * self.blank_count)
  542. outfd.write(line + '\n')
  543. self.blank_count = 0
  544. self.finalise()
  545. def insert_tags(msg, tags_to_emit):
  546. """Add extra tags to a commit message
  547. The tags are added after an existing block of tags if found, otherwise at
  548. the end.
  549. Args:
  550. msg (str): Commit message
  551. tags_to_emit (list): List of tags to emit, each a str
  552. Returns:
  553. (str) new message
  554. """
  555. out = []
  556. done = False
  557. emit_tags = False
  558. emit_blank = False
  559. for line in msg.splitlines():
  560. if not done:
  561. signoff_match = RE_SIGNOFF.match(line)
  562. tag_match = RE_TAG.match(line)
  563. if tag_match or signoff_match:
  564. emit_tags = True
  565. if emit_tags and not tag_match and not signoff_match:
  566. out += tags_to_emit
  567. emit_tags = False
  568. done = True
  569. emit_blank = not (signoff_match or tag_match)
  570. else:
  571. emit_blank = line
  572. out.append(line)
  573. if not done:
  574. if emit_blank:
  575. out.append('')
  576. out += tags_to_emit
  577. return '\n'.join(out)
  578. def get_list(commit_range, git_dir=None, count=None):
  579. """Get a log of a list of comments
  580. This returns the output of 'git log' for the selected commits
  581. Args:
  582. commit_range (str): Range of commits to count (e.g. 'HEAD..base')
  583. git_dir (str): Path to git repositiory (None to use default)
  584. count (int): Number of commits to list, or None for no limit
  585. Returns
  586. str: String containing the contents of the git log
  587. """
  588. params = gitutil.LogCmd(commit_range, reverse=True, count=count,
  589. git_dir=git_dir)
  590. return command.RunPipe([params], capture=True).stdout
  591. def get_metadata_for_list(commit_range, git_dir=None, count=None,
  592. series=None, allow_overwrite=False):
  593. """Reads out patch series metadata from the commits
  594. This does a 'git log' on the relevant commits and pulls out the tags we
  595. are interested in.
  596. Args:
  597. commit_range (str): Range of commits to count (e.g. 'HEAD..base')
  598. git_dir (str): Path to git repositiory (None to use default)
  599. count (int): Number of commits to list, or None for no limit
  600. series (Series): Object to add information into. By default a new series
  601. is started.
  602. allow_overwrite (bool): Allow tags to overwrite an existing tag
  603. Returns:
  604. Series: Object containing information about the commits.
  605. """
  606. if not series:
  607. series = Series()
  608. series.allow_overwrite = allow_overwrite
  609. stdout = get_list(commit_range, git_dir, count)
  610. pst = PatchStream(series, is_log=True)
  611. for line in stdout.splitlines():
  612. pst.process_line(line)
  613. pst.finalise()
  614. return series
  615. def get_metadata(branch, start, count):
  616. """Reads out patch series metadata from the commits
  617. This does a 'git log' on the relevant commits and pulls out the tags we
  618. are interested in.
  619. Args:
  620. branch (str): Branch to use (None for current branch)
  621. start (int): Commit to start from: 0=branch HEAD, 1=next one, etc.
  622. count (int): Number of commits to list
  623. Returns:
  624. Series: Object containing information about the commits.
  625. """
  626. return get_metadata_for_list(
  627. '%s~%d' % (branch if branch else 'HEAD', start), None, count)
  628. def get_metadata_for_test(text):
  629. """Process metadata from a file containing a git log. Used for tests
  630. Args:
  631. text:
  632. Returns:
  633. Series: Object containing information about the commits.
  634. """
  635. series = Series()
  636. pst = PatchStream(series, is_log=True)
  637. for line in text.splitlines():
  638. pst.process_line(line)
  639. pst.finalise()
  640. return series
  641. def fix_patch(backup_dir, fname, series, cmt):
  642. """Fix up a patch file, by adding/removing as required.
  643. We remove our tags from the patch file, insert changes lists, etc.
  644. The patch file is processed in place, and overwritten.
  645. A backup file is put into backup_dir (if not None).
  646. Args:
  647. backup_dir (str): Path to directory to use to backup the file
  648. fname (str): Filename to patch file to process
  649. series (Series): Series information about this patch set
  650. cmt (Commit): Commit object for this patch file
  651. Return:
  652. list: A list of errors, each str, or [] if all ok.
  653. """
  654. handle, tmpname = tempfile.mkstemp()
  655. outfd = os.fdopen(handle, 'w', encoding='utf-8')
  656. infd = open(fname, 'r', encoding='utf-8')
  657. pst = PatchStream(series)
  658. pst.commit = cmt
  659. pst.process_stream(infd, outfd)
  660. infd.close()
  661. outfd.close()
  662. # Create a backup file if required
  663. if backup_dir:
  664. shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
  665. shutil.move(tmpname, fname)
  666. return cmt.warn
  667. def fix_patches(series, fnames):
  668. """Fix up a list of patches identified by filenames
  669. The patch files are processed in place, and overwritten.
  670. Args:
  671. series (Series): The Series object
  672. fnames (:type: list of str): List of patch files to process
  673. """
  674. # Current workflow creates patches, so we shouldn't need a backup
  675. backup_dir = None #tempfile.mkdtemp('clean-patch')
  676. count = 0
  677. for fname in fnames:
  678. cmt = series.commits[count]
  679. cmt.patch = fname
  680. cmt.count = count
  681. result = fix_patch(backup_dir, fname, series, cmt)
  682. if result:
  683. print('%d warning%s for %s:' %
  684. (len(result), 's' if len(result) > 1 else '', fname))
  685. for warn in result:
  686. print('\t%s' % warn)
  687. print()
  688. count += 1
  689. print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
  690. def insert_cover_letter(fname, series, count):
  691. """Inserts a cover letter with the required info into patch 0
  692. Args:
  693. fname (str): Input / output filename of the cover letter file
  694. series (Series): Series object
  695. count (int): Number of patches in the series
  696. """
  697. fil = open(fname, 'r')
  698. lines = fil.readlines()
  699. fil.close()
  700. fil = open(fname, 'w')
  701. text = series.cover
  702. prefix = series.GetPatchPrefix()
  703. for line in lines:
  704. if line.startswith('Subject:'):
  705. # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
  706. zero_repeat = int(math.log10(count)) + 1
  707. zero = '0' * zero_repeat
  708. line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
  709. # Insert our cover letter
  710. elif line.startswith('*** BLURB HERE ***'):
  711. # First the blurb test
  712. line = '\n'.join(text[1:]) + '\n'
  713. if series.get('notes'):
  714. line += '\n'.join(series.notes) + '\n'
  715. # Now the change list
  716. out = series.MakeChangeLog(None)
  717. line += '\n' + '\n'.join(out)
  718. fil.write(line)
  719. fil.close()