status.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. # SPDX-License-Identifier: GPL-2.0+
  2. #
  3. # Copyright 2020 Google LLC
  4. #
  5. """Talks to the patchwork service to figure out what patches have been reviewed
  6. and commented on. Provides a way to display review tags and comments.
  7. Allows creation of a new branch based on the old but with the review tags
  8. collected from patchwork.
  9. """
  10. import collections
  11. import concurrent.futures
  12. from itertools import repeat
  13. import re
  14. import pygit2
  15. import requests
  16. from patman import patchstream
  17. from patman.patchstream import PatchStream
  18. from patman import terminal
  19. from patman import tout
  20. # Patches which are part of a multi-patch series are shown with a prefix like
  21. # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
  22. # part is optional. This decodes the string into groups. For single patches
  23. # the [] part is not present:
  24. # Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
  25. RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
  26. # This decodes the sequence string into a patch number and patch count
  27. RE_SEQ = re.compile(r'(\d+)/(\d+)')
  28. def to_int(vals):
  29. """Convert a list of strings into integers, using 0 if not an integer
  30. Args:
  31. vals (list): List of strings
  32. Returns:
  33. list: List of integers, one for each input string
  34. """
  35. out = [int(val) if val.isdigit() else 0 for val in vals]
  36. return out
  37. class Patch(dict):
  38. """Models a patch in patchwork
  39. This class records information obtained from patchwork
  40. Some of this information comes from the 'Patch' column:
  41. [RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
  42. This shows the prefix, version, seq, count and subject.
  43. The other properties come from other columns in the display.
  44. Properties:
  45. pid (str): ID of the patch (typically an integer)
  46. seq (int): Sequence number within series (1=first) parsed from sequence
  47. string
  48. count (int): Number of patches in series, parsed from sequence string
  49. raw_subject (str): Entire subject line, e.g.
  50. "[1/2,v2] efi_loader: Sort header file ordering"
  51. prefix (str): Prefix string or None (e.g. 'RFC')
  52. version (str): Version string or None (e.g. 'v2')
  53. raw_subject (str): Raw patch subject
  54. subject (str): Patch subject with [..] part removed (same as commit
  55. subject)
  56. """
  57. def __init__(self, pid):
  58. super().__init__()
  59. self.id = pid # Use 'id' to match what the Rest API provides
  60. self.seq = None
  61. self.count = None
  62. self.prefix = None
  63. self.version = None
  64. self.raw_subject = None
  65. self.subject = None
  66. # These make us more like a dictionary
  67. def __setattr__(self, name, value):
  68. self[name] = value
  69. def __getattr__(self, name):
  70. return self[name]
  71. def __hash__(self):
  72. return hash(frozenset(self.items()))
  73. def __str__(self):
  74. return self.raw_subject
  75. def parse_subject(self, raw_subject):
  76. """Parse the subject of a patch into its component parts
  77. See RE_PATCH for details. The parsed info is placed into seq, count,
  78. prefix, version, subject
  79. Args:
  80. raw_subject (str): Subject string to parse
  81. Raises:
  82. ValueError: the subject cannot be parsed
  83. """
  84. self.raw_subject = raw_subject.strip()
  85. mat = RE_PATCH.search(raw_subject.strip())
  86. if not mat:
  87. raise ValueError("Cannot parse subject '%s'" % raw_subject)
  88. self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
  89. mat_seq = RE_SEQ.match(seq_info) if seq_info else False
  90. if mat_seq is None:
  91. self.version = seq_info
  92. seq_info = None
  93. if self.version and not self.version.startswith('v'):
  94. self.prefix = self.version
  95. self.version = None
  96. if seq_info:
  97. if mat_seq:
  98. self.seq = int(mat_seq.group(1))
  99. self.count = int(mat_seq.group(2))
  100. else:
  101. self.seq = 1
  102. self.count = 1
  103. class Review:
  104. """Represents a single review email collected in Patchwork
  105. Patches can attract multiple reviews. Each consists of an author/date and
  106. a variable number of 'snippets', which are groups of quoted and unquoted
  107. text.
  108. """
  109. def __init__(self, meta, snippets):
  110. """Create new Review object
  111. Args:
  112. meta (str): Text containing review author and date
  113. snippets (list): List of snippets in th review, each a list of text
  114. lines
  115. """
  116. self.meta = ' : '.join([line for line in meta.splitlines() if line])
  117. self.snippets = snippets
  118. def compare_with_series(series, patches):
  119. """Compare a list of patches with a series it came from
  120. This prints any problems as warnings
  121. Args:
  122. series (Series): Series to compare against
  123. patches (:type: list of Patch): list of Patch objects to compare with
  124. Returns:
  125. tuple
  126. dict:
  127. key: Commit number (0...n-1)
  128. value: Patch object for that commit
  129. dict:
  130. key: Patch number (0...n-1)
  131. value: Commit object for that patch
  132. """
  133. # Check the names match
  134. warnings = []
  135. patch_for_commit = {}
  136. all_patches = set(patches)
  137. for seq, cmt in enumerate(series.commits):
  138. pmatch = [p for p in all_patches if p.subject == cmt.subject]
  139. if len(pmatch) == 1:
  140. patch_for_commit[seq] = pmatch[0]
  141. all_patches.remove(pmatch[0])
  142. elif len(pmatch) > 1:
  143. warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
  144. (seq + 1, cmt.subject,
  145. '\n '.join([p.subject for p in pmatch])))
  146. else:
  147. warnings.append("Cannot find patch for commit %d ('%s')" %
  148. (seq + 1, cmt.subject))
  149. # Check the names match
  150. commit_for_patch = {}
  151. all_commits = set(series.commits)
  152. for seq, patch in enumerate(patches):
  153. cmatch = [c for c in all_commits if c.subject == patch.subject]
  154. if len(cmatch) == 1:
  155. commit_for_patch[seq] = cmatch[0]
  156. all_commits.remove(cmatch[0])
  157. elif len(cmatch) > 1:
  158. warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
  159. (seq + 1, patch.subject,
  160. '\n '.join([c.subject for c in cmatch])))
  161. else:
  162. warnings.append("Cannot find commit for patch %d ('%s')" %
  163. (seq + 1, patch.subject))
  164. return patch_for_commit, commit_for_patch, warnings
  165. def call_rest_api(url, subpath):
  166. """Call the patchwork API and return the result as JSON
  167. Args:
  168. url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
  169. subpath (str): URL subpath to use
  170. Returns:
  171. dict: Json result
  172. Raises:
  173. ValueError: the URL could not be read
  174. """
  175. full_url = '%s/api/1.2/%s' % (url, subpath)
  176. response = requests.get(full_url)
  177. if response.status_code != 200:
  178. raise ValueError("Could not read URL '%s'" % full_url)
  179. return response.json()
  180. def collect_patches(series, series_id, url, rest_api=call_rest_api):
  181. """Collect patch information about a series from patchwork
  182. Uses the Patchwork REST API to collect information provided by patchwork
  183. about the status of each patch.
  184. Args:
  185. series (Series): Series object corresponding to the local branch
  186. containing the series
  187. series_id (str): Patch series ID number
  188. url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
  189. rest_api (function): API function to call to access Patchwork, for
  190. testing
  191. Returns:
  192. list: List of patches sorted by sequence number, each a Patch object
  193. Raises:
  194. ValueError: if the URL could not be read or the web page does not follow
  195. the expected structure
  196. """
  197. data = rest_api(url, 'series/%s/' % series_id)
  198. # Get all the rows, which are patches
  199. patch_dict = data['patches']
  200. count = len(patch_dict)
  201. num_commits = len(series.commits)
  202. if count != num_commits:
  203. tout.Warning('Warning: Patchwork reports %d patches, series has %d' %
  204. (count, num_commits))
  205. patches = []
  206. # Work through each row (patch) one at a time, collecting the information
  207. warn_count = 0
  208. for pw_patch in patch_dict:
  209. patch = Patch(pw_patch['id'])
  210. patch.parse_subject(pw_patch['name'])
  211. patches.append(patch)
  212. if warn_count > 1:
  213. tout.Warning(' (total of %d warnings)' % warn_count)
  214. # Sort patches by patch number
  215. patches = sorted(patches, key=lambda x: x.seq)
  216. return patches
  217. def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, url,
  218. rest_api=call_rest_api):
  219. """Find new rtags collected by patchwork that we don't know about
  220. This is designed to be run in parallel, once for each commit/patch
  221. Args:
  222. new_rtag_list (list): New rtags are written to new_rtag_list[seq]
  223. list, each a dict:
  224. key: Response tag (e.g. 'Reviewed-by')
  225. value: Set of people who gave that response, each a name/email
  226. string
  227. review_list (list): New reviews are written to review_list[seq]
  228. list, each a
  229. List of reviews for the patch, each a Review
  230. seq (int): Position in new_rtag_list to update
  231. cmt (Commit): Commit object for this commit
  232. patch (Patch): Corresponding Patch object for this patch
  233. url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
  234. rest_api (function): API function to call to access Patchwork, for
  235. testing
  236. """
  237. if not patch:
  238. return
  239. # Get the content for the patch email itself as well as all comments
  240. data = rest_api(url, 'patches/%s/' % patch.id)
  241. pstrm = PatchStream.process_text(data['content'], True)
  242. rtags = collections.defaultdict(set)
  243. for response, people in pstrm.commit.rtags.items():
  244. rtags[response].update(people)
  245. data = rest_api(url, 'patches/%s/comments/' % patch.id)
  246. reviews = []
  247. for comment in data:
  248. pstrm = PatchStream.process_text(comment['content'], True)
  249. if pstrm.snippets:
  250. submitter = comment['submitter']
  251. person = '%s <%s>' % (submitter['name'], submitter['email'])
  252. reviews.append(Review(person, pstrm.snippets))
  253. for response, people in pstrm.commit.rtags.items():
  254. rtags[response].update(people)
  255. # Find the tags that are not in the commit
  256. new_rtags = collections.defaultdict(set)
  257. base_rtags = cmt.rtags
  258. for tag, people in rtags.items():
  259. for who in people:
  260. is_new = (tag not in base_rtags or
  261. who not in base_rtags[tag])
  262. if is_new:
  263. new_rtags[tag].add(who)
  264. new_rtag_list[seq] = new_rtags
  265. review_list[seq] = reviews
  266. def show_responses(rtags, indent, is_new):
  267. """Show rtags collected
  268. Args:
  269. rtags (dict): review tags to show
  270. key: Response tag (e.g. 'Reviewed-by')
  271. value: Set of people who gave that response, each a name/email string
  272. indent (str): Indentation string to write before each line
  273. is_new (bool): True if this output should be highlighted
  274. Returns:
  275. int: Number of review tags displayed
  276. """
  277. col = terminal.Color()
  278. count = 0
  279. for tag in sorted(rtags.keys()):
  280. people = rtags[tag]
  281. for who in sorted(people):
  282. terminal.Print(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
  283. newline=False, colour=col.GREEN, bright=is_new)
  284. terminal.Print(who, colour=col.WHITE, bright=is_new)
  285. count += 1
  286. return count
  287. def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
  288. repo=None):
  289. """Create a new branch with review tags added
  290. Args:
  291. series (Series): Series object for the existing branch
  292. new_rtag_list (list): List of review tags to add, one for each commit,
  293. each a dict:
  294. key: Response tag (e.g. 'Reviewed-by')
  295. value: Set of people who gave that response, each a name/email
  296. string
  297. branch (str): Existing branch to update
  298. dest_branch (str): Name of new branch to create
  299. overwrite (bool): True to force overwriting dest_branch if it exists
  300. repo (pygit2.Repository): Repo to use (use None unless testing)
  301. Returns:
  302. int: Total number of review tags added across all commits
  303. Raises:
  304. ValueError: if the destination branch name is the same as the original
  305. branch, or it already exists and @overwrite is False
  306. """
  307. if branch == dest_branch:
  308. raise ValueError(
  309. 'Destination branch must not be the same as the original branch')
  310. if not repo:
  311. repo = pygit2.Repository('.')
  312. count = len(series.commits)
  313. new_br = repo.branches.get(dest_branch)
  314. if new_br:
  315. if not overwrite:
  316. raise ValueError("Branch '%s' already exists (-f to overwrite)" %
  317. dest_branch)
  318. new_br.delete()
  319. if not branch:
  320. branch = 'HEAD'
  321. target = repo.revparse_single('%s~%d' % (branch, count))
  322. repo.branches.local.create(dest_branch, target)
  323. num_added = 0
  324. for seq in range(count):
  325. parent = repo.branches.get(dest_branch)
  326. cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
  327. repo.merge_base(cherry.oid, parent.target)
  328. base_tree = cherry.parents[0].tree
  329. index = repo.merge_trees(base_tree, parent, cherry)
  330. tree_id = index.write_tree(repo)
  331. lines = []
  332. if new_rtag_list[seq]:
  333. for tag, people in new_rtag_list[seq].items():
  334. for who in people:
  335. lines.append('%s: %s' % (tag, who))
  336. num_added += 1
  337. message = patchstream.insert_tags(cherry.message.rstrip(),
  338. sorted(lines))
  339. repo.create_commit(
  340. parent.name, cherry.author, cherry.committer, message, tree_id,
  341. [parent.target])
  342. return num_added
  343. def check_patchwork_status(series, series_id, branch, dest_branch, force,
  344. show_comments, url, rest_api=call_rest_api,
  345. test_repo=None):
  346. """Check the status of a series on Patchwork
  347. This finds review tags and comments for a series in Patchwork, displaying
  348. them to show what is new compared to the local series.
  349. Args:
  350. series (Series): Series object for the existing branch
  351. series_id (str): Patch series ID number
  352. branch (str): Existing branch to update, or None
  353. dest_branch (str): Name of new branch to create, or None
  354. force (bool): True to force overwriting dest_branch if it exists
  355. show_comments (bool): True to show the comments on each patch
  356. url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
  357. rest_api (function): API function to call to access Patchwork, for
  358. testing
  359. test_repo (pygit2.Repository): Repo to use (use None unless testing)
  360. """
  361. patches = collect_patches(series, series_id, url, rest_api)
  362. col = terminal.Color()
  363. count = len(series.commits)
  364. new_rtag_list = [None] * count
  365. review_list = [None] * count
  366. patch_for_commit, _, warnings = compare_with_series(series, patches)
  367. for warn in warnings:
  368. tout.Warning(warn)
  369. patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
  370. with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
  371. futures = executor.map(
  372. find_new_responses, repeat(new_rtag_list), repeat(review_list),
  373. range(count), series.commits, patch_list, repeat(url),
  374. repeat(rest_api))
  375. for fresponse in futures:
  376. if fresponse:
  377. raise fresponse.exception()
  378. num_to_add = 0
  379. for seq, cmt in enumerate(series.commits):
  380. patch = patch_for_commit.get(seq)
  381. if not patch:
  382. continue
  383. terminal.Print('%3d %s' % (patch.seq, patch.subject[:50]),
  384. colour=col.BLUE)
  385. cmt = series.commits[seq]
  386. base_rtags = cmt.rtags
  387. new_rtags = new_rtag_list[seq]
  388. indent = ' ' * 2
  389. show_responses(base_rtags, indent, False)
  390. num_to_add += show_responses(new_rtags, indent, True)
  391. if show_comments:
  392. for review in review_list[seq]:
  393. terminal.Print('Review: %s' % review.meta, colour=col.RED)
  394. for snippet in review.snippets:
  395. for line in snippet:
  396. quoted = line.startswith('>')
  397. terminal.Print(' %s' % line,
  398. colour=col.MAGENTA if quoted else None)
  399. terminal.Print()
  400. terminal.Print("%d new response%s available in patchwork%s" %
  401. (num_to_add, 's' if num_to_add != 1 else '',
  402. '' if dest_branch
  403. else ' (use -d to write them to a new branch)'))
  404. if dest_branch:
  405. num_added = create_branch(series, new_rtag_list, branch,
  406. dest_branch, force, test_repo)
  407. terminal.Print(
  408. "%d response%s added from patchwork into new branch '%s'" %
  409. (num_added, 's' if num_added != 1 else '', dest_branch))