123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487 |
- # SPDX-License-Identifier: GPL-2.0+
- #
- # Copyright 2020 Google LLC
- #
- """Talks to the patchwork service to figure out what patches have been reviewed
- and commented on. Provides a way to display review tags and comments.
- Allows creation of a new branch based on the old but with the review tags
- collected from patchwork.
- """
- import collections
- import concurrent.futures
- from itertools import repeat
- import re
- import pygit2
- import requests
- from patman import patchstream
- from patman.patchstream import PatchStream
- from patman import terminal
- from patman import tout
- # Patches which are part of a multi-patch series are shown with a prefix like
- # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
- # part is optional. This decodes the string into groups. For single patches
- # the [] part is not present:
- # Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
- RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
- # This decodes the sequence string into a patch number and patch count
- RE_SEQ = re.compile(r'(\d+)/(\d+)')
- def to_int(vals):
- """Convert a list of strings into integers, using 0 if not an integer
- Args:
- vals (list): List of strings
- Returns:
- list: List of integers, one for each input string
- """
- out = [int(val) if val.isdigit() else 0 for val in vals]
- return out
- class Patch(dict):
- """Models a patch in patchwork
- This class records information obtained from patchwork
- Some of this information comes from the 'Patch' column:
- [RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
- This shows the prefix, version, seq, count and subject.
- The other properties come from other columns in the display.
- Properties:
- pid (str): ID of the patch (typically an integer)
- seq (int): Sequence number within series (1=first) parsed from sequence
- string
- count (int): Number of patches in series, parsed from sequence string
- raw_subject (str): Entire subject line, e.g.
- "[1/2,v2] efi_loader: Sort header file ordering"
- prefix (str): Prefix string or None (e.g. 'RFC')
- version (str): Version string or None (e.g. 'v2')
- raw_subject (str): Raw patch subject
- subject (str): Patch subject with [..] part removed (same as commit
- subject)
- """
- def __init__(self, pid):
- super().__init__()
- self.id = pid # Use 'id' to match what the Rest API provides
- self.seq = None
- self.count = None
- self.prefix = None
- self.version = None
- self.raw_subject = None
- self.subject = None
- # These make us more like a dictionary
- def __setattr__(self, name, value):
- self[name] = value
- def __getattr__(self, name):
- return self[name]
- def __hash__(self):
- return hash(frozenset(self.items()))
- def __str__(self):
- return self.raw_subject
- def parse_subject(self, raw_subject):
- """Parse the subject of a patch into its component parts
- See RE_PATCH for details. The parsed info is placed into seq, count,
- prefix, version, subject
- Args:
- raw_subject (str): Subject string to parse
- Raises:
- ValueError: the subject cannot be parsed
- """
- self.raw_subject = raw_subject.strip()
- mat = RE_PATCH.search(raw_subject.strip())
- if not mat:
- raise ValueError("Cannot parse subject '%s'" % raw_subject)
- self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
- mat_seq = RE_SEQ.match(seq_info) if seq_info else False
- if mat_seq is None:
- self.version = seq_info
- seq_info = None
- if self.version and not self.version.startswith('v'):
- self.prefix = self.version
- self.version = None
- if seq_info:
- if mat_seq:
- self.seq = int(mat_seq.group(1))
- self.count = int(mat_seq.group(2))
- else:
- self.seq = 1
- self.count = 1
- class Review:
- """Represents a single review email collected in Patchwork
- Patches can attract multiple reviews. Each consists of an author/date and
- a variable number of 'snippets', which are groups of quoted and unquoted
- text.
- """
- def __init__(self, meta, snippets):
- """Create new Review object
- Args:
- meta (str): Text containing review author and date
- snippets (list): List of snippets in th review, each a list of text
- lines
- """
- self.meta = ' : '.join([line for line in meta.splitlines() if line])
- self.snippets = snippets
- def compare_with_series(series, patches):
- """Compare a list of patches with a series it came from
- This prints any problems as warnings
- Args:
- series (Series): Series to compare against
- patches (:type: list of Patch): list of Patch objects to compare with
- Returns:
- tuple
- dict:
- key: Commit number (0...n-1)
- value: Patch object for that commit
- dict:
- key: Patch number (0...n-1)
- value: Commit object for that patch
- """
- # Check the names match
- warnings = []
- patch_for_commit = {}
- all_patches = set(patches)
- for seq, cmt in enumerate(series.commits):
- pmatch = [p for p in all_patches if p.subject == cmt.subject]
- if len(pmatch) == 1:
- patch_for_commit[seq] = pmatch[0]
- all_patches.remove(pmatch[0])
- elif len(pmatch) > 1:
- warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
- (seq + 1, cmt.subject,
- '\n '.join([p.subject for p in pmatch])))
- else:
- warnings.append("Cannot find patch for commit %d ('%s')" %
- (seq + 1, cmt.subject))
- # Check the names match
- commit_for_patch = {}
- all_commits = set(series.commits)
- for seq, patch in enumerate(patches):
- cmatch = [c for c in all_commits if c.subject == patch.subject]
- if len(cmatch) == 1:
- commit_for_patch[seq] = cmatch[0]
- all_commits.remove(cmatch[0])
- elif len(cmatch) > 1:
- warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
- (seq + 1, patch.subject,
- '\n '.join([c.subject for c in cmatch])))
- else:
- warnings.append("Cannot find commit for patch %d ('%s')" %
- (seq + 1, patch.subject))
- return patch_for_commit, commit_for_patch, warnings
- def call_rest_api(url, subpath):
- """Call the patchwork API and return the result as JSON
- Args:
- url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
- subpath (str): URL subpath to use
- Returns:
- dict: Json result
- Raises:
- ValueError: the URL could not be read
- """
- full_url = '%s/api/1.2/%s' % (url, subpath)
- response = requests.get(full_url)
- if response.status_code != 200:
- raise ValueError("Could not read URL '%s'" % full_url)
- return response.json()
- def collect_patches(series, series_id, url, rest_api=call_rest_api):
- """Collect patch information about a series from patchwork
- Uses the Patchwork REST API to collect information provided by patchwork
- about the status of each patch.
- Args:
- series (Series): Series object corresponding to the local branch
- containing the series
- series_id (str): Patch series ID number
- url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
- rest_api (function): API function to call to access Patchwork, for
- testing
- Returns:
- list: List of patches sorted by sequence number, each a Patch object
- Raises:
- ValueError: if the URL could not be read or the web page does not follow
- the expected structure
- """
- data = rest_api(url, 'series/%s/' % series_id)
- # Get all the rows, which are patches
- patch_dict = data['patches']
- count = len(patch_dict)
- num_commits = len(series.commits)
- if count != num_commits:
- tout.Warning('Warning: Patchwork reports %d patches, series has %d' %
- (count, num_commits))
- patches = []
- # Work through each row (patch) one at a time, collecting the information
- warn_count = 0
- for pw_patch in patch_dict:
- patch = Patch(pw_patch['id'])
- patch.parse_subject(pw_patch['name'])
- patches.append(patch)
- if warn_count > 1:
- tout.Warning(' (total of %d warnings)' % warn_count)
- # Sort patches by patch number
- patches = sorted(patches, key=lambda x: x.seq)
- return patches
- def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, url,
- rest_api=call_rest_api):
- """Find new rtags collected by patchwork that we don't know about
- This is designed to be run in parallel, once for each commit/patch
- Args:
- new_rtag_list (list): New rtags are written to new_rtag_list[seq]
- list, each a dict:
- key: Response tag (e.g. 'Reviewed-by')
- value: Set of people who gave that response, each a name/email
- string
- review_list (list): New reviews are written to review_list[seq]
- list, each a
- List of reviews for the patch, each a Review
- seq (int): Position in new_rtag_list to update
- cmt (Commit): Commit object for this commit
- patch (Patch): Corresponding Patch object for this patch
- url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
- rest_api (function): API function to call to access Patchwork, for
- testing
- """
- if not patch:
- return
- # Get the content for the patch email itself as well as all comments
- data = rest_api(url, 'patches/%s/' % patch.id)
- pstrm = PatchStream.process_text(data['content'], True)
- rtags = collections.defaultdict(set)
- for response, people in pstrm.commit.rtags.items():
- rtags[response].update(people)
- data = rest_api(url, 'patches/%s/comments/' % patch.id)
- reviews = []
- for comment in data:
- pstrm = PatchStream.process_text(comment['content'], True)
- if pstrm.snippets:
- submitter = comment['submitter']
- person = '%s <%s>' % (submitter['name'], submitter['email'])
- reviews.append(Review(person, pstrm.snippets))
- for response, people in pstrm.commit.rtags.items():
- rtags[response].update(people)
- # Find the tags that are not in the commit
- new_rtags = collections.defaultdict(set)
- base_rtags = cmt.rtags
- for tag, people in rtags.items():
- for who in people:
- is_new = (tag not in base_rtags or
- who not in base_rtags[tag])
- if is_new:
- new_rtags[tag].add(who)
- new_rtag_list[seq] = new_rtags
- review_list[seq] = reviews
- def show_responses(rtags, indent, is_new):
- """Show rtags collected
- Args:
- rtags (dict): review tags to show
- key: Response tag (e.g. 'Reviewed-by')
- value: Set of people who gave that response, each a name/email string
- indent (str): Indentation string to write before each line
- is_new (bool): True if this output should be highlighted
- Returns:
- int: Number of review tags displayed
- """
- col = terminal.Color()
- count = 0
- for tag in sorted(rtags.keys()):
- people = rtags[tag]
- for who in sorted(people):
- terminal.Print(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
- newline=False, colour=col.GREEN, bright=is_new)
- terminal.Print(who, colour=col.WHITE, bright=is_new)
- count += 1
- return count
- def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
- repo=None):
- """Create a new branch with review tags added
- Args:
- series (Series): Series object for the existing branch
- new_rtag_list (list): List of review tags to add, one for each commit,
- each a dict:
- key: Response tag (e.g. 'Reviewed-by')
- value: Set of people who gave that response, each a name/email
- string
- branch (str): Existing branch to update
- dest_branch (str): Name of new branch to create
- overwrite (bool): True to force overwriting dest_branch if it exists
- repo (pygit2.Repository): Repo to use (use None unless testing)
- Returns:
- int: Total number of review tags added across all commits
- Raises:
- ValueError: if the destination branch name is the same as the original
- branch, or it already exists and @overwrite is False
- """
- if branch == dest_branch:
- raise ValueError(
- 'Destination branch must not be the same as the original branch')
- if not repo:
- repo = pygit2.Repository('.')
- count = len(series.commits)
- new_br = repo.branches.get(dest_branch)
- if new_br:
- if not overwrite:
- raise ValueError("Branch '%s' already exists (-f to overwrite)" %
- dest_branch)
- new_br.delete()
- if not branch:
- branch = 'HEAD'
- target = repo.revparse_single('%s~%d' % (branch, count))
- repo.branches.local.create(dest_branch, target)
- num_added = 0
- for seq in range(count):
- parent = repo.branches.get(dest_branch)
- cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
- repo.merge_base(cherry.oid, parent.target)
- base_tree = cherry.parents[0].tree
- index = repo.merge_trees(base_tree, parent, cherry)
- tree_id = index.write_tree(repo)
- lines = []
- if new_rtag_list[seq]:
- for tag, people in new_rtag_list[seq].items():
- for who in people:
- lines.append('%s: %s' % (tag, who))
- num_added += 1
- message = patchstream.insert_tags(cherry.message.rstrip(),
- sorted(lines))
- repo.create_commit(
- parent.name, cherry.author, cherry.committer, message, tree_id,
- [parent.target])
- return num_added
- def check_patchwork_status(series, series_id, branch, dest_branch, force,
- show_comments, url, rest_api=call_rest_api,
- test_repo=None):
- """Check the status of a series on Patchwork
- This finds review tags and comments for a series in Patchwork, displaying
- them to show what is new compared to the local series.
- Args:
- series (Series): Series object for the existing branch
- series_id (str): Patch series ID number
- branch (str): Existing branch to update, or None
- dest_branch (str): Name of new branch to create, or None
- force (bool): True to force overwriting dest_branch if it exists
- show_comments (bool): True to show the comments on each patch
- url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
- rest_api (function): API function to call to access Patchwork, for
- testing
- test_repo (pygit2.Repository): Repo to use (use None unless testing)
- """
- patches = collect_patches(series, series_id, url, rest_api)
- col = terminal.Color()
- count = len(series.commits)
- new_rtag_list = [None] * count
- review_list = [None] * count
- patch_for_commit, _, warnings = compare_with_series(series, patches)
- for warn in warnings:
- tout.Warning(warn)
- patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
- with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
- futures = executor.map(
- find_new_responses, repeat(new_rtag_list), repeat(review_list),
- range(count), series.commits, patch_list, repeat(url),
- repeat(rest_api))
- for fresponse in futures:
- if fresponse:
- raise fresponse.exception()
- num_to_add = 0
- for seq, cmt in enumerate(series.commits):
- patch = patch_for_commit.get(seq)
- if not patch:
- continue
- terminal.Print('%3d %s' % (patch.seq, patch.subject[:50]),
- colour=col.BLUE)
- cmt = series.commits[seq]
- base_rtags = cmt.rtags
- new_rtags = new_rtag_list[seq]
- indent = ' ' * 2
- show_responses(base_rtags, indent, False)
- num_to_add += show_responses(new_rtags, indent, True)
- if show_comments:
- for review in review_list[seq]:
- terminal.Print('Review: %s' % review.meta, colour=col.RED)
- for snippet in review.snippets:
- for line in snippet:
- quoted = line.startswith('>')
- terminal.Print(' %s' % line,
- colour=col.MAGENTA if quoted else None)
- terminal.Print()
- terminal.Print("%d new response%s available in patchwork%s" %
- (num_to_add, 's' if num_to_add != 1 else '',
- '' if dest_branch
- else ' (use -d to write them to a new branch)'))
- if dest_branch:
- num_added = create_branch(series, new_rtag_list, branch,
- dest_branch, force, test_repo)
- terminal.Print(
- "%d response%s added from patchwork into new branch '%s'" %
- (num_added, 's' if num_added != 1 else '', dest_branch))
|