123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547 |
- # Copyright (c) 2013 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Top-level presubmit script for Skia.
- See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
- for more details about the presubmit API built into gcl.
- """
- import collections
- import csv
- import fnmatch
- import os
- import re
- import subprocess
- import sys
- import traceback
- REVERT_CL_SUBJECT_PREFIX = 'Revert '
- SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
- # Please add the complete email address here (and not just 'xyz@' or 'xyz').
- PUBLIC_API_OWNERS = (
- 'mtklein@chromium.org',
- 'mtklein@google.com',
- 'reed@chromium.org',
- 'reed@google.com',
- 'bsalomon@chromium.org',
- 'bsalomon@google.com',
- 'djsollen@chromium.org',
- 'djsollen@google.com',
- 'hcm@chromium.org',
- 'hcm@google.com',
- )
- AUTHORS_FILE_NAME = 'AUTHORS'
- DOCS_PREVIEW_URL = 'https://skia.org/?cl='
- GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
- SERVICE_ACCOUNT_SUFFIX = [
- '@%s.iam.gserviceaccount.com' % project for project in [
- 'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
- 'skia-corp.google.com', 'chops-service-accounts']]
- def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
- """Checks that files end with atleast one \n (LF)."""
- eof_files = []
- for f in input_api.AffectedSourceFiles(source_file_filter):
- contents = input_api.ReadFile(f, 'rb')
- # Check that the file ends in atleast one newline character.
- if len(contents) > 1 and contents[-1:] != '\n':
- eof_files.append(f.LocalPath())
- if eof_files:
- return [output_api.PresubmitPromptWarning(
- 'These files should end in a newline character:',
- items=eof_files)]
- return []
- def _JsonChecks(input_api, output_api):
- """Run checks on any modified json files."""
- failing_files = []
- for affected_file in input_api.AffectedFiles(None):
- affected_file_path = affected_file.LocalPath()
- is_json = affected_file_path.endswith('.json')
- is_metadata = (affected_file_path.startswith('site/') and
- affected_file_path.endswith('/METADATA'))
- if is_json or is_metadata:
- try:
- input_api.json.load(open(affected_file_path, 'r'))
- except ValueError:
- failing_files.append(affected_file_path)
- results = []
- if failing_files:
- results.append(
- output_api.PresubmitError(
- 'The following files contain invalid json:\n%s\n\n' %
- '\n'.join(failing_files)))
- return results
- def _IfDefChecks(input_api, output_api):
- """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
- comment_block_start_pattern = re.compile('^\s*\/\*.*$')
- comment_block_middle_pattern = re.compile('^\s+\*.*')
- comment_block_end_pattern = re.compile('^\s+\*\/.*$')
- single_line_comment_pattern = re.compile('^\s*//.*$')
- def is_comment(line):
- return (comment_block_start_pattern.match(line) or
- comment_block_middle_pattern.match(line) or
- comment_block_end_pattern.match(line) or
- single_line_comment_pattern.match(line))
- empty_line_pattern = re.compile('^\s*$')
- def is_empty_line(line):
- return empty_line_pattern.match(line)
- failing_files = []
- for affected_file in input_api.AffectedSourceFiles(None):
- affected_file_path = affected_file.LocalPath()
- if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
- f = open(affected_file_path)
- for line in f.xreadlines():
- if is_comment(line) or is_empty_line(line):
- continue
- # The below will be the first real line after comments and newlines.
- if line.startswith('#if 0 '):
- pass
- elif line.startswith('#if ') or line.startswith('#ifdef '):
- failing_files.append(affected_file_path)
- break
- results = []
- if failing_files:
- results.append(
- output_api.PresubmitError(
- 'The following files have #if or #ifdef before includes:\n%s\n\n'
- 'See https://bug.skia.org/3362 for why this should be fixed.' %
- '\n'.join(failing_files)))
- return results
- def _CopyrightChecks(input_api, output_api, source_file_filter=None):
- results = []
- year_pattern = r'\d{4}'
- year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
- years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
- copyright_pattern = (
- r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
- for affected_file in input_api.AffectedSourceFiles(source_file_filter):
- if 'third_party' in affected_file.LocalPath():
- continue
- contents = input_api.ReadFile(affected_file, 'rb')
- if not re.search(copyright_pattern, contents):
- results.append(output_api.PresubmitError(
- '%s is missing a correct copyright header.' % affected_file))
- return results
- def _ToolFlags(input_api, output_api):
- """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
- results = []
- sources = lambda x: ('dm_flags.py' in x.LocalPath() or
- 'nanobench_flags.py' in x.LocalPath())
- for f in input_api.AffectedSourceFiles(sources):
- if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
- results.append(output_api.PresubmitError('`python %s test` failed' % f))
- return results
- def _InfraTests(input_api, output_api):
- """Run the infra tests."""
- results = []
- if not any(f.LocalPath().startswith('infra')
- for f in input_api.AffectedFiles()):
- return results
- cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
- try:
- subprocess.check_output(cmd)
- except subprocess.CalledProcessError as e:
- results.append(output_api.PresubmitError(
- '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
- return results
- def _CheckGNFormatted(input_api, output_api):
- """Make sure any .gn files we're changing have been formatted."""
- results = []
- for f in input_api.AffectedFiles():
- if (not f.LocalPath().endswith('.gn') and
- not f.LocalPath().endswith('.gni')):
- continue
- gn = 'gn.bat' if 'win32' in sys.platform else 'gn'
- cmd = [gn, 'format', '--dry-run', f.LocalPath()]
- try:
- subprocess.check_output(cmd)
- except subprocess.CalledProcessError:
- fix = 'gn format ' + f.LocalPath()
- results.append(output_api.PresubmitError(
- '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
- return results
- def _CheckCompileIsolate(input_api, output_api):
- """Ensure that gen_compile_isolate.py does not change compile.isolate."""
- # Only run the check if files were added or removed.
- results = []
- script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
- isolate = os.path.join('infra', 'bots', 'compile.isolated')
- for f in input_api.AffectedFiles():
- if f.Action() in ('A', 'D', 'R'):
- break
- if f.LocalPath() in (script, isolate):
- break
- else:
- return results
- cmd = ['python', script, 'test']
- try:
- subprocess.check_output(cmd, stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError as e:
- results.append(output_api.PresubmitError(e.output))
- return results
- class _WarningsAsErrors():
- def __init__(self, output_api):
- self.output_api = output_api
- self.old_warning = None
- def __enter__(self):
- self.old_warning = self.output_api.PresubmitPromptWarning
- self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
- return self.output_api
- def __exit__(self, ex_type, ex_value, ex_traceback):
- self.output_api.PresubmitPromptWarning = self.old_warning
- def _CommonChecks(input_api, output_api):
- """Presubmit checks common to upload and commit."""
- results = []
- sources = lambda x: (x.LocalPath().endswith('.h') or
- x.LocalPath().endswith('.py') or
- x.LocalPath().endswith('.sh') or
- x.LocalPath().endswith('.m') or
- x.LocalPath().endswith('.mm') or
- x.LocalPath().endswith('.go') or
- x.LocalPath().endswith('.c') or
- x.LocalPath().endswith('.cc') or
- x.LocalPath().endswith('.cpp'))
- results.extend(_CheckChangeHasEol(
- input_api, output_api, source_file_filter=sources))
- with _WarningsAsErrors(output_api):
- results.extend(input_api.canned_checks.CheckChangeHasNoCR(
- input_api, output_api, source_file_filter=sources))
- results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
- input_api, output_api, source_file_filter=sources))
- results.extend(_JsonChecks(input_api, output_api))
- results.extend(_IfDefChecks(input_api, output_api))
- results.extend(_CopyrightChecks(input_api, output_api,
- source_file_filter=sources))
- results.extend(_ToolFlags(input_api, output_api))
- results.extend(_CheckCompileIsolate(input_api, output_api))
- return results
- def CheckChangeOnUpload(input_api, output_api):
- """Presubmit checks for the change on upload.
- The following are the presubmit checks:
- * Check change has one and only one EOL.
- """
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- # Run on upload, not commit, since the presubmit bot apparently doesn't have
- # coverage or Go installed.
- results.extend(_InfraTests(input_api, output_api))
- results.extend(_CheckGNFormatted(input_api, output_api))
- return results
- def _CheckTreeStatus(input_api, output_api, json_url):
- """Check whether to allow commit.
- Args:
- input_api: input related apis.
- output_api: output related apis.
- json_url: url to download json style status.
- """
- tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
- input_api, output_api, json_url=json_url)
- if not tree_status_results:
- # Check for caution state only if tree is not closed.
- connection = input_api.urllib2.urlopen(json_url)
- status = input_api.json.loads(connection.read())
- connection.close()
- if ('caution' in status['message'].lower() and
- os.isatty(sys.stdout.fileno())):
- # Display a prompt only if we are in an interactive shell. Without this
- # check the commit queue behaves incorrectly because it considers
- # prompts to be failures.
- short_text = 'Tree state is: ' + status['general_state']
- long_text = status['message'] + '\n' + json_url
- tree_status_results.append(
- output_api.PresubmitPromptWarning(
- message=short_text, long_text=long_text))
- else:
- # Tree status is closed. Put in message about contacting sheriff.
- connection = input_api.urllib2.urlopen(
- SKIA_TREE_STATUS_URL + '/current-sheriff')
- sheriff_details = input_api.json.loads(connection.read())
- if sheriff_details:
- tree_status_results[0]._message += (
- '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
- 'to submit a build fix\nand do not know how to submit because the '
- 'tree is closed') % sheriff_details['username']
- return tree_status_results
- class CodeReview(object):
- """Abstracts which codereview tool is used for the specified issue."""
- def __init__(self, input_api):
- self._issue = input_api.change.issue
- self._gerrit = input_api.gerrit
- def GetOwnerEmail(self):
- return self._gerrit.GetChangeOwner(self._issue)
- def GetSubject(self):
- return self._gerrit.GetChangeInfo(self._issue)['subject']
- def GetDescription(self):
- return self._gerrit.GetChangeDescription(self._issue)
- def IsDryRun(self):
- return self._gerrit.GetChangeInfo(
- self._issue)['labels']['Commit-Queue'].get('value', 0) == 1
- def GetReviewers(self):
- code_review_label = (
- self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
- return [r['email'] for r in code_review_label.get('all', [])]
- def GetApprovers(self):
- approvers = []
- code_review_label = (
- self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
- for m in code_review_label.get('all', []):
- if m.get("value") == 1:
- approvers.append(m["email"])
- return approvers
- def _CheckOwnerIsInAuthorsFile(input_api, output_api):
- results = []
- if input_api.change.issue:
- cr = CodeReview(input_api)
- owner_email = cr.GetOwnerEmail()
- # Service accounts don't need to be in AUTHORS.
- for suffix in SERVICE_ACCOUNT_SUFFIX:
- if owner_email.endswith(suffix):
- return results
- try:
- authors_content = ''
- for line in open(AUTHORS_FILE_NAME):
- if not line.startswith('#'):
- authors_content += line
- email_fnmatches = re.findall('<(.*)>', authors_content)
- for email_fnmatch in email_fnmatches:
- if fnmatch.fnmatch(owner_email, email_fnmatch):
- # Found a match, the user is in the AUTHORS file break out of the loop
- break
- else:
- results.append(
- output_api.PresubmitError(
- 'The email %s is not in Skia\'s AUTHORS file.\n'
- 'Issue owner, this CL must include an addition to the Skia AUTHORS '
- 'file.'
- % owner_email))
- except IOError:
- # Do not fail if authors file cannot be found.
- traceback.print_exc()
- input_api.logging.error('AUTHORS file not found!')
- return results
- def _CheckLGTMsForPublicAPI(input_api, output_api):
- """Check LGTMs for public API changes.
- For public API files make sure there is an LGTM from the list of owners in
- PUBLIC_API_OWNERS.
- """
- results = []
- requires_owner_check = False
- for affected_file in input_api.AffectedFiles():
- affected_file_path = affected_file.LocalPath()
- file_path, file_ext = os.path.splitext(affected_file_path)
- # We only care about files that end in .h and are under the top-level
- # include dir, but not include/private.
- if (file_ext == '.h' and
- 'include' == file_path.split(os.path.sep)[0] and
- 'private' not in file_path):
- requires_owner_check = True
- if not requires_owner_check:
- return results
- lgtm_from_owner = False
- if input_api.change.issue:
- cr = CodeReview(input_api)
- if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
- # It is a revert CL, ignore the public api owners check.
- return results
- if cr.IsDryRun():
- # Ignore public api owners check for dry run CLs since they are not
- # going to be committed.
- return results
- if input_api.gerrit:
- for reviewer in cr.GetReviewers():
- if reviewer in PUBLIC_API_OWNERS:
- # If an owner is specified as an reviewer in Gerrit then ignore the
- # public api owners check.
- return results
- else:
- match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
- if match:
- tbr_section = match.group(1).strip().split(' ')[0]
- tbr_entries = tbr_section.split(',')
- for owner in PUBLIC_API_OWNERS:
- if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
- # If an owner is specified in the TBR= line then ignore the public
- # api owners check.
- return results
- if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
- # An owner created the CL that is an automatic LGTM.
- lgtm_from_owner = True
- for approver in cr.GetApprovers():
- if approver in PUBLIC_API_OWNERS:
- # Found an lgtm in a message from an owner.
- lgtm_from_owner = True
- break
- if not lgtm_from_owner:
- results.append(
- output_api.PresubmitError(
- "If this CL adds to or changes Skia's public API, you need an LGTM "
- "from any of %s. If this CL only removes from or doesn't change "
- "Skia's public API, please add a short note to the CL saying so. "
- "Add one of the owners as a reviewer to your CL as well as to the "
- "TBR= line. If you don't know if this CL affects Skia's public "
- "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
- return results
- def _FooterExists(footers, key, value):
- for k, v in footers:
- if k == key and v == value:
- return True
- return False
- def PostUploadHook(cl, change, output_api):
- """git cl upload will call this hook after the issue is created/modified.
- This hook does the following:
- * Adds a link to preview docs changes if there are any docs changes in the CL.
- * Adds 'No-Try: true' if the CL contains only docs changes.
- """
- results = []
- atleast_one_docs_change = False
- all_docs_changes = True
- for affected_file in change.AffectedFiles():
- affected_file_path = affected_file.LocalPath()
- file_path, _ = os.path.splitext(affected_file_path)
- if 'site' == file_path.split(os.path.sep)[0]:
- atleast_one_docs_change = True
- else:
- all_docs_changes = False
- if atleast_one_docs_change and not all_docs_changes:
- break
- issue = cl.issue
- if issue:
- # Skip PostUploadHooks for all auto-commit service account bots. New
- # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
- # the "--use-commit-queue" flag to "git cl upload".
- for suffix in SERVICE_ACCOUNT_SUFFIX:
- if cl.GetIssueOwner().endswith(suffix):
- return results
- original_description_lines, footers = cl.GetDescriptionFooters()
- new_description_lines = list(original_description_lines)
- # If the change includes only doc changes then add No-Try: true in the
- # CL's description if it does not exist yet.
- if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'):
- new_description_lines.append('No-Try: true')
- results.append(
- output_api.PresubmitNotifyResult(
- 'This change has only doc changes. Automatically added '
- '\'No-Try: true\' to the CL\'s description'))
- # If there is atleast one docs change then add preview link in the CL's
- # description if it does not already exist there.
- docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue)
- docs_preview_line = 'Docs-Preview: %s' % docs_preview_link
- if (atleast_one_docs_change and
- not _FooterExists(footers, 'Docs-Preview', docs_preview_link)):
- # Automatically add a link to where the docs can be previewed.
- new_description_lines.append(docs_preview_line)
- results.append(
- output_api.PresubmitNotifyResult(
- 'Automatically added a link to preview the docs changes to the '
- 'CL\'s description'))
- # If the description has changed update it.
- if new_description_lines != original_description_lines:
- # Add a new line separating the new contents from the old contents.
- new_description_lines.insert(len(original_description_lines), '')
- cl.UpdateDescriptionFooters(new_description_lines, footers)
- return results
- def CheckChangeOnCommit(input_api, output_api):
- """Presubmit checks for the change on commit.
- The following are the presubmit checks:
- * Check change has one and only one EOL.
- * Ensures that the Skia tree is open in
- http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
- state and an error if it is in 'Closed' state.
- """
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- results.extend(
- _CheckTreeStatus(input_api, output_api, json_url=(
- SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
- results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
- results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
- # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
- # content of files.
- results.extend(
- input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
- return results
|