PRESUBMIT.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547
  1. # Copyright (c) 2013 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Top-level presubmit script for Skia.
  5. See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
  6. for more details about the presubmit API built into gcl.
  7. """
  8. import collections
  9. import csv
  10. import fnmatch
  11. import os
  12. import re
  13. import subprocess
  14. import sys
  15. import traceback
  16. REVERT_CL_SUBJECT_PREFIX = 'Revert '
  17. SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
  18. # Please add the complete email address here (and not just 'xyz@' or 'xyz').
  19. PUBLIC_API_OWNERS = (
  20. 'mtklein@chromium.org',
  21. 'mtklein@google.com',
  22. 'reed@chromium.org',
  23. 'reed@google.com',
  24. 'bsalomon@chromium.org',
  25. 'bsalomon@google.com',
  26. 'djsollen@chromium.org',
  27. 'djsollen@google.com',
  28. 'hcm@chromium.org',
  29. 'hcm@google.com',
  30. )
  31. AUTHORS_FILE_NAME = 'AUTHORS'
  32. DOCS_PREVIEW_URL = 'https://skia.org/?cl='
  33. GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
  34. SERVICE_ACCOUNT_SUFFIX = [
  35. '@%s.iam.gserviceaccount.com' % project for project in [
  36. 'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
  37. 'skia-corp.google.com', 'chops-service-accounts']]
  38. def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
  39. """Checks that files end with atleast one \n (LF)."""
  40. eof_files = []
  41. for f in input_api.AffectedSourceFiles(source_file_filter):
  42. contents = input_api.ReadFile(f, 'rb')
  43. # Check that the file ends in atleast one newline character.
  44. if len(contents) > 1 and contents[-1:] != '\n':
  45. eof_files.append(f.LocalPath())
  46. if eof_files:
  47. return [output_api.PresubmitPromptWarning(
  48. 'These files should end in a newline character:',
  49. items=eof_files)]
  50. return []
  51. def _JsonChecks(input_api, output_api):
  52. """Run checks on any modified json files."""
  53. failing_files = []
  54. for affected_file in input_api.AffectedFiles(None):
  55. affected_file_path = affected_file.LocalPath()
  56. is_json = affected_file_path.endswith('.json')
  57. is_metadata = (affected_file_path.startswith('site/') and
  58. affected_file_path.endswith('/METADATA'))
  59. if is_json or is_metadata:
  60. try:
  61. input_api.json.load(open(affected_file_path, 'r'))
  62. except ValueError:
  63. failing_files.append(affected_file_path)
  64. results = []
  65. if failing_files:
  66. results.append(
  67. output_api.PresubmitError(
  68. 'The following files contain invalid json:\n%s\n\n' %
  69. '\n'.join(failing_files)))
  70. return results
  71. def _IfDefChecks(input_api, output_api):
  72. """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
  73. comment_block_start_pattern = re.compile('^\s*\/\*.*$')
  74. comment_block_middle_pattern = re.compile('^\s+\*.*')
  75. comment_block_end_pattern = re.compile('^\s+\*\/.*$')
  76. single_line_comment_pattern = re.compile('^\s*//.*$')
  77. def is_comment(line):
  78. return (comment_block_start_pattern.match(line) or
  79. comment_block_middle_pattern.match(line) or
  80. comment_block_end_pattern.match(line) or
  81. single_line_comment_pattern.match(line))
  82. empty_line_pattern = re.compile('^\s*$')
  83. def is_empty_line(line):
  84. return empty_line_pattern.match(line)
  85. failing_files = []
  86. for affected_file in input_api.AffectedSourceFiles(None):
  87. affected_file_path = affected_file.LocalPath()
  88. if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
  89. f = open(affected_file_path)
  90. for line in f.xreadlines():
  91. if is_comment(line) or is_empty_line(line):
  92. continue
  93. # The below will be the first real line after comments and newlines.
  94. if line.startswith('#if 0 '):
  95. pass
  96. elif line.startswith('#if ') or line.startswith('#ifdef '):
  97. failing_files.append(affected_file_path)
  98. break
  99. results = []
  100. if failing_files:
  101. results.append(
  102. output_api.PresubmitError(
  103. 'The following files have #if or #ifdef before includes:\n%s\n\n'
  104. 'See https://bug.skia.org/3362 for why this should be fixed.' %
  105. '\n'.join(failing_files)))
  106. return results
  107. def _CopyrightChecks(input_api, output_api, source_file_filter=None):
  108. results = []
  109. year_pattern = r'\d{4}'
  110. year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
  111. years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
  112. copyright_pattern = (
  113. r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
  114. for affected_file in input_api.AffectedSourceFiles(source_file_filter):
  115. if 'third_party' in affected_file.LocalPath():
  116. continue
  117. contents = input_api.ReadFile(affected_file, 'rb')
  118. if not re.search(copyright_pattern, contents):
  119. results.append(output_api.PresubmitError(
  120. '%s is missing a correct copyright header.' % affected_file))
  121. return results
  122. def _ToolFlags(input_api, output_api):
  123. """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
  124. results = []
  125. sources = lambda x: ('dm_flags.py' in x.LocalPath() or
  126. 'nanobench_flags.py' in x.LocalPath())
  127. for f in input_api.AffectedSourceFiles(sources):
  128. if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
  129. results.append(output_api.PresubmitError('`python %s test` failed' % f))
  130. return results
  131. def _InfraTests(input_api, output_api):
  132. """Run the infra tests."""
  133. results = []
  134. if not any(f.LocalPath().startswith('infra')
  135. for f in input_api.AffectedFiles()):
  136. return results
  137. cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
  138. try:
  139. subprocess.check_output(cmd)
  140. except subprocess.CalledProcessError as e:
  141. results.append(output_api.PresubmitError(
  142. '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
  143. return results
  144. def _CheckGNFormatted(input_api, output_api):
  145. """Make sure any .gn files we're changing have been formatted."""
  146. results = []
  147. for f in input_api.AffectedFiles():
  148. if (not f.LocalPath().endswith('.gn') and
  149. not f.LocalPath().endswith('.gni')):
  150. continue
  151. gn = 'gn.bat' if 'win32' in sys.platform else 'gn'
  152. cmd = [gn, 'format', '--dry-run', f.LocalPath()]
  153. try:
  154. subprocess.check_output(cmd)
  155. except subprocess.CalledProcessError:
  156. fix = 'gn format ' + f.LocalPath()
  157. results.append(output_api.PresubmitError(
  158. '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
  159. return results
  160. def _CheckCompileIsolate(input_api, output_api):
  161. """Ensure that gen_compile_isolate.py does not change compile.isolate."""
  162. # Only run the check if files were added or removed.
  163. results = []
  164. script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
  165. isolate = os.path.join('infra', 'bots', 'compile.isolated')
  166. for f in input_api.AffectedFiles():
  167. if f.Action() in ('A', 'D', 'R'):
  168. break
  169. if f.LocalPath() in (script, isolate):
  170. break
  171. else:
  172. return results
  173. cmd = ['python', script, 'test']
  174. try:
  175. subprocess.check_output(cmd, stderr=subprocess.STDOUT)
  176. except subprocess.CalledProcessError as e:
  177. results.append(output_api.PresubmitError(e.output))
  178. return results
  179. class _WarningsAsErrors():
  180. def __init__(self, output_api):
  181. self.output_api = output_api
  182. self.old_warning = None
  183. def __enter__(self):
  184. self.old_warning = self.output_api.PresubmitPromptWarning
  185. self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
  186. return self.output_api
  187. def __exit__(self, ex_type, ex_value, ex_traceback):
  188. self.output_api.PresubmitPromptWarning = self.old_warning
  189. def _CommonChecks(input_api, output_api):
  190. """Presubmit checks common to upload and commit."""
  191. results = []
  192. sources = lambda x: (x.LocalPath().endswith('.h') or
  193. x.LocalPath().endswith('.py') or
  194. x.LocalPath().endswith('.sh') or
  195. x.LocalPath().endswith('.m') or
  196. x.LocalPath().endswith('.mm') or
  197. x.LocalPath().endswith('.go') or
  198. x.LocalPath().endswith('.c') or
  199. x.LocalPath().endswith('.cc') or
  200. x.LocalPath().endswith('.cpp'))
  201. results.extend(_CheckChangeHasEol(
  202. input_api, output_api, source_file_filter=sources))
  203. with _WarningsAsErrors(output_api):
  204. results.extend(input_api.canned_checks.CheckChangeHasNoCR(
  205. input_api, output_api, source_file_filter=sources))
  206. results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
  207. input_api, output_api, source_file_filter=sources))
  208. results.extend(_JsonChecks(input_api, output_api))
  209. results.extend(_IfDefChecks(input_api, output_api))
  210. results.extend(_CopyrightChecks(input_api, output_api,
  211. source_file_filter=sources))
  212. results.extend(_ToolFlags(input_api, output_api))
  213. results.extend(_CheckCompileIsolate(input_api, output_api))
  214. return results
  215. def CheckChangeOnUpload(input_api, output_api):
  216. """Presubmit checks for the change on upload.
  217. The following are the presubmit checks:
  218. * Check change has one and only one EOL.
  219. """
  220. results = []
  221. results.extend(_CommonChecks(input_api, output_api))
  222. # Run on upload, not commit, since the presubmit bot apparently doesn't have
  223. # coverage or Go installed.
  224. results.extend(_InfraTests(input_api, output_api))
  225. results.extend(_CheckGNFormatted(input_api, output_api))
  226. return results
  227. def _CheckTreeStatus(input_api, output_api, json_url):
  228. """Check whether to allow commit.
  229. Args:
  230. input_api: input related apis.
  231. output_api: output related apis.
  232. json_url: url to download json style status.
  233. """
  234. tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
  235. input_api, output_api, json_url=json_url)
  236. if not tree_status_results:
  237. # Check for caution state only if tree is not closed.
  238. connection = input_api.urllib2.urlopen(json_url)
  239. status = input_api.json.loads(connection.read())
  240. connection.close()
  241. if ('caution' in status['message'].lower() and
  242. os.isatty(sys.stdout.fileno())):
  243. # Display a prompt only if we are in an interactive shell. Without this
  244. # check the commit queue behaves incorrectly because it considers
  245. # prompts to be failures.
  246. short_text = 'Tree state is: ' + status['general_state']
  247. long_text = status['message'] + '\n' + json_url
  248. tree_status_results.append(
  249. output_api.PresubmitPromptWarning(
  250. message=short_text, long_text=long_text))
  251. else:
  252. # Tree status is closed. Put in message about contacting sheriff.
  253. connection = input_api.urllib2.urlopen(
  254. SKIA_TREE_STATUS_URL + '/current-sheriff')
  255. sheriff_details = input_api.json.loads(connection.read())
  256. if sheriff_details:
  257. tree_status_results[0]._message += (
  258. '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
  259. 'to submit a build fix\nand do not know how to submit because the '
  260. 'tree is closed') % sheriff_details['username']
  261. return tree_status_results
  262. class CodeReview(object):
  263. """Abstracts which codereview tool is used for the specified issue."""
  264. def __init__(self, input_api):
  265. self._issue = input_api.change.issue
  266. self._gerrit = input_api.gerrit
  267. def GetOwnerEmail(self):
  268. return self._gerrit.GetChangeOwner(self._issue)
  269. def GetSubject(self):
  270. return self._gerrit.GetChangeInfo(self._issue)['subject']
  271. def GetDescription(self):
  272. return self._gerrit.GetChangeDescription(self._issue)
  273. def IsDryRun(self):
  274. return self._gerrit.GetChangeInfo(
  275. self._issue)['labels']['Commit-Queue'].get('value', 0) == 1
  276. def GetReviewers(self):
  277. code_review_label = (
  278. self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
  279. return [r['email'] for r in code_review_label.get('all', [])]
  280. def GetApprovers(self):
  281. approvers = []
  282. code_review_label = (
  283. self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
  284. for m in code_review_label.get('all', []):
  285. if m.get("value") == 1:
  286. approvers.append(m["email"])
  287. return approvers
  288. def _CheckOwnerIsInAuthorsFile(input_api, output_api):
  289. results = []
  290. if input_api.change.issue:
  291. cr = CodeReview(input_api)
  292. owner_email = cr.GetOwnerEmail()
  293. # Service accounts don't need to be in AUTHORS.
  294. for suffix in SERVICE_ACCOUNT_SUFFIX:
  295. if owner_email.endswith(suffix):
  296. return results
  297. try:
  298. authors_content = ''
  299. for line in open(AUTHORS_FILE_NAME):
  300. if not line.startswith('#'):
  301. authors_content += line
  302. email_fnmatches = re.findall('<(.*)>', authors_content)
  303. for email_fnmatch in email_fnmatches:
  304. if fnmatch.fnmatch(owner_email, email_fnmatch):
  305. # Found a match, the user is in the AUTHORS file break out of the loop
  306. break
  307. else:
  308. results.append(
  309. output_api.PresubmitError(
  310. 'The email %s is not in Skia\'s AUTHORS file.\n'
  311. 'Issue owner, this CL must include an addition to the Skia AUTHORS '
  312. 'file.'
  313. % owner_email))
  314. except IOError:
  315. # Do not fail if authors file cannot be found.
  316. traceback.print_exc()
  317. input_api.logging.error('AUTHORS file not found!')
  318. return results
  319. def _CheckLGTMsForPublicAPI(input_api, output_api):
  320. """Check LGTMs for public API changes.
  321. For public API files make sure there is an LGTM from the list of owners in
  322. PUBLIC_API_OWNERS.
  323. """
  324. results = []
  325. requires_owner_check = False
  326. for affected_file in input_api.AffectedFiles():
  327. affected_file_path = affected_file.LocalPath()
  328. file_path, file_ext = os.path.splitext(affected_file_path)
  329. # We only care about files that end in .h and are under the top-level
  330. # include dir, but not include/private.
  331. if (file_ext == '.h' and
  332. 'include' == file_path.split(os.path.sep)[0] and
  333. 'private' not in file_path):
  334. requires_owner_check = True
  335. if not requires_owner_check:
  336. return results
  337. lgtm_from_owner = False
  338. if input_api.change.issue:
  339. cr = CodeReview(input_api)
  340. if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
  341. # It is a revert CL, ignore the public api owners check.
  342. return results
  343. if cr.IsDryRun():
  344. # Ignore public api owners check for dry run CLs since they are not
  345. # going to be committed.
  346. return results
  347. if input_api.gerrit:
  348. for reviewer in cr.GetReviewers():
  349. if reviewer in PUBLIC_API_OWNERS:
  350. # If an owner is specified as an reviewer in Gerrit then ignore the
  351. # public api owners check.
  352. return results
  353. else:
  354. match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
  355. if match:
  356. tbr_section = match.group(1).strip().split(' ')[0]
  357. tbr_entries = tbr_section.split(',')
  358. for owner in PUBLIC_API_OWNERS:
  359. if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
  360. # If an owner is specified in the TBR= line then ignore the public
  361. # api owners check.
  362. return results
  363. if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
  364. # An owner created the CL that is an automatic LGTM.
  365. lgtm_from_owner = True
  366. for approver in cr.GetApprovers():
  367. if approver in PUBLIC_API_OWNERS:
  368. # Found an lgtm in a message from an owner.
  369. lgtm_from_owner = True
  370. break
  371. if not lgtm_from_owner:
  372. results.append(
  373. output_api.PresubmitError(
  374. "If this CL adds to or changes Skia's public API, you need an LGTM "
  375. "from any of %s. If this CL only removes from or doesn't change "
  376. "Skia's public API, please add a short note to the CL saying so. "
  377. "Add one of the owners as a reviewer to your CL as well as to the "
  378. "TBR= line. If you don't know if this CL affects Skia's public "
  379. "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
  380. return results
  381. def _FooterExists(footers, key, value):
  382. for k, v in footers:
  383. if k == key and v == value:
  384. return True
  385. return False
  386. def PostUploadHook(cl, change, output_api):
  387. """git cl upload will call this hook after the issue is created/modified.
  388. This hook does the following:
  389. * Adds a link to preview docs changes if there are any docs changes in the CL.
  390. * Adds 'No-Try: true' if the CL contains only docs changes.
  391. """
  392. results = []
  393. atleast_one_docs_change = False
  394. all_docs_changes = True
  395. for affected_file in change.AffectedFiles():
  396. affected_file_path = affected_file.LocalPath()
  397. file_path, _ = os.path.splitext(affected_file_path)
  398. if 'site' == file_path.split(os.path.sep)[0]:
  399. atleast_one_docs_change = True
  400. else:
  401. all_docs_changes = False
  402. if atleast_one_docs_change and not all_docs_changes:
  403. break
  404. issue = cl.issue
  405. if issue:
  406. # Skip PostUploadHooks for all auto-commit service account bots. New
  407. # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
  408. # the "--use-commit-queue" flag to "git cl upload".
  409. for suffix in SERVICE_ACCOUNT_SUFFIX:
  410. if cl.GetIssueOwner().endswith(suffix):
  411. return results
  412. original_description_lines, footers = cl.GetDescriptionFooters()
  413. new_description_lines = list(original_description_lines)
  414. # If the change includes only doc changes then add No-Try: true in the
  415. # CL's description if it does not exist yet.
  416. if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'):
  417. new_description_lines.append('No-Try: true')
  418. results.append(
  419. output_api.PresubmitNotifyResult(
  420. 'This change has only doc changes. Automatically added '
  421. '\'No-Try: true\' to the CL\'s description'))
  422. # If there is atleast one docs change then add preview link in the CL's
  423. # description if it does not already exist there.
  424. docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue)
  425. docs_preview_line = 'Docs-Preview: %s' % docs_preview_link
  426. if (atleast_one_docs_change and
  427. not _FooterExists(footers, 'Docs-Preview', docs_preview_link)):
  428. # Automatically add a link to where the docs can be previewed.
  429. new_description_lines.append(docs_preview_line)
  430. results.append(
  431. output_api.PresubmitNotifyResult(
  432. 'Automatically added a link to preview the docs changes to the '
  433. 'CL\'s description'))
  434. # If the description has changed update it.
  435. if new_description_lines != original_description_lines:
  436. # Add a new line separating the new contents from the old contents.
  437. new_description_lines.insert(len(original_description_lines), '')
  438. cl.UpdateDescriptionFooters(new_description_lines, footers)
  439. return results
  440. def CheckChangeOnCommit(input_api, output_api):
  441. """Presubmit checks for the change on commit.
  442. The following are the presubmit checks:
  443. * Check change has one and only one EOL.
  444. * Ensures that the Skia tree is open in
  445. http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
  446. state and an error if it is in 'Closed' state.
  447. """
  448. results = []
  449. results.extend(_CommonChecks(input_api, output_api))
  450. results.extend(
  451. _CheckTreeStatus(input_api, output_api, json_url=(
  452. SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
  453. results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
  454. results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
  455. # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
  456. # content of files.
  457. results.extend(
  458. input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
  459. return results