PRESUBMIT.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371
  1. # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. import io
  5. import os
  6. import re
  7. import subprocess
  8. import sys
  9. # In this file `sys.executable` is used instead of
  10. # `input_api.python3_executable` because on Windows
  11. # `input_api.python3_executable` is `vpython3.bat` whereas `sys.executable` is
  12. # `python.exe`. If `input_api.python3_executable` is used, we need to explicitly
  13. # pass `shell=True` to `subprocess.Popen()`, which is a security risk
  14. # (https://docs.python.org/3/library/subprocess.html#security-considerations).
  15. #
  16. # TODO: Investigate the incompatibility of `input_api.python3_executable` on
  17. # Windows, for this particular PRESUBMIT script.
  18. USE_PYTHON3 = True
  19. def RunCmdAndCheck(cmd, err_string, output_api, cwd=None, warning=False):
  20. results = []
  21. p = subprocess.Popen(cmd, cwd=cwd,
  22. stdout=subprocess.PIPE,
  23. stderr=subprocess.PIPE)
  24. (_, p_stderr) = p.communicate()
  25. if p.returncode:
  26. if warning:
  27. results.append(output_api.PresubmitPromptWarning(
  28. '%s\n\n%s' % (err_string, p_stderr.decode('utf-8'))))
  29. else:
  30. results.append(
  31. output_api.PresubmitError(err_string,
  32. long_text=p_stderr.decode('utf-8')))
  33. return results
  34. def RunUnittests(input_api, output_api):
  35. # Run some Generator unittests if the generator source was changed.
  36. results = []
  37. files = input_api.LocalPaths()
  38. generator_files = []
  39. for filename in files:
  40. name_parts = filename.split(os.sep)
  41. if name_parts[0:2] == ['ppapi', 'generators']:
  42. generator_files.append(filename)
  43. if generator_files != []:
  44. cmd = [sys.executable, 'idl_tests.py']
  45. ppapi_dir = input_api.PresubmitLocalPath()
  46. results.extend(RunCmdAndCheck(cmd,
  47. 'PPAPI IDL unittests failed.',
  48. output_api,
  49. os.path.join(ppapi_dir, 'generators')))
  50. return results
  51. # Verify that the files do not contain a 'TODO' in them.
  52. RE_TODO = re.compile(r'\WTODO\W', flags=re.I)
  53. def CheckTODO(input_api, output_api):
  54. live_files = input_api.AffectedFiles(include_deletes=False)
  55. files = [f.LocalPath() for f in live_files]
  56. todo = []
  57. for filename in files:
  58. name, ext = os.path.splitext(filename)
  59. name_parts = name.split(os.sep)
  60. # Only check normal build sources.
  61. if ext not in ['.h', '.idl']:
  62. continue
  63. # Only examine the ppapi directory.
  64. if name_parts[0] != 'ppapi':
  65. continue
  66. # Only examine public plugin facing directories.
  67. if name_parts[1] not in ['api', 'c', 'cpp', 'utility']:
  68. continue
  69. # Only examine public stable interfaces.
  70. if name_parts[2] in ['dev', 'private', 'trusted']:
  71. continue
  72. filepath = os.path.join('..', filename)
  73. with io.open(filepath, encoding='utf-8') as f:
  74. if RE_TODO.search(f.read()):
  75. todo.append(filename)
  76. if todo:
  77. return [output_api.PresubmitPromptWarning(
  78. 'TODOs found in stable public PPAPI files:',
  79. long_text='\n'.join(todo))]
  80. return []
  81. # Verify that no CPP wrappers use un-versioned PPB interface name macros.
  82. RE_UNVERSIONED_PPB = re.compile(r'\bPPB_\w+_INTERFACE\b')
  83. def CheckUnversionedPPB(input_api, output_api):
  84. live_files = input_api.AffectedFiles(include_deletes=False)
  85. files = [f.LocalPath() for f in live_files]
  86. todo = []
  87. for filename in files:
  88. name, ext = os.path.splitext(filename)
  89. name_parts = name.split(os.sep)
  90. # Only check C++ sources.
  91. if ext not in ['.cc']:
  92. continue
  93. # Only examine the public plugin facing ppapi/cpp directory.
  94. if name_parts[0:2] != ['ppapi', 'cpp']:
  95. continue
  96. # Only examine public stable and trusted interfaces.
  97. if name_parts[2] in ['dev', 'private']:
  98. continue
  99. filepath = os.path.join('..', filename)
  100. with io.open(filepath, encoding='utf-8') as f:
  101. if RE_UNVERSIONED_PPB.search(f.read()):
  102. todo.append(filename)
  103. if todo:
  104. return [output_api.PresubmitError(
  105. 'Unversioned PPB interface references found in PPAPI C++ wrappers:',
  106. long_text='\n'.join(todo))]
  107. return []
  108. # Verify that changes to ppapi headers/sources are also made to NaCl SDK.
  109. def CheckUpdatedNaClSDK(input_api, output_api):
  110. files = input_api.LocalPaths()
  111. # PPAPI files the Native Client SDK cares about.
  112. nacl_sdk_files = []
  113. for filename in files:
  114. name, ext = os.path.splitext(filename)
  115. name_parts = name.split(os.sep)
  116. if len(name_parts) <= 2:
  117. continue
  118. if name_parts[0] != 'ppapi':
  119. continue
  120. if ((name_parts[1] == 'c' and ext == '.h') or
  121. (name_parts[1] in ('cpp', 'utility') and ext in ('.h', '.cc'))):
  122. if name_parts[2] in ('documentation', 'trusted'):
  123. continue
  124. nacl_sdk_files.append(filename)
  125. if not nacl_sdk_files:
  126. return []
  127. verify_ppapi_py = os.path.join(input_api.change.RepositoryRoot(),
  128. 'native_client_sdk', 'src', 'build_tools',
  129. 'verify_ppapi.py')
  130. # When running git cl presubmit --all this presubmit may be asked to check
  131. # ~300 files, leading to a command line that is ~9,500 characters, which
  132. # exceeds the Windows 8191 character cmd.exe limit and causes cryptic failures
  133. # with no context. To avoid these we break the command up into smaller pieces.
  134. # The error is:
  135. # The command line is too long.
  136. files_per_command = 25 if input_api.is_windows else 1000
  137. results = []
  138. for i in range(len(nacl_sdk_files), files_per_command):
  139. cmd = [sys.executable, verify_ppapi_py
  140. ] + nacl_sdk_files[i:i + files_per_command]
  141. results.extend(
  142. RunCmdAndCheck(
  143. cmd,'PPAPI Interface modified without updating NaCl SDK.\n'
  144. '(note that some dev interfaces should not be added '
  145. 'the NaCl SDK; when in doubt, ask a ppapi OWNER.\n'
  146. 'To ignore a file, add it to IGNORED_FILES in '
  147. 'native_client_sdk/src/build_tools/verify_ppapi.py)',
  148. output_api,
  149. warning=True))
  150. return results
  151. # Verify that changes to ppapi/thunk/interfaces_* files have a corresponding
  152. # change to tools/metrics/histograms/enums.xml for UMA tracking.
  153. def CheckHistogramXml(input_api, output_api):
  154. # We can't use input_api.LocalPaths() here because we need to know about
  155. # changes outside of ppapi/. See tools/depot_tools/presubmit_support.py for
  156. # details on input_api.
  157. files = input_api.change.AffectedFiles()
  158. INTERFACE_FILES = ('ppapi/thunk/interfaces_legacy.h',
  159. 'ppapi/thunk/interfaces_ppb_private.h',
  160. 'ppapi/thunk/interfaces_ppb_private_no_permissions.h',
  161. 'ppapi/thunk/interfaces_ppb_public_dev_channel.h',
  162. 'ppapi/thunk/interfaces_ppb_public_dev.h',
  163. 'ppapi/thunk/interfaces_ppb_public_stable.h',
  164. 'ppapi/thunk/interfaces_ppb_public_socket.h')
  165. HISTOGRAM_XML_FILE = 'tools/metrics/histograms/enums.xml'
  166. interface_changes = []
  167. has_histogram_xml_change = False
  168. for filename in files:
  169. path = filename.LocalPath()
  170. if path in INTERFACE_FILES:
  171. interface_changes.append(path)
  172. if path == HISTOGRAM_XML_FILE:
  173. has_histogram_xml_change = True
  174. if interface_changes and not has_histogram_xml_change:
  175. return [output_api.PresubmitNotifyResult(
  176. 'Missing change to tools/metrics/histograms/enums.xml.\n' +
  177. 'Run pepper_hash_for_uma to make get values for new interfaces.\n' +
  178. 'Interface changes:\n' + '\n'.join(interface_changes))]
  179. return []
  180. def CheckChange(input_api, output_api):
  181. results = []
  182. results.extend(RunUnittests(input_api, output_api))
  183. results.extend(CheckTODO(input_api, output_api))
  184. results.extend(CheckUnversionedPPB(input_api, output_api))
  185. results.extend(CheckUpdatedNaClSDK(input_api, output_api))
  186. results.extend(CheckHistogramXml(input_api, output_api))
  187. # Verify all modified *.idl have a matching *.h
  188. files = input_api.LocalPaths()
  189. h_files = []
  190. idl_files = []
  191. generators_changed = False
  192. # These are autogenerated by the command buffer generator, they don't go
  193. # through idl.
  194. whitelist = ['ppb_opengles2', 'ppb_opengles2ext_dev']
  195. # Find all relevant .h and .idl files.
  196. for filename in files:
  197. name, ext = os.path.splitext(filename)
  198. name_parts = name.split(os.sep)
  199. if name_parts[-1] in whitelist:
  200. continue
  201. if name_parts[0:2] == ['ppapi', 'c'] and ext == '.h':
  202. h_files.append('/'.join(name_parts[2:]))
  203. elif name_parts[0:2] == ['ppapi', 'api'] and ext == '.idl':
  204. idl_files.append('/'.join(name_parts[2:]))
  205. elif name_parts[0:2] == ['ppapi', 'generators']:
  206. generators_changed = True
  207. # Generate a list of all appropriate *.h and *.idl changes in this CL.
  208. both = h_files + idl_files
  209. # If there aren't any, we are done checking.
  210. if not both: return results
  211. missing = []
  212. for filename in idl_files:
  213. if filename not in set(h_files):
  214. missing.append('ppapi/api/%s.idl' % filename)
  215. # An IDL change that includes [generate_thunk] doesn't need to have
  216. # an update to the corresponding .h file.
  217. new_thunk_files = []
  218. for filename in missing:
  219. lines = input_api.RightHandSideLines(lambda f: f.LocalPath() == filename)
  220. for line in lines:
  221. if line[2].strip() == '[generate_thunk]':
  222. new_thunk_files.append(filename)
  223. for filename in new_thunk_files:
  224. missing.remove(filename)
  225. if missing:
  226. results.append(
  227. output_api.PresubmitPromptWarning(
  228. 'Missing PPAPI header, no change or skipped generation?',
  229. long_text='\n '.join(missing)))
  230. missing_dev = []
  231. missing_stable = []
  232. missing_priv = []
  233. for filename in h_files:
  234. if filename not in set(idl_files):
  235. name_parts = filename.split(os.sep)
  236. if name_parts[-1] == 'pp_macros':
  237. # The C header generator adds a PPAPI_RELEASE macro based on all the
  238. # IDL files, so pp_macros.h may change while its IDL does not.
  239. lines = input_api.RightHandSideLines(
  240. lambda f: f.LocalPath() == 'ppapi/c/%s.h' % filename)
  241. releaseChanged = False
  242. for line in lines:
  243. if line[2].split()[:2] == ['#define', 'PPAPI_RELEASE']:
  244. results.append(
  245. output_api.PresubmitPromptOrNotify(
  246. 'PPAPI_RELEASE has changed', long_text=line[2]))
  247. releaseChanged = True
  248. break
  249. if releaseChanged:
  250. continue
  251. if 'trusted' in name_parts:
  252. missing_priv.append(' ppapi/c/%s.h' % filename)
  253. continue
  254. if 'private' in name_parts:
  255. missing_priv.append(' ppapi/c/%s.h' % filename)
  256. continue
  257. if 'dev' in name_parts:
  258. missing_dev.append(' ppapi/c/%s.h' % filename)
  259. continue
  260. missing_stable.append(' ppapi/c/%s.h' % filename)
  261. if missing_priv:
  262. results.append(
  263. output_api.PresubmitPromptWarning(
  264. 'Missing PPAPI IDL for private interface, please generate IDL:',
  265. long_text='\n'.join(missing_priv)))
  266. if missing_dev:
  267. results.append(
  268. output_api.PresubmitPromptWarning(
  269. 'Missing PPAPI IDL for DEV, required before moving to stable:',
  270. long_text='\n'.join(missing_dev)))
  271. if missing_stable:
  272. # It might be okay that the header changed without a corresponding IDL
  273. # change. E.g., comment indenting may have been changed. Treat this as a
  274. # warning.
  275. if generators_changed:
  276. results.append(
  277. output_api.PresubmitPromptWarning(
  278. 'Missing PPAPI IDL for stable interface (due to change in ' +
  279. 'generators?):',
  280. long_text='\n'.join(missing_stable)))
  281. else:
  282. results.append(
  283. output_api.PresubmitError(
  284. 'Missing PPAPI IDL for stable interface:',
  285. long_text='\n'.join(missing_stable)))
  286. # Verify all *.h files match *.idl definitions, use:
  287. # --test to prevent output to disk
  288. # --diff to generate a unified diff
  289. # --out to pick which files to examine (only the ones in the CL)
  290. ppapi_dir = input_api.PresubmitLocalPath()
  291. cmd = [sys.executable, 'generator.py',
  292. '--wnone', '--diff', '--test','--cgen', '--range=start,end']
  293. # Only generate output for IDL files references (as *.h or *.idl) in this CL
  294. cmd.append('--out=' + ','.join([name + '.idl' for name in both]))
  295. cmd_results = RunCmdAndCheck(cmd,
  296. 'PPAPI IDL Diff detected: Run the generator.',
  297. output_api,
  298. os.path.join(ppapi_dir, 'generators'))
  299. if cmd_results:
  300. results.extend(cmd_results)
  301. return results
  302. def CheckChangeOnUpload(input_api, output_api):
  303. return CheckChange(input_api, output_api)
  304. def CheckChangeOnCommit(input_api, output_api):
  305. return CheckChange(input_api, output_api)