v8_presubmit.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802
  1. #!/usr/bin/env python3
  2. #
  3. # Copyright 2012 the V8 project authors. All rights reserved.
  4. # Redistribution and use in source and binary forms, with or without
  5. # modification, are permitted provided that the following conditions are
  6. # met:
  7. #
  8. # * Redistributions of source code must retain the above copyright
  9. # notice, this list of conditions and the following disclaimer.
  10. # * Redistributions in binary form must reproduce the above
  11. # copyright notice, this list of conditions and the following
  12. # disclaimer in the documentation and/or other materials provided
  13. # with the distribution.
  14. # * Neither the name of Google Inc. nor the names of its
  15. # contributors may be used to endorse or promote products derived
  16. # from this software without specific prior written permission.
  17. #
  18. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29. import hashlib
  30. md5er = hashlib.md5
  31. import json
  32. import multiprocessing
  33. import optparse
  34. import os
  35. from os.path import abspath, join, dirname, basename, exists
  36. import pickle
  37. import re
  38. import subprocess
  39. from subprocess import PIPE
  40. import sys
  41. from testrunner.local import statusfile
  42. from testrunner.local import testsuite
  43. from testrunner.local import utils
  44. def decode(arg, encoding="utf-8"):
  45. return arg.decode(encoding)
  46. # Special LINT rules diverging from default and reason.
  47. # build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
  48. # We now run our own header guard check in PRESUBMIT.py.
  49. # build/include_what_you_use: Started giving false positives for variables
  50. # named "string" and "map" assuming that you needed to include STL headers.
  51. # runtime/references: As of May 2020 the C++ style guide suggests using
  52. # references for out parameters, see
  53. # https://google.github.io/styleguide/cppguide.html#Inputs_and_Outputs.
  54. # whitespace/braces: Doesn't handle {}-initialization for custom types
  55. # well; also should be subsumed by clang-format.
  56. LINT_RULES = """
  57. -build/header_guard
  58. -build/include_what_you_use
  59. -readability/fn_size
  60. -readability/multiline_comment
  61. -runtime/references
  62. -whitespace/braces
  63. -whitespace/comments
  64. """.split()
  65. LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]')
  66. FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
  67. ASSERT_OPTIMIZED_PATTERN = re.compile("assertOptimized")
  68. FLAGS_ENABLE_OPT = re.compile("//\s*Flags:.*--opt[^-].*\n")
  69. ASSERT_UNOPTIMIZED_PATTERN = re.compile("assertUnoptimized")
  70. FLAGS_NO_ALWAYS_OPT = re.compile("//\s*Flags:.*--no-?always-opt.*\n")
  71. TOOLS_PATH = dirname(abspath(__file__))
  72. DEPS_DEPOT_TOOLS_PATH = abspath(
  73. join(TOOLS_PATH, '..', 'third_party', 'depot_tools'))
  74. def CppLintWorker(command):
  75. try:
  76. process = subprocess.Popen(command, stderr=subprocess.PIPE)
  77. process.wait()
  78. out_lines = ""
  79. error_count = -1
  80. while True:
  81. out_line = decode(process.stderr.readline())
  82. if out_line == '' and process.poll() != None:
  83. if error_count == -1:
  84. print("Failed to process %s" % command.pop())
  85. return 1
  86. break
  87. if out_line.strip() == 'Total errors found: 0':
  88. out_lines += "Done processing %s\n" % command.pop()
  89. error_count += 1
  90. else:
  91. m = LINT_OUTPUT_PATTERN.match(out_line)
  92. if m:
  93. out_lines += out_line
  94. error_count += 1
  95. sys.stdout.write(out_lines)
  96. return error_count
  97. except KeyboardInterrupt:
  98. process.kill()
  99. except:
  100. print('Error running cpplint.py. Please make sure you have depot_tools' +
  101. ' in your third_party directory. Lint check skipped.')
  102. process.kill()
  103. def TorqueLintWorker(command):
  104. try:
  105. process = subprocess.Popen(command, stderr=subprocess.PIPE)
  106. process.wait()
  107. out_lines = ""
  108. error_count = 0
  109. while True:
  110. out_line = decode(process.stderr.readline())
  111. if out_line == '' and process.poll() != None:
  112. break
  113. out_lines += out_line
  114. error_count += 1
  115. sys.stdout.write(out_lines)
  116. if error_count != 0:
  117. sys.stdout.write(
  118. "warning: formatting and overwriting unformatted Torque files\n")
  119. return error_count
  120. except KeyboardInterrupt:
  121. process.kill()
  122. except:
  123. print('Error running format-torque.py')
  124. process.kill()
  125. def JSLintWorker(command):
  126. def format_file(command):
  127. try:
  128. file_name = command[-1]
  129. with open(file_name, "r") as file_handle:
  130. contents = file_handle.read()
  131. process = subprocess.Popen(command, stdout=PIPE, stderr=subprocess.PIPE)
  132. output, err = process.communicate()
  133. rc = process.returncode
  134. if rc != 0:
  135. sys.stdout.write("error code " + str(rc) + " running clang-format.\n")
  136. return rc
  137. if decode(output) != contents:
  138. return 1
  139. return 0
  140. except KeyboardInterrupt:
  141. process.kill()
  142. except Exception:
  143. print(
  144. 'Error running clang-format. Please make sure you have depot_tools' +
  145. ' in your third_party directory. Lint check skipped.')
  146. process.kill()
  147. rc = format_file(command)
  148. if rc == 1:
  149. # There are files that need to be formatted, let's format them in place.
  150. file_name = command[-1]
  151. sys.stdout.write("Formatting %s.\n" % (file_name))
  152. rc = format_file(command[:-1] + ["-i", file_name])
  153. return rc
  154. class FileContentsCache(object):
  155. def __init__(self, sums_file_name):
  156. self.sums = {}
  157. self.sums_file_name = sums_file_name
  158. def Load(self):
  159. try:
  160. sums_file = None
  161. try:
  162. sums_file = open(self.sums_file_name, 'rb')
  163. self.sums = pickle.load(sums_file)
  164. except:
  165. # Cannot parse pickle for any reason. Not much we can do about it.
  166. pass
  167. finally:
  168. if sums_file:
  169. sums_file.close()
  170. def Save(self):
  171. try:
  172. sums_file = open(self.sums_file_name, 'wb')
  173. pickle.dump(self.sums, sums_file)
  174. except:
  175. # Failed to write pickle. Try to clean-up behind us.
  176. if sums_file:
  177. sums_file.close()
  178. try:
  179. os.unlink(self.sums_file_name)
  180. except:
  181. pass
  182. finally:
  183. sums_file.close()
  184. def FilterUnchangedFiles(self, files):
  185. changed_or_new = []
  186. for file in files:
  187. try:
  188. handle = open(file, "rb")
  189. file_sum = md5er(handle.read()).digest()
  190. if not file in self.sums or self.sums[file] != file_sum:
  191. changed_or_new.append(file)
  192. self.sums[file] = file_sum
  193. finally:
  194. handle.close()
  195. return changed_or_new
  196. def RemoveFile(self, file):
  197. if file in self.sums:
  198. self.sums.pop(file)
  199. class SourceFileProcessor(object):
  200. """
  201. Utility class that can run through a directory structure, find all relevant
  202. files and invoke a custom check on the files.
  203. """
  204. def RunOnPath(self, path):
  205. """Runs processor on all files under the given path."""
  206. all_files = []
  207. for file in self.GetPathsToSearch():
  208. all_files += self.FindFilesIn(join(path, file))
  209. return self.ProcessFiles(all_files)
  210. def RunOnFiles(self, files):
  211. """Runs processor only on affected files."""
  212. # Helper for getting directory pieces.
  213. dirs = lambda f: dirname(f).split(os.sep)
  214. # Path offsets where to look (to be in sync with RunOnPath).
  215. # Normalize '.' to check for it with str.startswith.
  216. search_paths = [('' if p == '.' else p) for p in self.GetPathsToSearch()]
  217. all_files = [
  218. f.AbsoluteLocalPath()
  219. for f in files
  220. if (not self.IgnoreFile(f.LocalPath()) and
  221. self.IsRelevant(f.LocalPath()) and
  222. all(not self.IgnoreDir(d) for d in dirs(f.LocalPath())) and
  223. any(map(f.LocalPath().startswith, search_paths)))
  224. ]
  225. return self.ProcessFiles(all_files)
  226. def IgnoreDir(self, name):
  227. return (name.startswith('.') or
  228. name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
  229. 'octane', 'sunspider', 'traces-arm64'))
  230. def IgnoreFile(self, name):
  231. return name.startswith('.')
  232. def FindFilesIn(self, path):
  233. result = []
  234. for (root, dirs, files) in os.walk(path):
  235. for ignored in [x for x in dirs if self.IgnoreDir(x)]:
  236. dirs.remove(ignored)
  237. for file in files:
  238. if not self.IgnoreFile(file) and self.IsRelevant(file):
  239. result.append(join(root, file))
  240. return result
  241. class CacheableSourceFileProcessor(SourceFileProcessor):
  242. """Utility class that allows caching ProcessFiles() method calls.
  243. In order to use it, create a ProcessFilesWithoutCaching method that returns
  244. the files requiring intervention after processing the source files.
  245. """
  246. def __init__(self, use_cache, cache_file_path, file_type):
  247. self.use_cache = use_cache
  248. self.cache_file_path = cache_file_path
  249. self.file_type = file_type
  250. def GetProcessorWorker(self):
  251. """Expected to return the worker function to run the formatter."""
  252. raise NotImplementedError
  253. def GetProcessorScript(self):
  254. """Expected to return a tuple
  255. (path to the format processor script, list of arguments)."""
  256. raise NotImplementedError
  257. def GetProcessorCommand(self):
  258. format_processor, options = self.GetProcessorScript()
  259. if not format_processor:
  260. print('Could not find the formatter for % files' % self.file_type)
  261. sys.exit(1)
  262. command = [sys.executable, format_processor]
  263. command.extend(options)
  264. return command
  265. def ProcessFiles(self, files):
  266. if self.use_cache:
  267. cache = FileContentsCache(self.cache_file_path)
  268. cache.Load()
  269. files = cache.FilterUnchangedFiles(files)
  270. if len(files) == 0:
  271. print('No changes in %s files detected. Skipping check' % self.file_type)
  272. return True
  273. files_requiring_changes = self.DetectFilesToChange(files)
  274. print (
  275. 'Total %s files found that require formatting: %d' %
  276. (self.file_type, len(files_requiring_changes)))
  277. if self.use_cache:
  278. for file in files_requiring_changes:
  279. cache.RemoveFile(file)
  280. cache.Save()
  281. return files_requiring_changes == []
  282. def DetectFilesToChange(self, files):
  283. command = self.GetProcessorCommand()
  284. worker = self.GetProcessorWorker()
  285. commands = [command + [file] for file in files]
  286. count = multiprocessing.cpu_count()
  287. pool = multiprocessing.Pool(count)
  288. try:
  289. results = pool.map_async(worker, commands).get(timeout=240)
  290. except KeyboardInterrupt:
  291. print("\nCaught KeyboardInterrupt, terminating workers.")
  292. pool.terminate()
  293. pool.join()
  294. sys.exit(1)
  295. unformatted_files = []
  296. for index, errors in enumerate(results):
  297. if errors > 0:
  298. unformatted_files.append(files[index])
  299. return unformatted_files
  300. class CppLintProcessor(CacheableSourceFileProcessor):
  301. """
  302. Lint files to check that they follow the google code style.
  303. """
  304. def __init__(self, use_cache=True):
  305. super(CppLintProcessor, self).__init__(
  306. use_cache=use_cache, cache_file_path='.cpplint-cache', file_type='C/C++')
  307. def IsRelevant(self, name):
  308. return name.endswith('.cc') or name.endswith('.h')
  309. def IgnoreDir(self, name):
  310. return (super(CppLintProcessor, self).IgnoreDir(name)
  311. or (name == 'third_party'))
  312. IGNORE_LINT = [
  313. 'export-template.h',
  314. 'flag-definitions.h',
  315. 'gay-fixed.cc',
  316. 'gay-precision.cc',
  317. 'gay-shortest.cc',
  318. ]
  319. def IgnoreFile(self, name):
  320. return (super(CppLintProcessor, self).IgnoreFile(name)
  321. or (name in CppLintProcessor.IGNORE_LINT))
  322. def GetPathsToSearch(self):
  323. dirs = ['include', 'samples', 'src']
  324. test_dirs = ['cctest', 'common', 'fuzzer', 'inspector', 'unittests']
  325. return dirs + [join('test', dir) for dir in test_dirs]
  326. def GetProcessorWorker(self):
  327. return CppLintWorker
  328. def GetProcessorScript(self):
  329. filters = ','.join([n for n in LINT_RULES])
  330. arguments = ['--filter', filters]
  331. cpplint = os.path.join(DEPS_DEPOT_TOOLS_PATH, 'cpplint.py')
  332. return cpplint, arguments
  333. class TorqueLintProcessor(CacheableSourceFileProcessor):
  334. """
  335. Check .tq files to verify they follow the Torque style guide.
  336. """
  337. def __init__(self, use_cache=True):
  338. super(TorqueLintProcessor, self).__init__(
  339. use_cache=use_cache, cache_file_path='.torquelint-cache',
  340. file_type='Torque')
  341. def IsRelevant(self, name):
  342. return name.endswith('.tq')
  343. def GetPathsToSearch(self):
  344. dirs = ['third_party', 'src']
  345. test_dirs = ['torque']
  346. return dirs + [join('test', dir) for dir in test_dirs]
  347. def GetProcessorWorker(self):
  348. return TorqueLintWorker
  349. def GetProcessorScript(self):
  350. torque_tools = os.path.join(TOOLS_PATH, "torque")
  351. torque_path = os.path.join(torque_tools, "format-torque.py")
  352. arguments = ["-il"]
  353. if os.path.isfile(torque_path):
  354. return torque_path, arguments
  355. return None, arguments
  356. class JSLintProcessor(CacheableSourceFileProcessor):
  357. """
  358. Check .{m}js file to verify they follow the JS Style guide.
  359. """
  360. def __init__(self, use_cache=True):
  361. super(JSLintProcessor, self).__init__(
  362. use_cache=use_cache, cache_file_path='.jslint-cache',
  363. file_type='JavaScript')
  364. def IsRelevant(self, name):
  365. return name.endswith('.js') or name.endswith('.mjs')
  366. def GetPathsToSearch(self):
  367. return ['tools/system-analyzer', 'tools/heap-layout', 'tools/js']
  368. def GetProcessorWorker(self):
  369. return JSLintWorker
  370. def GetProcessorScript(self):
  371. jslint = os.path.join(DEPS_DEPOT_TOOLS_PATH, 'clang_format.py')
  372. return jslint, []
  373. COPYRIGHT_HEADER_PATTERN = re.compile(
  374. r'Copyright [\d-]*20[0-2][0-9] the V8 project authors. All rights reserved.')
  375. class SourceProcessor(SourceFileProcessor):
  376. """
  377. Check that all files include a copyright notice and no trailing whitespaces.
  378. """
  379. RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', '.status', '.tq', '.g4']
  380. def __init__(self):
  381. self.runtime_function_call_pattern = self.CreateRuntimeFunctionCallMatcher()
  382. def CreateRuntimeFunctionCallMatcher(self):
  383. runtime_h_path = join(dirname(TOOLS_PATH), 'src/runtime/runtime.h')
  384. pattern = re.compile(r'\s+F\(([^,]*),.*\)')
  385. runtime_functions = []
  386. with open(runtime_h_path) as f:
  387. for line in f.readlines():
  388. m = pattern.match(line)
  389. if m:
  390. runtime_functions.append(m.group(1))
  391. if len(runtime_functions) < 250:
  392. print ("Runtime functions list is suspiciously short. "
  393. "Consider updating the presubmit script.")
  394. sys.exit(1)
  395. str = '(\%\s+(' + '|'.join(runtime_functions) + '))[\s\(]'
  396. return re.compile(str)
  397. # Overwriting the one in the parent class.
  398. def FindFilesIn(self, path):
  399. if os.path.exists(path+'/.git'):
  400. output = subprocess.Popen('git ls-files --full-name',
  401. stdout=PIPE, cwd=path, shell=True)
  402. result = []
  403. for file in decode(output.stdout.read()).split():
  404. for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
  405. if self.IgnoreDir(dir_part):
  406. break
  407. else:
  408. if (self.IsRelevant(file) and os.path.exists(file)
  409. and not self.IgnoreFile(file)):
  410. result.append(join(path, file))
  411. if output.wait() == 0:
  412. return result
  413. return super(SourceProcessor, self).FindFilesIn(path)
  414. def IsRelevant(self, name):
  415. for ext in SourceProcessor.RELEVANT_EXTENSIONS:
  416. if name.endswith(ext):
  417. return True
  418. return False
  419. def GetPathsToSearch(self):
  420. return ['.']
  421. def IgnoreDir(self, name):
  422. return (super(SourceProcessor, self).IgnoreDir(name) or
  423. name in ('third_party', 'out', 'obj', 'DerivedSources'))
  424. IGNORE_COPYRIGHTS = ['box2d.js',
  425. 'cpplint.py',
  426. 'copy.js',
  427. 'corrections.js',
  428. 'crypto.js',
  429. 'daemon.py',
  430. 'earley-boyer.js',
  431. 'fannkuch.js',
  432. 'fasta.js',
  433. 'injected-script.cc',
  434. 'injected-script.h',
  435. 'libraries.cc',
  436. 'libraries-empty.cc',
  437. 'lua_binarytrees.js',
  438. 'meta-123.js',
  439. 'memops.js',
  440. 'poppler.js',
  441. 'primes.js',
  442. 'raytrace.js',
  443. 'regexp-pcre.js',
  444. 'resources-123.js',
  445. 'sqlite.js',
  446. 'sqlite-change-heap.js',
  447. 'sqlite-pointer-masking.js',
  448. 'sqlite-safe-heap.js',
  449. 'v8-debugger-script.h',
  450. 'v8-inspector-impl.cc',
  451. 'v8-inspector-impl.h',
  452. 'v8-runtime-agent-impl.cc',
  453. 'v8-runtime-agent-impl.h',
  454. 'gnuplot-4.6.3-emscripten.js',
  455. 'zlib.js']
  456. IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
  457. IGNORE_COPYRIGHTS_DIRECTORIES = [
  458. "test/test262/local-tests",
  459. "test/mjsunit/wasm/bulk-memory-spec",
  460. ]
  461. def EndOfDeclaration(self, line):
  462. return line == "}" or line == "};"
  463. def StartOfDeclaration(self, line):
  464. return line.find("//") == 0 or \
  465. line.find("/*") == 0 or \
  466. line.find(") {") != -1
  467. def ProcessContents(self, name, contents):
  468. result = True
  469. base = basename(name)
  470. if not base in SourceProcessor.IGNORE_TABS:
  471. if '\t' in contents:
  472. print("%s contains tabs" % name)
  473. result = False
  474. if not base in SourceProcessor.IGNORE_COPYRIGHTS and \
  475. not any(ignore_dir in name for ignore_dir
  476. in SourceProcessor.IGNORE_COPYRIGHTS_DIRECTORIES):
  477. if not COPYRIGHT_HEADER_PATTERN.search(contents):
  478. print("%s is missing a correct copyright header." % name)
  479. result = False
  480. if ' \n' in contents or contents.endswith(' '):
  481. line = 0
  482. lines = []
  483. parts = contents.split(' \n')
  484. if not contents.endswith(' '):
  485. parts.pop()
  486. for part in parts:
  487. line += part.count('\n') + 1
  488. lines.append(str(line))
  489. linenumbers = ', '.join(lines)
  490. if len(lines) > 1:
  491. print("%s has trailing whitespaces in lines %s." % (name, linenumbers))
  492. else:
  493. print("%s has trailing whitespaces in line %s." % (name, linenumbers))
  494. result = False
  495. if not contents.endswith('\n') or contents.endswith('\n\n'):
  496. print("%s does not end with a single new line." % name)
  497. result = False
  498. # Sanitize flags for fuzzer.
  499. if (".js" in name or ".mjs" in name) and ("mjsunit" in name or "debugger" in name):
  500. match = FLAGS_LINE.search(contents)
  501. if match:
  502. print("%s Flags should use '-' (not '_')" % name)
  503. result = False
  504. if (not "mjsunit/mjsunit.js" in name and
  505. not "mjsunit/mjsunit_numfuzz.js" in name):
  506. if ASSERT_OPTIMIZED_PATTERN.search(contents) and \
  507. not FLAGS_ENABLE_OPT.search(contents):
  508. print("%s Flag --opt should be set if " \
  509. "assertOptimized() is used" % name)
  510. result = False
  511. if ASSERT_UNOPTIMIZED_PATTERN.search(contents) and \
  512. not FLAGS_NO_ALWAYS_OPT.search(contents):
  513. print("%s Flag --no-always-opt should be set if " \
  514. "assertUnoptimized() is used" % name)
  515. result = False
  516. match = self.runtime_function_call_pattern.search(contents)
  517. if match:
  518. print("%s has unexpected spaces in a runtime call '%s'" % (name, match.group(1)))
  519. result = False
  520. return result
  521. def ProcessFiles(self, files):
  522. success = True
  523. violations = 0
  524. for file in files:
  525. try:
  526. handle = open(file, "rb")
  527. contents = decode(handle.read(), "ISO-8859-1")
  528. if len(contents) > 0 and not self.ProcessContents(file, contents):
  529. success = False
  530. violations += 1
  531. finally:
  532. handle.close()
  533. print("Total violating files: %s" % violations)
  534. return success
  535. def _CheckStatusFileForDuplicateKeys(filepath):
  536. comma_space_bracket = re.compile(", *]")
  537. lines = []
  538. with open(filepath) as f:
  539. for line in f.readlines():
  540. # Skip all-comment lines.
  541. if line.lstrip().startswith("#"): continue
  542. # Strip away comments at the end of the line.
  543. comment_start = line.find("#")
  544. if comment_start != -1:
  545. line = line[:comment_start]
  546. line = line.strip()
  547. # Strip away trailing commas within the line.
  548. line = comma_space_bracket.sub("]", line)
  549. if len(line) > 0:
  550. lines.append(line)
  551. # Strip away trailing commas at line ends. Ugh.
  552. for i in range(len(lines) - 1):
  553. if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
  554. lines[i + 1][0] in ("}", "]")):
  555. lines[i] = lines[i][:-1]
  556. contents = "\n".join(lines)
  557. # JSON wants double-quotes.
  558. contents = contents.replace("'", '"')
  559. # Fill in keywords (like PASS, SKIP).
  560. for key in statusfile.KEYWORDS:
  561. contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
  562. status = {"success": True}
  563. def check_pairs(pairs):
  564. keys = {}
  565. for key, value in pairs:
  566. if key in keys:
  567. print("%s: Error: duplicate key %s" % (filepath, key))
  568. status["success"] = False
  569. keys[key] = True
  570. json.loads(contents, object_pairs_hook=check_pairs)
  571. return status["success"]
  572. class StatusFilesProcessor(SourceFileProcessor):
  573. """Checks status files for incorrect syntax and duplicate keys."""
  574. def IsRelevant(self, name):
  575. # Several changes to files under the test directories could impact status
  576. # files.
  577. return True
  578. def GetPathsToSearch(self):
  579. return ['test', 'tools/testrunner']
  580. def ProcessFiles(self, files):
  581. success = True
  582. for status_file_path in sorted(self._GetStatusFiles(files)):
  583. success &= statusfile.PresubmitCheck(status_file_path)
  584. success &= _CheckStatusFileForDuplicateKeys(status_file_path)
  585. return success
  586. def _GetStatusFiles(self, files):
  587. test_path = join(dirname(TOOLS_PATH), 'test')
  588. testrunner_path = join(TOOLS_PATH, 'testrunner')
  589. status_files = set()
  590. for file_path in files:
  591. if file_path.startswith(testrunner_path):
  592. for suitepath in os.listdir(test_path):
  593. suitename = os.path.basename(suitepath)
  594. status_file = os.path.join(
  595. test_path, suitename, suitename + ".status")
  596. if os.path.exists(status_file):
  597. status_files.add(status_file)
  598. return status_files
  599. for file_path in files:
  600. if file_path.startswith(test_path):
  601. # Strip off absolute path prefix pointing to test suites.
  602. pieces = file_path[len(test_path):].lstrip(os.sep).split(os.sep)
  603. if pieces:
  604. # Infer affected status file name. Only care for existing status
  605. # files. Some directories under "test" don't have any.
  606. if not os.path.isdir(join(test_path, pieces[0])):
  607. continue
  608. status_file = join(test_path, pieces[0], pieces[0] + ".status")
  609. if not os.path.exists(status_file):
  610. continue
  611. status_files.add(status_file)
  612. return status_files
  613. def CheckDeps(workspace):
  614. checkdeps_py = join(workspace, 'buildtools', 'checkdeps', 'checkdeps.py')
  615. return subprocess.call([sys.executable, checkdeps_py, workspace]) == 0
  616. def FindTests(workspace):
  617. scripts = []
  618. # TODO(almuthanna): unskip valid tests when they are properly migrated
  619. exclude = [
  620. 'tools/clang',
  621. 'tools/mb/mb_test.py',
  622. 'tools/cppgc/gen_cmake_test.py',
  623. 'tools/ignition/linux_perf_report_test.py',
  624. 'tools/ignition/bytecode_dispatches_report_test.py',
  625. 'tools/ignition/linux_perf_bytecode_annotate_test.py',
  626. ]
  627. scripts_without_excluded = []
  628. for root, dirs, files in os.walk(join(workspace, 'tools')):
  629. for f in files:
  630. if f.endswith('_test.py'):
  631. fullpath = os.path.join(root, f)
  632. scripts.append(fullpath)
  633. for script in scripts:
  634. if not any(exc_dir in script for exc_dir in exclude):
  635. scripts_without_excluded.append(script)
  636. return scripts_without_excluded
  637. def PyTests(workspace):
  638. result = True
  639. for script in FindTests(workspace):
  640. print('Running ' + script)
  641. result &= subprocess.call(
  642. [sys.executable, script], stdout=subprocess.PIPE) == 0
  643. return result
  644. def GetOptions():
  645. result = optparse.OptionParser()
  646. result.add_option('--no-lint', help="Do not run cpplint", default=False,
  647. action="store_true")
  648. result.add_option('--no-linter-cache', help="Do not cache linter results",
  649. default=False, action="store_true")
  650. return result
  651. def Main():
  652. workspace = abspath(join(dirname(sys.argv[0]), '..'))
  653. parser = GetOptions()
  654. (options, args) = parser.parse_args()
  655. success = True
  656. print("Running checkdeps...")
  657. success &= CheckDeps(workspace)
  658. use_linter_cache = not options.no_linter_cache
  659. if not options.no_lint:
  660. print("Running C++ lint check...")
  661. success &= CppLintProcessor(use_cache=use_linter_cache).RunOnPath(workspace)
  662. print("Running Torque formatting check...")
  663. success &= TorqueLintProcessor(use_cache=use_linter_cache).RunOnPath(
  664. workspace)
  665. print("Running JavaScript formatting check...")
  666. success &= JSLintProcessor(use_cache=use_linter_cache).RunOnPath(
  667. workspace)
  668. print("Running copyright header, trailing whitespaces and " \
  669. "two empty lines between declarations check...")
  670. success &= SourceProcessor().RunOnPath(workspace)
  671. print("Running status-files check...")
  672. success &= StatusFilesProcessor().RunOnPath(workspace)
  673. print("Running python tests...")
  674. success &= PyTests(workspace)
  675. if success:
  676. return 0
  677. else:
  678. return 1
  679. if __name__ == '__main__':
  680. sys.exit(Main())