run-clang-tidy.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2018 the V8 project authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. # for py2/py3 compatibility
  7. from __future__ import print_function
  8. import json
  9. import multiprocessing
  10. import optparse
  11. import os
  12. import re
  13. import subprocess
  14. import sys
  15. CLANG_TIDY_WARNING = re.compile(r'(\/.*?)\ .*\[(.*)\]$')
  16. CLANG_TIDY_CMDLINE_OUT = re.compile(r'^clang-tidy.*\ .*|^\./\.\*')
  17. FILE_REGEXS = ['../src/*', '../test/*']
  18. HEADER_REGEX = ['\.\.\/src\/.*|\.\.\/include\/.*|\.\.\/test\/.*']
  19. THREADS = multiprocessing.cpu_count()
  20. class ClangTidyWarning(object):
  21. """
  22. Wraps up a clang-tidy warning to present aggregated information.
  23. """
  24. def __init__(self, warning_type):
  25. self.warning_type = warning_type
  26. self.occurrences = set()
  27. def add_occurrence(self, file_path):
  28. self.occurrences.add(file_path.lstrip())
  29. def __hash__(self):
  30. return hash(self.warning_type)
  31. def to_string(self, file_loc):
  32. s = '[%s] #%d\n' % (self.warning_type, len(self.occurrences))
  33. if file_loc:
  34. s += ' ' + '\n '.join(self.occurrences)
  35. s += '\n'
  36. return s
  37. def __str__(self):
  38. return self.to_string(False)
  39. def __lt__(self, other):
  40. return len(self.occurrences) < len(other.occurrences)
  41. def GenerateCompileCommands(build_folder):
  42. """
  43. Generate a compilation database.
  44. Currently clang-tidy-4 does not understand all flags that are passed
  45. by the build system, therefore, we remove them from the generated file.
  46. """
  47. ninja_ps = subprocess.Popen(
  48. ['ninja', '-t', 'compdb', 'cxx', 'cc'],
  49. stdout=subprocess.PIPE,
  50. cwd=build_folder)
  51. out_filepath = os.path.join(build_folder, 'compile_commands.json')
  52. with open(out_filepath, 'w') as cc_file:
  53. while True:
  54. line = ninja_ps.stdout.readline()
  55. if line == '':
  56. break
  57. line = line.replace('-fcomplete-member-pointers', '')
  58. line = line.replace('-Wno-enum-compare-switch', '')
  59. line = line.replace('-Wno-ignored-pragma-optimize', '')
  60. line = line.replace('-Wno-null-pointer-arithmetic', '')
  61. line = line.replace('-Wno-unused-lambda-capture', '')
  62. line = line.replace('-Wno-defaulted-function-deleted', '')
  63. cc_file.write(line)
  64. def skip_line(line):
  65. """
  66. Check if a clang-tidy output line should be skipped.
  67. """
  68. return bool(CLANG_TIDY_CMDLINE_OUT.search(line))
  69. def ClangTidyRunFull(build_folder, skip_output_filter, checks, auto_fix):
  70. """
  71. Run clang-tidy on the full codebase and print warnings.
  72. """
  73. extra_args = []
  74. if auto_fix:
  75. extra_args.append('-fix')
  76. if checks is not None:
  77. extra_args.append('-checks')
  78. extra_args.append('-*, ' + checks)
  79. with open(os.devnull, 'w') as DEVNULL:
  80. ct_process = subprocess.Popen(
  81. ['run-clang-tidy', '-j' + str(THREADS), '-p', '.']
  82. + ['-header-filter'] + HEADER_REGEX + extra_args
  83. + FILE_REGEXS,
  84. cwd=build_folder,
  85. stdout=subprocess.PIPE,
  86. stderr=DEVNULL)
  87. removing_check_header = False
  88. empty_lines = 0
  89. while True:
  90. line = ct_process.stdout.readline()
  91. if line == '':
  92. break
  93. # Skip all lines after Enbale checks and before two newlines,
  94. # i.e., skip clang-tidy check list.
  95. if line.startswith('Enabled checks'):
  96. removing_check_header = True
  97. if removing_check_header and not skip_output_filter:
  98. if line == '\n':
  99. empty_lines += 1
  100. if empty_lines == 2:
  101. removing_check_header = False
  102. continue
  103. # Different lines get removed to ease output reading.
  104. if not skip_output_filter and skip_line(line):
  105. continue
  106. # Print line, because no filter was matched.
  107. if line != '\n':
  108. sys.stdout.write(line)
  109. def ClangTidyRunAggregate(build_folder, print_files):
  110. """
  111. Run clang-tidy on the full codebase and aggregate warnings into categories.
  112. """
  113. with open(os.devnull, 'w') as DEVNULL:
  114. ct_process = subprocess.Popen(
  115. ['run-clang-tidy', '-j' + str(THREADS), '-p', '.'] +
  116. ['-header-filter'] + HEADER_REGEX +
  117. FILE_REGEXS,
  118. cwd=build_folder,
  119. stdout=subprocess.PIPE,
  120. stderr=DEVNULL)
  121. warnings = dict()
  122. while True:
  123. line = ct_process.stdout.readline()
  124. if line == '':
  125. break
  126. res = CLANG_TIDY_WARNING.search(line)
  127. if res is not None:
  128. warnings.setdefault(
  129. res.group(2),
  130. ClangTidyWarning(res.group(2))).add_occurrence(res.group(1))
  131. for warning in sorted(warnings.values(), reverse=True):
  132. sys.stdout.write(warning.to_string(print_files))
  133. def ClangTidyRunDiff(build_folder, diff_branch, auto_fix):
  134. """
  135. Run clang-tidy on the diff between current and the diff_branch.
  136. """
  137. if diff_branch is None:
  138. diff_branch = subprocess.check_output(['git', 'merge-base',
  139. 'HEAD', 'origin/master']).strip()
  140. git_ps = subprocess.Popen(
  141. ['git', 'diff', '-U0', diff_branch], stdout=subprocess.PIPE)
  142. extra_args = []
  143. if auto_fix:
  144. extra_args.append('-fix')
  145. with open(os.devnull, 'w') as DEVNULL:
  146. """
  147. The script `clang-tidy-diff` does not provide support to add header-
  148. filters. To still analyze headers we use the build path option `-path` to
  149. inject our header-filter option. This works because the script just adds
  150. the passed path string to the commandline of clang-tidy.
  151. """
  152. modified_build_folder = build_folder
  153. modified_build_folder += ' -header-filter='
  154. modified_build_folder += '\'' + ''.join(HEADER_REGEX) + '\''
  155. ct_ps = subprocess.Popen(
  156. ['clang-tidy-diff.py', '-path', modified_build_folder, '-p1'] +
  157. extra_args,
  158. stdin=git_ps.stdout,
  159. stdout=subprocess.PIPE,
  160. stderr=DEVNULL)
  161. git_ps.wait()
  162. while True:
  163. line = ct_ps.stdout.readline()
  164. if line == '':
  165. break
  166. if skip_line(line):
  167. continue
  168. sys.stdout.write(line)
  169. def rm_prefix(string, prefix):
  170. """
  171. Removes prefix from a string until the new string
  172. no longer starts with the prefix.
  173. """
  174. while string.startswith(prefix):
  175. string = string[len(prefix):]
  176. return string
  177. def ClangTidyRunSingleFile(build_folder, filename_to_check, auto_fix,
  178. line_ranges=[]):
  179. """
  180. Run clang-tidy on a single file.
  181. """
  182. files_with_relative_path = []
  183. compdb_filepath = os.path.join(build_folder, 'compile_commands.json')
  184. with open(compdb_filepath) as raw_json_file:
  185. compdb = json.load(raw_json_file)
  186. for db_entry in compdb:
  187. if db_entry['file'].endswith(filename_to_check):
  188. files_with_relative_path.append(db_entry['file'])
  189. with open(os.devnull, 'w') as DEVNULL:
  190. for file_with_relative_path in files_with_relative_path:
  191. line_filter = None
  192. if len(line_ranges) != 0:
  193. line_filter = '['
  194. line_filter += '{ \"lines\":[' + ', '.join(line_ranges)
  195. line_filter += '], \"name\":\"'
  196. line_filter += rm_prefix(file_with_relative_path,
  197. '../') + '\"}'
  198. line_filter += ']'
  199. extra_args = ['-line-filter=' + line_filter] if line_filter else []
  200. if auto_fix:
  201. extra_args.append('-fix')
  202. subprocess.call(['clang-tidy', '-p', '.'] +
  203. extra_args +
  204. [file_with_relative_path],
  205. cwd=build_folder,
  206. stderr=DEVNULL)
  207. def CheckClangTidy():
  208. """
  209. Checks if a clang-tidy binary exists.
  210. """
  211. with open(os.devnull, 'w') as DEVNULL:
  212. return subprocess.call(['which', 'clang-tidy'], stdout=DEVNULL) == 0
  213. def CheckCompDB(build_folder):
  214. """
  215. Checks if a compilation database exists in the build_folder.
  216. """
  217. return os.path.isfile(os.path.join(build_folder, 'compile_commands.json'))
  218. def DetectBuildFolder():
  219. """
  220. Tries to auto detect the last used build folder in out/
  221. """
  222. outdirs_folder = 'out/'
  223. last_used = None
  224. last_timestamp = -1
  225. for outdir in [outdirs_folder + folder_name
  226. for folder_name in os.listdir(outdirs_folder)
  227. if os.path.isdir(outdirs_folder + folder_name)]:
  228. outdir_modified_timestamp = os.path.getmtime(outdir)
  229. if outdir_modified_timestamp > last_timestamp:
  230. last_timestamp = outdir_modified_timestamp
  231. last_used = outdir
  232. return last_used
  233. def GetOptions():
  234. """
  235. Generate the option parser for this script.
  236. """
  237. result = optparse.OptionParser()
  238. result.add_option(
  239. '-b',
  240. '--build-folder',
  241. help='Set V8 build folder',
  242. dest='build_folder',
  243. default=None)
  244. result.add_option(
  245. '-j',
  246. help='Set the amount of threads that should be used',
  247. dest='threads',
  248. default=None)
  249. result.add_option(
  250. '--gen-compdb',
  251. help='Generate a compilation database for clang-tidy',
  252. default=False,
  253. action='store_true')
  254. result.add_option(
  255. '--no-output-filter',
  256. help='Done use any output filterning',
  257. default=False,
  258. action='store_true')
  259. result.add_option(
  260. '--fix',
  261. help='Fix auto fixable issues',
  262. default=False,
  263. dest='auto_fix',
  264. action='store_true'
  265. )
  266. # Full clang-tidy.
  267. full_run_g = optparse.OptionGroup(result, 'Clang-tidy full', '')
  268. full_run_g.add_option(
  269. '--full',
  270. help='Run clang-tidy on the whole codebase',
  271. default=False,
  272. action='store_true')
  273. full_run_g.add_option('--checks',
  274. help='Clang-tidy checks to use.',
  275. default=None)
  276. result.add_option_group(full_run_g)
  277. # Aggregate clang-tidy.
  278. agg_run_g = optparse.OptionGroup(result, 'Clang-tidy aggregate', '')
  279. agg_run_g.add_option('--aggregate', help='Run clang-tidy on the whole '\
  280. 'codebase and aggregate the warnings',
  281. default=False, action='store_true')
  282. agg_run_g.add_option('--show-loc', help='Show file locations when running '\
  283. 'in aggregate mode', default=False,
  284. action='store_true')
  285. result.add_option_group(agg_run_g)
  286. # Diff clang-tidy.
  287. diff_run_g = optparse.OptionGroup(result, 'Clang-tidy diff', '')
  288. diff_run_g.add_option('--branch', help='Run clang-tidy on the diff '\
  289. 'between HEAD and the merge-base between HEAD '\
  290. 'and DIFF_BRANCH (origin/master by default).',
  291. default=None, dest='diff_branch')
  292. result.add_option_group(diff_run_g)
  293. # Single clang-tidy.
  294. single_run_g = optparse.OptionGroup(result, 'Clang-tidy single', '')
  295. single_run_g.add_option(
  296. '--single', help='', default=False, action='store_true')
  297. single_run_g.add_option(
  298. '--file', help='File name to check', default=None, dest='file_name')
  299. single_run_g.add_option('--lines', help='Limit checks to a line range. '\
  300. 'For example: --lines="[2,4], [5,6]"',
  301. default=[], dest='line_ranges')
  302. result.add_option_group(single_run_g)
  303. return result
  304. def main():
  305. parser = GetOptions()
  306. (options, _) = parser.parse_args()
  307. if options.threads is not None:
  308. global THREADS
  309. THREADS = options.threads
  310. if options.build_folder is None:
  311. options.build_folder = DetectBuildFolder()
  312. if not CheckClangTidy():
  313. print('Could not find clang-tidy')
  314. elif options.build_folder is None or not os.path.isdir(options.build_folder):
  315. print('Please provide a build folder with -b')
  316. elif options.gen_compdb:
  317. GenerateCompileCommands(options.build_folder)
  318. elif not CheckCompDB(options.build_folder):
  319. print('Could not find compilation database, ' \
  320. 'please generate it with --gen-compdb')
  321. else:
  322. print('Using build folder:', options.build_folder)
  323. if options.full:
  324. print('Running clang-tidy - full')
  325. ClangTidyRunFull(options.build_folder,
  326. options.no_output_filter,
  327. options.checks,
  328. options.auto_fix)
  329. elif options.aggregate:
  330. print('Running clang-tidy - aggregating warnings')
  331. if options.auto_fix:
  332. print('Auto fix not working in aggregate mode, running without.')
  333. ClangTidyRunAggregate(options.build_folder, options.show_loc)
  334. elif options.single:
  335. print('Running clang-tidy - single on ' + options.file_name)
  336. if options.file_name is not None:
  337. line_ranges = []
  338. for match in re.findall(r'(\[.*?\])', options.line_ranges):
  339. if match is not []:
  340. line_ranges.append(match)
  341. ClangTidyRunSingleFile(options.build_folder,
  342. options.file_name,
  343. options.auto_fix,
  344. line_ranges)
  345. else:
  346. print('Filename provided, please specify a filename with --file')
  347. else:
  348. print('Running clang-tidy')
  349. ClangTidyRunDiff(options.build_folder,
  350. options.diff_branch,
  351. options.auto_fix)
  352. if __name__ == '__main__':
  353. main()