check_gn_headers.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310
  1. #!/usr/bin/env python
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Find header files missing in GN.
  6. This script gets all the header files from ninja_deps, which is from the true
  7. dependency generated by the compiler, and report if they don't exist in GN.
  8. """
  9. from __future__ import print_function
  10. import argparse
  11. import json
  12. import os
  13. import re
  14. import shutil
  15. import subprocess
  16. import sys
  17. import tempfile
  18. from multiprocessing import Process, Queue
  19. SRC_DIR = os.path.abspath(
  20. os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
  21. DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
  22. def GetHeadersFromNinja(out_dir, skip_obj, q):
  23. """Return all the header files from ninja_deps"""
  24. def NinjaSource():
  25. cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps']
  26. # A negative bufsize means to use the system default, which usually
  27. # means fully buffered.
  28. popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
  29. for line in iter(popen.stdout.readline, ''):
  30. yield line.rstrip()
  31. popen.stdout.close()
  32. return_code = popen.wait()
  33. if return_code:
  34. raise subprocess.CalledProcessError(return_code, cmd)
  35. ans, err = set(), None
  36. try:
  37. ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
  38. except Exception as e:
  39. err = str(e)
  40. q.put((ans, err))
  41. def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
  42. """Parse ninja output and get the header files"""
  43. all_headers = {}
  44. # Ninja always uses "/", even on Windows.
  45. prefix = '../../'
  46. is_valid = False
  47. obj_file = ''
  48. for line in ninja_out:
  49. if line.startswith(' '):
  50. if not is_valid:
  51. continue
  52. if line.endswith('.h') or line.endswith('.hh'):
  53. f = line.strip()
  54. if f.startswith(prefix):
  55. f = f[6:] # Remove the '../../' prefix
  56. # build/ only contains build-specific files like build_config.h
  57. # and buildflag.h, and system header files, so they should be
  58. # skipped.
  59. if f.startswith(out_dir) or f.startswith('out'):
  60. continue
  61. if not f.startswith('build'):
  62. all_headers.setdefault(f, [])
  63. if not skip_obj:
  64. all_headers[f].append(obj_file)
  65. else:
  66. is_valid = line.endswith('(VALID)')
  67. obj_file = line.split(':')[0]
  68. return all_headers
  69. def GetHeadersFromGN(out_dir, q):
  70. """Return all the header files from GN"""
  71. tmp = None
  72. ans, err = set(), None
  73. try:
  74. # Argument |dir| is needed to make sure it's on the same drive on Windows.
  75. # dir='' means dir='.', but doesn't introduce an unneeded prefix.
  76. tmp = tempfile.mkdtemp(dir='')
  77. shutil.copy2(os.path.join(out_dir, 'args.gn'),
  78. os.path.join(tmp, 'args.gn'))
  79. # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
  80. gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
  81. subprocess.check_call([
  82. os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
  83. gn_json = json.load(open(os.path.join(tmp, 'project.json')))
  84. ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
  85. except Exception as e:
  86. err = str(e)
  87. finally:
  88. if tmp:
  89. shutil.rmtree(tmp)
  90. q.put((ans, err))
  91. def ParseGNProjectJSON(gn, out_dir, tmp_out):
  92. """Parse GN output and get the header files"""
  93. all_headers = set()
  94. for _target, properties in gn['targets'].iteritems():
  95. sources = properties.get('sources', [])
  96. public = properties.get('public', [])
  97. # Exclude '"public": "*"'.
  98. if type(public) is list:
  99. sources += public
  100. for f in sources:
  101. if f.endswith('.h') or f.endswith('.hh'):
  102. if f.startswith('//'):
  103. f = f[2:] # Strip the '//' prefix.
  104. if f.startswith(tmp_out):
  105. f = out_dir + f[len(tmp_out):]
  106. all_headers.add(f)
  107. return all_headers
  108. def GetDepsPrefixes(q):
  109. """Return all the folders controlled by DEPS file"""
  110. prefixes, err = set(), None
  111. try:
  112. gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
  113. gclient_out = subprocess.check_output([
  114. os.path.join(DEPOT_TOOLS_DIR, gclient_exe),
  115. 'recurse', '--no-progress', '-j1',
  116. 'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'],
  117. universal_newlines=True)
  118. for i in gclient_out.split('\n'):
  119. if i.startswith('src/'):
  120. i = i[4:]
  121. prefixes.add(i)
  122. except Exception as e:
  123. err = str(e)
  124. q.put((prefixes, err))
  125. def IsBuildClean(out_dir):
  126. cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-n']
  127. try:
  128. out = subprocess.check_output(cmd)
  129. return 'no work to do.' in out
  130. except Exception as e:
  131. print(e)
  132. return False
  133. def ParseWhiteList(whitelist):
  134. out = set()
  135. for line in whitelist.split('\n'):
  136. line = re.sub(r'#.*', '', line).strip()
  137. if line:
  138. out.add(line)
  139. return out
  140. def FilterOutDepsedRepo(files, deps):
  141. return {f for f in files if not any(f.startswith(d) for d in deps)}
  142. def GetNonExistingFiles(lst):
  143. out = set()
  144. for f in lst:
  145. if not os.path.isfile(f):
  146. out.add(f)
  147. return out
  148. def main():
  149. def DumpJson(data):
  150. if args.json:
  151. with open(args.json, 'w') as f:
  152. json.dump(data, f)
  153. def PrintError(msg):
  154. DumpJson([])
  155. parser.error(msg)
  156. parser = argparse.ArgumentParser(description='''
  157. NOTE: Use ninja to build all targets in OUT_DIR before running
  158. this script.''')
  159. parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
  160. help='output directory of the build')
  161. parser.add_argument('--json',
  162. help='JSON output filename for missing headers')
  163. parser.add_argument('--whitelist', help='file containing whitelist')
  164. parser.add_argument('--skip-dirty-check', action='store_true',
  165. help='skip checking whether the build is dirty')
  166. parser.add_argument('--verbose', action='store_true',
  167. help='print more diagnostic info')
  168. args, _extras = parser.parse_known_args()
  169. if not os.path.isdir(args.out_dir):
  170. parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
  171. if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
  172. dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
  173. if args.json:
  174. # Assume running on the bots. Silently skip this step.
  175. # This is possible because "analyze" step can be wrong due to
  176. # underspecified header files. See crbug.com/725877
  177. print(dirty_msg)
  178. DumpJson([])
  179. return 0
  180. else:
  181. # Assume running interactively.
  182. parser.error(dirty_msg)
  183. d_q = Queue()
  184. d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
  185. d_p.start()
  186. gn_q = Queue()
  187. gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
  188. gn_p.start()
  189. deps_q = Queue()
  190. deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
  191. deps_p.start()
  192. d, d_err = d_q.get()
  193. gn, gn_err = gn_q.get()
  194. missing = set(d.keys()) - gn
  195. nonexisting = GetNonExistingFiles(gn)
  196. deps, deps_err = deps_q.get()
  197. missing = FilterOutDepsedRepo(missing, deps)
  198. nonexisting = FilterOutDepsedRepo(nonexisting, deps)
  199. d_p.join()
  200. gn_p.join()
  201. deps_p.join()
  202. if d_err:
  203. PrintError(d_err)
  204. if gn_err:
  205. PrintError(gn_err)
  206. if deps_err:
  207. PrintError(deps_err)
  208. if len(GetNonExistingFiles(d)) > 0:
  209. print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
  210. PrintError('Found non-existing files in ninja deps. You should ' +
  211. 'build all in OUT_DIR.')
  212. if len(d) == 0:
  213. PrintError('OUT_DIR looks empty. You should build all there.')
  214. if any((('/gen/' in i) for i in nonexisting)):
  215. PrintError('OUT_DIR looks wrong. You should build all there.')
  216. if args.whitelist:
  217. whitelist = ParseWhiteList(open(args.whitelist).read())
  218. missing -= whitelist
  219. nonexisting -= whitelist
  220. missing = sorted(missing)
  221. nonexisting = sorted(nonexisting)
  222. DumpJson(sorted(missing + nonexisting))
  223. if len(missing) == 0 and len(nonexisting) == 0:
  224. return 0
  225. if len(missing) > 0:
  226. print('\nThe following files should be included in gn files:')
  227. for i in missing:
  228. print(i)
  229. if len(nonexisting) > 0:
  230. print('\nThe following non-existing files should be removed from gn files:')
  231. for i in nonexisting:
  232. print(i)
  233. if args.verbose:
  234. # Only get detailed obj dependency here since it is slower.
  235. GetHeadersFromNinja(args.out_dir, False, d_q)
  236. d, d_err = d_q.get()
  237. print('\nDetailed dependency info:')
  238. for f in missing:
  239. print(f)
  240. for cc in d[f]:
  241. print(' ', cc)
  242. print('\nMissing headers sorted by number of affected object files:')
  243. count = {k: len(v) for (k, v) in d.iteritems()}
  244. for f in sorted(count, key=count.get, reverse=True):
  245. if f in missing:
  246. print(count[f], f)
  247. if args.json:
  248. # Assume running on the bots. Temporarily return 0 before
  249. # https://crbug.com/937847 is fixed.
  250. return 0
  251. return 1
  252. if __name__ == '__main__':
  253. sys.exit(main())