123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310 |
- #!/usr/bin/env python
- # Copyright 2017 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Find header files missing in GN.
- This script gets all the header files from ninja_deps, which is from the true
- dependency generated by the compiler, and report if they don't exist in GN.
- """
- from __future__ import print_function
- import argparse
- import json
- import os
- import re
- import shutil
- import subprocess
- import sys
- import tempfile
- from multiprocessing import Process, Queue
- SRC_DIR = os.path.abspath(
- os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
- DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
- def GetHeadersFromNinja(out_dir, skip_obj, q):
- """Return all the header files from ninja_deps"""
- def NinjaSource():
- cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps']
- # A negative bufsize means to use the system default, which usually
- # means fully buffered.
- popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
- for line in iter(popen.stdout.readline, ''):
- yield line.rstrip()
- popen.stdout.close()
- return_code = popen.wait()
- if return_code:
- raise subprocess.CalledProcessError(return_code, cmd)
- ans, err = set(), None
- try:
- ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
- except Exception as e:
- err = str(e)
- q.put((ans, err))
- def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
- """Parse ninja output and get the header files"""
- all_headers = {}
- # Ninja always uses "/", even on Windows.
- prefix = '../../'
- is_valid = False
- obj_file = ''
- for line in ninja_out:
- if line.startswith(' '):
- if not is_valid:
- continue
- if line.endswith('.h') or line.endswith('.hh'):
- f = line.strip()
- if f.startswith(prefix):
- f = f[6:] # Remove the '../../' prefix
- # build/ only contains build-specific files like build_config.h
- # and buildflag.h, and system header files, so they should be
- # skipped.
- if f.startswith(out_dir) or f.startswith('out'):
- continue
- if not f.startswith('build'):
- all_headers.setdefault(f, [])
- if not skip_obj:
- all_headers[f].append(obj_file)
- else:
- is_valid = line.endswith('(VALID)')
- obj_file = line.split(':')[0]
- return all_headers
- def GetHeadersFromGN(out_dir, q):
- """Return all the header files from GN"""
- tmp = None
- ans, err = set(), None
- try:
- # Argument |dir| is needed to make sure it's on the same drive on Windows.
- # dir='' means dir='.', but doesn't introduce an unneeded prefix.
- tmp = tempfile.mkdtemp(dir='')
- shutil.copy2(os.path.join(out_dir, 'args.gn'),
- os.path.join(tmp, 'args.gn'))
- # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
- gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
- subprocess.check_call([
- os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
- gn_json = json.load(open(os.path.join(tmp, 'project.json')))
- ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
- except Exception as e:
- err = str(e)
- finally:
- if tmp:
- shutil.rmtree(tmp)
- q.put((ans, err))
- def ParseGNProjectJSON(gn, out_dir, tmp_out):
- """Parse GN output and get the header files"""
- all_headers = set()
- for _target, properties in gn['targets'].iteritems():
- sources = properties.get('sources', [])
- public = properties.get('public', [])
- # Exclude '"public": "*"'.
- if type(public) is list:
- sources += public
- for f in sources:
- if f.endswith('.h') or f.endswith('.hh'):
- if f.startswith('//'):
- f = f[2:] # Strip the '//' prefix.
- if f.startswith(tmp_out):
- f = out_dir + f[len(tmp_out):]
- all_headers.add(f)
- return all_headers
- def GetDepsPrefixes(q):
- """Return all the folders controlled by DEPS file"""
- prefixes, err = set(), None
- try:
- gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
- gclient_out = subprocess.check_output([
- os.path.join(DEPOT_TOOLS_DIR, gclient_exe),
- 'recurse', '--no-progress', '-j1',
- 'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'],
- universal_newlines=True)
- for i in gclient_out.split('\n'):
- if i.startswith('src/'):
- i = i[4:]
- prefixes.add(i)
- except Exception as e:
- err = str(e)
- q.put((prefixes, err))
- def IsBuildClean(out_dir):
- cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-n']
- try:
- out = subprocess.check_output(cmd)
- return 'no work to do.' in out
- except Exception as e:
- print(e)
- return False
- def ParseWhiteList(whitelist):
- out = set()
- for line in whitelist.split('\n'):
- line = re.sub(r'#.*', '', line).strip()
- if line:
- out.add(line)
- return out
- def FilterOutDepsedRepo(files, deps):
- return {f for f in files if not any(f.startswith(d) for d in deps)}
- def GetNonExistingFiles(lst):
- out = set()
- for f in lst:
- if not os.path.isfile(f):
- out.add(f)
- return out
- def main():
- def DumpJson(data):
- if args.json:
- with open(args.json, 'w') as f:
- json.dump(data, f)
- def PrintError(msg):
- DumpJson([])
- parser.error(msg)
- parser = argparse.ArgumentParser(description='''
- NOTE: Use ninja to build all targets in OUT_DIR before running
- this script.''')
- parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
- help='output directory of the build')
- parser.add_argument('--json',
- help='JSON output filename for missing headers')
- parser.add_argument('--whitelist', help='file containing whitelist')
- parser.add_argument('--skip-dirty-check', action='store_true',
- help='skip checking whether the build is dirty')
- parser.add_argument('--verbose', action='store_true',
- help='print more diagnostic info')
- args, _extras = parser.parse_known_args()
- if not os.path.isdir(args.out_dir):
- parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
- if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
- dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
- if args.json:
- # Assume running on the bots. Silently skip this step.
- # This is possible because "analyze" step can be wrong due to
- # underspecified header files. See crbug.com/725877
- print(dirty_msg)
- DumpJson([])
- return 0
- else:
- # Assume running interactively.
- parser.error(dirty_msg)
- d_q = Queue()
- d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
- d_p.start()
- gn_q = Queue()
- gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
- gn_p.start()
- deps_q = Queue()
- deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
- deps_p.start()
- d, d_err = d_q.get()
- gn, gn_err = gn_q.get()
- missing = set(d.keys()) - gn
- nonexisting = GetNonExistingFiles(gn)
- deps, deps_err = deps_q.get()
- missing = FilterOutDepsedRepo(missing, deps)
- nonexisting = FilterOutDepsedRepo(nonexisting, deps)
- d_p.join()
- gn_p.join()
- deps_p.join()
- if d_err:
- PrintError(d_err)
- if gn_err:
- PrintError(gn_err)
- if deps_err:
- PrintError(deps_err)
- if len(GetNonExistingFiles(d)) > 0:
- print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
- PrintError('Found non-existing files in ninja deps. You should ' +
- 'build all in OUT_DIR.')
- if len(d) == 0:
- PrintError('OUT_DIR looks empty. You should build all there.')
- if any((('/gen/' in i) for i in nonexisting)):
- PrintError('OUT_DIR looks wrong. You should build all there.')
- if args.whitelist:
- whitelist = ParseWhiteList(open(args.whitelist).read())
- missing -= whitelist
- nonexisting -= whitelist
- missing = sorted(missing)
- nonexisting = sorted(nonexisting)
- DumpJson(sorted(missing + nonexisting))
- if len(missing) == 0 and len(nonexisting) == 0:
- return 0
- if len(missing) > 0:
- print('\nThe following files should be included in gn files:')
- for i in missing:
- print(i)
- if len(nonexisting) > 0:
- print('\nThe following non-existing files should be removed from gn files:')
- for i in nonexisting:
- print(i)
- if args.verbose:
- # Only get detailed obj dependency here since it is slower.
- GetHeadersFromNinja(args.out_dir, False, d_q)
- d, d_err = d_q.get()
- print('\nDetailed dependency info:')
- for f in missing:
- print(f)
- for cc in d[f]:
- print(' ', cc)
- print('\nMissing headers sorted by number of affected object files:')
- count = {k: len(v) for (k, v) in d.iteritems()}
- for f in sorted(count, key=count.get, reverse=True):
- if f in missing:
- print(count[f], f)
- if args.json:
- # Assume running on the bots. Temporarily return 0 before
- # https://crbug.com/937847 is fixed.
- return 0
- return 1
- if __name__ == '__main__':
- sys.exit(main())
|