builddeps.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. #!/usr/bin/env python
  2. # Copyright 2013 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Traverses the source tree, parses all found DEPS files, and constructs
  6. a dependency rule table to be used by subclasses.
  7. See README.md for the format of the deps file.
  8. """
  9. import copy
  10. import os.path
  11. import posixpath
  12. import subprocess
  13. from rules import Rule, Rules
  14. # Variable name used in the DEPS file to add or subtract include files from
  15. # the module-level deps.
  16. INCLUDE_RULES_VAR_NAME = 'include_rules'
  17. # Variable name used in the DEPS file to add or subtract include files
  18. # from module-level deps specific to files whose basename (last
  19. # component of path) matches a given regular expression.
  20. SPECIFIC_INCLUDE_RULES_VAR_NAME = 'specific_include_rules'
  21. # Optionally present in the DEPS file to list subdirectories which should not
  22. # be checked. This allows us to skip third party code, for example.
  23. SKIP_SUBDIRS_VAR_NAME = 'skip_child_includes'
  24. # Optionally discard rules from parent directories, similar to "noparent" in
  25. # OWNERS files. For example, if //ash/components has "noparent = True" then
  26. # it will not inherit rules from //ash/DEPS, forcing each //ash/component/foo
  27. # to declare all its dependencies.
  28. NOPARENT_VAR_NAME = 'noparent'
  29. class DepsBuilderError(Exception):
  30. """Base class for exceptions in this module."""
  31. pass
  32. def NormalizePath(path):
  33. """Returns a path normalized to how we write DEPS rules and compare paths."""
  34. return os.path.normcase(path).replace(os.path.sep, posixpath.sep)
  35. def _GitSourceDirectories(base_directory):
  36. """Returns set of normalized paths to subdirectories containing sources
  37. managed by git."""
  38. base_dir_norm = NormalizePath(base_directory)
  39. git_source_directories = set([base_dir_norm])
  40. git_cmd = 'git.bat' if os.name == 'nt' else 'git'
  41. git_ls_files_cmd = [git_cmd, 'ls-files']
  42. # FIXME: Use a context manager in Python 3.2+
  43. popen = subprocess.Popen(git_ls_files_cmd,
  44. stdout=subprocess.PIPE,
  45. cwd=base_directory)
  46. try:
  47. try:
  48. for line in popen.stdout.read().decode('utf-8').splitlines():
  49. dir_path = os.path.join(base_directory, os.path.dirname(line))
  50. dir_path_norm = NormalizePath(dir_path)
  51. # Add the directory as well as all the parent directories,
  52. # stopping once we reach an already-listed directory.
  53. while dir_path_norm not in git_source_directories:
  54. git_source_directories.add(dir_path_norm)
  55. dir_path_norm = posixpath.dirname(dir_path_norm)
  56. finally:
  57. popen.stdout.close()
  58. finally:
  59. popen.wait()
  60. return git_source_directories
  61. class DepsBuilder(object):
  62. """Parses include_rules from DEPS files."""
  63. def __init__(self,
  64. base_directory=None,
  65. extra_repos=[],
  66. verbose=False,
  67. being_tested=False,
  68. ignore_temp_rules=False,
  69. ignore_specific_rules=False):
  70. """Creates a new DepsBuilder.
  71. Args:
  72. base_directory: local path to root of checkout, e.g. C:\chr\src.
  73. verbose: Set to True for debug output.
  74. being_tested: Set to True to ignore the DEPS file at
  75. buildtools/checkdeps/DEPS.
  76. ignore_temp_rules: Ignore rules that start with Rule.TEMP_ALLOW ("!").
  77. """
  78. base_directory = (base_directory or
  79. os.path.join(os.path.dirname(__file__),
  80. os.path.pardir, os.path.pardir))
  81. self.base_directory = os.path.abspath(base_directory) # Local absolute path
  82. self.extra_repos = extra_repos
  83. self.verbose = verbose
  84. self._under_test = being_tested
  85. self._ignore_temp_rules = ignore_temp_rules
  86. self._ignore_specific_rules = ignore_specific_rules
  87. self._git_source_directories = None
  88. if os.path.exists(os.path.join(base_directory, '.git')):
  89. self.is_git = True
  90. elif os.path.exists(os.path.join(base_directory, '.svn')):
  91. self.is_git = False
  92. else:
  93. raise DepsBuilderError("%s is not a repository root" % base_directory)
  94. # Map of normalized directory paths to rules to use for those
  95. # directories, or None for directories that should be skipped.
  96. # Normalized is: absolute, lowercase, / for separator.
  97. self.directory_rules = {}
  98. self._ApplyDirectoryRulesAndSkipSubdirs(Rules(), self.base_directory)
  99. def _ApplyRules(self, existing_rules, includes, specific_includes,
  100. cur_dir_norm):
  101. """Applies the given include rules, returning the new rules.
  102. Args:
  103. existing_rules: A set of existing rules that will be combined.
  104. include: The list of rules from the "include_rules" section of DEPS.
  105. specific_includes: E.g. {'.*_unittest\.cc': ['+foo', '-blat']} rules
  106. from the "specific_include_rules" section of DEPS.
  107. cur_dir_norm: The current directory, normalized path. We will create an
  108. implicit rule that allows inclusion from this directory.
  109. Returns: A new set of rules combining the existing_rules with the other
  110. arguments.
  111. """
  112. rules = copy.deepcopy(existing_rules)
  113. # First apply the implicit "allow" rule for the current directory.
  114. base_dir_norm = NormalizePath(self.base_directory)
  115. if not cur_dir_norm.startswith(base_dir_norm):
  116. raise Exception(
  117. 'Internal error: base directory is not at the beginning for\n'
  118. ' %s and base dir\n'
  119. ' %s' % (cur_dir_norm, base_dir_norm))
  120. relative_dir = posixpath.relpath(cur_dir_norm, base_dir_norm)
  121. # Make the help string a little more meaningful.
  122. source = relative_dir or 'top level'
  123. rules.AddRule('+' + relative_dir,
  124. relative_dir,
  125. 'Default rule for ' + source)
  126. def ApplyOneRule(rule_str, dependee_regexp=None):
  127. """Deduces a sensible description for the rule being added, and
  128. adds the rule with its description to |rules|.
  129. If we are ignoring temporary rules, this function does nothing
  130. for rules beginning with the Rule.TEMP_ALLOW character.
  131. """
  132. if self._ignore_temp_rules and rule_str.startswith(Rule.TEMP_ALLOW):
  133. return
  134. rule_block_name = 'include_rules'
  135. if dependee_regexp:
  136. rule_block_name = 'specific_include_rules'
  137. if relative_dir:
  138. rule_description = relative_dir + "'s %s" % rule_block_name
  139. else:
  140. rule_description = 'the top level %s' % rule_block_name
  141. rules.AddRule(rule_str, relative_dir, rule_description, dependee_regexp)
  142. # Apply the additional explicit rules.
  143. for rule_str in includes:
  144. ApplyOneRule(rule_str)
  145. # Finally, apply the specific rules.
  146. if self._ignore_specific_rules:
  147. return rules
  148. for regexp, specific_rules in specific_includes.items():
  149. for rule_str in specific_rules:
  150. ApplyOneRule(rule_str, regexp)
  151. return rules
  152. def _ApplyDirectoryRules(self, existing_rules, dir_path_local_abs):
  153. """Combines rules from the existing rules and the new directory.
  154. Any directory can contain a DEPS file. Top-level DEPS files can contain
  155. module dependencies which are used by gclient. We use these, along with
  156. additional include rules and implicit rules for the given directory, to
  157. come up with a combined set of rules to apply for the directory.
  158. Args:
  159. existing_rules: The rules for the parent directory. We'll add-on to these.
  160. dir_path_local_abs: The directory path that the DEPS file may live in (if
  161. it exists). This will also be used to generate the
  162. implicit rules. This is a local path.
  163. Returns: A 2-tuple of:
  164. (1) the combined set of rules to apply to the sub-tree,
  165. (2) a list of all subdirectories that should NOT be checked, as specified
  166. in the DEPS file (if any).
  167. Subdirectories are single words, hence no OS dependence.
  168. """
  169. dir_path_norm = NormalizePath(dir_path_local_abs)
  170. # Check the DEPS file in this directory.
  171. if self.verbose:
  172. print('Applying rules from', dir_path_local_abs)
  173. def FromImpl(*_):
  174. pass # NOP function so "From" doesn't fail.
  175. def FileImpl(_):
  176. pass # NOP function so "File" doesn't fail.
  177. class _VarImpl:
  178. def __init__(self, local_scope):
  179. self._local_scope = local_scope
  180. def Lookup(self, var_name):
  181. """Implements the Var syntax."""
  182. try:
  183. return self._local_scope['vars'][var_name]
  184. except KeyError:
  185. raise Exception('Var is not defined: %s' % var_name)
  186. local_scope = {}
  187. global_scope = {
  188. 'File': FileImpl,
  189. 'From': FromImpl,
  190. 'Var': _VarImpl(local_scope).Lookup,
  191. 'Str': str,
  192. }
  193. deps_file_path = os.path.join(dir_path_local_abs, 'DEPS')
  194. # The second conditional here is to disregard the
  195. # buildtools/checkdeps/DEPS file while running tests. This DEPS file
  196. # has a skip_child_includes for 'testdata' which is necessary for
  197. # running production tests, since there are intentional DEPS
  198. # violations under the testdata directory. On the other hand when
  199. # running tests, we absolutely need to verify the contents of that
  200. # directory to trigger those intended violations and see that they
  201. # are handled correctly.
  202. if os.path.isfile(deps_file_path) and not (
  203. self._under_test and
  204. os.path.basename(dir_path_local_abs) == 'checkdeps'):
  205. try:
  206. with open(deps_file_path) as file:
  207. exec(file.read(), global_scope, local_scope)
  208. except Exception as e:
  209. print(' Error reading %s: %s' % (deps_file_path, str(e)))
  210. raise
  211. elif self.verbose:
  212. print(' No deps file found in', dir_path_local_abs)
  213. # Even if a DEPS file does not exist we still invoke ApplyRules
  214. # to apply the implicit "allow" rule for the current directory
  215. include_rules = local_scope.get(INCLUDE_RULES_VAR_NAME, [])
  216. specific_include_rules = local_scope.get(SPECIFIC_INCLUDE_RULES_VAR_NAME,
  217. {})
  218. skip_subdirs = local_scope.get(SKIP_SUBDIRS_VAR_NAME, [])
  219. noparent = local_scope.get(NOPARENT_VAR_NAME, False)
  220. if noparent:
  221. parent_rules = Rules()
  222. else:
  223. parent_rules = existing_rules
  224. return (self._ApplyRules(parent_rules, include_rules,
  225. specific_include_rules, dir_path_norm),
  226. skip_subdirs)
  227. def _ApplyDirectoryRulesAndSkipSubdirs(self, parent_rules,
  228. dir_path_local_abs):
  229. """Given |parent_rules| and a subdirectory |dir_path_local_abs| of the
  230. directory that owns the |parent_rules|, add |dir_path_local_abs|'s rules to
  231. |self.directory_rules|, and add None entries for any of its
  232. subdirectories that should be skipped.
  233. """
  234. directory_rules, excluded_subdirs = self._ApplyDirectoryRules(
  235. parent_rules, dir_path_local_abs)
  236. dir_path_norm = NormalizePath(dir_path_local_abs)
  237. self.directory_rules[dir_path_norm] = directory_rules
  238. for subdir in excluded_subdirs:
  239. subdir_path_norm = posixpath.join(dir_path_norm, subdir)
  240. self.directory_rules[subdir_path_norm] = None
  241. def GetAllRulesAndFiles(self, dir_name=None):
  242. """Yields (rules, filenames) for each repository directory with DEPS rules.
  243. This walks the directory tree while staying in the repository. Specify
  244. |dir_name| to walk just one directory and its children; omit |dir_name| to
  245. walk the entire repository.
  246. Yields:
  247. Two-element (rules, filenames) tuples. |rules| is a rules.Rules object
  248. for a directory, and |filenames| is a list of the absolute local paths
  249. of all files in that directory.
  250. """
  251. if self.is_git and self._git_source_directories is None:
  252. self._git_source_directories = _GitSourceDirectories(self.base_directory)
  253. for repo in self.extra_repos:
  254. repo_path = os.path.join(self.base_directory, repo)
  255. self._git_source_directories.update(_GitSourceDirectories(repo_path))
  256. # Collect a list of all files and directories to check.
  257. if dir_name and not os.path.isabs(dir_name):
  258. dir_name = os.path.join(self.base_directory, dir_name)
  259. dirs_to_check = [dir_name or self.base_directory]
  260. while dirs_to_check:
  261. current_dir = dirs_to_check.pop()
  262. # Check that this directory is part of the source repository. This
  263. # prevents us from descending into third-party code or directories
  264. # generated by the build system.
  265. if self.is_git:
  266. if NormalizePath(current_dir) not in self._git_source_directories:
  267. continue
  268. elif not os.path.exists(os.path.join(current_dir, '.svn')):
  269. continue
  270. current_dir_rules = self.GetDirectoryRules(current_dir)
  271. if not current_dir_rules:
  272. continue # Handle the 'skip_child_includes' case.
  273. current_dir_contents = sorted(os.listdir(current_dir))
  274. file_names = []
  275. sub_dirs = []
  276. for file_name in current_dir_contents:
  277. full_name = os.path.join(current_dir, file_name)
  278. if os.path.isdir(full_name):
  279. sub_dirs.append(full_name)
  280. else:
  281. file_names.append(full_name)
  282. dirs_to_check.extend(reversed(sub_dirs))
  283. yield (current_dir_rules, file_names)
  284. def GetDirectoryRules(self, dir_path_local):
  285. """Returns a Rules object to use for the given directory, or None
  286. if the given directory should be skipped.
  287. Also modifies |self.directory_rules| to store the Rules.
  288. This takes care of first building rules for parent directories (up to
  289. |self.base_directory|) if needed, which may add rules for skipped
  290. subdirectories.
  291. Args:
  292. dir_path_local: A local path to the directory you want rules for.
  293. Can be relative and unnormalized. It is the caller's responsibility
  294. to ensure that this is part of the repository rooted at
  295. |self.base_directory|.
  296. """
  297. if os.path.isabs(dir_path_local):
  298. dir_path_local_abs = dir_path_local
  299. else:
  300. dir_path_local_abs = os.path.join(self.base_directory, dir_path_local)
  301. dir_path_norm = NormalizePath(dir_path_local_abs)
  302. if dir_path_norm in self.directory_rules:
  303. return self.directory_rules[dir_path_norm]
  304. parent_dir_local_abs = os.path.dirname(dir_path_local_abs)
  305. parent_rules = self.GetDirectoryRules(parent_dir_local_abs)
  306. # We need to check for an entry for our dir_path again, since
  307. # GetDirectoryRules can modify entries for subdirectories, namely setting
  308. # to None if they should be skipped, via _ApplyDirectoryRulesAndSkipSubdirs.
  309. # For example, if dir_path == 'A/B/C' and A/B/DEPS specifies that the C
  310. # subdirectory be skipped, GetDirectoryRules('A/B') will fill in the entry
  311. # for 'A/B/C' as None.
  312. if dir_path_norm in self.directory_rules:
  313. return self.directory_rules[dir_path_norm]
  314. if parent_rules:
  315. self._ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path_local_abs)
  316. else:
  317. # If the parent directory should be skipped, then the current
  318. # directory should also be skipped.
  319. self.directory_rules[dir_path_norm] = None
  320. return self.directory_rules[dir_path_norm]