print_python_deps.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186
  1. #!/usr/bin/env vpython3
  2. # Copyright 2016 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Prints all non-system dependencies for the given module.
  6. The primary use-case for this script is to generate the list of python modules
  7. required for .isolate files.
  8. """
  9. import argparse
  10. import os
  11. import pipes
  12. import sys
  13. # Don't use any helper modules, or else they will end up in the results.
  14. _SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
  15. def ComputePythonDependencies():
  16. """Gets the paths of imported non-system python modules.
  17. A path is assumed to be a "system" import if it is outside of chromium's
  18. src/. The paths will be relative to the current directory.
  19. """
  20. module_paths = (m.__file__ for m in sys.modules.values()
  21. if m and hasattr(m, '__file__') and m.__file__)
  22. src_paths = set()
  23. for path in module_paths:
  24. if path == __file__:
  25. continue
  26. path = os.path.abspath(path)
  27. if not path.startswith(_SRC_ROOT):
  28. continue
  29. if (path.endswith('.pyc')
  30. or (path.endswith('c') and not os.path.splitext(path)[1])):
  31. path = path[:-1]
  32. src_paths.add(path)
  33. return src_paths
  34. def quote(string):
  35. if string.count(' ') > 0:
  36. return '"%s"' % string
  37. else:
  38. return string
  39. def _NormalizeCommandLine(options):
  40. """Returns a string that when run from SRC_ROOT replicates the command."""
  41. args = ['build/print_python_deps.py']
  42. root = os.path.relpath(options.root, _SRC_ROOT)
  43. if root != '.':
  44. args.extend(('--root', root))
  45. if options.output:
  46. args.extend(('--output', os.path.relpath(options.output, _SRC_ROOT)))
  47. if options.gn_paths:
  48. args.extend(('--gn-paths',))
  49. for allowlist in sorted(options.allowlists):
  50. args.extend(('--allowlist', os.path.relpath(allowlist, _SRC_ROOT)))
  51. args.append(os.path.relpath(options.module, _SRC_ROOT))
  52. if os.name == 'nt':
  53. return ' '.join(quote(x) for x in args).replace('\\', '/')
  54. else:
  55. return ' '.join(pipes.quote(x) for x in args)
  56. def _FindPythonInDirectory(directory, allow_test):
  57. """Returns an iterable of all non-test python files in the given directory."""
  58. for root, _dirnames, filenames in os.walk(directory):
  59. for filename in filenames:
  60. if filename.endswith('.py') and (allow_test
  61. or not filename.endswith('_test.py')):
  62. yield os.path.join(root, filename)
  63. def _ImportModuleByPath(module_path):
  64. """Imports a module by its source file."""
  65. # Replace the path entry for print_python_deps.py with the one for the given
  66. # module.
  67. sys.path[0] = os.path.dirname(module_path)
  68. # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
  69. module_name = os.path.splitext(os.path.basename(module_path))[0]
  70. import importlib.util # Python 3 only, since it's unavailable in Python 2.
  71. spec = importlib.util.spec_from_file_location(module_name, module_path)
  72. module = importlib.util.module_from_spec(spec)
  73. sys.modules[module_name] = module
  74. spec.loader.exec_module(module)
  75. def main():
  76. parser = argparse.ArgumentParser(
  77. description='Prints all non-system dependencies for the given module.')
  78. parser.add_argument('module',
  79. help='The python module to analyze.')
  80. parser.add_argument('--root', default='.',
  81. help='Directory to make paths relative to.')
  82. parser.add_argument('--output',
  83. help='Write output to a file rather than stdout.')
  84. parser.add_argument('--inplace', action='store_true',
  85. help='Write output to a file with the same path as the '
  86. 'module, but with a .pydeps extension. Also sets the '
  87. 'root to the module\'s directory.')
  88. parser.add_argument('--no-header', action='store_true',
  89. help='Do not write the "# Generated by" header.')
  90. parser.add_argument('--gn-paths', action='store_true',
  91. help='Write paths as //foo/bar/baz.py')
  92. parser.add_argument('--did-relaunch', action='store_true',
  93. help=argparse.SUPPRESS)
  94. parser.add_argument('--allowlist',
  95. default=[],
  96. action='append',
  97. dest='allowlists',
  98. help='Recursively include all non-test python files '
  99. 'within this directory. May be specified multiple times.')
  100. options = parser.parse_args()
  101. if options.inplace:
  102. if options.output:
  103. parser.error('Cannot use --inplace and --output at the same time!')
  104. if not options.module.endswith('.py'):
  105. parser.error('Input module path should end with .py suffix!')
  106. options.output = options.module + 'deps'
  107. options.root = os.path.dirname(options.module)
  108. modules = [options.module]
  109. if os.path.isdir(options.module):
  110. modules = list(_FindPythonInDirectory(options.module, allow_test=True))
  111. if not modules:
  112. parser.error('Input directory does not contain any python files!')
  113. is_vpython = 'vpython' in sys.executable
  114. if not is_vpython:
  115. # Prevent infinite relaunch if something goes awry.
  116. assert not options.did_relaunch
  117. # Re-launch using vpython will cause us to pick up modules specified in
  118. # //.vpython, but does not cause it to pick up modules defined inline via
  119. # [VPYTHON:BEGIN] ... [VPYTHON:END] comments.
  120. # TODO(agrieve): Add support for this if the need ever arises.
  121. os.execvp('vpython3', ['vpython3'] + sys.argv + ['--did-relaunch'])
  122. # Work-around for protobuf library not being loadable via importlib
  123. # This is needed due to compile_resources.py.
  124. import importlib._bootstrap_external
  125. importlib._bootstrap_external._NamespacePath.sort = lambda self, **_: 0
  126. paths_set = set()
  127. try:
  128. for module in modules:
  129. _ImportModuleByPath(module)
  130. paths_set.update(ComputePythonDependencies())
  131. except Exception:
  132. # Output extra diagnostics when loading the script fails.
  133. sys.stderr.write('Error running print_python_deps.py.\n')
  134. sys.stderr.write('is_vpython={}\n'.format(is_vpython))
  135. sys.stderr.write('did_relanuch={}\n'.format(options.did_relaunch))
  136. sys.stderr.write('python={}\n'.format(sys.executable))
  137. raise
  138. for path in options.allowlists:
  139. paths_set.update(
  140. os.path.abspath(p)
  141. for p in _FindPythonInDirectory(path, allow_test=False))
  142. paths = [os.path.relpath(p, options.root) for p in paths_set]
  143. normalized_cmdline = _NormalizeCommandLine(options)
  144. out = open(options.output, 'w', newline='') if options.output else sys.stdout
  145. with out:
  146. if not options.no_header:
  147. out.write('# Generated by running:\n')
  148. out.write('# %s\n' % normalized_cmdline)
  149. prefix = '//' if options.gn_paths else ''
  150. for path in sorted(paths):
  151. out.write(prefix + path.replace('\\', '/') + '\n')
  152. if __name__ == '__main__':
  153. sys.exit(main())