pythondeps 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. #!/usr/bin/env python3
  2. #
  3. # Copyright OpenEmbedded Contributors
  4. #
  5. # SPDX-License-Identifier: GPL-2.0-only
  6. #
  7. # Determine dependencies of python scripts or available python modules in a search path.
  8. #
  9. # Given the -d argument and a filename/filenames, returns the modules imported by those files.
  10. # Given the -d argument and a directory/directories, recurses to find all
  11. # python packages and modules, returns the modules imported by these.
  12. # Given the -p argument and a path or paths, scans that path for available python modules/packages.
  13. import argparse
  14. import ast
  15. import importlib
  16. from importlib import machinery
  17. import logging
  18. import os.path
  19. import sys
  20. logger = logging.getLogger('pythondeps')
  21. suffixes = importlib.machinery.all_suffixes()
  22. class PythonDepError(Exception):
  23. pass
  24. class DependError(PythonDepError):
  25. def __init__(self, path, error):
  26. self.path = path
  27. self.error = error
  28. PythonDepError.__init__(self, error)
  29. def __str__(self):
  30. return "Failure determining dependencies of {}: {}".format(self.path, self.error)
  31. class ImportVisitor(ast.NodeVisitor):
  32. def __init__(self):
  33. self.imports = set()
  34. self.importsfrom = []
  35. def visit_Import(self, node):
  36. for alias in node.names:
  37. self.imports.add(alias.name)
  38. def visit_ImportFrom(self, node):
  39. self.importsfrom.append((node.module, [a.name for a in node.names], node.level))
  40. def walk_up(path):
  41. while path:
  42. yield path
  43. path, _, _ = path.rpartition(os.sep)
  44. def get_provides(path):
  45. path = os.path.realpath(path)
  46. def get_fn_name(fn):
  47. for suffix in suffixes:
  48. if fn.endswith(suffix):
  49. return fn[:-len(suffix)]
  50. isdir = os.path.isdir(path)
  51. if isdir:
  52. pkg_path = path
  53. walk_path = path
  54. else:
  55. pkg_path = get_fn_name(path)
  56. if pkg_path is None:
  57. return
  58. walk_path = os.path.dirname(path)
  59. for curpath in walk_up(walk_path):
  60. if not os.path.exists(os.path.join(curpath, '__init__.py')):
  61. libdir = curpath
  62. break
  63. else:
  64. libdir = ''
  65. package_relpath = pkg_path[len(libdir)+1:]
  66. package = '.'.join(package_relpath.split(os.sep))
  67. if not isdir:
  68. yield package, path
  69. else:
  70. if os.path.exists(os.path.join(path, '__init__.py')):
  71. yield package, path
  72. for dirpath, dirnames, filenames in os.walk(path):
  73. relpath = dirpath[len(path)+1:]
  74. if relpath:
  75. if '__init__.py' not in filenames:
  76. dirnames[:] = []
  77. continue
  78. else:
  79. context = '.'.join(relpath.split(os.sep))
  80. if package:
  81. context = package + '.' + context
  82. yield context, dirpath
  83. else:
  84. context = package
  85. for fn in filenames:
  86. adjusted_fn = get_fn_name(fn)
  87. if not adjusted_fn or adjusted_fn == '__init__':
  88. continue
  89. fullfn = os.path.join(dirpath, fn)
  90. if context:
  91. yield context + '.' + adjusted_fn, fullfn
  92. else:
  93. yield adjusted_fn, fullfn
  94. def get_code_depends(code_string, path=None, provide=None, ispkg=False):
  95. try:
  96. code = ast.parse(code_string, path)
  97. except TypeError as exc:
  98. raise DependError(path, exc)
  99. except SyntaxError as exc:
  100. raise DependError(path, exc)
  101. visitor = ImportVisitor()
  102. visitor.visit(code)
  103. for builtin_module in sys.builtin_module_names:
  104. if builtin_module in visitor.imports:
  105. visitor.imports.remove(builtin_module)
  106. if provide:
  107. provide_elements = provide.split('.')
  108. if ispkg:
  109. provide_elements.append("__self__")
  110. context = '.'.join(provide_elements[:-1])
  111. package_path = os.path.dirname(path)
  112. else:
  113. context = None
  114. package_path = None
  115. levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
  116. if level == 0)
  117. for module in visitor.imports | set(levelzero_importsfrom):
  118. if context and path:
  119. module_basepath = os.path.join(package_path, module.replace('.', '/'))
  120. if os.path.exists(module_basepath):
  121. # Implicit relative import
  122. yield context + '.' + module, path
  123. continue
  124. for suffix in suffixes:
  125. if os.path.exists(module_basepath + suffix):
  126. # Implicit relative import
  127. yield context + '.' + module, path
  128. break
  129. else:
  130. yield module, path
  131. else:
  132. yield module, path
  133. for module, names, level in visitor.importsfrom:
  134. if level == 0:
  135. continue
  136. elif not provide:
  137. raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
  138. elif level > len(provide_elements):
  139. raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
  140. else:
  141. context = '.'.join(provide_elements[:-level])
  142. if module:
  143. if context:
  144. yield context + '.' + module, path
  145. else:
  146. yield module, path
  147. def get_file_depends(path):
  148. try:
  149. code_string = open(path, 'r').read()
  150. except (OSError, IOError) as exc:
  151. raise DependError(path, exc)
  152. return get_code_depends(code_string, path)
  153. def get_depends_recursive(directory):
  154. directory = os.path.realpath(directory)
  155. provides = dict((v, k) for k, v in get_provides(directory))
  156. for filename, provide in provides.items():
  157. if os.path.isdir(filename):
  158. filename = os.path.join(filename, '__init__.py')
  159. ispkg = True
  160. elif not filename.endswith('.py'):
  161. continue
  162. else:
  163. ispkg = False
  164. with open(filename, 'r') as f:
  165. source = f.read()
  166. depends = get_code_depends(source, filename, provide, ispkg)
  167. for depend, by in depends:
  168. yield depend, by
  169. def get_depends(path):
  170. if os.path.isdir(path):
  171. return get_depends_recursive(path)
  172. else:
  173. return get_file_depends(path)
  174. def main():
  175. logging.basicConfig()
  176. parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
  177. parser.add_argument('path', nargs='+', help='full path to content to be processed')
  178. group = parser.add_mutually_exclusive_group()
  179. group.add_argument('-p', '--provides', action='store_true',
  180. help='given a path, display the provided python modules')
  181. group.add_argument('-d', '--depends', action='store_true',
  182. help='given a filename, display the imported python modules')
  183. args = parser.parse_args()
  184. if args.provides:
  185. modules = set()
  186. for path in args.path:
  187. for provide, fn in get_provides(path):
  188. modules.add(provide)
  189. for module in sorted(modules):
  190. print(module)
  191. elif args.depends:
  192. for path in args.path:
  193. try:
  194. modules = get_depends(path)
  195. except PythonDepError as exc:
  196. logger.error(str(exc))
  197. sys.exit(1)
  198. for module, imp_by in modules:
  199. print("{}\t{}".format(module, imp_by))
  200. else:
  201. parser.print_help()
  202. sys.exit(2)
  203. if __name__ == '__main__':
  204. main()