pythondeps 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250
  1. #!/usr/bin/env python3
  2. #
  3. # Determine dependencies of python scripts or available python modules in a search path.
  4. #
  5. # Given the -d argument and a filename/filenames, returns the modules imported by those files.
  6. # Given the -d argument and a directory/directories, recurses to find all
  7. # python packages and modules, returns the modules imported by these.
  8. # Given the -p argument and a path or paths, scans that path for available python modules/packages.
  9. import argparse
  10. import ast
  11. import imp
  12. import logging
  13. import os.path
  14. import sys
  15. logger = logging.getLogger('pythondeps')
  16. suffixes = []
  17. for triple in imp.get_suffixes():
  18. suffixes.append(triple[0])
  19. class PythonDepError(Exception):
  20. pass
  21. class DependError(PythonDepError):
  22. def __init__(self, path, error):
  23. self.path = path
  24. self.error = error
  25. PythonDepError.__init__(self, error)
  26. def __str__(self):
  27. return "Failure determining dependencies of {}: {}".format(self.path, self.error)
  28. class ImportVisitor(ast.NodeVisitor):
  29. def __init__(self):
  30. self.imports = set()
  31. self.importsfrom = []
  32. def visit_Import(self, node):
  33. for alias in node.names:
  34. self.imports.add(alias.name)
  35. def visit_ImportFrom(self, node):
  36. self.importsfrom.append((node.module, [a.name for a in node.names], node.level))
  37. def walk_up(path):
  38. while path:
  39. yield path
  40. path, _, _ = path.rpartition(os.sep)
  41. def get_provides(path):
  42. path = os.path.realpath(path)
  43. def get_fn_name(fn):
  44. for suffix in suffixes:
  45. if fn.endswith(suffix):
  46. return fn[:-len(suffix)]
  47. isdir = os.path.isdir(path)
  48. if isdir:
  49. pkg_path = path
  50. walk_path = path
  51. else:
  52. pkg_path = get_fn_name(path)
  53. if pkg_path is None:
  54. return
  55. walk_path = os.path.dirname(path)
  56. for curpath in walk_up(walk_path):
  57. if not os.path.exists(os.path.join(curpath, '__init__.py')):
  58. libdir = curpath
  59. break
  60. else:
  61. libdir = ''
  62. package_relpath = pkg_path[len(libdir)+1:]
  63. package = '.'.join(package_relpath.split(os.sep))
  64. if not isdir:
  65. yield package, path
  66. else:
  67. if os.path.exists(os.path.join(path, '__init__.py')):
  68. yield package, path
  69. for dirpath, dirnames, filenames in os.walk(path):
  70. relpath = dirpath[len(path)+1:]
  71. if relpath:
  72. if '__init__.py' not in filenames:
  73. dirnames[:] = []
  74. continue
  75. else:
  76. context = '.'.join(relpath.split(os.sep))
  77. if package:
  78. context = package + '.' + context
  79. yield context, dirpath
  80. else:
  81. context = package
  82. for fn in filenames:
  83. adjusted_fn = get_fn_name(fn)
  84. if not adjusted_fn or adjusted_fn == '__init__':
  85. continue
  86. fullfn = os.path.join(dirpath, fn)
  87. if context:
  88. yield context + '.' + adjusted_fn, fullfn
  89. else:
  90. yield adjusted_fn, fullfn
  91. def get_code_depends(code_string, path=None, provide=None, ispkg=False):
  92. try:
  93. code = ast.parse(code_string, path)
  94. except TypeError as exc:
  95. raise DependError(path, exc)
  96. except SyntaxError as exc:
  97. raise DependError(path, exc)
  98. visitor = ImportVisitor()
  99. visitor.visit(code)
  100. for builtin_module in sys.builtin_module_names:
  101. if builtin_module in visitor.imports:
  102. visitor.imports.remove(builtin_module)
  103. if provide:
  104. provide_elements = provide.split('.')
  105. if ispkg:
  106. provide_elements.append("__self__")
  107. context = '.'.join(provide_elements[:-1])
  108. package_path = os.path.dirname(path)
  109. else:
  110. context = None
  111. package_path = None
  112. levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
  113. if level == 0)
  114. for module in visitor.imports | set(levelzero_importsfrom):
  115. if context and path:
  116. module_basepath = os.path.join(package_path, module.replace('.', '/'))
  117. if os.path.exists(module_basepath):
  118. # Implicit relative import
  119. yield context + '.' + module, path
  120. continue
  121. for suffix in suffixes:
  122. if os.path.exists(module_basepath + suffix):
  123. # Implicit relative import
  124. yield context + '.' + module, path
  125. break
  126. else:
  127. yield module, path
  128. else:
  129. yield module, path
  130. for module, names, level in visitor.importsfrom:
  131. if level == 0:
  132. continue
  133. elif not provide:
  134. raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
  135. elif level > len(provide_elements):
  136. raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
  137. else:
  138. context = '.'.join(provide_elements[:-level])
  139. if module:
  140. if context:
  141. yield context + '.' + module, path
  142. else:
  143. yield module, path
  144. def get_file_depends(path):
  145. try:
  146. code_string = open(path, 'r').read()
  147. except (OSError, IOError) as exc:
  148. raise DependError(path, exc)
  149. return get_code_depends(code_string, path)
  150. def get_depends_recursive(directory):
  151. directory = os.path.realpath(directory)
  152. provides = dict((v, k) for k, v in get_provides(directory))
  153. for filename, provide in provides.items():
  154. if os.path.isdir(filename):
  155. filename = os.path.join(filename, '__init__.py')
  156. ispkg = True
  157. elif not filename.endswith('.py'):
  158. continue
  159. else:
  160. ispkg = False
  161. with open(filename, 'r') as f:
  162. source = f.read()
  163. depends = get_code_depends(source, filename, provide, ispkg)
  164. for depend, by in depends:
  165. yield depend, by
  166. def get_depends(path):
  167. if os.path.isdir(path):
  168. return get_depends_recursive(path)
  169. else:
  170. return get_file_depends(path)
  171. def main():
  172. logging.basicConfig()
  173. parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
  174. parser.add_argument('path', nargs='+', help='full path to content to be processed')
  175. group = parser.add_mutually_exclusive_group()
  176. group.add_argument('-p', '--provides', action='store_true',
  177. help='given a path, display the provided python modules')
  178. group.add_argument('-d', '--depends', action='store_true',
  179. help='given a filename, display the imported python modules')
  180. args = parser.parse_args()
  181. if args.provides:
  182. modules = set()
  183. for path in args.path:
  184. for provide, fn in get_provides(path):
  185. modules.add(provide)
  186. for module in sorted(modules):
  187. print(module)
  188. elif args.depends:
  189. for path in args.path:
  190. try:
  191. modules = get_depends(path)
  192. except PythonDepError as exc:
  193. logger.error(str(exc))
  194. sys.exit(1)
  195. for module, imp_by in modules:
  196. print("{}\t{}".format(module, imp_by))
  197. else:
  198. parser.print_help()
  199. sys.exit(2)
  200. if __name__ == '__main__':
  201. main()