getdeveloperlib.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. from __future__ import print_function
  2. import os
  3. import re
  4. import glob
  5. import subprocess
  6. import sys
  7. import unittest
  8. #
  9. # Patch parsing functions
  10. #
  11. FIND_INFRA_IN_PATCH = re.compile(r"^\+\$\(eval \$\((host-)?([^-]*)-package\)\)$")
  12. def analyze_patch(patch):
  13. """Parse one patch and return the list of files modified, added or
  14. removed by the patch."""
  15. files = set()
  16. infras = set()
  17. for line in patch:
  18. # If the patch is adding a package, find which infra it is
  19. m = FIND_INFRA_IN_PATCH.match(line)
  20. if m:
  21. infras.add(m.group(2))
  22. if not line.startswith("+++ "):
  23. continue
  24. line.strip()
  25. fname = line[line.find("/") + 1:].strip()
  26. if fname == "dev/null":
  27. continue
  28. files.add(fname)
  29. return (files, infras)
  30. FIND_INFRA_IN_MK = re.compile(r"^\$\(eval \$\((host-)?([^-]*)-package\)\)$")
  31. def fname_get_package_infra(fname):
  32. """Checks whether the file name passed as argument is a Buildroot .mk
  33. file describing a package, and find the infrastructure it's using."""
  34. if not fname.endswith(".mk"):
  35. return None
  36. if not os.path.exists(fname):
  37. return None
  38. with open(fname, "r") as f:
  39. for line in f:
  40. line = line.strip()
  41. m = FIND_INFRA_IN_MK.match(line)
  42. if m:
  43. return m.group(2)
  44. return None
  45. def get_infras(files):
  46. """Search in the list of files for .mk files, and collect the package
  47. infrastructures used by those .mk files."""
  48. infras = set()
  49. for fname in files:
  50. infra = fname_get_package_infra(fname)
  51. if infra:
  52. infras.add(infra)
  53. return infras
  54. def analyze_patches(patches):
  55. """Parse a list of patches and returns the list of files modified,
  56. added or removed by the patches, as well as the list of package
  57. infrastructures used by those patches (if any)"""
  58. allfiles = set()
  59. allinfras = set()
  60. for patch in patches:
  61. (files, infras) = analyze_patch(patch)
  62. allfiles = allfiles | files
  63. allinfras = allinfras | infras
  64. allinfras = allinfras | get_infras(allfiles)
  65. return (allfiles, allinfras)
  66. #
  67. # Unit-test parsing functions
  68. #
  69. def get_all_test_cases(suite):
  70. """Generate all test-cases from a given test-suite.
  71. :return: (test.module, test.name)"""
  72. if issubclass(type(suite), unittest.TestSuite):
  73. for test in suite:
  74. for res in get_all_test_cases(test):
  75. yield res
  76. else:
  77. yield (suite.__module__, suite.__class__.__name__)
  78. def list_unittests(path):
  79. """Use the unittest module to retreive all test cases from a given
  80. directory"""
  81. loader = unittest.TestLoader()
  82. suite = loader.discover(path)
  83. tests = {}
  84. for module, test in get_all_test_cases(suite):
  85. module_path = os.path.join(path, *module.split('.'))
  86. tests.setdefault(module_path, []).append('%s.%s' % (module, test))
  87. return tests
  88. unittests = {}
  89. #
  90. # DEVELOPERS file parsing functions
  91. #
  92. class Developer:
  93. def __init__(self, name, files):
  94. self.name = name
  95. self.files = files
  96. self.packages = parse_developer_packages(files)
  97. self.architectures = parse_developer_architectures(files)
  98. self.infras = parse_developer_infras(files)
  99. self.runtime_tests = parse_developer_runtime_tests(files)
  100. self.defconfigs = parse_developer_defconfigs(files)
  101. def hasfile(self, f):
  102. f = os.path.abspath(f)
  103. for fs in self.files:
  104. fs = os.path.abspath(fs)
  105. if f.startswith(fs):
  106. return True
  107. return False
  108. def __repr__(self):
  109. name = '\'' + self.name.split(' <')[0][:20] + '\''
  110. things = []
  111. if len(self.files):
  112. things.append('{} files'.format(len(self.files)))
  113. if len(self.packages):
  114. things.append('{} pkgs'.format(len(self.packages)))
  115. if len(self.architectures):
  116. things.append('{} archs'.format(len(self.architectures)))
  117. if len(self.infras):
  118. things.append('{} infras'.format(len(self.infras)))
  119. if len(self.runtime_tests):
  120. things.append('{} tests'.format(len(self.runtime_tests)))
  121. if len(self.defconfigs):
  122. things.append('{} defconfigs'.format(len(self.defconfigs)))
  123. if things:
  124. return 'Developer <{} ({})>'.format(name, ', '.join(things))
  125. else:
  126. return 'Developer <' + name + '>'
  127. def parse_developer_packages(fnames):
  128. """Given a list of file patterns, travel through the Buildroot source
  129. tree to find which packages are implemented by those file
  130. patterns, and return a list of those packages."""
  131. packages = set()
  132. for fname in fnames:
  133. for root, dirs, files in os.walk(fname):
  134. for f in files:
  135. path = os.path.join(root, f)
  136. if fname_get_package_infra(path):
  137. pkg = os.path.splitext(f)[0]
  138. packages.add(pkg)
  139. return packages
  140. def parse_arches_from_config_in(fname):
  141. """Given a path to an arch/Config.in.* file, parse it to get the list
  142. of BR2_ARCH values for this architecture."""
  143. arches = set()
  144. with open(fname, "r") as f:
  145. parsing_arches = False
  146. for line in f:
  147. line = line.strip()
  148. if line == "config BR2_ARCH":
  149. parsing_arches = True
  150. continue
  151. if parsing_arches:
  152. m = re.match(r"^\s*default \"([^\"]*)\".*", line)
  153. if m:
  154. arches.add(m.group(1))
  155. else:
  156. parsing_arches = False
  157. return arches
  158. def parse_developer_architectures(fnames):
  159. """Given a list of file names, find the ones starting by
  160. 'arch/Config.in.', and use that to determine the architecture a
  161. developer is working on."""
  162. arches = set()
  163. for fname in fnames:
  164. if not re.match(r"^.*/arch/Config\.in\..*$", fname):
  165. continue
  166. arches = arches | parse_arches_from_config_in(fname)
  167. return arches
  168. def parse_developer_infras(fnames):
  169. infras = set()
  170. for fname in fnames:
  171. m = re.match(r"^package/pkg-([^.]*).mk$", fname)
  172. if m:
  173. infras.add(m.group(1))
  174. return infras
  175. def parse_developer_defconfigs(fnames):
  176. """Given a list of file names, returns the config names
  177. corresponding to defconfigs."""
  178. return {os.path.basename(fname[:-10])
  179. for fname in fnames
  180. if fname.endswith('_defconfig')}
  181. def parse_developer_runtime_tests(fnames):
  182. """Given a list of file names, returns the runtime tests
  183. corresponding to the file."""
  184. all_files = []
  185. # List all files recursively
  186. for fname in fnames:
  187. if os.path.isdir(fname):
  188. for root, _dirs, files in os.walk(fname):
  189. all_files += [os.path.join(root, f) for f in files]
  190. else:
  191. all_files.append(fname)
  192. # Get all runtime tests
  193. runtimes = set()
  194. for f in all_files:
  195. name = os.path.splitext(f)[0]
  196. if name in unittests:
  197. runtimes |= set(unittests[name])
  198. return runtimes
  199. def parse_developers(basepath=None):
  200. """Parse the DEVELOPERS file and return a list of Developer objects."""
  201. developers = []
  202. linen = 0
  203. if basepath is None:
  204. basepath = os.getcwd()
  205. global unittests
  206. unittests = list_unittests(os.path.join(basepath, 'support/testing'))
  207. with open(os.path.join(basepath, "DEVELOPERS"), "r") as f:
  208. files = []
  209. name = None
  210. for line in f:
  211. line = line.strip()
  212. if line.startswith("#"):
  213. continue
  214. elif line.startswith("N:"):
  215. if name is not None or len(files) != 0:
  216. print("Syntax error in DEVELOPERS file, line %d" % linen,
  217. file=sys.stderr)
  218. name = line[2:].strip()
  219. elif line.startswith("F:"):
  220. fname = line[2:].strip()
  221. dev_files = glob.glob(os.path.join(basepath, fname))
  222. if len(dev_files) == 0:
  223. print("WARNING: '%s' doesn't match any file" % fname,
  224. file=sys.stderr)
  225. files += dev_files
  226. elif line == "":
  227. if not name:
  228. continue
  229. developers.append(Developer(name, files))
  230. files = []
  231. name = None
  232. else:
  233. print("Syntax error in DEVELOPERS file, line %d: '%s'" % (linen, line),
  234. file=sys.stderr)
  235. return None
  236. linen += 1
  237. # handle last developer
  238. if name is not None:
  239. developers.append(Developer(name, files))
  240. return developers
  241. def check_developers(developers, basepath=None):
  242. """Look at the list of files versioned in Buildroot, and returns the
  243. list of files that are not handled by any developer"""
  244. if basepath is None:
  245. basepath = os.getcwd()
  246. cmd = ["git", "--git-dir", os.path.join(basepath, ".git"), "ls-files"]
  247. files = subprocess.check_output(cmd).strip().split("\n")
  248. unhandled_files = []
  249. for f in files:
  250. handled = False
  251. for d in developers:
  252. if d.hasfile(os.path.join(basepath, f)):
  253. handled = True
  254. break
  255. if not handled:
  256. unhandled_files.append(f)
  257. return unhandled_files