create_buildsys_python.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714
  1. # Recipe creation tool - create build system handler for python
  2. #
  3. # Copyright (C) 2015 Mentor Graphics Corporation
  4. #
  5. # SPDX-License-Identifier: GPL-2.0-only
  6. #
  7. import ast
  8. import codecs
  9. import collections
  10. import distutils.command.build_py
  11. import email
  12. import imp
  13. import glob
  14. import itertools
  15. import logging
  16. import os
  17. import re
  18. import sys
  19. import subprocess
  20. from recipetool.create import RecipeHandler
  21. logger = logging.getLogger('recipetool')
  22. tinfoil = None
  23. def tinfoil_init(instance):
  24. global tinfoil
  25. tinfoil = instance
  26. class PythonRecipeHandler(RecipeHandler):
  27. base_pkgdeps = ['python3-core']
  28. excluded_pkgdeps = ['python3-dbg']
  29. # os.path is provided by python3-core
  30. assume_provided = ['builtins', 'os.path']
  31. # Assumes that the host python3 builtin_module_names is sane for target too
  32. assume_provided = assume_provided + list(sys.builtin_module_names)
  33. bbvar_map = {
  34. 'Name': 'PN',
  35. 'Version': 'PV',
  36. 'Home-page': 'HOMEPAGE',
  37. 'Summary': 'SUMMARY',
  38. 'Description': 'DESCRIPTION',
  39. 'License': 'LICENSE',
  40. 'Requires': 'RDEPENDS_${PN}',
  41. 'Provides': 'RPROVIDES_${PN}',
  42. 'Obsoletes': 'RREPLACES_${PN}',
  43. }
  44. # PN/PV are already set by recipetool core & desc can be extremely long
  45. excluded_fields = [
  46. 'Description',
  47. ]
  48. setup_parse_map = {
  49. 'Url': 'Home-page',
  50. 'Classifiers': 'Classifier',
  51. 'Description': 'Summary',
  52. }
  53. setuparg_map = {
  54. 'Home-page': 'url',
  55. 'Classifier': 'classifiers',
  56. 'Summary': 'description',
  57. 'Description': 'long-description',
  58. }
  59. # Values which are lists, used by the setup.py argument based metadata
  60. # extraction method, to determine how to process the setup.py output.
  61. setuparg_list_fields = [
  62. 'Classifier',
  63. 'Requires',
  64. 'Provides',
  65. 'Obsoletes',
  66. 'Platform',
  67. 'Supported-Platform',
  68. ]
  69. setuparg_multi_line_values = ['Description']
  70. replacements = [
  71. ('License', r' +$', ''),
  72. ('License', r'^ +', ''),
  73. ('License', r' ', '-'),
  74. ('License', r'^GNU-', ''),
  75. ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
  76. ('License', r'^UNKNOWN$', ''),
  77. # Remove currently unhandled version numbers from these variables
  78. ('Requires', r' *\([^)]*\)', ''),
  79. ('Provides', r' *\([^)]*\)', ''),
  80. ('Obsoletes', r' *\([^)]*\)', ''),
  81. ('Install-requires', r'^([^><= ]+).*', r'\1'),
  82. ('Extras-require', r'^([^><= ]+).*', r'\1'),
  83. ('Tests-require', r'^([^><= ]+).*', r'\1'),
  84. # Remove unhandled dependency on particular features (e.g. foo[PDF])
  85. ('Install-requires', r'\[[^\]]+\]$', ''),
  86. ]
  87. classifier_license_map = {
  88. 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
  89. 'License :: OSI Approved :: Apache Software License': 'Apache',
  90. 'License :: OSI Approved :: Apple Public Source License': 'APSL',
  91. 'License :: OSI Approved :: Artistic License': 'Artistic',
  92. 'License :: OSI Approved :: Attribution Assurance License': 'AAL',
  93. 'License :: OSI Approved :: BSD License': 'BSD',
  94. 'License :: OSI Approved :: Common Public License': 'CPL',
  95. 'License :: OSI Approved :: Eiffel Forum License': 'EFL',
  96. 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
  97. 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
  98. 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
  99. 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
  100. 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
  101. 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
  102. 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
  103. 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
  104. 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
  105. 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
  106. 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
  107. 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
  108. 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
  109. 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
  110. 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
  111. 'License :: OSI Approved :: IBM Public License': 'IPL',
  112. 'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
  113. 'License :: OSI Approved :: Intel Open Source License': 'Intel',
  114. 'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
  115. 'License :: OSI Approved :: MIT License': 'MIT',
  116. 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
  117. 'License :: OSI Approved :: Motosoto License': 'Motosoto',
  118. 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
  119. 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
  120. 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
  121. 'License :: OSI Approved :: Nethack General Public License': 'NGPL',
  122. 'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
  123. 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
  124. 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
  125. 'License :: OSI Approved :: Python Software Foundation License': 'PSF',
  126. 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
  127. 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
  128. 'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
  129. 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)',
  130. 'License :: OSI Approved :: Sun Public License': 'SPL',
  131. 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
  132. 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
  133. 'License :: OSI Approved :: W3C License': 'W3C',
  134. 'License :: OSI Approved :: X.Net License': 'Xnet',
  135. 'License :: OSI Approved :: Zope Public License': 'ZPL',
  136. 'License :: OSI Approved :: zlib/libpng License': 'Zlib',
  137. }
  138. def __init__(self):
  139. pass
  140. def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
  141. if 'buildsystem' in handled:
  142. return False
  143. # Check for non-zero size setup.py files
  144. setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
  145. for fn in setupfiles:
  146. if os.path.getsize(fn):
  147. break
  148. else:
  149. return False
  150. # setup.py is always parsed to get at certain required information, such as
  151. # distutils vs setuptools
  152. #
  153. # If egg info is available, we use it for both its PKG-INFO metadata
  154. # and for its requires.txt for install_requires.
  155. # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
  156. # the parsed setup.py, but use the install_requires info from the
  157. # parsed setup.py.
  158. setupscript = os.path.join(srctree, 'setup.py')
  159. try:
  160. setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
  161. except Exception:
  162. logger.exception("Failed to parse setup.py")
  163. setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
  164. egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
  165. if egginfo:
  166. info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
  167. requires_txt = os.path.join(egginfo[0], 'requires.txt')
  168. if os.path.exists(requires_txt):
  169. with codecs.open(requires_txt) as f:
  170. inst_req = []
  171. extras_req = collections.defaultdict(list)
  172. current_feature = None
  173. for line in f.readlines():
  174. line = line.rstrip()
  175. if not line:
  176. continue
  177. if line.startswith('['):
  178. current_feature = line[1:-1]
  179. elif current_feature:
  180. extras_req[current_feature].append(line)
  181. else:
  182. inst_req.append(line)
  183. info['Install-requires'] = inst_req
  184. info['Extras-require'] = extras_req
  185. elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
  186. info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
  187. if setup_info:
  188. if 'Install-requires' in setup_info:
  189. info['Install-requires'] = setup_info['Install-requires']
  190. if 'Extras-require' in setup_info:
  191. info['Extras-require'] = setup_info['Extras-require']
  192. else:
  193. if setup_info:
  194. info = setup_info
  195. else:
  196. info = self.get_setup_args_info(setupscript)
  197. # Grab the license value before applying replacements
  198. license_str = info.get('License', '').strip()
  199. self.apply_info_replacements(info)
  200. if uses_setuptools:
  201. classes.append('setuptools3')
  202. else:
  203. classes.append('distutils3')
  204. if license_str:
  205. for i, line in enumerate(lines_before):
  206. if line.startswith('LICENSE = '):
  207. lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
  208. break
  209. if 'Classifier' in info:
  210. existing_licenses = info.get('License', '')
  211. licenses = []
  212. for classifier in info['Classifier']:
  213. if classifier in self.classifier_license_map:
  214. license = self.classifier_license_map[classifier]
  215. if license == 'Apache' and 'Apache-2.0' in existing_licenses:
  216. license = 'Apache-2.0'
  217. elif license == 'GPL':
  218. if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
  219. license = 'GPL-2.0'
  220. elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
  221. license = 'GPL-3.0'
  222. elif license == 'LGPL':
  223. if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
  224. license = 'LGPL-2.1'
  225. elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
  226. license = 'LGPL-2.0'
  227. elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
  228. license = 'LGPL-3.0'
  229. licenses.append(license)
  230. if licenses:
  231. info['License'] = ' & '.join(licenses)
  232. # Map PKG-INFO & setup.py fields to bitbake variables
  233. for field, values in info.items():
  234. if field in self.excluded_fields:
  235. continue
  236. if field not in self.bbvar_map:
  237. continue
  238. if isinstance(values, str):
  239. value = values
  240. else:
  241. value = ' '.join(str(v) for v in values if v)
  242. bbvar = self.bbvar_map[field]
  243. if bbvar not in extravalues and value:
  244. extravalues[bbvar] = value
  245. mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
  246. extras_req = set()
  247. if 'Extras-require' in info:
  248. extras_req = info['Extras-require']
  249. if extras_req:
  250. lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
  251. lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
  252. lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
  253. lines_after.append('#')
  254. lines_after.append('# Uncomment this line to enable all the optional features.')
  255. lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
  256. for feature, feature_reqs in extras_req.items():
  257. unmapped_deps.difference_update(feature_reqs)
  258. feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
  259. lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
  260. inst_reqs = set()
  261. if 'Install-requires' in info:
  262. if extras_req:
  263. lines_after.append('')
  264. inst_reqs = info['Install-requires']
  265. if inst_reqs:
  266. unmapped_deps.difference_update(inst_reqs)
  267. inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
  268. lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
  269. lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
  270. lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
  271. if mapped_deps:
  272. name = info.get('Name')
  273. if name and name[0] in mapped_deps:
  274. # Attempt to avoid self-reference
  275. mapped_deps.remove(name[0])
  276. mapped_deps -= set(self.excluded_pkgdeps)
  277. if inst_reqs or extras_req:
  278. lines_after.append('')
  279. lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
  280. lines_after.append('# python sources, and might not be 100% accurate.')
  281. lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
  282. unmapped_deps -= set(extensions)
  283. unmapped_deps -= set(self.assume_provided)
  284. if unmapped_deps:
  285. if mapped_deps:
  286. lines_after.append('')
  287. lines_after.append('# WARNING: We were unable to map the following python package/module')
  288. lines_after.append('# dependencies to the bitbake packages which include them:')
  289. lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
  290. handled.append('buildsystem')
  291. def get_pkginfo(self, pkginfo_fn):
  292. msg = email.message_from_file(open(pkginfo_fn, 'r'))
  293. msginfo = {}
  294. for field in msg.keys():
  295. values = msg.get_all(field)
  296. if len(values) == 1:
  297. msginfo[field] = values[0]
  298. else:
  299. msginfo[field] = values
  300. return msginfo
  301. def parse_setup_py(self, setupscript='./setup.py'):
  302. with codecs.open(setupscript) as f:
  303. info, imported_modules, non_literals, extensions = gather_setup_info(f)
  304. def _map(key):
  305. key = key.replace('_', '-')
  306. key = key[0].upper() + key[1:]
  307. if key in self.setup_parse_map:
  308. key = self.setup_parse_map[key]
  309. return key
  310. # Naive mapping of setup() arguments to PKG-INFO field names
  311. for d in [info, non_literals]:
  312. for key, value in list(d.items()):
  313. if key is None:
  314. continue
  315. new_key = _map(key)
  316. if new_key != key:
  317. del d[key]
  318. d[new_key] = value
  319. return info, 'setuptools' in imported_modules, non_literals, extensions
  320. def get_setup_args_info(self, setupscript='./setup.py'):
  321. cmd = ['python3', setupscript]
  322. info = {}
  323. keys = set(self.bbvar_map.keys())
  324. keys |= set(self.setuparg_list_fields)
  325. keys |= set(self.setuparg_multi_line_values)
  326. grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
  327. for index, keys in grouped_keys:
  328. if index == (True, False):
  329. # Splitlines output for each arg as a list value
  330. for key in keys:
  331. arg = self.setuparg_map.get(key, key.lower())
  332. try:
  333. arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
  334. except (OSError, subprocess.CalledProcessError):
  335. pass
  336. else:
  337. info[key] = [l.rstrip() for l in arg_info.splitlines()]
  338. elif index == (False, True):
  339. # Entire output for each arg
  340. for key in keys:
  341. arg = self.setuparg_map.get(key, key.lower())
  342. try:
  343. arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
  344. except (OSError, subprocess.CalledProcessError):
  345. pass
  346. else:
  347. info[key] = arg_info
  348. else:
  349. info.update(self.get_setup_byline(list(keys), setupscript))
  350. return info
  351. def get_setup_byline(self, fields, setupscript='./setup.py'):
  352. info = {}
  353. cmd = ['python3', setupscript]
  354. cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
  355. try:
  356. info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
  357. except (OSError, subprocess.CalledProcessError):
  358. pass
  359. else:
  360. if len(fields) != len(info_lines):
  361. logger.error('Mismatch between setup.py output lines and number of fields')
  362. sys.exit(1)
  363. for lineno, line in enumerate(info_lines):
  364. line = line.rstrip()
  365. info[fields[lineno]] = line
  366. return info
  367. def apply_info_replacements(self, info):
  368. for variable, search, replace in self.replacements:
  369. if variable not in info:
  370. continue
  371. def replace_value(search, replace, value):
  372. if replace is None:
  373. if re.search(search, value):
  374. return None
  375. else:
  376. new_value = re.sub(search, replace, value)
  377. if value != new_value:
  378. return new_value
  379. return value
  380. value = info[variable]
  381. if isinstance(value, str):
  382. new_value = replace_value(search, replace, value)
  383. if new_value is None:
  384. del info[variable]
  385. elif new_value != value:
  386. info[variable] = new_value
  387. elif hasattr(value, 'items'):
  388. for dkey, dvalue in list(value.items()):
  389. new_list = []
  390. for pos, a_value in enumerate(dvalue):
  391. new_value = replace_value(search, replace, a_value)
  392. if new_value is not None and new_value != value:
  393. new_list.append(new_value)
  394. if value != new_list:
  395. value[dkey] = new_list
  396. else:
  397. new_list = []
  398. for pos, a_value in enumerate(value):
  399. new_value = replace_value(search, replace, a_value)
  400. if new_value is not None and new_value != value:
  401. new_list.append(new_value)
  402. if value != new_list:
  403. info[variable] = new_list
  404. def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
  405. if 'Package-dir' in setup_info:
  406. package_dir = setup_info['Package-dir']
  407. else:
  408. package_dir = {}
  409. class PackageDir(distutils.command.build_py.build_py):
  410. def __init__(self, package_dir):
  411. self.package_dir = package_dir
  412. pd = PackageDir(package_dir)
  413. to_scan = []
  414. if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
  415. if 'Py-modules' in setup_info:
  416. for module in setup_info['Py-modules']:
  417. try:
  418. package, module = module.rsplit('.', 1)
  419. except ValueError:
  420. package, module = '.', module
  421. module_path = os.path.join(pd.get_package_dir(package), module + '.py')
  422. to_scan.append(module_path)
  423. if 'Packages' in setup_info:
  424. for package in setup_info['Packages']:
  425. to_scan.append(pd.get_package_dir(package))
  426. if 'Scripts' in setup_info:
  427. to_scan.extend(setup_info['Scripts'])
  428. else:
  429. logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
  430. if not to_scan:
  431. to_scan = ['.']
  432. logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
  433. provided_packages = self.parse_pkgdata_for_python_packages()
  434. scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
  435. mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
  436. for dep in scanned_deps:
  437. mapped = provided_packages.get(dep)
  438. if mapped:
  439. logger.debug('Mapped %s to %s' % (dep, mapped))
  440. mapped_deps.add(mapped)
  441. else:
  442. logger.debug('Could not map %s' % dep)
  443. unmapped_deps.add(dep)
  444. return mapped_deps, unmapped_deps
  445. def scan_python_dependencies(self, paths):
  446. deps = set()
  447. try:
  448. dep_output = self.run_command(['pythondeps', '-d'] + paths)
  449. except (OSError, subprocess.CalledProcessError):
  450. pass
  451. else:
  452. for line in dep_output.splitlines():
  453. line = line.rstrip()
  454. dep, filename = line.split('\t', 1)
  455. if filename.endswith('/setup.py'):
  456. continue
  457. deps.add(dep)
  458. try:
  459. provides_output = self.run_command(['pythondeps', '-p'] + paths)
  460. except (OSError, subprocess.CalledProcessError):
  461. pass
  462. else:
  463. provides_lines = (l.rstrip() for l in provides_output.splitlines())
  464. provides = set(l for l in provides_lines if l and l != 'setup')
  465. deps -= provides
  466. return deps
  467. def parse_pkgdata_for_python_packages(self):
  468. suffixes = [t[0] for t in imp.get_suffixes()]
  469. pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
  470. ldata = tinfoil.config_data.createCopy()
  471. bb.parse.handle('classes/python3-dir.bbclass', ldata, True)
  472. python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
  473. dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
  474. python_dirs = [python_sitedir + os.sep,
  475. os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
  476. os.path.dirname(python_sitedir) + os.sep]
  477. packages = {}
  478. for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
  479. files_info = None
  480. with open(pkgdatafile, 'r') as f:
  481. for line in f.readlines():
  482. field, value = line.split(': ', 1)
  483. if field == 'FILES_INFO':
  484. files_info = ast.literal_eval(value)
  485. break
  486. else:
  487. continue
  488. for fn in files_info:
  489. for suffix in suffixes:
  490. if fn.endswith(suffix):
  491. break
  492. else:
  493. continue
  494. if fn.startswith(dynload_dir + os.sep):
  495. if '/.debug/' in fn:
  496. continue
  497. base = os.path.basename(fn)
  498. provided = base.split('.', 1)[0]
  499. packages[provided] = os.path.basename(pkgdatafile)
  500. continue
  501. for python_dir in python_dirs:
  502. if fn.startswith(python_dir):
  503. relpath = fn[len(python_dir):]
  504. relstart, _, relremaining = relpath.partition(os.sep)
  505. if relstart.endswith('.egg'):
  506. relpath = relremaining
  507. base, _ = os.path.splitext(relpath)
  508. if '/.debug/' in base:
  509. continue
  510. if os.path.basename(base) == '__init__':
  511. base = os.path.dirname(base)
  512. base = base.replace(os.sep + os.sep, os.sep)
  513. provided = base.replace(os.sep, '.')
  514. packages[provided] = os.path.basename(pkgdatafile)
  515. return packages
  516. @classmethod
  517. def run_command(cls, cmd, **popenargs):
  518. if 'stderr' not in popenargs:
  519. popenargs['stderr'] = subprocess.STDOUT
  520. try:
  521. return subprocess.check_output(cmd, **popenargs).decode('utf-8')
  522. except OSError as exc:
  523. logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
  524. raise
  525. except subprocess.CalledProcessError as exc:
  526. logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
  527. raise
  528. def gather_setup_info(fileobj):
  529. parsed = ast.parse(fileobj.read(), fileobj.name)
  530. visitor = SetupScriptVisitor()
  531. visitor.visit(parsed)
  532. non_literals, extensions = {}, []
  533. for key, value in list(visitor.keywords.items()):
  534. if key == 'ext_modules':
  535. if isinstance(value, list):
  536. for ext in value:
  537. if (isinstance(ext, ast.Call) and
  538. isinstance(ext.func, ast.Name) and
  539. ext.func.id == 'Extension' and
  540. not has_non_literals(ext.args)):
  541. extensions.append(ext.args[0])
  542. elif has_non_literals(value):
  543. non_literals[key] = value
  544. del visitor.keywords[key]
  545. return visitor.keywords, visitor.imported_modules, non_literals, extensions
  546. class SetupScriptVisitor(ast.NodeVisitor):
  547. def __init__(self):
  548. ast.NodeVisitor.__init__(self)
  549. self.keywords = {}
  550. self.non_literals = []
  551. self.imported_modules = set()
  552. def visit_Expr(self, node):
  553. if isinstance(node.value, ast.Call) and \
  554. isinstance(node.value.func, ast.Name) and \
  555. node.value.func.id == 'setup':
  556. self.visit_setup(node.value)
  557. def visit_setup(self, node):
  558. call = LiteralAstTransform().visit(node)
  559. self.keywords = call.keywords
  560. for k, v in self.keywords.items():
  561. if has_non_literals(v):
  562. self.non_literals.append(k)
  563. def visit_Import(self, node):
  564. for alias in node.names:
  565. self.imported_modules.add(alias.name)
  566. def visit_ImportFrom(self, node):
  567. self.imported_modules.add(node.module)
  568. class LiteralAstTransform(ast.NodeTransformer):
  569. """Simplify the ast through evaluation of literals."""
  570. excluded_fields = ['ctx']
  571. def visit(self, node):
  572. if not isinstance(node, ast.AST):
  573. return node
  574. else:
  575. return ast.NodeTransformer.visit(self, node)
  576. def generic_visit(self, node):
  577. try:
  578. return ast.literal_eval(node)
  579. except ValueError:
  580. for field, value in ast.iter_fields(node):
  581. if field in self.excluded_fields:
  582. delattr(node, field)
  583. if value is None:
  584. continue
  585. if isinstance(value, list):
  586. if field in ('keywords', 'kwargs'):
  587. new_value = dict((kw.arg, self.visit(kw.value)) for kw in value)
  588. else:
  589. new_value = [self.visit(i) for i in value]
  590. else:
  591. new_value = self.visit(value)
  592. setattr(node, field, new_value)
  593. return node
  594. def visit_Name(self, node):
  595. if hasattr('__builtins__', node.id):
  596. return getattr(__builtins__, node.id)
  597. else:
  598. return self.generic_visit(node)
  599. def visit_Tuple(self, node):
  600. return tuple(self.visit(v) for v in node.elts)
  601. def visit_List(self, node):
  602. return [self.visit(v) for v in node.elts]
  603. def visit_Set(self, node):
  604. return set(self.visit(v) for v in node.elts)
  605. def visit_Dict(self, node):
  606. keys = (self.visit(k) for k in node.keys)
  607. values = (self.visit(v) for v in node.values)
  608. return dict(zip(keys, values))
  609. def has_non_literals(value):
  610. if isinstance(value, ast.AST):
  611. return True
  612. elif isinstance(value, str):
  613. return False
  614. elif hasattr(value, 'values'):
  615. return any(has_non_literals(v) for v in value.values())
  616. elif hasattr(value, '__iter__'):
  617. return any(has_non_literals(v) for v in value)
  618. def register_recipe_handlers(handlers):
  619. # We need to make sure this is ahead of the makefile fallback handler
  620. handlers.append((PythonRecipeHandler(), 70))