scanpypi 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754
  1. #!/usr/bin/env python
  2. """
  3. Utility for building Buildroot packages for existing PyPI packages
  4. Any package built by scanpypi should be manually checked for
  5. errors.
  6. """
  7. from __future__ import print_function
  8. from __future__ import absolute_import
  9. import argparse
  10. import json
  11. import sys
  12. import os
  13. import shutil
  14. import tarfile
  15. import zipfile
  16. import errno
  17. import hashlib
  18. import re
  19. import textwrap
  20. import tempfile
  21. import imp
  22. from functools import wraps
  23. import six.moves.urllib.request
  24. import six.moves.urllib.error
  25. import six.moves.urllib.parse
  26. from six.moves import map
  27. from six.moves import zip
  28. from six.moves import input
  29. if six.PY2:
  30. import StringIO
  31. else:
  32. import io
  33. BUF_SIZE = 65536
  34. try:
  35. import spdx_lookup as liclookup
  36. except ImportError:
  37. # spdx_lookup is not installed
  38. print('spdx_lookup module is not installed. This can lead to an '
  39. 'inaccurate licence detection. Please install it via\n'
  40. 'pip install spdx_lookup')
  41. liclookup = None
  42. def setup_decorator(func, method):
  43. """
  44. Decorator for distutils.core.setup and setuptools.setup.
  45. Puts the arguments with which setup is called as a dict
  46. Add key 'method' which should be either 'setuptools' or 'distutils'.
  47. Keyword arguments:
  48. func -- either setuptools.setup or distutils.core.setup
  49. method -- either 'setuptools' or 'distutils'
  50. """
  51. @wraps(func)
  52. def closure(*args, **kwargs):
  53. # Any python packages calls its setup function to be installed.
  54. # Argument 'name' of this setup function is the package's name
  55. BuildrootPackage.setup_args[kwargs['name']] = kwargs
  56. BuildrootPackage.setup_args[kwargs['name']]['method'] = method
  57. return closure
  58. # monkey patch
  59. import setuptools # noqa E402
  60. setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
  61. import distutils # noqa E402
  62. distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
  63. def find_file_upper_case(filenames, path='./'):
  64. """
  65. List generator:
  66. Recursively find files that matches one of the specified filenames.
  67. Returns a relative path starting with path argument.
  68. Keyword arguments:
  69. filenames -- List of filenames to be found
  70. path -- Path to the directory to search
  71. """
  72. for root, dirs, files in os.walk(path):
  73. for file in files:
  74. if file.upper() in filenames:
  75. yield (os.path.join(root, file))
  76. def pkg_buildroot_name(pkg_name):
  77. """
  78. Returns the Buildroot package name for the PyPI package pkg_name.
  79. Remove all non alphanumeric characters except -
  80. Also lowers the name and adds 'python-' suffix
  81. Keyword arguments:
  82. pkg_name -- String to rename
  83. """
  84. name = re.sub(r'[^\w-]', '', pkg_name.lower())
  85. name = name.replace('_', '-')
  86. prefix = 'python-'
  87. pattern = re.compile(r'^(?!' + prefix + ')(.+?)$')
  88. name = pattern.sub(r'python-\1', name)
  89. return name
  90. class DownloadFailed(Exception):
  91. pass
  92. class BuildrootPackage():
  93. """This class's methods are not meant to be used individually please
  94. use them in the correct order:
  95. __init__
  96. download_package
  97. extract_package
  98. load_module
  99. get_requirements
  100. create_package_mk
  101. create_hash_file
  102. create_config_in
  103. """
  104. setup_args = {}
  105. def __init__(self, real_name, pkg_folder):
  106. self.real_name = real_name
  107. self.buildroot_name = pkg_buildroot_name(self.real_name)
  108. self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name)
  109. self.mk_name = self.buildroot_name.upper().replace('-', '_')
  110. self.as_string = None
  111. self.md5_sum = None
  112. self.metadata = None
  113. self.metadata_name = None
  114. self.metadata_url = None
  115. self.pkg_req = None
  116. self.setup_metadata = None
  117. self.tmp_extract = None
  118. self.used_url = None
  119. self.filename = None
  120. self.url = None
  121. self.version = None
  122. self.license_files = []
  123. def fetch_package_info(self):
  124. """
  125. Fetch a package's metadata from the python package index
  126. """
  127. self.metadata_url = 'https://pypi.org/pypi/{pkg}/json'.format(
  128. pkg=self.real_name)
  129. try:
  130. pkg_json = six.moves.urllib.request.urlopen(self.metadata_url).read().decode()
  131. except six.moves.urllib.error.HTTPError as error:
  132. print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
  133. print('ERROR: Could not find package {pkg}.\n'
  134. 'Check syntax inside the python package index:\n'
  135. 'https://pypi.python.org/pypi/ '
  136. .format(pkg=self.real_name))
  137. raise
  138. except six.moves.urllib.error.URLError:
  139. print('ERROR: Could not find package {pkg}.\n'
  140. 'Check syntax inside the python package index:\n'
  141. 'https://pypi.python.org/pypi/ '
  142. .format(pkg=self.real_name))
  143. raise
  144. self.metadata = json.loads(pkg_json)
  145. self.version = self.metadata['info']['version']
  146. self.metadata_name = self.metadata['info']['name']
  147. def download_package(self):
  148. """
  149. Download a package using metadata from pypi
  150. """
  151. download = None
  152. try:
  153. self.metadata['urls'][0]['filename']
  154. except IndexError:
  155. print(
  156. 'Non-conventional package, ',
  157. 'please check carefully after creation')
  158. self.metadata['urls'] = [{
  159. 'packagetype': 'sdist',
  160. 'url': self.metadata['info']['download_url'],
  161. 'digests': None}]
  162. # In this case, we can't get the name of the downloaded file
  163. # from the pypi api, so we need to find it, this should work
  164. urlpath = six.moves.urllib.parse.urlparse(
  165. self.metadata['info']['download_url']).path
  166. # urlparse().path give something like
  167. # /path/to/file-version.tar.gz
  168. # We use basename to remove /path/to
  169. self.metadata['urls'][0]['filename'] = os.path.basename(urlpath)
  170. for download_url in self.metadata['urls']:
  171. if 'bdist' in download_url['packagetype']:
  172. continue
  173. try:
  174. print('Downloading package {pkg} from {url}...'.format(
  175. pkg=self.real_name, url=download_url['url']))
  176. download = six.moves.urllib.request.urlopen(download_url['url'])
  177. except six.moves.urllib.error.HTTPError as http_error:
  178. download = http_error
  179. else:
  180. self.used_url = download_url
  181. self.as_string = download.read()
  182. if not download_url['digests']['md5']:
  183. break
  184. self.md5_sum = hashlib.md5(self.as_string).hexdigest()
  185. if self.md5_sum == download_url['digests']['md5']:
  186. break
  187. if download is None:
  188. raise DownloadFailed('Failed to download package {pkg}: '
  189. 'No source archive available'
  190. .format(pkg=self.real_name))
  191. elif download.__class__ == six.moves.urllib.error.HTTPError:
  192. raise download
  193. self.filename = self.used_url['filename']
  194. self.url = self.used_url['url']
  195. def check_archive(self, members):
  196. """
  197. Check archive content before extracting
  198. Keyword arguments:
  199. members -- list of archive members
  200. """
  201. # Protect against https://github.com/snyk/zip-slip-vulnerability
  202. # Older python versions do not validate that the extracted files are
  203. # inside the target directory. Detect and error out on evil paths
  204. evil = [e for e in members if os.path.relpath(e).startswith(('/', '..'))]
  205. if evil:
  206. print('ERROR: Refusing to extract {} with suspicious members {}'.format(
  207. self.filename, evil))
  208. sys.exit(1)
  209. def extract_package(self, tmp_path):
  210. """
  211. Extract the package contents into a directrory
  212. Keyword arguments:
  213. tmp_path -- directory where you want the package to be extracted
  214. """
  215. if six.PY2:
  216. as_file = StringIO.StringIO(self.as_string)
  217. else:
  218. as_file = io.BytesIO(self.as_string)
  219. if self.filename[-3:] == 'zip':
  220. with zipfile.ZipFile(as_file) as as_zipfile:
  221. tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
  222. try:
  223. os.makedirs(tmp_pkg)
  224. except OSError as exception:
  225. if exception.errno != errno.EEXIST:
  226. print("ERROR: ", exception.strerror, file=sys.stderr)
  227. return
  228. print('WARNING:', exception.strerror, file=sys.stderr)
  229. print('Removing {pkg}...'.format(pkg=tmp_pkg))
  230. shutil.rmtree(tmp_pkg)
  231. os.makedirs(tmp_pkg)
  232. self.check_archive(as_zipfile.namelist())
  233. as_zipfile.extractall(tmp_pkg)
  234. pkg_filename = self.filename.split(".zip")[0]
  235. else:
  236. with tarfile.open(fileobj=as_file) as as_tarfile:
  237. tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
  238. try:
  239. os.makedirs(tmp_pkg)
  240. except OSError as exception:
  241. if exception.errno != errno.EEXIST:
  242. print("ERROR: ", exception.strerror, file=sys.stderr)
  243. return
  244. print('WARNING:', exception.strerror, file=sys.stderr)
  245. print('Removing {pkg}...'.format(pkg=tmp_pkg))
  246. shutil.rmtree(tmp_pkg)
  247. os.makedirs(tmp_pkg)
  248. self.check_archive(as_tarfile.getnames())
  249. as_tarfile.extractall(tmp_pkg)
  250. pkg_filename = self.filename.split(".tar")[0]
  251. tmp_extract = '{folder}/{name}'
  252. self.tmp_extract = tmp_extract.format(
  253. folder=tmp_pkg,
  254. name=pkg_filename)
  255. def load_setup(self):
  256. """
  257. Loads the corresponding setup and store its metadata
  258. """
  259. current_dir = os.getcwd()
  260. os.chdir(self.tmp_extract)
  261. sys.path.append(self.tmp_extract)
  262. s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
  263. setup = imp.load_module('setup', s_file, s_path, s_desc)
  264. if self.metadata_name in self.setup_args:
  265. pass
  266. elif self.metadata_name.replace('_', '-') in self.setup_args:
  267. self.metadata_name = self.metadata_name.replace('_', '-')
  268. elif self.metadata_name.replace('-', '_') in self.setup_args:
  269. self.metadata_name = self.metadata_name.replace('-', '_')
  270. try:
  271. self.setup_metadata = self.setup_args[self.metadata_name]
  272. except KeyError:
  273. # This means setup was not called which most likely mean that it is
  274. # called through the if __name__ == '__main__' directive.
  275. # In this case, we can only pray that it is called through a
  276. # function called main() in setup.py.
  277. setup.main() # Will raise AttributeError if not found
  278. self.setup_metadata = self.setup_args[self.metadata_name]
  279. # Here we must remove the module the hard way.
  280. # We must do this because of a very specific case: if a package calls
  281. # setup from the __main__ but does not come with a 'main()' function,
  282. # for some reason setup.main() will successfully call the main
  283. # function of a previous package...
  284. sys.modules.pop('setup', None)
  285. del setup
  286. os.chdir(current_dir)
  287. sys.path.remove(self.tmp_extract)
  288. def get_requirements(self, pkg_folder):
  289. """
  290. Retrieve dependencies from the metadata found in the setup.py script of
  291. a pypi package.
  292. Keyword Arguments:
  293. pkg_folder -- location of the already created packages
  294. """
  295. if 'install_requires' not in self.setup_metadata:
  296. self.pkg_req = None
  297. return set()
  298. self.pkg_req = self.setup_metadata['install_requires']
  299. self.pkg_req = [re.sub(r'([-.\w]+).*', r'\1', req)
  300. for req in self.pkg_req]
  301. # get rid of commented lines and also strip the package strings
  302. self.pkg_req = [item.strip() for item in self.pkg_req
  303. if len(item) > 0 and item[0] != '#']
  304. req_not_found = self.pkg_req
  305. self.pkg_req = list(map(pkg_buildroot_name, self.pkg_req))
  306. pkg_tuples = list(zip(req_not_found, self.pkg_req))
  307. # pkg_tuples is a list of tuples that looks like
  308. # ('werkzeug','python-werkzeug') because I need both when checking if
  309. # dependencies already exist or are already in the download list
  310. req_not_found = set(
  311. pkg[0] for pkg in pkg_tuples
  312. if not os.path.isdir(pkg[1])
  313. )
  314. return req_not_found
  315. def __create_mk_header(self):
  316. """
  317. Create the header of the <package_name>.mk file
  318. """
  319. header = ['#' * 80 + '\n']
  320. header.append('#\n')
  321. header.append('# {name}\n'.format(name=self.buildroot_name))
  322. header.append('#\n')
  323. header.append('#' * 80 + '\n')
  324. header.append('\n')
  325. return header
  326. def __create_mk_download_info(self):
  327. """
  328. Create the lines refering to the download information of the
  329. <package_name>.mk file
  330. """
  331. lines = []
  332. version_line = '{name}_VERSION = {version}\n'.format(
  333. name=self.mk_name,
  334. version=self.version)
  335. lines.append(version_line)
  336. if self.buildroot_name != self.real_name:
  337. targz = self.filename.replace(
  338. self.version,
  339. '$({name}_VERSION)'.format(name=self.mk_name))
  340. targz_line = '{name}_SOURCE = {filename}\n'.format(
  341. name=self.mk_name,
  342. filename=targz)
  343. lines.append(targz_line)
  344. if self.filename not in self.url:
  345. # Sometimes the filename is in the url, sometimes it's not
  346. site_url = self.url
  347. else:
  348. site_url = self.url[:self.url.find(self.filename)]
  349. site_line = '{name}_SITE = {url}'.format(name=self.mk_name,
  350. url=site_url)
  351. site_line = site_line.rstrip('/') + '\n'
  352. lines.append(site_line)
  353. return lines
  354. def __create_mk_setup(self):
  355. """
  356. Create the line refering to the setup method of the package of the
  357. <package_name>.mk file
  358. There are two things you can use to make an installer
  359. for a python package: distutils or setuptools
  360. distutils comes with python but does not support dependencies.
  361. distutils is mostly still there for backward support.
  362. setuptools is what smart people use,
  363. but it is not shipped with python :(
  364. """
  365. lines = []
  366. setup_type_line = '{name}_SETUP_TYPE = {method}\n'.format(
  367. name=self.mk_name,
  368. method=self.setup_metadata['method'])
  369. lines.append(setup_type_line)
  370. return lines
  371. def __get_license_names(self, license_files):
  372. """
  373. Try to determine the related license name.
  374. There are two possibilities. Either the script tries to
  375. get license name from package's metadata or, if spdx_lookup
  376. package is available, the script compares license files with
  377. SPDX database.
  378. """
  379. license_line = ''
  380. if liclookup is None:
  381. license_dict = {
  382. 'Apache Software License': 'Apache-2.0',
  383. 'BSD License': 'FIXME: please specify the exact BSD version',
  384. 'European Union Public Licence 1.0': 'EUPL-1.0',
  385. 'European Union Public Licence 1.1': 'EUPL-1.1',
  386. "GNU General Public License": "GPL",
  387. "GNU General Public License v2": "GPL-2.0",
  388. "GNU General Public License v2 or later": "GPL-2.0+",
  389. "GNU General Public License v3": "GPL-3.0",
  390. "GNU General Public License v3 or later": "GPL-3.0+",
  391. "GNU Lesser General Public License v2": "LGPL-2.1",
  392. "GNU Lesser General Public License v2 or later": "LGPL-2.1+",
  393. "GNU Lesser General Public License v3": "LGPL-3.0",
  394. "GNU Lesser General Public License v3 or later": "LGPL-3.0+",
  395. "GNU Library or Lesser General Public License": "LGPL-2.0",
  396. "ISC License": "ISC",
  397. "MIT License": "MIT",
  398. "Mozilla Public License 1.0": "MPL-1.0",
  399. "Mozilla Public License 1.1": "MPL-1.1",
  400. "Mozilla Public License 2.0": "MPL-2.0",
  401. "Zope Public License": "ZPL"
  402. }
  403. regexp = re.compile(r'^License :* *.* *:+ (.*)( \(.*\))?$')
  404. classifiers_licenses = [regexp.sub(r"\1", lic)
  405. for lic in self.metadata['info']['classifiers']
  406. if regexp.match(lic)]
  407. licenses = [license_dict[x] if x in license_dict else x for x in classifiers_licenses]
  408. if not len(licenses):
  409. print('WARNING: License has been set to "{license}". It is most'
  410. ' likely wrong, please change it if need be'.format(
  411. license=', '.join(licenses)))
  412. licenses = [self.metadata['info']['license']]
  413. licenses = set(licenses)
  414. license_line = '{name}_LICENSE = {license}\n'.format(
  415. name=self.mk_name,
  416. license=', '.join(licenses))
  417. else:
  418. license_names = []
  419. for license_file in license_files:
  420. with open(license_file) as lic_file:
  421. match = liclookup.match(lic_file.read())
  422. if match is not None and match.confidence >= 90.0:
  423. license_names.append(match.license.id)
  424. else:
  425. license_names.append("FIXME: license id couldn't be detected")
  426. license_names = set(license_names)
  427. if len(license_names) > 0:
  428. license_line = ('{name}_LICENSE ='
  429. ' {names}\n'.format(
  430. name=self.mk_name,
  431. names=', '.join(license_names)))
  432. return license_line
  433. def __create_mk_license(self):
  434. """
  435. Create the lines referring to the package's license informations of the
  436. <package_name>.mk file
  437. The license's files are found by searching the package (case insensitive)
  438. for files named license, license.txt etc. If more than one license file
  439. is found, the user is asked to select which ones he wants to use.
  440. """
  441. lines = []
  442. filenames = ['LICENCE', 'LICENSE', 'LICENSE.RST', 'LICENSE.TXT',
  443. 'COPYING', 'COPYING.TXT']
  444. self.license_files = list(find_file_upper_case(filenames, self.tmp_extract))
  445. lines.append(self.__get_license_names(self.license_files))
  446. license_files = [license.replace(self.tmp_extract, '')[1:]
  447. for license in self.license_files]
  448. if len(license_files) > 0:
  449. if len(license_files) > 1:
  450. print('More than one file found for license:',
  451. ', '.join(license_files))
  452. license_files = [filename
  453. for index, filename in enumerate(license_files)]
  454. license_file_line = ('{name}_LICENSE_FILES ='
  455. ' {files}\n'.format(
  456. name=self.mk_name,
  457. files=' '.join(license_files)))
  458. lines.append(license_file_line)
  459. else:
  460. print('WARNING: No license file found,'
  461. ' please specify it manually afterwards')
  462. license_file_line = '# No license file found\n'
  463. return lines
  464. def __create_mk_requirements(self):
  465. """
  466. Create the lines referring to the dependencies of the of the
  467. <package_name>.mk file
  468. Keyword Arguments:
  469. pkg_name -- name of the package
  470. pkg_req -- dependencies of the package
  471. """
  472. lines = []
  473. dependencies_line = ('{name}_DEPENDENCIES ='
  474. ' {reqs}\n'.format(
  475. name=self.mk_name,
  476. reqs=' '.join(self.pkg_req)))
  477. lines.append(dependencies_line)
  478. return lines
  479. def create_package_mk(self):
  480. """
  481. Create the lines corresponding to the <package_name>.mk file
  482. """
  483. pkg_mk = '{name}.mk'.format(name=self.buildroot_name)
  484. path_to_mk = os.path.join(self.pkg_dir, pkg_mk)
  485. print('Creating {file}...'.format(file=path_to_mk))
  486. lines = self.__create_mk_header()
  487. lines += self.__create_mk_download_info()
  488. lines += self.__create_mk_setup()
  489. lines += self.__create_mk_license()
  490. lines.append('\n')
  491. lines.append('$(eval $(python-package))')
  492. lines.append('\n')
  493. with open(path_to_mk, 'w') as mk_file:
  494. mk_file.writelines(lines)
  495. def create_hash_file(self):
  496. """
  497. Create the lines corresponding to the <package_name>.hash files
  498. """
  499. pkg_hash = '{name}.hash'.format(name=self.buildroot_name)
  500. path_to_hash = os.path.join(self.pkg_dir, pkg_hash)
  501. print('Creating {filename}...'.format(filename=path_to_hash))
  502. lines = []
  503. if self.used_url['digests']['md5'] and self.used_url['digests']['sha256']:
  504. hash_header = '# md5, sha256 from {url}\n'.format(
  505. url=self.metadata_url)
  506. lines.append(hash_header)
  507. hash_line = '{method} {digest} {filename}\n'.format(
  508. method='md5',
  509. digest=self.used_url['digests']['md5'],
  510. filename=self.filename)
  511. lines.append(hash_line)
  512. hash_line = '{method} {digest} {filename}\n'.format(
  513. method='sha256',
  514. digest=self.used_url['digests']['sha256'],
  515. filename=self.filename)
  516. lines.append(hash_line)
  517. if self.license_files:
  518. lines.append('# Locally computed sha256 checksums\n')
  519. for license_file in self.license_files:
  520. sha256 = hashlib.sha256()
  521. with open(license_file, 'rb') as lic_f:
  522. while True:
  523. data = lic_f.read(BUF_SIZE)
  524. if not data:
  525. break
  526. sha256.update(data)
  527. hash_line = '{method} {digest} {filename}\n'.format(
  528. method='sha256',
  529. digest=sha256.hexdigest(),
  530. filename=license_file.replace(self.tmp_extract, '')[1:])
  531. lines.append(hash_line)
  532. with open(path_to_hash, 'w') as hash_file:
  533. hash_file.writelines(lines)
  534. def create_config_in(self):
  535. """
  536. Creates the Config.in file of a package
  537. """
  538. path_to_config = os.path.join(self.pkg_dir, 'Config.in')
  539. print('Creating {file}...'.format(file=path_to_config))
  540. lines = []
  541. config_line = 'config BR2_PACKAGE_{name}\n'.format(
  542. name=self.mk_name)
  543. lines.append(config_line)
  544. bool_line = '\tbool "{name}"\n'.format(name=self.buildroot_name)
  545. lines.append(bool_line)
  546. if self.pkg_req:
  547. self.pkg_req.sort()
  548. for dep in self.pkg_req:
  549. dep_line = '\tselect BR2_PACKAGE_{req} # runtime\n'.format(
  550. req=dep.upper().replace('-', '_'))
  551. lines.append(dep_line)
  552. lines.append('\thelp\n')
  553. help_lines = textwrap.wrap(self.metadata['info']['summary'], 62,
  554. initial_indent='\t ',
  555. subsequent_indent='\t ')
  556. # make sure a help text is terminated with a full stop
  557. if help_lines[-1][-1] != '.':
  558. help_lines[-1] += '.'
  559. # \t + two spaces is 3 char long
  560. help_lines.append('')
  561. help_lines.append('\t ' + self.metadata['info']['home_page'])
  562. help_lines = [x + '\n' for x in help_lines]
  563. lines += help_lines
  564. with open(path_to_config, 'w') as config_file:
  565. config_file.writelines(lines)
  566. def main():
  567. # Building the parser
  568. parser = argparse.ArgumentParser(
  569. description="Creates buildroot packages from the metadata of "
  570. "an existing PyPI packages and include it "
  571. "in menuconfig")
  572. parser.add_argument("packages",
  573. help="list of packages to be created",
  574. nargs='+')
  575. parser.add_argument("-o", "--output",
  576. help="""
  577. Output directory for packages.
  578. Default is ./package
  579. """,
  580. default='./package')
  581. args = parser.parse_args()
  582. packages = list(set(args.packages))
  583. # tmp_path is where we'll extract the files later
  584. tmp_prefix = 'scanpypi-'
  585. pkg_folder = args.output
  586. tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
  587. try:
  588. for real_pkg_name in packages:
  589. package = BuildrootPackage(real_pkg_name, pkg_folder)
  590. print('buildroot package name for {}:'.format(package.real_name),
  591. package.buildroot_name)
  592. # First we download the package
  593. # Most of the info we need can only be found inside the package
  594. print('Package:', package.buildroot_name)
  595. print('Fetching package', package.real_name)
  596. try:
  597. package.fetch_package_info()
  598. except (six.moves.urllib.error.URLError, six.moves.urllib.error.HTTPError):
  599. continue
  600. if package.metadata_name.lower() == 'setuptools':
  601. # setuptools imports itself, that does not work very well
  602. # with the monkey path at the begining
  603. print('Error: setuptools cannot be built using scanPyPI')
  604. continue
  605. try:
  606. package.download_package()
  607. except six.moves.urllib.error.HTTPError as error:
  608. print('Error: {code} {reason}'.format(code=error.code,
  609. reason=error.reason))
  610. print('Error downloading package :', package.buildroot_name)
  611. print()
  612. continue
  613. # extract the tarball
  614. try:
  615. package.extract_package(tmp_path)
  616. except (tarfile.ReadError, zipfile.BadZipfile):
  617. print('Error extracting package {}'.format(package.real_name))
  618. print()
  619. continue
  620. # Loading the package install info from the package
  621. try:
  622. package.load_setup()
  623. except ImportError as err:
  624. if 'buildutils' in err.message:
  625. print('This package needs buildutils')
  626. else:
  627. raise
  628. continue
  629. except AttributeError as error:
  630. print('Error: Could not install package {pkg}: {error}'.format(
  631. pkg=package.real_name, error=error))
  632. continue
  633. # Package requirement are an argument of the setup function
  634. req_not_found = package.get_requirements(pkg_folder)
  635. req_not_found = req_not_found.difference(packages)
  636. packages += req_not_found
  637. if req_not_found:
  638. print('Added packages \'{pkgs}\' as dependencies of {pkg}'
  639. .format(pkgs=", ".join(req_not_found),
  640. pkg=package.buildroot_name))
  641. print('Checking if package {name} already exists...'.format(
  642. name=package.pkg_dir))
  643. try:
  644. os.makedirs(package.pkg_dir)
  645. except OSError as exception:
  646. if exception.errno != errno.EEXIST:
  647. print("ERROR: ", exception.message, file=sys.stderr)
  648. continue
  649. print('Error: Package {name} already exists'
  650. .format(name=package.pkg_dir))
  651. del_pkg = input(
  652. 'Do you want to delete existing package ? [y/N]')
  653. if del_pkg.lower() == 'y':
  654. shutil.rmtree(package.pkg_dir)
  655. os.makedirs(package.pkg_dir)
  656. else:
  657. continue
  658. package.create_package_mk()
  659. package.create_hash_file()
  660. package.create_config_in()
  661. print("NOTE: Remember to also make an update to the DEVELOPERS file")
  662. print(" and include an entry for the pkg in packages/Config.in")
  663. print()
  664. # printing an empty line for visual confort
  665. finally:
  666. shutil.rmtree(tmp_path)
  667. if __name__ == "__main__":
  668. main()