pkg-stats 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170
  1. #!/usr/bin/env python3
  2. # Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation; either version 2 of the License, or
  7. # (at your option) any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  12. # General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program; if not, write to the Free Software
  16. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17. import aiohttp
  18. import argparse
  19. import asyncio
  20. import datetime
  21. import fnmatch
  22. import os
  23. from collections import defaultdict
  24. import re
  25. import subprocess
  26. import json
  27. import sys
  28. brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
  29. sys.path.append(os.path.join(brpath, "utils"))
  30. from getdeveloperlib import parse_developers # noqa: E402
  31. from cpedb import CPEDB # noqa: E402
  32. INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
  33. URL_RE = re.compile(r"\s*https?://\S*\s*$")
  34. RM_API_STATUS_ERROR = 1
  35. RM_API_STATUS_FOUND_BY_DISTRO = 2
  36. RM_API_STATUS_FOUND_BY_PATTERN = 3
  37. RM_API_STATUS_NOT_FOUND = 4
  38. class Defconfig:
  39. def __init__(self, name, path):
  40. self.name = name
  41. self.path = path
  42. self.developers = None
  43. def set_developers(self, developers):
  44. """
  45. Fills in the .developers field
  46. """
  47. self.developers = [
  48. developer.name
  49. for developer in developers
  50. if developer.hasfile(self.path)
  51. ]
  52. def get_defconfig_list():
  53. """
  54. Builds the list of Buildroot defconfigs, returning a list of Defconfig
  55. objects.
  56. """
  57. return [
  58. Defconfig(name[:-len('_defconfig')], os.path.join('configs', name))
  59. for name in os.listdir(os.path.join(brpath, 'configs'))
  60. if name.endswith('_defconfig')
  61. ]
  62. class Package:
  63. all_licenses = dict()
  64. all_license_files = list()
  65. all_versions = dict()
  66. all_ignored_cves = dict()
  67. all_cpeids = dict()
  68. # This is the list of all possible checks. Add new checks to this list so
  69. # a tool that post-processeds the json output knows the checks before
  70. # iterating over the packages.
  71. status_checks = ['cve', 'developers', 'hash', 'license',
  72. 'license-files', 'patches', 'pkg-check', 'url', 'version']
  73. def __init__(self, name, path):
  74. self.name = name
  75. self.path = path
  76. self.pkg_path = os.path.dirname(path)
  77. self.infras = None
  78. self.license = None
  79. self.has_license = False
  80. self.has_license_files = False
  81. self.has_hash = False
  82. self.patch_files = []
  83. self.warnings = 0
  84. self.current_version = None
  85. self.url = None
  86. self.url_worker = None
  87. self.cpeid = None
  88. self.cves = list()
  89. self.ignored_cves = list()
  90. self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
  91. self.status = {}
  92. def pkgvar(self):
  93. return self.name.upper().replace("-", "_")
  94. def set_url(self):
  95. """
  96. Fills in the .url field
  97. """
  98. self.status['url'] = ("warning", "no Config.in")
  99. pkgdir = os.path.dirname(os.path.join(brpath, self.path))
  100. for filename in os.listdir(pkgdir):
  101. if fnmatch.fnmatch(filename, 'Config.*'):
  102. fp = open(os.path.join(pkgdir, filename), "r")
  103. for config_line in fp:
  104. if URL_RE.match(config_line):
  105. self.url = config_line.strip()
  106. self.status['url'] = ("ok", "found")
  107. fp.close()
  108. return
  109. self.status['url'] = ("error", "missing")
  110. fp.close()
  111. @property
  112. def patch_count(self):
  113. return len(self.patch_files)
  114. @property
  115. def has_valid_infra(self):
  116. if self.infras is None:
  117. return False
  118. return len(self.infras) > 0
  119. @property
  120. def is_actual_package(self):
  121. try:
  122. if not self.has_valid_infra:
  123. return False
  124. if self.infras[0][1] == 'virtual':
  125. return False
  126. except IndexError:
  127. return False
  128. return True
  129. def set_infra(self):
  130. """
  131. Fills in the .infras field
  132. """
  133. self.infras = list()
  134. with open(os.path.join(brpath, self.path), 'r') as f:
  135. lines = f.readlines()
  136. for line in lines:
  137. match = INFRA_RE.match(line)
  138. if not match:
  139. continue
  140. infra = match.group(1)
  141. if infra.startswith("host-"):
  142. self.infras.append(("host", infra[5:]))
  143. else:
  144. self.infras.append(("target", infra))
  145. def set_license(self):
  146. """
  147. Fills in the .status['license'] and .status['license-files'] fields
  148. """
  149. if not self.is_actual_package:
  150. self.status['license'] = ("na", "no valid package infra")
  151. self.status['license-files'] = ("na", "no valid package infra")
  152. return
  153. var = self.pkgvar()
  154. self.status['license'] = ("error", "missing")
  155. self.status['license-files'] = ("error", "missing")
  156. if var in self.all_licenses:
  157. self.license = self.all_licenses[var]
  158. self.status['license'] = ("ok", "found")
  159. if var in self.all_license_files:
  160. self.status['license-files'] = ("ok", "found")
  161. def set_hash_info(self):
  162. """
  163. Fills in the .status['hash'] field
  164. """
  165. if not self.is_actual_package:
  166. self.status['hash'] = ("na", "no valid package infra")
  167. self.status['hash-license'] = ("na", "no valid package infra")
  168. return
  169. hashpath = self.path.replace(".mk", ".hash")
  170. if os.path.exists(os.path.join(brpath, hashpath)):
  171. self.status['hash'] = ("ok", "found")
  172. else:
  173. self.status['hash'] = ("error", "missing")
  174. def set_patch_count(self):
  175. """
  176. Fills in the .patch_count, .patch_files and .status['patches'] fields
  177. """
  178. if not self.is_actual_package:
  179. self.status['patches'] = ("na", "no valid package infra")
  180. return
  181. pkgdir = os.path.dirname(os.path.join(brpath, self.path))
  182. for subdir, _, _ in os.walk(pkgdir):
  183. self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch')
  184. if self.patch_count == 0:
  185. self.status['patches'] = ("ok", "no patches")
  186. elif self.patch_count < 5:
  187. self.status['patches'] = ("warning", "some patches")
  188. else:
  189. self.status['patches'] = ("error", "lots of patches")
  190. def set_current_version(self):
  191. """
  192. Fills in the .current_version field
  193. """
  194. var = self.pkgvar()
  195. if var in self.all_versions:
  196. self.current_version = self.all_versions[var]
  197. def set_cpeid(self):
  198. """
  199. Fills in the .cpeid field
  200. """
  201. var = self.pkgvar()
  202. if not self.is_actual_package:
  203. self.status['cpe'] = ("na", "N/A - virtual pkg")
  204. return
  205. if not self.current_version:
  206. self.status['cpe'] = ("na", "no version information available")
  207. return
  208. if var in self.all_cpeids:
  209. self.cpeid = self.all_cpeids[var]
  210. # Set a preliminary status, it might be overridden by check_package_cpes()
  211. self.status['cpe'] = ("warning", "not checked against CPE dictionnary")
  212. else:
  213. self.status['cpe'] = ("error", "no verified CPE identifier")
  214. def set_check_package_warnings(self):
  215. """
  216. Fills in the .warnings and .status['pkg-check'] fields
  217. """
  218. cmd = [os.path.join(brpath, "utils/check-package")]
  219. pkgdir = os.path.dirname(os.path.join(brpath, self.path))
  220. self.status['pkg-check'] = ("error", "Missing")
  221. for root, dirs, files in os.walk(pkgdir):
  222. for f in files:
  223. if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
  224. cmd.append(os.path.join(root, f))
  225. o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
  226. lines = o.splitlines()
  227. for line in lines:
  228. m = re.match("^([0-9]*) warnings generated", line.decode())
  229. if m:
  230. self.warnings = int(m.group(1))
  231. if self.warnings == 0:
  232. self.status['pkg-check'] = ("ok", "no warnings")
  233. else:
  234. self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings))
  235. return
  236. def set_ignored_cves(self):
  237. """
  238. Give the list of CVEs ignored by the package
  239. """
  240. self.ignored_cves = list(self.all_ignored_cves.get(self.pkgvar(), []))
  241. def set_developers(self, developers):
  242. """
  243. Fills in the .developers and .status['developers'] field
  244. """
  245. self.developers = [
  246. dev.name
  247. for dev in developers
  248. if dev.hasfile(self.path)
  249. ]
  250. if self.developers:
  251. self.status['developers'] = ("ok", "{} developers".format(len(self.developers)))
  252. else:
  253. self.status['developers'] = ("warning", "no developers")
  254. def is_status_ok(self, name):
  255. return name in self.status and self.status[name][0] == 'ok'
  256. def is_status_error(self, name):
  257. return name in self.status and self.status[name][0] == 'error'
  258. def is_status_na(self, name):
  259. return name in self.status and self.status[name][0] == 'na'
  260. def __eq__(self, other):
  261. return self.path == other.path
  262. def __lt__(self, other):
  263. return self.path < other.path
  264. def __str__(self):
  265. return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
  266. (self.name, self.path, self.is_status_ok('license'),
  267. self.is_status_ok('license-files'), self.status['hash'], self.patch_count)
  268. def get_pkglist(npackages, package_list):
  269. """
  270. Builds the list of Buildroot packages, returning a list of Package
  271. objects. Only the .name and .path fields of the Package object are
  272. initialized.
  273. npackages: limit to N packages
  274. package_list: limit to those packages in this list
  275. """
  276. WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"]
  277. WALK_EXCLUDES = ["boot/common.mk",
  278. "linux/linux-ext-.*.mk",
  279. "package/freescale-imx/freescale-imx.mk",
  280. "package/gcc/gcc.mk",
  281. "package/gstreamer/gstreamer.mk",
  282. "package/gstreamer1/gstreamer1.mk",
  283. "package/gtk2-themes/gtk2-themes.mk",
  284. "package/matchbox/matchbox.mk",
  285. "package/opengl/opengl.mk",
  286. "package/qt5/qt5.mk",
  287. "package/x11r7/x11r7.mk",
  288. "package/doc-asciidoc.mk",
  289. "package/pkg-.*.mk",
  290. "toolchain/toolchain-external/pkg-toolchain-external.mk",
  291. "toolchain/toolchain-external/toolchain-external.mk",
  292. "toolchain/toolchain.mk",
  293. "toolchain/helpers.mk",
  294. "toolchain/toolchain-wrapper.mk"]
  295. packages = list()
  296. count = 0
  297. for root, dirs, files in os.walk(brpath):
  298. root = os.path.relpath(root, brpath)
  299. rootdir = root.split("/")
  300. if len(rootdir) < 1:
  301. continue
  302. if rootdir[0] not in WALK_USEFUL_SUBDIRS:
  303. continue
  304. for f in files:
  305. if not f.endswith(".mk"):
  306. continue
  307. # Strip ending ".mk"
  308. pkgname = f[:-3]
  309. if package_list and pkgname not in package_list:
  310. continue
  311. pkgpath = os.path.join(root, f)
  312. skip = False
  313. for exclude in WALK_EXCLUDES:
  314. if re.match(exclude, pkgpath):
  315. skip = True
  316. continue
  317. if skip:
  318. continue
  319. p = Package(pkgname, pkgpath)
  320. packages.append(p)
  321. count += 1
  322. if npackages and count == npackages:
  323. return packages
  324. return packages
  325. def get_config_packages():
  326. cmd = ["make", "--no-print-directory", "show-info"]
  327. js = json.loads(subprocess.check_output(cmd))
  328. return set([v["name"] for v in js.values()])
  329. def package_init_make_info():
  330. # Fetch all variables at once
  331. variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
  332. "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"])
  333. variable_list = variables.decode().splitlines()
  334. # We process first the host package VERSION, and then the target
  335. # package VERSION. This means that if a package exists in both
  336. # target and host variants, with different values (eg. version
  337. # numbers (unlikely)), we'll report the target one.
  338. variable_list = [x[5:] for x in variable_list if x.startswith("HOST_")] + \
  339. [x for x in variable_list if not x.startswith("HOST_")]
  340. for item in variable_list:
  341. # Get variable name and value
  342. pkgvar, value = item.split("=", maxsplit=1)
  343. # Strip the suffix according to the variable
  344. if pkgvar.endswith("_LICENSE"):
  345. # If value is "unknown", no license details available
  346. if value == "unknown":
  347. continue
  348. pkgvar = pkgvar[:-8]
  349. Package.all_licenses[pkgvar] = value
  350. elif pkgvar.endswith("_LICENSE_FILES"):
  351. if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
  352. continue
  353. pkgvar = pkgvar[:-14]
  354. Package.all_license_files.append(pkgvar)
  355. elif pkgvar.endswith("_VERSION"):
  356. if pkgvar.endswith("_DL_VERSION"):
  357. continue
  358. pkgvar = pkgvar[:-8]
  359. Package.all_versions[pkgvar] = value
  360. elif pkgvar.endswith("_IGNORE_CVES"):
  361. pkgvar = pkgvar[:-12]
  362. Package.all_ignored_cves[pkgvar] = value.split()
  363. elif pkgvar.endswith("_CPE_ID"):
  364. pkgvar = pkgvar[:-7]
  365. Package.all_cpeids[pkgvar] = value
  366. check_url_count = 0
  367. async def check_url_status(session, pkg, npkgs, retry=True):
  368. global check_url_count
  369. try:
  370. async with session.get(pkg.url) as resp:
  371. if resp.status >= 400:
  372. pkg.status['url'] = ("error", "invalid {}".format(resp.status))
  373. check_url_count += 1
  374. print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
  375. return
  376. except (aiohttp.ClientError, asyncio.TimeoutError):
  377. if retry:
  378. return await check_url_status(session, pkg, npkgs, retry=False)
  379. else:
  380. pkg.status['url'] = ("error", "invalid (err)")
  381. check_url_count += 1
  382. print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
  383. return
  384. pkg.status['url'] = ("ok", "valid")
  385. check_url_count += 1
  386. print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name))
  387. async def check_package_urls(packages):
  388. tasks = []
  389. connector = aiohttp.TCPConnector(limit_per_host=5)
  390. async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
  391. packages = [p for p in packages if p.status['url'][0] == 'ok']
  392. for pkg in packages:
  393. tasks.append(asyncio.ensure_future(check_url_status(sess, pkg, len(packages))))
  394. await asyncio.wait(tasks)
  395. def check_package_latest_version_set_status(pkg, status, version, identifier):
  396. pkg.latest_version = {
  397. "status": status,
  398. "version": version,
  399. "id": identifier,
  400. }
  401. if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
  402. pkg.status['version'] = ('warning', "Release Monitoring API error")
  403. elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND:
  404. pkg.status['version'] = ('warning', "Package not found on Release Monitoring")
  405. if pkg.latest_version['version'] is None:
  406. pkg.status['version'] = ('warning', "No upstream version available on Release Monitoring")
  407. elif pkg.latest_version['version'] != pkg.current_version:
  408. pkg.status['version'] = ('error', "The newer version {} is available upstream".format(pkg.latest_version['version']))
  409. else:
  410. pkg.status['version'] = ('ok', 'up-to-date')
  411. async def check_package_get_latest_version_by_distro(session, pkg, retry=True):
  412. url = "https://release-monitoring.org//api/project/Buildroot/%s" % pkg.name
  413. try:
  414. async with session.get(url) as resp:
  415. if resp.status != 200:
  416. return False
  417. data = await resp.json()
  418. version = data['stable_versions'][0] if 'stable_versions' in data else data['version'] if 'version' in data else None
  419. check_package_latest_version_set_status(pkg,
  420. RM_API_STATUS_FOUND_BY_DISTRO,
  421. version,
  422. data['id'])
  423. return True
  424. except (aiohttp.ClientError, asyncio.TimeoutError):
  425. if retry:
  426. return await check_package_get_latest_version_by_distro(session, pkg, retry=False)
  427. else:
  428. return False
  429. async def check_package_get_latest_version_by_guess(session, pkg, retry=True):
  430. url = "https://release-monitoring.org/api/projects/?pattern=%s" % pkg.name
  431. try:
  432. async with session.get(url) as resp:
  433. if resp.status != 200:
  434. return False
  435. data = await resp.json()
  436. # filter projects that have the right name and a version defined
  437. projects = [p for p in data['projects'] if p['name'] == pkg.name and 'stable_versions' in p]
  438. projects.sort(key=lambda x: x['id'])
  439. if len(projects) > 0:
  440. check_package_latest_version_set_status(pkg,
  441. RM_API_STATUS_FOUND_BY_PATTERN,
  442. projects[0]['stable_versions'][0],
  443. projects[0]['id'])
  444. return True
  445. except (aiohttp.ClientError, asyncio.TimeoutError):
  446. if retry:
  447. return await check_package_get_latest_version_by_guess(session, pkg, retry=False)
  448. else:
  449. return False
  450. check_latest_count = 0
  451. async def check_package_latest_version_get(session, pkg, npkgs):
  452. global check_latest_count
  453. if await check_package_get_latest_version_by_distro(session, pkg):
  454. check_latest_count += 1
  455. print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
  456. return
  457. if await check_package_get_latest_version_by_guess(session, pkg):
  458. check_latest_count += 1
  459. print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
  460. return
  461. check_package_latest_version_set_status(pkg,
  462. RM_API_STATUS_NOT_FOUND,
  463. None, None)
  464. check_latest_count += 1
  465. print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name))
  466. async def check_package_latest_version(packages):
  467. """
  468. Fills in the .latest_version field of all Package objects
  469. This field is a dict and has the following keys:
  470. - status: one of RM_API_STATUS_ERROR,
  471. RM_API_STATUS_FOUND_BY_DISTRO, RM_API_STATUS_FOUND_BY_PATTERN,
  472. RM_API_STATUS_NOT_FOUND
  473. - version: string containing the latest version known by
  474. release-monitoring.org for this package
  475. - id: string containing the id of the project corresponding to this
  476. package, as known by release-monitoring.org
  477. """
  478. for pkg in [p for p in packages if not p.is_actual_package]:
  479. pkg.status['version'] = ("na", "no valid package infra")
  480. tasks = []
  481. connector = aiohttp.TCPConnector(limit_per_host=5)
  482. async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
  483. packages = [p for p in packages if p.is_actual_package]
  484. for pkg in packages:
  485. tasks.append(asyncio.ensure_future(check_package_latest_version_get(sess, pkg, len(packages))))
  486. await asyncio.wait(tasks)
  487. def check_package_cve_affects(cve, cpe_product_pkgs):
  488. for product in cve.affected_products:
  489. if product not in cpe_product_pkgs:
  490. continue
  491. for pkg in cpe_product_pkgs[product]:
  492. if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS:
  493. pkg.cves.append(cve.identifier)
  494. def check_package_cves(nvd_path, packages):
  495. if not os.path.isdir(nvd_path):
  496. os.makedirs(nvd_path)
  497. cpe_product_pkgs = defaultdict(list)
  498. for pkg in packages:
  499. if not pkg.is_actual_package:
  500. pkg.status['cve'] = ("na", "N/A")
  501. continue
  502. if not pkg.current_version:
  503. pkg.status['cve'] = ("na", "no version information available")
  504. continue
  505. if pkg.cpeid:
  506. cpe_product = cvecheck.cpe_product(pkg.cpeid)
  507. cpe_product_pkgs[cpe_product].append(pkg)
  508. else:
  509. cpe_product_pkgs[pkg.name].append(pkg)
  510. for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
  511. check_package_cve_affects(cve, cpe_product_pkgs)
  512. for pkg in packages:
  513. if 'cve' not in pkg.status:
  514. if pkg.cves:
  515. pkg.status['cve'] = ("error", "affected by CVEs")
  516. else:
  517. pkg.status['cve'] = ("ok", "not affected by CVEs")
  518. def check_package_cpes(nvd_path, packages):
  519. cpedb = CPEDB(nvd_path)
  520. cpedb.get_xml_dict()
  521. for p in packages:
  522. if not p.cpeid:
  523. continue
  524. if cpedb.find(p.cpeid):
  525. p.status['cpe'] = ("ok", "verified CPE identifier")
  526. else:
  527. p.status['cpe'] = ("error", "CPE version unknown in CPE database")
  528. def calculate_stats(packages):
  529. stats = defaultdict(int)
  530. stats['packages'] = len(packages)
  531. for pkg in packages:
  532. # If packages have multiple infra, take the first one. For the
  533. # vast majority of packages, the target and host infra are the
  534. # same. There are very few packages that use a different infra
  535. # for the host and target variants.
  536. if len(pkg.infras) > 0:
  537. infra = pkg.infras[0][1]
  538. stats["infra-%s" % infra] += 1
  539. else:
  540. stats["infra-unknown"] += 1
  541. if pkg.is_status_ok('license'):
  542. stats["license"] += 1
  543. else:
  544. stats["no-license"] += 1
  545. if pkg.is_status_ok('license-files'):
  546. stats["license-files"] += 1
  547. else:
  548. stats["no-license-files"] += 1
  549. if pkg.is_status_ok('hash'):
  550. stats["hash"] += 1
  551. else:
  552. stats["no-hash"] += 1
  553. if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO:
  554. stats["rmo-mapping"] += 1
  555. else:
  556. stats["rmo-no-mapping"] += 1
  557. if not pkg.latest_version['version']:
  558. stats["version-unknown"] += 1
  559. elif pkg.latest_version['version'] == pkg.current_version:
  560. stats["version-uptodate"] += 1
  561. else:
  562. stats["version-not-uptodate"] += 1
  563. stats["patches"] += pkg.patch_count
  564. stats["total-cves"] += len(pkg.cves)
  565. if len(pkg.cves) != 0:
  566. stats["pkg-cves"] += 1
  567. if pkg.cpeid:
  568. stats["cpe-id"] += 1
  569. else:
  570. stats["no-cpe-id"] += 1
  571. return stats
  572. html_header = """
  573. <head>
  574. <script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
  575. <style type=\"text/css\">
  576. table {
  577. width: 100%;
  578. }
  579. td {
  580. border: 1px solid black;
  581. }
  582. td.centered {
  583. text-align: center;
  584. }
  585. td.wrong {
  586. background: #ff9a69;
  587. }
  588. td.correct {
  589. background: #d2ffc4;
  590. }
  591. td.nopatches {
  592. background: #d2ffc4;
  593. }
  594. td.somepatches {
  595. background: #ffd870;
  596. }
  597. td.lotsofpatches {
  598. background: #ff9a69;
  599. }
  600. td.good_url {
  601. background: #d2ffc4;
  602. }
  603. td.missing_url {
  604. background: #ffd870;
  605. }
  606. td.invalid_url {
  607. background: #ff9a69;
  608. }
  609. td.version-good {
  610. background: #d2ffc4;
  611. }
  612. td.version-needs-update {
  613. background: #ff9a69;
  614. }
  615. td.version-unknown {
  616. background: #ffd870;
  617. }
  618. td.version-error {
  619. background: #ccc;
  620. }
  621. td.cpe-ok {
  622. background: #d2ffc4;
  623. }
  624. td.cpe-nok {
  625. background: #ff9a69;
  626. }
  627. td.cpe-unknown {
  628. background: #ffd870;
  629. }
  630. td.cve-ok {
  631. background: #d2ffc4;
  632. }
  633. td.cve-nok {
  634. background: #ff9a69;
  635. }
  636. td.cve-unknown {
  637. background: #ffd870;
  638. }
  639. td.cve_ignored {
  640. background: #ccc;
  641. }
  642. </style>
  643. <title>Statistics of Buildroot packages</title>
  644. </head>
  645. <a href=\"#results\">Results</a><br/>
  646. <p id=\"sortable_hint\"></p>
  647. """
  648. html_footer = """
  649. </body>
  650. <script>
  651. if (typeof sorttable === \"object\") {
  652. document.getElementById(\"sortable_hint\").innerHTML =
  653. \"hint: the table can be sorted by clicking the column headers\"
  654. }
  655. </script>
  656. </html>
  657. """
  658. def infra_str(infra_list):
  659. if not infra_list:
  660. return "Unknown"
  661. elif len(infra_list) == 1:
  662. return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
  663. elif infra_list[0][1] == infra_list[1][1]:
  664. return "<b>%s</b><br/>%s + %s" % \
  665. (infra_list[0][1], infra_list[0][0], infra_list[1][0])
  666. else:
  667. return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
  668. (infra_list[0][1], infra_list[0][0],
  669. infra_list[1][1], infra_list[1][0])
  670. def boolean_str(b):
  671. if b:
  672. return "Yes"
  673. else:
  674. return "No"
  675. def dump_html_pkg(f, pkg):
  676. f.write(" <tr>\n")
  677. f.write(" <td>%s</td>\n" % pkg.path)
  678. # Patch count
  679. td_class = ["centered"]
  680. if pkg.patch_count == 0:
  681. td_class.append("nopatches")
  682. elif pkg.patch_count < 5:
  683. td_class.append("somepatches")
  684. else:
  685. td_class.append("lotsofpatches")
  686. f.write(" <td class=\"%s\">%s</td>\n" %
  687. (" ".join(td_class), str(pkg.patch_count)))
  688. # Infrastructure
  689. infra = infra_str(pkg.infras)
  690. td_class = ["centered"]
  691. if infra == "Unknown":
  692. td_class.append("wrong")
  693. else:
  694. td_class.append("correct")
  695. f.write(" <td class=\"%s\">%s</td>\n" %
  696. (" ".join(td_class), infra_str(pkg.infras)))
  697. # License
  698. td_class = ["centered"]
  699. if pkg.is_status_ok('license'):
  700. td_class.append("correct")
  701. else:
  702. td_class.append("wrong")
  703. f.write(" <td class=\"%s\">%s</td>\n" %
  704. (" ".join(td_class), boolean_str(pkg.is_status_ok('license'))))
  705. # License files
  706. td_class = ["centered"]
  707. if pkg.is_status_ok('license-files'):
  708. td_class.append("correct")
  709. else:
  710. td_class.append("wrong")
  711. f.write(" <td class=\"%s\">%s</td>\n" %
  712. (" ".join(td_class), boolean_str(pkg.is_status_ok('license-files'))))
  713. # Hash
  714. td_class = ["centered"]
  715. if pkg.is_status_ok('hash'):
  716. td_class.append("correct")
  717. else:
  718. td_class.append("wrong")
  719. f.write(" <td class=\"%s\">%s</td>\n" %
  720. (" ".join(td_class), boolean_str(pkg.is_status_ok('hash'))))
  721. # Current version
  722. if len(pkg.current_version) > 20:
  723. current_version = pkg.current_version[:20] + "..."
  724. else:
  725. current_version = pkg.current_version
  726. f.write(" <td class=\"centered\">%s</td>\n" % current_version)
  727. # Latest version
  728. if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
  729. td_class.append("version-error")
  730. if pkg.latest_version['version'] is None:
  731. td_class.append("version-unknown")
  732. elif pkg.latest_version['version'] != pkg.current_version:
  733. td_class.append("version-needs-update")
  734. else:
  735. td_class.append("version-good")
  736. if pkg.latest_version['status'] == RM_API_STATUS_ERROR:
  737. latest_version_text = "<b>Error</b>"
  738. elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND:
  739. latest_version_text = "<b>Not found</b>"
  740. else:
  741. if pkg.latest_version['version'] is None:
  742. latest_version_text = "<b>Found, but no version</b>"
  743. else:
  744. latest_version_text = "<a href=\"https://release-monitoring.org/project/%s\"><b>%s</b></a>" % \
  745. (pkg.latest_version['id'], str(pkg.latest_version['version']))
  746. latest_version_text += "<br/>"
  747. if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO:
  748. latest_version_text += "found by <a href=\"https://release-monitoring.org/distro/Buildroot/\">distro</a>"
  749. else:
  750. latest_version_text += "found by guess"
  751. f.write(" <td class=\"%s\">%s</td>\n" %
  752. (" ".join(td_class), latest_version_text))
  753. # Warnings
  754. td_class = ["centered"]
  755. if pkg.warnings == 0:
  756. td_class.append("correct")
  757. else:
  758. td_class.append("wrong")
  759. f.write(" <td class=\"%s\">%d</td>\n" %
  760. (" ".join(td_class), pkg.warnings))
  761. # URL status
  762. td_class = ["centered"]
  763. url_str = pkg.status['url'][1]
  764. if pkg.status['url'][0] in ("error", "warning"):
  765. td_class.append("missing_url")
  766. if pkg.status['url'][0] == "error":
  767. td_class.append("invalid_url")
  768. url_str = "<a href=%s>%s</a>" % (pkg.url, pkg.status['url'][1])
  769. else:
  770. td_class.append("good_url")
  771. url_str = "<a href=%s>Link</a>" % pkg.url
  772. f.write(" <td class=\"%s\">%s</td>\n" %
  773. (" ".join(td_class), url_str))
  774. # CVEs
  775. td_class = ["centered"]
  776. if pkg.is_status_ok("cve"):
  777. td_class.append("cve-ok")
  778. elif pkg.is_status_error("cve"):
  779. td_class.append("cve-nok")
  780. elif pkg.is_status_na("cve") and not pkg.is_actual_package:
  781. td_class.append("cve-ok")
  782. else:
  783. td_class.append("cve-unknown")
  784. f.write(" <td class=\"%s\">\n" % " ".join(td_class))
  785. if pkg.is_status_error("cve"):
  786. for cve in pkg.cves:
  787. f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
  788. elif pkg.is_status_na("cve"):
  789. f.write(" %s" % pkg.status['cve'][1])
  790. else:
  791. f.write(" N/A\n")
  792. f.write(" </td>\n")
  793. # CVEs Ignored
  794. td_class = ["centered"]
  795. if pkg.ignored_cves:
  796. td_class.append("cve_ignored")
  797. f.write(" <td class=\"%s\">\n" % " ".join(td_class))
  798. for ignored_cve in pkg.ignored_cves:
  799. f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (ignored_cve, ignored_cve))
  800. f.write(" </td>\n")
  801. # CPE ID
  802. td_class = ["left"]
  803. if pkg.is_status_ok("cpe"):
  804. td_class.append("cpe-ok")
  805. elif pkg.is_status_error("cpe"):
  806. td_class.append("cpe-nok")
  807. elif pkg.is_status_na("cpe") and not pkg.is_actual_package:
  808. td_class.append("cpe-ok")
  809. else:
  810. td_class.append("cpe-unknown")
  811. f.write(" <td class=\"%s\">\n" % " ".join(td_class))
  812. if pkg.cpeid:
  813. f.write(" <code>%s</code>\n" % pkg.cpeid)
  814. if not pkg.is_status_ok("cpe"):
  815. if pkg.is_actual_package and pkg.current_version:
  816. if pkg.cpeid:
  817. f.write(" <br/>%s <a href=\"https://nvd.nist.gov/products/cpe/search/results?namingFormat=2.3&keyword=%s\">(Search)</a>\n" % # noqa: E501
  818. (pkg.status['cpe'][1], ":".join(pkg.cpeid.split(":")[0:5])))
  819. else:
  820. f.write(" %s <a href=\"https://nvd.nist.gov/products/cpe/search/results?namingFormat=2.3&keyword=%s\">(Search)</a>\n" % # noqa: E501
  821. (pkg.status['cpe'][1], pkg.name))
  822. else:
  823. f.write(" %s\n" % pkg.status['cpe'][1])
  824. f.write(" </td>\n")
  825. f.write(" </tr>\n")
  826. def dump_html_all_pkgs(f, packages):
  827. f.write("""
  828. <table class=\"sortable\">
  829. <tr>
  830. <td>Package</td>
  831. <td class=\"centered\">Patch count</td>
  832. <td class=\"centered\">Infrastructure</td>
  833. <td class=\"centered\">License</td>
  834. <td class=\"centered\">License files</td>
  835. <td class=\"centered\">Hash file</td>
  836. <td class=\"centered\">Current version</td>
  837. <td class=\"centered\">Latest version</td>
  838. <td class=\"centered\">Warnings</td>
  839. <td class=\"centered\">Upstream URL</td>
  840. <td class=\"centered\">CVEs</td>
  841. <td class=\"centered\">CVEs Ignored</td>
  842. <td class=\"centered\">CPE ID</td>
  843. </tr>
  844. """)
  845. for pkg in sorted(packages):
  846. dump_html_pkg(f, pkg)
  847. f.write("</table>")
  848. def dump_html_stats(f, stats):
  849. f.write("<a id=\"results\"></a>\n")
  850. f.write("<table>\n")
  851. infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")]
  852. for infra in infras:
  853. f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" %
  854. (infra, stats["infra-%s" % infra]))
  855. f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" %
  856. stats["license"])
  857. f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" %
  858. stats["no-license"])
  859. f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" %
  860. stats["license-files"])
  861. f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" %
  862. stats["no-license-files"])
  863. f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" %
  864. stats["hash"])
  865. f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" %
  866. stats["no-hash"])
  867. f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" %
  868. stats["patches"])
  869. f.write("<tr><td>Packages having a mapping on <i>release-monitoring.org</i></td><td>%s</td></tr>\n" %
  870. stats["rmo-mapping"])
  871. f.write("<tr><td>Packages lacking a mapping on <i>release-monitoring.org</i></td><td>%s</td></tr>\n" %
  872. stats["rmo-no-mapping"])
  873. f.write("<tr><td>Packages that are up-to-date</td><td>%s</td></tr>\n" %
  874. stats["version-uptodate"])
  875. f.write("<tr><td>Packages that are not up-to-date</td><td>%s</td></tr>\n" %
  876. stats["version-not-uptodate"])
  877. f.write("<tr><td>Packages with no known upstream version</td><td>%s</td></tr>\n" %
  878. stats["version-unknown"])
  879. f.write("<tr><td>Packages affected by CVEs</td><td>%s</td></tr>\n" %
  880. stats["pkg-cves"])
  881. f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
  882. stats["total-cves"])
  883. f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" %
  884. stats["cpe-id"])
  885. f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" %
  886. stats["no-cpe-id"])
  887. f.write("</table>\n")
  888. def dump_html_gen_info(f, date, commit):
  889. # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
  890. f.write("<p><i>Updated on %s, git commit %s</i></p>\n" % (str(date), commit))
  891. def dump_html(packages, stats, date, commit, output):
  892. with open(output, 'w') as f:
  893. f.write(html_header)
  894. dump_html_all_pkgs(f, packages)
  895. dump_html_stats(f, stats)
  896. dump_html_gen_info(f, date, commit)
  897. f.write(html_footer)
  898. def dump_json(packages, defconfigs, stats, date, commit, output):
  899. # Format packages as a dictionnary instead of a list
  900. # Exclude local field that does not contains real date
  901. excluded_fields = ['url_worker', 'name']
  902. pkgs = {
  903. pkg.name: {
  904. k: v
  905. for k, v in pkg.__dict__.items()
  906. if k not in excluded_fields
  907. } for pkg in packages
  908. }
  909. defconfigs = {
  910. d.name: {
  911. k: v
  912. for k, v in d.__dict__.items()
  913. } for d in defconfigs
  914. }
  915. # Aggregate infrastructures into a single dict entry
  916. statistics = {
  917. k: v
  918. for k, v in stats.items()
  919. if not k.startswith('infra-')
  920. }
  921. statistics['infra'] = {k[6:]: v for k, v in stats.items() if k.startswith('infra-')}
  922. # The actual structure to dump, add commit and date to it
  923. final = {'packages': pkgs,
  924. 'stats': statistics,
  925. 'defconfigs': defconfigs,
  926. 'package_status_checks': Package.status_checks,
  927. 'commit': commit,
  928. 'date': str(date)}
  929. with open(output, 'w') as f:
  930. json.dump(final, f, indent=2, separators=(',', ': '))
  931. f.write('\n')
  932. def resolvepath(path):
  933. return os.path.abspath(os.path.expanduser(path))
  934. def parse_args():
  935. parser = argparse.ArgumentParser()
  936. output = parser.add_argument_group('output', 'Output file(s)')
  937. output.add_argument('--html', dest='html', type=resolvepath,
  938. help='HTML output file')
  939. output.add_argument('--json', dest='json', type=resolvepath,
  940. help='JSON output file')
  941. packages = parser.add_mutually_exclusive_group()
  942. packages.add_argument('-c', dest='configpackages', action='store_true',
  943. help='Apply to packages enabled in current configuration')
  944. packages.add_argument('-n', dest='npackages', type=int, action='store',
  945. help='Number of packages')
  946. packages.add_argument('-p', dest='packages', action='store',
  947. help='List of packages (comma separated)')
  948. parser.add_argument('--nvd-path', dest='nvd_path',
  949. help='Path to the local NVD database', type=resolvepath)
  950. args = parser.parse_args()
  951. if not args.html and not args.json:
  952. parser.error('at least one of --html or --json (or both) is required')
  953. return args
  954. def __main__():
  955. global cvecheck
  956. args = parse_args()
  957. if args.nvd_path:
  958. import cve as cvecheck
  959. if args.packages:
  960. package_list = args.packages.split(",")
  961. elif args.configpackages:
  962. package_list = get_config_packages()
  963. else:
  964. package_list = None
  965. date = datetime.datetime.utcnow()
  966. commit = subprocess.check_output(['git', '-C', brpath,
  967. 'rev-parse',
  968. 'HEAD']).splitlines()[0].decode()
  969. print("Build package list ...")
  970. packages = get_pkglist(args.npackages, package_list)
  971. print("Getting developers ...")
  972. developers = parse_developers()
  973. print("Build defconfig list ...")
  974. defconfigs = get_defconfig_list()
  975. for d in defconfigs:
  976. d.set_developers(developers)
  977. print("Getting package make info ...")
  978. package_init_make_info()
  979. print("Getting package details ...")
  980. for pkg in packages:
  981. pkg.set_infra()
  982. pkg.set_license()
  983. pkg.set_hash_info()
  984. pkg.set_patch_count()
  985. pkg.set_check_package_warnings()
  986. pkg.set_current_version()
  987. pkg.set_cpeid()
  988. pkg.set_url()
  989. pkg.set_ignored_cves()
  990. pkg.set_developers(developers)
  991. print("Checking URL status")
  992. loop = asyncio.get_event_loop()
  993. loop.run_until_complete(check_package_urls(packages))
  994. print("Getting latest versions ...")
  995. loop = asyncio.get_event_loop()
  996. loop.run_until_complete(check_package_latest_version(packages))
  997. if args.nvd_path:
  998. print("Checking packages CVEs")
  999. check_package_cves(args.nvd_path, packages)
  1000. check_package_cpes(args.nvd_path, packages)
  1001. print("Calculate stats")
  1002. stats = calculate_stats(packages)
  1003. if args.html:
  1004. print("Write HTML")
  1005. dump_html(packages, stats, date, commit, args.html)
  1006. if args.json:
  1007. print("Write JSON")
  1008. dump_json(packages, defconfigs, stats, date, commit, args.json)
  1009. __main__()