cpedb.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174
  1. #!/usr/bin/env python3
  2. import xml.etree.ElementTree as ET
  3. from xml.etree.ElementTree import Element, SubElement
  4. import gzip
  5. import os
  6. import requests
  7. import time
  8. from xml.dom import minidom
  9. VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
  10. CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
  11. ns = {
  12. '': 'http://cpe.mitre.org/dictionary/2.0',
  13. 'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3',
  14. 'xml': 'http://www.w3.org/XML/1998/namespace'
  15. }
  16. class CPE:
  17. def __init__(self, cpe_str, titles, refs):
  18. self.cpe_str = cpe_str
  19. self.titles = titles
  20. self.references = refs
  21. self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6])
  22. def update_xml_dict(self):
  23. ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0')
  24. cpes = Element('cpe-list')
  25. cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3")
  26. cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1")
  27. cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3")
  28. cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1")
  29. cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
  30. cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2")
  31. cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3",
  32. "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd",
  33. "http://cpe.mitre.org/dictionary/2.0",
  34. "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd",
  35. "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
  36. "https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd",
  37. "http://scap.nist.gov/schema/scap-core/0.3",
  38. "https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd",
  39. "http://scap.nist.gov/schema/configuration/0.1",
  40. "https://scap.nist.gov/schema/nvd/configuration_0.1.xsd",
  41. "http://scap.nist.gov/schema/scap-core/0.1",
  42. "https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"]))
  43. item = SubElement(cpes, 'cpe-item')
  44. cpe_short_name = CPE.short_name(self.cpe_str)
  45. cpe_new_ver = CPE.version_update(self.cpe_str)
  46. item.set('name', 'cpe:/' + cpe_short_name)
  47. self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver)
  48. for title in self.titles:
  49. item.append(title)
  50. if self.references:
  51. item.append(self.references)
  52. cpe23item = SubElement(item, 'cpe-23:cpe23-item')
  53. cpe23item.set('name', self.cpe_str)
  54. # Generate the XML as a string
  55. xmlstr = ET.tostring(cpes)
  56. # And use minidom to pretty print the XML
  57. return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8")
  58. @staticmethod
  59. def version(cpe):
  60. return cpe.split(":")[5]
  61. @staticmethod
  62. def product(cpe):
  63. return cpe.split(":")[4]
  64. @staticmethod
  65. def short_name(cpe):
  66. return ":".join(cpe.split(":")[2:6])
  67. @staticmethod
  68. def version_update(cpe):
  69. return ":".join(cpe.split(":")[5:6])
  70. @staticmethod
  71. def no_version(cpe):
  72. return ":".join(cpe.split(":")[:5])
  73. class CPEDB:
  74. def __init__(self, nvd_path):
  75. self.all_cpes = dict()
  76. self.all_cpes_no_version = dict()
  77. self.nvd_path = nvd_path
  78. def get_xml_dict(self):
  79. print("CPE: Setting up NIST dictionary")
  80. if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
  81. os.makedirs(os.path.join(self.nvd_path, "cpe"))
  82. cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
  83. if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
  84. print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
  85. cpe_dict = requests.get(CPEDB_URL)
  86. open(cpe_dict_local, "wb").write(cpe_dict.content)
  87. print("CPE: Unzipping xml manifest...")
  88. nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
  89. print("CPE: Converting xml manifest to dict...")
  90. tree = ET.parse(nist_cpe_file)
  91. all_cpedb = tree.getroot()
  92. self.parse_dict(all_cpedb)
  93. def parse_dict(self, all_cpedb):
  94. # Cycle through the dict and build two dict to be used for custom
  95. # lookups of partial and complete CPE objects
  96. # The objects are then used to create new proposed XML updates if
  97. # if is determined one is required
  98. # Out of the different language titles, select English
  99. for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
  100. cpe_titles = []
  101. for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns):
  102. title.tail = None
  103. cpe_titles.append(title)
  104. # Some older CPE don't include references, if they do, make
  105. # sure we handle the case of one ref needing to be packed
  106. # in a list
  107. cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references")
  108. if cpe_ref:
  109. for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"):
  110. ref.tail = None
  111. ref.text = ref.text.upper()
  112. if ref.text not in VALID_REFS:
  113. ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501
  114. cpe_ref.tail = None
  115. cpe_ref.text = None
  116. cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name')
  117. item = CPE(cpe_str, cpe_titles, cpe_ref)
  118. cpe_str_no_version = CPE.no_version(cpe_str)
  119. # This dict must have a unique key for every CPE version
  120. # which allows matching to the specific obj data of that
  121. # NIST dict entry
  122. self.all_cpes.update({cpe_str: item})
  123. # This dict has one entry for every CPE (w/o version) to allow
  124. # partial match (no valid version) check (the obj is saved and
  125. # used as seed for suggested xml updates. By updating the same
  126. # non-version'd entry, it assumes the last update here is the
  127. # latest version in the NIST dict)
  128. self.all_cpes_no_version.update({cpe_str_no_version: item})
  129. def find_partial(self, cpe_str):
  130. cpe_str_no_version = CPE.no_version(cpe_str)
  131. if cpe_str_no_version in self.all_cpes_no_version:
  132. return cpe_str_no_version
  133. def find_partial_obj(self, cpe_str):
  134. cpe_str_no_version = CPE.no_version(cpe_str)
  135. if cpe_str_no_version in self.all_cpes_no_version:
  136. return self.all_cpes_no_version[cpe_str_no_version]
  137. def find_partial_latest_version(self, cpe_str_partial):
  138. cpe_obj = self.find_partial_obj(cpe_str_partial)
  139. return cpe_obj.cpe_cur_ver
  140. def find(self, cpe_str):
  141. if self.find_partial(cpe_str):
  142. if cpe_str in self.all_cpes:
  143. return cpe_str
  144. def gen_update_xml(self, cpe_str):
  145. cpe = self.find_partial_obj(cpe_str)
  146. return cpe.update_xml_dict()