|
@@ -17,7 +17,7 @@ python do_populate_cve_db() {
|
|
|
Update NVD database with json data feed
|
|
|
"""
|
|
|
|
|
|
- import sqlite3, urllib, shutil, gzip
|
|
|
+ import sqlite3, urllib, urllib.parse, shutil, gzip
|
|
|
from datetime import date
|
|
|
|
|
|
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
|
|
@@ -27,6 +27,16 @@ python do_populate_cve_db() {
|
|
|
db_file = os.path.join(db_dir, 'nvdcve_1.0.db')
|
|
|
json_tmpfile = os.path.join(db_dir, 'nvd.json.gz')
|
|
|
proxy = d.getVar("https_proxy")
|
|
|
+
|
|
|
+ if proxy:
|
|
|
+ # instantiate an opener but do not install it as the global
|
|
|
+ # opener unless if we're really sure it's applicable for all
|
|
|
+ # urllib requests
|
|
|
+ proxy_handler = urllib.request.ProxyHandler({'https': proxy})
|
|
|
+ proxy_opener = urllib.request.build_opener(proxy_handler)
|
|
|
+ else:
|
|
|
+ proxy_opener = None
|
|
|
+
|
|
|
cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')
|
|
|
|
|
|
if not os.path.isdir(db_dir):
|
|
@@ -44,11 +54,17 @@ python do_populate_cve_db() {
|
|
|
json_url = year_url + ".json.gz"
|
|
|
|
|
|
# Retrieve meta last modified date
|
|
|
- req = urllib.request.Request(meta_url)
|
|
|
- if proxy:
|
|
|
- req.set_proxy(proxy, 'https')
|
|
|
- with urllib.request.urlopen(req) as r:
|
|
|
- for l in r.read().decode("utf-8").splitlines():
|
|
|
+
|
|
|
+ response = None
|
|
|
+
|
|
|
+ if proxy_opener:
|
|
|
+ response = proxy_opener.open(meta_url)
|
|
|
+ else:
|
|
|
+ req = urllib.request.Request(meta_url)
|
|
|
+ response = urllib.request.urlopen(req)
|
|
|
+
|
|
|
+ if response:
|
|
|
+ for l in response.read().decode("utf-8").splitlines():
|
|
|
key, value = l.split(":", 1)
|
|
|
if key == "lastModifiedDate":
|
|
|
last_modified = value
|
|
@@ -66,11 +82,14 @@ python do_populate_cve_db() {
|
|
|
|
|
|
# Update db with current year json file
|
|
|
try:
|
|
|
- req = urllib.request.Request(json_url)
|
|
|
- if proxy:
|
|
|
- req.set_proxy(proxy, 'https')
|
|
|
- with urllib.request.urlopen(req) as r:
|
|
|
- update_db(c, gzip.decompress(r.read()))
|
|
|
+ if proxy_opener:
|
|
|
+ response = proxy_opener.open(json_url)
|
|
|
+ else:
|
|
|
+ req = urllib.request.Request(json_url)
|
|
|
+ response = urllib.request.urlopen(req)
|
|
|
+
|
|
|
+ if response:
|
|
|
+ update_db(c, gzip.decompress(response.read()).decode('utf-8'))
|
|
|
c.execute("insert or replace into META values (?, ?)", [year, last_modified])
|
|
|
except urllib.error.URLError as e:
|
|
|
cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
|