|
@@ -1,7 +1,28 @@
|
|
|
-def get_links_from_url(url):
|
|
|
+from contextlib import contextmanager
|
|
|
+@contextmanager
|
|
|
+def create_socket(url, d):
|
|
|
+ import urllib
|
|
|
+ socket = urllib.urlopen(url, proxies=get_proxies(d))
|
|
|
+ try:
|
|
|
+ yield socket
|
|
|
+ finally:
|
|
|
+ socket.close()
|
|
|
+
|
|
|
+def get_proxies(d):
|
|
|
+ import os
|
|
|
+ proxykeys = ['HTTP_PROXY', 'http_proxy',
|
|
|
+ 'HTTPS_PROXY', 'https_proxy',
|
|
|
+ 'FTP_PROXY', 'ftp_proxy',
|
|
|
+ 'FTPS_PROXY', 'ftps_proxy',
|
|
|
+ 'NO_PROXY', 'no_proxy',
|
|
|
+ 'ALL_PROXY', 'all_proxy']
|
|
|
+ proxyvalues = map(lambda key: d.getVar(key, True), proxykeys)
|
|
|
+ return dict(zip(proxykeys, proxyvalues))
|
|
|
+
|
|
|
+def get_links_from_url(url, d):
|
|
|
"Return all the href links found on the web location"
|
|
|
|
|
|
- import urllib, sgmllib
|
|
|
+ import sgmllib
|
|
|
|
|
|
class LinksParser(sgmllib.SGMLParser):
|
|
|
def parse(self, s):
|
|
@@ -24,19 +45,18 @@ def get_links_from_url(url):
|
|
|
"Return the list of hyperlinks."
|
|
|
return self.hyperlinks
|
|
|
|
|
|
- sock = urllib.urlopen(url)
|
|
|
- webpage = sock.read()
|
|
|
- sock.close()
|
|
|
+ with create_socket(url,d) as sock:
|
|
|
+ webpage = sock.read()
|
|
|
|
|
|
linksparser = LinksParser()
|
|
|
linksparser.parse(webpage)
|
|
|
return linksparser.get_hyperlinks()
|
|
|
|
|
|
-def find_latest_numeric_release(url):
|
|
|
+def find_latest_numeric_release(url, d):
|
|
|
"Find the latest listed numeric release on the given url"
|
|
|
max=0
|
|
|
maxstr=""
|
|
|
- for link in get_links_from_url(url):
|
|
|
+ for link in get_links_from_url(url, d):
|
|
|
try:
|
|
|
release = float(link)
|
|
|
except:
|
|
@@ -70,7 +90,7 @@ def clean_package_list(package_list):
|
|
|
return set.keys()
|
|
|
|
|
|
|
|
|
-def get_latest_released_meego_source_package_list():
|
|
|
+def get_latest_released_meego_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest meego distro"
|
|
|
|
|
|
package_names = []
|
|
@@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list():
|
|
|
package_list=clean_package_list(package_names)
|
|
|
return "1.0", package_list
|
|
|
|
|
|
-def get_source_package_list_from_url(url, section):
|
|
|
+def get_source_package_list_from_url(url, section, d):
|
|
|
"Return a sectioned list of package names from a URL list"
|
|
|
|
|
|
bb.note("Reading %s: %s" % (url, section))
|
|
|
- links = get_links_from_url(url)
|
|
|
+ links = get_links_from_url(url, d)
|
|
|
srpms = filter(is_src_rpm, links)
|
|
|
names_list = map(package_name_from_srpm, srpms)
|
|
|
|
|
@@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section):
|
|
|
|
|
|
return new_pkgs
|
|
|
|
|
|
-def get_latest_released_fedora_source_package_list():
|
|
|
+def get_latest_released_fedora_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest fedora distro"
|
|
|
- latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
|
|
|
+ latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
|
|
|
|
|
|
- package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
|
|
|
+ package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d)
|
|
|
|
|
|
# package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
|
|
|
- package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
|
|
|
+ package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
|
|
|
|
|
|
package_list=clean_package_list(package_names)
|
|
|
|
|
|
return latest, package_list
|
|
|
|
|
|
-def get_latest_released_opensuse_source_package_list():
|
|
|
+def get_latest_released_opensuse_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest opensuse distro"
|
|
|
- latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
|
|
|
+ latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
|
|
|
|
|
|
- package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
|
|
|
- package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
|
|
|
+ package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
|
|
|
+ package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d)
|
|
|
|
|
|
package_list=clean_package_list(package_names)
|
|
|
return latest, package_list
|
|
|
|
|
|
-def get_latest_released_mandriva_source_package_list():
|
|
|
+def get_latest_released_mandriva_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest mandriva distro"
|
|
|
- latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
|
|
|
- package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
|
|
|
+ latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
|
|
|
+ package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
|
|
|
# package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
|
|
|
- package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
|
|
|
+ package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
|
|
|
|
|
|
package_list=clean_package_list(package_names)
|
|
|
return latest, package_list
|
|
|
|
|
|
-def find_latest_debian_release(url):
|
|
|
+def find_latest_debian_release(url, d):
|
|
|
"Find the latest listed debian release on the given url"
|
|
|
|
|
|
releases = []
|
|
|
- for link in get_links_from_url(url):
|
|
|
+ for link in get_links_from_url(url, d):
|
|
|
if link[:6] == "Debian":
|
|
|
if ';' not in link:
|
|
|
releases.append(link)
|
|
@@ -143,16 +163,15 @@ def find_latest_debian_release(url):
|
|
|
except:
|
|
|
return "_NotFound_"
|
|
|
|
|
|
-def get_debian_style_source_package_list(url, section):
|
|
|
+def get_debian_style_source_package_list(url, section, d):
|
|
|
"Return the list of package-names stored in the debian style Sources.gz file"
|
|
|
- import urllib
|
|
|
- sock = urllib.urlopen(url)
|
|
|
- import tempfile
|
|
|
- tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
|
|
|
- tmpfilename=tmpfile.name
|
|
|
- tmpfile.write(sock.read())
|
|
|
- sock.close()
|
|
|
- tmpfile.close()
|
|
|
+ with create_socket(url,d) as sock:
|
|
|
+ webpage = sock.read()
|
|
|
+ import tempfile
|
|
|
+ tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
|
|
|
+ tmpfilename=tmpfile.name
|
|
|
+ tmpfile.write(sock.read())
|
|
|
+ tmpfile.close()
|
|
|
import gzip
|
|
|
bb.note("Reading %s: %s" % (url, section))
|
|
|
|
|
@@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url, section):
|
|
|
|
|
|
return package_names
|
|
|
|
|
|
-def get_latest_released_debian_source_package_list():
|
|
|
+def get_latest_released_debian_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest debian distro"
|
|
|
- latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
|
|
|
+ latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
|
|
|
url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
|
|
|
- package_names = get_debian_style_source_package_list(url, "main")
|
|
|
+ package_names = get_debian_style_source_package_list(url, "main", d)
|
|
|
# url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
|
|
|
# package_names += get_debian_style_source_package_list(url, "contrib")
|
|
|
url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
|
|
|
- package_names += get_debian_style_source_package_list(url, "updates")
|
|
|
+ package_names += get_debian_style_source_package_list(url, "updates", d)
|
|
|
package_list=clean_package_list(package_names)
|
|
|
return latest, package_list
|
|
|
|
|
|
-def find_latest_ubuntu_release(url):
|
|
|
+def find_latest_ubuntu_release(url, d):
|
|
|
"Find the latest listed ubuntu release on the given url"
|
|
|
url += "?C=M;O=D" # Descending Sort by Last Modified
|
|
|
- for link in get_links_from_url(url):
|
|
|
+ for link in get_links_from_url(url, d):
|
|
|
if link[-8:] == "-updates":
|
|
|
return link[:-8]
|
|
|
return "_NotFound_"
|
|
|
|
|
|
-def get_latest_released_ubuntu_source_package_list():
|
|
|
+def get_latest_released_ubuntu_source_package_list(d):
|
|
|
"Returns list of all the name os packages in the latest ubuntu distro"
|
|
|
- latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
|
|
|
+ latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
|
|
|
url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
|
|
|
- package_names = get_debian_style_source_package_list(url, "main")
|
|
|
+ package_names = get_debian_style_source_package_list(url, "main", d)
|
|
|
# url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
|
|
|
# package_names += get_debian_style_source_package_list(url, "multiverse")
|
|
|
# url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
|
|
|
# package_names += get_debian_style_source_package_list(url, "universe")
|
|
|
url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
|
|
|
- package_names += get_debian_style_source_package_list(url, "updates")
|
|
|
+ package_names += get_debian_style_source_package_list(url, "updates", d)
|
|
|
package_list=clean_package_list(package_names)
|
|
|
return latest, package_list
|
|
|
|
|
|
-def create_distro_packages_list(distro_check_dir):
|
|
|
+def create_distro_packages_list(distro_check_dir, d):
|
|
|
pkglst_dir = os.path.join(distro_check_dir, "package_lists")
|
|
|
if not os.path.isdir (pkglst_dir):
|
|
|
os.makedirs(pkglst_dir)
|
|
@@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir):
|
|
|
begin = datetime.now()
|
|
|
for distro in per_distro_functions:
|
|
|
name = distro[0]
|
|
|
- release, package_list = distro[1]()
|
|
|
+ release, package_list = distro[1](d)
|
|
|
bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
|
|
|
package_list_file = os.path.join(pkglst_dir, name + "-" + release)
|
|
|
f = open(package_list_file, "w+b")
|
|
@@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir):
|
|
|
delta = end - begin
|
|
|
bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
|
|
|
|
|
|
-def update_distro_data(distro_check_dir, datetime):
|
|
|
+def update_distro_data(distro_check_dir, datetime, d):
|
|
|
"""
|
|
|
If distro packages list data is old then rebuild it.
|
|
|
The operations has to be protected by a lock so that
|
|
@@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime):
|
|
|
if saved_datetime[0:8] != datetime[0:8]:
|
|
|
bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
|
|
|
bb.note("Regenerating distro package lists")
|
|
|
- create_distro_packages_list(distro_check_dir)
|
|
|
+ create_distro_packages_list(distro_check_dir, d)
|
|
|
f.seek(0)
|
|
|
f.write(datetime)
|
|
|
|