distro_check.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. def create_socket(url, d):
  5. import urllib
  6. from bb.utils import export_proxies
  7. export_proxies(d)
  8. return urllib.request.urlopen(url)
  9. def get_links_from_url(url, d):
  10. "Return all the href links found on the web location"
  11. from bs4 import BeautifulSoup, SoupStrainer
  12. soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a"))
  13. hyperlinks = []
  14. for line in soup.find_all('a', href=True):
  15. hyperlinks.append(line['href'].strip('/'))
  16. return hyperlinks
  17. def find_latest_numeric_release(url, d):
  18. "Find the latest listed numeric release on the given url"
  19. max=0
  20. maxstr=""
  21. for link in get_links_from_url(url, d):
  22. try:
  23. # TODO use LooseVersion
  24. release = float(link)
  25. except:
  26. release = 0
  27. if release > max:
  28. max = release
  29. maxstr = link
  30. return maxstr
  31. def is_src_rpm(name):
  32. "Check if the link is pointing to a src.rpm file"
  33. return name.endswith(".src.rpm")
  34. def package_name_from_srpm(srpm):
  35. "Strip out the package name from the src.rpm filename"
  36. # ca-certificates-2016.2.7-1.0.fc24.src.rpm
  37. # ^name ^ver ^release^removed
  38. (name, version, release) = srpm.replace(".src.rpm", "").rsplit("-", 2)
  39. return name
  40. def get_source_package_list_from_url(url, section, d):
  41. "Return a sectioned list of package names from a URL list"
  42. bb.note("Reading %s: %s" % (url, section))
  43. links = get_links_from_url(url, d)
  44. srpms = filter(is_src_rpm, links)
  45. names_list = map(package_name_from_srpm, srpms)
  46. new_pkgs = set()
  47. for pkgs in names_list:
  48. new_pkgs.add(pkgs + ":" + section)
  49. return new_pkgs
  50. def get_source_package_list_from_url_by_letter(url, section, d):
  51. import string
  52. from urllib.error import HTTPError
  53. packages = set()
  54. for letter in (string.ascii_lowercase + string.digits):
  55. # Not all subfolders may exist, so silently handle 404
  56. try:
  57. packages |= get_source_package_list_from_url(url + "/" + letter, section, d)
  58. except HTTPError as e:
  59. if e.code != 404: raise
  60. return packages
  61. def get_latest_released_fedora_source_package_list(d):
  62. "Returns list of all the name os packages in the latest fedora distro"
  63. latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
  64. package_names = get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Everything/source/tree/Packages/" % latest, "main", d)
  65. package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
  66. return latest, package_names
  67. def get_latest_released_opensuse_source_package_list(d):
  68. "Returns list of all the name os packages in the latest opensuse distro"
  69. latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d)
  70. package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d)
  71. package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d)
  72. return latest, package_names
  73. def get_latest_released_clear_source_package_list(d):
  74. latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d)
  75. package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d)
  76. return latest, package_names
  77. def find_latest_debian_release(url, d):
  78. "Find the latest listed debian release on the given url"
  79. releases = [link.replace("Debian", "")
  80. for link in get_links_from_url(url, d)
  81. if link.startswith("Debian")]
  82. releases.sort()
  83. try:
  84. return releases[-1]
  85. except:
  86. return "_NotFound_"
  87. def get_debian_style_source_package_list(url, section, d):
  88. "Return the list of package-names stored in the debian style Sources.gz file"
  89. import gzip
  90. package_names = set()
  91. for line in gzip.open(create_socket(url, d), mode="rt"):
  92. if line.startswith("Package:"):
  93. pkg = line.split(":", 1)[1].strip()
  94. package_names.add(pkg + ":" + section)
  95. return package_names
  96. def get_latest_released_debian_source_package_list(d):
  97. "Returns list of all the name of packages in the latest debian distro"
  98. latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
  99. url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
  100. package_names = get_debian_style_source_package_list(url, "main", d)
  101. url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
  102. package_names |= get_debian_style_source_package_list(url, "updates", d)
  103. return latest, package_names
  104. def find_latest_ubuntu_release(url, d):
  105. """
  106. Find the latest listed Ubuntu release on the given ubuntu/dists/ URL.
  107. To avoid matching development releases look for distributions that have
  108. updates, so the resulting distro could be any supported release.
  109. """
  110. url += "?C=M;O=D" # Descending Sort by Last Modified
  111. for link in get_links_from_url(url, d):
  112. if "-updates" in link:
  113. distro = link.replace("-updates", "")
  114. return distro
  115. return "_NotFound_"
  116. def get_latest_released_ubuntu_source_package_list(d):
  117. "Returns list of all the name os packages in the latest ubuntu distro"
  118. latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
  119. url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
  120. package_names = get_debian_style_source_package_list(url, "main", d)
  121. url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
  122. package_names |= get_debian_style_source_package_list(url, "updates", d)
  123. return latest, package_names
  124. def create_distro_packages_list(distro_check_dir, d):
  125. import shutil
  126. pkglst_dir = os.path.join(distro_check_dir, "package_lists")
  127. bb.utils.remove(pkglst_dir, True)
  128. bb.utils.mkdirhier(pkglst_dir)
  129. per_distro_functions = (
  130. ("Debian", get_latest_released_debian_source_package_list),
  131. ("Ubuntu", get_latest_released_ubuntu_source_package_list),
  132. ("Fedora", get_latest_released_fedora_source_package_list),
  133. ("openSUSE", get_latest_released_opensuse_source_package_list),
  134. ("Clear", get_latest_released_clear_source_package_list),
  135. )
  136. for name, fetcher_func in per_distro_functions:
  137. try:
  138. release, package_list = fetcher_func(d)
  139. except Exception as e:
  140. bb.warn("Cannot fetch packages for %s: %s" % (name, e))
  141. bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
  142. if len(package_list) == 0:
  143. bb.error("Didn't fetch any packages for %s %s" % (name, release))
  144. package_list_file = os.path.join(pkglst_dir, name + "-" + release)
  145. with open(package_list_file, 'w') as f:
  146. for pkg in sorted(package_list):
  147. f.write(pkg + "\n")
  148. def update_distro_data(distro_check_dir, datetime, d):
  149. """
  150. If distro packages list data is old then rebuild it.
  151. The operations has to be protected by a lock so that
  152. only one thread performes it at a time.
  153. """
  154. if not os.path.isdir (distro_check_dir):
  155. try:
  156. bb.note ("Making new directory: %s" % distro_check_dir)
  157. os.makedirs (distro_check_dir)
  158. except OSError:
  159. raise Exception('Unable to create directory %s' % (distro_check_dir))
  160. datetime_file = os.path.join(distro_check_dir, "build_datetime")
  161. saved_datetime = "_invalid_"
  162. import fcntl
  163. try:
  164. if not os.path.exists(datetime_file):
  165. open(datetime_file, 'w+').close() # touch the file so that the next open won't fail
  166. f = open(datetime_file, "r+")
  167. fcntl.lockf(f, fcntl.LOCK_EX)
  168. saved_datetime = f.read()
  169. if saved_datetime[0:8] != datetime[0:8]:
  170. bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
  171. bb.note("Regenerating distro package lists")
  172. create_distro_packages_list(distro_check_dir, d)
  173. f.seek(0)
  174. f.write(datetime)
  175. except OSError as e:
  176. raise Exception('Unable to open timestamp: %s' % e)
  177. finally:
  178. fcntl.lockf(f, fcntl.LOCK_UN)
  179. f.close()
  180. def compare_in_distro_packages_list(distro_check_dir, d):
  181. if not os.path.isdir(distro_check_dir):
  182. raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
  183. localdata = bb.data.createCopy(d)
  184. pkglst_dir = os.path.join(distro_check_dir, "package_lists")
  185. matching_distros = []
  186. pn = recipe_name = d.getVar('PN')
  187. bb.note("Checking: %s" % pn)
  188. if pn.find("-native") != -1:
  189. pnstripped = pn.split("-native")
  190. localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
  191. recipe_name = pnstripped[0]
  192. if pn.startswith("nativesdk-"):
  193. pnstripped = pn.split("nativesdk-")
  194. localdata.setVar('OVERRIDES', "pn-" + pnstripped[1] + ":" + d.getVar('OVERRIDES'))
  195. recipe_name = pnstripped[1]
  196. if pn.find("-cross") != -1:
  197. pnstripped = pn.split("-cross")
  198. localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
  199. recipe_name = pnstripped[0]
  200. if pn.find("-initial") != -1:
  201. pnstripped = pn.split("-initial")
  202. localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES'))
  203. recipe_name = pnstripped[0]
  204. bb.note("Recipe: %s" % recipe_name)
  205. distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
  206. tmp = localdata.getVar('DISTRO_PN_ALIAS') or ""
  207. for str in tmp.split():
  208. if str and str.find("=") == -1 and distro_exceptions[str]:
  209. matching_distros.append(str)
  210. distro_pn_aliases = {}
  211. for str in tmp.split():
  212. if "=" in str:
  213. (dist, pn_alias) = str.split('=')
  214. distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
  215. for file in os.listdir(pkglst_dir):
  216. (distro, distro_release) = file.split("-")
  217. f = open(os.path.join(pkglst_dir, file), "r")
  218. for line in f:
  219. (pkg, section) = line.split(":")
  220. if distro.lower() in distro_pn_aliases:
  221. pn = distro_pn_aliases[distro.lower()]
  222. else:
  223. pn = recipe_name
  224. if pn == pkg:
  225. matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
  226. f.close()
  227. break
  228. f.close()
  229. for item in tmp.split():
  230. matching_distros.append(item)
  231. bb.note("Matching: %s" % matching_distros)
  232. return matching_distros
  233. def create_log_file(d, logname):
  234. logpath = d.getVar('LOG_DIR')
  235. bb.utils.mkdirhier(logpath)
  236. logfn, logsuffix = os.path.splitext(logname)
  237. logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME'), logsuffix))
  238. if not os.path.exists(logfile):
  239. slogfile = os.path.join(logpath, logname)
  240. if os.path.exists(slogfile):
  241. os.remove(slogfile)
  242. open(logfile, 'w+').close()
  243. os.symlink(logfile, slogfile)
  244. d.setVar('LOG_FILE', logfile)
  245. return logfile
  246. def save_distro_check_result(result, datetime, result_file, d):
  247. pn = d.getVar('PN')
  248. logdir = d.getVar('LOG_DIR')
  249. if not logdir:
  250. bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
  251. return
  252. bb.utils.mkdirhier(logdir)
  253. line = pn
  254. for i in result:
  255. line = line + "," + i
  256. f = open(result_file, "a")
  257. import fcntl
  258. fcntl.lockf(f, fcntl.LOCK_EX)
  259. f.seek(0, os.SEEK_END) # seek to the end of file
  260. f.write(line + "\n")
  261. fcntl.lockf(f, fcntl.LOCK_UN)
  262. f.close()