lsupdates.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. #
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # BitBake Toaster Implementation
  6. #
  7. # Copyright (C) 2016-2017 Intel Corporation
  8. #
  9. # SPDX-License-Identifier: GPL-2.0-only
  10. #
  11. from django.core.management.base import BaseCommand
  12. from orm.models import LayerSource, Layer, Release, Layer_Version
  13. from orm.models import LayerVersionDependency, Machine, Recipe
  14. from orm.models import Distro
  15. from orm.models import ToasterSetting
  16. import os
  17. import sys
  18. import logging
  19. import threading
  20. import time
  21. logger = logging.getLogger("toaster")
  22. DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
  23. # Add path to bitbake modules for layerindexlib
  24. # lib/toaster/orm/management/commands/lsupdates.py (abspath)
  25. # lib/toaster/orm/management/commands (dirname)
  26. # lib/toaster/orm/management (dirname)
  27. # lib/toaster/orm (dirname)
  28. # lib/toaster/ (dirname)
  29. # lib/ (dirname)
  30. path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
  31. sys.path.insert(0, path)
  32. import layerindexlib
  33. class Spinner(threading.Thread):
  34. """ A simple progress spinner to indicate download/parsing is happening"""
  35. def __init__(self, *args, **kwargs):
  36. super(Spinner, self).__init__(*args, **kwargs)
  37. self.setDaemon(True)
  38. self.signal = True
  39. def run(self):
  40. os.system('setterm -cursor off')
  41. while self.signal:
  42. for char in ["/", "-", "\\", "|"]:
  43. sys.stdout.write("\r" + char)
  44. sys.stdout.flush()
  45. time.sleep(0.25)
  46. os.system('setterm -cursor on')
  47. def stop(self):
  48. self.signal = False
  49. class Command(BaseCommand):
  50. args = ""
  51. help = "Updates locally cached information from a layerindex server"
  52. def mini_progress(self, what, i, total):
  53. i = i + 1
  54. pec = (float(i)/float(total))*100
  55. sys.stdout.write("\rUpdating %s %d%%" %
  56. (what,
  57. pec))
  58. sys.stdout.flush()
  59. if int(pec) is 100:
  60. sys.stdout.write("\n")
  61. sys.stdout.flush()
  62. def update(self):
  63. """
  64. Fetches layer, recipe and machine information from a layerindex
  65. server
  66. """
  67. os.system('setterm -cursor off')
  68. self.apiurl = DEFAULT_LAYERINDEX_SERVER
  69. if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
  70. self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
  71. assert self.apiurl is not None
  72. # update branches; only those that we already have names listed in the
  73. # Releases table
  74. whitelist_branch_names = [rel.branch_name
  75. for rel in Release.objects.all()]
  76. if len(whitelist_branch_names) == 0:
  77. raise Exception("Failed to make list of branches to fetch")
  78. logger.info("Fetching metadata for %s",
  79. " ".join(whitelist_branch_names))
  80. # We require a non-empty bb.data, but we can fake it with a dictionary
  81. layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
  82. http_progress = Spinner()
  83. http_progress.start()
  84. if whitelist_branch_names:
  85. url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
  86. else:
  87. url_branches = ""
  88. layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
  89. http_progress.stop()
  90. # We know we're only processing one entry, so we reference it here
  91. # (this is cheating...)
  92. index = layerindex.indexes[0]
  93. # Map the layer index branches to toaster releases
  94. li_branch_id_to_toaster_release = {}
  95. logger.info("Processing releases")
  96. total = len(index.branches)
  97. for i, id in enumerate(index.branches):
  98. li_branch_id_to_toaster_release[id] = \
  99. Release.objects.get(name=index.branches[id].name)
  100. self.mini_progress("Releases", i, total)
  101. # keep a track of the layerindex (li) id mappings so that
  102. # layer_versions can be created for these layers later on
  103. li_layer_id_to_toaster_layer_id = {}
  104. logger.info("Processing layers")
  105. total = len(index.layerItems)
  106. for i, id in enumerate(index.layerItems):
  107. try:
  108. l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
  109. l.up_date = index.layerItems[id].updated
  110. l.summary = index.layerItems[id].summary
  111. l.description = index.layerItems[id].description
  112. if created:
  113. # predefined layers in the fixtures (for example poky.xml)
  114. # always preempt the Layer Index for these values
  115. l.vcs_url = index.layerItems[id].vcs_url
  116. l.vcs_web_url = index.layerItems[id].vcs_web_url
  117. l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
  118. l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
  119. l.save()
  120. except Layer.MultipleObjectsReturned:
  121. logger.info("Skipped %s as we found multiple layers and "
  122. "don't know which to update" %
  123. index.layerItems[id].name)
  124. li_layer_id_to_toaster_layer_id[id] = l.pk
  125. self.mini_progress("layers", i, total)
  126. # update layer_versions
  127. logger.info("Processing layer versions")
  128. # Map Layer index layer_branch object id to
  129. # layer_version toaster object id
  130. li_layer_branch_id_to_toaster_lv_id = {}
  131. total = len(index.layerBranches)
  132. for i, id in enumerate(index.layerBranches):
  133. # release as defined by toaster map to layerindex branch
  134. release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
  135. try:
  136. lv, created = Layer_Version.objects.get_or_create(
  137. layer=Layer.objects.get(
  138. pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
  139. release=release
  140. )
  141. except KeyError:
  142. logger.warning(
  143. "No such layerindex layer referenced by layerbranch %d" %
  144. index.layerBranches[id].layer_id)
  145. continue
  146. if created:
  147. lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
  148. lv.up_date = index.layerBranches[id].updated
  149. lv.commit = index.layerBranches[id].actual_branch
  150. lv.dirpath = index.layerBranches[id].vcs_subdir
  151. lv.save()
  152. li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
  153. lv.pk
  154. self.mini_progress("layer versions", i, total)
  155. logger.info("Processing layer version dependencies")
  156. dependlist = {}
  157. for id in index.layerDependencies:
  158. try:
  159. lv = Layer_Version.objects.get(
  160. pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
  161. except Layer_Version.DoesNotExist as e:
  162. continue
  163. if lv not in dependlist:
  164. dependlist[lv] = []
  165. try:
  166. layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
  167. dependlist[lv].append(
  168. Layer_Version.objects.get(layer__pk=layer_id,
  169. release=lv.release))
  170. except Layer_Version.DoesNotExist:
  171. logger.warning("Cannot find layer version (ls:%s),"
  172. "up_id:%s lv:%s" %
  173. (self, index.layerDependencies[id].dependency_id, lv))
  174. total = len(dependlist)
  175. for i, lv in enumerate(dependlist):
  176. LayerVersionDependency.objects.filter(layer_version=lv).delete()
  177. for lvd in dependlist[lv]:
  178. LayerVersionDependency.objects.get_or_create(layer_version=lv,
  179. depends_on=lvd)
  180. self.mini_progress("Layer version dependencies", i, total)
  181. # update Distros
  182. logger.info("Processing distro information")
  183. total = len(index.distros)
  184. for i, id in enumerate(index.distros):
  185. distro, created = Distro.objects.get_or_create(
  186. name=index.distros[id].name,
  187. layer_version=Layer_Version.objects.get(
  188. pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
  189. distro.up_date = index.distros[id].updated
  190. distro.name = index.distros[id].name
  191. distro.description = index.distros[id].description
  192. distro.save()
  193. self.mini_progress("distros", i, total)
  194. # update machines
  195. logger.info("Processing machine information")
  196. total = len(index.machines)
  197. for i, id in enumerate(index.machines):
  198. mo, created = Machine.objects.get_or_create(
  199. name=index.machines[id].name,
  200. layer_version=Layer_Version.objects.get(
  201. pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
  202. mo.up_date = index.machines[id].updated
  203. mo.name = index.machines[id].name
  204. mo.description = index.machines[id].description
  205. mo.save()
  206. self.mini_progress("machines", i, total)
  207. # update recipes; paginate by layer version / layer branch
  208. logger.info("Processing recipe information")
  209. total = len(index.recipes)
  210. for i, id in enumerate(index.recipes):
  211. try:
  212. lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
  213. lv = Layer_Version.objects.get(pk=lv_id)
  214. ro, created = Recipe.objects.get_or_create(
  215. layer_version=lv,
  216. name=index.recipes[id].pn
  217. )
  218. ro.layer_version = lv
  219. ro.up_date = index.recipes[id].updated
  220. ro.name = index.recipes[id].pn
  221. ro.version = index.recipes[id].pv
  222. ro.summary = index.recipes[id].summary
  223. ro.description = index.recipes[id].description
  224. ro.section = index.recipes[id].section
  225. ro.license = index.recipes[id].license
  226. ro.homepage = index.recipes[id].homepage
  227. ro.bugtracker = index.recipes[id].bugtracker
  228. ro.file_path = index.recipes[id].fullpath
  229. ro.is_image = 'image' in index.recipes[id].inherits.split()
  230. ro.save()
  231. except Exception as e:
  232. logger.warning("Failed saving recipe %s", e)
  233. self.mini_progress("recipes", i, total)
  234. os.system('setterm -cursor on')
  235. def handle(self, **options):
  236. self.update()