lsupdates.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. #
  2. # BitBake Toaster Implementation
  3. #
  4. # Copyright (C) 2016-2017 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. from django.core.management.base import BaseCommand
  9. from orm.models import LayerSource, Layer, Release, Layer_Version
  10. from orm.models import LayerVersionDependency, Machine, Recipe
  11. from orm.models import Distro
  12. from orm.models import ToasterSetting
  13. import os
  14. import sys
  15. import logging
  16. import threading
  17. import time
  18. logger = logging.getLogger("toaster")
  19. DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
  20. # Add path to bitbake modules for layerindexlib
  21. # lib/toaster/orm/management/commands/lsupdates.py (abspath)
  22. # lib/toaster/orm/management/commands (dirname)
  23. # lib/toaster/orm/management (dirname)
  24. # lib/toaster/orm (dirname)
  25. # lib/toaster/ (dirname)
  26. # lib/ (dirname)
  27. path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
  28. sys.path.insert(0, path)
  29. import layerindexlib
  30. class Spinner(threading.Thread):
  31. """ A simple progress spinner to indicate download/parsing is happening"""
  32. def __init__(self, *args, **kwargs):
  33. super(Spinner, self).__init__(*args, **kwargs)
  34. self.setDaemon(True)
  35. self.signal = True
  36. def run(self):
  37. os.system('setterm -cursor off')
  38. while self.signal:
  39. for char in ["/", "-", "\\", "|"]:
  40. sys.stdout.write("\r" + char)
  41. sys.stdout.flush()
  42. time.sleep(0.25)
  43. os.system('setterm -cursor on')
  44. def stop(self):
  45. self.signal = False
  46. class Command(BaseCommand):
  47. args = ""
  48. help = "Updates locally cached information from a layerindex server"
  49. def mini_progress(self, what, i, total):
  50. i = i + 1
  51. pec = (float(i)/float(total))*100
  52. sys.stdout.write("\rUpdating %s %d%%" %
  53. (what,
  54. pec))
  55. sys.stdout.flush()
  56. if int(pec) is 100:
  57. sys.stdout.write("\n")
  58. sys.stdout.flush()
  59. def update(self):
  60. """
  61. Fetches layer, recipe and machine information from a layerindex
  62. server
  63. """
  64. os.system('setterm -cursor off')
  65. self.apiurl = DEFAULT_LAYERINDEX_SERVER
  66. if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
  67. self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
  68. assert self.apiurl is not None
  69. # update branches; only those that we already have names listed in the
  70. # Releases table
  71. whitelist_branch_names = [rel.branch_name
  72. for rel in Release.objects.all()]
  73. if len(whitelist_branch_names) == 0:
  74. raise Exception("Failed to make list of branches to fetch")
  75. logger.info("Fetching metadata for %s",
  76. " ".join(whitelist_branch_names))
  77. # We require a non-empty bb.data, but we can fake it with a dictionary
  78. layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
  79. http_progress = Spinner()
  80. http_progress.start()
  81. if whitelist_branch_names:
  82. url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
  83. else:
  84. url_branches = ""
  85. layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
  86. http_progress.stop()
  87. # We know we're only processing one entry, so we reference it here
  88. # (this is cheating...)
  89. index = layerindex.indexes[0]
  90. # Map the layer index branches to toaster releases
  91. li_branch_id_to_toaster_release = {}
  92. logger.info("Processing releases")
  93. total = len(index.branches)
  94. for i, id in enumerate(index.branches):
  95. li_branch_id_to_toaster_release[id] = \
  96. Release.objects.get(name=index.branches[id].name)
  97. self.mini_progress("Releases", i, total)
  98. # keep a track of the layerindex (li) id mappings so that
  99. # layer_versions can be created for these layers later on
  100. li_layer_id_to_toaster_layer_id = {}
  101. logger.info("Processing layers")
  102. total = len(index.layerItems)
  103. for i, id in enumerate(index.layerItems):
  104. try:
  105. l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
  106. l.up_date = index.layerItems[id].updated
  107. l.summary = index.layerItems[id].summary
  108. l.description = index.layerItems[id].description
  109. if created:
  110. # predefined layers in the fixtures (for example poky.xml)
  111. # always preempt the Layer Index for these values
  112. l.vcs_url = index.layerItems[id].vcs_url
  113. l.vcs_web_url = index.layerItems[id].vcs_web_url
  114. l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
  115. l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
  116. l.save()
  117. except Layer.MultipleObjectsReturned:
  118. logger.info("Skipped %s as we found multiple layers and "
  119. "don't know which to update" %
  120. index.layerItems[id].name)
  121. li_layer_id_to_toaster_layer_id[id] = l.pk
  122. self.mini_progress("layers", i, total)
  123. # update layer_versions
  124. logger.info("Processing layer versions")
  125. # Map Layer index layer_branch object id to
  126. # layer_version toaster object id
  127. li_layer_branch_id_to_toaster_lv_id = {}
  128. total = len(index.layerBranches)
  129. for i, id in enumerate(index.layerBranches):
  130. # release as defined by toaster map to layerindex branch
  131. release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
  132. try:
  133. lv, created = Layer_Version.objects.get_or_create(
  134. layer=Layer.objects.get(
  135. pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
  136. release=release
  137. )
  138. except KeyError:
  139. logger.warning(
  140. "No such layerindex layer referenced by layerbranch %d" %
  141. index.layerBranches[id].layer_id)
  142. continue
  143. if created:
  144. lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
  145. lv.up_date = index.layerBranches[id].updated
  146. lv.commit = index.layerBranches[id].actual_branch
  147. lv.dirpath = index.layerBranches[id].vcs_subdir
  148. lv.save()
  149. li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
  150. lv.pk
  151. self.mini_progress("layer versions", i, total)
  152. logger.info("Processing layer version dependencies")
  153. dependlist = {}
  154. for id in index.layerDependencies:
  155. try:
  156. lv = Layer_Version.objects.get(
  157. pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
  158. except Layer_Version.DoesNotExist as e:
  159. continue
  160. if lv not in dependlist:
  161. dependlist[lv] = []
  162. try:
  163. layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
  164. dependlist[lv].append(
  165. Layer_Version.objects.get(layer__pk=layer_id,
  166. release=lv.release))
  167. except Layer_Version.DoesNotExist:
  168. logger.warning("Cannot find layer version (ls:%s),"
  169. "up_id:%s lv:%s" %
  170. (self, index.layerDependencies[id].dependency_id, lv))
  171. total = len(dependlist)
  172. for i, lv in enumerate(dependlist):
  173. LayerVersionDependency.objects.filter(layer_version=lv).delete()
  174. for lvd in dependlist[lv]:
  175. LayerVersionDependency.objects.get_or_create(layer_version=lv,
  176. depends_on=lvd)
  177. self.mini_progress("Layer version dependencies", i, total)
  178. # update Distros
  179. logger.info("Processing distro information")
  180. total = len(index.distros)
  181. for i, id in enumerate(index.distros):
  182. distro, created = Distro.objects.get_or_create(
  183. name=index.distros[id].name,
  184. layer_version=Layer_Version.objects.get(
  185. pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
  186. distro.up_date = index.distros[id].updated
  187. distro.name = index.distros[id].name
  188. distro.description = index.distros[id].description
  189. distro.save()
  190. self.mini_progress("distros", i, total)
  191. # update machines
  192. logger.info("Processing machine information")
  193. total = len(index.machines)
  194. for i, id in enumerate(index.machines):
  195. mo, created = Machine.objects.get_or_create(
  196. name=index.machines[id].name,
  197. layer_version=Layer_Version.objects.get(
  198. pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
  199. mo.up_date = index.machines[id].updated
  200. mo.name = index.machines[id].name
  201. mo.description = index.machines[id].description
  202. mo.save()
  203. self.mini_progress("machines", i, total)
  204. # update recipes; paginate by layer version / layer branch
  205. logger.info("Processing recipe information")
  206. total = len(index.recipes)
  207. for i, id in enumerate(index.recipes):
  208. try:
  209. lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
  210. lv = Layer_Version.objects.get(pk=lv_id)
  211. ro, created = Recipe.objects.get_or_create(
  212. layer_version=lv,
  213. name=index.recipes[id].pn
  214. )
  215. ro.layer_version = lv
  216. ro.up_date = index.recipes[id].updated
  217. ro.name = index.recipes[id].pn
  218. ro.version = index.recipes[id].pv
  219. ro.summary = index.recipes[id].summary
  220. ro.description = index.recipes[id].description
  221. ro.section = index.recipes[id].section
  222. ro.license = index.recipes[id].license
  223. ro.homepage = index.recipes[id].homepage
  224. ro.bugtracker = index.recipes[id].bugtracker
  225. ro.file_path = index.recipes[id].fullpath
  226. ro.is_image = 'image' in index.recipes[id].inherits.split()
  227. ro.save()
  228. except Exception as e:
  229. logger.warning("Failed saving recipe %s", e)
  230. self.mini_progress("recipes", i, total)
  231. os.system('setterm -cursor on')
  232. def handle(self, **options):
  233. self.update()