123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290 |
- #
- # BitBake Toaster Implementation
- #
- # Copyright (C) 2016-2017 Intel Corporation
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- from django.core.management.base import BaseCommand
- from orm.models import LayerSource, Layer, Release, Layer_Version
- from orm.models import LayerVersionDependency, Machine, Recipe
- from orm.models import Distro
- from orm.models import ToasterSetting
- import os
- import sys
- import logging
- import threading
- import time
- logger = logging.getLogger("toaster")
- DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
- # Add path to bitbake modules for layerindexlib
- # lib/toaster/orm/management/commands/lsupdates.py (abspath)
- # lib/toaster/orm/management/commands (dirname)
- # lib/toaster/orm/management (dirname)
- # lib/toaster/orm (dirname)
- # lib/toaster/ (dirname)
- # lib/ (dirname)
- path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
- sys.path.insert(0, path)
- import layerindexlib
- class Spinner(threading.Thread):
- """ A simple progress spinner to indicate download/parsing is happening"""
- def __init__(self, *args, **kwargs):
- super(Spinner, self).__init__(*args, **kwargs)
- self.setDaemon(True)
- self.signal = True
- def run(self):
- os.system('setterm -cursor off')
- while self.signal:
- for char in ["/", "-", "\\", "|"]:
- sys.stdout.write("\r" + char)
- sys.stdout.flush()
- time.sleep(0.25)
- os.system('setterm -cursor on')
- def stop(self):
- self.signal = False
- class Command(BaseCommand):
- args = ""
- help = "Updates locally cached information from a layerindex server"
- def mini_progress(self, what, i, total):
- i = i + 1
- pec = (float(i)/float(total))*100
- sys.stdout.write("\rUpdating %s %d%%" %
- (what,
- pec))
- sys.stdout.flush()
- if int(pec) is 100:
- sys.stdout.write("\n")
- sys.stdout.flush()
- def update(self):
- """
- Fetches layer, recipe and machine information from a layerindex
- server
- """
- os.system('setterm -cursor off')
- self.apiurl = DEFAULT_LAYERINDEX_SERVER
- if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
- self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
- assert self.apiurl is not None
- # update branches; only those that we already have names listed in the
- # Releases table
- whitelist_branch_names = [rel.branch_name
- for rel in Release.objects.all()]
- if len(whitelist_branch_names) == 0:
- raise Exception("Failed to make list of branches to fetch")
- logger.info("Fetching metadata for %s",
- " ".join(whitelist_branch_names))
- # We require a non-empty bb.data, but we can fake it with a dictionary
- layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
- http_progress = Spinner()
- http_progress.start()
- if whitelist_branch_names:
- url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
- else:
- url_branches = ""
- layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
- http_progress.stop()
- # We know we're only processing one entry, so we reference it here
- # (this is cheating...)
- index = layerindex.indexes[0]
- # Map the layer index branches to toaster releases
- li_branch_id_to_toaster_release = {}
- logger.info("Processing releases")
- total = len(index.branches)
- for i, id in enumerate(index.branches):
- li_branch_id_to_toaster_release[id] = \
- Release.objects.get(name=index.branches[id].name)
- self.mini_progress("Releases", i, total)
- # keep a track of the layerindex (li) id mappings so that
- # layer_versions can be created for these layers later on
- li_layer_id_to_toaster_layer_id = {}
- logger.info("Processing layers")
- total = len(index.layerItems)
- for i, id in enumerate(index.layerItems):
- try:
- l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
- l.up_date = index.layerItems[id].updated
- l.summary = index.layerItems[id].summary
- l.description = index.layerItems[id].description
- if created:
- # predefined layers in the fixtures (for example poky.xml)
- # always preempt the Layer Index for these values
- l.vcs_url = index.layerItems[id].vcs_url
- l.vcs_web_url = index.layerItems[id].vcs_web_url
- l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
- l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
- l.save()
- except Layer.MultipleObjectsReturned:
- logger.info("Skipped %s as we found multiple layers and "
- "don't know which to update" %
- index.layerItems[id].name)
- li_layer_id_to_toaster_layer_id[id] = l.pk
- self.mini_progress("layers", i, total)
- # update layer_versions
- logger.info("Processing layer versions")
- # Map Layer index layer_branch object id to
- # layer_version toaster object id
- li_layer_branch_id_to_toaster_lv_id = {}
- total = len(index.layerBranches)
- for i, id in enumerate(index.layerBranches):
- # release as defined by toaster map to layerindex branch
- release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
- try:
- lv, created = Layer_Version.objects.get_or_create(
- layer=Layer.objects.get(
- pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
- release=release
- )
- except KeyError:
- logger.warning(
- "No such layerindex layer referenced by layerbranch %d" %
- index.layerBranches[id].layer_id)
- continue
- if created:
- lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
- lv.up_date = index.layerBranches[id].updated
- lv.commit = index.layerBranches[id].actual_branch
- lv.dirpath = index.layerBranches[id].vcs_subdir
- lv.save()
- li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
- lv.pk
- self.mini_progress("layer versions", i, total)
- logger.info("Processing layer version dependencies")
- dependlist = {}
- for id in index.layerDependencies:
- try:
- lv = Layer_Version.objects.get(
- pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
- except Layer_Version.DoesNotExist as e:
- continue
- if lv not in dependlist:
- dependlist[lv] = []
- try:
- layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
- dependlist[lv].append(
- Layer_Version.objects.get(layer__pk=layer_id,
- release=lv.release))
- except Layer_Version.DoesNotExist:
- logger.warning("Cannot find layer version (ls:%s),"
- "up_id:%s lv:%s" %
- (self, index.layerDependencies[id].dependency_id, lv))
- total = len(dependlist)
- for i, lv in enumerate(dependlist):
- LayerVersionDependency.objects.filter(layer_version=lv).delete()
- for lvd in dependlist[lv]:
- LayerVersionDependency.objects.get_or_create(layer_version=lv,
- depends_on=lvd)
- self.mini_progress("Layer version dependencies", i, total)
- # update Distros
- logger.info("Processing distro information")
- total = len(index.distros)
- for i, id in enumerate(index.distros):
- distro, created = Distro.objects.get_or_create(
- name=index.distros[id].name,
- layer_version=Layer_Version.objects.get(
- pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
- distro.up_date = index.distros[id].updated
- distro.name = index.distros[id].name
- distro.description = index.distros[id].description
- distro.save()
- self.mini_progress("distros", i, total)
- # update machines
- logger.info("Processing machine information")
- total = len(index.machines)
- for i, id in enumerate(index.machines):
- mo, created = Machine.objects.get_or_create(
- name=index.machines[id].name,
- layer_version=Layer_Version.objects.get(
- pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
- mo.up_date = index.machines[id].updated
- mo.name = index.machines[id].name
- mo.description = index.machines[id].description
- mo.save()
- self.mini_progress("machines", i, total)
- # update recipes; paginate by layer version / layer branch
- logger.info("Processing recipe information")
- total = len(index.recipes)
- for i, id in enumerate(index.recipes):
- try:
- lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
- lv = Layer_Version.objects.get(pk=lv_id)
- ro, created = Recipe.objects.get_or_create(
- layer_version=lv,
- name=index.recipes[id].pn
- )
- ro.layer_version = lv
- ro.up_date = index.recipes[id].updated
- ro.name = index.recipes[id].pn
- ro.version = index.recipes[id].pv
- ro.summary = index.recipes[id].summary
- ro.description = index.recipes[id].description
- ro.section = index.recipes[id].section
- ro.license = index.recipes[id].license
- ro.homepage = index.recipes[id].homepage
- ro.bugtracker = index.recipes[id].bugtracker
- ro.file_path = index.recipes[id].fullpath
- ro.is_image = 'image' in index.recipes[id].inherits.split()
- ro.save()
- except Exception as e:
- logger.warning("Failed saving recipe %s", e)
- self.mini_progress("recipes", i, total)
- os.system('setterm -cursor on')
- def handle(self, **options):
- self.update()
|