123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212 |
- #
- # BitBake Toaster Implementation
- #
- # Copyright (C) 2016 Intel Corporation
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- # Please run flake8 on this file before sending patches
- import os
- import re
- import logging
- import json
- import subprocess
- from collections import Counter
- from orm.models import Project, ProjectTarget, Build, Layer_Version
- from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
- from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
- from orm.models import Layer, Target, Package, Package_Dependency
- from orm.models import ProjectVariable
- from bldcontrol.models import BuildRequest, BuildEnvironment
- from bldcontrol import bbcontroller
- from django.http import HttpResponse, JsonResponse
- from django.views.generic import View
- from django.urls import reverse
- from django.db.models import Q, F
- from django.db import Error
- from toastergui.templatetags.projecttags import filtered_filesizeformat
- # development/debugging support
- verbose = 2
- def _log(msg):
- if 1 == verbose:
- print(msg)
- elif 2 == verbose:
- f1=open('/tmp/toaster.log', 'a')
- f1.write("|" + msg + "|\n" )
- f1.close()
- logger = logging.getLogger("toaster")
- def error_response(error):
- return JsonResponse({"error": error})
- class XhrBuildRequest(View):
- def get(self, request, *args, **kwargs):
- return HttpResponse()
- @staticmethod
- def cancel_build(br):
- """Cancel a build request"""
- try:
- bbctrl = bbcontroller.BitbakeController(br.environment)
- bbctrl.forceShutDown()
- except:
- # We catch a bunch of exceptions here because
- # this is where the server has not had time to start up
- # and the build request or build is in transit between
- # processes.
- # We can safely just set the build as cancelled
- # already as it never got started
- build = br.build
- build.outcome = Build.CANCELLED
- build.save()
- # We now hand over to the buildinfohelper to update the
- # build state once we've finished cancelling
- br.state = BuildRequest.REQ_CANCELLING
- br.save()
- def post(self, request, *args, **kwargs):
- """
- Build control
- Entry point: /xhr_buildrequest/<project_id>
- Method: POST
- Args:
- id: id of build to change
- buildCancel = build_request_id ...
- buildDelete = id ...
- targets = recipe_name ...
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- project = Project.objects.get(pk=kwargs['pid'])
- if 'buildCancel' in request.POST:
- for i in request.POST['buildCancel'].strip().split(" "):
- try:
- br = BuildRequest.objects.get(project=project, pk=i)
- self.cancel_build(br)
- except BuildRequest.DoesNotExist:
- return error_response('No such build request id %s' % i)
- return error_response('ok')
- if 'buildDelete' in request.POST:
- for i in request.POST['buildDelete'].strip().split(" "):
- try:
- BuildRequest.objects.select_for_update().get(
- project=project,
- pk=i,
- state__lte=BuildRequest.REQ_DELETED).delete()
- except BuildRequest.DoesNotExist:
- pass
- return error_response("ok")
- if 'targets' in request.POST:
- ProjectTarget.objects.filter(project=project).delete()
- s = str(request.POST['targets'])
- for t in re.sub(r'[;%|"]', '', s).split(" "):
- if ":" in t:
- target, task = t.split(":")
- else:
- target = t
- task = ""
- ProjectTarget.objects.create(project=project,
- target=target,
- task=task)
- project.schedule_build()
- return error_response('ok')
- response = HttpResponse()
- response.status_code = 500
- return response
- class XhrProjectUpdate(View):
- def get(self, request, *args, **kwargs):
- return HttpResponse()
- def post(self, request, *args, **kwargs):
- """
- Project Update
- Entry point: /xhr_projectupdate/<project_id>
- Method: POST
- Args:
- pid: pid of project to update
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- project = Project.objects.get(pk=kwargs['pid'])
- logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
- if 'do_update' in request.POST:
- # Extract any default image recipe
- if 'default_image' in request.POST:
- project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
- else:
- project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
- logger.debug("ProjectUpdateCallback:Chain to the build request")
- # Chain to the build request
- xhrBuildRequest = XhrBuildRequest()
- return xhrBuildRequest.post(request, *args, **kwargs)
- logger.warning("ERROR:XhrProjectUpdate")
- response = HttpResponse()
- response.status_code = 500
- return response
- class XhrSetDefaultImageUrl(View):
- def get(self, request, *args, **kwargs):
- return HttpResponse()
- def post(self, request, *args, **kwargs):
- """
- Project Update
- Entry point: /xhr_setdefaultimage/<project_id>
- Method: POST
- Args:
- pid: pid of project to update default image
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- project = Project.objects.get(pk=kwargs['pid'])
- logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
- # set any default image recipe
- if 'targets' in request.POST:
- default_target = str(request.POST['targets'])
- project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
- logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
- return error_response('ok')
- logger.warning("ERROR:XhrSetDefaultImageUrl")
- response = HttpResponse()
- response.status_code = 500
- return response
- #
- # Layer Management
- #
- # Rules for 'local_source_dir' layers
- # * Layers must have a unique name in the Layers table
- # * A 'local_source_dir' layer is supposed to be shared
- # by all projects that use it, so that it can have the
- # same logical name
- # * Each project that uses a layer will have its own
- # LayerVersion and Project Layer for it
- # * During the Paroject delete process, when the last
- # LayerVersion for a 'local_source_dir' layer is deleted
- # then the Layer record is deleted to remove orphans
- #
- def scan_layer_content(layer,layer_version):
- # if this is a local layer directory, we can immediately scan its content
- if layer.local_source_dir:
- try:
- # recipes-*/*/*.bb
- cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
- recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
- recipes_list = recipes_list.decode("utf-8").strip()
- if recipes_list and 'No such' not in recipes_list:
- for recipe in recipes_list.split('\n'):
- recipe_path = recipe[recipe.rfind('recipes-'):]
- recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
- recipe_ver = recipe_name.rfind('_')
- if recipe_ver > 0:
- recipe_name = recipe_name[0:recipe_ver]
- if recipe_name:
- ro, created = Recipe.objects.get_or_create(
- layer_version=layer_version,
- name=recipe_name
- )
- if created:
- ro.file_path = recipe_path
- ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
- ro.description = ro.summary
- ro.save()
- except Exception as e:
- logger.warning("ERROR:scan_layer_content: %s" % e)
- class XhrLayer(View):
- """ Delete, Get, Add and Update Layer information
- Methods: GET POST DELETE PUT
- """
- def get(self, request, *args, **kwargs):
- """
- Get layer information
- Method: GET
- Entry point: /xhr_layer/<project id>/<layerversion_id>
- """
- try:
- layer_version = Layer_Version.objects.get(
- pk=kwargs['layerversion_id'])
- project = Project.objects.get(pk=kwargs['pid'])
- project_layers = ProjectLayer.objects.filter(
- project=project).values_list("layercommit_id",
- flat=True)
- ret = {
- 'error': 'ok',
- 'id': layer_version.pk,
- 'name': layer_version.layer.name,
- 'layerdetailurl':
- layer_version.get_detailspage_url(project.pk),
- 'vcs_ref': layer_version.get_vcs_reference(),
- 'vcs_url': layer_version.layer.vcs_url,
- 'local_source_dir': layer_version.layer.local_source_dir,
- 'layerdeps': {
- "list": [
- {
- "id": dep.id,
- "name": dep.layer.name,
- "layerdetailurl":
- dep.get_detailspage_url(project.pk),
- "vcs_url": dep.layer.vcs_url,
- "vcs_reference": dep.get_vcs_reference()
- }
- for dep in layer_version.get_alldeps(project.id)]
- },
- 'projectlayers': list(project_layers)
- }
- return JsonResponse(ret)
- except Layer_Version.DoesNotExist:
- error_response("No such layer")
- def post(self, request, *args, **kwargs):
- """
- Update a layer
- Method: POST
- Entry point: /xhr_layer/<layerversion_id>
- Args:
- vcs_url, dirpath, commit, up_branch, summary, description,
- local_source_dir
- add_dep = append a layerversion_id as a dependency
- rm_dep = remove a layerversion_id as a depedency
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- try:
- # We currently only allow Imported layers to be edited
- layer_version = Layer_Version.objects.get(
- id=kwargs['layerversion_id'],
- project=kwargs['pid'],
- layer_source=LayerSource.TYPE_IMPORTED)
- except Layer_Version.DoesNotExist:
- return error_response("Cannot find imported layer to update")
- if "vcs_url" in request.POST:
- layer_version.layer.vcs_url = request.POST["vcs_url"]
- if "dirpath" in request.POST:
- layer_version.dirpath = request.POST["dirpath"]
- if "commit" in request.POST:
- layer_version.commit = request.POST["commit"]
- layer_version.branch = request.POST["commit"]
- if "summary" in request.POST:
- layer_version.layer.summary = request.POST["summary"]
- if "description" in request.POST:
- layer_version.layer.description = request.POST["description"]
- if "local_source_dir" in request.POST:
- layer_version.layer.local_source_dir = \
- request.POST["local_source_dir"]
- if "add_dep" in request.POST:
- lvd = LayerVersionDependency(
- layer_version=layer_version,
- depends_on_id=request.POST["add_dep"])
- lvd.save()
- if "rm_dep" in request.POST:
- rm_dep = LayerVersionDependency.objects.get(
- layer_version=layer_version,
- depends_on_id=request.POST["rm_dep"])
- rm_dep.delete()
- try:
- layer_version.layer.save()
- layer_version.save()
- except Exception as e:
- return error_response("Could not update layer version entry: %s"
- % e)
- return error_response("ok")
- def put(self, request, *args, **kwargs):
- """ Add a new layer
- Method: PUT
- Entry point: /xhr_layer/<project id>/
- Args:
- project_id, name,
- [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
- (csv)]
- """
- try:
- project = Project.objects.get(pk=kwargs['pid'])
- layer_data = json.loads(request.body.decode('utf-8'))
- # We require a unique layer name as otherwise the lists of layers
- # becomes very confusing
- existing_layers = \
- project.get_all_compatible_layer_versions().values_list(
- "layer__name",
- flat=True)
- add_to_project = False
- layer_deps_added = []
- if 'add_to_project' in layer_data:
- add_to_project = True
- if layer_data['name'] in existing_layers:
- return JsonResponse({"error": "layer-name-exists"})
- if ('local_source_dir' in layer_data):
- # Local layer can be shared across projects. They have no 'release'
- # and are not included in get_all_compatible_layer_versions() above
- layer,created = Layer.objects.get_or_create(name=layer_data['name'])
- _log("Local Layer created=%s" % created)
- else:
- layer = Layer.objects.create(name=layer_data['name'])
- layer_version = Layer_Version.objects.create(
- layer=layer,
- project=project,
- layer_source=LayerSource.TYPE_IMPORTED)
- # Local layer
- if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
- layer.local_source_dir = layer_data['local_source_dir']
- # git layer
- elif 'vcs_url' in layer_data:
- layer.vcs_url = layer_data['vcs_url']
- layer_version.dirpath = layer_data['dir_path']
- layer_version.commit = layer_data['git_ref']
- layer_version.branch = layer_data['git_ref']
- layer.save()
- layer_version.save()
- if add_to_project:
- ProjectLayer.objects.get_or_create(
- layercommit=layer_version, project=project)
- # Add the layer dependencies
- if 'layer_deps' in layer_data:
- for layer_dep_id in layer_data['layer_deps'].split(","):
- layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
- LayerVersionDependency.objects.get_or_create(
- layer_version=layer_version, depends_on=layer_dep)
- # Add layer deps to the project if specified
- if add_to_project:
- created, pl = ProjectLayer.objects.get_or_create(
- layercommit=layer_dep, project=project)
- layer_deps_added.append(
- {'name': layer_dep.layer.name,
- 'layerdetailurl':
- layer_dep.get_detailspage_url(project.pk)})
- # Scan the layer's content and update components
- scan_layer_content(layer,layer_version)
- except Layer_Version.DoesNotExist:
- return error_response("layer-dep-not-found")
- except Project.DoesNotExist:
- return error_response("project-not-found")
- except KeyError:
- return error_response("incorrect-parameters")
- return JsonResponse({'error': "ok",
- 'imported_layer': {
- 'name': layer.name,
- 'layerdetailurl':
- layer_version.get_detailspage_url()},
- 'deps_added': layer_deps_added})
- def delete(self, request, *args, **kwargs):
- """ Delete an imported layer
- Method: DELETE
- Entry point: /xhr_layer/<projed id>/<layerversion_id>
- """
- try:
- # We currently only allow Imported layers to be deleted
- layer_version = Layer_Version.objects.get(
- id=kwargs['layerversion_id'],
- project=kwargs['pid'],
- layer_source=LayerSource.TYPE_IMPORTED)
- except Layer_Version.DoesNotExist:
- return error_response("Cannot find imported layer to delete")
- try:
- ProjectLayer.objects.get(project=kwargs['pid'],
- layercommit=layer_version).delete()
- except ProjectLayer.DoesNotExist:
- pass
- layer_version.layer.delete()
- layer_version.delete()
- return JsonResponse({
- "error": "ok",
- "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],))
- })
- class XhrCustomRecipe(View):
- """ Create a custom image recipe """
- def post(self, request, *args, **kwargs):
- """
- Custom image recipe REST API
- Entry point: /xhr_customrecipe/
- Method: POST
- Args:
- name: name of custom recipe to create
- project: target project id of orm.models.Project
- base: base recipe id of orm.models.Recipe
- Returns:
- {"error": "ok",
- "url": <url of the created recipe>}
- or
- {"error": <error message>}
- """
- # check if request has all required parameters
- for param in ('name', 'project', 'base'):
- if param not in request.POST:
- return error_response("Missing parameter '%s'" % param)
- # get project and baserecipe objects
- params = {}
- for name, model in [("project", Project),
- ("base", Recipe)]:
- value = request.POST[name]
- try:
- params[name] = model.objects.get(id=value)
- except model.DoesNotExist:
- return error_response("Invalid %s id %s" % (name, value))
- # create custom recipe
- try:
- # Only allowed chars in name are a-z, 0-9 and -
- if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
- return error_response("invalid-name")
- custom_images = CustomImageRecipe.objects.all()
- # Are there any recipes with this name already in our project?
- existing_image_recipes_in_project = custom_images.filter(
- name=request.POST["name"], project=params["project"])
- if existing_image_recipes_in_project.count() > 0:
- return error_response("image-already-exists")
- # Are there any recipes with this name which aren't custom
- # image recipes?
- custom_image_ids = custom_images.values_list('id', flat=True)
- existing_non_image_recipes = Recipe.objects.filter(
- Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
- )
- if existing_non_image_recipes.count() > 0:
- return error_response("recipe-already-exists")
- # create layer 'Custom layer' and verion if needed
- layer, l_created = Layer.objects.get_or_create(
- name=CustomImageRecipe.LAYER_NAME,
- summary="Layer for custom recipes")
- if l_created:
- layer.local_source_dir = "toaster_created_layer"
- layer.save()
- # Check if we have a layer version already
- # We don't use get_or_create here because the dirpath will change
- # and is a required field
- lver = Layer_Version.objects.filter(Q(project=params['project']) &
- Q(layer=layer) &
- Q(build=None)).last()
- if lver is None:
- lver, lv_created = Layer_Version.objects.get_or_create(
- project=params['project'],
- layer=layer,
- layer_source=LayerSource.TYPE_LOCAL,
- dirpath="toaster_created_layer")
- # Add a dependency on our layer to the base recipe's layer
- LayerVersionDependency.objects.get_or_create(
- layer_version=lver,
- depends_on=params["base"].layer_version)
- # Add it to our current project if needed
- ProjectLayer.objects.get_or_create(project=params['project'],
- layercommit=lver,
- optional=False)
- # Create the actual recipe
- recipe, r_created = CustomImageRecipe.objects.get_or_create(
- name=request.POST["name"],
- base_recipe=params["base"],
- project=params["project"],
- layer_version=lver,
- is_image=True)
- # If we created the object then setup these fields. They may get
- # overwritten later on and cause the get_or_create to create a
- # duplicate if they've changed.
- if r_created:
- recipe.file_path = request.POST["name"]
- recipe.license = "MIT"
- recipe.version = "0.1"
- recipe.save()
- except Error as err:
- return error_response("Can't create custom recipe: %s" % err)
- # Find the package list from the last build of this recipe/target
- target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
- Q(build__project=params['project']) &
- (Q(target=params['base'].name) |
- Q(target=recipe.name))).last()
- if target:
- # Copy in every package
- # We don't want these packages to be linked to anything because
- # that underlying data may change e.g. delete a build
- for tpackage in target.target_installed_package_set.all():
- try:
- built_package = tpackage.package
- # The package had no recipe information so is a ghost
- # package skip it
- if built_package.recipe is None:
- continue
- config_package = CustomImagePackage.objects.get(
- name=built_package.name)
- recipe.includes_set.add(config_package)
- except Exception as e:
- logger.warning("Error adding package %s %s" %
- (tpackage.package.name, e))
- pass
- # pre-create layer directory structure, so that other builds
- # are not blocked by this new recipe dependecy
- # NOTE: this is parallel code to 'localhostbecontroller.py'
- be = BuildEnvironment.objects.all()[0]
- layerpath = os.path.join(be.builddir,
- CustomImageRecipe.LAYER_NAME)
- for name in ("conf", "recipes"):
- path = os.path.join(layerpath, name)
- if not os.path.isdir(path):
- os.makedirs(path)
- # pre-create layer.conf
- config = os.path.join(layerpath, "conf", "layer.conf")
- if not os.path.isfile(config):
- with open(config, "w") as conf:
- conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
- # pre-create new image's recipe file
- recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
- recipe.name)
- with open(recipe_path, "w") as recipef:
- content = recipe.generate_recipe_file_contents()
- if not content:
- # Delete this incomplete image recipe object
- recipe.delete()
- return error_response("recipe-parent-not-exist")
- else:
- recipef.write(recipe.generate_recipe_file_contents())
- return JsonResponse(
- {"error": "ok",
- "packages": recipe.get_all_packages().count(),
- "url": reverse('customrecipe', args=(params['project'].pk,
- recipe.id))})
- class XhrCustomRecipeId(View):
- """
- Set of ReST API processors working with recipe id.
- Entry point: /xhr_customrecipe/<recipe_id>
- Methods:
- GET - Get details of custom image recipe
- DELETE - Delete custom image recipe
- Returns:
- GET:
- {"error": "ok",
- "info": dictionary of field name -> value pairs
- of the CustomImageRecipe model}
- DELETE:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- @staticmethod
- def _get_ci_recipe(recipe_id):
- """ Get Custom Image recipe or return an error response"""
- try:
- custom_recipe = \
- CustomImageRecipe.objects.get(pk=recipe_id)
- return custom_recipe, None
- except CustomImageRecipe.DoesNotExist:
- return None, error_response("Custom recipe with id=%s "
- "not found" % recipe_id)
- def get(self, request, *args, **kwargs):
- custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
- if error:
- return error
- if request.method == 'GET':
- info = {"id": custom_recipe.id,
- "name": custom_recipe.name,
- "base_recipe_id": custom_recipe.base_recipe.id,
- "project_id": custom_recipe.project.id}
- return JsonResponse({"error": "ok", "info": info})
- def delete(self, request, *args, **kwargs):
- custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
- if error:
- return error
- project = custom_recipe.project
- custom_recipe.delete()
- return JsonResponse({"error": "ok",
- "gotoUrl": reverse("projectcustomimages",
- args=(project.pk,))})
- class XhrCustomRecipePackages(View):
- """
- ReST API to add/remove packages to/from custom recipe.
- Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
- Methods:
- PUT - Add package to the recipe
- DELETE - Delete package from the recipe
- GET - Get package information
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- @staticmethod
- def _get_package(package_id):
- try:
- package = CustomImagePackage.objects.get(pk=package_id)
- return package, None
- except Package.DoesNotExist:
- return None, error_response("Package with id=%s "
- "not found" % package_id)
- def _traverse_dependents(self, next_package_id,
- rev_deps, all_current_packages, tree_level=0):
- """
- Recurse through reverse dependency tree for next_package_id.
- Limit the reverse dependency search to packages not already scanned,
- that is, not already in rev_deps.
- Limit the scan to a depth (tree_level) not exceeding the count of
- all packages in the custom image, and if that depth is exceeded
- return False, pop out of the recursion, and write a warning
- to the log, but this is unlikely, suggesting a dependency loop
- not caught by bitbake.
- On return, the input/output arg rev_deps is appended with queryset
- dictionary elements, annotated for use in the customimage template.
- The list has unsorted, but unique elements.
- """
- max_dependency_tree_depth = all_current_packages.count()
- if tree_level >= max_dependency_tree_depth:
- logger.warning(
- "The number of reverse dependencies "
- "for this package exceeds " + max_dependency_tree_depth +
- " and the remaining reverse dependencies will not be removed")
- return True
- package = CustomImagePackage.objects.get(id=next_package_id)
- dependents = \
- package.package_dependencies_target.annotate(
- name=F('package__name'),
- pk=F('package__pk'),
- size=F('package__size'),
- ).values("name", "pk", "size").exclude(
- ~Q(pk__in=all_current_packages)
- )
- for pkg in dependents:
- if pkg in rev_deps:
- # already seen, skip dependent search
- continue
- rev_deps.append(pkg)
- if (self._traverse_dependents(pkg["pk"], rev_deps,
- all_current_packages,
- tree_level+1)):
- return True
- return False
- def _get_all_dependents(self, package_id, all_current_packages):
- """
- Returns sorted list of recursive reverse dependencies for package_id,
- as a list of dictionary items, by recursing through dependency
- relationships.
- """
- rev_deps = []
- self._traverse_dependents(package_id, rev_deps, all_current_packages)
- rev_deps = sorted(rev_deps, key=lambda x: x["name"])
- return rev_deps
- def get(self, request, *args, **kwargs):
- recipe, error = XhrCustomRecipeId._get_ci_recipe(
- kwargs['recipe_id'])
- if error:
- return error
- # If no package_id then list all the current packages
- if not kwargs['package_id']:
- total_size = 0
- packages = recipe.get_all_packages().values("id",
- "name",
- "version",
- "size")
- for package in packages:
- package['size_formatted'] = \
- filtered_filesizeformat(package['size'])
- total_size += package['size']
- return JsonResponse({"error": "ok",
- "packages": list(packages),
- "total": len(packages),
- "total_size": total_size,
- "total_size_formatted":
- filtered_filesizeformat(total_size)})
- else:
- package, error = XhrCustomRecipePackages._get_package(
- kwargs['package_id'])
- if error:
- return error
- all_current_packages = recipe.get_all_packages()
- # Dependencies for package which aren't satisfied by the
- # current packages in the custom image recipe
- deps = package.package_dependencies_source.for_target_or_none(
- recipe.name)['packages'].annotate(
- name=F('depends_on__name'),
- pk=F('depends_on__pk'),
- size=F('depends_on__size'),
- ).values("name", "pk", "size").filter(
- # There are two depends types we don't know why
- (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
- Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
- ~Q(pk__in=all_current_packages)
- )
- # Reverse dependencies which are needed by packages that are
- # in the image. Recursive search providing all dependents,
- # not just immediate dependents.
- reverse_deps = self._get_all_dependents(kwargs['package_id'],
- all_current_packages)
- total_size_deps = 0
- total_size_reverse_deps = 0
- for dep in deps:
- dep['size_formatted'] = \
- filtered_filesizeformat(dep['size'])
- total_size_deps += dep['size']
- for dep in reverse_deps:
- dep['size_formatted'] = \
- filtered_filesizeformat(dep['size'])
- total_size_reverse_deps += dep['size']
- return JsonResponse(
- {"error": "ok",
- "id": package.pk,
- "name": package.name,
- "version": package.version,
- "unsatisfied_dependencies": list(deps),
- "unsatisfied_dependencies_size": total_size_deps,
- "unsatisfied_dependencies_size_formatted":
- filtered_filesizeformat(total_size_deps),
- "reverse_dependencies": list(reverse_deps),
- "reverse_dependencies_size": total_size_reverse_deps,
- "reverse_dependencies_size_formatted":
- filtered_filesizeformat(total_size_reverse_deps)})
- def put(self, request, *args, **kwargs):
- recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
- package, error = self._get_package(kwargs['package_id'])
- if error:
- return error
- included_packages = recipe.includes_set.values_list('pk',
- flat=True)
- # If we're adding back a package which used to be included in this
- # image all we need to do is remove it from the excludes
- if package.pk in included_packages:
- try:
- recipe.excludes_set.remove(package)
- return {"error": "ok"}
- except Package.DoesNotExist:
- return error_response("Package %s not found in excludes"
- " but was in included list" %
- package.name)
- else:
- recipe.appends_set.add(package)
- # Make sure that package is not in the excludes set
- try:
- recipe.excludes_set.remove(package)
- except:
- pass
- # Add the dependencies we think will be added to the recipe
- # as a result of appending this package.
- # TODO this should recurse down the entire deps tree
- for dep in package.package_dependencies_source.all_depends():
- try:
- cust_package = CustomImagePackage.objects.get(
- name=dep.depends_on.name)
- recipe.includes_set.add(cust_package)
- try:
- # When adding the pre-requisite package, make
- # sure it's not in the excluded list from a
- # prior removal.
- recipe.excludes_set.remove(cust_package)
- except package.DoesNotExist:
- # Don't care if the package had never been excluded
- pass
- except:
- logger.warning("Could not add package's suggested"
- "dependencies to the list")
- return JsonResponse({"error": "ok"})
- def delete(self, request, *args, **kwargs):
- recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
- package, error = self._get_package(kwargs['package_id'])
- if error:
- return error
- try:
- included_packages = recipe.includes_set.values_list('pk',
- flat=True)
- # If we're deleting a package which is included we need to
- # Add it to the excludes list.
- if package.pk in included_packages:
- recipe.excludes_set.add(package)
- else:
- recipe.appends_set.remove(package)
- # remove dependencies as well
- all_current_packages = recipe.get_all_packages()
- reverse_deps_dictlist = self._get_all_dependents(
- package.pk,
- all_current_packages)
- ids = [entry['pk'] for entry in reverse_deps_dictlist]
- reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
- for r in reverse_deps:
- try:
- if r.id in included_packages:
- recipe.excludes_set.add(r)
- else:
- recipe.appends_set.remove(r)
- except:
- pass
- return JsonResponse({"error": "ok"})
- except CustomImageRecipe.DoesNotExist:
- return error_response("Tried to remove package that wasn't"
- " present")
- class XhrProject(View):
- """ Create, delete or edit a project
- Entry point: /xhr_project/<project_id>
- """
- def post(self, request, *args, **kwargs):
- """
- Edit project control
- Args:
- layerAdd = layer_version_id layer_version_id ...
- layerDel = layer_version_id layer_version_id ...
- projectName = new_project_name
- machineName = new_machine_name
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- try:
- prj = Project.objects.get(pk=kwargs['project_id'])
- except Project.DoesNotExist:
- return error_response("No such project")
- # Add layers
- if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0:
- for layer_version_id in request.POST['layerAdd'].split(','):
- try:
- lv = Layer_Version.objects.get(pk=int(layer_version_id))
- ProjectLayer.objects.get_or_create(project=prj,
- layercommit=lv)
- except Layer_Version.DoesNotExist:
- return error_response("Layer version %s asked to add "
- "doesn't exist" % layer_version_id)
- # Remove layers
- if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0:
- layer_version_ids = request.POST['layerDel'].split(',')
- ProjectLayer.objects.filter(
- project=prj,
- layercommit_id__in=layer_version_ids).delete()
- # Project name change
- if 'projectName' in request.POST:
- prj.name = request.POST['projectName']
- prj.save()
- # Machine name change
- if 'machineName' in request.POST:
- machinevar = prj.projectvariable_set.get(name="MACHINE")
- machinevar.value = request.POST['machineName']
- machinevar.save()
- # Distro name change
- if 'distroName' in request.POST:
- distrovar = prj.projectvariable_set.get(name="DISTRO")
- distrovar.value = request.POST['distroName']
- distrovar.save()
- return JsonResponse({"error": "ok"})
- def get(self, request, *args, **kwargs):
- """
- Returns:
- json object representing the current project
- or:
- {"error": <error message>}
- """
- try:
- project = Project.objects.get(pk=kwargs['project_id'])
- except Project.DoesNotExist:
- return error_response("Project %s does not exist" %
- kwargs['project_id'])
- # Create the frequently built targets list
- freqtargets = Counter(Target.objects.filter(
- Q(build__project=project),
- ~Q(build__outcome=Build.IN_PROGRESS)
- ).order_by("target").values_list("target", flat=True))
- freqtargets = freqtargets.most_common(5)
- # We now have the targets in order of frequency but if there are two
- # with the same frequency then we need to make sure those are in
- # alphabetical order without losing the frequency ordering
- tmp = []
- switch = None
- for i, freqtartget in enumerate(freqtargets):
- target, count = freqtartget
- try:
- target_next, count_next = freqtargets[i+1]
- if count == count_next and target > target_next:
- switch = target
- continue
- except IndexError:
- pass
- tmp.append(target)
- if switch:
- tmp.append(switch)
- switch = None
- freqtargets = tmp
- layers = []
- for layer in project.projectlayer_set.all():
- layers.append({
- "id": layer.layercommit.pk,
- "name": layer.layercommit.layer.name,
- "vcs_url": layer.layercommit.layer.vcs_url,
- "local_source_dir": layer.layercommit.layer.local_source_dir,
- "vcs_reference": layer.layercommit.get_vcs_reference(),
- "url": layer.layercommit.layer.layer_index_url,
- "layerdetailurl": layer.layercommit.get_detailspage_url(
- project.pk),
- "xhrLayerUrl": reverse("xhr_layer",
- args=(project.pk,
- layer.layercommit.pk)),
- "layersource": layer.layercommit.layer_source
- })
- data = {
- "name": project.name,
- "layers": layers,
- "freqtargets": freqtargets,
- }
- if project.release is not None:
- data['release'] = {
- "id": project.release.pk,
- "name": project.release.name,
- "description": project.release.description
- }
- try:
- data["machine"] = {"name":
- project.projectvariable_set.get(
- name="MACHINE").value}
- except ProjectVariable.DoesNotExist:
- data["machine"] = None
- try:
- data["distro"] = {"name":
- project.projectvariable_set.get(
- name="DISTRO").value}
- except ProjectVariable.DoesNotExist:
- data["distro"] = None
- data['error'] = "ok"
- return JsonResponse(data)
- def put(self, request, *args, **kwargs):
- # TODO create new project api
- return HttpResponse()
- def delete(self, request, *args, **kwargs):
- """Delete a project. Cancels any builds in progress"""
- try:
- project = Project.objects.get(pk=kwargs['project_id'])
- # Cancel any builds in progress
- for br in BuildRequest.objects.filter(
- project=project,
- state=BuildRequest.REQ_INPROGRESS):
- XhrBuildRequest.cancel_build(br)
- # gather potential orphaned local layers attached to this project
- project_local_layer_list = []
- for pl in ProjectLayer.objects.filter(project=project):
- if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
- project_local_layer_list.append(pl.layercommit.layer)
- # deep delete the project and its dependencies
- project.delete()
- # delete any local layers now orphaned
- _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
- for layer in project_local_layer_list:
- layer_refs = Layer_Version.objects.filter(layer=layer)
- _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
- if 0 == len(layer_refs):
- _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
- Layer.objects.filter(pk=layer.id).delete()
- except Project.DoesNotExist:
- return error_response("Project %s does not exist" %
- kwargs['project_id'])
- return JsonResponse({
- "error": "ok",
- "gotoUrl": reverse("all-projects", args=[])
- })
- class XhrBuild(View):
- """ Delete a build object
- Entry point: /xhr_build/<build_id>
- """
- def delete(self, request, *args, **kwargs):
- """
- Delete build data
- Args:
- build_id = build_id
- Returns:
- {"error": "ok"}
- or
- {"error": <error message>}
- """
- try:
- build = Build.objects.get(pk=kwargs['build_id'])
- project = build.project
- build.delete()
- except Build.DoesNotExist:
- return error_response("Build %s does not exist" %
- kwargs['build_id'])
- return JsonResponse({
- "error": "ok",
- "gotoUrl": reverse("projectbuilds", args=(project.pk,))
- })
|