123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875 |
- #
- # BitBake Toaster Implementation
- #
- # Copyright (C) 2013 Intel Corporation
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- from __future__ import unicode_literals
- from django.db import models, IntegrityError, DataError
- from django.db.models import F, Q, Sum, Count
- from django.utils import timezone
- from django.utils.encoding import force_bytes
- from django.core.urlresolvers import reverse
- from django.core import validators
- from django.conf import settings
- import django.db.models.signals
- import sys
- import os
- import re
- import itertools
- from signal import SIGUSR1
- import logging
- logger = logging.getLogger("toaster")
- if 'sqlite' in settings.DATABASES['default']['ENGINE']:
- from django.db import transaction, OperationalError
- from time import sleep
- _base_save = models.Model.save
- def save(self, *args, **kwargs):
- while True:
- try:
- with transaction.atomic():
- return _base_save(self, *args, **kwargs)
- except OperationalError as err:
- if 'database is locked' in str(err):
- logger.warning("%s, model: %s, args: %s, kwargs: %s",
- err, self.__class__, args, kwargs)
- sleep(0.5)
- continue
- raise
- models.Model.save = save
- # HACK: Monkey patch Django to fix 'database is locked' issue
- from django.db.models.query import QuerySet
- _base_insert = QuerySet._insert
- def _insert(self, *args, **kwargs):
- with transaction.atomic(using=self.db, savepoint=False):
- return _base_insert(self, *args, **kwargs)
- QuerySet._insert = _insert
- from django.utils import six
- def _create_object_from_params(self, lookup, params):
- """
- Tries to create an object using passed params.
- Used by get_or_create and update_or_create
- """
- try:
- obj = self.create(**params)
- return obj, True
- except (IntegrityError, DataError):
- exc_info = sys.exc_info()
- try:
- return self.get(**lookup), False
- except self.model.DoesNotExist:
- pass
- six.reraise(*exc_info)
- QuerySet._create_object_from_params = _create_object_from_params
- # end of HACK
- class GitURLValidator(validators.URLValidator):
- import re
- regex = re.compile(
- r'^(?:ssh|git|http|ftp)s?://' # http:// or https://
- r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
- r'localhost|' # localhost...
- r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
- r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
- r'(?::\d+)?' # optional port
- r'(?:/?|[/?]\S+)$', re.IGNORECASE)
- def GitURLField(**kwargs):
- r = models.URLField(**kwargs)
- for i in range(len(r.validators)):
- if isinstance(r.validators[i], validators.URLValidator):
- r.validators[i] = GitURLValidator()
- return r
- class ToasterSetting(models.Model):
- name = models.CharField(max_length=63)
- helptext = models.TextField()
- value = models.CharField(max_length=255)
- def __unicode__(self):
- return "Setting %s = %s" % (self.name, self.value)
- class ProjectManager(models.Manager):
- def create_project(self, name, release, existing_project=None):
- if existing_project and (release is not None):
- prj = existing_project
- prj.bitbake_version = release.bitbake_version
- prj.release = release
- # Delete the previous ProjectLayer mappings
- for pl in ProjectLayer.objects.filter(project=prj):
- pl.delete()
- elif release is not None:
- prj = self.model(name=name,
- bitbake_version=release.bitbake_version,
- release=release)
- else:
- prj = self.model(name=name,
- bitbake_version=None,
- release=None)
- prj.save()
- for defaultconf in ToasterSetting.objects.filter(
- name__startswith="DEFCONF_"):
- name = defaultconf.name[8:]
- pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name)
- pv.value = defaultconf.value
- pv.save()
- if release is None:
- return prj
- for rdl in release.releasedefaultlayer_set.all():
- lv = Layer_Version.objects.filter(
- layer__name=rdl.layer_name,
- release=release).first()
- if lv:
- ProjectLayer.objects.create(project=prj,
- layercommit=lv,
- optional=False)
- else:
- logger.warning("Default project layer %s not found" %
- rdl.layer_name)
- return prj
- # return single object with is_default = True
- def get_or_create_default_project(self):
- projects = super(ProjectManager, self).filter(is_default=True)
- if len(projects) > 1:
- raise Exception('Inconsistent project data: multiple ' +
- 'default projects (i.e. with is_default=True)')
- elif len(projects) < 1:
- options = {
- 'name': 'Command line builds',
- 'short_description':
- 'Project for builds started outside Toaster',
- 'is_default': True
- }
- project = Project.objects.create(**options)
- project.save()
- return project
- else:
- return projects[0]
- class Project(models.Model):
- search_allowed_fields = ['name', 'short_description', 'release__name',
- 'release__branch_name']
- name = models.CharField(max_length=100)
- short_description = models.CharField(max_length=50, blank=True)
- bitbake_version = models.ForeignKey('BitbakeVersion', null=True)
- release = models.ForeignKey("Release", null=True)
- created = models.DateTimeField(auto_now_add=True)
- updated = models.DateTimeField(auto_now=True)
- # This is a horrible hack; since Toaster has no "User" model available when
- # running in interactive mode, we can't reference the field here directly
- # Instead, we keep a possible null reference to the User id,
- # as not to force
- # hard links to possibly missing models
- user_id = models.IntegerField(null=True)
- objects = ProjectManager()
- # build directory override (e.g. imported)
- builddir = models.TextField()
- # merge the Toaster configure attributes directly into the standard conf files
- merged_attr = models.BooleanField(default=False)
- # set to True for the project which is the default container
- # for builds initiated by the command line etc.
- is_default= models.BooleanField(default=False)
- def __unicode__(self):
- return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
- def get_current_machine_name(self):
- try:
- return self.projectvariable_set.get(name="MACHINE").value
- except (ProjectVariable.DoesNotExist,IndexError):
- return None;
- def get_number_of_builds(self):
- """Return the number of builds which have ended"""
- return self.build_set.exclude(
- Q(outcome=Build.IN_PROGRESS) |
- Q(outcome=Build.CANCELLED)
- ).count()
- def get_last_build_id(self):
- try:
- return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id
- except (Build.DoesNotExist,IndexError):
- return( -1 )
- def get_last_outcome(self):
- build_id = self.get_last_build_id()
- if (-1 == build_id):
- return( "" )
- try:
- return Build.objects.filter( id = build_id )[ 0 ].outcome
- except (Build.DoesNotExist,IndexError):
- return( "not_found" )
- def get_last_target(self):
- build_id = self.get_last_build_id()
- if (-1 == build_id):
- return( "" )
- try:
- return Target.objects.filter(build = build_id)[0].target
- except (Target.DoesNotExist,IndexError):
- return( "not_found" )
- def get_last_errors(self):
- build_id = self.get_last_build_id()
- if (-1 == build_id):
- return( 0 )
- try:
- return Build.objects.filter(id = build_id)[ 0 ].errors.count()
- except (Build.DoesNotExist,IndexError):
- return( "not_found" )
- def get_last_warnings(self):
- build_id = self.get_last_build_id()
- if (-1 == build_id):
- return( 0 )
- try:
- return Build.objects.filter(id = build_id)[ 0 ].warnings.count()
- except (Build.DoesNotExist,IndexError):
- return( "not_found" )
- def get_last_build_extensions(self):
- """
- Get list of file name extensions for images produced by the most
- recent build
- """
- last_build = Build.objects.get(pk = self.get_last_build_id())
- return last_build.get_image_file_extensions()
- def get_last_imgfiles(self):
- build_id = self.get_last_build_id()
- if (-1 == build_id):
- return( "" )
- try:
- return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value
- except (Variable.DoesNotExist,IndexError):
- return( "not_found" )
- def get_all_compatible_layer_versions(self):
- """ Returns Queryset of all Layer_Versions which are compatible with
- this project"""
- queryset = None
- # guard on release, as it can be null
- if self.release:
- queryset = Layer_Version.objects.filter(
- (Q(release=self.release) &
- Q(build=None) &
- Q(project=None)) |
- Q(project=self))
- else:
- queryset = Layer_Version.objects.none()
- return queryset
- def get_project_layer_versions(self, pk=False):
- """ Returns the Layer_Versions currently added to this project """
- layer_versions = self.projectlayer_set.all().values_list('layercommit',
- flat=True)
- if pk is False:
- return Layer_Version.objects.filter(pk__in=layer_versions)
- else:
- return layer_versions
- def get_default_image_recipe(self):
- try:
- return self.projectvariable_set.get(name="DEFAULT_IMAGE").value
- except (ProjectVariable.DoesNotExist,IndexError):
- return None;
- def get_is_new(self):
- return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW)
- def get_available_machines(self):
- """ Returns QuerySet of all Machines which are provided by the
- Layers currently added to the Project """
- queryset = Machine.objects.filter(
- layer_version__in=self.get_project_layer_versions())
- return queryset
- def get_all_compatible_machines(self):
- """ Returns QuerySet of all the compatible machines available to the
- project including ones from Layers not currently added """
- queryset = Machine.objects.filter(
- layer_version__in=self.get_all_compatible_layer_versions())
- return queryset
- def get_available_distros(self):
- """ Returns QuerySet of all Distros which are provided by the
- Layers currently added to the Project """
- queryset = Distro.objects.filter(
- layer_version__in=self.get_project_layer_versions())
- return queryset
- def get_all_compatible_distros(self):
- """ Returns QuerySet of all the compatible Wind River distros available to the
- project including ones from Layers not currently added """
- queryset = Distro.objects.filter(
- layer_version__in=self.get_all_compatible_layer_versions())
- return queryset
- def get_available_recipes(self):
- """ Returns QuerySet of all the recipes that are provided by layers
- added to this project """
- queryset = Recipe.objects.filter(
- layer_version__in=self.get_project_layer_versions())
- return queryset
- def get_all_compatible_recipes(self):
- """ Returns QuerySet of all the compatible Recipes available to the
- project including ones from Layers not currently added """
- queryset = Recipe.objects.filter(
- layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='')
- return queryset
- # Project Specific status management
- PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS'
- PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK'
- PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW'
- PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE'
- PROJECT_SPECIFIC_NONE = ''
- PROJECT_SPECIFIC_NEW = '1'
- PROJECT_SPECIFIC_EDIT = '2'
- PROJECT_SPECIFIC_CLONING = '3'
- PROJECT_SPECIFIC_CLONING_SUCCESS = '4'
- PROJECT_SPECIFIC_CLONING_FAIL = '5'
- def get_variable(self,variable,default_value = ''):
- try:
- return self.projectvariable_set.get(name=variable).value
- except (ProjectVariable.DoesNotExist,IndexError):
- return default_value
- def set_variable(self,variable,value):
- pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable)
- pv.value = value
- pv.save()
- def get_default_image(self):
- return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
- def schedule_build(self):
- from bldcontrol.models import BuildRequest, BRTarget, BRLayer
- from bldcontrol.models import BRBitbake, BRVariable
- try:
- now = timezone.now()
- build = Build.objects.create(project=self,
- completed_on=now,
- started_on=now)
- br = BuildRequest.objects.create(project=self,
- state=BuildRequest.REQ_QUEUED,
- build=build)
- BRBitbake.objects.create(req=br,
- giturl=self.bitbake_version.giturl,
- commit=self.bitbake_version.branch,
- dirpath=self.bitbake_version.dirpath)
- for t in self.projecttarget_set.all():
- BRTarget.objects.create(req=br, target=t.target, task=t.task)
- Target.objects.create(build=br.build, target=t.target,
- task=t.task)
- # If we're about to build a custom image recipe make sure
- # that layer is currently in the project before we create the
- # BRLayer objects
- customrecipe = CustomImageRecipe.objects.filter(
- name=t.target,
- project=self).first()
- if customrecipe:
- ProjectLayer.objects.get_or_create(
- project=self,
- layercommit=customrecipe.layer_version,
- optional=False)
- for l in self.projectlayer_set.all().order_by("pk"):
- commit = l.layercommit.get_vcs_reference()
- logger.debug("Adding layer to build %s" %
- l.layercommit.layer.name)
- BRLayer.objects.create(
- req=br,
- name=l.layercommit.layer.name,
- giturl=l.layercommit.layer.vcs_url,
- commit=commit,
- dirpath=l.layercommit.dirpath,
- layer_version=l.layercommit,
- local_source_dir=l.layercommit.layer.local_source_dir
- )
- for v in self.projectvariable_set.all():
- BRVariable.objects.create(req=br, name=v.name, value=v.value)
- try:
- br.build.machine = self.projectvariable_set.get(
- name='MACHINE').value
- br.build.save()
- except ProjectVariable.DoesNotExist:
- pass
- br.save()
- signal_runbuilds()
- except Exception:
- # revert the build request creation since we're not done cleanly
- br.delete()
- raise
- return br
- class Build(models.Model):
- SUCCEEDED = 0
- FAILED = 1
- IN_PROGRESS = 2
- CANCELLED = 3
- BUILD_OUTCOME = (
- (SUCCEEDED, 'Succeeded'),
- (FAILED, 'Failed'),
- (IN_PROGRESS, 'In Progress'),
- (CANCELLED, 'Cancelled'),
- )
- search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"]
- project = models.ForeignKey(Project) # must have a project
- machine = models.CharField(max_length=100)
- distro = models.CharField(max_length=100)
- distro_version = models.CharField(max_length=100)
- started_on = models.DateTimeField()
- completed_on = models.DateTimeField()
- outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
- cooker_log_path = models.CharField(max_length=500)
- build_name = models.CharField(max_length=100, default='')
- bitbake_version = models.CharField(max_length=50)
- # number of recipes to parse for this build
- recipes_to_parse = models.IntegerField(default=1)
- # number of recipes parsed so far for this build
- recipes_parsed = models.IntegerField(default=1)
- # number of repos to clone for this build
- repos_to_clone = models.IntegerField(default=1)
- # number of repos cloned so far for this build (default off)
- repos_cloned = models.IntegerField(default=1)
- # Hint on current progress item
- progress_item = models.CharField(max_length=40)
- @staticmethod
- def get_recent(project=None):
- """
- Return recent builds as a list; if project is set, only return
- builds for that project
- """
- builds = Build.objects.all()
- if project:
- builds = builds.filter(project=project)
- finished_criteria = \
- Q(outcome=Build.SUCCEEDED) | \
- Q(outcome=Build.FAILED) | \
- Q(outcome=Build.CANCELLED)
- recent_builds = list(itertools.chain(
- builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"),
- builds.filter(finished_criteria).order_by("-completed_on")[:3]
- ))
- # add percentage done property to each build; this is used
- # to show build progress in mrb_section.html
- for build in recent_builds:
- build.percentDone = build.completeper()
- build.outcomeText = build.get_outcome_text()
- return recent_builds
- def started(self):
- """
- As build variables are only added for a build when its BuildStarted event
- is received, a build with no build variables is counted as
- "in preparation" and not properly started yet. This method
- will return False if a build has no build variables (it never properly
- started), or True otherwise.
- Note that this is a temporary workaround for the fact that we don't
- have a fine-grained state variable on a build which would allow us
- to record "in progress" (BuildStarted received) vs. "in preparation".
- """
- variables = Variable.objects.filter(build=self)
- return len(variables) > 0
- def completeper(self):
- tf = Task.objects.filter(build = self)
- tfc = tf.count()
- if tfc > 0:
- completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
- else:
- completeper = 0
- return completeper
- def eta(self):
- eta = timezone.now()
- completeper = self.completeper()
- if self.completeper() > 0:
- eta += ((eta - self.started_on)*(100-completeper))/completeper
- return eta
- def has_images(self):
- """
- Returns True if at least one of the targets for this build has an
- image file associated with it, False otherwise
- """
- targets = Target.objects.filter(build_id=self.id)
- has_images = False
- for target in targets:
- if target.has_images():
- has_images = True
- break
- return has_images
- def has_image_recipes(self):
- """
- Returns True if a build has any targets which were built from
- image recipes.
- """
- image_recipes = self.get_image_recipes()
- return len(image_recipes) > 0
- def get_image_file_extensions(self):
- """
- Get string of file name extensions for images produced by this build;
- note that this is the actual list of extensions stored on Target objects
- for this build, and not the value of IMAGE_FSTYPES.
- Returns comma-separated string, e.g. "vmdk, ext4"
- """
- extensions = []
- targets = Target.objects.filter(build_id = self.id)
- for target in targets:
- if not target.is_image:
- continue
- target_image_files = Target_Image_File.objects.filter(
- target_id=target.id)
- for target_image_file in target_image_files:
- extensions.append(target_image_file.suffix)
- extensions = list(set(extensions))
- extensions.sort()
- return ', '.join(extensions)
- def get_image_fstypes(self):
- """
- Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
- list of image file suffixes.
- """
- image_fstypes = Variable.objects.get(
- build=self, variable_name='IMAGE_FSTYPES').variable_value
- return list(set(re.split(r' {1,}', image_fstypes)))
- def get_sorted_target_list(self):
- tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
- return( tgts );
- def get_recipes(self):
- """
- Get the recipes related to this build;
- note that the related layer versions and layers are also prefetched
- by this query, as this queryset can be sorted by these objects in the
- build recipes view; prefetching them here removes the need
- for another query in that view
- """
- layer_versions = Layer_Version.objects.filter(build=self)
- criteria = Q(layer_version__id__in=layer_versions)
- return Recipe.objects.filter(criteria) \
- .select_related('layer_version', 'layer_version__layer')
- def get_image_recipes(self):
- """
- Returns a list of image Recipes (custom and built-in) related to this
- build, sorted by name; note that this has to be done in two steps, as
- there's no way to get all the custom image recipes and image recipes
- in one query
- """
- custom_image_recipes = self.get_custom_image_recipes()
- custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True)
- not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \
- Q(is_image=True)
- built_image_recipes = self.get_recipes().filter(not_custom_image_recipes)
- # append to the custom image recipes and sort
- customisable_image_recipes = list(
- itertools.chain(custom_image_recipes, built_image_recipes)
- )
- return sorted(customisable_image_recipes, key=lambda recipe: recipe.name)
- def get_custom_image_recipes(self):
- """
- Returns a queryset of CustomImageRecipes related to this build,
- sorted by name
- """
- built_recipe_names = self.get_recipes().values_list('name', flat=True)
- criteria = Q(name__in=built_recipe_names) & Q(project=self.project)
- queryset = CustomImageRecipe.objects.filter(criteria).order_by('name')
- return queryset
- def get_outcome_text(self):
- return Build.BUILD_OUTCOME[int(self.outcome)][1]
- @property
- def failed_tasks(self):
- """ Get failed tasks for the build """
- tasks = self.task_build.all()
- return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED)
- @property
- def errors(self):
- return (self.logmessage_set.filter(level=LogMessage.ERROR) |
- self.logmessage_set.filter(level=LogMessage.EXCEPTION) |
- self.logmessage_set.filter(level=LogMessage.CRITICAL))
- @property
- def warnings(self):
- return self.logmessage_set.filter(level=LogMessage.WARNING)
- @property
- def timespent(self):
- return self.completed_on - self.started_on
- @property
- def timespent_seconds(self):
- return self.timespent.total_seconds()
- @property
- def target_labels(self):
- """
- Sorted (a-z) "target1:task, target2, target3" etc. string for all
- targets in this build
- """
- targets = self.target_set.all()
- target_labels = [target.target +
- (':' + target.task if target.task else '')
- for target in targets]
- target_labels.sort()
- return target_labels
- def get_buildrequest(self):
- buildrequest = None
- if hasattr(self, 'buildrequest'):
- buildrequest = self.buildrequest
- return buildrequest
- def is_queued(self):
- from bldcontrol.models import BuildRequest
- buildrequest = self.get_buildrequest()
- if buildrequest:
- return buildrequest.state == BuildRequest.REQ_QUEUED
- else:
- return False
- def is_cancelling(self):
- from bldcontrol.models import BuildRequest
- buildrequest = self.get_buildrequest()
- if buildrequest:
- return self.outcome == Build.IN_PROGRESS and \
- buildrequest.state == BuildRequest.REQ_CANCELLING
- else:
- return False
- def is_cloning(self):
- """
- True if the build is still cloning repos
- """
- return self.outcome == Build.IN_PROGRESS and \
- self.repos_cloned < self.repos_to_clone
- def is_parsing(self):
- """
- True if the build is still parsing recipes
- """
- return self.outcome == Build.IN_PROGRESS and \
- self.recipes_parsed < self.recipes_to_parse
- def is_starting(self):
- """
- True if the build has no completed tasks yet and is still just starting
- tasks.
- Note that the mechanism for testing whether a Task is "done" is whether
- its outcome field is set, as per the completeper() method.
- """
- return self.outcome == Build.IN_PROGRESS and \
- self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
- def get_state(self):
- """
- Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
- 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
- dependent on the BuildRequest state).
- This works around the fact that we have BuildRequest states as well
- as Build states, but really we just want to know the state of the build.
- """
- if self.is_cancelling():
- return 'Cancelling';
- elif self.is_queued():
- return 'Queued'
- elif self.is_cloning():
- return 'Cloning'
- elif self.is_parsing():
- return 'Parsing'
- elif self.is_starting():
- return 'Starting'
- else:
- return self.get_outcome_text()
- def __str__(self):
- return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
- class ProjectTarget(models.Model):
- project = models.ForeignKey(Project)
- target = models.CharField(max_length=100)
- task = models.CharField(max_length=100, null=True)
- class Target(models.Model):
- search_allowed_fields = ['target', 'file_name']
- build = models.ForeignKey(Build)
- target = models.CharField(max_length=100)
- task = models.CharField(max_length=100, null=True)
- is_image = models.BooleanField(default = False)
- image_size = models.IntegerField(default=0)
- license_manifest_path = models.CharField(max_length=500, null=True)
- package_manifest_path = models.CharField(max_length=500, null=True)
- def package_count(self):
- return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
- def __unicode__(self):
- return self.target
- def get_similar_targets(self):
- """
- Get target sfor the same machine, task and target name
- (e.g. 'core-image-minimal') from a successful build for this project
- (but excluding this target).
- Note that we only look for targets built by this project because
- projects can have different configurations from each other, and put
- their artifacts in different directories.
- The possibility of error when retrieving candidate targets
- is minimised by the fact that bitbake will rebuild artifacts if MACHINE
- (or various other variables) change. In this case, there is no need to
- clone artifacts from another target, as those artifacts will have
- been re-generated for this target anyway.
- """
- query = ~Q(pk=self.pk) & \
- Q(target=self.target) & \
- Q(build__machine=self.build.machine) & \
- Q(build__outcome=Build.SUCCEEDED) & \
- Q(build__project=self.build.project)
- return Target.objects.filter(query)
- def get_similar_target_with_image_files(self):
- """
- Get the most recent similar target with Target_Image_Files associated
- with it, for the purpose of cloning those files onto this target.
- """
- similar_target = None
- candidates = self.get_similar_targets()
- if candidates.count() == 0:
- return similar_target
- task_subquery = Q(task=self.task)
- # we can look for a 'build' task if this task is a 'populate_sdk_ext'
- # task, as the latter also creates images; and vice versa; note that
- # 'build' targets can have their task set to '';
- # also note that 'populate_sdk' does not produce image files
- image_tasks = [
- '', # aka 'build'
- 'build',
- 'image',
- 'populate_sdk_ext'
- ]
- if self.task in image_tasks:
- task_subquery = Q(task__in=image_tasks)
- # annotate with the count of files, to exclude any targets which
- # don't have associated files
- candidates = candidates.annotate(num_files=Count('target_image_file'))
- query = task_subquery & Q(num_files__gt=0)
- candidates = candidates.filter(query)
- if candidates.count() > 0:
- candidates.order_by('build__completed_on')
- similar_target = candidates.last()
- return similar_target
- def get_similar_target_with_sdk_files(self):
- """
- Get the most recent similar target with TargetSDKFiles associated
- with it, for the purpose of cloning those files onto this target.
- """
- similar_target = None
- candidates = self.get_similar_targets()
- if candidates.count() == 0:
- return similar_target
- # annotate with the count of files, to exclude any targets which
- # don't have associated files
- candidates = candidates.annotate(num_files=Count('targetsdkfile'))
- query = Q(task=self.task) & Q(num_files__gt=0)
- candidates = candidates.filter(query)
- if candidates.count() > 0:
- candidates.order_by('build__completed_on')
- similar_target = candidates.last()
- return similar_target
- def clone_image_artifacts_from(self, target):
- """
- Make clones of the Target_Image_Files and TargetKernelFile objects
- associated with Target target, then associate them with this target.
- Note that for Target_Image_Files, we only want files from the previous
- build whose suffix matches one of the suffixes defined in this
- target's build's IMAGE_FSTYPES configuration variable. This prevents the
- Target_Image_File object for an ext4 image being associated with a
- target for a project which didn't produce an ext4 image (for example).
- Also sets the license_manifest_path and package_manifest_path
- of this target to the same path as that of target being cloned from, as
- the manifests are also build artifacts but are treated differently.
- """
- image_fstypes = self.build.get_image_fstypes()
- # filter out any image files whose suffixes aren't in the
- # IMAGE_FSTYPES suffixes variable for this target's build
- image_files = [target_image_file \
- for target_image_file in target.target_image_file_set.all() \
- if target_image_file.suffix in image_fstypes]
- for image_file in image_files:
- image_file.pk = None
- image_file.target = self
- image_file.save()
- kernel_files = target.targetkernelfile_set.all()
- for kernel_file in kernel_files:
- kernel_file.pk = None
- kernel_file.target = self
- kernel_file.save()
- self.license_manifest_path = target.license_manifest_path
- self.package_manifest_path = target.package_manifest_path
- self.save()
- def clone_sdk_artifacts_from(self, target):
- """
- Clone TargetSDKFile objects from target and associate them with this
- target.
- """
- sdk_files = target.targetsdkfile_set.all()
- for sdk_file in sdk_files:
- sdk_file.pk = None
- sdk_file.target = self
- sdk_file.save()
- def has_images(self):
- """
- Returns True if this target has one or more image files attached to it.
- """
- return self.target_image_file_set.all().count() > 0
- # kernel artifacts for a target: bzImage and modules*
- class TargetKernelFile(models.Model):
- target = models.ForeignKey(Target)
- file_name = models.FilePathField()
- file_size = models.IntegerField()
- @property
- def basename(self):
- return os.path.basename(self.file_name)
- # SDK artifacts for a target: sh and manifest files
- class TargetSDKFile(models.Model):
- target = models.ForeignKey(Target)
- file_name = models.FilePathField()
- file_size = models.IntegerField()
- @property
- def basename(self):
- return os.path.basename(self.file_name)
- class Target_Image_File(models.Model):
- # valid suffixes for image files produced by a build
- SUFFIXES = {
- 'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma',
- 'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma',
- 'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2',
- 'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo',
- 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
- 'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
- }
- target = models.ForeignKey(Target)
- file_name = models.FilePathField(max_length=254)
- file_size = models.IntegerField()
- @property
- def suffix(self):
- """
- Suffix for image file, minus leading "."
- """
- for suffix in Target_Image_File.SUFFIXES:
- if self.file_name.endswith(suffix):
- return suffix
- filename, suffix = os.path.splitext(self.file_name)
- suffix = suffix.lstrip('.')
- return suffix
- class Target_File(models.Model):
- ITYPE_REGULAR = 1
- ITYPE_DIRECTORY = 2
- ITYPE_SYMLINK = 3
- ITYPE_SOCKET = 4
- ITYPE_FIFO = 5
- ITYPE_CHARACTER = 6
- ITYPE_BLOCK = 7
- ITYPES = ( (ITYPE_REGULAR ,'regular'),
- ( ITYPE_DIRECTORY ,'directory'),
- ( ITYPE_SYMLINK ,'symlink'),
- ( ITYPE_SOCKET ,'socket'),
- ( ITYPE_FIFO ,'fifo'),
- ( ITYPE_CHARACTER ,'character'),
- ( ITYPE_BLOCK ,'block'),
- )
- target = models.ForeignKey(Target)
- path = models.FilePathField()
- size = models.IntegerField()
- inodetype = models.IntegerField(choices = ITYPES)
- permission = models.CharField(max_length=16)
- owner = models.CharField(max_length=128)
- group = models.CharField(max_length=128)
- directory = models.ForeignKey('Target_File', related_name="directory_set", null=True)
- sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True)
- class Task(models.Model):
- SSTATE_NA = 0
- SSTATE_MISS = 1
- SSTATE_FAILED = 2
- SSTATE_RESTORED = 3
- SSTATE_RESULT = (
- (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
- (SSTATE_MISS, 'File not in cache'), # the sstate object was not found
- (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
- (SSTATE_RESTORED, 'Succeeded'), # successfully restored
- )
- CODING_NA = 0
- CODING_PYTHON = 2
- CODING_SHELL = 3
- TASK_CODING = (
- (CODING_NA, 'N/A'),
- (CODING_PYTHON, 'Python'),
- (CODING_SHELL, 'Shell'),
- )
- OUTCOME_NA = -1
- OUTCOME_SUCCESS = 0
- OUTCOME_COVERED = 1
- OUTCOME_CACHED = 2
- OUTCOME_PREBUILT = 3
- OUTCOME_FAILED = 4
- OUTCOME_EMPTY = 5
- TASK_OUTCOME = (
- (OUTCOME_NA, 'Not Available'),
- (OUTCOME_SUCCESS, 'Succeeded'),
- (OUTCOME_COVERED, 'Covered'),
- (OUTCOME_CACHED, 'Cached'),
- (OUTCOME_PREBUILT, 'Prebuilt'),
- (OUTCOME_FAILED, 'Failed'),
- (OUTCOME_EMPTY, 'Empty'),
- )
- TASK_OUTCOME_HELP = (
- (OUTCOME_SUCCESS, 'This task successfully completed'),
- (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'),
- (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'),
- (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'),
- (OUTCOME_FAILED, 'This task did not complete'),
- (OUTCOME_EMPTY, 'This task has no executable content'),
- (OUTCOME_NA, ''),
- )
- search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ]
- def __init__(self, *args, **kwargs):
- super(Task, self).__init__(*args, **kwargs)
- try:
- self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text
- except HelpText.DoesNotExist:
- self._helptext = None
- def get_related_setscene(self):
- return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene")
- def get_outcome_text(self):
- return Task.TASK_OUTCOME[int(self.outcome) + 1][1]
- def get_outcome_help(self):
- return Task.TASK_OUTCOME_HELP[int(self.outcome)][1]
- def get_sstate_text(self):
- if self.sstate_result==Task.SSTATE_NA:
- return ''
- else:
- return Task.SSTATE_RESULT[int(self.sstate_result)][1]
- def get_executed_display(self):
- if self.task_executed:
- return "Executed"
- return "Not Executed"
- def get_description(self):
- return self._helptext
- build = models.ForeignKey(Build, related_name='task_build')
- order = models.IntegerField(null=True)
- task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed
- outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
- sstate_checksum = models.CharField(max_length=100, blank=True)
- path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
- recipe = models.ForeignKey('Recipe', related_name='tasks')
- task_name = models.CharField(max_length=100)
- source_url = models.FilePathField(max_length=255, blank=True)
- work_directory = models.FilePathField(max_length=255, blank=True)
- script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
- line_number = models.IntegerField(default=0)
- # start/end times
- started = models.DateTimeField(null=True)
- ended = models.DateTimeField(null=True)
- # in seconds; this is stored to enable sorting
- elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
- # in bytes; note that disk_io is stored to enable sorting
- disk_io = models.IntegerField(null=True)
- disk_io_read = models.IntegerField(null=True)
- disk_io_write = models.IntegerField(null=True)
- # in seconds
- cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
- cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
- sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
- message = models.CharField(max_length=240)
- logfile = models.FilePathField(max_length=255, blank=True)
- outcome_text = property(get_outcome_text)
- sstate_text = property(get_sstate_text)
- def __unicode__(self):
- return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name)
- class Meta:
- ordering = ('order', 'recipe' ,)
- unique_together = ('build', 'recipe', 'task_name', )
- class Task_Dependency(models.Model):
- task = models.ForeignKey(Task, related_name='task_dependencies_task')
- depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends')
- class Package(models.Model):
- search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name']
- build = models.ForeignKey('Build', null=True)
- recipe = models.ForeignKey('Recipe', null=True)
- name = models.CharField(max_length=100)
- installed_name = models.CharField(max_length=100, default='')
- version = models.CharField(max_length=100, blank=True)
- revision = models.CharField(max_length=32, blank=True)
- summary = models.TextField(blank=True)
- description = models.TextField(blank=True)
- size = models.IntegerField(default=0)
- installed_size = models.IntegerField(default=0)
- section = models.CharField(max_length=80, blank=True)
- license = models.CharField(max_length=80, blank=True)
- @property
- def is_locale_package(self):
- """ Returns True if this package is identifiable as a locale package """
- if self.name.find('locale') != -1:
- return True
- return False
- @property
- def is_packagegroup(self):
- """ Returns True is this package is identifiable as a packagegroup """
- if self.name.find('packagegroup') != -1:
- return True
- return False
- class CustomImagePackage(Package):
- # CustomImageRecipe fields to track pacakges appended,
- # included and excluded from a CustomImageRecipe
- recipe_includes = models.ManyToManyField('CustomImageRecipe',
- related_name='includes_set')
- recipe_excludes = models.ManyToManyField('CustomImageRecipe',
- related_name='excludes_set')
- recipe_appends = models.ManyToManyField('CustomImageRecipe',
- related_name='appends_set')
- class Package_DependencyManager(models.Manager):
- use_for_related_fields = True
- TARGET_LATEST = "use-latest-target-for-target"
- def get_queryset(self):
- return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id'))
- def for_target_or_none(self, target):
- """ filter the dependencies to be displayed by the supplied target
- if no dependences are found for the target then try None as the target
- which will return the dependences calculated without the context of a
- target e.g. non image recipes.
- returns: { size, packages }
- """
- package_dependencies = self.all_depends().order_by('depends_on__name')
- if target is self.TARGET_LATEST:
- installed_deps =\
- package_dependencies.filter(~Q(target__target=None))
- else:
- installed_deps =\
- package_dependencies.filter(Q(target__target=target))
- packages_list = None
- total_size = 0
- # If we have installed depdencies for this package and target then use
- # these to display
- if installed_deps.count() > 0:
- packages_list = installed_deps
- total_size = installed_deps.aggregate(
- Sum('depends_on__size'))['depends_on__size__sum']
- else:
- new_list = []
- package_names = []
- # Find dependencies for the package that we know about even if
- # it's not installed on a target e.g. from a non-image recipe
- for p in package_dependencies.filter(Q(target=None)):
- if p.depends_on.name in package_names:
- continue
- else:
- package_names.append(p.depends_on.name)
- new_list.append(p.pk)
- # while we're here we may as well total up the size to
- # avoid iterating again
- total_size += p.depends_on.size
- # We want to return a queryset here for consistency so pick the
- # deps from the new_list
- packages_list = package_dependencies.filter(Q(pk__in=new_list))
- return {'packages': packages_list,
- 'size': total_size}
- def all_depends(self):
- """ Returns just the depends packages and not any other dep_type
- Note that this is for any target
- """
- return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) |
- Q(dep_type=Package_Dependency.TYPE_TRDEPENDS))
- class Package_Dependency(models.Model):
- TYPE_RDEPENDS = 0
- TYPE_TRDEPENDS = 1
- TYPE_RRECOMMENDS = 2
- TYPE_TRECOMMENDS = 3
- TYPE_RSUGGESTS = 4
- TYPE_RPROVIDES = 5
- TYPE_RREPLACES = 6
- TYPE_RCONFLICTS = 7
- ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
- DEPENDS_TYPE = (
- (TYPE_RDEPENDS, "depends"),
- (TYPE_TRDEPENDS, "depends"),
- (TYPE_TRECOMMENDS, "recommends"),
- (TYPE_RRECOMMENDS, "recommends"),
- (TYPE_RSUGGESTS, "suggests"),
- (TYPE_RPROVIDES, "provides"),
- (TYPE_RREPLACES, "replaces"),
- (TYPE_RCONFLICTS, "conflicts"),
- )
- """ Indexed by dep_type, in view order, key for short name and help
- description which when viewed will be printf'd with the
- package name.
- """
- DEPENDS_DICT = {
- TYPE_RDEPENDS : ("depends", "%s is required to run %s"),
- TYPE_TRDEPENDS : ("depends", "%s is required to run %s"),
- TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"),
- TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"),
- TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"),
- TYPE_RPROVIDES : ("provides", "%s is provided by %s"),
- TYPE_RREPLACES : ("replaces", "%s is replaced by %s"),
- TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"),
- }
- package = models.ForeignKey(Package, related_name='package_dependencies_source')
- depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency
- dep_type = models.IntegerField(choices=DEPENDS_TYPE)
- target = models.ForeignKey(Target, null=True)
- objects = Package_DependencyManager()
- class Target_Installed_Package(models.Model):
- target = models.ForeignKey(Target)
- package = models.ForeignKey(Package, related_name='buildtargetlist_package')
- class Package_File(models.Model):
- package = models.ForeignKey(Package, related_name='buildfilelist_package')
- path = models.FilePathField(max_length=255, blank=True)
- size = models.IntegerField()
- class Recipe(models.Model):
- search_allowed_fields = ['name', 'version', 'file_path', 'section',
- 'summary', 'description', 'license',
- 'layer_version__layer__name',
- 'layer_version__branch', 'layer_version__commit',
- 'layer_version__local_path',
- 'layer_version__layer_source']
- up_date = models.DateTimeField(null=True, default=None)
- name = models.CharField(max_length=100, blank=True)
- version = models.CharField(max_length=100, blank=True)
- layer_version = models.ForeignKey('Layer_Version',
- related_name='recipe_layer_version')
- summary = models.TextField(blank=True)
- description = models.TextField(blank=True)
- section = models.CharField(max_length=100, blank=True)
- license = models.CharField(max_length=200, blank=True)
- homepage = models.URLField(blank=True)
- bugtracker = models.URLField(blank=True)
- file_path = models.FilePathField(max_length=255)
- pathflags = models.CharField(max_length=200, blank=True)
- is_image = models.BooleanField(default=False)
- def __unicode__(self):
- return "Recipe " + self.name + ":" + self.version
- def get_vcs_recipe_file_link_url(self):
- return self.layer_version.get_vcs_file_link_url(self.file_path)
- def get_description_or_summary(self):
- if self.description:
- return self.description
- elif self.summary:
- return self.summary
- else:
- return ""
- class Meta:
- unique_together = (("layer_version", "file_path", "pathflags"), )
- class Recipe_DependencyManager(models.Manager):
- use_for_related_fields = True
- def get_queryset(self):
- return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id'))
- class Provides(models.Model):
- name = models.CharField(max_length=100)
- recipe = models.ForeignKey(Recipe)
- class Recipe_Dependency(models.Model):
- TYPE_DEPENDS = 0
- TYPE_RDEPENDS = 1
- DEPENDS_TYPE = (
- (TYPE_DEPENDS, "depends"),
- (TYPE_RDEPENDS, "rdepends"),
- )
- recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe')
- depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends')
- via = models.ForeignKey(Provides, null=True, default=None)
- dep_type = models.IntegerField(choices=DEPENDS_TYPE)
- objects = Recipe_DependencyManager()
- class Machine(models.Model):
- search_allowed_fields = ["name", "description", "layer_version__layer__name"]
- up_date = models.DateTimeField(null = True, default = None)
- layer_version = models.ForeignKey('Layer_Version')
- name = models.CharField(max_length=255)
- description = models.CharField(max_length=255)
- def get_vcs_machine_file_link_url(self):
- path = 'conf/machine/'+self.name+'.conf'
- return self.layer_version.get_vcs_file_link_url(path)
- def __unicode__(self):
- return "Machine " + self.name + "(" + self.description + ")"
- class BitbakeVersion(models.Model):
- name = models.CharField(max_length=32, unique = True)
- giturl = GitURLField()
- branch = models.CharField(max_length=32)
- dirpath = models.CharField(max_length=255)
- def __unicode__(self):
- return "%s (Branch: %s)" % (self.name, self.branch)
- class Release(models.Model):
- """ A release is a project template, used to pre-populate Project settings with a configuration set """
- name = models.CharField(max_length=32, unique = True)
- description = models.CharField(max_length=255)
- bitbake_version = models.ForeignKey(BitbakeVersion)
- branch_name = models.CharField(max_length=50, default = "")
- helptext = models.TextField(null=True)
- def __unicode__(self):
- return "%s (%s)" % (self.name, self.branch_name)
- def __str__(self):
- return self.name
- class ReleaseDefaultLayer(models.Model):
- release = models.ForeignKey(Release)
- layer_name = models.CharField(max_length=100, default="")
- class LayerSource(object):
- """ Where the layer metadata came from """
- TYPE_LOCAL = 0
- TYPE_LAYERINDEX = 1
- TYPE_IMPORTED = 2
- TYPE_BUILD = 3
- SOURCE_TYPE = (
- (TYPE_LOCAL, "local"),
- (TYPE_LAYERINDEX, "layerindex"),
- (TYPE_IMPORTED, "imported"),
- (TYPE_BUILD, "build"),
- )
- def types_dict():
- """ Turn the TYPES enums into a simple dictionary """
- dictionary = {}
- for key in LayerSource.__dict__:
- if "TYPE" in key:
- dictionary[key] = getattr(LayerSource, key)
- return dictionary
- class Layer(models.Model):
- up_date = models.DateTimeField(null=True, default=timezone.now)
- name = models.CharField(max_length=100)
- layer_index_url = models.URLField()
- vcs_url = GitURLField(default=None, null=True)
- local_source_dir = models.TextField(null=True, default=None)
- vcs_web_url = models.URLField(null=True, default=None)
- vcs_web_tree_base_url = models.URLField(null=True, default=None)
- vcs_web_file_base_url = models.URLField(null=True, default=None)
- summary = models.TextField(help_text='One-line description of the layer',
- null=True, default=None)
- description = models.TextField(null=True, default=None)
- def __unicode__(self):
- return "%s / %s " % (self.name, self.summary)
- class Layer_Version(models.Model):
- """
- A Layer_Version either belongs to a single project or no project
- """
- search_allowed_fields = ["layer__name", "layer__summary",
- "layer__description", "layer__vcs_url",
- "dirpath", "release__name", "commit", "branch"]
- build = models.ForeignKey(Build, related_name='layer_version_build',
- default=None, null=True)
- layer = models.ForeignKey(Layer, related_name='layer_version_layer')
- layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
- default=0)
- up_date = models.DateTimeField(null=True, default=timezone.now)
- # To which metadata release does this layer version belong to
- release = models.ForeignKey(Release, null=True, default=None)
- branch = models.CharField(max_length=80)
- commit = models.CharField(max_length=100)
- # If the layer is in a subdir
- dirpath = models.CharField(max_length=255, null=True, default=None)
- # if -1, this is a default layer
- priority = models.IntegerField(default=0)
- # where this layer exists on the filesystem
- local_path = models.FilePathField(max_length=1024, default="/")
- # Set if this layer is restricted to a particular project
- project = models.ForeignKey('Project', null=True, default=None)
- # code lifted, with adaptations, from the layerindex-web application
- # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
- def _handle_url_path(self, base_url, path):
- import re, posixpath
- if base_url:
- if self.dirpath:
- if path:
- extra_path = self.dirpath + '/' + path
- # Normalise out ../ in path for usage URL
- extra_path = posixpath.normpath(extra_path)
- # Minor workaround to handle case where subdirectory has been added between branches
- # (should probably support usage URL per branch to handle this... sigh...)
- if extra_path.startswith('../'):
- extra_path = extra_path[3:]
- else:
- extra_path = self.dirpath
- else:
- extra_path = path
- branchname = self.release.name
- url = base_url.replace('%branch%', branchname)
- # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
- if extra_path:
- extra_path = extra_path.replace('%', '%25')
- if '%path%' in base_url:
- if extra_path:
- url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url)
- else:
- url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url)
- return url.replace('%path%', extra_path)
- else:
- return url + extra_path
- return None
- def get_vcs_link_url(self):
- if self.layer.vcs_web_url is None:
- return None
- return self.layer.vcs_web_url
- def get_vcs_file_link_url(self, file_path=""):
- if self.layer.vcs_web_file_base_url is None:
- return None
- return self._handle_url_path(self.layer.vcs_web_file_base_url,
- file_path)
- def get_vcs_dirpath_link_url(self):
- if self.layer.vcs_web_tree_base_url is None:
- return None
- return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
- def get_vcs_reference(self):
- if self.commit is not None and len(self.commit) > 0:
- return self.commit
- if self.branch is not None and len(self.branch) > 0:
- return self.branch
- if self.release is not None:
- return self.release.name
- return 'N/A'
- def get_detailspage_url(self, project_id=None):
- """ returns the url to the layer details page uses own project
- field if project_id is not specified """
- if project_id is None:
- project_id = self.project.pk
- return reverse('layerdetails', args=(project_id, self.pk))
- def get_alldeps(self, project_id):
- """Get full list of unique layer dependencies."""
- def gen_layerdeps(lver, project, depth):
- if depth == 0:
- return
- for ldep in lver.dependencies.all():
- yield ldep.depends_on
- # get next level of deps recursively calling gen_layerdeps
- for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
- yield subdep
- project = Project.objects.get(pk=project_id)
- result = []
- projectlvers = [player.layercommit for player in
- project.projectlayer_set.all()]
- # protect against infinite layer dependency loops
- maxdepth = 20
- for dep in gen_layerdeps(self, project, maxdepth):
- # filter out duplicates and layers already belonging to the project
- if dep not in result + projectlvers:
- result.append(dep)
- return sorted(result, key=lambda x: x.layer.name)
- def __unicode__(self):
- return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
- def __str__(self):
- if self.release:
- release = self.release.name
- else:
- release = "No release set"
- return "%d %s (%s)" % (self.pk, self.layer.name, release)
- class LayerVersionDependency(models.Model):
- layer_version = models.ForeignKey(Layer_Version,
- related_name="dependencies")
- depends_on = models.ForeignKey(Layer_Version,
- related_name="dependees")
- class ProjectLayer(models.Model):
- project = models.ForeignKey(Project)
- layercommit = models.ForeignKey(Layer_Version, null=True)
- optional = models.BooleanField(default = True)
- def __unicode__(self):
- return "%s, %s" % (self.project.name, self.layercommit)
- class Meta:
- unique_together = (("project", "layercommit"),)
- class CustomImageRecipe(Recipe):
- # CustomImageRecipe's belong to layers called:
- LAYER_NAME = "toaster-custom-images"
- search_allowed_fields = ['name']
- base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe')
- project = models.ForeignKey(Project)
- last_updated = models.DateTimeField(null=True, default=None)
- def get_last_successful_built_target(self):
- """ Return the last successful built target object if one exists
- otherwise return None """
- return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
- Q(build__project=self.project) &
- Q(target=self.name)).last()
- def update_package_list(self):
- """ Update the package list from the last good build of this
- CustomImageRecipe
- """
- # Check if we're aldready up-to-date or not
- target = self.get_last_successful_built_target()
- if target is None:
- # So we've never actually built this Custom recipe but what about
- # the recipe it's based on?
- target = \
- Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
- Q(build__project=self.project) &
- Q(target=self.base_recipe.name)).last()
- if target is None:
- return
- if target.build.completed_on == self.last_updated:
- return
- self.includes_set.clear()
- excludes_list = self.excludes_set.values_list('name', flat=True)
- appends_list = self.appends_set.values_list('name', flat=True)
- built_packages_list = \
- target.target_installed_package_set.values_list('package__name',
- flat=True)
- for built_package in built_packages_list:
- # Is the built package in the custom packages list?
- if built_package in excludes_list:
- continue
- if built_package in appends_list:
- continue
- cust_img_p = \
- CustomImagePackage.objects.get(name=built_package)
- self.includes_set.add(cust_img_p)
- self.last_updated = target.build.completed_on
- self.save()
- def get_all_packages(self):
- """Get the included packages and any appended packages"""
- self.update_package_list()
- return CustomImagePackage.objects.filter((Q(recipe_appends=self) |
- Q(recipe_includes=self)) &
- ~Q(recipe_excludes=self))
- def get_base_recipe_file(self):
- """Get the base recipe file path if it exists on the file system"""
- path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
- self.base_recipe.file_path)
- path_schema_two = self.base_recipe.file_path
- path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
- self.base_recipe.file_path)
- if os.path.exists(path_schema_one):
- return path_schema_one
- # The path may now be the full path if the recipe has been built
- if os.path.exists(path_schema_two):
- return path_schema_two
- # Or a local path if all layers are local
- if os.path.exists(path_schema_three):
- return path_schema_three
- return None
- def generate_recipe_file_contents(self):
- """Generate the contents for the recipe file."""
- # If we have no excluded packages we only need to _append
- if self.excludes_set.count() == 0:
- packages_conf = "IMAGE_INSTALL_append = \" "
- for pkg in self.appends_set.all():
- packages_conf += pkg.name+' '
- else:
- packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \""
- # We add all the known packages to be built by this recipe apart
- # from locale packages which are are controlled with IMAGE_LINGUAS.
- for pkg in self.get_all_packages().exclude(
- name__icontains="locale"):
- packages_conf += pkg.name+' '
- packages_conf += "\""
- base_recipe_path = self.get_base_recipe_file()
- if base_recipe_path:
- base_recipe = open(base_recipe_path, 'r').read()
- else:
- # Pass back None to trigger error message to user
- return None
- # Add a special case for when the recipe we have based a custom image
- # recipe on requires another recipe.
- # For example:
- # "require core-image-minimal.bb" is changed to:
- # "require recipes-core/images/core-image-minimal.bb"
- req_search = re.search(r'(require\s+)(.+\.bb\s*$)',
- base_recipe,
- re.MULTILINE)
- if req_search:
- require_filename = req_search.group(2).strip()
- corrected_location = Recipe.objects.filter(
- Q(layer_version=self.base_recipe.layer_version) &
- Q(file_path__icontains=require_filename)).last().file_path
- new_require_line = "require %s" % corrected_location
- base_recipe = base_recipe.replace(req_search.group(0),
- new_require_line)
- info = {
- "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
- "base_recipe": base_recipe,
- "recipe_name": self.name,
- "base_recipe_name": self.base_recipe.name,
- "license": self.license,
- "summary": self.summary,
- "description": self.description,
- "packages_conf": packages_conf.strip()
- }
- recipe_contents = ("# Original recipe %(base_recipe_name)s \n"
- "%(base_recipe)s\n\n"
- "# Recipe %(recipe_name)s \n"
- "# Customisation Generated by Toaster on %(date)s\n"
- "SUMMARY = \"%(summary)s\"\n"
- "DESCRIPTION = \"%(description)s\"\n"
- "LICENSE = \"%(license)s\"\n"
- "%(packages_conf)s") % info
- return recipe_contents
- class ProjectVariable(models.Model):
- project = models.ForeignKey(Project)
- name = models.CharField(max_length=100)
- value = models.TextField(blank = True)
- class Variable(models.Model):
- search_allowed_fields = ['variable_name', 'variable_value',
- 'vhistory__file_name', "description"]
- build = models.ForeignKey(Build, related_name='variable_build')
- variable_name = models.CharField(max_length=100)
- variable_value = models.TextField(blank=True)
- changed = models.BooleanField(default=False)
- human_readable_name = models.CharField(max_length=200)
- description = models.TextField(blank=True)
- class VariableHistory(models.Model):
- variable = models.ForeignKey(Variable, related_name='vhistory')
- value = models.TextField(blank=True)
- file_name = models.FilePathField(max_length=255)
- line_number = models.IntegerField(null=True)
- operation = models.CharField(max_length=64)
- class HelpText(models.Model):
- VARIABLE = 0
- HELPTEXT_AREA = ((VARIABLE, 'variable'), )
- build = models.ForeignKey(Build, related_name='helptext_build')
- area = models.IntegerField(choices=HELPTEXT_AREA)
- key = models.CharField(max_length=100)
- text = models.TextField()
- class LogMessage(models.Model):
- EXCEPTION = -1 # used to signal self-toaster-exceptions
- INFO = 0
- WARNING = 1
- ERROR = 2
- CRITICAL = 3
- LOG_LEVEL = (
- (INFO, "info"),
- (WARNING, "warn"),
- (ERROR, "error"),
- (CRITICAL, "critical"),
- (EXCEPTION, "toaster exception")
- )
- build = models.ForeignKey(Build)
- task = models.ForeignKey(Task, blank = True, null=True)
- level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
- message = models.TextField(blank=True, null=True)
- pathname = models.FilePathField(max_length=255, blank=True)
- lineno = models.IntegerField(null=True)
- def __str__(self):
- return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build))
- def invalidate_cache(**kwargs):
- from django.core.cache import cache
- try:
- cache.clear()
- except Exception as e:
- logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
- def signal_runbuilds():
- """Send SIGUSR1 to runbuilds process"""
- try:
- with open(os.path.join(os.getenv('BUILDDIR', '.'),
- '.runbuilds.pid')) as pidf:
- os.kill(int(pidf.read()), SIGUSR1)
- except FileNotFoundError:
- logger.info("Stopping existing runbuilds: no current process found")
- class Distro(models.Model):
- search_allowed_fields = ["name", "description", "layer_version__layer__name"]
- up_date = models.DateTimeField(null = True, default = None)
- layer_version = models.ForeignKey('Layer_Version')
- name = models.CharField(max_length=255)
- description = models.CharField(max_length=255)
- def get_vcs_distro_file_link_url(self):
- path = 'conf/distro/%s.conf' % self.name
- return self.layer_version.get_vcs_file_link_url(path)
- def __unicode__(self):
- return "Distro " + self.name + "(" + self.description + ")"
- django.db.models.signals.post_save.connect(invalidate_cache)
- django.db.models.signals.post_delete.connect(invalidate_cache)
- django.db.models.signals.m2m_changed.connect(invalidate_cache)
|