123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569 |
- #
- # BitBake Toaster Implementation
- #
- # Copyright (C) 2018 Wind River Systems
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- # buildimport: import a project for project specific configuration
- #
- # Usage:
- # (a) Set up Toaster environent
- #
- # (b) Call buildimport
- # $ /path/to/bitbake/lib/toaster/manage.py buildimport \
- # --name=$PROJECTNAME \
- # --path=$BUILD_DIRECTORY \
- # --callback="$CALLBACK_SCRIPT" \
- # --command="configure|reconfigure|import"
- #
- # (c) Return is "|Default_image=%s|Project_id=%d"
- #
- # (d) Open Toaster to this project using for example:
- # $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
- #
- # (e) To delete a project:
- # $ /path/to/bitbake/lib/toaster/manage.py buildimport \
- # --name=$PROJECTNAME --delete-project
- #
- # ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
- from django.core.management.base import BaseCommand
- from orm.models import Project, Release, ProjectVariable
- from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
- from toastergui.api import scan_layer_content
- import os
- import re
- import os.path
- import subprocess
- import shutil
- # Toaster variable section delimiters
- TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
- TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
- # quick development/debugging support
- verbose = 2
- def _log(msg):
- if 1 == verbose:
- print(msg)
- elif 2 == verbose:
- f1=open('/tmp/toaster.log', 'a')
- f1.write("|" + msg + "|\n" )
- f1.close()
- __config_regexp__ = re.compile( r"""
- ^
- (?P<exp>export\s+)?
- (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
- (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
- \s* (
- (?P<colon>:=) |
- (?P<lazyques>\?\?=) |
- (?P<ques>\?=) |
- (?P<append>\+=) |
- (?P<prepend>=\+) |
- (?P<predot>=\.) |
- (?P<postdot>\.=) |
- =
- ) \s*
- (?!'[^']*'[^']*'$)
- (?!\"[^\"]*\"[^\"]*\"$)
- (?P<apo>['\"])
- (?P<value>.*)
- (?P=apo)
- $
- """, re.X)
- class Command(BaseCommand):
- args = "<name> <path> <release>"
- help = "Import a command line build directory"
- vars = {}
- toaster_vars = {}
- def add_arguments(self, parser):
- parser.add_argument(
- '--name', dest='name', required=True,
- help='name of the project',
- )
- parser.add_argument(
- '--path', dest='path', required=True,
- help='path to the project',
- )
- parser.add_argument(
- '--release', dest='release', required=False,
- help='release for the project',
- )
- parser.add_argument(
- '--callback', dest='callback', required=False,
- help='callback for project config update',
- )
- parser.add_argument(
- '--delete-project', dest='delete_project', required=False,
- help='delete this project from the database',
- )
- parser.add_argument(
- '--command', dest='command', required=False,
- help='command (configure,reconfigure,import)',
- )
- # Extract the bb variables from a conf file
- def scan_conf(self,fn):
- vars = self.vars
- toaster_vars = self.toaster_vars
- #_log("scan_conf:%s" % fn)
- if not os.path.isfile(fn):
- return
- f = open(fn, 'r')
- #statements = ast.StatementGroup()
- lineno = 0
- is_toaster_section = False
- while True:
- lineno = lineno + 1
- s = f.readline()
- if not s:
- break
- w = s.strip()
- # skip empty lines
- if not w:
- continue
- # evaluate Toaster sections
- if w.startswith(TOASTER_PROLOG):
- is_toaster_section = True
- continue
- if w.startswith(TOASTER_EPILOG):
- is_toaster_section = False
- continue
- s = s.rstrip()
- while s[-1] == '\\':
- s2 = f.readline().strip()
- lineno = lineno + 1
- if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
- echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
- s = s[:-1] + s2
- # skip comments
- if s[0] == '#':
- continue
- # process the line for just assignments
- m = __config_regexp__.match(s)
- if m:
- groupd = m.groupdict()
- var = groupd['var']
- value = groupd['value']
- if groupd['lazyques']:
- if not var in vars:
- vars[var] = value
- continue
- if groupd['ques']:
- if not var in vars:
- vars[var] = value
- continue
- # preset empty blank for remaining operators
- if not var in vars:
- vars[var] = ''
- if groupd['append']:
- vars[var] += value
- elif groupd['prepend']:
- vars[var] = "%s%s" % (value,vars[var])
- elif groupd['predot']:
- vars[var] = "%s %s" % (value,vars[var])
- elif groupd['postdot']:
- vars[var] = "%s %s" % (vars[var],value)
- else:
- vars[var] = "%s" % (value)
- # capture vars in a Toaster section
- if is_toaster_section:
- toaster_vars[var] = vars[var]
- # DONE WITH PARSING
- f.close()
- self.vars = vars
- self.toaster_vars = toaster_vars
- # Update the scanned project variables
- def update_project_vars(self,project,name):
- pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
- if (not name in self.vars.keys()) or (not self.vars[name]):
- self.vars[name] = pv.value
- else:
- if pv.value != self.vars[name]:
- pv.value = self.vars[name]
- pv.save()
- # Find the git version of the installation
- def find_layer_dir_version(self,path):
- # * rocko ...
- install_version = ''
- cwd = os.getcwd()
- os.chdir(path)
- p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = p.communicate()
- out = out.decode("utf-8")
- for branch in out.split('\n'):
- if ('*' == branch[0:1]) and ('no branch' not in branch):
- install_version = re.sub(' .*','',branch[2:])
- break
- if 'remotes/m/master' in branch:
- install_version = re.sub('.*base/','',branch)
- break
- os.chdir(cwd)
- return install_version
- # Compute table of the installation's registered layer versions (branch or commit)
- def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
- lv_dict = {}
- layer_versions = Layer_Version.objects.all()
- for lv in layer_versions:
- layer = Layer.objects.filter(pk=lv.layer.pk)[0]
- if layer.vcs_url:
- url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
- else:
- url_short = ''
- # register the core, branch, and the version variations
- lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
- lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
- lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
- #_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
- return lv_dict
- # Apply table of all layer versions
- def extract_bblayers(self):
- # set up the constants
- bblayer_str = self.vars['BBLAYERS']
- TOASTER_DIR = os.environ.get('TOASTER_DIR')
- INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
- TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
- INSTALL_URL_PREFIX = ''
- layers = Layer.objects.filter(name='openembedded-core')
- for layer in layers:
- if layer.vcs_url:
- INSTALL_URL_PREFIX = layer.vcs_url
- break
- INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
- INSTALL_VERSION_DIR = TOASTER_DIR
- INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
- INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
- INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
- # get the set of available layer:layer_versions
- lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
- # compute the layer matches
- layers_list = []
- for line in bblayer_str.split(' '):
- if not line:
- continue
- if line.endswith('/local'):
- continue
- # isolate the repo
- layer_path = line
- line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
- # isolate the sub-path
- path_index = line.rfind('/')
- if path_index > 0:
- sub_path = line[path_index+1:]
- line = line[0:path_index]
- else:
- sub_path = ''
- # isolate the version
- if TOASTER_CLONE_PREFIX in layer_path:
- is_toaster_clone = True
- # extract version from name syntax
- version_index = line.find('_')
- if version_index > 0:
- version = line[version_index+1:]
- line = line[0:version_index]
- else:
- version = ''
- _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
- else:
- is_toaster_clone = False
- # version is from the installation
- version = self.find_layer_dir_version(layer_path)
- _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
- # capture the layer information into layers_list
- layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
- return layers_list,lv_dict
- #
- def find_import_release(self,layers_list,lv_dict,default_release):
- # poky,meta,rocko => 4;openembedded-core
- release = default_release
- for line,path,version,layer_path,is_toaster_clone in layers_list:
- key = "%s,%s,%s" % (line,path,version)
- if key in lv_dict:
- lv_id = lv_dict[key]
- if 'openembedded-core' == lv_id[1]:
- _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
- # only versions in Toaster managed layers are accepted
- if not is_toaster_clone:
- break
- try:
- release = Release.objects.get(name=version)
- except:
- pass
- break
- _log("Find_import_release:RELEASE=%s" % release.name)
- return release
- # Apply the found conf layers
- def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
- for line,path,version,layer_path,is_toaster_clone in layers_list:
- # Assert release promote if present
- if release:
- version = release
- # try to match the key to a layer_version
- key = "%s,%s,%s" % (line,path,version)
- key_short = "%s,%s,%s" % (line,path,'')
- lv_id = ''
- if key in lv_dict:
- lv_id = lv_dict[key]
- lv = Layer_Version.objects.get(pk=int(lv_id[0]))
- pl,created = ProjectLayer.objects.get_or_create(project=project,
- layercommit=lv)
- pl.optional=False
- pl.save()
- _log(" %s => %s;%s" % (key,lv_id[0],lv_id[1]))
- elif key_short in lv_dict:
- lv_id = lv_dict[key_short]
- lv = Layer_Version.objects.get(pk=int(lv_id[0]))
- pl,created = ProjectLayer.objects.get_or_create(project=project,
- layercommit=lv)
- pl.optional=False
- pl.save()
- _log(" %s ?> %s" % (key,lv_dict[key_short]))
- else:
- _log("%s <= %s" % (key,layer_path))
- found = False
- # does local layer already exist in this project?
- try:
- for pl in ProjectLayer.objects.filter(project=project):
- if pl.layercommit.layer.local_source_dir == layer_path:
- found = True
- _log(" Project Local Layer found!")
- except Exception as e:
- _log("ERROR: Local Layer '%s'" % e)
- pass
- if not found:
- # Does Layer name+path already exist?
- try:
- layer_name_base = os.path.basename(layer_path)
- _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
- layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
- # Found! Attach layer_version and ProjectLayer
- layer_version = Layer_Version.objects.create(
- layer=layer,
- project=project,
- layer_source=LayerSource.TYPE_IMPORTED)
- layer_version.save()
- pl,created = ProjectLayer.objects.get_or_create(project=project,
- layercommit=layer_version)
- pl.optional=False
- pl.save()
- found = True
- # add layer contents to this layer version
- scan_layer_content(layer,layer_version)
- _log(" Parent Local Layer found in db!")
- except Exception as e:
- _log("Layer_exists_test_failed: Local Layer '%s'" % e)
- pass
- if not found:
- # Insure that layer path exists, in case of user typo
- if not os.path.isdir(layer_path):
- _log("ERROR:Layer path '%s' not found" % layer_path)
- continue
- # Add layer to db and attach project to it
- layer_name_base = os.path.basename(layer_path)
- # generate a unique layer name
- layer_name_matches = {}
- for layer in Layer.objects.filter(name__contains=layer_name_base):
- layer_name_matches[layer.name] = '1'
- layer_name_idx = 0
- layer_name_test = layer_name_base
- while layer_name_test in layer_name_matches.keys():
- layer_name_idx += 1
- layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
- # create the layer and layer_verion objects
- layer = Layer.objects.create(name=layer_name_test)
- layer.local_source_dir = layer_path
- layer_version = Layer_Version.objects.create(
- layer=layer,
- project=project,
- layer_source=LayerSource.TYPE_IMPORTED)
- layer.save()
- layer_version.save()
- pl,created = ProjectLayer.objects.get_or_create(project=project,
- layercommit=layer_version)
- pl.optional=False
- pl.save()
- # register the layer's content
- _log(" Local Layer Add content")
- scan_layer_content(layer,layer_version)
- _log(" Local Layer Added '%s'!" % layer_name_test)
- # Scan the project's conf files (if any)
- def scan_conf_variables(self,project_path):
- # scan the project's settings, add any new layers or variables
- if os.path.isfile("%s/conf/local.conf" % project_path):
- self.scan_conf("%s/conf/local.conf" % project_path)
- self.scan_conf("%s/conf/bblayers.conf" % project_path)
- # Import then disable old style Toaster conf files (before 'merged_attr')
- old_toaster_local = "%s/conf/toaster.conf" % project_path
- if os.path.isfile(old_toaster_local):
- self.scan_conf(old_toaster_local)
- shutil.move(old_toaster_local, old_toaster_local+"_old")
- old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
- if os.path.isfile(old_toaster_layer):
- self.scan_conf(old_toaster_layer)
- shutil.move(old_toaster_layer, old_toaster_layer+"_old")
- # Scan the found conf variables (if any)
- def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
- if self.vars:
- # Catch vars relevant to Toaster (in case no Toaster section)
- self.update_project_vars(project,'DISTRO')
- self.update_project_vars(project,'MACHINE')
- self.update_project_vars(project,'IMAGE_INSTALL_append')
- self.update_project_vars(project,'IMAGE_FSTYPES')
- self.update_project_vars(project,'PACKAGE_CLASSES')
- # These vars are typically only assigned by Toaster
- #self.update_project_vars(project,'DL_DIR')
- #self.update_project_vars(project,'SSTATE_DIR')
- # Assert found Toaster vars
- for var in self.toaster_vars.keys():
- pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
- pv.value = self.toaster_vars[var]
- _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
- pv.save()
- # Assert found BBLAYERS
- if 0 < verbose:
- for pl in ProjectLayer.objects.filter(project=project):
- release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
- print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
- self.apply_conf_bblayers(layers_list,lv_dict,project,release)
- if 0 < verbose:
- for pl in ProjectLayer.objects.filter(project=project):
- release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
- print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
- def handle(self, *args, **options):
- project_name = options['name']
- project_path = options['path']
- project_callback = options['callback'] if options['callback'] else ''
- release_name = options['release'] if options['release'] else ''
- #
- # Delete project
- #
- if options['delete_project']:
- try:
- print("Project '%s' delete from Toaster database" % (project_name))
- project = Project.objects.get(name=project_name)
- # TODO: deep project delete
- project.delete()
- print("Project '%s' Deleted" % (project_name))
- return
- except Exception as e:
- print("Project '%s' not found, not deleted (%s)" % (project_name,e))
- return
- #
- # Create/Update/Import project
- #
- # See if project (by name) exists
- project = None
- try:
- # Project already exists
- project = Project.objects.get(name=project_name)
- except Exception as e:
- pass
- # Find the installation's default release
- default_release = Release.objects.get(id=1)
- # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
- if ("reconfigure" == options['command']) and project is None:
- options['command'] = 'import'
- # 'Configure':
- if "configure" == options['command']:
- # Note: ignore any existing conf files
- # create project, SANITY: reuse any project of same name
- project = Project.objects.create_project(project_name,default_release,project)
- # 'Re-configure':
- if "reconfigure" == options['command']:
- # Scan the directory's conf files
- self.scan_conf_variables(project_path)
- # Scan the layer list
- layers_list,lv_dict = self.extract_bblayers()
- # Apply any new layers or variables
- self.apply_conf_variables(project,layers_list,lv_dict)
- # 'Import':
- if "import" == options['command']:
- # Scan the directory's conf files
- self.scan_conf_variables(project_path)
- # Remove these Toaster controlled variables
- for var in ('DL_DIR','SSTATE_DIR'):
- self.vars.pop(var, None)
- self.toaster_vars.pop(var, None)
- # Scan the layer list
- layers_list,lv_dict = self.extract_bblayers()
- # Find the directory's release, and promote to default_release if local paths
- release = self.find_import_release(layers_list,lv_dict,default_release)
- # create project, SANITY: reuse any project of same name
- project = Project.objects.create_project(project_name,release,project)
- # Apply any new layers or variables
- self.apply_conf_variables(project,layers_list,lv_dict,release)
- # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
- project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
- # Set up the project's meta data
- project.builddir = project_path
- project.merged_attr = True
- project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
- project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
- if ("configure" == options['command']) or ("import" == options['command']):
- # preset the mode and default image recipe
- project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
- project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
- # Assert any extended/custom actions or variables for new non-Toaster projects
- if not len(self.toaster_vars):
- pass
- else:
- project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
- # Save the updated Project
- project.save()
- _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
- if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
- print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
- else:
- print("|Project_id=%d" % (project.id))
|