buildimport.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569
  1. #
  2. # BitBake Toaster Implementation
  3. #
  4. # Copyright (C) 2018 Wind River Systems
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # buildimport: import a project for project specific configuration
  9. #
  10. # Usage:
  11. # (a) Set up Toaster environent
  12. #
  13. # (b) Call buildimport
  14. # $ /path/to/bitbake/lib/toaster/manage.py buildimport \
  15. # --name=$PROJECTNAME \
  16. # --path=$BUILD_DIRECTORY \
  17. # --callback="$CALLBACK_SCRIPT" \
  18. # --command="configure|reconfigure|import"
  19. #
  20. # (c) Return is "|Default_image=%s|Project_id=%d"
  21. #
  22. # (d) Open Toaster to this project using for example:
  23. # $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
  24. #
  25. # (e) To delete a project:
  26. # $ /path/to/bitbake/lib/toaster/manage.py buildimport \
  27. # --name=$PROJECTNAME --delete-project
  28. #
  29. # ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
  30. from django.core.management.base import BaseCommand
  31. from orm.models import Project, Release, ProjectVariable
  32. from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
  33. from toastergui.api import scan_layer_content
  34. import os
  35. import re
  36. import os.path
  37. import subprocess
  38. import shutil
  39. # Toaster variable section delimiters
  40. TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
  41. TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
  42. # quick development/debugging support
  43. verbose = 2
  44. def _log(msg):
  45. if 1 == verbose:
  46. print(msg)
  47. elif 2 == verbose:
  48. f1=open('/tmp/toaster.log', 'a')
  49. f1.write("|" + msg + "|\n" )
  50. f1.close()
  51. __config_regexp__ = re.compile( r"""
  52. ^
  53. (?P<exp>export\s+)?
  54. (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
  55. (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
  56. \s* (
  57. (?P<colon>:=) |
  58. (?P<lazyques>\?\?=) |
  59. (?P<ques>\?=) |
  60. (?P<append>\+=) |
  61. (?P<prepend>=\+) |
  62. (?P<predot>=\.) |
  63. (?P<postdot>\.=) |
  64. =
  65. ) \s*
  66. (?!'[^']*'[^']*'$)
  67. (?!\"[^\"]*\"[^\"]*\"$)
  68. (?P<apo>['\"])
  69. (?P<value>.*)
  70. (?P=apo)
  71. $
  72. """, re.X)
  73. class Command(BaseCommand):
  74. args = "<name> <path> <release>"
  75. help = "Import a command line build directory"
  76. vars = {}
  77. toaster_vars = {}
  78. def add_arguments(self, parser):
  79. parser.add_argument(
  80. '--name', dest='name', required=True,
  81. help='name of the project',
  82. )
  83. parser.add_argument(
  84. '--path', dest='path', required=True,
  85. help='path to the project',
  86. )
  87. parser.add_argument(
  88. '--release', dest='release', required=False,
  89. help='release for the project',
  90. )
  91. parser.add_argument(
  92. '--callback', dest='callback', required=False,
  93. help='callback for project config update',
  94. )
  95. parser.add_argument(
  96. '--delete-project', dest='delete_project', required=False,
  97. help='delete this project from the database',
  98. )
  99. parser.add_argument(
  100. '--command', dest='command', required=False,
  101. help='command (configure,reconfigure,import)',
  102. )
  103. # Extract the bb variables from a conf file
  104. def scan_conf(self,fn):
  105. vars = self.vars
  106. toaster_vars = self.toaster_vars
  107. #_log("scan_conf:%s" % fn)
  108. if not os.path.isfile(fn):
  109. return
  110. f = open(fn, 'r')
  111. #statements = ast.StatementGroup()
  112. lineno = 0
  113. is_toaster_section = False
  114. while True:
  115. lineno = lineno + 1
  116. s = f.readline()
  117. if not s:
  118. break
  119. w = s.strip()
  120. # skip empty lines
  121. if not w:
  122. continue
  123. # evaluate Toaster sections
  124. if w.startswith(TOASTER_PROLOG):
  125. is_toaster_section = True
  126. continue
  127. if w.startswith(TOASTER_EPILOG):
  128. is_toaster_section = False
  129. continue
  130. s = s.rstrip()
  131. while s[-1] == '\\':
  132. s2 = f.readline().strip()
  133. lineno = lineno + 1
  134. if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
  135. echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
  136. s = s[:-1] + s2
  137. # skip comments
  138. if s[0] == '#':
  139. continue
  140. # process the line for just assignments
  141. m = __config_regexp__.match(s)
  142. if m:
  143. groupd = m.groupdict()
  144. var = groupd['var']
  145. value = groupd['value']
  146. if groupd['lazyques']:
  147. if not var in vars:
  148. vars[var] = value
  149. continue
  150. if groupd['ques']:
  151. if not var in vars:
  152. vars[var] = value
  153. continue
  154. # preset empty blank for remaining operators
  155. if not var in vars:
  156. vars[var] = ''
  157. if groupd['append']:
  158. vars[var] += value
  159. elif groupd['prepend']:
  160. vars[var] = "%s%s" % (value,vars[var])
  161. elif groupd['predot']:
  162. vars[var] = "%s %s" % (value,vars[var])
  163. elif groupd['postdot']:
  164. vars[var] = "%s %s" % (vars[var],value)
  165. else:
  166. vars[var] = "%s" % (value)
  167. # capture vars in a Toaster section
  168. if is_toaster_section:
  169. toaster_vars[var] = vars[var]
  170. # DONE WITH PARSING
  171. f.close()
  172. self.vars = vars
  173. self.toaster_vars = toaster_vars
  174. # Update the scanned project variables
  175. def update_project_vars(self,project,name):
  176. pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
  177. if (not name in self.vars.keys()) or (not self.vars[name]):
  178. self.vars[name] = pv.value
  179. else:
  180. if pv.value != self.vars[name]:
  181. pv.value = self.vars[name]
  182. pv.save()
  183. # Find the git version of the installation
  184. def find_layer_dir_version(self,path):
  185. # * rocko ...
  186. install_version = ''
  187. cwd = os.getcwd()
  188. os.chdir(path)
  189. p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
  190. stderr=subprocess.PIPE)
  191. out, err = p.communicate()
  192. out = out.decode("utf-8")
  193. for branch in out.split('\n'):
  194. if ('*' == branch[0:1]) and ('no branch' not in branch):
  195. install_version = re.sub(' .*','',branch[2:])
  196. break
  197. if 'remotes/m/master' in branch:
  198. install_version = re.sub('.*base/','',branch)
  199. break
  200. os.chdir(cwd)
  201. return install_version
  202. # Compute table of the installation's registered layer versions (branch or commit)
  203. def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
  204. lv_dict = {}
  205. layer_versions = Layer_Version.objects.all()
  206. for lv in layer_versions:
  207. layer = Layer.objects.filter(pk=lv.layer.pk)[0]
  208. if layer.vcs_url:
  209. url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
  210. else:
  211. url_short = ''
  212. # register the core, branch, and the version variations
  213. lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
  214. lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
  215. lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
  216. #_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
  217. return lv_dict
  218. # Apply table of all layer versions
  219. def extract_bblayers(self):
  220. # set up the constants
  221. bblayer_str = self.vars['BBLAYERS']
  222. TOASTER_DIR = os.environ.get('TOASTER_DIR')
  223. INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
  224. TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
  225. INSTALL_URL_PREFIX = ''
  226. layers = Layer.objects.filter(name='openembedded-core')
  227. for layer in layers:
  228. if layer.vcs_url:
  229. INSTALL_URL_PREFIX = layer.vcs_url
  230. break
  231. INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
  232. INSTALL_VERSION_DIR = TOASTER_DIR
  233. INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
  234. INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
  235. INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
  236. # get the set of available layer:layer_versions
  237. lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
  238. # compute the layer matches
  239. layers_list = []
  240. for line in bblayer_str.split(' '):
  241. if not line:
  242. continue
  243. if line.endswith('/local'):
  244. continue
  245. # isolate the repo
  246. layer_path = line
  247. line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
  248. # isolate the sub-path
  249. path_index = line.rfind('/')
  250. if path_index > 0:
  251. sub_path = line[path_index+1:]
  252. line = line[0:path_index]
  253. else:
  254. sub_path = ''
  255. # isolate the version
  256. if TOASTER_CLONE_PREFIX in layer_path:
  257. is_toaster_clone = True
  258. # extract version from name syntax
  259. version_index = line.find('_')
  260. if version_index > 0:
  261. version = line[version_index+1:]
  262. line = line[0:version_index]
  263. else:
  264. version = ''
  265. _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
  266. else:
  267. is_toaster_clone = False
  268. # version is from the installation
  269. version = self.find_layer_dir_version(layer_path)
  270. _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
  271. # capture the layer information into layers_list
  272. layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
  273. return layers_list,lv_dict
  274. #
  275. def find_import_release(self,layers_list,lv_dict,default_release):
  276. # poky,meta,rocko => 4;openembedded-core
  277. release = default_release
  278. for line,path,version,layer_path,is_toaster_clone in layers_list:
  279. key = "%s,%s,%s" % (line,path,version)
  280. if key in lv_dict:
  281. lv_id = lv_dict[key]
  282. if 'openembedded-core' == lv_id[1]:
  283. _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
  284. # only versions in Toaster managed layers are accepted
  285. if not is_toaster_clone:
  286. break
  287. try:
  288. release = Release.objects.get(name=version)
  289. except:
  290. pass
  291. break
  292. _log("Find_import_release:RELEASE=%s" % release.name)
  293. return release
  294. # Apply the found conf layers
  295. def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
  296. for line,path,version,layer_path,is_toaster_clone in layers_list:
  297. # Assert release promote if present
  298. if release:
  299. version = release
  300. # try to match the key to a layer_version
  301. key = "%s,%s,%s" % (line,path,version)
  302. key_short = "%s,%s,%s" % (line,path,'')
  303. lv_id = ''
  304. if key in lv_dict:
  305. lv_id = lv_dict[key]
  306. lv = Layer_Version.objects.get(pk=int(lv_id[0]))
  307. pl,created = ProjectLayer.objects.get_or_create(project=project,
  308. layercommit=lv)
  309. pl.optional=False
  310. pl.save()
  311. _log(" %s => %s;%s" % (key,lv_id[0],lv_id[1]))
  312. elif key_short in lv_dict:
  313. lv_id = lv_dict[key_short]
  314. lv = Layer_Version.objects.get(pk=int(lv_id[0]))
  315. pl,created = ProjectLayer.objects.get_or_create(project=project,
  316. layercommit=lv)
  317. pl.optional=False
  318. pl.save()
  319. _log(" %s ?> %s" % (key,lv_dict[key_short]))
  320. else:
  321. _log("%s <= %s" % (key,layer_path))
  322. found = False
  323. # does local layer already exist in this project?
  324. try:
  325. for pl in ProjectLayer.objects.filter(project=project):
  326. if pl.layercommit.layer.local_source_dir == layer_path:
  327. found = True
  328. _log(" Project Local Layer found!")
  329. except Exception as e:
  330. _log("ERROR: Local Layer '%s'" % e)
  331. pass
  332. if not found:
  333. # Does Layer name+path already exist?
  334. try:
  335. layer_name_base = os.path.basename(layer_path)
  336. _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
  337. layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
  338. # Found! Attach layer_version and ProjectLayer
  339. layer_version = Layer_Version.objects.create(
  340. layer=layer,
  341. project=project,
  342. layer_source=LayerSource.TYPE_IMPORTED)
  343. layer_version.save()
  344. pl,created = ProjectLayer.objects.get_or_create(project=project,
  345. layercommit=layer_version)
  346. pl.optional=False
  347. pl.save()
  348. found = True
  349. # add layer contents to this layer version
  350. scan_layer_content(layer,layer_version)
  351. _log(" Parent Local Layer found in db!")
  352. except Exception as e:
  353. _log("Layer_exists_test_failed: Local Layer '%s'" % e)
  354. pass
  355. if not found:
  356. # Insure that layer path exists, in case of user typo
  357. if not os.path.isdir(layer_path):
  358. _log("ERROR:Layer path '%s' not found" % layer_path)
  359. continue
  360. # Add layer to db and attach project to it
  361. layer_name_base = os.path.basename(layer_path)
  362. # generate a unique layer name
  363. layer_name_matches = {}
  364. for layer in Layer.objects.filter(name__contains=layer_name_base):
  365. layer_name_matches[layer.name] = '1'
  366. layer_name_idx = 0
  367. layer_name_test = layer_name_base
  368. while layer_name_test in layer_name_matches.keys():
  369. layer_name_idx += 1
  370. layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
  371. # create the layer and layer_verion objects
  372. layer = Layer.objects.create(name=layer_name_test)
  373. layer.local_source_dir = layer_path
  374. layer_version = Layer_Version.objects.create(
  375. layer=layer,
  376. project=project,
  377. layer_source=LayerSource.TYPE_IMPORTED)
  378. layer.save()
  379. layer_version.save()
  380. pl,created = ProjectLayer.objects.get_or_create(project=project,
  381. layercommit=layer_version)
  382. pl.optional=False
  383. pl.save()
  384. # register the layer's content
  385. _log(" Local Layer Add content")
  386. scan_layer_content(layer,layer_version)
  387. _log(" Local Layer Added '%s'!" % layer_name_test)
  388. # Scan the project's conf files (if any)
  389. def scan_conf_variables(self,project_path):
  390. # scan the project's settings, add any new layers or variables
  391. if os.path.isfile("%s/conf/local.conf" % project_path):
  392. self.scan_conf("%s/conf/local.conf" % project_path)
  393. self.scan_conf("%s/conf/bblayers.conf" % project_path)
  394. # Import then disable old style Toaster conf files (before 'merged_attr')
  395. old_toaster_local = "%s/conf/toaster.conf" % project_path
  396. if os.path.isfile(old_toaster_local):
  397. self.scan_conf(old_toaster_local)
  398. shutil.move(old_toaster_local, old_toaster_local+"_old")
  399. old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
  400. if os.path.isfile(old_toaster_layer):
  401. self.scan_conf(old_toaster_layer)
  402. shutil.move(old_toaster_layer, old_toaster_layer+"_old")
  403. # Scan the found conf variables (if any)
  404. def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
  405. if self.vars:
  406. # Catch vars relevant to Toaster (in case no Toaster section)
  407. self.update_project_vars(project,'DISTRO')
  408. self.update_project_vars(project,'MACHINE')
  409. self.update_project_vars(project,'IMAGE_INSTALL_append')
  410. self.update_project_vars(project,'IMAGE_FSTYPES')
  411. self.update_project_vars(project,'PACKAGE_CLASSES')
  412. # These vars are typically only assigned by Toaster
  413. #self.update_project_vars(project,'DL_DIR')
  414. #self.update_project_vars(project,'SSTATE_DIR')
  415. # Assert found Toaster vars
  416. for var in self.toaster_vars.keys():
  417. pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
  418. pv.value = self.toaster_vars[var]
  419. _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
  420. pv.save()
  421. # Assert found BBLAYERS
  422. if 0 < verbose:
  423. for pl in ProjectLayer.objects.filter(project=project):
  424. release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
  425. print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
  426. self.apply_conf_bblayers(layers_list,lv_dict,project,release)
  427. if 0 < verbose:
  428. for pl in ProjectLayer.objects.filter(project=project):
  429. release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
  430. print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
  431. def handle(self, *args, **options):
  432. project_name = options['name']
  433. project_path = options['path']
  434. project_callback = options['callback'] if options['callback'] else ''
  435. release_name = options['release'] if options['release'] else ''
  436. #
  437. # Delete project
  438. #
  439. if options['delete_project']:
  440. try:
  441. print("Project '%s' delete from Toaster database" % (project_name))
  442. project = Project.objects.get(name=project_name)
  443. # TODO: deep project delete
  444. project.delete()
  445. print("Project '%s' Deleted" % (project_name))
  446. return
  447. except Exception as e:
  448. print("Project '%s' not found, not deleted (%s)" % (project_name,e))
  449. return
  450. #
  451. # Create/Update/Import project
  452. #
  453. # See if project (by name) exists
  454. project = None
  455. try:
  456. # Project already exists
  457. project = Project.objects.get(name=project_name)
  458. except Exception as e:
  459. pass
  460. # Find the installation's default release
  461. default_release = Release.objects.get(id=1)
  462. # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
  463. if ("reconfigure" == options['command']) and project is None:
  464. options['command'] = 'import'
  465. # 'Configure':
  466. if "configure" == options['command']:
  467. # Note: ignore any existing conf files
  468. # create project, SANITY: reuse any project of same name
  469. project = Project.objects.create_project(project_name,default_release,project)
  470. # 'Re-configure':
  471. if "reconfigure" == options['command']:
  472. # Scan the directory's conf files
  473. self.scan_conf_variables(project_path)
  474. # Scan the layer list
  475. layers_list,lv_dict = self.extract_bblayers()
  476. # Apply any new layers or variables
  477. self.apply_conf_variables(project,layers_list,lv_dict)
  478. # 'Import':
  479. if "import" == options['command']:
  480. # Scan the directory's conf files
  481. self.scan_conf_variables(project_path)
  482. # Remove these Toaster controlled variables
  483. for var in ('DL_DIR','SSTATE_DIR'):
  484. self.vars.pop(var, None)
  485. self.toaster_vars.pop(var, None)
  486. # Scan the layer list
  487. layers_list,lv_dict = self.extract_bblayers()
  488. # Find the directory's release, and promote to default_release if local paths
  489. release = self.find_import_release(layers_list,lv_dict,default_release)
  490. # create project, SANITY: reuse any project of same name
  491. project = Project.objects.create_project(project_name,release,project)
  492. # Apply any new layers or variables
  493. self.apply_conf_variables(project,layers_list,lv_dict,release)
  494. # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
  495. project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
  496. # Set up the project's meta data
  497. project.builddir = project_path
  498. project.merged_attr = True
  499. project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
  500. project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
  501. if ("configure" == options['command']) or ("import" == options['command']):
  502. # preset the mode and default image recipe
  503. project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
  504. project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
  505. # Assert any extended/custom actions or variables for new non-Toaster projects
  506. if not len(self.toaster_vars):
  507. pass
  508. else:
  509. project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
  510. # Save the updated Project
  511. project.save()
  512. _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
  513. if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
  514. print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
  515. else:
  516. print("|Project_id=%d" % (project.id))