api.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212
  1. #
  2. # BitBake Toaster Implementation
  3. #
  4. # Copyright (C) 2016 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. # Please run flake8 on this file before sending patches
  9. import os
  10. import re
  11. import logging
  12. import json
  13. import subprocess
  14. from collections import Counter
  15. from orm.models import Project, ProjectTarget, Build, Layer_Version
  16. from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
  17. from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
  18. from orm.models import Layer, Target, Package, Package_Dependency
  19. from orm.models import ProjectVariable
  20. from bldcontrol.models import BuildRequest, BuildEnvironment
  21. from bldcontrol import bbcontroller
  22. from django.http import HttpResponse, JsonResponse
  23. from django.views.generic import View
  24. from django.urls import reverse
  25. from django.db.models import Q, F
  26. from django.db import Error
  27. from toastergui.templatetags.projecttags import filtered_filesizeformat
  28. # development/debugging support
  29. verbose = 2
  30. def _log(msg):
  31. if 1 == verbose:
  32. print(msg)
  33. elif 2 == verbose:
  34. f1=open('/tmp/toaster.log', 'a')
  35. f1.write("|" + msg + "|\n" )
  36. f1.close()
  37. logger = logging.getLogger("toaster")
  38. def error_response(error):
  39. return JsonResponse({"error": error})
  40. class XhrBuildRequest(View):
  41. def get(self, request, *args, **kwargs):
  42. return HttpResponse()
  43. @staticmethod
  44. def cancel_build(br):
  45. """Cancel a build request"""
  46. try:
  47. bbctrl = bbcontroller.BitbakeController(br.environment)
  48. bbctrl.forceShutDown()
  49. except:
  50. # We catch a bunch of exceptions here because
  51. # this is where the server has not had time to start up
  52. # and the build request or build is in transit between
  53. # processes.
  54. # We can safely just set the build as cancelled
  55. # already as it never got started
  56. build = br.build
  57. build.outcome = Build.CANCELLED
  58. build.save()
  59. # We now hand over to the buildinfohelper to update the
  60. # build state once we've finished cancelling
  61. br.state = BuildRequest.REQ_CANCELLING
  62. br.save()
  63. def post(self, request, *args, **kwargs):
  64. """
  65. Build control
  66. Entry point: /xhr_buildrequest/<project_id>
  67. Method: POST
  68. Args:
  69. id: id of build to change
  70. buildCancel = build_request_id ...
  71. buildDelete = id ...
  72. targets = recipe_name ...
  73. Returns:
  74. {"error": "ok"}
  75. or
  76. {"error": <error message>}
  77. """
  78. project = Project.objects.get(pk=kwargs['pid'])
  79. if 'buildCancel' in request.POST:
  80. for i in request.POST['buildCancel'].strip().split(" "):
  81. try:
  82. br = BuildRequest.objects.get(project=project, pk=i)
  83. self.cancel_build(br)
  84. except BuildRequest.DoesNotExist:
  85. return error_response('No such build request id %s' % i)
  86. return error_response('ok')
  87. if 'buildDelete' in request.POST:
  88. for i in request.POST['buildDelete'].strip().split(" "):
  89. try:
  90. BuildRequest.objects.select_for_update().get(
  91. project=project,
  92. pk=i,
  93. state__lte=BuildRequest.REQ_DELETED).delete()
  94. except BuildRequest.DoesNotExist:
  95. pass
  96. return error_response("ok")
  97. if 'targets' in request.POST:
  98. ProjectTarget.objects.filter(project=project).delete()
  99. s = str(request.POST['targets'])
  100. for t in re.sub(r'[;%|"]', '', s).split(" "):
  101. if ":" in t:
  102. target, task = t.split(":")
  103. else:
  104. target = t
  105. task = ""
  106. ProjectTarget.objects.create(project=project,
  107. target=target,
  108. task=task)
  109. project.schedule_build()
  110. return error_response('ok')
  111. response = HttpResponse()
  112. response.status_code = 500
  113. return response
  114. class XhrProjectUpdate(View):
  115. def get(self, request, *args, **kwargs):
  116. return HttpResponse()
  117. def post(self, request, *args, **kwargs):
  118. """
  119. Project Update
  120. Entry point: /xhr_projectupdate/<project_id>
  121. Method: POST
  122. Args:
  123. pid: pid of project to update
  124. Returns:
  125. {"error": "ok"}
  126. or
  127. {"error": <error message>}
  128. """
  129. project = Project.objects.get(pk=kwargs['pid'])
  130. logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
  131. if 'do_update' in request.POST:
  132. # Extract any default image recipe
  133. if 'default_image' in request.POST:
  134. project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
  135. else:
  136. project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
  137. logger.debug("ProjectUpdateCallback:Chain to the build request")
  138. # Chain to the build request
  139. xhrBuildRequest = XhrBuildRequest()
  140. return xhrBuildRequest.post(request, *args, **kwargs)
  141. logger.warning("ERROR:XhrProjectUpdate")
  142. response = HttpResponse()
  143. response.status_code = 500
  144. return response
  145. class XhrSetDefaultImageUrl(View):
  146. def get(self, request, *args, **kwargs):
  147. return HttpResponse()
  148. def post(self, request, *args, **kwargs):
  149. """
  150. Project Update
  151. Entry point: /xhr_setdefaultimage/<project_id>
  152. Method: POST
  153. Args:
  154. pid: pid of project to update default image
  155. Returns:
  156. {"error": "ok"}
  157. or
  158. {"error": <error message>}
  159. """
  160. project = Project.objects.get(pk=kwargs['pid'])
  161. logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
  162. # set any default image recipe
  163. if 'targets' in request.POST:
  164. default_target = str(request.POST['targets'])
  165. project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
  166. logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
  167. return error_response('ok')
  168. logger.warning("ERROR:XhrSetDefaultImageUrl")
  169. response = HttpResponse()
  170. response.status_code = 500
  171. return response
  172. #
  173. # Layer Management
  174. #
  175. # Rules for 'local_source_dir' layers
  176. # * Layers must have a unique name in the Layers table
  177. # * A 'local_source_dir' layer is supposed to be shared
  178. # by all projects that use it, so that it can have the
  179. # same logical name
  180. # * Each project that uses a layer will have its own
  181. # LayerVersion and Project Layer for it
  182. # * During the Paroject delete process, when the last
  183. # LayerVersion for a 'local_source_dir' layer is deleted
  184. # then the Layer record is deleted to remove orphans
  185. #
  186. def scan_layer_content(layer,layer_version):
  187. # if this is a local layer directory, we can immediately scan its content
  188. if layer.local_source_dir:
  189. try:
  190. # recipes-*/*/*.bb
  191. cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
  192. recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
  193. recipes_list = recipes_list.decode("utf-8").strip()
  194. if recipes_list and 'No such' not in recipes_list:
  195. for recipe in recipes_list.split('\n'):
  196. recipe_path = recipe[recipe.rfind('recipes-'):]
  197. recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
  198. recipe_ver = recipe_name.rfind('_')
  199. if recipe_ver > 0:
  200. recipe_name = recipe_name[0:recipe_ver]
  201. if recipe_name:
  202. ro, created = Recipe.objects.get_or_create(
  203. layer_version=layer_version,
  204. name=recipe_name
  205. )
  206. if created:
  207. ro.file_path = recipe_path
  208. ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
  209. ro.description = ro.summary
  210. ro.save()
  211. except Exception as e:
  212. logger.warning("ERROR:scan_layer_content: %s" % e)
  213. class XhrLayer(View):
  214. """ Delete, Get, Add and Update Layer information
  215. Methods: GET POST DELETE PUT
  216. """
  217. def get(self, request, *args, **kwargs):
  218. """
  219. Get layer information
  220. Method: GET
  221. Entry point: /xhr_layer/<project id>/<layerversion_id>
  222. """
  223. try:
  224. layer_version = Layer_Version.objects.get(
  225. pk=kwargs['layerversion_id'])
  226. project = Project.objects.get(pk=kwargs['pid'])
  227. project_layers = ProjectLayer.objects.filter(
  228. project=project).values_list("layercommit_id",
  229. flat=True)
  230. ret = {
  231. 'error': 'ok',
  232. 'id': layer_version.pk,
  233. 'name': layer_version.layer.name,
  234. 'layerdetailurl':
  235. layer_version.get_detailspage_url(project.pk),
  236. 'vcs_ref': layer_version.get_vcs_reference(),
  237. 'vcs_url': layer_version.layer.vcs_url,
  238. 'local_source_dir': layer_version.layer.local_source_dir,
  239. 'layerdeps': {
  240. "list": [
  241. {
  242. "id": dep.id,
  243. "name": dep.layer.name,
  244. "layerdetailurl":
  245. dep.get_detailspage_url(project.pk),
  246. "vcs_url": dep.layer.vcs_url,
  247. "vcs_reference": dep.get_vcs_reference()
  248. }
  249. for dep in layer_version.get_alldeps(project.id)]
  250. },
  251. 'projectlayers': list(project_layers)
  252. }
  253. return JsonResponse(ret)
  254. except Layer_Version.DoesNotExist:
  255. error_response("No such layer")
  256. def post(self, request, *args, **kwargs):
  257. """
  258. Update a layer
  259. Method: POST
  260. Entry point: /xhr_layer/<layerversion_id>
  261. Args:
  262. vcs_url, dirpath, commit, up_branch, summary, description,
  263. local_source_dir
  264. add_dep = append a layerversion_id as a dependency
  265. rm_dep = remove a layerversion_id as a depedency
  266. Returns:
  267. {"error": "ok"}
  268. or
  269. {"error": <error message>}
  270. """
  271. try:
  272. # We currently only allow Imported layers to be edited
  273. layer_version = Layer_Version.objects.get(
  274. id=kwargs['layerversion_id'],
  275. project=kwargs['pid'],
  276. layer_source=LayerSource.TYPE_IMPORTED)
  277. except Layer_Version.DoesNotExist:
  278. return error_response("Cannot find imported layer to update")
  279. if "vcs_url" in request.POST:
  280. layer_version.layer.vcs_url = request.POST["vcs_url"]
  281. if "dirpath" in request.POST:
  282. layer_version.dirpath = request.POST["dirpath"]
  283. if "commit" in request.POST:
  284. layer_version.commit = request.POST["commit"]
  285. layer_version.branch = request.POST["commit"]
  286. if "summary" in request.POST:
  287. layer_version.layer.summary = request.POST["summary"]
  288. if "description" in request.POST:
  289. layer_version.layer.description = request.POST["description"]
  290. if "local_source_dir" in request.POST:
  291. layer_version.layer.local_source_dir = \
  292. request.POST["local_source_dir"]
  293. if "add_dep" in request.POST:
  294. lvd = LayerVersionDependency(
  295. layer_version=layer_version,
  296. depends_on_id=request.POST["add_dep"])
  297. lvd.save()
  298. if "rm_dep" in request.POST:
  299. rm_dep = LayerVersionDependency.objects.get(
  300. layer_version=layer_version,
  301. depends_on_id=request.POST["rm_dep"])
  302. rm_dep.delete()
  303. try:
  304. layer_version.layer.save()
  305. layer_version.save()
  306. except Exception as e:
  307. return error_response("Could not update layer version entry: %s"
  308. % e)
  309. return error_response("ok")
  310. def put(self, request, *args, **kwargs):
  311. """ Add a new layer
  312. Method: PUT
  313. Entry point: /xhr_layer/<project id>/
  314. Args:
  315. project_id, name,
  316. [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
  317. (csv)]
  318. """
  319. try:
  320. project = Project.objects.get(pk=kwargs['pid'])
  321. layer_data = json.loads(request.body.decode('utf-8'))
  322. # We require a unique layer name as otherwise the lists of layers
  323. # becomes very confusing
  324. existing_layers = \
  325. project.get_all_compatible_layer_versions().values_list(
  326. "layer__name",
  327. flat=True)
  328. add_to_project = False
  329. layer_deps_added = []
  330. if 'add_to_project' in layer_data:
  331. add_to_project = True
  332. if layer_data['name'] in existing_layers:
  333. return JsonResponse({"error": "layer-name-exists"})
  334. if ('local_source_dir' in layer_data):
  335. # Local layer can be shared across projects. They have no 'release'
  336. # and are not included in get_all_compatible_layer_versions() above
  337. layer,created = Layer.objects.get_or_create(name=layer_data['name'])
  338. _log("Local Layer created=%s" % created)
  339. else:
  340. layer = Layer.objects.create(name=layer_data['name'])
  341. layer_version = Layer_Version.objects.create(
  342. layer=layer,
  343. project=project,
  344. layer_source=LayerSource.TYPE_IMPORTED)
  345. # Local layer
  346. if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
  347. layer.local_source_dir = layer_data['local_source_dir']
  348. # git layer
  349. elif 'vcs_url' in layer_data:
  350. layer.vcs_url = layer_data['vcs_url']
  351. layer_version.dirpath = layer_data['dir_path']
  352. layer_version.commit = layer_data['git_ref']
  353. layer_version.branch = layer_data['git_ref']
  354. layer.save()
  355. layer_version.save()
  356. if add_to_project:
  357. ProjectLayer.objects.get_or_create(
  358. layercommit=layer_version, project=project)
  359. # Add the layer dependencies
  360. if 'layer_deps' in layer_data:
  361. for layer_dep_id in layer_data['layer_deps'].split(","):
  362. layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
  363. LayerVersionDependency.objects.get_or_create(
  364. layer_version=layer_version, depends_on=layer_dep)
  365. # Add layer deps to the project if specified
  366. if add_to_project:
  367. created, pl = ProjectLayer.objects.get_or_create(
  368. layercommit=layer_dep, project=project)
  369. layer_deps_added.append(
  370. {'name': layer_dep.layer.name,
  371. 'layerdetailurl':
  372. layer_dep.get_detailspage_url(project.pk)})
  373. # Scan the layer's content and update components
  374. scan_layer_content(layer,layer_version)
  375. except Layer_Version.DoesNotExist:
  376. return error_response("layer-dep-not-found")
  377. except Project.DoesNotExist:
  378. return error_response("project-not-found")
  379. except KeyError:
  380. return error_response("incorrect-parameters")
  381. return JsonResponse({'error': "ok",
  382. 'imported_layer': {
  383. 'name': layer.name,
  384. 'layerdetailurl':
  385. layer_version.get_detailspage_url()},
  386. 'deps_added': layer_deps_added})
  387. def delete(self, request, *args, **kwargs):
  388. """ Delete an imported layer
  389. Method: DELETE
  390. Entry point: /xhr_layer/<projed id>/<layerversion_id>
  391. """
  392. try:
  393. # We currently only allow Imported layers to be deleted
  394. layer_version = Layer_Version.objects.get(
  395. id=kwargs['layerversion_id'],
  396. project=kwargs['pid'],
  397. layer_source=LayerSource.TYPE_IMPORTED)
  398. except Layer_Version.DoesNotExist:
  399. return error_response("Cannot find imported layer to delete")
  400. try:
  401. ProjectLayer.objects.get(project=kwargs['pid'],
  402. layercommit=layer_version).delete()
  403. except ProjectLayer.DoesNotExist:
  404. pass
  405. layer_version.layer.delete()
  406. layer_version.delete()
  407. return JsonResponse({
  408. "error": "ok",
  409. "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],))
  410. })
  411. class XhrCustomRecipe(View):
  412. """ Create a custom image recipe """
  413. def post(self, request, *args, **kwargs):
  414. """
  415. Custom image recipe REST API
  416. Entry point: /xhr_customrecipe/
  417. Method: POST
  418. Args:
  419. name: name of custom recipe to create
  420. project: target project id of orm.models.Project
  421. base: base recipe id of orm.models.Recipe
  422. Returns:
  423. {"error": "ok",
  424. "url": <url of the created recipe>}
  425. or
  426. {"error": <error message>}
  427. """
  428. # check if request has all required parameters
  429. for param in ('name', 'project', 'base'):
  430. if param not in request.POST:
  431. return error_response("Missing parameter '%s'" % param)
  432. # get project and baserecipe objects
  433. params = {}
  434. for name, model in [("project", Project),
  435. ("base", Recipe)]:
  436. value = request.POST[name]
  437. try:
  438. params[name] = model.objects.get(id=value)
  439. except model.DoesNotExist:
  440. return error_response("Invalid %s id %s" % (name, value))
  441. # create custom recipe
  442. try:
  443. # Only allowed chars in name are a-z, 0-9 and -
  444. if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
  445. return error_response("invalid-name")
  446. custom_images = CustomImageRecipe.objects.all()
  447. # Are there any recipes with this name already in our project?
  448. existing_image_recipes_in_project = custom_images.filter(
  449. name=request.POST["name"], project=params["project"])
  450. if existing_image_recipes_in_project.count() > 0:
  451. return error_response("image-already-exists")
  452. # Are there any recipes with this name which aren't custom
  453. # image recipes?
  454. custom_image_ids = custom_images.values_list('id', flat=True)
  455. existing_non_image_recipes = Recipe.objects.filter(
  456. Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
  457. )
  458. if existing_non_image_recipes.count() > 0:
  459. return error_response("recipe-already-exists")
  460. # create layer 'Custom layer' and verion if needed
  461. layer, l_created = Layer.objects.get_or_create(
  462. name=CustomImageRecipe.LAYER_NAME,
  463. summary="Layer for custom recipes")
  464. if l_created:
  465. layer.local_source_dir = "toaster_created_layer"
  466. layer.save()
  467. # Check if we have a layer version already
  468. # We don't use get_or_create here because the dirpath will change
  469. # and is a required field
  470. lver = Layer_Version.objects.filter(Q(project=params['project']) &
  471. Q(layer=layer) &
  472. Q(build=None)).last()
  473. if lver is None:
  474. lver, lv_created = Layer_Version.objects.get_or_create(
  475. project=params['project'],
  476. layer=layer,
  477. layer_source=LayerSource.TYPE_LOCAL,
  478. dirpath="toaster_created_layer")
  479. # Add a dependency on our layer to the base recipe's layer
  480. LayerVersionDependency.objects.get_or_create(
  481. layer_version=lver,
  482. depends_on=params["base"].layer_version)
  483. # Add it to our current project if needed
  484. ProjectLayer.objects.get_or_create(project=params['project'],
  485. layercommit=lver,
  486. optional=False)
  487. # Create the actual recipe
  488. recipe, r_created = CustomImageRecipe.objects.get_or_create(
  489. name=request.POST["name"],
  490. base_recipe=params["base"],
  491. project=params["project"],
  492. layer_version=lver,
  493. is_image=True)
  494. # If we created the object then setup these fields. They may get
  495. # overwritten later on and cause the get_or_create to create a
  496. # duplicate if they've changed.
  497. if r_created:
  498. recipe.file_path = request.POST["name"]
  499. recipe.license = "MIT"
  500. recipe.version = "0.1"
  501. recipe.save()
  502. except Error as err:
  503. return error_response("Can't create custom recipe: %s" % err)
  504. # Find the package list from the last build of this recipe/target
  505. target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
  506. Q(build__project=params['project']) &
  507. (Q(target=params['base'].name) |
  508. Q(target=recipe.name))).last()
  509. if target:
  510. # Copy in every package
  511. # We don't want these packages to be linked to anything because
  512. # that underlying data may change e.g. delete a build
  513. for tpackage in target.target_installed_package_set.all():
  514. try:
  515. built_package = tpackage.package
  516. # The package had no recipe information so is a ghost
  517. # package skip it
  518. if built_package.recipe is None:
  519. continue
  520. config_package = CustomImagePackage.objects.get(
  521. name=built_package.name)
  522. recipe.includes_set.add(config_package)
  523. except Exception as e:
  524. logger.warning("Error adding package %s %s" %
  525. (tpackage.package.name, e))
  526. pass
  527. # pre-create layer directory structure, so that other builds
  528. # are not blocked by this new recipe dependecy
  529. # NOTE: this is parallel code to 'localhostbecontroller.py'
  530. be = BuildEnvironment.objects.all()[0]
  531. layerpath = os.path.join(be.builddir,
  532. CustomImageRecipe.LAYER_NAME)
  533. for name in ("conf", "recipes"):
  534. path = os.path.join(layerpath, name)
  535. if not os.path.isdir(path):
  536. os.makedirs(path)
  537. # pre-create layer.conf
  538. config = os.path.join(layerpath, "conf", "layer.conf")
  539. if not os.path.isfile(config):
  540. with open(config, "w") as conf:
  541. conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
  542. # pre-create new image's recipe file
  543. recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
  544. recipe.name)
  545. with open(recipe_path, "w") as recipef:
  546. content = recipe.generate_recipe_file_contents()
  547. if not content:
  548. # Delete this incomplete image recipe object
  549. recipe.delete()
  550. return error_response("recipe-parent-not-exist")
  551. else:
  552. recipef.write(recipe.generate_recipe_file_contents())
  553. return JsonResponse(
  554. {"error": "ok",
  555. "packages": recipe.get_all_packages().count(),
  556. "url": reverse('customrecipe', args=(params['project'].pk,
  557. recipe.id))})
  558. class XhrCustomRecipeId(View):
  559. """
  560. Set of ReST API processors working with recipe id.
  561. Entry point: /xhr_customrecipe/<recipe_id>
  562. Methods:
  563. GET - Get details of custom image recipe
  564. DELETE - Delete custom image recipe
  565. Returns:
  566. GET:
  567. {"error": "ok",
  568. "info": dictionary of field name -> value pairs
  569. of the CustomImageRecipe model}
  570. DELETE:
  571. {"error": "ok"}
  572. or
  573. {"error": <error message>}
  574. """
  575. @staticmethod
  576. def _get_ci_recipe(recipe_id):
  577. """ Get Custom Image recipe or return an error response"""
  578. try:
  579. custom_recipe = \
  580. CustomImageRecipe.objects.get(pk=recipe_id)
  581. return custom_recipe, None
  582. except CustomImageRecipe.DoesNotExist:
  583. return None, error_response("Custom recipe with id=%s "
  584. "not found" % recipe_id)
  585. def get(self, request, *args, **kwargs):
  586. custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
  587. if error:
  588. return error
  589. if request.method == 'GET':
  590. info = {"id": custom_recipe.id,
  591. "name": custom_recipe.name,
  592. "base_recipe_id": custom_recipe.base_recipe.id,
  593. "project_id": custom_recipe.project.id}
  594. return JsonResponse({"error": "ok", "info": info})
  595. def delete(self, request, *args, **kwargs):
  596. custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
  597. if error:
  598. return error
  599. project = custom_recipe.project
  600. custom_recipe.delete()
  601. return JsonResponse({"error": "ok",
  602. "gotoUrl": reverse("projectcustomimages",
  603. args=(project.pk,))})
  604. class XhrCustomRecipePackages(View):
  605. """
  606. ReST API to add/remove packages to/from custom recipe.
  607. Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
  608. Methods:
  609. PUT - Add package to the recipe
  610. DELETE - Delete package from the recipe
  611. GET - Get package information
  612. Returns:
  613. {"error": "ok"}
  614. or
  615. {"error": <error message>}
  616. """
  617. @staticmethod
  618. def _get_package(package_id):
  619. try:
  620. package = CustomImagePackage.objects.get(pk=package_id)
  621. return package, None
  622. except Package.DoesNotExist:
  623. return None, error_response("Package with id=%s "
  624. "not found" % package_id)
  625. def _traverse_dependents(self, next_package_id,
  626. rev_deps, all_current_packages, tree_level=0):
  627. """
  628. Recurse through reverse dependency tree for next_package_id.
  629. Limit the reverse dependency search to packages not already scanned,
  630. that is, not already in rev_deps.
  631. Limit the scan to a depth (tree_level) not exceeding the count of
  632. all packages in the custom image, and if that depth is exceeded
  633. return False, pop out of the recursion, and write a warning
  634. to the log, but this is unlikely, suggesting a dependency loop
  635. not caught by bitbake.
  636. On return, the input/output arg rev_deps is appended with queryset
  637. dictionary elements, annotated for use in the customimage template.
  638. The list has unsorted, but unique elements.
  639. """
  640. max_dependency_tree_depth = all_current_packages.count()
  641. if tree_level >= max_dependency_tree_depth:
  642. logger.warning(
  643. "The number of reverse dependencies "
  644. "for this package exceeds " + max_dependency_tree_depth +
  645. " and the remaining reverse dependencies will not be removed")
  646. return True
  647. package = CustomImagePackage.objects.get(id=next_package_id)
  648. dependents = \
  649. package.package_dependencies_target.annotate(
  650. name=F('package__name'),
  651. pk=F('package__pk'),
  652. size=F('package__size'),
  653. ).values("name", "pk", "size").exclude(
  654. ~Q(pk__in=all_current_packages)
  655. )
  656. for pkg in dependents:
  657. if pkg in rev_deps:
  658. # already seen, skip dependent search
  659. continue
  660. rev_deps.append(pkg)
  661. if (self._traverse_dependents(pkg["pk"], rev_deps,
  662. all_current_packages,
  663. tree_level+1)):
  664. return True
  665. return False
  666. def _get_all_dependents(self, package_id, all_current_packages):
  667. """
  668. Returns sorted list of recursive reverse dependencies for package_id,
  669. as a list of dictionary items, by recursing through dependency
  670. relationships.
  671. """
  672. rev_deps = []
  673. self._traverse_dependents(package_id, rev_deps, all_current_packages)
  674. rev_deps = sorted(rev_deps, key=lambda x: x["name"])
  675. return rev_deps
  676. def get(self, request, *args, **kwargs):
  677. recipe, error = XhrCustomRecipeId._get_ci_recipe(
  678. kwargs['recipe_id'])
  679. if error:
  680. return error
  681. # If no package_id then list all the current packages
  682. if not kwargs['package_id']:
  683. total_size = 0
  684. packages = recipe.get_all_packages().values("id",
  685. "name",
  686. "version",
  687. "size")
  688. for package in packages:
  689. package['size_formatted'] = \
  690. filtered_filesizeformat(package['size'])
  691. total_size += package['size']
  692. return JsonResponse({"error": "ok",
  693. "packages": list(packages),
  694. "total": len(packages),
  695. "total_size": total_size,
  696. "total_size_formatted":
  697. filtered_filesizeformat(total_size)})
  698. else:
  699. package, error = XhrCustomRecipePackages._get_package(
  700. kwargs['package_id'])
  701. if error:
  702. return error
  703. all_current_packages = recipe.get_all_packages()
  704. # Dependencies for package which aren't satisfied by the
  705. # current packages in the custom image recipe
  706. deps = package.package_dependencies_source.for_target_or_none(
  707. recipe.name)['packages'].annotate(
  708. name=F('depends_on__name'),
  709. pk=F('depends_on__pk'),
  710. size=F('depends_on__size'),
  711. ).values("name", "pk", "size").filter(
  712. # There are two depends types we don't know why
  713. (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
  714. Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
  715. ~Q(pk__in=all_current_packages)
  716. )
  717. # Reverse dependencies which are needed by packages that are
  718. # in the image. Recursive search providing all dependents,
  719. # not just immediate dependents.
  720. reverse_deps = self._get_all_dependents(kwargs['package_id'],
  721. all_current_packages)
  722. total_size_deps = 0
  723. total_size_reverse_deps = 0
  724. for dep in deps:
  725. dep['size_formatted'] = \
  726. filtered_filesizeformat(dep['size'])
  727. total_size_deps += dep['size']
  728. for dep in reverse_deps:
  729. dep['size_formatted'] = \
  730. filtered_filesizeformat(dep['size'])
  731. total_size_reverse_deps += dep['size']
  732. return JsonResponse(
  733. {"error": "ok",
  734. "id": package.pk,
  735. "name": package.name,
  736. "version": package.version,
  737. "unsatisfied_dependencies": list(deps),
  738. "unsatisfied_dependencies_size": total_size_deps,
  739. "unsatisfied_dependencies_size_formatted":
  740. filtered_filesizeformat(total_size_deps),
  741. "reverse_dependencies": list(reverse_deps),
  742. "reverse_dependencies_size": total_size_reverse_deps,
  743. "reverse_dependencies_size_formatted":
  744. filtered_filesizeformat(total_size_reverse_deps)})
  745. def put(self, request, *args, **kwargs):
  746. recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
  747. package, error = self._get_package(kwargs['package_id'])
  748. if error:
  749. return error
  750. included_packages = recipe.includes_set.values_list('pk',
  751. flat=True)
  752. # If we're adding back a package which used to be included in this
  753. # image all we need to do is remove it from the excludes
  754. if package.pk in included_packages:
  755. try:
  756. recipe.excludes_set.remove(package)
  757. return {"error": "ok"}
  758. except Package.DoesNotExist:
  759. return error_response("Package %s not found in excludes"
  760. " but was in included list" %
  761. package.name)
  762. else:
  763. recipe.appends_set.add(package)
  764. # Make sure that package is not in the excludes set
  765. try:
  766. recipe.excludes_set.remove(package)
  767. except:
  768. pass
  769. # Add the dependencies we think will be added to the recipe
  770. # as a result of appending this package.
  771. # TODO this should recurse down the entire deps tree
  772. for dep in package.package_dependencies_source.all_depends():
  773. try:
  774. cust_package = CustomImagePackage.objects.get(
  775. name=dep.depends_on.name)
  776. recipe.includes_set.add(cust_package)
  777. try:
  778. # When adding the pre-requisite package, make
  779. # sure it's not in the excluded list from a
  780. # prior removal.
  781. recipe.excludes_set.remove(cust_package)
  782. except package.DoesNotExist:
  783. # Don't care if the package had never been excluded
  784. pass
  785. except:
  786. logger.warning("Could not add package's suggested"
  787. "dependencies to the list")
  788. return JsonResponse({"error": "ok"})
  789. def delete(self, request, *args, **kwargs):
  790. recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
  791. package, error = self._get_package(kwargs['package_id'])
  792. if error:
  793. return error
  794. try:
  795. included_packages = recipe.includes_set.values_list('pk',
  796. flat=True)
  797. # If we're deleting a package which is included we need to
  798. # Add it to the excludes list.
  799. if package.pk in included_packages:
  800. recipe.excludes_set.add(package)
  801. else:
  802. recipe.appends_set.remove(package)
  803. # remove dependencies as well
  804. all_current_packages = recipe.get_all_packages()
  805. reverse_deps_dictlist = self._get_all_dependents(
  806. package.pk,
  807. all_current_packages)
  808. ids = [entry['pk'] for entry in reverse_deps_dictlist]
  809. reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
  810. for r in reverse_deps:
  811. try:
  812. if r.id in included_packages:
  813. recipe.excludes_set.add(r)
  814. else:
  815. recipe.appends_set.remove(r)
  816. except:
  817. pass
  818. return JsonResponse({"error": "ok"})
  819. except CustomImageRecipe.DoesNotExist:
  820. return error_response("Tried to remove package that wasn't"
  821. " present")
  822. class XhrProject(View):
  823. """ Create, delete or edit a project
  824. Entry point: /xhr_project/<project_id>
  825. """
  826. def post(self, request, *args, **kwargs):
  827. """
  828. Edit project control
  829. Args:
  830. layerAdd = layer_version_id layer_version_id ...
  831. layerDel = layer_version_id layer_version_id ...
  832. projectName = new_project_name
  833. machineName = new_machine_name
  834. Returns:
  835. {"error": "ok"}
  836. or
  837. {"error": <error message>}
  838. """
  839. try:
  840. prj = Project.objects.get(pk=kwargs['project_id'])
  841. except Project.DoesNotExist:
  842. return error_response("No such project")
  843. # Add layers
  844. if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0:
  845. for layer_version_id in request.POST['layerAdd'].split(','):
  846. try:
  847. lv = Layer_Version.objects.get(pk=int(layer_version_id))
  848. ProjectLayer.objects.get_or_create(project=prj,
  849. layercommit=lv)
  850. except Layer_Version.DoesNotExist:
  851. return error_response("Layer version %s asked to add "
  852. "doesn't exist" % layer_version_id)
  853. # Remove layers
  854. if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0:
  855. layer_version_ids = request.POST['layerDel'].split(',')
  856. ProjectLayer.objects.filter(
  857. project=prj,
  858. layercommit_id__in=layer_version_ids).delete()
  859. # Project name change
  860. if 'projectName' in request.POST:
  861. prj.name = request.POST['projectName']
  862. prj.save()
  863. # Machine name change
  864. if 'machineName' in request.POST:
  865. machinevar = prj.projectvariable_set.get(name="MACHINE")
  866. machinevar.value = request.POST['machineName']
  867. machinevar.save()
  868. # Distro name change
  869. if 'distroName' in request.POST:
  870. distrovar = prj.projectvariable_set.get(name="DISTRO")
  871. distrovar.value = request.POST['distroName']
  872. distrovar.save()
  873. return JsonResponse({"error": "ok"})
  874. def get(self, request, *args, **kwargs):
  875. """
  876. Returns:
  877. json object representing the current project
  878. or:
  879. {"error": <error message>}
  880. """
  881. try:
  882. project = Project.objects.get(pk=kwargs['project_id'])
  883. except Project.DoesNotExist:
  884. return error_response("Project %s does not exist" %
  885. kwargs['project_id'])
  886. # Create the frequently built targets list
  887. freqtargets = Counter(Target.objects.filter(
  888. Q(build__project=project),
  889. ~Q(build__outcome=Build.IN_PROGRESS)
  890. ).order_by("target").values_list("target", flat=True))
  891. freqtargets = freqtargets.most_common(5)
  892. # We now have the targets in order of frequency but if there are two
  893. # with the same frequency then we need to make sure those are in
  894. # alphabetical order without losing the frequency ordering
  895. tmp = []
  896. switch = None
  897. for i, freqtartget in enumerate(freqtargets):
  898. target, count = freqtartget
  899. try:
  900. target_next, count_next = freqtargets[i+1]
  901. if count == count_next and target > target_next:
  902. switch = target
  903. continue
  904. except IndexError:
  905. pass
  906. tmp.append(target)
  907. if switch:
  908. tmp.append(switch)
  909. switch = None
  910. freqtargets = tmp
  911. layers = []
  912. for layer in project.projectlayer_set.all():
  913. layers.append({
  914. "id": layer.layercommit.pk,
  915. "name": layer.layercommit.layer.name,
  916. "vcs_url": layer.layercommit.layer.vcs_url,
  917. "local_source_dir": layer.layercommit.layer.local_source_dir,
  918. "vcs_reference": layer.layercommit.get_vcs_reference(),
  919. "url": layer.layercommit.layer.layer_index_url,
  920. "layerdetailurl": layer.layercommit.get_detailspage_url(
  921. project.pk),
  922. "xhrLayerUrl": reverse("xhr_layer",
  923. args=(project.pk,
  924. layer.layercommit.pk)),
  925. "layersource": layer.layercommit.layer_source
  926. })
  927. data = {
  928. "name": project.name,
  929. "layers": layers,
  930. "freqtargets": freqtargets,
  931. }
  932. if project.release is not None:
  933. data['release'] = {
  934. "id": project.release.pk,
  935. "name": project.release.name,
  936. "description": project.release.description
  937. }
  938. try:
  939. data["machine"] = {"name":
  940. project.projectvariable_set.get(
  941. name="MACHINE").value}
  942. except ProjectVariable.DoesNotExist:
  943. data["machine"] = None
  944. try:
  945. data["distro"] = {"name":
  946. project.projectvariable_set.get(
  947. name="DISTRO").value}
  948. except ProjectVariable.DoesNotExist:
  949. data["distro"] = None
  950. data['error'] = "ok"
  951. return JsonResponse(data)
  952. def put(self, request, *args, **kwargs):
  953. # TODO create new project api
  954. return HttpResponse()
  955. def delete(self, request, *args, **kwargs):
  956. """Delete a project. Cancels any builds in progress"""
  957. try:
  958. project = Project.objects.get(pk=kwargs['project_id'])
  959. # Cancel any builds in progress
  960. for br in BuildRequest.objects.filter(
  961. project=project,
  962. state=BuildRequest.REQ_INPROGRESS):
  963. XhrBuildRequest.cancel_build(br)
  964. # gather potential orphaned local layers attached to this project
  965. project_local_layer_list = []
  966. for pl in ProjectLayer.objects.filter(project=project):
  967. if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
  968. project_local_layer_list.append(pl.layercommit.layer)
  969. # deep delete the project and its dependencies
  970. project.delete()
  971. # delete any local layers now orphaned
  972. _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
  973. for layer in project_local_layer_list:
  974. layer_refs = Layer_Version.objects.filter(layer=layer)
  975. _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
  976. if 0 == len(layer_refs):
  977. _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
  978. Layer.objects.filter(pk=layer.id).delete()
  979. except Project.DoesNotExist:
  980. return error_response("Project %s does not exist" %
  981. kwargs['project_id'])
  982. return JsonResponse({
  983. "error": "ok",
  984. "gotoUrl": reverse("all-projects", args=[])
  985. })
  986. class XhrBuild(View):
  987. """ Delete a build object
  988. Entry point: /xhr_build/<build_id>
  989. """
  990. def delete(self, request, *args, **kwargs):
  991. """
  992. Delete build data
  993. Args:
  994. build_id = build_id
  995. Returns:
  996. {"error": "ok"}
  997. or
  998. {"error": <error message>}
  999. """
  1000. try:
  1001. build = Build.objects.get(pk=kwargs['build_id'])
  1002. project = build.project
  1003. build.delete()
  1004. except Build.DoesNotExist:
  1005. return error_response("Build %s does not exist" %
  1006. kwargs['build_id'])
  1007. return JsonResponse({
  1008. "error": "ok",
  1009. "gotoUrl": reverse("projectbuilds", args=(project.pk,))
  1010. })