widgets.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. #
  2. # BitBake Toaster Implementation
  3. #
  4. # Copyright (C) 2015 Intel Corporation
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. from django.views.generic import View, TemplateView
  9. from django.views.decorators.cache import cache_control
  10. from django.shortcuts import HttpResponse
  11. from django.core.cache import cache
  12. from django.core.paginator import Paginator, EmptyPage
  13. from django.db.models import Q
  14. from orm.models import Project, Build
  15. from django.template import Context, Template
  16. from django.template import VariableDoesNotExist
  17. from django.template import TemplateSyntaxError
  18. from django.core.serializers.json import DjangoJSONEncoder
  19. from django.core.exceptions import FieldError
  20. from django.utils import timezone
  21. from toastergui.templatetags.projecttags import sectohms, get_tasks
  22. from toastergui.templatetags.projecttags import json as template_json
  23. from django.http import JsonResponse
  24. from django.urls import reverse
  25. import types
  26. import json
  27. import collections
  28. import re
  29. import os
  30. from toastergui.tablefilter import TableFilterMap
  31. try:
  32. from urllib import unquote_plus
  33. except ImportError:
  34. from urllib.parse import unquote_plus
  35. import logging
  36. logger = logging.getLogger("toaster")
  37. class NoFieldOrDataName(Exception):
  38. pass
  39. class ToasterTable(TemplateView):
  40. def __init__(self, *args, **kwargs):
  41. super(ToasterTable, self).__init__()
  42. if 'template_name' in kwargs:
  43. self.template_name = kwargs['template_name']
  44. self.title = "Table"
  45. self.queryset = None
  46. self.columns = []
  47. # map from field names to Filter instances
  48. self.filter_map = TableFilterMap()
  49. self.total_count = 0
  50. self.static_context_extra = {}
  51. self.empty_state = "Sorry - no data found"
  52. self.default_orderby = ""
  53. # prevent HTTP caching of table data
  54. @cache_control(must_revalidate=True,
  55. max_age=0, no_store=True, no_cache=True)
  56. def dispatch(self, *args, **kwargs):
  57. return super(ToasterTable, self).dispatch(*args, **kwargs)
  58. def get_context_data(self, **kwargs):
  59. context = super(ToasterTable, self).get_context_data(**kwargs)
  60. context['title'] = self.title
  61. context['table_name'] = type(self).__name__.lower()
  62. context['empty_state'] = self.empty_state
  63. # global variables
  64. context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
  65. try:
  66. context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
  67. except:
  68. context['project_specific'] = ''
  69. return context
  70. def get(self, request, *args, **kwargs):
  71. if request.GET.get('format', None) == 'json':
  72. self.setup_queryset(*args, **kwargs)
  73. # Put the project id into the context for the static_data_template
  74. if 'pid' in kwargs:
  75. self.static_context_extra['pid'] = kwargs['pid']
  76. cmd = request.GET.get('cmd', None)
  77. if cmd and 'filterinfo' in cmd:
  78. data = self.get_filter_info(request, **kwargs)
  79. else:
  80. # If no cmd is specified we give you the table data
  81. data = self.get_data(request, **kwargs)
  82. return HttpResponse(data, content_type="application/json")
  83. return super(ToasterTable, self).get(request, *args, **kwargs)
  84. def get_filter_info(self, request, **kwargs):
  85. self.setup_filters(**kwargs)
  86. search = request.GET.get("search", None)
  87. if search:
  88. self.apply_search(search)
  89. name = request.GET.get("name", None)
  90. table_filter = self.filter_map.get_filter(name)
  91. return json.dumps(table_filter.to_json(self.queryset),
  92. indent=2,
  93. cls=DjangoJSONEncoder)
  94. def setup_columns(self, *args, **kwargs):
  95. """ function to implement in the subclass which sets up
  96. the columns """
  97. pass
  98. def setup_filters(self, *args, **kwargs):
  99. """ function to implement in the subclass which sets up the
  100. filters """
  101. pass
  102. def setup_queryset(self, *args, **kwargs):
  103. """ function to implement in the subclass which sets up the
  104. queryset"""
  105. pass
  106. def add_filter(self, table_filter):
  107. """Add a filter to the table.
  108. Args:
  109. table_filter: Filter instance
  110. """
  111. self.filter_map.add_filter(table_filter.name, table_filter)
  112. def add_column(self, title="", help_text="",
  113. orderable=False, hideable=True, hidden=False,
  114. field_name="", filter_name=None, static_data_name=None,
  115. static_data_template=None):
  116. """Add a column to the table.
  117. Args:
  118. title (str): Title for the table header
  119. help_text (str): Optional help text to describe the column
  120. orderable (bool): Whether the column can be ordered.
  121. We order on the field_name.
  122. hideable (bool): Whether the user can hide the column
  123. hidden (bool): Whether the column is default hidden
  124. field_name (str or list): field(s) required for this column's data
  125. static_data_name (str, optional): The column's main identifier
  126. which will replace the field_name.
  127. static_data_template(str, optional): The template to be rendered
  128. as data
  129. """
  130. self.columns.append({'title': title,
  131. 'help_text': help_text,
  132. 'orderable': orderable,
  133. 'hideable': hideable,
  134. 'hidden': hidden,
  135. 'field_name': field_name,
  136. 'filter_name': filter_name,
  137. 'static_data_name': static_data_name,
  138. 'static_data_template': static_data_template})
  139. def set_column_hidden(self, title, hidden):
  140. """
  141. Set the hidden state of the column to the value of hidden
  142. """
  143. for col in self.columns:
  144. if col['title'] == title:
  145. col['hidden'] = hidden
  146. break
  147. def set_column_hideable(self, title, hideable):
  148. """
  149. Set the hideable state of the column to the value of hideable
  150. """
  151. for col in self.columns:
  152. if col['title'] == title:
  153. col['hideable'] = hideable
  154. break
  155. def render_static_data(self, template, row):
  156. """Utility function to render the static data template"""
  157. context = {
  158. 'extra': self.static_context_extra,
  159. 'data': row,
  160. }
  161. context = Context(context)
  162. template = Template(template)
  163. return template.render(context)
  164. def apply_filter(self, filters, filter_value, **kwargs):
  165. """
  166. Apply a filter submitted in the querystring to the ToasterTable
  167. filters: (str) in the format:
  168. '<filter name>:<action name>'
  169. filter_value: (str) parameters to pass to the named filter
  170. <filter name> and <action name> are used to look up the correct filter
  171. in the ToasterTable's filter map; the <action params> are set on
  172. TableFilterAction* before its filter is applied and may modify the
  173. queryset returned by the filter
  174. """
  175. self.setup_filters(**kwargs)
  176. try:
  177. filter_name, action_name = filters.split(':')
  178. action_params = unquote_plus(filter_value)
  179. except ValueError:
  180. return
  181. if "all" in action_name:
  182. return
  183. try:
  184. table_filter = self.filter_map.get_filter(filter_name)
  185. action = table_filter.get_action(action_name)
  186. action.set_filter_params(action_params)
  187. self.queryset = action.filter(self.queryset)
  188. except KeyError:
  189. # pass it to the user - programming error here
  190. raise
  191. def apply_orderby(self, orderby):
  192. # Note that django will execute this when we try to retrieve the data
  193. self.queryset = self.queryset.order_by(orderby)
  194. def apply_search(self, search_term):
  195. """Creates a query based on the model's search_allowed_fields"""
  196. if not hasattr(self.queryset.model, 'search_allowed_fields'):
  197. raise Exception("Search fields aren't defined in the model %s"
  198. % self.queryset.model)
  199. search_queries = None
  200. for st in search_term.split(" "):
  201. queries = None
  202. for field in self.queryset.model.search_allowed_fields:
  203. query = Q(**{field + '__icontains': st})
  204. if queries:
  205. queries |= query
  206. else:
  207. queries = query
  208. if search_queries:
  209. search_queries &= queries
  210. else:
  211. search_queries = queries
  212. self.queryset = self.queryset.filter(search_queries)
  213. def get_data(self, request, **kwargs):
  214. """
  215. Returns the data for the page requested with the specified
  216. parameters applied
  217. filters: filter and action name, e.g. "outcome:build_succeeded"
  218. filter_value: value to pass to the named filter+action, e.g. "on"
  219. (for a toggle filter) or "2015-12-11,2015-12-12"
  220. (for a date range filter)
  221. """
  222. page_num = request.GET.get("page", 1)
  223. limit = request.GET.get("limit", 10)
  224. search = request.GET.get("search", None)
  225. filters = request.GET.get("filter", None)
  226. filter_value = request.GET.get("filter_value", "on")
  227. orderby = request.GET.get("orderby", None)
  228. nocache = request.GET.get("nocache", None)
  229. # Make a unique cache name
  230. cache_name = self.__class__.__name__
  231. for key, val in request.GET.items():
  232. if key == 'nocache':
  233. continue
  234. cache_name = cache_name + str(key) + str(val)
  235. for key, val in kwargs.items():
  236. cache_name = cache_name + str(key) + str(val)
  237. # No special chars allowed in the cache name apart from dash
  238. cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)
  239. if nocache:
  240. cache.delete(cache_name)
  241. data = cache.get(cache_name)
  242. if data:
  243. logger.debug("Got cache data for table '%s'" % self.title)
  244. return data
  245. self.setup_columns(**kwargs)
  246. if search:
  247. self.apply_search(search)
  248. if filters:
  249. self.apply_filter(filters, filter_value, **kwargs)
  250. if orderby:
  251. self.apply_orderby(orderby)
  252. paginator = Paginator(self.queryset, limit)
  253. try:
  254. page = paginator.page(page_num)
  255. except EmptyPage:
  256. page = paginator.page(1)
  257. data = {
  258. 'total': self.queryset.count(),
  259. 'default_orderby': self.default_orderby,
  260. 'columns': self.columns,
  261. 'rows': [],
  262. 'error': "ok",
  263. }
  264. try:
  265. for model_obj in page.object_list:
  266. # Use collection to maintain the order
  267. required_data = collections.OrderedDict()
  268. for col in self.columns:
  269. field = col['field_name']
  270. if not field:
  271. field = col['static_data_name']
  272. if not field:
  273. raise NoFieldOrDataName("Must supply a field_name or"
  274. "static_data_name for column"
  275. "%s.%s" %
  276. (self.__class__.__name__, col)
  277. )
  278. # Check if we need to process some static data
  279. if "static_data_name" in col and col['static_data_name']:
  280. # Overwrite the field_name with static_data_name
  281. # so that this can be used as the html class name
  282. col['field_name'] = col['static_data_name']
  283. try:
  284. # Render the template given
  285. required_data[col['static_data_name']] = \
  286. self.render_static_data(
  287. col['static_data_template'], model_obj)
  288. except (TemplateSyntaxError,
  289. VariableDoesNotExist) as e:
  290. logger.error("could not render template code"
  291. "%s %s %s",
  292. col['static_data_template'],
  293. e, self.__class__.__name__)
  294. required_data[col['static_data_name']] =\
  295. '<!--error-->'
  296. else:
  297. # Traverse to any foriegn key in the field
  298. # e.g. recipe__layer_version__name
  299. model_data = None
  300. if "__" in field:
  301. for subfield in field.split("__"):
  302. if not model_data:
  303. # The first iteration is always going to
  304. # be on the actual model object instance.
  305. # Subsequent ones are on the result of
  306. # that. e.g. forieng key objects
  307. model_data = getattr(model_obj,
  308. subfield)
  309. else:
  310. model_data = getattr(model_data,
  311. subfield)
  312. else:
  313. model_data = getattr(model_obj,
  314. col['field_name'])
  315. # We might have a model function as the field so
  316. # call it to return the data needed
  317. if isinstance(model_data, types.MethodType):
  318. model_data = model_data()
  319. required_data[col['field_name']] = model_data
  320. data['rows'].append(required_data)
  321. except FieldError:
  322. # pass it to the user - programming-error here
  323. raise
  324. data = json.dumps(data, indent=2, cls=DjangoJSONEncoder)
  325. cache.set(cache_name, data, 60*30)
  326. return data
  327. class ToasterTypeAhead(View):
  328. """ A typeahead mechanism to support the front end typeahead widgets """
  329. MAX_RESULTS = 6
  330. class MissingFieldsException(Exception):
  331. pass
  332. def __init__(self, *args, **kwargs):
  333. super(ToasterTypeAhead, self).__init__()
  334. def get(self, request, *args, **kwargs):
  335. def response(data):
  336. return HttpResponse(json.dumps(data,
  337. indent=2,
  338. cls=DjangoJSONEncoder),
  339. content_type="application/json")
  340. error = "ok"
  341. search_term = request.GET.get("search", None)
  342. if search_term is None:
  343. # We got no search value so return empty reponse
  344. return response({'error': error, 'results': []})
  345. try:
  346. prj = Project.objects.get(pk=kwargs['pid'])
  347. except KeyError:
  348. prj = None
  349. results = self.apply_search(search_term,
  350. prj,
  351. request)[:ToasterTypeAhead.MAX_RESULTS]
  352. if len(results) > 0:
  353. try:
  354. self.validate_fields(results[0])
  355. except self.MissingFieldsException as e:
  356. error = e
  357. data = {'results': results,
  358. 'error': error}
  359. return response(data)
  360. def validate_fields(self, result):
  361. if 'name' in result is False or 'detail' in result is False:
  362. raise self.MissingFieldsException(
  363. "name and detail are required fields")
  364. def apply_search(self, search_term, prj):
  365. """ Override this function to implement search. Return an array of
  366. dictionaries with a minium of a name and detail field"""
  367. pass
  368. class MostRecentBuildsView(View):
  369. def _was_yesterday_or_earlier(self, completed_on):
  370. now = timezone.now()
  371. delta = now - completed_on
  372. if delta.days >= 1:
  373. return True
  374. return False
  375. def get(self, request, *args, **kwargs):
  376. """
  377. Returns a list of builds in JSON format.
  378. """
  379. project = None
  380. project_id = request.GET.get('project_id', None)
  381. if project_id:
  382. try:
  383. project = Project.objects.get(pk=project_id)
  384. except:
  385. # if project lookup fails, assume no project
  386. pass
  387. recent_build_objs = Build.get_recent(project)
  388. recent_builds = []
  389. for build_obj in recent_build_objs:
  390. dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
  391. buildtime_url = reverse('buildtime', args=(build_obj.pk,))
  392. rebuild_url = \
  393. reverse('xhr_buildrequest', args=(build_obj.project.pk,))
  394. cancel_url = \
  395. reverse('xhr_buildrequest', args=(build_obj.project.pk,))
  396. build = {}
  397. build['id'] = build_obj.pk
  398. build['dashboard_url'] = dashboard_url
  399. buildrequest_id = None
  400. if hasattr(build_obj, 'buildrequest'):
  401. buildrequest_id = build_obj.buildrequest.pk
  402. build['buildrequest_id'] = buildrequest_id
  403. if build_obj.recipes_to_parse > 0:
  404. build['recipes_parsed_percentage'] = \
  405. int((build_obj.recipes_parsed /
  406. build_obj.recipes_to_parse) * 100)
  407. else:
  408. build['recipes_parsed_percentage'] = 0
  409. if build_obj.repos_to_clone > 0:
  410. build['repos_cloned_percentage'] = \
  411. int((build_obj.repos_cloned /
  412. build_obj.repos_to_clone) * 100)
  413. else:
  414. build['repos_cloned_percentage'] = 0
  415. build['progress_item'] = build_obj.progress_item
  416. tasks_complete_percentage = 0
  417. if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
  418. tasks_complete_percentage = 100
  419. elif build_obj.outcome == Build.IN_PROGRESS:
  420. tasks_complete_percentage = build_obj.completeper()
  421. build['tasks_complete_percentage'] = tasks_complete_percentage
  422. build['state'] = build_obj.get_state()
  423. build['errors'] = build_obj.errors.count()
  424. build['dashboard_errors_url'] = dashboard_url + '#errors'
  425. build['warnings'] = build_obj.warnings.count()
  426. build['dashboard_warnings_url'] = dashboard_url + '#warnings'
  427. build['buildtime'] = sectohms(build_obj.timespent_seconds)
  428. build['buildtime_url'] = buildtime_url
  429. build['rebuild_url'] = rebuild_url
  430. build['cancel_url'] = cancel_url
  431. build['is_default_project_build'] = build_obj.project.is_default
  432. build['build_targets_json'] = \
  433. template_json(get_tasks(build_obj.target_set.all()))
  434. # convert completed_on time to user's timezone
  435. completed_on = timezone.localtime(build_obj.completed_on)
  436. completed_on_template = '%H:%M'
  437. if self._was_yesterday_or_earlier(completed_on):
  438. completed_on_template = '%d/%m/%Y ' + completed_on_template
  439. build['completed_on'] = completed_on.strftime(
  440. completed_on_template)
  441. targets = []
  442. target_objs = build_obj.get_sorted_target_list()
  443. for target_obj in target_objs:
  444. if target_obj.task:
  445. targets.append(target_obj.target + ':' + target_obj.task)
  446. else:
  447. targets.append(target_obj.target)
  448. build['targets'] = ' '.join(targets)
  449. # abbreviated form of the full target list
  450. abbreviated_targets = ''
  451. num_targets = len(targets)
  452. if num_targets > 0:
  453. abbreviated_targets = targets[0]
  454. if num_targets > 1:
  455. abbreviated_targets += (' +%s' % (num_targets - 1))
  456. build['targets_abbreviated'] = abbreviated_targets
  457. recent_builds.append(build)
  458. return JsonResponse(recent_builds, safe=False)