data_smart.py 38 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. """
  4. BitBake Smart Dictionary Implementation
  5. Functions for interacting with the data structure used by the
  6. BitBake build tools.
  7. """
  8. # Copyright (C) 2003, 2004 Chris Larson
  9. # Copyright (C) 2004, 2005 Seb Frankengul
  10. # Copyright (C) 2005, 2006 Holger Hans Peter Freyther
  11. # Copyright (C) 2005 Uli Luckas
  12. # Copyright (C) 2005 ROAD GmbH
  13. #
  14. # SPDX-License-Identifier: GPL-2.0-only
  15. #
  16. # This program is free software; you can redistribute it and/or modify
  17. # it under the terms of the GNU General Public License version 2 as
  18. # published by the Free Software Foundation.
  19. #
  20. # This program is distributed in the hope that it will be useful,
  21. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  22. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  23. # GNU General Public License for more details.
  24. #
  25. # You should have received a copy of the GNU General Public License along
  26. # with this program; if not, write to the Free Software Foundation, Inc.,
  27. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  28. # Based on functions from the base bb module, Copyright 2003 Holger Schurig
  29. import copy, re, sys, traceback
  30. from collections import MutableMapping
  31. import logging
  32. import hashlib
  33. import bb, bb.codeparser
  34. from bb import utils
  35. from bb.COW import COWDictBase
  36. logger = logging.getLogger("BitBake.Data")
  37. __setvar_keyword__ = ["_append", "_prepend", "_remove"]
  38. __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
  39. __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
  40. __expand_python_regexp__ = re.compile(r"\${@.+?}")
  41. __whitespace_split__ = re.compile(r'(\s)')
  42. __override_regexp__ = re.compile(r'[a-z0-9]+')
  43. def infer_caller_details(loginfo, parent = False, varval = True):
  44. """Save the caller the trouble of specifying everything."""
  45. # Save effort.
  46. if 'ignore' in loginfo and loginfo['ignore']:
  47. return
  48. # If nothing was provided, mark this as possibly unneeded.
  49. if not loginfo:
  50. loginfo['ignore'] = True
  51. return
  52. # Infer caller's likely values for variable (var) and value (value),
  53. # to reduce clutter in the rest of the code.
  54. above = None
  55. def set_above():
  56. try:
  57. raise Exception
  58. except Exception:
  59. tb = sys.exc_info()[2]
  60. if parent:
  61. return tb.tb_frame.f_back.f_back.f_back
  62. else:
  63. return tb.tb_frame.f_back.f_back
  64. if varval and ('variable' not in loginfo or 'detail' not in loginfo):
  65. if not above:
  66. above = set_above()
  67. lcls = above.f_locals.items()
  68. for k, v in lcls:
  69. if k == 'value' and 'detail' not in loginfo:
  70. loginfo['detail'] = v
  71. if k == 'var' and 'variable' not in loginfo:
  72. loginfo['variable'] = v
  73. # Infer file/line/function from traceback
  74. # Don't use traceback.extract_stack() since it fills the line contents which
  75. # we don't need and that hits stat syscalls
  76. if 'file' not in loginfo:
  77. if not above:
  78. above = set_above()
  79. f = above.f_back
  80. line = f.f_lineno
  81. file = f.f_code.co_filename
  82. func = f.f_code.co_name
  83. loginfo['file'] = file
  84. loginfo['line'] = line
  85. if func not in loginfo:
  86. loginfo['func'] = func
  87. class VariableParse:
  88. def __init__(self, varname, d, val = None):
  89. self.varname = varname
  90. self.d = d
  91. self.value = val
  92. self.references = set()
  93. self.execs = set()
  94. self.contains = {}
  95. def var_sub(self, match):
  96. key = match.group()[2:-1]
  97. if self.varname and key:
  98. if self.varname == key:
  99. raise Exception("variable %s references itself!" % self.varname)
  100. var = self.d.getVarFlag(key, "_content")
  101. self.references.add(key)
  102. if var is not None:
  103. return var
  104. else:
  105. return match.group()
  106. def python_sub(self, match):
  107. if isinstance(match, str):
  108. code = match
  109. else:
  110. code = match.group()[3:-1]
  111. if "_remote_data" in self.d:
  112. connector = self.d["_remote_data"]
  113. return connector.expandPythonRef(self.varname, code, self.d)
  114. if self.varname:
  115. varname = 'Var <%s>' % self.varname
  116. else:
  117. varname = '<expansion>'
  118. codeobj = compile(code.strip(), varname, "eval")
  119. parser = bb.codeparser.PythonParser(self.varname, logger)
  120. parser.parse_python(code)
  121. if self.varname:
  122. vardeps = self.d.getVarFlag(self.varname, "vardeps")
  123. if vardeps is None:
  124. parser.log.flush()
  125. else:
  126. parser.log.flush()
  127. self.references |= parser.references
  128. self.execs |= parser.execs
  129. for k in parser.contains:
  130. if k not in self.contains:
  131. self.contains[k] = parser.contains[k].copy()
  132. else:
  133. self.contains[k].update(parser.contains[k])
  134. value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
  135. return str(value)
  136. class DataContext(dict):
  137. def __init__(self, metadata, **kwargs):
  138. self.metadata = metadata
  139. dict.__init__(self, **kwargs)
  140. self['d'] = metadata
  141. def __missing__(self, key):
  142. value = self.metadata.getVar(key)
  143. if value is None or self.metadata.getVarFlag(key, 'func', False):
  144. raise KeyError(key)
  145. else:
  146. return value
  147. class ExpansionError(Exception):
  148. def __init__(self, varname, expression, exception):
  149. self.expression = expression
  150. self.variablename = varname
  151. self.exception = exception
  152. if varname:
  153. if expression:
  154. self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
  155. else:
  156. self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
  157. else:
  158. self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
  159. Exception.__init__(self, self.msg)
  160. self.args = (varname, expression, exception)
  161. def __str__(self):
  162. return self.msg
  163. class IncludeHistory(object):
  164. def __init__(self, parent = None, filename = '[TOP LEVEL]'):
  165. self.parent = parent
  166. self.filename = filename
  167. self.children = []
  168. self.current = self
  169. def copy(self):
  170. new = IncludeHistory(self.parent, self.filename)
  171. for c in self.children:
  172. new.children.append(c)
  173. return new
  174. def include(self, filename):
  175. newfile = IncludeHistory(self.current, filename)
  176. self.current.children.append(newfile)
  177. self.current = newfile
  178. return self
  179. def __enter__(self):
  180. pass
  181. def __exit__(self, a, b, c):
  182. if self.current.parent:
  183. self.current = self.current.parent
  184. else:
  185. bb.warn("Include log: Tried to finish '%s' at top level." % filename)
  186. return False
  187. def emit(self, o, level = 0):
  188. """Emit an include history file, and its children."""
  189. if level:
  190. spaces = " " * (level - 1)
  191. o.write("# %s%s" % (spaces, self.filename))
  192. if len(self.children) > 0:
  193. o.write(" includes:")
  194. else:
  195. o.write("#\n# INCLUDE HISTORY:\n#")
  196. level = level + 1
  197. for child in self.children:
  198. o.write("\n")
  199. child.emit(o, level)
  200. class VariableHistory(object):
  201. def __init__(self, dataroot):
  202. self.dataroot = dataroot
  203. self.variables = COWDictBase.copy()
  204. def copy(self):
  205. new = VariableHistory(self.dataroot)
  206. new.variables = self.variables.copy()
  207. return new
  208. def __getstate__(self):
  209. vardict = {}
  210. for k, v in self.variables.iteritems():
  211. vardict[k] = v
  212. return {'dataroot': self.dataroot,
  213. 'variables': vardict}
  214. def __setstate__(self, state):
  215. self.dataroot = state['dataroot']
  216. self.variables = COWDictBase.copy()
  217. for k, v in state['variables'].items():
  218. self.variables[k] = v
  219. def record(self, *kwonly, **loginfo):
  220. if not self.dataroot._tracking:
  221. return
  222. if len(kwonly) > 0:
  223. raise TypeError
  224. infer_caller_details(loginfo, parent = True)
  225. if 'ignore' in loginfo and loginfo['ignore']:
  226. return
  227. if 'op' not in loginfo or not loginfo['op']:
  228. loginfo['op'] = 'set'
  229. if 'detail' in loginfo:
  230. loginfo['detail'] = str(loginfo['detail'])
  231. if 'variable' not in loginfo or 'file' not in loginfo:
  232. raise ValueError("record() missing variable or file.")
  233. var = loginfo['variable']
  234. if var not in self.variables:
  235. self.variables[var] = []
  236. if not isinstance(self.variables[var], list):
  237. return
  238. if 'nodups' in loginfo and loginfo in self.variables[var]:
  239. return
  240. self.variables[var].append(loginfo.copy())
  241. def rename_variable_hist(self, oldvar, newvar):
  242. if not self.dataroot._tracking:
  243. return
  244. if oldvar not in self.variables:
  245. return
  246. if newvar not in self.variables:
  247. self.variables[newvar] = []
  248. for i in self.variables[oldvar]:
  249. self.variables[newvar].append(i.copy())
  250. def variable(self, var):
  251. remote_connector = self.dataroot.getVar('_remote_data', False)
  252. if remote_connector:
  253. varhistory = remote_connector.getVarHistory(var)
  254. else:
  255. varhistory = []
  256. if var in self.variables:
  257. varhistory.extend(self.variables[var])
  258. return varhistory
  259. def emit(self, var, oval, val, o, d):
  260. history = self.variable(var)
  261. # Append override history
  262. if var in d.overridedata:
  263. for (r, override) in d.overridedata[var]:
  264. for event in self.variable(r):
  265. loginfo = event.copy()
  266. if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
  267. continue
  268. loginfo['variable'] = var
  269. loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
  270. history.append(loginfo)
  271. commentVal = re.sub('\n', '\n#', str(oval))
  272. if history:
  273. if len(history) == 1:
  274. o.write("#\n# $%s\n" % var)
  275. else:
  276. o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
  277. for event in history:
  278. # o.write("# %s\n" % str(event))
  279. if 'func' in event:
  280. # If we have a function listed, this is internal
  281. # code, not an operation in a config file, and the
  282. # full path is distracting.
  283. event['file'] = re.sub('.*/', '', event['file'])
  284. display_func = ' [%s]' % event['func']
  285. else:
  286. display_func = ''
  287. if 'flag' in event:
  288. flag = '[%s] ' % (event['flag'])
  289. else:
  290. flag = ''
  291. o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
  292. if len(history) > 1:
  293. o.write("# pre-expansion value:\n")
  294. o.write('# "%s"\n' % (commentVal))
  295. else:
  296. o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
  297. o.write('# "%s"\n' % (commentVal))
  298. def get_variable_files(self, var):
  299. """Get the files where operations are made on a variable"""
  300. var_history = self.variable(var)
  301. files = []
  302. for event in var_history:
  303. files.append(event['file'])
  304. return files
  305. def get_variable_lines(self, var, f):
  306. """Get the line where a operation is made on a variable in file f"""
  307. var_history = self.variable(var)
  308. lines = []
  309. for event in var_history:
  310. if f== event['file']:
  311. line = event['line']
  312. lines.append(line)
  313. return lines
  314. def get_variable_items_files(self, var, d):
  315. """
  316. Use variable history to map items added to a list variable and
  317. the files in which they were added.
  318. """
  319. history = self.variable(var)
  320. finalitems = (d.getVar(var) or '').split()
  321. filemap = {}
  322. isset = False
  323. for event in history:
  324. if 'flag' in event:
  325. continue
  326. if event['op'] == '_remove':
  327. continue
  328. if isset and event['op'] == 'set?':
  329. continue
  330. isset = True
  331. items = d.expand(event['detail']).split()
  332. for item in items:
  333. # This is a little crude but is belt-and-braces to avoid us
  334. # having to handle every possible operation type specifically
  335. if item in finalitems and not item in filemap:
  336. filemap[item] = event['file']
  337. return filemap
  338. def del_var_history(self, var, f=None, line=None):
  339. """If file f and line are not given, the entire history of var is deleted"""
  340. if var in self.variables:
  341. if f and line:
  342. self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
  343. else:
  344. self.variables[var] = []
  345. class DataSmart(MutableMapping):
  346. def __init__(self):
  347. self.dict = {}
  348. self.inchistory = IncludeHistory()
  349. self.varhistory = VariableHistory(self)
  350. self._tracking = False
  351. self.expand_cache = {}
  352. # cookie monster tribute
  353. # Need to be careful about writes to overridedata as
  354. # its only a shallow copy, could influence other data store
  355. # copies!
  356. self.overridedata = {}
  357. self.overrides = None
  358. self.overridevars = set(["OVERRIDES", "FILE"])
  359. self.inoverride = False
  360. def enableTracking(self):
  361. self._tracking = True
  362. def disableTracking(self):
  363. self._tracking = False
  364. def expandWithRefs(self, s, varname):
  365. if not isinstance(s, str): # sanity check
  366. return VariableParse(varname, self, s)
  367. varparse = VariableParse(varname, self)
  368. while s.find('${') != -1:
  369. olds = s
  370. try:
  371. s = __expand_var_regexp__.sub(varparse.var_sub, s)
  372. try:
  373. s = __expand_python_regexp__.sub(varparse.python_sub, s)
  374. except SyntaxError as e:
  375. # Likely unmatched brackets, just don't expand the expression
  376. if e.msg != "EOL while scanning string literal":
  377. raise
  378. if s == olds:
  379. break
  380. except ExpansionError:
  381. raise
  382. except bb.parse.SkipRecipe:
  383. raise
  384. except Exception as exc:
  385. tb = sys.exc_info()[2]
  386. raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
  387. varparse.value = s
  388. return varparse
  389. def expand(self, s, varname = None):
  390. return self.expandWithRefs(s, varname).value
  391. def finalize(self, parent = False):
  392. return
  393. def internal_finalize(self, parent = False):
  394. """Performs final steps upon the datastore, including application of overrides"""
  395. self.overrides = None
  396. def need_overrides(self):
  397. if self.overrides is not None:
  398. return
  399. if self.inoverride:
  400. return
  401. for count in range(5):
  402. self.inoverride = True
  403. # Can end up here recursively so setup dummy values
  404. self.overrides = []
  405. self.overridesset = set()
  406. self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
  407. self.overridesset = set(self.overrides)
  408. self.inoverride = False
  409. self.expand_cache = {}
  410. newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
  411. if newoverrides == self.overrides:
  412. break
  413. self.overrides = newoverrides
  414. self.overridesset = set(self.overrides)
  415. else:
  416. bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
  417. def initVar(self, var):
  418. self.expand_cache = {}
  419. if not var in self.dict:
  420. self.dict[var] = {}
  421. def _findVar(self, var):
  422. dest = self.dict
  423. while dest:
  424. if var in dest:
  425. return dest[var], self.overridedata.get(var, None)
  426. if "_remote_data" in dest:
  427. connector = dest["_remote_data"]["_content"]
  428. return connector.getVar(var)
  429. if "_data" not in dest:
  430. break
  431. dest = dest["_data"]
  432. return None, self.overridedata.get(var, None)
  433. def _makeShadowCopy(self, var):
  434. if var in self.dict:
  435. return
  436. local_var, _ = self._findVar(var)
  437. if local_var:
  438. self.dict[var] = copy.copy(local_var)
  439. else:
  440. self.initVar(var)
  441. def setVar(self, var, value, **loginfo):
  442. #print("var=" + str(var) + " val=" + str(value))
  443. self.expand_cache = {}
  444. parsing=False
  445. if 'parsing' in loginfo:
  446. parsing=True
  447. if '_remote_data' in self.dict:
  448. connector = self.dict["_remote_data"]["_content"]
  449. res = connector.setVar(var, value)
  450. if not res:
  451. return
  452. if 'op' not in loginfo:
  453. loginfo['op'] = "set"
  454. match = __setvar_regexp__.match(var)
  455. if match and match.group("keyword") in __setvar_keyword__:
  456. base = match.group('base')
  457. keyword = match.group("keyword")
  458. override = match.group('add')
  459. l = self.getVarFlag(base, keyword, False) or []
  460. l.append([value, override])
  461. self.setVarFlag(base, keyword, l, ignore=True)
  462. # And cause that to be recorded:
  463. loginfo['detail'] = value
  464. loginfo['variable'] = base
  465. if override:
  466. loginfo['op'] = '%s[%s]' % (keyword, override)
  467. else:
  468. loginfo['op'] = keyword
  469. self.varhistory.record(**loginfo)
  470. # todo make sure keyword is not __doc__ or __module__
  471. # pay the cookie monster
  472. # more cookies for the cookie monster
  473. if '_' in var:
  474. self._setvar_update_overrides(base, **loginfo)
  475. if base in self.overridevars:
  476. self._setvar_update_overridevars(var, value)
  477. return
  478. if not var in self.dict:
  479. self._makeShadowCopy(var)
  480. if not parsing:
  481. if "_append" in self.dict[var]:
  482. del self.dict[var]["_append"]
  483. if "_prepend" in self.dict[var]:
  484. del self.dict[var]["_prepend"]
  485. if "_remove" in self.dict[var]:
  486. del self.dict[var]["_remove"]
  487. if var in self.overridedata:
  488. active = []
  489. self.need_overrides()
  490. for (r, o) in self.overridedata[var]:
  491. if o in self.overridesset:
  492. active.append(r)
  493. elif "_" in o:
  494. if set(o.split("_")).issubset(self.overridesset):
  495. active.append(r)
  496. for a in active:
  497. self.delVar(a)
  498. del self.overridedata[var]
  499. # more cookies for the cookie monster
  500. if '_' in var:
  501. self._setvar_update_overrides(var, **loginfo)
  502. # setting var
  503. self.dict[var]["_content"] = value
  504. self.varhistory.record(**loginfo)
  505. if var in self.overridevars:
  506. self._setvar_update_overridevars(var, value)
  507. def _setvar_update_overridevars(self, var, value):
  508. vardata = self.expandWithRefs(value, var)
  509. new = vardata.references
  510. new.update(vardata.contains.keys())
  511. while not new.issubset(self.overridevars):
  512. nextnew = set()
  513. self.overridevars.update(new)
  514. for i in new:
  515. vardata = self.expandWithRefs(self.getVar(i), i)
  516. nextnew.update(vardata.references)
  517. nextnew.update(vardata.contains.keys())
  518. new = nextnew
  519. self.internal_finalize(True)
  520. def _setvar_update_overrides(self, var, **loginfo):
  521. # aka pay the cookie monster
  522. override = var[var.rfind('_')+1:]
  523. shortvar = var[:var.rfind('_')]
  524. while override and __override_regexp__.match(override):
  525. if shortvar not in self.overridedata:
  526. self.overridedata[shortvar] = []
  527. if [var, override] not in self.overridedata[shortvar]:
  528. # Force CoW by recreating the list first
  529. self.overridedata[shortvar] = list(self.overridedata[shortvar])
  530. self.overridedata[shortvar].append([var, override])
  531. override = None
  532. if "_" in shortvar:
  533. override = var[shortvar.rfind('_')+1:]
  534. shortvar = var[:shortvar.rfind('_')]
  535. if len(shortvar) == 0:
  536. override = None
  537. def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
  538. return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
  539. def renameVar(self, key, newkey, **loginfo):
  540. """
  541. Rename the variable key to newkey
  542. """
  543. if '_remote_data' in self.dict:
  544. connector = self.dict["_remote_data"]["_content"]
  545. res = connector.renameVar(key, newkey)
  546. if not res:
  547. return
  548. val = self.getVar(key, 0, parsing=True)
  549. if val is not None:
  550. self.varhistory.rename_variable_hist(key, newkey)
  551. loginfo['variable'] = newkey
  552. loginfo['op'] = 'rename from %s' % key
  553. loginfo['detail'] = val
  554. self.varhistory.record(**loginfo)
  555. self.setVar(newkey, val, ignore=True, parsing=True)
  556. for i in (__setvar_keyword__):
  557. src = self.getVarFlag(key, i, False)
  558. if src is None:
  559. continue
  560. dest = self.getVarFlag(newkey, i, False) or []
  561. dest.extend(src)
  562. self.setVarFlag(newkey, i, dest, ignore=True)
  563. if key in self.overridedata:
  564. self.overridedata[newkey] = []
  565. for (v, o) in self.overridedata[key]:
  566. self.overridedata[newkey].append([v.replace(key, newkey), o])
  567. self.renameVar(v, v.replace(key, newkey))
  568. if '_' in newkey and val is None:
  569. self._setvar_update_overrides(newkey, **loginfo)
  570. loginfo['variable'] = key
  571. loginfo['op'] = 'rename (to)'
  572. loginfo['detail'] = newkey
  573. self.varhistory.record(**loginfo)
  574. self.delVar(key, ignore=True)
  575. def appendVar(self, var, value, **loginfo):
  576. loginfo['op'] = 'append'
  577. self.varhistory.record(**loginfo)
  578. self.setVar(var + "_append", value, ignore=True, parsing=True)
  579. def prependVar(self, var, value, **loginfo):
  580. loginfo['op'] = 'prepend'
  581. self.varhistory.record(**loginfo)
  582. self.setVar(var + "_prepend", value, ignore=True, parsing=True)
  583. def delVar(self, var, **loginfo):
  584. self.expand_cache = {}
  585. if '_remote_data' in self.dict:
  586. connector = self.dict["_remote_data"]["_content"]
  587. res = connector.delVar(var)
  588. if not res:
  589. return
  590. loginfo['detail'] = ""
  591. loginfo['op'] = 'del'
  592. self.varhistory.record(**loginfo)
  593. self.dict[var] = {}
  594. if var in self.overridedata:
  595. del self.overridedata[var]
  596. if '_' in var:
  597. override = var[var.rfind('_')+1:]
  598. shortvar = var[:var.rfind('_')]
  599. while override and override.islower():
  600. try:
  601. if shortvar in self.overridedata:
  602. # Force CoW by recreating the list first
  603. self.overridedata[shortvar] = list(self.overridedata[shortvar])
  604. self.overridedata[shortvar].remove([var, override])
  605. except ValueError as e:
  606. pass
  607. override = None
  608. if "_" in shortvar:
  609. override = var[shortvar.rfind('_')+1:]
  610. shortvar = var[:shortvar.rfind('_')]
  611. if len(shortvar) == 0:
  612. override = None
  613. def setVarFlag(self, var, flag, value, **loginfo):
  614. self.expand_cache = {}
  615. if '_remote_data' in self.dict:
  616. connector = self.dict["_remote_data"]["_content"]
  617. res = connector.setVarFlag(var, flag, value)
  618. if not res:
  619. return
  620. if 'op' not in loginfo:
  621. loginfo['op'] = "set"
  622. loginfo['flag'] = flag
  623. self.varhistory.record(**loginfo)
  624. if not var in self.dict:
  625. self._makeShadowCopy(var)
  626. self.dict[var][flag] = value
  627. if flag == "_defaultval" and '_' in var:
  628. self._setvar_update_overrides(var, **loginfo)
  629. if flag == "_defaultval" and var in self.overridevars:
  630. self._setvar_update_overridevars(var, value)
  631. if flag == "unexport" or flag == "export":
  632. if not "__exportlist" in self.dict:
  633. self._makeShadowCopy("__exportlist")
  634. if not "_content" in self.dict["__exportlist"]:
  635. self.dict["__exportlist"]["_content"] = set()
  636. self.dict["__exportlist"]["_content"].add(var)
  637. def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
  638. if flag == "_content":
  639. cachename = var
  640. else:
  641. if not flag:
  642. bb.warn("Calling getVarFlag with flag unset is invalid")
  643. return None
  644. cachename = var + "[" + flag + "]"
  645. if expand and cachename in self.expand_cache:
  646. return self.expand_cache[cachename].value
  647. local_var, overridedata = self._findVar(var)
  648. value = None
  649. removes = set()
  650. if flag == "_content" and overridedata is not None and not parsing:
  651. match = False
  652. active = {}
  653. self.need_overrides()
  654. for (r, o) in overridedata:
  655. # What about double overrides both with "_" in the name?
  656. if o in self.overridesset:
  657. active[o] = r
  658. elif "_" in o:
  659. if set(o.split("_")).issubset(self.overridesset):
  660. active[o] = r
  661. mod = True
  662. while mod:
  663. mod = False
  664. for o in self.overrides:
  665. for a in active.copy():
  666. if a.endswith("_" + o):
  667. t = active[a]
  668. del active[a]
  669. active[a.replace("_" + o, "")] = t
  670. mod = True
  671. elif a == o:
  672. match = active[a]
  673. del active[a]
  674. if match:
  675. value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
  676. if hasattr(subparser, "removes"):
  677. # We have to carry the removes from the overridden variable to apply at the
  678. # end of processing
  679. removes = subparser.removes
  680. if local_var is not None and value is None:
  681. if flag in local_var:
  682. value = copy.copy(local_var[flag])
  683. elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
  684. value = copy.copy(local_var["_defaultval"])
  685. if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
  686. if not value:
  687. value = ""
  688. self.need_overrides()
  689. for (r, o) in local_var["_append"]:
  690. match = True
  691. if o:
  692. for o2 in o.split("_"):
  693. if not o2 in self.overrides:
  694. match = False
  695. if match:
  696. value = value + r
  697. if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
  698. if not value:
  699. value = ""
  700. self.need_overrides()
  701. for (r, o) in local_var["_prepend"]:
  702. match = True
  703. if o:
  704. for o2 in o.split("_"):
  705. if not o2 in self.overrides:
  706. match = False
  707. if match:
  708. value = r + value
  709. parser = None
  710. if expand or retparser:
  711. parser = self.expandWithRefs(value, cachename)
  712. if expand:
  713. value = parser.value
  714. if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
  715. self.need_overrides()
  716. for (r, o) in local_var["_remove"]:
  717. match = True
  718. if o:
  719. for o2 in o.split("_"):
  720. if not o2 in self.overrides:
  721. match = False
  722. if match:
  723. removes.add(r)
  724. if value and flag == "_content" and not parsing:
  725. if removes and parser:
  726. expanded_removes = {}
  727. for r in removes:
  728. expanded_removes[r] = self.expand(r).split()
  729. parser.removes = set()
  730. val = ""
  731. for v in __whitespace_split__.split(parser.value):
  732. skip = False
  733. for r in removes:
  734. if v in expanded_removes[r]:
  735. parser.removes.add(r)
  736. skip = True
  737. if skip:
  738. continue
  739. val = val + v
  740. parser.value = val
  741. if expand:
  742. value = parser.value
  743. if parser:
  744. self.expand_cache[cachename] = parser
  745. if retparser:
  746. return value, parser
  747. return value
  748. def delVarFlag(self, var, flag, **loginfo):
  749. self.expand_cache = {}
  750. if '_remote_data' in self.dict:
  751. connector = self.dict["_remote_data"]["_content"]
  752. res = connector.delVarFlag(var, flag)
  753. if not res:
  754. return
  755. local_var, _ = self._findVar(var)
  756. if not local_var:
  757. return
  758. if not var in self.dict:
  759. self._makeShadowCopy(var)
  760. if var in self.dict and flag in self.dict[var]:
  761. loginfo['detail'] = ""
  762. loginfo['op'] = 'delFlag'
  763. loginfo['flag'] = flag
  764. self.varhistory.record(**loginfo)
  765. del self.dict[var][flag]
  766. def appendVarFlag(self, var, flag, value, **loginfo):
  767. loginfo['op'] = 'append'
  768. loginfo['flag'] = flag
  769. self.varhistory.record(**loginfo)
  770. newvalue = (self.getVarFlag(var, flag, False) or "") + value
  771. self.setVarFlag(var, flag, newvalue, ignore=True)
  772. def prependVarFlag(self, var, flag, value, **loginfo):
  773. loginfo['op'] = 'prepend'
  774. loginfo['flag'] = flag
  775. self.varhistory.record(**loginfo)
  776. newvalue = value + (self.getVarFlag(var, flag, False) or "")
  777. self.setVarFlag(var, flag, newvalue, ignore=True)
  778. def setVarFlags(self, var, flags, **loginfo):
  779. self.expand_cache = {}
  780. infer_caller_details(loginfo)
  781. if not var in self.dict:
  782. self._makeShadowCopy(var)
  783. for i in flags:
  784. if i == "_content":
  785. continue
  786. loginfo['flag'] = i
  787. loginfo['detail'] = flags[i]
  788. self.varhistory.record(**loginfo)
  789. self.dict[var][i] = flags[i]
  790. def getVarFlags(self, var, expand = False, internalflags=False):
  791. local_var, _ = self._findVar(var)
  792. flags = {}
  793. if local_var:
  794. for i in local_var:
  795. if i.startswith("_") and not internalflags:
  796. continue
  797. flags[i] = local_var[i]
  798. if expand and i in expand:
  799. flags[i] = self.expand(flags[i], var + "[" + i + "]")
  800. if len(flags) == 0:
  801. return None
  802. return flags
  803. def delVarFlags(self, var, **loginfo):
  804. self.expand_cache = {}
  805. if not var in self.dict:
  806. self._makeShadowCopy(var)
  807. if var in self.dict:
  808. content = None
  809. loginfo['op'] = 'delete flags'
  810. self.varhistory.record(**loginfo)
  811. # try to save the content
  812. if "_content" in self.dict[var]:
  813. content = self.dict[var]["_content"]
  814. self.dict[var] = {}
  815. self.dict[var]["_content"] = content
  816. else:
  817. del self.dict[var]
  818. def createCopy(self):
  819. """
  820. Create a copy of self by setting _data to self
  821. """
  822. # we really want this to be a DataSmart...
  823. data = DataSmart()
  824. data.dict["_data"] = self.dict
  825. data.varhistory = self.varhistory.copy()
  826. data.varhistory.dataroot = data
  827. data.inchistory = self.inchistory.copy()
  828. data._tracking = self._tracking
  829. data.overrides = None
  830. data.overridevars = copy.copy(self.overridevars)
  831. # Should really be a deepcopy but has heavy overhead.
  832. # Instead, we're careful with writes.
  833. data.overridedata = copy.copy(self.overridedata)
  834. return data
  835. def expandVarref(self, variable, parents=False):
  836. """Find all references to variable in the data and expand it
  837. in place, optionally descending to parent datastores."""
  838. if parents:
  839. keys = iter(self)
  840. else:
  841. keys = self.localkeys()
  842. ref = '${%s}' % variable
  843. value = self.getVar(variable, False)
  844. for key in keys:
  845. referrervalue = self.getVar(key, False)
  846. if referrervalue and ref in referrervalue:
  847. self.setVar(key, referrervalue.replace(ref, value))
  848. def localkeys(self):
  849. for key in self.dict:
  850. if key not in ['_data', '_remote_data']:
  851. yield key
  852. def __iter__(self):
  853. deleted = set()
  854. overrides = set()
  855. def keylist(d):
  856. klist = set()
  857. for key in d:
  858. if key in ["_data", "_remote_data"]:
  859. continue
  860. if key in deleted:
  861. continue
  862. if key in overrides:
  863. continue
  864. if not d[key]:
  865. deleted.add(key)
  866. continue
  867. klist.add(key)
  868. if "_data" in d:
  869. klist |= keylist(d["_data"])
  870. if "_remote_data" in d:
  871. connector = d["_remote_data"]["_content"]
  872. for key in connector.getKeys():
  873. if key in deleted:
  874. continue
  875. klist.add(key)
  876. return klist
  877. self.need_overrides()
  878. for var in self.overridedata:
  879. for (r, o) in self.overridedata[var]:
  880. if o in self.overridesset:
  881. overrides.add(var)
  882. elif "_" in o:
  883. if set(o.split("_")).issubset(self.overridesset):
  884. overrides.add(var)
  885. for k in keylist(self.dict):
  886. yield k
  887. for k in overrides:
  888. yield k
  889. def __len__(self):
  890. return len(frozenset(iter(self)))
  891. def __getitem__(self, item):
  892. value = self.getVar(item, False)
  893. if value is None:
  894. raise KeyError(item)
  895. else:
  896. return value
  897. def __setitem__(self, var, value):
  898. self.setVar(var, value)
  899. def __delitem__(self, var):
  900. self.delVar(var)
  901. def get_hash(self):
  902. data = {}
  903. d = self.createCopy()
  904. bb.data.expandKeys(d)
  905. config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
  906. keys = set(key for key in iter(d) if not key.startswith("__"))
  907. for key in keys:
  908. if key in config_whitelist:
  909. continue
  910. value = d.getVar(key, False) or ""
  911. data.update({key:value})
  912. varflags = d.getVarFlags(key, internalflags = True)
  913. if not varflags:
  914. continue
  915. for f in varflags:
  916. if f == "_content":
  917. continue
  918. data.update({'%s[%s]' % (key, f):varflags[f]})
  919. for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
  920. bb_list = d.getVar(key, False) or []
  921. data.update({key:str(bb_list)})
  922. if key == "__BBANONFUNCS":
  923. for i in bb_list:
  924. value = d.getVar(i, False) or ""
  925. data.update({i:value})
  926. data_str = str([(k, data[k]) for k in sorted(data.keys())])
  927. return hashlib.sha256(data_str.encode("utf-8")).hexdigest()