data.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. #!/usr/bin/env python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. """
  5. BitBake 'Data' implementations
  6. Functions for interacting with the data structure used by the
  7. BitBake build tools.
  8. The expandData and update_data are the most expensive
  9. operations. At night the cookie monster came by and
  10. suggested 'give me cookies on setting the variables and
  11. things will work out'. Taking this suggestion into account
  12. applying the skills from the not yet passed 'Entwurf und
  13. Analyse von Algorithmen' lecture and the cookie
  14. monster seems to be right. We will track setVar more carefully
  15. to have faster update_data and expandKeys operations.
  16. This is a treade-off between speed and memory again but
  17. the speed is more critical here.
  18. """
  19. # Copyright (C) 2003, 2004 Chris Larson
  20. # Copyright (C) 2005 Holger Hans Peter Freyther
  21. #
  22. # This program is free software; you can redistribute it and/or modify
  23. # it under the terms of the GNU General Public License version 2 as
  24. # published by the Free Software Foundation.
  25. #
  26. # This program is distributed in the hope that it will be useful,
  27. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  28. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  29. # GNU General Public License for more details.
  30. #
  31. # You should have received a copy of the GNU General Public License along
  32. # with this program; if not, write to the Free Software Foundation, Inc.,
  33. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  34. #
  35. #Based on functions from the base bb module, Copyright 2003 Holger Schurig
  36. import sys, os, re, time, types
  37. if sys.argv[0][-5:] == "pydoc":
  38. path = os.path.dirname(os.path.dirname(sys.argv[1]))
  39. else:
  40. path = os.path.dirname(os.path.dirname(sys.argv[0]))
  41. sys.path.insert(0,path)
  42. from bb import data_smart
  43. import bb
  44. _dict_type = data_smart.DataSmart
  45. def init():
  46. return _dict_type()
  47. def init_db(parent = None):
  48. if parent:
  49. return parent.createCopy()
  50. else:
  51. return _dict_type()
  52. def createCopy(source):
  53. """Link the source set to the destination
  54. If one does not find the value in the destination set,
  55. search will go on to the source set to get the value.
  56. Value from source are copy-on-write. i.e. any try to
  57. modify one of them will end up putting the modified value
  58. in the destination set.
  59. """
  60. return source.createCopy()
  61. def initVar(var, d):
  62. """Non-destructive var init for data structure"""
  63. d.initVar(var)
  64. def setVar(var, value, d):
  65. """Set a variable to a given value
  66. Example:
  67. >>> d = init()
  68. >>> setVar('TEST', 'testcontents', d)
  69. >>> print getVar('TEST', d)
  70. testcontents
  71. """
  72. d.setVar(var,value)
  73. def getVar(var, d, exp = 0):
  74. """Gets the value of a variable
  75. Example:
  76. >>> d = init()
  77. >>> setVar('TEST', 'testcontents', d)
  78. >>> print getVar('TEST', d)
  79. testcontents
  80. """
  81. return d.getVar(var,exp)
  82. def delVar(var, d):
  83. """Removes a variable from the data set
  84. Example:
  85. >>> d = init()
  86. >>> setVar('TEST', 'testcontents', d)
  87. >>> print getVar('TEST', d)
  88. testcontents
  89. >>> delVar('TEST', d)
  90. >>> print getVar('TEST', d)
  91. None
  92. """
  93. d.delVar(var)
  94. def setVarFlag(var, flag, flagvalue, d):
  95. """Set a flag for a given variable to a given value
  96. Example:
  97. >>> d = init()
  98. >>> setVarFlag('TEST', 'python', 1, d)
  99. >>> print getVarFlag('TEST', 'python', d)
  100. 1
  101. """
  102. d.setVarFlag(var,flag,flagvalue)
  103. def getVarFlag(var, flag, d):
  104. """Gets given flag from given var
  105. Example:
  106. >>> d = init()
  107. >>> setVarFlag('TEST', 'python', 1, d)
  108. >>> print getVarFlag('TEST', 'python', d)
  109. 1
  110. """
  111. return d.getVarFlag(var,flag)
  112. def delVarFlag(var, flag, d):
  113. """Removes a given flag from the variable's flags
  114. Example:
  115. >>> d = init()
  116. >>> setVarFlag('TEST', 'testflag', 1, d)
  117. >>> print getVarFlag('TEST', 'testflag', d)
  118. 1
  119. >>> delVarFlag('TEST', 'testflag', d)
  120. >>> print getVarFlag('TEST', 'testflag', d)
  121. None
  122. """
  123. d.delVarFlag(var,flag)
  124. def setVarFlags(var, flags, d):
  125. """Set the flags for a given variable
  126. Note:
  127. setVarFlags will not clear previous
  128. flags. Think of this method as
  129. addVarFlags
  130. Example:
  131. >>> d = init()
  132. >>> myflags = {}
  133. >>> myflags['test'] = 'blah'
  134. >>> setVarFlags('TEST', myflags, d)
  135. >>> print getVarFlag('TEST', 'test', d)
  136. blah
  137. """
  138. d.setVarFlags(var,flags)
  139. def getVarFlags(var, d):
  140. """Gets a variable's flags
  141. Example:
  142. >>> d = init()
  143. >>> setVarFlag('TEST', 'test', 'blah', d)
  144. >>> print getVarFlags('TEST', d)['test']
  145. blah
  146. """
  147. return d.getVarFlags(var)
  148. def delVarFlags(var, d):
  149. """Removes a variable's flags
  150. Example:
  151. >>> data = init()
  152. >>> setVarFlag('TEST', 'testflag', 1, data)
  153. >>> print getVarFlag('TEST', 'testflag', data)
  154. 1
  155. >>> delVarFlags('TEST', data)
  156. >>> print getVarFlags('TEST', data)
  157. None
  158. """
  159. d.delVarFlags(var)
  160. def keys(d):
  161. """Return a list of keys in d
  162. Example:
  163. >>> d = init()
  164. >>> setVar('TEST', 1, d)
  165. >>> setVar('MOO' , 2, d)
  166. >>> setVarFlag('TEST', 'test', 1, d)
  167. >>> keys(d)
  168. ['TEST', 'MOO']
  169. """
  170. return d.keys()
  171. def getData(d):
  172. """Returns the data object used"""
  173. return d
  174. def setData(newData, d):
  175. """Sets the data object to the supplied value"""
  176. d = newData
  177. ##
  178. ## Cookie Monsters' query functions
  179. ##
  180. def _get_override_vars(d, override):
  181. """
  182. Internal!!!
  183. Get the Names of Variables that have a specific
  184. override. This function returns a iterable
  185. Set or an empty list
  186. """
  187. return []
  188. def _get_var_flags_triple(d):
  189. """
  190. Internal!!!
  191. """
  192. return []
  193. __expand_var_regexp__ = re.compile(r"\${[^{}]+}")
  194. __expand_python_regexp__ = re.compile(r"\${@.+?}")
  195. def expand(s, d, varname = None):
  196. """Variable expansion using the data store.
  197. Example:
  198. Standard expansion:
  199. >>> d = init()
  200. >>> setVar('A', 'sshd', d)
  201. >>> print expand('/usr/bin/${A}', d)
  202. /usr/bin/sshd
  203. Python expansion:
  204. >>> d = init()
  205. >>> print expand('result: ${@37 * 72}', d)
  206. result: 2664
  207. Shell expansion:
  208. >>> d = init()
  209. >>> print expand('${TARGET_MOO}', d)
  210. ${TARGET_MOO}
  211. >>> setVar('TARGET_MOO', 'yupp', d)
  212. >>> print expand('${TARGET_MOO}',d)
  213. yupp
  214. >>> setVar('SRC_URI', 'http://somebug.${TARGET_MOO}', d)
  215. >>> delVar('TARGET_MOO', d)
  216. >>> print expand('${SRC_URI}', d)
  217. http://somebug.${TARGET_MOO}
  218. """
  219. return d.expand(s, varname)
  220. def expandKeys(alterdata, readdata = None):
  221. if readdata == None:
  222. readdata = alterdata
  223. for key in keys(alterdata):
  224. if not '${' in key:
  225. continue
  226. ekey = expand(key, readdata)
  227. if key == ekey:
  228. continue
  229. val = getVar(key, alterdata)
  230. if val is None:
  231. continue
  232. # import copy
  233. # setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata)
  234. setVar(ekey, val, alterdata)
  235. for i in ('_append', '_prepend'):
  236. dest = getVarFlag(ekey, i, alterdata) or []
  237. src = getVarFlag(key, i, readdata) or []
  238. dest.extend(src)
  239. setVarFlag(ekey, i, dest, alterdata)
  240. delVar(key, alterdata)
  241. def expandData(alterdata, readdata = None):
  242. """For each variable in alterdata, expand it, and update the var contents.
  243. Replacements use data from readdata.
  244. Example:
  245. >>> a=init()
  246. >>> b=init()
  247. >>> setVar("dlmsg", "dl_dir is ${DL_DIR}", a)
  248. >>> setVar("DL_DIR", "/path/to/whatever", b)
  249. >>> expandData(a, b)
  250. >>> print getVar("dlmsg", a)
  251. dl_dir is /path/to/whatever
  252. """
  253. if readdata == None:
  254. readdata = alterdata
  255. for key in keys(alterdata):
  256. val = getVar(key, alterdata)
  257. if type(val) is not types.StringType:
  258. continue
  259. expanded = expand(val, readdata)
  260. # print "key is %s, val is %s, expanded is %s" % (key, val, expanded)
  261. if val != expanded:
  262. setVar(key, expanded, alterdata)
  263. import os
  264. def inheritFromOS(d):
  265. """Inherit variables from the environment."""
  266. # fakeroot needs to be able to set these
  267. non_inherit_vars = [ "LD_LIBRARY_PATH", "LD_PRELOAD" ]
  268. for s in os.environ.keys():
  269. if not s in non_inherit_vars:
  270. try:
  271. setVar(s, os.environ[s], d)
  272. setVarFlag(s, 'matchesenv', '1', d)
  273. except TypeError:
  274. pass
  275. import sys
  276. def emit_var(var, o=sys.__stdout__, d = init(), all=False):
  277. """Emit a variable to be sourced by a shell."""
  278. if getVarFlag(var, "python", d):
  279. return 0
  280. try:
  281. if all:
  282. oval = getVar(var, d, 0)
  283. val = getVar(var, d, 1)
  284. except KeyboardInterrupt:
  285. raise
  286. except:
  287. excname = str(sys.exc_info()[0])
  288. if excname == "bb.build.FuncFailed":
  289. raise
  290. o.write('# expansion of %s threw %s\n' % (var, excname))
  291. return 0
  292. if all:
  293. o.write('# %s=%s\n' % (var, oval))
  294. if type(val) is not types.StringType:
  295. return 0
  296. if getVarFlag(var, 'matchesenv', d):
  297. return 0
  298. if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
  299. return 0
  300. val.rstrip()
  301. if not val:
  302. return 0
  303. varExpanded = expand(var, d)
  304. if getVarFlag(var, "func", d):
  305. # NOTE: should probably check for unbalanced {} within the var
  306. o.write("%s() {\n%s\n}\n" % (varExpanded, val))
  307. else:
  308. if getVarFlag(var, "unexport", d):
  309. o.write('unset %s\n' % varExpanded)
  310. return 1
  311. if getVarFlag(var, "export", d):
  312. o.write('export ')
  313. else:
  314. if not all:
  315. return 0
  316. # if we're going to output this within doublequotes,
  317. # to a shell, we need to escape the quotes in the var
  318. alter = re.sub('"', '\\"', val.strip())
  319. o.write('%s="%s"\n' % (varExpanded, alter))
  320. return 1
  321. def emit_env(o=sys.__stdout__, d = init(), all=False):
  322. """Emits all items in the data store in a format such that it can be sourced by a shell."""
  323. env = keys(d)
  324. for e in env:
  325. if getVarFlag(e, "func", d):
  326. continue
  327. emit_var(e, o, d, all) and o.write('\n')
  328. for e in env:
  329. if not getVarFlag(e, "func", d):
  330. continue
  331. emit_var(e, o, d) and o.write('\n')
  332. def update_data(d):
  333. """Modifies the environment vars according to local overrides and commands.
  334. Examples:
  335. Appending to a variable:
  336. >>> d = init()
  337. >>> setVar('TEST', 'this is a', d)
  338. >>> setVar('TEST_append', ' test', d)
  339. >>> setVar('TEST_append', ' of the emergency broadcast system.', d)
  340. >>> update_data(d)
  341. >>> print getVar('TEST', d)
  342. this is a test of the emergency broadcast system.
  343. Prepending to a variable:
  344. >>> setVar('TEST', 'virtual/libc', d)
  345. >>> setVar('TEST_prepend', 'virtual/tmake ', d)
  346. >>> setVar('TEST_prepend', 'virtual/patcher ', d)
  347. >>> update_data(d)
  348. >>> print getVar('TEST', d)
  349. virtual/patcher virtual/tmake virtual/libc
  350. Overrides:
  351. >>> setVar('TEST_arm', 'target', d)
  352. >>> setVar('TEST_ramses', 'machine', d)
  353. >>> setVar('TEST_local', 'local', d)
  354. >>> setVar('OVERRIDES', 'arm', d)
  355. >>> setVar('TEST', 'original', d)
  356. >>> update_data(d)
  357. >>> print getVar('TEST', d)
  358. target
  359. >>> setVar('OVERRIDES', 'arm:ramses:local', d)
  360. >>> setVar('TEST', 'original', d)
  361. >>> update_data(d)
  362. >>> print getVar('TEST', d)
  363. local
  364. CopyMonster:
  365. >>> e = d.createCopy()
  366. >>> setVar('TEST_foo', 'foo', e)
  367. >>> update_data(e)
  368. >>> print getVar('TEST', e)
  369. local
  370. >>> setVar('OVERRIDES', 'arm:ramses:local:foo', e)
  371. >>> update_data(e)
  372. >>> print getVar('TEST', e)
  373. foo
  374. >>> f = d.createCopy()
  375. >>> setVar('TEST_moo', 'something', f)
  376. >>> setVar('OVERRIDES', 'moo:arm:ramses:local:foo', e)
  377. >>> update_data(e)
  378. >>> print getVar('TEST', e)
  379. foo
  380. >>> h = init()
  381. >>> setVar('SRC_URI', 'file://append.foo;patch=1 ', h)
  382. >>> g = h.createCopy()
  383. >>> setVar('SRC_URI_append_arm', 'file://other.foo;patch=1', g)
  384. >>> setVar('OVERRIDES', 'arm:moo', g)
  385. >>> update_data(g)
  386. >>> print getVar('SRC_URI', g)
  387. file://append.foo;patch=1 file://other.foo;patch=1
  388. """
  389. bb.msg.debug(2, bb.msg.domain.Data, "update_data()")
  390. # now ask the cookie monster for help
  391. #print "Cookie Monster"
  392. #print "Append/Prepend %s" % d._special_values
  393. #print "Overrides %s" % d._seen_overrides
  394. overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or []
  395. #
  396. # Well let us see what breaks here. We used to iterate
  397. # over each variable and apply the override and then
  398. # do the line expanding.
  399. # If we have bad luck - which we will have - the keys
  400. # where in some order that is so important for this
  401. # method which we don't have anymore.
  402. # Anyway we will fix that and write test cases this
  403. # time.
  404. #
  405. # First we apply all overrides
  406. # Then we will handle _append and _prepend
  407. #
  408. for o in overrides:
  409. # calculate '_'+override
  410. l = len(o)+1
  411. # see if one should even try
  412. if not d._seen_overrides.has_key(o):
  413. continue
  414. vars = d._seen_overrides[o]
  415. for var in vars:
  416. name = var[:-l]
  417. try:
  418. d[name] = d[var]
  419. except:
  420. bb.msg.note(1, bb.msg.domain.Data, "Untracked delVar")
  421. # now on to the appends and prepends
  422. if d._special_values.has_key('_append'):
  423. appends = d._special_values['_append'] or []
  424. for append in appends:
  425. for (a, o) in getVarFlag(append, '_append', d) or []:
  426. # maybe the OVERRIDE was not yet added so keep the append
  427. if (o and o in overrides) or not o:
  428. delVarFlag(append, '_append', d)
  429. if o and not o in overrides:
  430. continue
  431. sval = getVar(append,d) or ""
  432. sval+=a
  433. setVar(append, sval, d)
  434. if d._special_values.has_key('_prepend'):
  435. prepends = d._special_values['_prepend'] or []
  436. for prepend in prepends:
  437. for (a, o) in getVarFlag(prepend, '_prepend', d) or []:
  438. # maybe the OVERRIDE was not yet added so keep the prepend
  439. if (o and o in overrides) or not o:
  440. delVarFlag(prepend, '_prepend', d)
  441. if o and not o in overrides:
  442. continue
  443. sval = a + (getVar(prepend,d) or "")
  444. setVar(prepend, sval, d)
  445. def inherits_class(klass, d):
  446. val = getVar('__inherit_cache', d) or []
  447. if os.path.join('classes', '%s.bbclass' % klass) in val:
  448. return True
  449. return False
  450. def _test():
  451. """Start a doctest run on this module"""
  452. import doctest
  453. from bb import data
  454. doctest.testmod(data)
  455. if __name__ == "__main__":
  456. _test()