codeparser.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. #
  4. # BitBake Test for codeparser.py
  5. #
  6. # Copyright (C) 2010 Chris Larson
  7. # Copyright (C) 2012 Richard Purdie
  8. #
  9. # SPDX-License-Identifier: GPL-2.0-only
  10. #
  11. # This program is free software; you can redistribute it and/or modify
  12. # it under the terms of the GNU General Public License version 2 as
  13. # published by the Free Software Foundation.
  14. #
  15. # This program is distributed in the hope that it will be useful,
  16. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. # GNU General Public License for more details.
  19. #
  20. # You should have received a copy of the GNU General Public License along
  21. # with this program; if not, write to the Free Software Foundation, Inc.,
  22. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  23. #
  24. import unittest
  25. import logging
  26. import bb
  27. logger = logging.getLogger('BitBake.TestCodeParser')
  28. # bb.data references bb.parse but can't directly import due to circular dependencies.
  29. # Hack around it for now :(
  30. import bb.parse
  31. import bb.data
  32. class ReferenceTest(unittest.TestCase):
  33. def setUp(self):
  34. self.d = bb.data.init()
  35. def setEmptyVars(self, varlist):
  36. for k in varlist:
  37. self.d.setVar(k, "")
  38. def setValues(self, values):
  39. for k, v in values.items():
  40. self.d.setVar(k, v)
  41. def assertReferences(self, refs):
  42. self.assertEqual(self.references, refs)
  43. def assertExecs(self, execs):
  44. self.assertEqual(self.execs, execs)
  45. def assertContains(self, contains):
  46. self.assertEqual(self.contains, contains)
  47. class VariableReferenceTest(ReferenceTest):
  48. def parseExpression(self, exp):
  49. parsedvar = self.d.expandWithRefs(exp, None)
  50. self.references = parsedvar.references
  51. def test_simple_reference(self):
  52. self.setEmptyVars(["FOO"])
  53. self.parseExpression("${FOO}")
  54. self.assertReferences(set(["FOO"]))
  55. def test_nested_reference(self):
  56. self.setEmptyVars(["BAR"])
  57. self.d.setVar("FOO", "BAR")
  58. self.parseExpression("${${FOO}}")
  59. self.assertReferences(set(["FOO", "BAR"]))
  60. def test_python_reference(self):
  61. self.setEmptyVars(["BAR"])
  62. self.parseExpression("${@d.getVar('BAR') + 'foo'}")
  63. self.assertReferences(set(["BAR"]))
  64. class ShellReferenceTest(ReferenceTest):
  65. def parseExpression(self, exp):
  66. parsedvar = self.d.expandWithRefs(exp, None)
  67. parser = bb.codeparser.ShellParser("ParserTest", logger)
  68. parser.parse_shell(parsedvar.value)
  69. self.references = parsedvar.references
  70. self.execs = parser.execs
  71. def test_quotes_inside_assign(self):
  72. self.parseExpression('foo=foo"bar"baz')
  73. self.assertReferences(set([]))
  74. def test_quotes_inside_arg(self):
  75. self.parseExpression('sed s#"bar baz"#"alpha beta"#g')
  76. self.assertExecs(set(["sed"]))
  77. def test_arg_continuation(self):
  78. self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc")
  79. self.assertExecs(set(["sed"]))
  80. def test_dollar_in_quoted(self):
  81. self.parseExpression('sed -i -e "foo$" *.pc')
  82. self.assertExecs(set(["sed"]))
  83. def test_quotes_inside_arg_continuation(self):
  84. self.setEmptyVars(["bindir", "D", "libdir"])
  85. self.parseExpression("""
  86. sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\
  87. -e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\
  88. ${D}${libdir}/pkgconfig/*.pc
  89. """)
  90. self.assertReferences(set(["bindir", "D", "libdir"]))
  91. def test_assign_subshell_expansion(self):
  92. self.parseExpression("foo=$(echo bar)")
  93. self.assertExecs(set(["echo"]))
  94. def test_shell_unexpanded(self):
  95. self.setEmptyVars(["QT_BASE_NAME"])
  96. self.parseExpression('echo "${QT_BASE_NAME}"')
  97. self.assertExecs(set(["echo"]))
  98. self.assertReferences(set(["QT_BASE_NAME"]))
  99. def test_incomplete_varexp_single_quotes(self):
  100. self.parseExpression("sed -i -e 's:IP{:I${:g' $pc")
  101. self.assertExecs(set(["sed"]))
  102. def test_parameter_expansion_modifiers(self):
  103. # - and + are also valid modifiers for parameter expansion, but are
  104. # valid characters in bitbake variable names, so are not included here
  105. for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
  106. name = "foo%sbar" % i
  107. self.parseExpression("${%s}" % name)
  108. self.assertNotIn(name, self.references)
  109. def test_until(self):
  110. self.parseExpression("until false; do echo true; done")
  111. self.assertExecs(set(["false", "echo"]))
  112. self.assertReferences(set())
  113. def test_case(self):
  114. self.parseExpression("""
  115. case $foo in
  116. *)
  117. bar
  118. ;;
  119. esac
  120. """)
  121. self.assertExecs(set(["bar"]))
  122. self.assertReferences(set())
  123. def test_assign_exec(self):
  124. self.parseExpression("a=b c='foo bar' alpha 1 2 3")
  125. self.assertExecs(set(["alpha"]))
  126. def test_redirect_to_file(self):
  127. self.setEmptyVars(["foo"])
  128. self.parseExpression("echo foo >${foo}/bar")
  129. self.assertExecs(set(["echo"]))
  130. self.assertReferences(set(["foo"]))
  131. def test_heredoc(self):
  132. self.setEmptyVars(["theta"])
  133. self.parseExpression("""
  134. cat <<END
  135. alpha
  136. beta
  137. ${theta}
  138. END
  139. """)
  140. self.assertReferences(set(["theta"]))
  141. def test_redirect_from_heredoc(self):
  142. v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"]
  143. self.setEmptyVars(v)
  144. self.parseExpression("""
  145. cat <<END >${B}/cachedpaths
  146. shadow_cv_maildir=${SHADOW_MAILDIR}
  147. shadow_cv_mailfile=${SHADOW_MAILFILE}
  148. shadow_cv_utmpdir=${SHADOW_UTMPDIR}
  149. shadow_cv_logdir=${SHADOW_LOGDIR}
  150. shadow_cv_passwd_dir=${bindir}
  151. END
  152. """)
  153. self.assertReferences(set(v))
  154. self.assertExecs(set(["cat"]))
  155. # def test_incomplete_command_expansion(self):
  156. # self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs,
  157. # bbvalue.shparse("cp foo`", self.d), self.d)
  158. # def test_rogue_dollarsign(self):
  159. # self.setValues({"D" : "/tmp"})
  160. # self.parseExpression("install -d ${D}$")
  161. # self.assertReferences(set(["D"]))
  162. # self.assertExecs(set(["install"]))
  163. class PythonReferenceTest(ReferenceTest):
  164. def setUp(self):
  165. self.d = bb.data.init()
  166. if hasattr(bb.utils, "_context"):
  167. self.context = bb.utils._context
  168. else:
  169. import builtins
  170. self.context = builtins.__dict__
  171. def parseExpression(self, exp):
  172. parsedvar = self.d.expandWithRefs(exp, None)
  173. parser = bb.codeparser.PythonParser("ParserTest", logger)
  174. parser.parse_python(parsedvar.value)
  175. self.references = parsedvar.references | parser.references
  176. self.execs = parser.execs
  177. self.contains = parser.contains
  178. @staticmethod
  179. def indent(value):
  180. """Python Snippets have to be indented, python values don't have to
  181. be. These unit tests are testing snippets."""
  182. return " " + value
  183. def test_getvar_reference(self):
  184. self.parseExpression("d.getVar('foo')")
  185. self.assertReferences(set(["foo"]))
  186. self.assertExecs(set())
  187. def test_getvar_computed_reference(self):
  188. self.parseExpression("d.getVar('f' + 'o' + 'o')")
  189. self.assertReferences(set())
  190. self.assertExecs(set())
  191. def test_getvar_exec_reference(self):
  192. self.parseExpression("eval('d.getVar(\"foo\")')")
  193. self.assertReferences(set())
  194. self.assertExecs(set(["eval"]))
  195. def test_var_reference(self):
  196. self.context["foo"] = lambda x: x
  197. self.setEmptyVars(["FOO"])
  198. self.parseExpression("foo('${FOO}')")
  199. self.assertReferences(set(["FOO"]))
  200. self.assertExecs(set(["foo"]))
  201. del self.context["foo"]
  202. def test_var_exec(self):
  203. for etype in ("func", "task"):
  204. self.d.setVar("do_something", "echo 'hi mom! ${FOO}'")
  205. self.d.setVarFlag("do_something", etype, True)
  206. self.parseExpression("bb.build.exec_func('do_something', d)")
  207. self.assertReferences(set([]))
  208. self.assertExecs(set(["do_something"]))
  209. def test_function_reference(self):
  210. self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg)
  211. self.d.setVar("FOO", "Hello, World!")
  212. self.parseExpression("testfunc('${FOO}')")
  213. self.assertReferences(set(["FOO"]))
  214. self.assertExecs(set(["testfunc"]))
  215. del self.context["testfunc"]
  216. def test_qualified_function_reference(self):
  217. self.parseExpression("time.time()")
  218. self.assertExecs(set(["time.time"]))
  219. def test_qualified_function_reference_2(self):
  220. self.parseExpression("os.path.dirname('/foo/bar')")
  221. self.assertExecs(set(["os.path.dirname"]))
  222. def test_qualified_function_reference_nested(self):
  223. self.parseExpression("time.strftime('%Y%m%d',time.gmtime())")
  224. self.assertExecs(set(["time.strftime", "time.gmtime"]))
  225. def test_function_reference_chained(self):
  226. self.context["testget"] = lambda: "\tstrip me "
  227. self.parseExpression("testget().strip()")
  228. self.assertExecs(set(["testget"]))
  229. del self.context["testget"]
  230. def test_contains(self):
  231. self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
  232. self.assertContains({'TESTVAR': {'one'}})
  233. def test_contains_multi(self):
  234. self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
  235. self.assertContains({'TESTVAR': {'one two'}})
  236. def test_contains_any(self):
  237. self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
  238. self.assertContains({'TESTVAR': {'hello'}})
  239. def test_contains_any_multi(self):
  240. self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
  241. self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
  242. def test_contains_filter(self):
  243. self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
  244. self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
  245. class DependencyReferenceTest(ReferenceTest):
  246. pydata = """
  247. d.getVar('somevar')
  248. def test(d):
  249. foo = 'bar %s' % 'foo'
  250. def test2(d):
  251. d.getVar(foo)
  252. d.getVar('bar', False)
  253. test2(d)
  254. def a():
  255. \"\"\"some
  256. stuff
  257. \"\"\"
  258. return "heh"
  259. test(d)
  260. d.expand(d.getVar("something", False))
  261. d.expand("${inexpand} somethingelse")
  262. d.getVar(a(), False)
  263. """
  264. def test_python(self):
  265. self.d.setVar("FOO", self.pydata)
  266. self.setEmptyVars(["inexpand", "a", "test2", "test"])
  267. self.d.setVarFlags("FOO", {
  268. "func": True,
  269. "python": True,
  270. "lineno": 1,
  271. "filename": "example.bb",
  272. })
  273. deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
  274. self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
  275. shelldata = """
  276. foo () {
  277. bar
  278. }
  279. {
  280. echo baz
  281. $(heh)
  282. eval `moo`
  283. }
  284. a=b
  285. c=d
  286. (
  287. true && false
  288. test -f foo
  289. testval=something
  290. $testval
  291. ) || aiee
  292. ! inverted
  293. echo ${somevar}
  294. case foo in
  295. bar)
  296. echo bar
  297. ;;
  298. baz)
  299. echo baz
  300. ;;
  301. foo*)
  302. echo foo
  303. ;;
  304. esac
  305. """
  306. def test_shell(self):
  307. execs = ["bar", "echo", "heh", "moo", "true", "aiee"]
  308. self.d.setVar("somevar", "heh")
  309. self.d.setVar("inverted", "echo inverted...")
  310. self.d.setVarFlag("inverted", "func", True)
  311. self.d.setVar("FOO", self.shelldata)
  312. self.d.setVarFlags("FOO", {"func": True})
  313. self.setEmptyVars(execs)
  314. deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
  315. self.assertEqual(deps, set(["somevar", "inverted"] + execs))
  316. def test_vardeps(self):
  317. self.d.setVar("oe_libinstall", "echo test")
  318. self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
  319. self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
  320. deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
  321. self.assertEqual(deps, set(["oe_libinstall"]))
  322. def test_vardeps_expand(self):
  323. self.d.setVar("oe_libinstall", "echo test")
  324. self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
  325. self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
  326. deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
  327. self.assertEqual(deps, set(["oe_libinstall"]))
  328. def test_contains_vardeps(self):
  329. expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
  330. ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
  331. ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
  332. ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
  333. parsedvar = self.d.expandWithRefs(expr, None)
  334. # Check contains
  335. self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
  336. # Check dependencies
  337. self.d.setVar('ANOTHERVAR', expr)
  338. self.d.setVar('TESTVAR', 'anothervalue testval testval2')
  339. deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
  340. self.assertEqual(sorted(values.splitlines()),
  341. sorted([expr,
  342. 'TESTVAR{anothervalue} = Set',
  343. 'TESTVAR{somevalue} = Unset',
  344. 'TESTVAR{testval testval2} = Set',
  345. 'TESTVAR{testval2 testval3} = Unset',
  346. 'TESTVAR{testval2} = Set',
  347. 'TESTVAR{testval3} = Unset'
  348. ]))
  349. # Check final value
  350. self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
  351. #Currently no wildcard support
  352. #def test_vardeps_wildcards(self):
  353. # self.d.setVar("oe_libinstall", "echo test")
  354. # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
  355. # self.d.setVarFlag("FOO", "vardeps", "oe_*")
  356. # self.assertEquals(deps, set(["oe_libinstall"]))