utils.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. #
  2. # BitBake Tests for utils.py
  3. #
  4. # Copyright (C) 2012 Richard Purdie
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. import unittest
  9. import bb
  10. import os
  11. import tempfile
  12. import re
  13. class VerCmpString(unittest.TestCase):
  14. def test_vercmpstring(self):
  15. result = bb.utils.vercmp_string('1', '2')
  16. self.assertTrue(result < 0)
  17. result = bb.utils.vercmp_string('2', '1')
  18. self.assertTrue(result > 0)
  19. result = bb.utils.vercmp_string('1', '1.0')
  20. self.assertTrue(result < 0)
  21. result = bb.utils.vercmp_string('1', '1.1')
  22. self.assertTrue(result < 0)
  23. result = bb.utils.vercmp_string('1.1', '1_p2')
  24. self.assertTrue(result < 0)
  25. result = bb.utils.vercmp_string('1.0', '1.0+1.1-beta1')
  26. self.assertTrue(result < 0)
  27. result = bb.utils.vercmp_string('1.1', '1.0+1.1-beta1')
  28. self.assertTrue(result > 0)
  29. result = bb.utils.vercmp_string('1a', '1a1')
  30. self.assertTrue(result < 0)
  31. result = bb.utils.vercmp_string('1a1', '1a')
  32. self.assertTrue(result > 0)
  33. result = bb.utils.vercmp_string('1.', '1.1')
  34. self.assertTrue(result < 0)
  35. result = bb.utils.vercmp_string('1.1', '1.')
  36. self.assertTrue(result > 0)
  37. def test_explode_dep_versions(self):
  38. correctresult = {"foo" : ["= 1.10"]}
  39. result = bb.utils.explode_dep_versions2("foo (= 1.10)")
  40. self.assertEqual(result, correctresult)
  41. result = bb.utils.explode_dep_versions2("foo (=1.10)")
  42. self.assertEqual(result, correctresult)
  43. result = bb.utils.explode_dep_versions2("foo ( = 1.10)")
  44. self.assertEqual(result, correctresult)
  45. result = bb.utils.explode_dep_versions2("foo ( =1.10)")
  46. self.assertEqual(result, correctresult)
  47. result = bb.utils.explode_dep_versions2("foo ( = 1.10 )")
  48. self.assertEqual(result, correctresult)
  49. result = bb.utils.explode_dep_versions2("foo ( =1.10 )")
  50. self.assertEqual(result, correctresult)
  51. def test_vercmp_string_op(self):
  52. compareops = [('1', '1', '=', True),
  53. ('1', '1', '==', True),
  54. ('1', '1', '!=', False),
  55. ('1', '1', '>', False),
  56. ('1', '1', '<', False),
  57. ('1', '1', '>=', True),
  58. ('1', '1', '<=', True),
  59. ('1', '0', '=', False),
  60. ('1', '0', '==', False),
  61. ('1', '0', '!=', True),
  62. ('1', '0', '>', True),
  63. ('1', '0', '<', False),
  64. ('1', '0', '>>', True),
  65. ('1', '0', '<<', False),
  66. ('1', '0', '>=', True),
  67. ('1', '0', '<=', False),
  68. ('0', '1', '=', False),
  69. ('0', '1', '==', False),
  70. ('0', '1', '!=', True),
  71. ('0', '1', '>', False),
  72. ('0', '1', '<', True),
  73. ('0', '1', '>>', False),
  74. ('0', '1', '<<', True),
  75. ('0', '1', '>=', False),
  76. ('0', '1', '<=', True)]
  77. for arg1, arg2, op, correctresult in compareops:
  78. result = bb.utils.vercmp_string_op(arg1, arg2, op)
  79. self.assertEqual(result, correctresult, 'vercmp_string_op("%s", "%s", "%s") != %s' % (arg1, arg2, op, correctresult))
  80. # Check that clearly invalid operator raises an exception
  81. self.assertRaises(bb.utils.VersionStringException, bb.utils.vercmp_string_op, '0', '0', '$')
  82. class Path(unittest.TestCase):
  83. def test_unsafe_delete_path(self):
  84. checkitems = [('/', True),
  85. ('//', True),
  86. ('///', True),
  87. (os.getcwd().count(os.sep) * ('..' + os.sep), True),
  88. (os.environ.get('HOME', '/home/test'), True),
  89. ('/home/someone', True),
  90. ('/home/other/', True),
  91. ('/home/other/subdir', False),
  92. ('', False)]
  93. for arg1, correctresult in checkitems:
  94. result = bb.utils._check_unsafe_delete_path(arg1)
  95. self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
  96. class Checksum(unittest.TestCase):
  97. filler = b"Shiver me timbers square-rigged spike Gold Road galleon bilge water boatswain wherry jack pirate. Mizzenmast rum lad Privateer jack salmagundi hang the jib piracy Pieces of Eight Corsair. Parrel marooned black spot yawl provost quarterdeck cable no prey, no pay spirits lateen sail."
  98. def test_md5(self):
  99. import hashlib
  100. with tempfile.NamedTemporaryFile() as f:
  101. f.write(self.filler)
  102. f.flush()
  103. checksum = bb.utils.md5_file(f.name)
  104. self.assertEqual(checksum, "bd572cd5de30a785f4efcb6eaf5089e3")
  105. def test_sha1(self):
  106. import hashlib
  107. with tempfile.NamedTemporaryFile() as f:
  108. f.write(self.filler)
  109. f.flush()
  110. checksum = bb.utils.sha1_file(f.name)
  111. self.assertEqual(checksum, "249eb8fd654732ea836d5e702d7aa567898eca71")
  112. def test_sha256(self):
  113. import hashlib
  114. with tempfile.NamedTemporaryFile() as f:
  115. f.write(self.filler)
  116. f.flush()
  117. checksum = bb.utils.sha256_file(f.name)
  118. self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f")
  119. class EditMetadataFile(unittest.TestCase):
  120. _origfile = """
  121. # A comment
  122. HELLO = "oldvalue"
  123. THIS = "that"
  124. # Another comment
  125. NOCHANGE = "samevalue"
  126. OTHER = 'anothervalue'
  127. MULTILINE = "a1 \\
  128. a2 \\
  129. a3"
  130. MULTILINE2 := " \\
  131. b1 \\
  132. b2 \\
  133. b3 \\
  134. "
  135. MULTILINE3 = " \\
  136. c1 \\
  137. c2 \\
  138. c3 \\
  139. "
  140. do_functionname() {
  141. command1 ${VAL1} ${VAL2}
  142. command2 ${VAL3} ${VAL4}
  143. }
  144. """
  145. def _testeditfile(self, varvalues, compareto, dummyvars=None):
  146. if dummyvars is None:
  147. dummyvars = []
  148. with tempfile.NamedTemporaryFile('w', delete=False) as tf:
  149. tf.write(self._origfile)
  150. tf.close()
  151. try:
  152. varcalls = []
  153. def handle_file(varname, origvalue, op, newlines):
  154. self.assertIn(varname, varvalues, 'Callback called for variable %s not in the list!' % varname)
  155. self.assertNotIn(varname, dummyvars, 'Callback called for variable %s in dummy list!' % varname)
  156. varcalls.append(varname)
  157. return varvalues[varname]
  158. bb.utils.edit_metadata_file(tf.name, varvalues.keys(), handle_file)
  159. with open(tf.name) as f:
  160. modfile = f.readlines()
  161. # Ensure the output matches the expected output
  162. self.assertEqual(compareto.splitlines(True), modfile)
  163. # Ensure the callback function was called for every variable we asked for
  164. # (plus allow testing behaviour when a requested variable is not present)
  165. self.assertEqual(sorted(varvalues.keys()), sorted(varcalls + dummyvars))
  166. finally:
  167. os.remove(tf.name)
  168. def test_edit_metadata_file_nochange(self):
  169. # Test file doesn't get modified with nothing to do
  170. self._testeditfile({}, self._origfile)
  171. # Test file doesn't get modified with only dummy variables
  172. self._testeditfile({'DUMMY1': ('should_not_set', None, 0, True),
  173. 'DUMMY2': ('should_not_set_again', None, 0, True)}, self._origfile, dummyvars=['DUMMY1', 'DUMMY2'])
  174. # Test file doesn't get modified with some the same values
  175. self._testeditfile({'THIS': ('that', None, 0, True),
  176. 'OTHER': ('anothervalue', None, 0, True),
  177. 'MULTILINE3': (' c1 c2 c3 ', None, 4, False)}, self._origfile)
  178. def test_edit_metadata_file_1(self):
  179. newfile1 = """
  180. # A comment
  181. HELLO = "newvalue"
  182. THIS = "that"
  183. # Another comment
  184. NOCHANGE = "samevalue"
  185. OTHER = 'anothervalue'
  186. MULTILINE = "a1 \\
  187. a2 \\
  188. a3"
  189. MULTILINE2 := " \\
  190. b1 \\
  191. b2 \\
  192. b3 \\
  193. "
  194. MULTILINE3 = " \\
  195. c1 \\
  196. c2 \\
  197. c3 \\
  198. "
  199. do_functionname() {
  200. command1 ${VAL1} ${VAL2}
  201. command2 ${VAL3} ${VAL4}
  202. }
  203. """
  204. self._testeditfile({'HELLO': ('newvalue', None, 4, True)}, newfile1)
  205. def test_edit_metadata_file_2(self):
  206. newfile2 = """
  207. # A comment
  208. HELLO = "oldvalue"
  209. THIS = "that"
  210. # Another comment
  211. NOCHANGE = "samevalue"
  212. OTHER = 'anothervalue'
  213. MULTILINE = " \\
  214. d1 \\
  215. d2 \\
  216. d3 \\
  217. "
  218. MULTILINE2 := " \\
  219. b1 \\
  220. b2 \\
  221. b3 \\
  222. "
  223. MULTILINE3 = "nowsingle"
  224. do_functionname() {
  225. command1 ${VAL1} ${VAL2}
  226. command2 ${VAL3} ${VAL4}
  227. }
  228. """
  229. self._testeditfile({'MULTILINE': (['d1','d2','d3'], None, 4, False),
  230. 'MULTILINE3': ('nowsingle', None, 4, True),
  231. 'NOTPRESENT': (['a', 'b'], None, 4, False)}, newfile2, dummyvars=['NOTPRESENT'])
  232. def test_edit_metadata_file_3(self):
  233. newfile3 = """
  234. # A comment
  235. HELLO = "oldvalue"
  236. # Another comment
  237. NOCHANGE = "samevalue"
  238. OTHER = "yetanothervalue"
  239. MULTILINE = "e1 \\
  240. e2 \\
  241. e3 \\
  242. "
  243. MULTILINE2 := "f1 \\
  244. \tf2 \\
  245. \t"
  246. MULTILINE3 = " \\
  247. c1 \\
  248. c2 \\
  249. c3 \\
  250. "
  251. do_functionname() {
  252. othercommand_one a b c
  253. othercommand_two d e f
  254. }
  255. """
  256. self._testeditfile({'do_functionname()': (['othercommand_one a b c', 'othercommand_two d e f'], None, 4, False),
  257. 'MULTILINE2': (['f1', 'f2'], None, '\t', True),
  258. 'MULTILINE': (['e1', 'e2', 'e3'], None, -1, True),
  259. 'THIS': (None, None, 0, False),
  260. 'OTHER': ('yetanothervalue', None, 0, True)}, newfile3)
  261. def test_edit_metadata_file_4(self):
  262. newfile4 = """
  263. # A comment
  264. HELLO = "oldvalue"
  265. THIS = "that"
  266. # Another comment
  267. OTHER = 'anothervalue'
  268. MULTILINE = "a1 \\
  269. a2 \\
  270. a3"
  271. MULTILINE2 := " \\
  272. b1 \\
  273. b2 \\
  274. b3 \\
  275. "
  276. """
  277. self._testeditfile({'NOCHANGE': (None, None, 0, False),
  278. 'MULTILINE3': (None, None, 0, False),
  279. 'THIS': ('that', None, 0, False),
  280. 'do_functionname()': (None, None, 0, False)}, newfile4)
  281. def test_edit_metadata(self):
  282. newfile5 = """
  283. # A comment
  284. HELLO = "hithere"
  285. # A new comment
  286. THIS += "that"
  287. # Another comment
  288. NOCHANGE = "samevalue"
  289. OTHER = 'anothervalue'
  290. MULTILINE = "a1 \\
  291. a2 \\
  292. a3"
  293. MULTILINE2 := " \\
  294. b1 \\
  295. b2 \\
  296. b3 \\
  297. "
  298. MULTILINE3 = " \\
  299. c1 \\
  300. c2 \\
  301. c3 \\
  302. "
  303. NEWVAR = "value"
  304. do_functionname() {
  305. command1 ${VAL1} ${VAL2}
  306. command2 ${VAL3} ${VAL4}
  307. }
  308. """
  309. def handle_var(varname, origvalue, op, newlines):
  310. if varname == 'THIS':
  311. newlines.append('# A new comment\n')
  312. elif varname == 'do_functionname()':
  313. newlines.append('NEWVAR = "value"\n')
  314. newlines.append('\n')
  315. valueitem = varvalues.get(varname, None)
  316. if valueitem:
  317. return valueitem
  318. else:
  319. return (origvalue, op, 0, True)
  320. varvalues = {'HELLO': ('hithere', None, 0, True), 'THIS': ('that', '+=', 0, True)}
  321. varlist = ['HELLO', 'THIS', 'do_functionname()']
  322. (updated, newlines) = bb.utils.edit_metadata(self._origfile.splitlines(True), varlist, handle_var)
  323. self.assertTrue(updated, 'List should be updated but isn\'t')
  324. self.assertEqual(newlines, newfile5.splitlines(True))
  325. # Make sure the orig value matches what we expect it to be
  326. def test_edit_metadata_origvalue(self):
  327. origfile = """
  328. MULTILINE = " stuff \\
  329. morestuff"
  330. """
  331. expected_value = "stuff morestuff"
  332. global value_in_callback
  333. value_in_callback = ""
  334. def handle_var(varname, origvalue, op, newlines):
  335. global value_in_callback
  336. value_in_callback = origvalue
  337. return (origvalue, op, -1, False)
  338. bb.utils.edit_metadata(origfile.splitlines(True),
  339. ['MULTILINE'],
  340. handle_var)
  341. testvalue = re.sub('\s+', ' ', value_in_callback.strip())
  342. self.assertEqual(expected_value, testvalue)
  343. class EditBbLayersConf(unittest.TestCase):
  344. def _test_bblayers_edit(self, before, after, add, remove, notadded, notremoved):
  345. with tempfile.NamedTemporaryFile('w', delete=False) as tf:
  346. tf.write(before)
  347. tf.close()
  348. try:
  349. actual_notadded, actual_notremoved = bb.utils.edit_bblayers_conf(tf.name, add, remove)
  350. with open(tf.name) as f:
  351. actual_after = f.readlines()
  352. self.assertEqual(after.splitlines(True), actual_after)
  353. self.assertEqual(notadded, actual_notadded)
  354. self.assertEqual(notremoved, actual_notremoved)
  355. finally:
  356. os.remove(tf.name)
  357. def test_bblayers_remove(self):
  358. before = r"""
  359. # A comment
  360. BBPATH = "${TOPDIR}"
  361. BBFILES ?= ""
  362. BBLAYERS = " \
  363. /home/user/path/layer1 \
  364. /home/user/path/layer2 \
  365. /home/user/path/subpath/layer3 \
  366. /home/user/path/layer4 \
  367. "
  368. """
  369. after = r"""
  370. # A comment
  371. BBPATH = "${TOPDIR}"
  372. BBFILES ?= ""
  373. BBLAYERS = " \
  374. /home/user/path/layer1 \
  375. /home/user/path/subpath/layer3 \
  376. /home/user/path/layer4 \
  377. "
  378. """
  379. self._test_bblayers_edit(before, after,
  380. None,
  381. '/home/user/path/layer2',
  382. [],
  383. [])
  384. def test_bblayers_add(self):
  385. before = r"""
  386. # A comment
  387. BBPATH = "${TOPDIR}"
  388. BBFILES ?= ""
  389. BBLAYERS = " \
  390. /home/user/path/layer1 \
  391. /home/user/path/layer2 \
  392. /home/user/path/subpath/layer3 \
  393. /home/user/path/layer4 \
  394. "
  395. """
  396. after = r"""
  397. # A comment
  398. BBPATH = "${TOPDIR}"
  399. BBFILES ?= ""
  400. BBLAYERS = " \
  401. /home/user/path/layer1 \
  402. /home/user/path/layer2 \
  403. /home/user/path/subpath/layer3 \
  404. /home/user/path/layer4 \
  405. /other/path/to/layer5 \
  406. "
  407. """
  408. self._test_bblayers_edit(before, after,
  409. '/other/path/to/layer5/',
  410. None,
  411. [],
  412. [])
  413. def test_bblayers_add_remove(self):
  414. before = r"""
  415. # A comment
  416. BBPATH = "${TOPDIR}"
  417. BBFILES ?= ""
  418. BBLAYERS = " \
  419. /home/user/path/layer1 \
  420. /home/user/path/layer2 \
  421. /home/user/path/subpath/layer3 \
  422. /home/user/path/layer4 \
  423. "
  424. """
  425. after = r"""
  426. # A comment
  427. BBPATH = "${TOPDIR}"
  428. BBFILES ?= ""
  429. BBLAYERS = " \
  430. /home/user/path/layer1 \
  431. /home/user/path/layer2 \
  432. /home/user/path/layer4 \
  433. /other/path/to/layer5 \
  434. "
  435. """
  436. self._test_bblayers_edit(before, after,
  437. ['/other/path/to/layer5', '/home/user/path/layer2/'], '/home/user/path/subpath/layer3/',
  438. ['/home/user/path/layer2'],
  439. [])
  440. def test_bblayers_add_remove_home(self):
  441. before = r"""
  442. # A comment
  443. BBPATH = "${TOPDIR}"
  444. BBFILES ?= ""
  445. BBLAYERS = " \
  446. ~/path/layer1 \
  447. ~/path/layer2 \
  448. ~/otherpath/layer3 \
  449. ~/path/layer4 \
  450. "
  451. """
  452. after = r"""
  453. # A comment
  454. BBPATH = "${TOPDIR}"
  455. BBFILES ?= ""
  456. BBLAYERS = " \
  457. ~/path/layer2 \
  458. ~/path/layer4 \
  459. ~/path2/layer5 \
  460. "
  461. """
  462. self._test_bblayers_edit(before, after,
  463. [os.environ['HOME'] + '/path/layer4', '~/path2/layer5'],
  464. [os.environ['HOME'] + '/otherpath/layer3', '~/path/layer1', '~/path/notinlist'],
  465. [os.environ['HOME'] + '/path/layer4'],
  466. ['~/path/notinlist'])
  467. def test_bblayers_add_remove_plusequals(self):
  468. before = r"""
  469. # A comment
  470. BBPATH = "${TOPDIR}"
  471. BBFILES ?= ""
  472. BBLAYERS += " \
  473. /home/user/path/layer1 \
  474. /home/user/path/layer2 \
  475. "
  476. """
  477. after = r"""
  478. # A comment
  479. BBPATH = "${TOPDIR}"
  480. BBFILES ?= ""
  481. BBLAYERS += " \
  482. /home/user/path/layer2 \
  483. /home/user/path/layer3 \
  484. "
  485. """
  486. self._test_bblayers_edit(before, after,
  487. '/home/user/path/layer3',
  488. '/home/user/path/layer1',
  489. [],
  490. [])
  491. def test_bblayers_add_remove_plusequals2(self):
  492. before = r"""
  493. # A comment
  494. BBPATH = "${TOPDIR}"
  495. BBFILES ?= ""
  496. BBLAYERS += " \
  497. /home/user/path/layer1 \
  498. /home/user/path/layer2 \
  499. /home/user/path/layer3 \
  500. "
  501. BBLAYERS += "/home/user/path/layer4"
  502. BBLAYERS += "/home/user/path/layer5"
  503. """
  504. after = r"""
  505. # A comment
  506. BBPATH = "${TOPDIR}"
  507. BBFILES ?= ""
  508. BBLAYERS += " \
  509. /home/user/path/layer2 \
  510. /home/user/path/layer3 \
  511. "
  512. BBLAYERS += "/home/user/path/layer5"
  513. BBLAYERS += "/home/user/otherpath/layer6"
  514. """
  515. self._test_bblayers_edit(before, after,
  516. ['/home/user/otherpath/layer6', '/home/user/path/layer3'], ['/home/user/path/layer1', '/home/user/path/layer4', '/home/user/path/layer7'],
  517. ['/home/user/path/layer3'],
  518. ['/home/user/path/layer7'])
  519. class GetReferencedVars(unittest.TestCase):
  520. def setUp(self):
  521. self.d = bb.data.init()
  522. def check_referenced(self, expression, expected_layers):
  523. vars = bb.utils.get_referenced_vars(expression, self.d)
  524. # Do the easy check first - is every variable accounted for?
  525. expected_vars = set.union(set(), *expected_layers)
  526. got_vars = set(vars)
  527. self.assertSetEqual(got_vars, expected_vars)
  528. # Now test the order of the layers
  529. start = 0
  530. for i, expected_layer in enumerate(expected_layers):
  531. got_layer = set(vars[start:len(expected_layer)+start])
  532. start += len(expected_layer)
  533. self.assertSetEqual(got_layer, expected_layer)
  534. def test_no_vars(self):
  535. self.check_referenced("", [])
  536. self.check_referenced(" ", [])
  537. self.check_referenced(" no vars here! ", [])
  538. def test_single_layer(self):
  539. self.check_referenced("${VAR}", [{"VAR"}])
  540. self.check_referenced("${VAR} ${VAR}", [{"VAR"}])
  541. def test_two_layer(self):
  542. self.d.setVar("VAR", "${B}")
  543. self.check_referenced("${VAR}", [{"VAR"}, {"B"}])
  544. self.check_referenced("${@d.getVar('VAR')}", [{"VAR"}, {"B"}])
  545. def test_more_complicated(self):
  546. self.d["SRC_URI"] = "${QT_GIT}/${QT_MODULE}.git;name=${QT_MODULE};${QT_MODULE_BRANCH_PARAM};protocol=${QT_GIT_PROTOCOL}"
  547. self.d["QT_GIT"] = "git://code.qt.io/${QT_GIT_PROJECT}"
  548. self.d["QT_MODULE_BRANCH_PARAM"] = "branch=${QT_MODULE_BRANCH}"
  549. self.d["QT_MODULE"] = "${BPN}"
  550. self.d["BPN"] = "something to do with ${PN} and ${SPECIAL_PKGSUFFIX}"
  551. layers = [{"SRC_URI"}, {"QT_GIT", "QT_MODULE", "QT_MODULE_BRANCH_PARAM", "QT_GIT_PROTOCOL"}, {"QT_GIT_PROJECT", "QT_MODULE_BRANCH", "BPN"}, {"PN", "SPECIAL_PKGSUFFIX"}]
  552. self.check_referenced("${SRC_URI}", layers)