recipetool.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718
  1. #
  2. # SPDX-License-Identifier: MIT
  3. #
  4. import os
  5. import shutil
  6. import tempfile
  7. import urllib.parse
  8. from oeqa.utils.commands import runCmd, bitbake, get_bb_var
  9. from oeqa.utils.commands import get_bb_vars, create_temp_layer
  10. from oeqa.selftest.cases import devtool
  11. templayerdir = None
  12. def setUpModule():
  13. global templayerdir
  14. templayerdir = tempfile.mkdtemp(prefix='recipetoolqa')
  15. create_temp_layer(templayerdir, 'selftestrecipetool')
  16. runCmd('bitbake-layers add-layer %s' % templayerdir)
  17. def tearDownModule():
  18. runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True)
  19. runCmd('rm -rf %s' % templayerdir)
  20. class RecipetoolBase(devtool.DevtoolBase):
  21. def setUpLocal(self):
  22. super(RecipetoolBase, self).setUpLocal()
  23. self.templayerdir = templayerdir
  24. self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa')
  25. self.track_for_cleanup(self.tempdir)
  26. self.testfile = os.path.join(self.tempdir, 'testfile')
  27. with open(self.testfile, 'w') as f:
  28. f.write('Test file\n')
  29. def tearDownLocal(self):
  30. runCmd('rm -rf %s/recipes-*' % self.templayerdir)
  31. super(RecipetoolBase, self).tearDownLocal()
  32. def _try_recipetool_appendcmd(self, cmd, testrecipe, expectedfiles, expectedlines=None):
  33. result = runCmd(cmd)
  34. self.assertNotIn('Traceback', result.output)
  35. # Check the bbappend was created and applies properly
  36. recipefile = get_bb_var('FILE', testrecipe)
  37. bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir)
  38. # Check the bbappend contents
  39. if expectedlines is not None:
  40. with open(bbappendfile, 'r') as f:
  41. self.assertEqual(expectedlines, f.readlines(), "Expected lines are not present in %s" % bbappendfile)
  42. # Check file was copied
  43. filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe)
  44. for expectedfile in expectedfiles:
  45. self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile)
  46. # Check no other files created
  47. createdfiles = []
  48. for root, _, files in os.walk(filesdir):
  49. for f in files:
  50. createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir))
  51. self.assertTrue(sorted(createdfiles), sorted(expectedfiles))
  52. return bbappendfile, result.output
  53. class RecipetoolTests(RecipetoolBase):
  54. @classmethod
  55. def setUpClass(cls):
  56. super(RecipetoolTests, cls).setUpClass()
  57. # Ensure we have the right data in shlibs/pkgdata
  58. cls.logger.info('Running bitbake to generate pkgdata')
  59. bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile')
  60. bb_vars = get_bb_vars(['COREBASE', 'BBPATH'])
  61. cls.corebase = bb_vars['COREBASE']
  62. cls.bbpath = bb_vars['BBPATH']
  63. def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles):
  64. cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options)
  65. return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines)
  66. def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror):
  67. cmd = 'recipetool appendfile %s %s %s' % (self.templayerdir, destfile, newfile)
  68. result = runCmd(cmd, ignore_status=True)
  69. self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd)
  70. self.assertNotIn('Traceback', result.output)
  71. for errorstr in checkerror:
  72. self.assertIn(errorstr, result.output)
  73. def test_recipetool_appendfile_basic(self):
  74. # Basic test
  75. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  76. '\n']
  77. _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd'])
  78. self.assertNotIn('WARNING: ', output)
  79. def test_recipetool_appendfile_invalid(self):
  80. # Test some commands that should error
  81. self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers'])
  82. self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool'])
  83. self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool'])
  84. def test_recipetool_appendfile_alternatives(self):
  85. # Now try with a file we know should be an alternative
  86. # (this is very much a fake example, but one we know is reliably an alternative)
  87. self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox'])
  88. # Need a test file - should be executable
  89. testfile2 = os.path.join(self.corebase, 'oe-init-build-env')
  90. testfile2name = os.path.basename(testfile2)
  91. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  92. '\n',
  93. 'SRC_URI += "file://%s"\n' % testfile2name,
  94. '\n',
  95. 'do_install_append() {\n',
  96. ' install -d ${D}${base_bindir}\n',
  97. ' install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name,
  98. '}\n']
  99. self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name])
  100. # Now try bbappending the same file again, contents should not change
  101. bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name])
  102. # But file should have
  103. copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name)
  104. result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True)
  105. self.assertNotEqual(result.status, 0, 'New file should have been copied but was not %s' % result.output)
  106. def test_recipetool_appendfile_binary(self):
  107. # Try appending a binary file
  108. # /bin/ls can be a symlink to /usr/bin/ls
  109. ls = os.path.realpath("/bin/ls")
  110. result = runCmd('recipetool appendfile %s /bin/ls %s -r coreutils' % (self.templayerdir, ls))
  111. self.assertIn('WARNING: ', result.output)
  112. self.assertIn('is a binary', result.output)
  113. def test_recipetool_appendfile_add(self):
  114. # Try arbitrary file add to a recipe
  115. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  116. '\n',
  117. 'SRC_URI += "file://testfile"\n',
  118. '\n',
  119. 'do_install_append() {\n',
  120. ' install -d ${D}${datadir}\n',
  121. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
  122. '}\n']
  123. self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile'])
  124. # Try adding another file, this time where the source file is executable
  125. # (so we're testing that, plus modifying an existing bbappend)
  126. testfile2 = os.path.join(self.corebase, 'oe-init-build-env')
  127. testfile2name = os.path.basename(testfile2)
  128. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  129. '\n',
  130. 'SRC_URI += "file://testfile \\\n',
  131. ' file://%s \\\n' % testfile2name,
  132. ' "\n',
  133. '\n',
  134. 'do_install_append() {\n',
  135. ' install -d ${D}${datadir}\n',
  136. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
  137. ' install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name,
  138. '}\n']
  139. self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name])
  140. def test_recipetool_appendfile_add_bindir(self):
  141. # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
  142. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  143. '\n',
  144. 'SRC_URI += "file://testfile"\n',
  145. '\n',
  146. 'do_install_append() {\n',
  147. ' install -d ${D}${bindir}\n',
  148. ' install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n',
  149. '}\n']
  150. _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile'])
  151. self.assertNotIn('WARNING: ', output)
  152. def test_recipetool_appendfile_add_machine(self):
  153. # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
  154. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  155. '\n',
  156. 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n',
  157. '\n',
  158. 'SRC_URI_append_mymachine = " file://testfile"\n',
  159. '\n',
  160. 'do_install_append_mymachine() {\n',
  161. ' install -d ${D}${datadir}\n',
  162. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
  163. '}\n']
  164. _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile'])
  165. self.assertNotIn('WARNING: ', output)
  166. def test_recipetool_appendfile_orig(self):
  167. # A file that's in SRC_URI and in do_install with the same name
  168. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  169. '\n']
  170. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig'])
  171. self.assertNotIn('WARNING: ', output)
  172. def test_recipetool_appendfile_todir(self):
  173. # A file that's in SRC_URI and in do_install with destination directory rather than file
  174. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  175. '\n']
  176. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir'])
  177. self.assertNotIn('WARNING: ', output)
  178. def test_recipetool_appendfile_renamed(self):
  179. # A file that's in SRC_URI with a different name to the destination file
  180. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  181. '\n']
  182. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1'])
  183. self.assertNotIn('WARNING: ', output)
  184. def test_recipetool_appendfile_subdir(self):
  185. # A file that's in SRC_URI in a subdir
  186. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  187. '\n',
  188. 'SRC_URI += "file://testfile"\n',
  189. '\n',
  190. 'do_install_append() {\n',
  191. ' install -d ${D}${datadir}\n',
  192. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n',
  193. '}\n']
  194. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile'])
  195. self.assertNotIn('WARNING: ', output)
  196. def test_recipetool_appendfile_inst_glob(self):
  197. # A file that's in do_install as a glob
  198. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  199. '\n']
  200. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile'])
  201. self.assertNotIn('WARNING: ', output)
  202. def test_recipetool_appendfile_inst_todir_glob(self):
  203. # A file that's in do_install as a glob with destination as a directory
  204. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  205. '\n']
  206. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile'])
  207. self.assertNotIn('WARNING: ', output)
  208. def test_recipetool_appendfile_patch(self):
  209. # A file that's added by a patch in SRC_URI
  210. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  211. '\n',
  212. 'SRC_URI += "file://testfile"\n',
  213. '\n',
  214. 'do_install_append() {\n',
  215. ' install -d ${D}${sysconfdir}\n',
  216. ' install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n',
  217. '}\n']
  218. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile'])
  219. for line in output.splitlines():
  220. if 'WARNING: ' in line:
  221. self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line)
  222. break
  223. else:
  224. self.fail('Patch warning not found in output:\n%s' % output)
  225. def test_recipetool_appendfile_script(self):
  226. # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install)
  227. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  228. '\n',
  229. 'SRC_URI += "file://testfile"\n',
  230. '\n',
  231. 'do_install_append() {\n',
  232. ' install -d ${D}${datadir}\n',
  233. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n',
  234. '}\n']
  235. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile'])
  236. self.assertNotIn('WARNING: ', output)
  237. def test_recipetool_appendfile_inst_func(self):
  238. # A file that's installed from a function called by do_install
  239. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  240. '\n']
  241. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func'])
  242. self.assertNotIn('WARNING: ', output)
  243. def test_recipetool_appendfile_postinstall(self):
  244. # A file that's created by a postinstall script (and explicitly mentioned in it)
  245. # First try without specifying recipe
  246. self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile'])
  247. # Now specify recipe
  248. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  249. '\n',
  250. 'SRC_URI += "file://testfile"\n',
  251. '\n',
  252. 'do_install_append() {\n',
  253. ' install -d ${D}${datadir}\n',
  254. ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n',
  255. '}\n']
  256. _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile'])
  257. def test_recipetool_appendfile_extlayer(self):
  258. # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure
  259. exttemplayerdir = os.path.join(self.tempdir, 'extlayer')
  260. self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*')
  261. result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile))
  262. self.assertNotIn('Traceback', result.output)
  263. createdfiles = []
  264. for root, _, files in os.walk(exttemplayerdir):
  265. for f in files:
  266. createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir))
  267. createdfiles.remove('conf/layer.conf')
  268. expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend',
  269. 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig']
  270. self.assertEqual(sorted(createdfiles), sorted(expectedfiles))
  271. def test_recipetool_appendfile_wildcard(self):
  272. def try_appendfile_wc(options):
  273. result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (self.templayerdir, self.testfile, options))
  274. self.assertNotIn('Traceback', result.output)
  275. bbappendfile = None
  276. for root, _, files in os.walk(self.templayerdir):
  277. for f in files:
  278. if f.endswith('.bbappend'):
  279. bbappendfile = f
  280. break
  281. if not bbappendfile:
  282. self.fail('No bbappend file created')
  283. runCmd('rm -rf %s/recipes-*' % self.templayerdir)
  284. return bbappendfile
  285. # Check without wildcard option
  286. recipefn = os.path.basename(get_bb_var('FILE', 'base-files'))
  287. filename = try_appendfile_wc('')
  288. self.assertEqual(filename, recipefn.replace('.bb', '.bbappend'))
  289. # Now check with wildcard option
  290. filename = try_appendfile_wc('-w')
  291. self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend')
  292. def test_recipetool_create(self):
  293. # Try adding a recipe
  294. tempsrc = os.path.join(self.tempdir, 'srctree')
  295. os.makedirs(tempsrc)
  296. recipefile = os.path.join(self.tempdir, 'logrotate_3.12.3.bb')
  297. srcuri = 'https://github.com/logrotate/logrotate/releases/download/3.12.3/logrotate-3.12.3.tar.xz'
  298. result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc))
  299. self.assertTrue(os.path.isfile(recipefile))
  300. checkvars = {}
  301. checkvars['LICENSE'] = 'GPLv2'
  302. checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263'
  303. checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/releases/download/${PV}/logrotate-${PV}.tar.xz'
  304. checkvars['SRC_URI[md5sum]'] = 'a560c57fac87c45b2fc17406cdf79288'
  305. checkvars['SRC_URI[sha256sum]'] = '2e6a401cac9024db2288297e3be1a8ab60e7401ba8e91225218aaf4a27e82a07'
  306. self._test_recipe_contents(recipefile, checkvars, [])
  307. def test_recipetool_create_git(self):
  308. if 'x11' not in get_bb_var('DISTRO_FEATURES'):
  309. self.skipTest('Test requires x11 as distro feature')
  310. # Ensure we have the right data in shlibs/pkgdata
  311. bitbake('libpng pango libx11 libxext jpeg libcheck')
  312. # Try adding a recipe
  313. tempsrc = os.path.join(self.tempdir, 'srctree')
  314. os.makedirs(tempsrc)
  315. recipefile = os.path.join(self.tempdir, 'libmatchbox.bb')
  316. srcuri = 'git://git.yoctoproject.org/libmatchbox'
  317. result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc])
  318. self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output)
  319. checkvars = {}
  320. checkvars['LICENSE'] = 'LGPLv2.1'
  321. checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34'
  322. checkvars['S'] = '${WORKDIR}/git'
  323. checkvars['PV'] = '1.11+git${SRCPV}'
  324. checkvars['SRC_URI'] = srcuri
  325. checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango'])
  326. inherits = ['autotools', 'pkgconfig']
  327. self._test_recipe_contents(recipefile, checkvars, inherits)
  328. def test_recipetool_create_simple(self):
  329. # Try adding a recipe
  330. temprecipe = os.path.join(self.tempdir, 'recipe')
  331. os.makedirs(temprecipe)
  332. pv = '1.7.3.0'
  333. srcuri = 'http://www.dest-unreach.org/socat/download/socat-%s.tar.bz2' % pv
  334. result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe))
  335. dirlist = os.listdir(temprecipe)
  336. if len(dirlist) > 1:
  337. self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist)))
  338. if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])):
  339. self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist)))
  340. self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named')
  341. checkvars = {}
  342. checkvars['LICENSE'] = set(['Unknown', 'GPLv2'])
  343. checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263'])
  344. # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot
  345. checkvars['S'] = None
  346. checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}')
  347. inherits = ['autotools']
  348. self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits)
  349. def test_recipetool_create_cmake(self):
  350. temprecipe = os.path.join(self.tempdir, 'recipe')
  351. os.makedirs(temprecipe)
  352. recipefile = os.path.join(temprecipe, 'taglib_1.11.1.bb')
  353. srcuri = 'http://taglib.github.io/releases/taglib-1.11.1.tar.gz'
  354. result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri))
  355. self.assertTrue(os.path.isfile(recipefile))
  356. checkvars = {}
  357. checkvars['LICENSE'] = set(['LGPLv2.1', 'MPL-1.1'])
  358. checkvars['SRC_URI'] = 'http://taglib.github.io/releases/taglib-${PV}.tar.gz'
  359. checkvars['SRC_URI[md5sum]'] = 'cee7be0ccfc892fa433d6c837df9522a'
  360. checkvars['SRC_URI[sha256sum]'] = 'b6d1a5a610aae6ff39d93de5efd0fdc787aa9e9dc1e7026fa4c961b26563526b'
  361. checkvars['DEPENDS'] = set(['boost', 'zlib'])
  362. inherits = ['cmake']
  363. self._test_recipe_contents(recipefile, checkvars, inherits)
  364. def test_recipetool_create_npm(self):
  365. temprecipe = os.path.join(self.tempdir, 'recipe')
  366. os.makedirs(temprecipe)
  367. recipefile = os.path.join(temprecipe, 'savoirfairelinux-node-server-example_1.0.0.bb')
  368. shrinkwrap = os.path.join(temprecipe, 'savoirfairelinux-node-server-example', 'npm-shrinkwrap.json')
  369. srcuri = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  370. result = runCmd('recipetool create -o %s \'%s\'' % (temprecipe, srcuri))
  371. self.assertTrue(os.path.isfile(recipefile))
  372. self.assertTrue(os.path.isfile(shrinkwrap))
  373. checkvars = {}
  374. checkvars['SUMMARY'] = 'Node Server Example'
  375. checkvars['HOMEPAGE'] = 'https://github.com/savoirfairelinux/node-server-example#readme'
  376. checkvars['LICENSE'] = set(['MIT', 'ISC', 'Unknown'])
  377. urls = []
  378. urls.append('npm://registry.npmjs.org/;package=@savoirfairelinux/node-server-example;version=${PV}')
  379. urls.append('npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json')
  380. checkvars['SRC_URI'] = set(urls)
  381. checkvars['S'] = '${WORKDIR}/npm'
  382. checkvars['LICENSE_${PN}'] = 'MIT'
  383. checkvars['LICENSE_${PN}-base64'] = 'Unknown'
  384. checkvars['LICENSE_${PN}-accepts'] = 'MIT'
  385. checkvars['LICENSE_${PN}-inherits'] = 'ISC'
  386. inherits = ['npm']
  387. self._test_recipe_contents(recipefile, checkvars, inherits)
  388. def test_recipetool_create_github(self):
  389. # Basic test to see if github URL mangling works
  390. temprecipe = os.path.join(self.tempdir, 'recipe')
  391. os.makedirs(temprecipe)
  392. recipefile = os.path.join(temprecipe, 'meson_git.bb')
  393. srcuri = 'https://github.com/mesonbuild/meson;rev=0.32.0'
  394. result = runCmd(['recipetool', 'create', '-o', temprecipe, srcuri])
  395. self.assertTrue(os.path.isfile(recipefile))
  396. checkvars = {}
  397. checkvars['LICENSE'] = set(['Apache-2.0'])
  398. checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https'
  399. inherits = ['setuptools3']
  400. self._test_recipe_contents(recipefile, checkvars, inherits)
  401. def test_recipetool_create_python3_setuptools(self):
  402. # Test creating python3 package from tarball (using setuptools3 class)
  403. temprecipe = os.path.join(self.tempdir, 'recipe')
  404. os.makedirs(temprecipe)
  405. pn = 'python-magic'
  406. pv = '0.4.15'
  407. recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv))
  408. srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv
  409. result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri))
  410. self.assertTrue(os.path.isfile(recipefile))
  411. checkvars = {}
  412. checkvars['LICENSE'] = set(['MIT'])
  413. checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88'
  414. checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-${PV}.tar.gz'
  415. checkvars['SRC_URI[md5sum]'] = 'e384c95a47218f66c6501cd6dd45ff59'
  416. checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5'
  417. inherits = ['setuptools3']
  418. self._test_recipe_contents(recipefile, checkvars, inherits)
  419. def test_recipetool_create_python3_distutils(self):
  420. # Test creating python3 package from tarball (using distutils3 class)
  421. temprecipe = os.path.join(self.tempdir, 'recipe')
  422. os.makedirs(temprecipe)
  423. pn = 'docutils'
  424. pv = '0.14'
  425. recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv))
  426. srcuri = 'https://files.pythonhosted.org/packages/84/f4/5771e41fdf52aabebbadecc9381d11dea0fa34e4759b4071244fa094804c/docutils-%s.tar.gz' % pv
  427. result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri))
  428. self.assertTrue(os.path.isfile(recipefile))
  429. checkvars = {}
  430. checkvars['LICENSE'] = set(['PSF', '&', 'BSD', 'GPL'])
  431. checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING.txt;md5=35a23d42b615470583563132872c97d6'
  432. checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/f4/5771e41fdf52aabebbadecc9381d11dea0fa34e4759b4071244fa094804c/docutils-${PV}.tar.gz'
  433. checkvars['SRC_URI[md5sum]'] = 'c53768d63db3873b7d452833553469de'
  434. checkvars['SRC_URI[sha256sum]'] = '51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274'
  435. inherits = ['distutils3']
  436. self._test_recipe_contents(recipefile, checkvars, inherits)
  437. def test_recipetool_create_github_tarball(self):
  438. # Basic test to ensure github URL mangling doesn't apply to release tarballs
  439. temprecipe = os.path.join(self.tempdir, 'recipe')
  440. os.makedirs(temprecipe)
  441. pv = '0.32.0'
  442. recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv)
  443. srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv)
  444. result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri))
  445. self.assertTrue(os.path.isfile(recipefile))
  446. checkvars = {}
  447. checkvars['LICENSE'] = set(['Apache-2.0'])
  448. checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz'
  449. inherits = ['setuptools3']
  450. self._test_recipe_contents(recipefile, checkvars, inherits)
  451. def test_recipetool_create_git_http(self):
  452. # Basic test to check http git URL mangling works
  453. temprecipe = os.path.join(self.tempdir, 'recipe')
  454. os.makedirs(temprecipe)
  455. recipefile = os.path.join(temprecipe, 'matchbox-terminal_git.bb')
  456. srcuri = 'http://git.yoctoproject.org/git/matchbox-terminal'
  457. result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri))
  458. self.assertTrue(os.path.isfile(recipefile))
  459. checkvars = {}
  460. checkvars['LICENSE'] = set(['GPLv2'])
  461. checkvars['SRC_URI'] = 'git://git.yoctoproject.org/git/matchbox-terminal;protocol=http'
  462. inherits = ['pkgconfig', 'autotools']
  463. self._test_recipe_contents(recipefile, checkvars, inherits)
  464. def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
  465. dstdir = basedstdir
  466. self.assertTrue(os.path.exists(dstdir))
  467. for p in paths:
  468. dstdir = os.path.join(dstdir, p)
  469. if not os.path.exists(dstdir):
  470. os.makedirs(dstdir)
  471. if p == "lib":
  472. # Can race with other tests
  473. self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir)
  474. else:
  475. self.track_for_cleanup(dstdir)
  476. dstfile = os.path.join(dstdir, os.path.basename(srcfile))
  477. if srcfile != dstfile:
  478. shutil.copy(srcfile, dstfile)
  479. self.track_for_cleanup(dstfile)
  480. def test_recipetool_load_plugin(self):
  481. """Test that recipetool loads only the first found plugin in BBPATH."""
  482. recipetool = runCmd("which recipetool")
  483. fromname = runCmd("recipetool --quiet pluginfile")
  484. srcfile = fromname.output
  485. searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)]
  486. plugincontent = []
  487. with open(srcfile) as fh:
  488. plugincontent = fh.readlines()
  489. try:
  490. self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found')
  491. for path in searchpath:
  492. self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool')
  493. result = runCmd("recipetool --quiet count")
  494. self.assertEqual(result.output, '1')
  495. result = runCmd("recipetool --quiet multiloaded")
  496. self.assertEqual(result.output, "no")
  497. for path in searchpath:
  498. result = runCmd("recipetool --quiet bbdir")
  499. self.assertEqual(result.output, path)
  500. os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py'))
  501. finally:
  502. with open(srcfile, 'w') as fh:
  503. fh.writelines(plugincontent)
  504. class RecipetoolAppendsrcBase(RecipetoolBase):
  505. def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles):
  506. cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile)
  507. return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines)
  508. def _try_recipetool_appendsrcfiles(self, testrecipe, newfiles, expectedlines=None, expectedfiles=None, destdir=None, options=''):
  509. if destdir:
  510. options += ' -D %s' % destdir
  511. if expectedfiles is None:
  512. expectedfiles = [os.path.basename(f) for f in newfiles]
  513. cmd = 'recipetool appendsrcfiles %s %s %s %s' % (options, self.templayerdir, testrecipe, ' '.join(newfiles))
  514. return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines)
  515. def _try_recipetool_appendsrcfile_fail(self, testrecipe, newfile, destfile, checkerror):
  516. cmd = 'recipetool appendsrcfile %s %s %s %s' % (self.templayerdir, testrecipe, newfile, destfile or '')
  517. result = runCmd(cmd, ignore_status=True)
  518. self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd)
  519. self.assertNotIn('Traceback', result.output)
  520. for errorstr in checkerror:
  521. self.assertIn(errorstr, result.output)
  522. @staticmethod
  523. def _get_first_file_uri(recipe):
  524. '''Return the first file:// in SRC_URI for the specified recipe.'''
  525. src_uri = get_bb_var('SRC_URI', recipe).split()
  526. for uri in src_uri:
  527. p = urllib.parse.urlparse(uri)
  528. if p.scheme == 'file':
  529. return p.netloc + p.path
  530. def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''):
  531. if newfile is None:
  532. newfile = self.testfile
  533. if srcdir:
  534. if destdir:
  535. expected_subdir = os.path.join(srcdir, destdir)
  536. else:
  537. expected_subdir = srcdir
  538. else:
  539. options += " -W"
  540. expected_subdir = destdir
  541. if filename:
  542. if destdir:
  543. destpath = os.path.join(destdir, filename)
  544. else:
  545. destpath = filename
  546. else:
  547. filename = os.path.basename(newfile)
  548. if destdir:
  549. destpath = destdir + os.sep
  550. else:
  551. destpath = '.' + os.sep
  552. expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
  553. '\n']
  554. if has_src_uri:
  555. uri = 'file://%s' % filename
  556. if expected_subdir:
  557. uri += ';subdir=%s' % expected_subdir
  558. expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri,
  559. '\n']
  560. return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename])
  561. def _test_appendsrcfiles(self, testrecipe, newfiles, expectedfiles=None, destdir=None, options=''):
  562. if expectedfiles is None:
  563. expectedfiles = [os.path.basename(n) for n in newfiles]
  564. self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options)
  565. bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe)
  566. src_uri = bb_vars['SRC_URI'].split()
  567. for f in expectedfiles:
  568. if destdir:
  569. self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri)
  570. else:
  571. self.assertIn('file://%s' % f, src_uri)
  572. recipefile = bb_vars['FILE']
  573. bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir)
  574. filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe)
  575. filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':')
  576. self.assertIn(filesdir, filesextrapaths)
  577. class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase):
  578. def test_recipetool_appendsrcfile_basic(self):
  579. self._test_appendsrcfile('base-files', 'a-file')
  580. def test_recipetool_appendsrcfile_basic_wildcard(self):
  581. testrecipe = 'base-files'
  582. self._test_appendsrcfile(testrecipe, 'a-file', options='-w')
  583. recipefile = get_bb_var('FILE', testrecipe)
  584. bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir)
  585. self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe)
  586. def test_recipetool_appendsrcfile_subdir_basic(self):
  587. self._test_appendsrcfile('base-files', 'a-file', 'tmp')
  588. def test_recipetool_appendsrcfile_subdir_basic_dirdest(self):
  589. self._test_appendsrcfile('base-files', destdir='tmp')
  590. def test_recipetool_appendsrcfile_srcdir_basic(self):
  591. testrecipe = 'bash'
  592. bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe)
  593. srcdir = bb_vars['S']
  594. workdir = bb_vars['WORKDIR']
  595. subdir = os.path.relpath(srcdir, workdir)
  596. self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir)
  597. def test_recipetool_appendsrcfile_existing_in_src_uri(self):
  598. testrecipe = 'base-files'
  599. filepath = self._get_first_file_uri(testrecipe)
  600. self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe)
  601. self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False)
  602. def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self):
  603. testrecipe = 'base-files'
  604. subdir = 'tmp'
  605. filepath = self._get_first_file_uri(testrecipe)
  606. self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe)
  607. output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False)
  608. self.assertTrue(any('with different parameters' in l for l in output))
  609. def test_recipetool_appendsrcfile_replace_file_srcdir(self):
  610. testrecipe = 'bash'
  611. filepath = 'Makefile.in'
  612. bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe)
  613. srcdir = bb_vars['S']
  614. workdir = bb_vars['WORKDIR']
  615. subdir = os.path.relpath(srcdir, workdir)
  616. self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir)
  617. bitbake('%s:do_unpack' % testrecipe)
  618. with open(self.testfile, 'r') as testfile:
  619. with open(os.path.join(srcdir, filepath), 'r') as makefilein:
  620. self.assertEqual(testfile.read(), makefilein.read())
  621. def test_recipetool_appendsrcfiles_basic(self, destdir=None):
  622. newfiles = [self.testfile]
  623. for i in range(1, 5):
  624. testfile = os.path.join(self.tempdir, 'testfile%d' % i)
  625. with open(testfile, 'w') as f:
  626. f.write('Test file %d\n' % i)
  627. newfiles.append(testfile)
  628. self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W')
  629. def test_recipetool_appendsrcfiles_basic_subdir(self):
  630. self.test_recipetool_appendsrcfiles_basic(destdir='testdir')