fetch.py 113 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567
  1. #
  2. # BitBake Tests for the Fetcher (fetch2/)
  3. #
  4. # Copyright (C) 2012 Richard Purdie
  5. #
  6. # SPDX-License-Identifier: GPL-2.0-only
  7. #
  8. import unittest
  9. import hashlib
  10. import tempfile
  11. import collections
  12. import os
  13. from bb.fetch2 import URI
  14. from bb.fetch2 import FetchMethod
  15. import bb
  16. from bb.tests.support.httpserver import HTTPService
  17. def skipIfNoNetwork():
  18. if os.environ.get("BB_SKIP_NETTESTS") == "yes":
  19. return unittest.skip("Network tests being skipped")
  20. return lambda f: f
  21. class URITest(unittest.TestCase):
  22. test_uris = {
  23. "http://www.google.com/index.html" : {
  24. 'uri': 'http://www.google.com/index.html',
  25. 'scheme': 'http',
  26. 'hostname': 'www.google.com',
  27. 'port': None,
  28. 'hostport': 'www.google.com',
  29. 'path': '/index.html',
  30. 'userinfo': '',
  31. 'username': '',
  32. 'password': '',
  33. 'params': {},
  34. 'query': {},
  35. 'relative': False
  36. },
  37. "http://www.google.com/index.html;param1=value1" : {
  38. 'uri': 'http://www.google.com/index.html;param1=value1',
  39. 'scheme': 'http',
  40. 'hostname': 'www.google.com',
  41. 'port': None,
  42. 'hostport': 'www.google.com',
  43. 'path': '/index.html',
  44. 'userinfo': '',
  45. 'username': '',
  46. 'password': '',
  47. 'params': {
  48. 'param1': 'value1'
  49. },
  50. 'query': {},
  51. 'relative': False
  52. },
  53. "http://www.example.org/index.html?param1=value1" : {
  54. 'uri': 'http://www.example.org/index.html?param1=value1',
  55. 'scheme': 'http',
  56. 'hostname': 'www.example.org',
  57. 'port': None,
  58. 'hostport': 'www.example.org',
  59. 'path': '/index.html',
  60. 'userinfo': '',
  61. 'username': '',
  62. 'password': '',
  63. 'params': {},
  64. 'query': {
  65. 'param1': 'value1'
  66. },
  67. 'relative': False
  68. },
  69. "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
  70. 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
  71. 'scheme': 'http',
  72. 'hostname': 'www.example.org',
  73. 'port': None,
  74. 'hostport': 'www.example.org',
  75. 'path': '/index.html',
  76. 'userinfo': '',
  77. 'username': '',
  78. 'password': '',
  79. 'params': {
  80. 'param2': 'value2'
  81. },
  82. 'query': {
  83. 'qparam1': 'qvalue1'
  84. },
  85. 'relative': False
  86. },
  87. "http://www.example.com:8080/index.html" : {
  88. 'uri': 'http://www.example.com:8080/index.html',
  89. 'scheme': 'http',
  90. 'hostname': 'www.example.com',
  91. 'port': 8080,
  92. 'hostport': 'www.example.com:8080',
  93. 'path': '/index.html',
  94. 'userinfo': '',
  95. 'username': '',
  96. 'password': '',
  97. 'params': {},
  98. 'query': {},
  99. 'relative': False
  100. },
  101. "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
  102. 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
  103. 'scheme': 'cvs',
  104. 'hostname': 'cvs.handhelds.org',
  105. 'port': None,
  106. 'hostport': 'cvs.handhelds.org',
  107. 'path': '/cvs',
  108. 'userinfo': 'anoncvs',
  109. 'username': 'anoncvs',
  110. 'password': '',
  111. 'params': {
  112. 'module': 'familiar/dist/ipkg'
  113. },
  114. 'query': {},
  115. 'relative': False
  116. },
  117. "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
  118. 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
  119. 'scheme': 'cvs',
  120. 'hostname': 'cvs.handhelds.org',
  121. 'port': None,
  122. 'hostport': 'cvs.handhelds.org',
  123. 'path': '/cvs',
  124. 'userinfo': 'anoncvs:anonymous',
  125. 'username': 'anoncvs',
  126. 'password': 'anonymous',
  127. 'params': collections.OrderedDict([
  128. ('tag', 'V0-99-81'),
  129. ('module', 'familiar/dist/ipkg')
  130. ]),
  131. 'query': {},
  132. 'relative': False
  133. },
  134. "file://example.diff": { # NOTE: Not RFC compliant!
  135. 'uri': 'file:example.diff',
  136. 'scheme': 'file',
  137. 'hostname': '',
  138. 'port': None,
  139. 'hostport': '',
  140. 'path': 'example.diff',
  141. 'userinfo': '',
  142. 'username': '',
  143. 'password': '',
  144. 'params': {},
  145. 'query': {},
  146. 'relative': True
  147. },
  148. "file:example.diff": { # NOTE: RFC compliant version of the former
  149. 'uri': 'file:example.diff',
  150. 'scheme': 'file',
  151. 'hostname': '',
  152. 'port': None,
  153. 'hostport': '',
  154. 'path': 'example.diff',
  155. 'userinfo': '',
  156. 'userinfo': '',
  157. 'username': '',
  158. 'password': '',
  159. 'params': {},
  160. 'query': {},
  161. 'relative': True
  162. },
  163. "file:///tmp/example.diff": {
  164. 'uri': 'file:///tmp/example.diff',
  165. 'scheme': 'file',
  166. 'hostname': '',
  167. 'port': None,
  168. 'hostport': '',
  169. 'path': '/tmp/example.diff',
  170. 'userinfo': '',
  171. 'userinfo': '',
  172. 'username': '',
  173. 'password': '',
  174. 'params': {},
  175. 'query': {},
  176. 'relative': False
  177. },
  178. "git:///path/example.git": {
  179. 'uri': 'git:///path/example.git',
  180. 'scheme': 'git',
  181. 'hostname': '',
  182. 'port': None,
  183. 'hostport': '',
  184. 'path': '/path/example.git',
  185. 'userinfo': '',
  186. 'userinfo': '',
  187. 'username': '',
  188. 'password': '',
  189. 'params': {},
  190. 'query': {},
  191. 'relative': False
  192. },
  193. "git:path/example.git": {
  194. 'uri': 'git:path/example.git',
  195. 'scheme': 'git',
  196. 'hostname': '',
  197. 'port': None,
  198. 'hostport': '',
  199. 'path': 'path/example.git',
  200. 'userinfo': '',
  201. 'userinfo': '',
  202. 'username': '',
  203. 'password': '',
  204. 'params': {},
  205. 'query': {},
  206. 'relative': True
  207. },
  208. "git://example.net/path/example.git": {
  209. 'uri': 'git://example.net/path/example.git',
  210. 'scheme': 'git',
  211. 'hostname': 'example.net',
  212. 'port': None,
  213. 'hostport': 'example.net',
  214. 'path': '/path/example.git',
  215. 'userinfo': '',
  216. 'userinfo': '',
  217. 'username': '',
  218. 'password': '',
  219. 'params': {},
  220. 'query': {},
  221. 'relative': False
  222. },
  223. "git://tfs-example.org:22/tfs/example%20path/example.git": {
  224. 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
  225. 'scheme': 'git',
  226. 'hostname': 'tfs-example.org',
  227. 'port': 22,
  228. 'hostport': 'tfs-example.org:22',
  229. 'path': '/tfs/example path/example.git',
  230. 'userinfo': '',
  231. 'userinfo': '',
  232. 'username': '',
  233. 'password': '',
  234. 'params': {},
  235. 'query': {},
  236. 'relative': False
  237. },
  238. "http://somesite.net;someparam=1": {
  239. 'uri': 'http://somesite.net;someparam=1',
  240. 'scheme': 'http',
  241. 'hostname': 'somesite.net',
  242. 'port': None,
  243. 'hostport': 'somesite.net',
  244. 'path': '',
  245. 'userinfo': '',
  246. 'userinfo': '',
  247. 'username': '',
  248. 'password': '',
  249. 'params': {"someparam" : "1"},
  250. 'query': {},
  251. 'relative': False
  252. },
  253. "file://somelocation;someparam=1": {
  254. 'uri': 'file:somelocation;someparam=1',
  255. 'scheme': 'file',
  256. 'hostname': '',
  257. 'port': None,
  258. 'hostport': '',
  259. 'path': 'somelocation',
  260. 'userinfo': '',
  261. 'userinfo': '',
  262. 'username': '',
  263. 'password': '',
  264. 'params': {"someparam" : "1"},
  265. 'query': {},
  266. 'relative': True
  267. }
  268. }
  269. def test_uri(self):
  270. for test_uri, ref in self.test_uris.items():
  271. uri = URI(test_uri)
  272. self.assertEqual(str(uri), ref['uri'])
  273. # expected attributes
  274. self.assertEqual(uri.scheme, ref['scheme'])
  275. self.assertEqual(uri.userinfo, ref['userinfo'])
  276. self.assertEqual(uri.username, ref['username'])
  277. self.assertEqual(uri.password, ref['password'])
  278. self.assertEqual(uri.hostname, ref['hostname'])
  279. self.assertEqual(uri.port, ref['port'])
  280. self.assertEqual(uri.hostport, ref['hostport'])
  281. self.assertEqual(uri.path, ref['path'])
  282. self.assertEqual(uri.params, ref['params'])
  283. self.assertEqual(uri.relative, ref['relative'])
  284. def test_dict(self):
  285. for test in self.test_uris.values():
  286. uri = URI()
  287. self.assertEqual(uri.scheme, '')
  288. self.assertEqual(uri.userinfo, '')
  289. self.assertEqual(uri.username, '')
  290. self.assertEqual(uri.password, '')
  291. self.assertEqual(uri.hostname, '')
  292. self.assertEqual(uri.port, None)
  293. self.assertEqual(uri.path, '')
  294. self.assertEqual(uri.params, {})
  295. uri.scheme = test['scheme']
  296. self.assertEqual(uri.scheme, test['scheme'])
  297. uri.userinfo = test['userinfo']
  298. self.assertEqual(uri.userinfo, test['userinfo'])
  299. self.assertEqual(uri.username, test['username'])
  300. self.assertEqual(uri.password, test['password'])
  301. # make sure changing the values doesn't do anything unexpected
  302. uri.username = 'changeme'
  303. self.assertEqual(uri.username, 'changeme')
  304. self.assertEqual(uri.password, test['password'])
  305. uri.password = 'insecure'
  306. self.assertEqual(uri.username, 'changeme')
  307. self.assertEqual(uri.password, 'insecure')
  308. # reset back after our trickery
  309. uri.userinfo = test['userinfo']
  310. self.assertEqual(uri.userinfo, test['userinfo'])
  311. self.assertEqual(uri.username, test['username'])
  312. self.assertEqual(uri.password, test['password'])
  313. uri.hostname = test['hostname']
  314. self.assertEqual(uri.hostname, test['hostname'])
  315. self.assertEqual(uri.hostport, test['hostname'])
  316. uri.port = test['port']
  317. self.assertEqual(uri.port, test['port'])
  318. self.assertEqual(uri.hostport, test['hostport'])
  319. uri.path = test['path']
  320. self.assertEqual(uri.path, test['path'])
  321. uri.params = test['params']
  322. self.assertEqual(uri.params, test['params'])
  323. uri.query = test['query']
  324. self.assertEqual(uri.query, test['query'])
  325. self.assertEqual(str(uri), test['uri'])
  326. uri.params = {}
  327. self.assertEqual(uri.params, {})
  328. self.assertEqual(str(uri), (str(uri).split(";"))[0])
  329. class FetcherTest(unittest.TestCase):
  330. def setUp(self):
  331. self.origdir = os.getcwd()
  332. self.d = bb.data.init()
  333. self.tempdir = tempfile.mkdtemp()
  334. self.dldir = os.path.join(self.tempdir, "download")
  335. os.mkdir(self.dldir)
  336. self.d.setVar("DL_DIR", self.dldir)
  337. self.unpackdir = os.path.join(self.tempdir, "unpacked")
  338. os.mkdir(self.unpackdir)
  339. persistdir = os.path.join(self.tempdir, "persistdata")
  340. self.d.setVar("PERSISTENT_DIR", persistdir)
  341. def tearDown(self):
  342. os.chdir(self.origdir)
  343. if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
  344. print("Not cleaning up %s. Please remove manually." % self.tempdir)
  345. else:
  346. bb.utils.prunedir(self.tempdir)
  347. class MirrorUriTest(FetcherTest):
  348. replaceuris = {
  349. ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
  350. : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
  351. ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
  352. : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  353. ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
  354. : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  355. ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
  356. : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  357. ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
  358. : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
  359. ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
  360. : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
  361. ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
  362. : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
  363. ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
  364. : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
  365. ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
  366. : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
  367. ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
  368. : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
  369. ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
  370. : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
  371. ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
  372. : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  373. ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
  374. : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  375. ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
  376. : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
  377. ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
  378. : "http://somewhere2.org/somefile_1.2.3.tar.gz",
  379. ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
  380. : "http://somewhere2.org/somefile_1.2.3.tar.gz",
  381. ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
  382. : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
  383. #Renaming files doesn't work
  384. #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
  385. #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
  386. }
  387. mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
  388. "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
  389. "https://.*/.* file:///someotherpath/downloads/ \n" \
  390. "http://.*/.* file:///someotherpath/downloads/ \n"
  391. def test_urireplace(self):
  392. for k, v in self.replaceuris.items():
  393. ud = bb.fetch.FetchData(k[0], self.d)
  394. ud.setup_localpath(self.d)
  395. mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
  396. newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
  397. self.assertEqual([v], newuris)
  398. def test_urilist1(self):
  399. fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
  400. mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
  401. uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
  402. self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
  403. def test_urilist2(self):
  404. # Catch https:// -> files:// bug
  405. fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
  406. mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
  407. uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
  408. self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
  409. def test_mirror_of_mirror(self):
  410. # Test if mirror of a mirror works
  411. mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
  412. mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
  413. fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
  414. mirrors = bb.fetch2.mirror_from_string(mirrorvar)
  415. uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
  416. self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
  417. 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
  418. 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
  419. 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
  420. recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
  421. "https://.*/[^/]* https://BBBB/B/B/B/ \n"
  422. def test_recursive(self):
  423. fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
  424. mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
  425. uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
  426. self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
  427. 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
  428. 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
  429. class GitDownloadDirectoryNamingTest(FetcherTest):
  430. def setUp(self):
  431. super(GitDownloadDirectoryNamingTest, self).setUp()
  432. self.recipe_url = "git://git.openembedded.org/bitbake"
  433. self.recipe_dir = "git.openembedded.org.bitbake"
  434. self.mirror_url = "git://github.com/openembedded/bitbake.git"
  435. self.mirror_dir = "github.com.openembedded.bitbake.git"
  436. self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
  437. def setup_mirror_rewrite(self):
  438. self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
  439. @skipIfNoNetwork()
  440. def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
  441. self.setup_mirror_rewrite()
  442. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  443. fetcher.download()
  444. dir = os.listdir(self.dldir + "/git2")
  445. self.assertIn(self.recipe_dir, dir)
  446. @skipIfNoNetwork()
  447. def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
  448. self.setup_mirror_rewrite()
  449. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  450. fetcher.download()
  451. dir = os.listdir(self.dldir + "/git2")
  452. self.assertIn(self.mirror_dir, dir)
  453. self.assertIn(self.recipe_dir, dir)
  454. @skipIfNoNetwork()
  455. def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
  456. self.setup_mirror_rewrite()
  457. fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
  458. fetcher.download()
  459. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  460. fetcher.download()
  461. dir = os.listdir(self.dldir + "/git2")
  462. self.assertIn(self.mirror_dir, dir)
  463. self.assertIn(self.recipe_dir, dir)
  464. class TarballNamingTest(FetcherTest):
  465. def setUp(self):
  466. super(TarballNamingTest, self).setUp()
  467. self.recipe_url = "git://git.openembedded.org/bitbake"
  468. self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
  469. self.mirror_url = "git://github.com/openembedded/bitbake.git"
  470. self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
  471. self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
  472. self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
  473. def setup_mirror_rewrite(self):
  474. self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
  475. @skipIfNoNetwork()
  476. def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
  477. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  478. fetcher.download()
  479. dir = os.listdir(self.dldir)
  480. self.assertIn(self.recipe_tarball, dir)
  481. @skipIfNoNetwork()
  482. def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
  483. self.setup_mirror_rewrite()
  484. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  485. fetcher.download()
  486. dir = os.listdir(self.dldir)
  487. self.assertIn(self.mirror_tarball, dir)
  488. class GitShallowTarballNamingTest(FetcherTest):
  489. def setUp(self):
  490. super(GitShallowTarballNamingTest, self).setUp()
  491. self.recipe_url = "git://git.openembedded.org/bitbake"
  492. self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
  493. self.mirror_url = "git://github.com/openembedded/bitbake.git"
  494. self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
  495. self.d.setVar('BB_GIT_SHALLOW', '1')
  496. self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
  497. self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
  498. def setup_mirror_rewrite(self):
  499. self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
  500. @skipIfNoNetwork()
  501. def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
  502. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  503. fetcher.download()
  504. dir = os.listdir(self.dldir)
  505. self.assertIn(self.recipe_tarball, dir)
  506. @skipIfNoNetwork()
  507. def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
  508. self.setup_mirror_rewrite()
  509. fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
  510. fetcher.download()
  511. dir = os.listdir(self.dldir)
  512. self.assertIn(self.mirror_tarball, dir)
  513. class FetcherLocalTest(FetcherTest):
  514. def setUp(self):
  515. def touch(fn):
  516. with open(fn, 'a'):
  517. os.utime(fn, None)
  518. super(FetcherLocalTest, self).setUp()
  519. self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
  520. os.makedirs(self.localsrcdir)
  521. touch(os.path.join(self.localsrcdir, 'a'))
  522. touch(os.path.join(self.localsrcdir, 'b'))
  523. os.makedirs(os.path.join(self.localsrcdir, 'dir'))
  524. touch(os.path.join(self.localsrcdir, 'dir', 'c'))
  525. touch(os.path.join(self.localsrcdir, 'dir', 'd'))
  526. os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
  527. touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
  528. touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
  529. self.d.setVar("FILESPATH", self.localsrcdir)
  530. def fetchUnpack(self, uris):
  531. fetcher = bb.fetch.Fetch(uris, self.d)
  532. fetcher.download()
  533. fetcher.unpack(self.unpackdir)
  534. flst = []
  535. for root, dirs, files in os.walk(self.unpackdir):
  536. for f in files:
  537. flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
  538. flst.sort()
  539. return flst
  540. def test_local(self):
  541. tree = self.fetchUnpack(['file://a', 'file://dir/c'])
  542. self.assertEqual(tree, ['a', 'dir/c'])
  543. def test_local_backslash(self):
  544. tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
  545. self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
  546. def test_local_wildcard(self):
  547. with self.assertRaises(bb.fetch2.ParameterError):
  548. tree = self.fetchUnpack(['file://a', 'file://dir/*'])
  549. def test_local_dir(self):
  550. tree = self.fetchUnpack(['file://a', 'file://dir'])
  551. self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
  552. def test_local_subdir(self):
  553. tree = self.fetchUnpack(['file://dir/subdir'])
  554. self.assertEqual(tree, ['dir/subdir/e'])
  555. def test_local_subdir_file(self):
  556. tree = self.fetchUnpack(['file://dir/subdir/e'])
  557. self.assertEqual(tree, ['dir/subdir/e'])
  558. def test_local_subdirparam(self):
  559. tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
  560. self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
  561. def test_local_deepsubdirparam(self):
  562. tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
  563. self.assertEqual(tree, ['bar/dir/subdir/e'])
  564. def test_local_absolutedir(self):
  565. # Unpacking to an absolute path that is a subdirectory of the root
  566. # should work
  567. tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
  568. # Unpacking to an absolute path outside of the root should fail
  569. with self.assertRaises(bb.fetch2.UnpackError):
  570. self.fetchUnpack(['file://a;subdir=/bin/sh'])
  571. class FetcherNoNetworkTest(FetcherTest):
  572. def setUp(self):
  573. super().setUp()
  574. # all test cases are based on not having network
  575. self.d.setVar("BB_NO_NETWORK", "1")
  576. def test_missing(self):
  577. string = "this is a test file\n".encode("utf-8")
  578. self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
  579. self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
  580. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  581. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  582. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  583. with self.assertRaises(bb.fetch2.NetworkAccess):
  584. fetcher.download()
  585. def test_valid_missing_donestamp(self):
  586. # create the file in the download directory with correct hash
  587. string = "this is a test file\n".encode("utf-8")
  588. with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
  589. f.write(string)
  590. self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
  591. self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
  592. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  593. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  594. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  595. fetcher.download()
  596. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  597. def test_invalid_missing_donestamp(self):
  598. # create an invalid file in the download directory with incorrect hash
  599. string = "this is a test file\n".encode("utf-8")
  600. with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
  601. pass
  602. self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
  603. self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
  604. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  605. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  606. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  607. with self.assertRaises(bb.fetch2.NetworkAccess):
  608. fetcher.download()
  609. # the existing file should not exist or should have be moved to "bad-checksum"
  610. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  611. def test_nochecksums_missing(self):
  612. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  613. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  614. # ssh fetch does not support checksums
  615. fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  616. # attempts to download with missing donestamp
  617. with self.assertRaises(bb.fetch2.NetworkAccess):
  618. fetcher.download()
  619. def test_nochecksums_missing_donestamp(self):
  620. # create a file in the download directory
  621. with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
  622. pass
  623. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  624. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  625. # ssh fetch does not support checksums
  626. fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  627. # attempts to download with missing donestamp
  628. with self.assertRaises(bb.fetch2.NetworkAccess):
  629. fetcher.download()
  630. def test_nochecksums_has_donestamp(self):
  631. # create a file in the download directory with the donestamp
  632. with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
  633. pass
  634. with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
  635. pass
  636. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  637. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  638. # ssh fetch does not support checksums
  639. fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  640. # should not fetch
  641. fetcher.download()
  642. # both files should still exist
  643. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  644. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  645. def test_nochecksums_missing_has_donestamp(self):
  646. # create a file in the download directory with the donestamp
  647. with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
  648. pass
  649. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  650. self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  651. # ssh fetch does not support checksums
  652. fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
  653. with self.assertRaises(bb.fetch2.NetworkAccess):
  654. fetcher.download()
  655. # both files should still exist
  656. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
  657. self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
  658. class FetcherNetworkTest(FetcherTest):
  659. @skipIfNoNetwork()
  660. def test_fetch(self):
  661. fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
  662. fetcher.download()
  663. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
  664. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
  665. self.d.setVar("BB_NO_NETWORK", "1")
  666. fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
  667. fetcher.download()
  668. fetcher.unpack(self.unpackdir)
  669. self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
  670. self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
  671. @skipIfNoNetwork()
  672. def test_fetch_mirror(self):
  673. self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
  674. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
  675. fetcher.download()
  676. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
  677. @skipIfNoNetwork()
  678. def test_fetch_mirror_of_mirror(self):
  679. self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
  680. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
  681. fetcher.download()
  682. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
  683. @skipIfNoNetwork()
  684. def test_fetch_file_mirror_of_mirror(self):
  685. self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
  686. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
  687. os.mkdir(self.dldir + "/some2where")
  688. fetcher.download()
  689. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
  690. @skipIfNoNetwork()
  691. def test_fetch_premirror(self):
  692. self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
  693. fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
  694. fetcher.download()
  695. self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
  696. @skipIfNoNetwork()
  697. def gitfetcher(self, url1, url2):
  698. def checkrevision(self, fetcher):
  699. fetcher.unpack(self.unpackdir)
  700. revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
  701. self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
  702. self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
  703. self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
  704. fetcher = bb.fetch.Fetch([url1], self.d)
  705. fetcher.download()
  706. checkrevision(self, fetcher)
  707. # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
  708. bb.utils.prunedir(self.dldir + "/git2/")
  709. bb.utils.prunedir(self.unpackdir)
  710. self.d.setVar("BB_NO_NETWORK", "1")
  711. fetcher = bb.fetch.Fetch([url2], self.d)
  712. fetcher.download()
  713. checkrevision(self, fetcher)
  714. @skipIfNoNetwork()
  715. def test_gitfetch(self):
  716. url1 = url2 = "git://git.openembedded.org/bitbake"
  717. self.gitfetcher(url1, url2)
  718. @skipIfNoNetwork()
  719. def test_gitfetch_goodsrcrev(self):
  720. # SRCREV is set but matches rev= parameter
  721. url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
  722. self.gitfetcher(url1, url2)
  723. @skipIfNoNetwork()
  724. def test_gitfetch_badsrcrev(self):
  725. # SRCREV is set but does not match rev= parameter
  726. url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
  727. self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
  728. @skipIfNoNetwork()
  729. def test_gitfetch_tagandrev(self):
  730. # SRCREV is set but does not match rev= parameter
  731. url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
  732. self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
  733. @skipIfNoNetwork()
  734. def test_gitfetch_localusehead(self):
  735. # Create dummy local Git repo
  736. src_dir = tempfile.mkdtemp(dir=self.tempdir,
  737. prefix='gitfetch_localusehead_')
  738. src_dir = os.path.abspath(src_dir)
  739. bb.process.run("git init", cwd=src_dir)
  740. bb.process.run("git commit --allow-empty -m'Dummy commit'",
  741. cwd=src_dir)
  742. # Use other branch than master
  743. bb.process.run("git checkout -b my-devel", cwd=src_dir)
  744. bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
  745. cwd=src_dir)
  746. stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
  747. orig_rev = stdout[0].strip()
  748. # Fetch and check revision
  749. self.d.setVar("SRCREV", "AUTOINC")
  750. url = "git://" + src_dir + ";protocol=file;usehead=1"
  751. fetcher = bb.fetch.Fetch([url], self.d)
  752. fetcher.download()
  753. fetcher.unpack(self.unpackdir)
  754. stdout = bb.process.run("git rev-parse HEAD",
  755. cwd=os.path.join(self.unpackdir, 'git'))
  756. unpack_rev = stdout[0].strip()
  757. self.assertEqual(orig_rev, unpack_rev)
  758. @skipIfNoNetwork()
  759. def test_gitfetch_remoteusehead(self):
  760. url = "git://git.openembedded.org/bitbake;usehead=1"
  761. self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
  762. @skipIfNoNetwork()
  763. def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
  764. recipeurl = "git://git.openembedded.org/bitbake"
  765. mirrorurl = "git://someserver.org/bitbake"
  766. self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
  767. self.gitfetcher(recipeurl, mirrorurl)
  768. @skipIfNoNetwork()
  769. def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
  770. recipeurl = "git://someserver.org/bitbake"
  771. self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
  772. self.gitfetcher(recipeurl, recipeurl)
  773. @skipIfNoNetwork()
  774. def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
  775. realurl = "git://git.openembedded.org/bitbake"
  776. recipeurl = "git://someserver.org/bitbake"
  777. self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
  778. os.chdir(self.tempdir)
  779. bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
  780. self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
  781. self.gitfetcher(recipeurl, recipeurl)
  782. @skipIfNoNetwork()
  783. def test_git_submodule(self):
  784. # URL with ssh submodules
  785. url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
  786. # Original URL (comment this if you have ssh access to git.yoctoproject.org)
  787. url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
  788. fetcher = bb.fetch.Fetch([url], self.d)
  789. fetcher.download()
  790. # Previous cwd has been deleted
  791. os.chdir(os.path.dirname(self.unpackdir))
  792. fetcher.unpack(self.unpackdir)
  793. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  794. self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
  795. self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
  796. self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
  797. # Only when we're running the extended test with a submodule's submodule, can we check this.
  798. if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
  799. self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
  800. @skipIfNoNetwork()
  801. def test_git_submodule_dbus_broker(self):
  802. # The following external repositories have show failures in fetch and unpack operations
  803. # We want to avoid regressions!
  804. url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
  805. fetcher = bb.fetch.Fetch([url], self.d)
  806. fetcher.download()
  807. # Previous cwd has been deleted
  808. os.chdir(os.path.dirname(self.unpackdir))
  809. fetcher.unpack(self.unpackdir)
  810. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  811. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
  812. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
  813. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
  814. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
  815. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
  816. @skipIfNoNetwork()
  817. def test_git_submodule_CLI11(self):
  818. url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
  819. fetcher = bb.fetch.Fetch([url], self.d)
  820. fetcher.download()
  821. # Previous cwd has been deleted
  822. os.chdir(os.path.dirname(self.unpackdir))
  823. fetcher.unpack(self.unpackdir)
  824. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  825. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
  826. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
  827. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
  828. @skipIfNoNetwork()
  829. def test_git_submodule_update_CLI11(self):
  830. """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
  831. url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
  832. fetcher = bb.fetch.Fetch([url], self.d)
  833. fetcher.download()
  834. # CLI11 that pulls in a newer nlohmann-json
  835. url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
  836. fetcher = bb.fetch.Fetch([url], self.d)
  837. fetcher.download()
  838. # Previous cwd has been deleted
  839. os.chdir(os.path.dirname(self.unpackdir))
  840. fetcher.unpack(self.unpackdir)
  841. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  842. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
  843. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
  844. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
  845. @skipIfNoNetwork()
  846. def test_git_submodule_aktualizr(self):
  847. url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
  848. fetcher = bb.fetch.Fetch([url], self.d)
  849. fetcher.download()
  850. # Previous cwd has been deleted
  851. os.chdir(os.path.dirname(self.unpackdir))
  852. fetcher.unpack(self.unpackdir)
  853. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  854. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
  855. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
  856. self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
  857. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
  858. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
  859. self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
  860. @skipIfNoNetwork()
  861. def test_git_submodule_iotedge(self):
  862. """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
  863. # This repository also has submodules where the module (name), path and url do not align
  864. url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
  865. fetcher = bb.fetch.Fetch([url], self.d)
  866. fetcher.download()
  867. # Previous cwd has been deleted
  868. os.chdir(os.path.dirname(self.unpackdir))
  869. fetcher.unpack(self.unpackdir)
  870. repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
  871. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
  872. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
  873. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
  874. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
  875. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
  876. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
  877. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
  878. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
  879. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
  880. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
  881. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
  882. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
  883. self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
  884. class SVNTest(FetcherTest):
  885. def skipIfNoSvn():
  886. import shutil
  887. if not shutil.which("svn"):
  888. return unittest.skip("svn not installed, tests being skipped")
  889. if not shutil.which("svnadmin"):
  890. return unittest.skip("svnadmin not installed, tests being skipped")
  891. return lambda f: f
  892. @skipIfNoSvn()
  893. def setUp(self):
  894. """ Create a local repository """
  895. super(SVNTest, self).setUp()
  896. # Create something we can fetch
  897. src_dir = tempfile.mkdtemp(dir=self.tempdir,
  898. prefix='svnfetch_srcdir_')
  899. src_dir = os.path.abspath(src_dir)
  900. bb.process.run("echo readme > README.md", cwd=src_dir)
  901. # Store it in a local SVN repository
  902. repo_dir = tempfile.mkdtemp(dir=self.tempdir,
  903. prefix='svnfetch_localrepo_')
  904. repo_dir = os.path.abspath(repo_dir)
  905. bb.process.run("svnadmin create project", cwd=repo_dir)
  906. self.repo_url = "file://%s/project" % repo_dir
  907. bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
  908. cwd=repo_dir)
  909. bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
  910. # Github will emulate SVN. Use this to check if we're downloding...
  911. bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
  912. cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
  913. bb.process.run("svn commit --non-interactive -m 'Add external'",
  914. cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
  915. self.src_dir = src_dir
  916. self.repo_dir = repo_dir
  917. @skipIfNoSvn()
  918. def tearDown(self):
  919. os.chdir(self.origdir)
  920. if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
  921. print("Not cleaning up %s. Please remove manually." % self.tempdir)
  922. else:
  923. bb.utils.prunedir(self.tempdir)
  924. @skipIfNoSvn()
  925. @skipIfNoNetwork()
  926. def test_noexternal_svn(self):
  927. # Always match the rev count from setUp (currently rev 2)
  928. url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
  929. fetcher = bb.fetch.Fetch([url], self.d)
  930. fetcher.download()
  931. os.chdir(os.path.dirname(self.unpackdir))
  932. fetcher.unpack(self.unpackdir)
  933. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
  934. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
  935. self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
  936. self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
  937. @skipIfNoSvn()
  938. def test_external_svn(self):
  939. # Always match the rev count from setUp (currently rev 2)
  940. url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
  941. fetcher = bb.fetch.Fetch([url], self.d)
  942. fetcher.download()
  943. os.chdir(os.path.dirname(self.unpackdir))
  944. fetcher.unpack(self.unpackdir)
  945. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
  946. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
  947. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
  948. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
  949. class TrustedNetworksTest(FetcherTest):
  950. def test_trusted_network(self):
  951. # Ensure trusted_network returns False when the host IS in the list.
  952. url = "git://Someserver.org/foo;rev=1"
  953. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
  954. self.assertTrue(bb.fetch.trusted_network(self.d, url))
  955. def test_wild_trusted_network(self):
  956. # Ensure trusted_network returns true when the *.host IS in the list.
  957. url = "git://Someserver.org/foo;rev=1"
  958. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
  959. self.assertTrue(bb.fetch.trusted_network(self.d, url))
  960. def test_prefix_wild_trusted_network(self):
  961. # Ensure trusted_network returns true when the prefix matches *.host.
  962. url = "git://git.Someserver.org/foo;rev=1"
  963. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
  964. self.assertTrue(bb.fetch.trusted_network(self.d, url))
  965. def test_two_prefix_wild_trusted_network(self):
  966. # Ensure trusted_network returns true when the prefix matches *.host.
  967. url = "git://something.git.Someserver.org/foo;rev=1"
  968. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
  969. self.assertTrue(bb.fetch.trusted_network(self.d, url))
  970. def test_port_trusted_network(self):
  971. # Ensure trusted_network returns True, even if the url specifies a port.
  972. url = "git://someserver.org:8080/foo;rev=1"
  973. self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
  974. self.assertTrue(bb.fetch.trusted_network(self.d, url))
  975. def test_untrusted_network(self):
  976. # Ensure trusted_network returns False when the host is NOT in the list.
  977. url = "git://someserver.org/foo;rev=1"
  978. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
  979. self.assertFalse(bb.fetch.trusted_network(self.d, url))
  980. def test_wild_untrusted_network(self):
  981. # Ensure trusted_network returns False when the host is NOT in the list.
  982. url = "git://*.someserver.org/foo;rev=1"
  983. self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
  984. self.assertFalse(bb.fetch.trusted_network(self.d, url))
  985. class URLHandle(unittest.TestCase):
  986. datatable = {
  987. "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
  988. "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
  989. "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
  990. "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
  991. "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
  992. }
  993. # we require a pathname to encodeurl but users can still pass such urls to
  994. # decodeurl and we need to handle them
  995. decodedata = datatable.copy()
  996. decodedata.update({
  997. "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
  998. })
  999. def test_decodeurl(self):
  1000. for k, v in self.decodedata.items():
  1001. result = bb.fetch.decodeurl(k)
  1002. self.assertEqual(result, v)
  1003. def test_encodeurl(self):
  1004. for k, v in self.datatable.items():
  1005. result = bb.fetch.encodeurl(v)
  1006. self.assertEqual(result, k)
  1007. class FetchLatestVersionTest(FetcherTest):
  1008. test_git_uris = {
  1009. # version pattern "X.Y.Z"
  1010. ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
  1011. : "1.99.4",
  1012. # version pattern "vX.Y"
  1013. # mirror of git.infradead.org since network issues interfered with testing
  1014. ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
  1015. : "1.5.0",
  1016. # version pattern "pkg_name-X.Y"
  1017. # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
  1018. ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
  1019. : "1.0",
  1020. # version pattern "pkg_name-vX.Y.Z"
  1021. ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
  1022. : "1.4.0",
  1023. # combination version pattern
  1024. ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
  1025. : "1.2.0",
  1026. ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
  1027. : "2014.01",
  1028. # version pattern "yyyymmdd"
  1029. ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
  1030. : "20120614",
  1031. # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
  1032. # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
  1033. ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
  1034. : "0.4.3",
  1035. ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
  1036. : "11.0.0",
  1037. ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
  1038. : "1.3.59",
  1039. ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
  1040. : "3.82+dbg0.9",
  1041. }
  1042. test_wget_uris = {
  1043. #
  1044. # packages with versions inside directory name
  1045. #
  1046. # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
  1047. ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
  1048. : "2.24.2",
  1049. # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
  1050. ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
  1051. : "1.6.0",
  1052. # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
  1053. ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
  1054. : "2.8.12.1",
  1055. #
  1056. # packages with versions only in current directory
  1057. #
  1058. # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
  1059. ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
  1060. : "2.19",
  1061. # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
  1062. ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
  1063. : "20120814",
  1064. #
  1065. # packages with "99" in the name of possible version
  1066. #
  1067. # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
  1068. ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
  1069. : "5.0",
  1070. # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
  1071. ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
  1072. : "1.15.1",
  1073. #
  1074. # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
  1075. #
  1076. # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
  1077. # https://github.com/apple/cups/releases
  1078. ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
  1079. : "2.0.0",
  1080. # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
  1081. # http://ftp.debian.org/debian/pool/main/d/db5.3/
  1082. ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
  1083. : "5.3.10",
  1084. }
  1085. @skipIfNoNetwork()
  1086. def test_git_latest_versionstring(self):
  1087. for k, v in self.test_git_uris.items():
  1088. self.d.setVar("PN", k[0])
  1089. self.d.setVar("SRCREV", k[2])
  1090. self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
  1091. ud = bb.fetch2.FetchData(k[1], self.d)
  1092. pupver= ud.method.latest_versionstring(ud, self.d)
  1093. verstring = pupver[0]
  1094. self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
  1095. r = bb.utils.vercmp_string(v, verstring)
  1096. self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
  1097. def test_wget_latest_versionstring(self):
  1098. testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
  1099. server = HTTPService(testdata)
  1100. server.start()
  1101. port = server.port
  1102. try:
  1103. for k, v in self.test_wget_uris.items():
  1104. self.d.setVar("PN", k[0])
  1105. checkuri = ""
  1106. if k[2]:
  1107. checkuri = "http://localhost:%s/" % port + k[2]
  1108. self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
  1109. self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
  1110. url = "http://localhost:%s/" % port + k[1]
  1111. ud = bb.fetch2.FetchData(url, self.d)
  1112. pupver = ud.method.latest_versionstring(ud, self.d)
  1113. verstring = pupver[0]
  1114. self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
  1115. r = bb.utils.vercmp_string(v, verstring)
  1116. self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
  1117. finally:
  1118. server.stop()
  1119. class FetchCheckStatusTest(FetcherTest):
  1120. test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
  1121. "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
  1122. "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
  1123. "https://yoctoproject.org/",
  1124. "https://yoctoproject.org/documentation",
  1125. "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
  1126. "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
  1127. "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
  1128. "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
  1129. "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
  1130. "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
  1131. # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
  1132. "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
  1133. ]
  1134. @skipIfNoNetwork()
  1135. def test_wget_checkstatus(self):
  1136. fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
  1137. for u in self.test_wget_uris:
  1138. with self.subTest(url=u):
  1139. ud = fetch.ud[u]
  1140. m = ud.method
  1141. ret = m.checkstatus(fetch, ud, self.d)
  1142. self.assertTrue(ret, msg="URI %s, can't check status" % (u))
  1143. @skipIfNoNetwork()
  1144. def test_wget_checkstatus_connection_cache(self):
  1145. from bb.fetch2 import FetchConnectionCache
  1146. connection_cache = FetchConnectionCache()
  1147. fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
  1148. connection_cache = connection_cache)
  1149. for u in self.test_wget_uris:
  1150. with self.subTest(url=u):
  1151. ud = fetch.ud[u]
  1152. m = ud.method
  1153. ret = m.checkstatus(fetch, ud, self.d)
  1154. self.assertTrue(ret, msg="URI %s, can't check status" % (u))
  1155. connection_cache.close_connections()
  1156. class GitMakeShallowTest(FetcherTest):
  1157. def setUp(self):
  1158. FetcherTest.setUp(self)
  1159. self.gitdir = os.path.join(self.tempdir, 'gitshallow')
  1160. bb.utils.mkdirhier(self.gitdir)
  1161. bb.process.run('git init', cwd=self.gitdir)
  1162. def assertRefs(self, expected_refs):
  1163. actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
  1164. full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
  1165. self.assertEqual(sorted(full_expected), sorted(actual_refs))
  1166. def assertRevCount(self, expected_count, args=None):
  1167. if args is None:
  1168. args = ['HEAD']
  1169. revs = self.git(['rev-list'] + args)
  1170. actual_count = len(revs.splitlines())
  1171. self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
  1172. def git(self, cmd):
  1173. if isinstance(cmd, str):
  1174. cmd = 'git ' + cmd
  1175. else:
  1176. cmd = ['git'] + cmd
  1177. return bb.process.run(cmd, cwd=self.gitdir)[0]
  1178. def make_shallow(self, args=None):
  1179. if args is None:
  1180. args = ['HEAD']
  1181. return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
  1182. def add_empty_file(self, path, msg=None):
  1183. if msg is None:
  1184. msg = path
  1185. open(os.path.join(self.gitdir, path), 'w').close()
  1186. self.git(['add', path])
  1187. self.git(['commit', '-m', msg, path])
  1188. def test_make_shallow_single_branch_no_merge(self):
  1189. self.add_empty_file('a')
  1190. self.add_empty_file('b')
  1191. self.assertRevCount(2)
  1192. self.make_shallow()
  1193. self.assertRevCount(1)
  1194. def test_make_shallow_single_branch_one_merge(self):
  1195. self.add_empty_file('a')
  1196. self.add_empty_file('b')
  1197. self.git('checkout -b a_branch')
  1198. self.add_empty_file('c')
  1199. self.git('checkout master')
  1200. self.add_empty_file('d')
  1201. self.git('merge --no-ff --no-edit a_branch')
  1202. self.git('branch -d a_branch')
  1203. self.add_empty_file('e')
  1204. self.assertRevCount(6)
  1205. self.make_shallow(['HEAD~2'])
  1206. self.assertRevCount(5)
  1207. def test_make_shallow_at_merge(self):
  1208. self.add_empty_file('a')
  1209. self.git('checkout -b a_branch')
  1210. self.add_empty_file('b')
  1211. self.git('checkout master')
  1212. self.git('merge --no-ff --no-edit a_branch')
  1213. self.git('branch -d a_branch')
  1214. self.assertRevCount(3)
  1215. self.make_shallow()
  1216. self.assertRevCount(1)
  1217. def test_make_shallow_annotated_tag(self):
  1218. self.add_empty_file('a')
  1219. self.add_empty_file('b')
  1220. self.git('tag -a -m a_tag a_tag')
  1221. self.assertRevCount(2)
  1222. self.make_shallow(['a_tag'])
  1223. self.assertRevCount(1)
  1224. def test_make_shallow_multi_ref(self):
  1225. self.add_empty_file('a')
  1226. self.add_empty_file('b')
  1227. self.git('checkout -b a_branch')
  1228. self.add_empty_file('c')
  1229. self.git('checkout master')
  1230. self.add_empty_file('d')
  1231. self.git('checkout -b a_branch_2')
  1232. self.add_empty_file('a_tag')
  1233. self.git('tag a_tag')
  1234. self.git('checkout master')
  1235. self.git('branch -D a_branch_2')
  1236. self.add_empty_file('e')
  1237. self.assertRevCount(6, ['--all'])
  1238. self.make_shallow()
  1239. self.assertRevCount(5, ['--all'])
  1240. def test_make_shallow_multi_ref_trim(self):
  1241. self.add_empty_file('a')
  1242. self.git('checkout -b a_branch')
  1243. self.add_empty_file('c')
  1244. self.git('checkout master')
  1245. self.assertRevCount(1)
  1246. self.assertRevCount(2, ['--all'])
  1247. self.assertRefs(['master', 'a_branch'])
  1248. self.make_shallow(['-r', 'master', 'HEAD'])
  1249. self.assertRevCount(1, ['--all'])
  1250. self.assertRefs(['master'])
  1251. def test_make_shallow_noop(self):
  1252. self.add_empty_file('a')
  1253. self.assertRevCount(1)
  1254. self.make_shallow()
  1255. self.assertRevCount(1)
  1256. @skipIfNoNetwork()
  1257. def test_make_shallow_bitbake(self):
  1258. self.git('remote add origin https://github.com/openembedded/bitbake')
  1259. self.git('fetch --tags origin')
  1260. orig_revs = len(self.git('rev-list --all').splitlines())
  1261. self.make_shallow(['refs/tags/1.10.0'])
  1262. self.assertRevCount(orig_revs - 1746, ['--all'])
  1263. class GitShallowTest(FetcherTest):
  1264. def setUp(self):
  1265. FetcherTest.setUp(self)
  1266. self.gitdir = os.path.join(self.tempdir, 'git')
  1267. self.srcdir = os.path.join(self.tempdir, 'gitsource')
  1268. bb.utils.mkdirhier(self.srcdir)
  1269. self.git('init', cwd=self.srcdir)
  1270. self.d.setVar('WORKDIR', self.tempdir)
  1271. self.d.setVar('S', self.gitdir)
  1272. self.d.delVar('PREMIRRORS')
  1273. self.d.delVar('MIRRORS')
  1274. uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
  1275. self.d.setVar('SRC_URI', uri)
  1276. self.d.setVar('SRCREV', '${AUTOREV}')
  1277. self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
  1278. self.d.setVar('BB_GIT_SHALLOW', '1')
  1279. self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
  1280. self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
  1281. def assertRefs(self, expected_refs, cwd=None):
  1282. if cwd is None:
  1283. cwd = self.gitdir
  1284. actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
  1285. full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
  1286. self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
  1287. def assertRevCount(self, expected_count, args=None, cwd=None):
  1288. if args is None:
  1289. args = ['HEAD']
  1290. if cwd is None:
  1291. cwd = self.gitdir
  1292. revs = self.git(['rev-list'] + args, cwd=cwd)
  1293. actual_count = len(revs.splitlines())
  1294. self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
  1295. def git(self, cmd, cwd=None):
  1296. if isinstance(cmd, str):
  1297. cmd = 'git ' + cmd
  1298. else:
  1299. cmd = ['git'] + cmd
  1300. if cwd is None:
  1301. cwd = self.gitdir
  1302. return bb.process.run(cmd, cwd=cwd)[0]
  1303. def add_empty_file(self, path, cwd=None, msg=None):
  1304. if msg is None:
  1305. msg = path
  1306. if cwd is None:
  1307. cwd = self.srcdir
  1308. open(os.path.join(cwd, path), 'w').close()
  1309. self.git(['add', path], cwd)
  1310. self.git(['commit', '-m', msg, path], cwd)
  1311. def fetch(self, uri=None):
  1312. if uri is None:
  1313. uris = self.d.getVar('SRC_URI').split()
  1314. uri = uris[0]
  1315. d = self.d
  1316. else:
  1317. d = self.d.createCopy()
  1318. d.setVar('SRC_URI', uri)
  1319. uri = d.expand(uri)
  1320. uris = [uri]
  1321. fetcher = bb.fetch2.Fetch(uris, d)
  1322. fetcher.download()
  1323. ud = fetcher.ud[uri]
  1324. return fetcher, ud
  1325. def fetch_and_unpack(self, uri=None):
  1326. fetcher, ud = self.fetch(uri)
  1327. fetcher.unpack(self.d.getVar('WORKDIR'))
  1328. assert os.path.exists(self.d.getVar('S'))
  1329. return fetcher, ud
  1330. def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
  1331. """Fetch a uri, generating a shallow tarball, then unpack using it"""
  1332. fetcher, ud = self.fetch_and_unpack(uri)
  1333. assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
  1334. # Confirm that the unpacked repo is unshallow
  1335. if not disabled:
  1336. assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
  1337. # fetch and unpack, from the shallow tarball
  1338. bb.utils.remove(self.gitdir, recurse=True)
  1339. bb.utils.remove(ud.clonedir, recurse=True)
  1340. bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
  1341. # confirm that the unpacked repo is used when no git clone or git
  1342. # mirror tarball is available
  1343. fetcher, ud = self.fetch_and_unpack(uri)
  1344. if not disabled:
  1345. assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
  1346. else:
  1347. assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
  1348. return fetcher, ud
  1349. def test_shallow_disabled(self):
  1350. self.add_empty_file('a')
  1351. self.add_empty_file('b')
  1352. self.assertRevCount(2, cwd=self.srcdir)
  1353. self.d.setVar('BB_GIT_SHALLOW', '0')
  1354. self.fetch_shallow(disabled=True)
  1355. self.assertRevCount(2)
  1356. def test_shallow_nobranch(self):
  1357. self.add_empty_file('a')
  1358. self.add_empty_file('b')
  1359. self.assertRevCount(2, cwd=self.srcdir)
  1360. srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
  1361. self.d.setVar('SRCREV', srcrev)
  1362. uri = self.d.getVar('SRC_URI').split()[0]
  1363. uri = '%s;nobranch=1;bare=1' % uri
  1364. self.fetch_shallow(uri)
  1365. self.assertRevCount(1)
  1366. # shallow refs are used to ensure the srcrev sticks around when we
  1367. # have no other branches referencing it
  1368. self.assertRefs(['refs/shallow/default'])
  1369. def test_shallow_default_depth_1(self):
  1370. # Create initial git repo
  1371. self.add_empty_file('a')
  1372. self.add_empty_file('b')
  1373. self.assertRevCount(2, cwd=self.srcdir)
  1374. self.fetch_shallow()
  1375. self.assertRevCount(1)
  1376. def test_shallow_depth_0_disables(self):
  1377. self.add_empty_file('a')
  1378. self.add_empty_file('b')
  1379. self.assertRevCount(2, cwd=self.srcdir)
  1380. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1381. self.fetch_shallow(disabled=True)
  1382. self.assertRevCount(2)
  1383. def test_shallow_depth_default_override(self):
  1384. self.add_empty_file('a')
  1385. self.add_empty_file('b')
  1386. self.assertRevCount(2, cwd=self.srcdir)
  1387. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
  1388. self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
  1389. self.fetch_shallow()
  1390. self.assertRevCount(1)
  1391. def test_shallow_depth_default_override_disable(self):
  1392. self.add_empty_file('a')
  1393. self.add_empty_file('b')
  1394. self.add_empty_file('c')
  1395. self.assertRevCount(3, cwd=self.srcdir)
  1396. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1397. self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
  1398. self.fetch_shallow()
  1399. self.assertRevCount(2)
  1400. def test_current_shallow_out_of_date_clone(self):
  1401. # Create initial git repo
  1402. self.add_empty_file('a')
  1403. self.add_empty_file('b')
  1404. self.add_empty_file('c')
  1405. self.assertRevCount(3, cwd=self.srcdir)
  1406. # Clone and generate mirror tarball
  1407. fetcher, ud = self.fetch()
  1408. # Ensure we have a current mirror tarball, but an out of date clone
  1409. self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
  1410. self.assertRevCount(2, cwd=ud.clonedir)
  1411. # Fetch and unpack, from the current tarball, not the out of date clone
  1412. bb.utils.remove(self.gitdir, recurse=True)
  1413. fetcher, ud = self.fetch()
  1414. fetcher.unpack(self.d.getVar('WORKDIR'))
  1415. self.assertRevCount(1)
  1416. def test_shallow_single_branch_no_merge(self):
  1417. self.add_empty_file('a')
  1418. self.add_empty_file('b')
  1419. self.assertRevCount(2, cwd=self.srcdir)
  1420. self.fetch_shallow()
  1421. self.assertRevCount(1)
  1422. assert os.path.exists(os.path.join(self.gitdir, 'a'))
  1423. assert os.path.exists(os.path.join(self.gitdir, 'b'))
  1424. def test_shallow_no_dangling(self):
  1425. self.add_empty_file('a')
  1426. self.add_empty_file('b')
  1427. self.assertRevCount(2, cwd=self.srcdir)
  1428. self.fetch_shallow()
  1429. self.assertRevCount(1)
  1430. assert not self.git('fsck --dangling')
  1431. def test_shallow_srcrev_branch_truncation(self):
  1432. self.add_empty_file('a')
  1433. self.add_empty_file('b')
  1434. b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
  1435. self.add_empty_file('c')
  1436. self.assertRevCount(3, cwd=self.srcdir)
  1437. self.d.setVar('SRCREV', b_commit)
  1438. self.fetch_shallow()
  1439. # The 'c' commit was removed entirely, and 'a' was removed from history
  1440. self.assertRevCount(1, ['--all'])
  1441. self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
  1442. assert os.path.exists(os.path.join(self.gitdir, 'a'))
  1443. assert os.path.exists(os.path.join(self.gitdir, 'b'))
  1444. assert not os.path.exists(os.path.join(self.gitdir, 'c'))
  1445. def test_shallow_ref_pruning(self):
  1446. self.add_empty_file('a')
  1447. self.add_empty_file('b')
  1448. self.git('branch a_branch', cwd=self.srcdir)
  1449. self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
  1450. self.assertRevCount(2, cwd=self.srcdir)
  1451. self.fetch_shallow()
  1452. self.assertRefs(['master', 'origin/master'])
  1453. self.assertRevCount(1)
  1454. def test_shallow_submodules(self):
  1455. self.add_empty_file('a')
  1456. self.add_empty_file('b')
  1457. smdir = os.path.join(self.tempdir, 'gitsubmodule')
  1458. bb.utils.mkdirhier(smdir)
  1459. self.git('init', cwd=smdir)
  1460. # Make this look like it was cloned from a remote...
  1461. self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
  1462. self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
  1463. self.add_empty_file('asub', cwd=smdir)
  1464. self.add_empty_file('bsub', cwd=smdir)
  1465. self.git('submodule init', cwd=self.srcdir)
  1466. self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
  1467. self.git('submodule update', cwd=self.srcdir)
  1468. self.git('commit -m submodule -a', cwd=self.srcdir)
  1469. uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
  1470. fetcher, ud = self.fetch_shallow(uri)
  1471. # Verify the main repository is shallow
  1472. self.assertRevCount(1)
  1473. # Verify the gitsubmodule directory is present
  1474. assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
  1475. # Verify the submodule is also shallow
  1476. self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
  1477. def test_shallow_submodule_mirrors(self):
  1478. self.add_empty_file('a')
  1479. self.add_empty_file('b')
  1480. smdir = os.path.join(self.tempdir, 'gitsubmodule')
  1481. bb.utils.mkdirhier(smdir)
  1482. self.git('init', cwd=smdir)
  1483. # Make this look like it was cloned from a remote...
  1484. self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
  1485. self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
  1486. self.add_empty_file('asub', cwd=smdir)
  1487. self.add_empty_file('bsub', cwd=smdir)
  1488. self.git('submodule init', cwd=self.srcdir)
  1489. self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
  1490. self.git('submodule update', cwd=self.srcdir)
  1491. self.git('commit -m submodule -a', cwd=self.srcdir)
  1492. uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
  1493. # Fetch once to generate the shallow tarball
  1494. fetcher, ud = self.fetch(uri)
  1495. # Set up the mirror
  1496. mirrordir = os.path.join(self.tempdir, 'mirror')
  1497. os.rename(self.dldir, mirrordir)
  1498. self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
  1499. # Fetch from the mirror
  1500. bb.utils.remove(self.dldir, recurse=True)
  1501. bb.utils.remove(self.gitdir, recurse=True)
  1502. self.fetch_and_unpack(uri)
  1503. # Verify the main repository is shallow
  1504. self.assertRevCount(1)
  1505. # Verify the gitsubmodule directory is present
  1506. assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
  1507. # Verify the submodule is also shallow
  1508. self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
  1509. if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
  1510. def test_shallow_annex(self):
  1511. self.add_empty_file('a')
  1512. self.add_empty_file('b')
  1513. self.git('annex init', cwd=self.srcdir)
  1514. open(os.path.join(self.srcdir, 'c'), 'w').close()
  1515. self.git('annex add c', cwd=self.srcdir)
  1516. self.git('commit -m annex-c -a', cwd=self.srcdir)
  1517. bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
  1518. uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
  1519. fetcher, ud = self.fetch_shallow(uri)
  1520. self.assertRevCount(1)
  1521. assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
  1522. assert os.path.exists(os.path.join(self.gitdir, 'c'))
  1523. def test_shallow_multi_one_uri(self):
  1524. # Create initial git repo
  1525. self.add_empty_file('a')
  1526. self.add_empty_file('b')
  1527. self.git('checkout -b a_branch', cwd=self.srcdir)
  1528. self.add_empty_file('c')
  1529. self.add_empty_file('d')
  1530. self.git('checkout master', cwd=self.srcdir)
  1531. self.git('tag v0.0 a_branch', cwd=self.srcdir)
  1532. self.add_empty_file('e')
  1533. self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
  1534. self.add_empty_file('f')
  1535. self.assertRevCount(7, cwd=self.srcdir)
  1536. uri = self.d.getVar('SRC_URI').split()[0]
  1537. uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
  1538. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1539. self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
  1540. self.d.setVar('SRCREV_master', '${AUTOREV}')
  1541. self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
  1542. self.fetch_shallow(uri)
  1543. self.assertRevCount(5)
  1544. self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
  1545. def test_shallow_multi_one_uri_depths(self):
  1546. # Create initial git repo
  1547. self.add_empty_file('a')
  1548. self.add_empty_file('b')
  1549. self.git('checkout -b a_branch', cwd=self.srcdir)
  1550. self.add_empty_file('c')
  1551. self.add_empty_file('d')
  1552. self.git('checkout master', cwd=self.srcdir)
  1553. self.add_empty_file('e')
  1554. self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
  1555. self.add_empty_file('f')
  1556. self.assertRevCount(7, cwd=self.srcdir)
  1557. uri = self.d.getVar('SRC_URI').split()[0]
  1558. uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
  1559. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1560. self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
  1561. self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
  1562. self.d.setVar('SRCREV_master', '${AUTOREV}')
  1563. self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
  1564. self.fetch_shallow(uri)
  1565. self.assertRevCount(4, ['--all'])
  1566. self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
  1567. def test_shallow_clone_preferred_over_shallow(self):
  1568. self.add_empty_file('a')
  1569. self.add_empty_file('b')
  1570. # Fetch once to generate the shallow tarball
  1571. fetcher, ud = self.fetch()
  1572. assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
  1573. # Fetch and unpack with both the clonedir and shallow tarball available
  1574. bb.utils.remove(self.gitdir, recurse=True)
  1575. fetcher, ud = self.fetch_and_unpack()
  1576. # The unpacked tree should *not* be shallow
  1577. self.assertRevCount(2)
  1578. assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
  1579. def test_shallow_mirrors(self):
  1580. self.add_empty_file('a')
  1581. self.add_empty_file('b')
  1582. # Fetch once to generate the shallow tarball
  1583. fetcher, ud = self.fetch()
  1584. mirrortarball = ud.mirrortarballs[0]
  1585. assert os.path.exists(os.path.join(self.dldir, mirrortarball))
  1586. # Set up the mirror
  1587. mirrordir = os.path.join(self.tempdir, 'mirror')
  1588. bb.utils.mkdirhier(mirrordir)
  1589. self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
  1590. os.rename(os.path.join(self.dldir, mirrortarball),
  1591. os.path.join(mirrordir, mirrortarball))
  1592. # Fetch from the mirror
  1593. bb.utils.remove(self.dldir, recurse=True)
  1594. bb.utils.remove(self.gitdir, recurse=True)
  1595. self.fetch_and_unpack()
  1596. self.assertRevCount(1)
  1597. def test_shallow_invalid_depth(self):
  1598. self.add_empty_file('a')
  1599. self.add_empty_file('b')
  1600. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
  1601. with self.assertRaises(bb.fetch2.FetchError):
  1602. self.fetch()
  1603. def test_shallow_invalid_depth_default(self):
  1604. self.add_empty_file('a')
  1605. self.add_empty_file('b')
  1606. self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
  1607. with self.assertRaises(bb.fetch2.FetchError):
  1608. self.fetch()
  1609. def test_shallow_extra_refs(self):
  1610. self.add_empty_file('a')
  1611. self.add_empty_file('b')
  1612. self.git('branch a_branch', cwd=self.srcdir)
  1613. self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
  1614. self.assertRevCount(2, cwd=self.srcdir)
  1615. self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
  1616. self.fetch_shallow()
  1617. self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
  1618. self.assertRevCount(1)
  1619. def test_shallow_extra_refs_wildcard(self):
  1620. self.add_empty_file('a')
  1621. self.add_empty_file('b')
  1622. self.git('branch a_branch', cwd=self.srcdir)
  1623. self.git('tag v1.0', cwd=self.srcdir)
  1624. self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
  1625. self.assertRevCount(2, cwd=self.srcdir)
  1626. self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
  1627. self.fetch_shallow()
  1628. self.assertRefs(['master', 'origin/master', 'v1.0'])
  1629. self.assertRevCount(1)
  1630. def test_shallow_missing_extra_refs(self):
  1631. self.add_empty_file('a')
  1632. self.add_empty_file('b')
  1633. self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
  1634. with self.assertRaises(bb.fetch2.FetchError):
  1635. self.fetch()
  1636. def test_shallow_missing_extra_refs_wildcard(self):
  1637. self.add_empty_file('a')
  1638. self.add_empty_file('b')
  1639. self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
  1640. self.fetch()
  1641. def test_shallow_remove_revs(self):
  1642. # Create initial git repo
  1643. self.add_empty_file('a')
  1644. self.add_empty_file('b')
  1645. self.git('checkout -b a_branch', cwd=self.srcdir)
  1646. self.add_empty_file('c')
  1647. self.add_empty_file('d')
  1648. self.git('checkout master', cwd=self.srcdir)
  1649. self.git('tag v0.0 a_branch', cwd=self.srcdir)
  1650. self.add_empty_file('e')
  1651. self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
  1652. self.git('branch -d a_branch', cwd=self.srcdir)
  1653. self.add_empty_file('f')
  1654. self.assertRevCount(7, cwd=self.srcdir)
  1655. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1656. self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
  1657. self.fetch_shallow()
  1658. self.assertRevCount(5)
  1659. def test_shallow_invalid_revs(self):
  1660. self.add_empty_file('a')
  1661. self.add_empty_file('b')
  1662. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1663. self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
  1664. with self.assertRaises(bb.fetch2.FetchError):
  1665. self.fetch()
  1666. def test_shallow_fetch_missing_revs(self):
  1667. self.add_empty_file('a')
  1668. self.add_empty_file('b')
  1669. fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
  1670. self.git('tag v0.0 master', cwd=self.srcdir)
  1671. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1672. self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
  1673. self.fetch_shallow()
  1674. def test_shallow_fetch_missing_revs_fails(self):
  1675. self.add_empty_file('a')
  1676. self.add_empty_file('b')
  1677. fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
  1678. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1679. self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
  1680. with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
  1681. self.fetch_shallow()
  1682. self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
  1683. @skipIfNoNetwork()
  1684. def test_bitbake(self):
  1685. self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
  1686. self.git('config core.bare true', cwd=self.srcdir)
  1687. self.git('fetch', cwd=self.srcdir)
  1688. self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
  1689. # Note that the 1.10.0 tag is annotated, so this also tests
  1690. # reference of an annotated vs unannotated tag
  1691. self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
  1692. self.fetch_shallow()
  1693. # Confirm that the history of 1.10.0 was removed
  1694. orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
  1695. revs = len(self.git('rev-list master').splitlines())
  1696. self.assertNotEqual(orig_revs, revs)
  1697. self.assertRefs(['master', 'origin/master'])
  1698. self.assertRevCount(orig_revs - 1758)
  1699. def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
  1700. self.add_empty_file('a')
  1701. fetcher, ud = self.fetch()
  1702. bb.utils.remove(self.gitdir, recurse=True)
  1703. bb.utils.remove(self.dldir, recurse=True)
  1704. with self.assertRaises(bb.fetch2.UnpackError) as context:
  1705. fetcher.unpack(self.d.getVar('WORKDIR'))
  1706. self.assertIn("No up to date source found", context.exception.msg)
  1707. self.assertIn("clone directory not available or not up to date", context.exception.msg)
  1708. @skipIfNoNetwork()
  1709. def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
  1710. self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
  1711. self.d.setVar('BB_GIT_SHALLOW', '1')
  1712. self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
  1713. fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
  1714. fetcher.download()
  1715. bb.utils.remove(self.dldir + "/*.tar.gz")
  1716. fetcher.unpack(self.unpackdir)
  1717. dir = os.listdir(self.unpackdir + "/git/")
  1718. self.assertIn("fstests.doap", dir)
  1719. class GitLfsTest(FetcherTest):
  1720. def setUp(self):
  1721. FetcherTest.setUp(self)
  1722. self.gitdir = os.path.join(self.tempdir, 'git')
  1723. self.srcdir = os.path.join(self.tempdir, 'gitsource')
  1724. self.d.setVar('WORKDIR', self.tempdir)
  1725. self.d.setVar('S', self.gitdir)
  1726. self.d.delVar('PREMIRRORS')
  1727. self.d.delVar('MIRRORS')
  1728. self.d.setVar('SRCREV', '${AUTOREV}')
  1729. self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
  1730. bb.utils.mkdirhier(self.srcdir)
  1731. self.git('init', cwd=self.srcdir)
  1732. with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
  1733. attrs.write('*.mp3 filter=lfs -text')
  1734. self.git(['add', '.gitattributes'], cwd=self.srcdir)
  1735. self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
  1736. def git(self, cmd, cwd=None):
  1737. if isinstance(cmd, str):
  1738. cmd = 'git ' + cmd
  1739. else:
  1740. cmd = ['git'] + cmd
  1741. if cwd is None:
  1742. cwd = self.gitdir
  1743. return bb.process.run(cmd, cwd=cwd)[0]
  1744. def fetch(self, uri=None):
  1745. uris = self.d.getVar('SRC_URI').split()
  1746. uri = uris[0]
  1747. d = self.d
  1748. fetcher = bb.fetch2.Fetch(uris, d)
  1749. fetcher.download()
  1750. ud = fetcher.ud[uri]
  1751. return fetcher, ud
  1752. def test_lfs_enabled(self):
  1753. import shutil
  1754. uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
  1755. self.d.setVar('SRC_URI', uri)
  1756. fetcher, ud = self.fetch()
  1757. self.assertIsNotNone(ud.method._find_git_lfs)
  1758. # If git-lfs can be found, the unpack should be successful
  1759. ud.method._find_git_lfs = lambda d: True
  1760. shutil.rmtree(self.gitdir, ignore_errors=True)
  1761. fetcher.unpack(self.d.getVar('WORKDIR'))
  1762. # If git-lfs cannot be found, the unpack should throw an error
  1763. with self.assertRaises(bb.fetch2.FetchError):
  1764. ud.method._find_git_lfs = lambda d: False
  1765. shutil.rmtree(self.gitdir, ignore_errors=True)
  1766. fetcher.unpack(self.d.getVar('WORKDIR'))
  1767. def test_lfs_disabled(self):
  1768. import shutil
  1769. uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
  1770. self.d.setVar('SRC_URI', uri)
  1771. fetcher, ud = self.fetch()
  1772. self.assertIsNotNone(ud.method._find_git_lfs)
  1773. # If git-lfs can be found, the unpack should be successful
  1774. ud.method._find_git_lfs = lambda d: True
  1775. shutil.rmtree(self.gitdir, ignore_errors=True)
  1776. fetcher.unpack(self.d.getVar('WORKDIR'))
  1777. # If git-lfs cannot be found, the unpack should be successful
  1778. ud.method._find_git_lfs = lambda d: False
  1779. shutil.rmtree(self.gitdir, ignore_errors=True)
  1780. fetcher.unpack(self.d.getVar('WORKDIR'))
  1781. class GitURLWithSpacesTest(FetcherTest):
  1782. test_git_urls = {
  1783. "git://tfs-example.org:22/tfs/example%20path/example.git" : {
  1784. 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
  1785. 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
  1786. 'path': '/tfs/example path/example.git'
  1787. },
  1788. "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
  1789. 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
  1790. 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
  1791. 'path': '/tfs/example path/example repo.git'
  1792. }
  1793. }
  1794. def test_urls(self):
  1795. # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
  1796. self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
  1797. for test_git_url, ref in self.test_git_urls.items():
  1798. fetcher = bb.fetch.Fetch([test_git_url], self.d)
  1799. ud = fetcher.ud[fetcher.urls[0]]
  1800. self.assertEqual(ud.url, ref['url'])
  1801. self.assertEqual(ud.path, ref['path'])
  1802. self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
  1803. self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
  1804. self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
  1805. self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
  1806. self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
  1807. class NPMTest(FetcherTest):
  1808. def skipIfNoNpm():
  1809. import shutil
  1810. if not shutil.which('npm'):
  1811. return unittest.skip('npm not installed, tests being skipped')
  1812. return lambda f: f
  1813. @skipIfNoNpm()
  1814. @skipIfNoNetwork()
  1815. def test_npm(self):
  1816. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1817. fetcher = bb.fetch.Fetch([url], self.d)
  1818. ud = fetcher.ud[fetcher.urls[0]]
  1819. fetcher.download()
  1820. self.assertTrue(os.path.exists(ud.localpath))
  1821. self.assertTrue(os.path.exists(ud.localpath + '.done'))
  1822. self.assertTrue(os.path.exists(ud.resolvefile))
  1823. fetcher.unpack(self.unpackdir)
  1824. unpackdir = os.path.join(self.unpackdir, 'npm')
  1825. self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
  1826. @skipIfNoNpm()
  1827. @skipIfNoNetwork()
  1828. def test_npm_bad_checksum(self):
  1829. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1830. # Fetch once to get a tarball
  1831. fetcher = bb.fetch.Fetch([url], self.d)
  1832. ud = fetcher.ud[fetcher.urls[0]]
  1833. fetcher.download()
  1834. self.assertTrue(os.path.exists(ud.localpath))
  1835. # Modify the tarball
  1836. bad = b'bad checksum'
  1837. with open(ud.localpath, 'wb') as f:
  1838. f.write(bad)
  1839. # Verify that the tarball is fetched again
  1840. fetcher.download()
  1841. badsum = hashlib.sha512(bad).hexdigest()
  1842. self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
  1843. self.assertTrue(os.path.exists(ud.localpath))
  1844. @skipIfNoNpm()
  1845. @skipIfNoNetwork()
  1846. def test_npm_premirrors(self):
  1847. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1848. # Fetch once to get a tarball
  1849. fetcher = bb.fetch.Fetch([url], self.d)
  1850. ud = fetcher.ud[fetcher.urls[0]]
  1851. fetcher.download()
  1852. self.assertTrue(os.path.exists(ud.localpath))
  1853. # Setup the mirror
  1854. mirrordir = os.path.join(self.tempdir, 'mirror')
  1855. bb.utils.mkdirhier(mirrordir)
  1856. os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
  1857. self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
  1858. self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
  1859. # Fetch again
  1860. self.assertFalse(os.path.exists(ud.localpath))
  1861. fetcher.download()
  1862. self.assertTrue(os.path.exists(ud.localpath))
  1863. @skipIfNoNpm()
  1864. @skipIfNoNetwork()
  1865. def test_npm_mirrors(self):
  1866. # Fetch once to get a tarball
  1867. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1868. fetcher = bb.fetch.Fetch([url], self.d)
  1869. ud = fetcher.ud[fetcher.urls[0]]
  1870. fetcher.download()
  1871. self.assertTrue(os.path.exists(ud.localpath))
  1872. # Setup the mirror
  1873. mirrordir = os.path.join(self.tempdir, 'mirror')
  1874. bb.utils.mkdirhier(mirrordir)
  1875. os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
  1876. self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
  1877. # Update the resolved url to an invalid url
  1878. with open(ud.resolvefile, 'r') as f:
  1879. url = f.read()
  1880. uri = URI(url)
  1881. uri.path = '/invalid'
  1882. with open(ud.resolvefile, 'w') as f:
  1883. f.write(str(uri))
  1884. # Fetch again
  1885. self.assertFalse(os.path.exists(ud.localpath))
  1886. fetcher.download()
  1887. self.assertTrue(os.path.exists(ud.localpath))
  1888. @skipIfNoNpm()
  1889. @skipIfNoNetwork()
  1890. def test_npm_destsuffix_downloadfilename(self):
  1891. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
  1892. fetcher = bb.fetch.Fetch([url], self.d)
  1893. fetcher.download()
  1894. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
  1895. fetcher.unpack(self.unpackdir)
  1896. unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
  1897. self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
  1898. def test_npm_no_network_no_tarball(self):
  1899. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1900. self.d.setVar('BB_NO_NETWORK', '1')
  1901. fetcher = bb.fetch.Fetch([url], self.d)
  1902. with self.assertRaises(bb.fetch2.NetworkAccess):
  1903. fetcher.download()
  1904. @skipIfNoNpm()
  1905. @skipIfNoNetwork()
  1906. def test_npm_no_network_with_tarball(self):
  1907. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1908. # Fetch once to get a tarball
  1909. fetcher = bb.fetch.Fetch([url], self.d)
  1910. fetcher.download()
  1911. # Disable network access
  1912. self.d.setVar('BB_NO_NETWORK', '1')
  1913. # Fetch again
  1914. fetcher.download()
  1915. fetcher.unpack(self.unpackdir)
  1916. unpackdir = os.path.join(self.unpackdir, 'npm')
  1917. self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
  1918. @skipIfNoNpm()
  1919. @skipIfNoNetwork()
  1920. def test_npm_registry_alternate(self):
  1921. url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1922. fetcher = bb.fetch.Fetch([url], self.d)
  1923. fetcher.download()
  1924. fetcher.unpack(self.unpackdir)
  1925. unpackdir = os.path.join(self.unpackdir, 'npm')
  1926. self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
  1927. @skipIfNoNpm()
  1928. @skipIfNoNetwork()
  1929. def test_npm_version_latest(self):
  1930. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
  1931. fetcher = bb.fetch.Fetch([url], self.d)
  1932. fetcher.download()
  1933. fetcher.unpack(self.unpackdir)
  1934. unpackdir = os.path.join(self.unpackdir, 'npm')
  1935. self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
  1936. @skipIfNoNpm()
  1937. @skipIfNoNetwork()
  1938. def test_npm_registry_invalid(self):
  1939. url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1940. fetcher = bb.fetch.Fetch([url], self.d)
  1941. with self.assertRaises(bb.fetch2.FetchError):
  1942. fetcher.download()
  1943. @skipIfNoNpm()
  1944. @skipIfNoNetwork()
  1945. def test_npm_package_invalid(self):
  1946. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
  1947. fetcher = bb.fetch.Fetch([url], self.d)
  1948. with self.assertRaises(bb.fetch2.FetchError):
  1949. fetcher.download()
  1950. @skipIfNoNpm()
  1951. @skipIfNoNetwork()
  1952. def test_npm_version_invalid(self):
  1953. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
  1954. with self.assertRaises(bb.fetch2.ParameterError):
  1955. fetcher = bb.fetch.Fetch([url], self.d)
  1956. @skipIfNoNpm()
  1957. @skipIfNoNetwork()
  1958. def test_npm_registry_none(self):
  1959. url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
  1960. with self.assertRaises(bb.fetch2.MalformedUrl):
  1961. fetcher = bb.fetch.Fetch([url], self.d)
  1962. @skipIfNoNpm()
  1963. @skipIfNoNetwork()
  1964. def test_npm_package_none(self):
  1965. url = 'npm://registry.npmjs.org;version=1.0.0'
  1966. with self.assertRaises(bb.fetch2.MissingParameterError):
  1967. fetcher = bb.fetch.Fetch([url], self.d)
  1968. @skipIfNoNpm()
  1969. @skipIfNoNetwork()
  1970. def test_npm_version_none(self):
  1971. url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
  1972. with self.assertRaises(bb.fetch2.MissingParameterError):
  1973. fetcher = bb.fetch.Fetch([url], self.d)
  1974. def create_shrinkwrap_file(self, data):
  1975. import json
  1976. datadir = os.path.join(self.tempdir, 'data')
  1977. swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
  1978. bb.utils.mkdirhier(datadir)
  1979. with open(swfile, 'w') as f:
  1980. json.dump(data, f)
  1981. # Also configure the S directory
  1982. self.sdir = os.path.join(self.unpackdir, 'S')
  1983. self.d.setVar('S', self.sdir)
  1984. return swfile
  1985. @skipIfNoNpm()
  1986. @skipIfNoNetwork()
  1987. def test_npmsw(self):
  1988. swfile = self.create_shrinkwrap_file({
  1989. 'dependencies': {
  1990. 'array-flatten': {
  1991. 'version': '1.1.1',
  1992. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  1993. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
  1994. 'dependencies': {
  1995. 'content-type': {
  1996. 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
  1997. 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
  1998. 'dependencies': {
  1999. 'cookie': {
  2000. 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
  2001. 'from': 'git+https://github.com/jshttp/cookie.git'
  2002. }
  2003. }
  2004. }
  2005. }
  2006. }
  2007. }
  2008. })
  2009. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2010. fetcher.download()
  2011. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
  2012. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
  2013. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
  2014. fetcher.unpack(self.unpackdir)
  2015. self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
  2016. self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
  2017. self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
  2018. self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
  2019. @skipIfNoNpm()
  2020. @skipIfNoNetwork()
  2021. def test_npmsw_dev(self):
  2022. swfile = self.create_shrinkwrap_file({
  2023. 'dependencies': {
  2024. 'array-flatten': {
  2025. 'version': '1.1.1',
  2026. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2027. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2028. },
  2029. 'content-type': {
  2030. 'version': '1.0.4',
  2031. 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
  2032. 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
  2033. 'dev': True
  2034. }
  2035. }
  2036. })
  2037. # Fetch with dev disabled
  2038. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2039. fetcher.download()
  2040. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
  2041. self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
  2042. # Fetch with dev enabled
  2043. fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
  2044. fetcher.download()
  2045. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
  2046. self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
  2047. @skipIfNoNpm()
  2048. @skipIfNoNetwork()
  2049. def test_npmsw_destsuffix(self):
  2050. swfile = self.create_shrinkwrap_file({
  2051. 'dependencies': {
  2052. 'array-flatten': {
  2053. 'version': '1.1.1',
  2054. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2055. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2056. }
  2057. }
  2058. })
  2059. fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
  2060. fetcher.download()
  2061. fetcher.unpack(self.unpackdir)
  2062. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
  2063. def test_npmsw_no_network_no_tarball(self):
  2064. swfile = self.create_shrinkwrap_file({
  2065. 'dependencies': {
  2066. 'array-flatten': {
  2067. 'version': '1.1.1',
  2068. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2069. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2070. }
  2071. }
  2072. })
  2073. self.d.setVar('BB_NO_NETWORK', '1')
  2074. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2075. with self.assertRaises(bb.fetch2.NetworkAccess):
  2076. fetcher.download()
  2077. @skipIfNoNpm()
  2078. @skipIfNoNetwork()
  2079. def test_npmsw_no_network_with_tarball(self):
  2080. # Fetch once to get a tarball
  2081. fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
  2082. fetcher.download()
  2083. # Disable network access
  2084. self.d.setVar('BB_NO_NETWORK', '1')
  2085. # Fetch again
  2086. swfile = self.create_shrinkwrap_file({
  2087. 'dependencies': {
  2088. 'array-flatten': {
  2089. 'version': '1.1.1',
  2090. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2091. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2092. }
  2093. }
  2094. })
  2095. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2096. fetcher.download()
  2097. fetcher.unpack(self.unpackdir)
  2098. self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
  2099. @skipIfNoNpm()
  2100. @skipIfNoNetwork()
  2101. def test_npmsw_npm_reusability(self):
  2102. # Fetch once with npmsw
  2103. swfile = self.create_shrinkwrap_file({
  2104. 'dependencies': {
  2105. 'array-flatten': {
  2106. 'version': '1.1.1',
  2107. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2108. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2109. }
  2110. }
  2111. })
  2112. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2113. fetcher.download()
  2114. # Disable network access
  2115. self.d.setVar('BB_NO_NETWORK', '1')
  2116. # Fetch again with npm
  2117. fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
  2118. fetcher.download()
  2119. fetcher.unpack(self.unpackdir)
  2120. self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
  2121. @skipIfNoNpm()
  2122. @skipIfNoNetwork()
  2123. def test_npmsw_bad_checksum(self):
  2124. # Try to fetch with bad checksum
  2125. swfile = self.create_shrinkwrap_file({
  2126. 'dependencies': {
  2127. 'array-flatten': {
  2128. 'version': '1.1.1',
  2129. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2130. 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
  2131. }
  2132. }
  2133. })
  2134. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2135. with self.assertRaises(bb.fetch2.FetchError):
  2136. fetcher.download()
  2137. # Fetch correctly to get a tarball
  2138. swfile = self.create_shrinkwrap_file({
  2139. 'dependencies': {
  2140. 'array-flatten': {
  2141. 'version': '1.1.1',
  2142. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2143. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2144. }
  2145. }
  2146. })
  2147. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2148. fetcher.download()
  2149. localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
  2150. self.assertTrue(os.path.exists(localpath))
  2151. # Modify the tarball
  2152. bad = b'bad checksum'
  2153. with open(localpath, 'wb') as f:
  2154. f.write(bad)
  2155. # Verify that the tarball is fetched again
  2156. fetcher.download()
  2157. badsum = hashlib.sha1(bad).hexdigest()
  2158. self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
  2159. self.assertTrue(os.path.exists(localpath))
  2160. @skipIfNoNpm()
  2161. @skipIfNoNetwork()
  2162. def test_npmsw_premirrors(self):
  2163. # Fetch once to get a tarball
  2164. fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
  2165. ud = fetcher.ud[fetcher.urls[0]]
  2166. fetcher.download()
  2167. self.assertTrue(os.path.exists(ud.localpath))
  2168. # Setup the mirror
  2169. mirrordir = os.path.join(self.tempdir, 'mirror')
  2170. bb.utils.mkdirhier(mirrordir)
  2171. os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
  2172. self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
  2173. self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
  2174. # Fetch again
  2175. self.assertFalse(os.path.exists(ud.localpath))
  2176. swfile = self.create_shrinkwrap_file({
  2177. 'dependencies': {
  2178. 'array-flatten': {
  2179. 'version': '1.1.1',
  2180. 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
  2181. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2182. }
  2183. }
  2184. })
  2185. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2186. fetcher.download()
  2187. self.assertTrue(os.path.exists(ud.localpath))
  2188. @skipIfNoNpm()
  2189. @skipIfNoNetwork()
  2190. def test_npmsw_mirrors(self):
  2191. # Fetch once to get a tarball
  2192. fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
  2193. ud = fetcher.ud[fetcher.urls[0]]
  2194. fetcher.download()
  2195. self.assertTrue(os.path.exists(ud.localpath))
  2196. # Setup the mirror
  2197. mirrordir = os.path.join(self.tempdir, 'mirror')
  2198. bb.utils.mkdirhier(mirrordir)
  2199. os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
  2200. self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
  2201. # Fetch again with invalid url
  2202. self.assertFalse(os.path.exists(ud.localpath))
  2203. swfile = self.create_shrinkwrap_file({
  2204. 'dependencies': {
  2205. 'array-flatten': {
  2206. 'version': '1.1.1',
  2207. 'resolved': 'https://invalid',
  2208. 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
  2209. }
  2210. }
  2211. })
  2212. fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
  2213. fetcher.download()
  2214. self.assertTrue(os.path.exists(ud.localpath))