__init__.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. """
  4. BitBake 'Fetch' implementations
  5. Classes for obtaining upstream sources for the
  6. BitBake build tools.
  7. """
  8. # Copyright (C) 2003, 2004 Chris Larson
  9. #
  10. # This program is free software; you can redistribute it and/or modify
  11. # it under the terms of the GNU General Public License version 2 as
  12. # published by the Free Software Foundation.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU General Public License along
  20. # with this program; if not, write to the Free Software Foundation, Inc.,
  21. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  22. #
  23. # Based on functions from the base bb module, Copyright 2003 Holger Schurig
  24. import os, re, fcntl
  25. import bb
  26. from bb import data
  27. from bb import persist_data
  28. try:
  29. import cPickle as pickle
  30. except ImportError:
  31. import pickle
  32. class FetchError(Exception):
  33. """Exception raised when a download fails"""
  34. class NoMethodError(Exception):
  35. """Exception raised when there is no method to obtain a supplied url or set of urls"""
  36. class MissingParameterError(Exception):
  37. """Exception raised when a fetch method is missing a critical parameter in the url"""
  38. class ParameterError(Exception):
  39. """Exception raised when a url cannot be proccessed due to invalid parameters."""
  40. class MD5SumError(Exception):
  41. """Exception raised when a MD5SUM of a file does not match the expected one"""
  42. class InvalidSRCREV(Exception):
  43. """Exception raised when an invalid SRCREV is encountered"""
  44. def uri_replace(uri, uri_find, uri_replace, d):
  45. # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: operating on %s" % uri)
  46. if not uri or not uri_find or not uri_replace:
  47. bb.msg.debug(1, bb.msg.domain.Fetcher, "uri_replace: passed an undefined value, not replacing")
  48. uri_decoded = list(bb.decodeurl(uri))
  49. uri_find_decoded = list(bb.decodeurl(uri_find))
  50. uri_replace_decoded = list(bb.decodeurl(uri_replace))
  51. result_decoded = ['','','','','',{}]
  52. for i in uri_find_decoded:
  53. loc = uri_find_decoded.index(i)
  54. result_decoded[loc] = uri_decoded[loc]
  55. import types
  56. if type(i) == types.StringType:
  57. import re
  58. if (re.match(i, uri_decoded[loc])):
  59. result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
  60. if uri_find_decoded.index(i) == 2:
  61. if d:
  62. localfn = bb.fetch.localpath(uri, d)
  63. if localfn:
  64. result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(bb.fetch.localpath(uri, d))
  65. # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
  66. else:
  67. # bb.msg.note(1, bb.msg.domain.Fetcher, "uri_replace: no match")
  68. return uri
  69. # else:
  70. # for j in i.keys():
  71. # FIXME: apply replacements against options
  72. return bb.encodeurl(result_decoded)
  73. methods = []
  74. urldata_cache = {}
  75. def fetcher_init(d):
  76. """
  77. Called to initilize the fetchers once the configuration data is known
  78. Calls before this must not hit the cache.
  79. """
  80. pd = persist_data.PersistData(d)
  81. # When to drop SCM head revisions controled by user policy
  82. srcrev_policy = bb.data.getVar('BB_SRCREV_POLICY', d, 1) or "clear"
  83. if srcrev_policy == "cache":
  84. bb.msg.debug(1, bb.msg.domain.Fetcher, "Keeping SRCREV cache due to cache policy of: %s" % srcrev_policy)
  85. elif srcrev_policy == "clear":
  86. bb.msg.debug(1, bb.msg.domain.Fetcher, "Clearing SRCREV cache due to cache policy of: %s" % srcrev_policy)
  87. pd.delDomain("BB_URI_HEADREVS")
  88. else:
  89. bb.msg.fatal(bb.msg.domain.Fetcher, "Invalid SRCREV cache policy of: %s" % srcrev_policy)
  90. # Make sure our domains exist
  91. pd.addDomain("BB_URI_HEADREVS")
  92. pd.addDomain("BB_URI_LOCALCOUNT")
  93. # Function call order is usually:
  94. # 1. init
  95. # 2. go
  96. # 3. localpaths
  97. # localpath can be called at any time
  98. def init(urls, d, setup = True):
  99. urldata = {}
  100. fn = bb.data.getVar('FILE', d, 1)
  101. if fn in urldata_cache:
  102. urldata = urldata_cache[fn]
  103. for url in urls:
  104. if url not in urldata:
  105. urldata[url] = FetchData(url, d)
  106. if setup:
  107. for url in urldata:
  108. if not urldata[url].setup:
  109. urldata[url].setup_localpath(d)
  110. urldata_cache[fn] = urldata
  111. return urldata
  112. def go(d):
  113. """
  114. Fetch all urls
  115. init must have previously been called
  116. """
  117. urldata = init([], d, True)
  118. for u in urldata:
  119. ud = urldata[u]
  120. m = ud.method
  121. if ud.localfile:
  122. if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
  123. # File already present along with md5 stamp file
  124. # Touch md5 file to show activity
  125. try:
  126. os.utime(ud.md5, None)
  127. except:
  128. # Errors aren't fatal here
  129. pass
  130. continue
  131. lf = bb.utils.lockfile(ud.lockfile)
  132. if not m.forcefetch(u, ud, d) and os.path.exists(ud.md5):
  133. # If someone else fetched this before we got the lock,
  134. # notice and don't try again
  135. try:
  136. os.utime(ud.md5, None)
  137. except:
  138. # Errors aren't fatal here
  139. pass
  140. bb.utils.unlockfile(lf)
  141. continue
  142. m.go(u, ud, d)
  143. if ud.localfile:
  144. if not m.forcefetch(u, ud, d):
  145. Fetch.write_md5sum(u, ud, d)
  146. bb.utils.unlockfile(lf)
  147. def checkstatus(d):
  148. """
  149. Check all urls exist upstream
  150. init must have previously been called
  151. """
  152. urldata = init([], d, True)
  153. for u in urldata:
  154. ud = urldata[u]
  155. m = ud.method
  156. bb.msg.note(1, bb.msg.domain.Fetcher, "Testing URL %s" % u)
  157. ret = m.checkstatus(u, ud, d)
  158. if not ret:
  159. bb.msg.fatal(bb.msg.domain.Fetcher, "URL %s doesn't work" % u)
  160. def localpaths(d):
  161. """
  162. Return a list of the local filenames, assuming successful fetch
  163. """
  164. local = []
  165. urldata = init([], d, True)
  166. for u in urldata:
  167. ud = urldata[u]
  168. local.append(ud.localpath)
  169. return local
  170. srcrev_internal_call = False
  171. def get_srcrev(d):
  172. """
  173. Return the version string for the current package
  174. (usually to be used as PV)
  175. Most packages usually only have one SCM so we just pass on the call.
  176. In the multi SCM case, we build a value based on SRCREV_FORMAT which must
  177. have been set.
  178. """
  179. #
  180. # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which
  181. # could translate into a call to here. If it does, we need to catch this
  182. # and provide some way so it knows get_srcrev is active instead of being
  183. # some number etc. hence the srcrev_internal_call tracking and the magic
  184. # "SRCREVINACTION" return value.
  185. #
  186. # Neater solutions welcome!
  187. #
  188. if bb.fetch.srcrev_internal_call:
  189. return "SRCREVINACTION"
  190. scms = []
  191. # Only call setup_localpath on URIs which suppports_srcrev()
  192. urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d, False)
  193. for u in urldata:
  194. ud = urldata[u]
  195. if ud.method.suppports_srcrev():
  196. if not ud.setup:
  197. ud.setup_localpath(d)
  198. scms.append(u)
  199. if len(scms) == 0:
  200. bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI")
  201. raise ParameterError
  202. if len(scms) == 1:
  203. return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d)
  204. #
  205. # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
  206. #
  207. format = bb.data.getVar('SRCREV_FORMAT', d, 1)
  208. if not format:
  209. bb.msg.error(bb.msg.domain.Fetcher, "The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
  210. raise ParameterError
  211. for scm in scms:
  212. if 'name' in urldata[scm].parm:
  213. name = urldata[scm].parm["name"]
  214. rev = urldata[scm].method.sortable_revision(scm, urldata[scm], d)
  215. format = format.replace(name, rev)
  216. return format
  217. def localpath(url, d, cache = True):
  218. """
  219. Called from the parser with cache=False since the cache isn't ready
  220. at this point. Also called from classed in OE e.g. patch.bbclass
  221. """
  222. ud = init([url], d)
  223. if ud[url].method:
  224. return ud[url].localpath
  225. return url
  226. def runfetchcmd(cmd, d, quiet = False):
  227. """
  228. Run cmd returning the command output
  229. Raise an error if interrupted or cmd fails
  230. Optionally echo command output to stdout
  231. """
  232. # Need to export PATH as binary could be in metadata paths
  233. # rather than host provided
  234. # Also include some other variables.
  235. # FIXME: Should really include all export varaiables?
  236. exportvars = ['PATH', 'GIT_PROXY_HOST', 'GIT_PROXY_PORT', 'GIT_PROXY_COMMAND']
  237. for var in exportvars:
  238. val = data.getVar(var, d, True)
  239. if val:
  240. cmd = 'export ' + var + '=%s; %s' % (val, cmd)
  241. bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cmd)
  242. # redirect stderr to stdout
  243. stdout_handle = os.popen(cmd + " 2>&1", "r")
  244. output = ""
  245. while 1:
  246. line = stdout_handle.readline()
  247. if not line:
  248. break
  249. if not quiet:
  250. print line,
  251. output += line
  252. status = stdout_handle.close() or 0
  253. signal = status >> 8
  254. exitstatus = status & 0xff
  255. if signal:
  256. raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output))
  257. elif status != 0:
  258. raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output))
  259. return output
  260. class FetchData(object):
  261. """
  262. A class which represents the fetcher state for a given URI.
  263. """
  264. def __init__(self, url, d):
  265. self.localfile = ""
  266. (self.type, self.host, self.path, self.user, self.pswd, self.parm) = bb.decodeurl(data.expand(url, d))
  267. self.date = Fetch.getSRCDate(self, d)
  268. self.url = url
  269. self.setup = False
  270. for m in methods:
  271. if m.supports(url, self, d):
  272. self.method = m
  273. return
  274. raise NoMethodError("Missing implementation for url %s" % url)
  275. def setup_localpath(self, d):
  276. self.setup = True
  277. if "localpath" in self.parm:
  278. # if user sets localpath for file, use it instead.
  279. self.localpath = self.parm["localpath"]
  280. else:
  281. bb.fetch.srcrev_internal_call = True
  282. self.localpath = self.method.localpath(self.url, self, d)
  283. bb.fetch.srcrev_internal_call = False
  284. # We have to clear data's internal caches since the cached value of SRCREV is now wrong.
  285. # Horrible...
  286. bb.data.delVar("ISHOULDNEVEREXIST", d)
  287. self.md5 = self.localpath + '.md5'
  288. self.lockfile = self.localpath + '.lock'
  289. class Fetch(object):
  290. """Base class for 'fetch'ing data"""
  291. def __init__(self, urls = []):
  292. self.urls = []
  293. def supports(self, url, urldata, d):
  294. """
  295. Check to see if this fetch class supports a given url.
  296. """
  297. return 0
  298. def localpath(self, url, urldata, d):
  299. """
  300. Return the local filename of a given url assuming a successful fetch.
  301. Can also setup variables in urldata for use in go (saving code duplication
  302. and duplicate code execution)
  303. """
  304. return url
  305. def setUrls(self, urls):
  306. self.__urls = urls
  307. def getUrls(self):
  308. return self.__urls
  309. urls = property(getUrls, setUrls, None, "Urls property")
  310. def forcefetch(self, url, urldata, d):
  311. """
  312. Force a fetch, even if localpath exists?
  313. """
  314. return False
  315. def suppports_srcrev(self):
  316. """
  317. The fetcher supports auto source revisions (SRCREV)
  318. """
  319. return False
  320. def go(self, url, urldata, d):
  321. """
  322. Fetch urls
  323. Assumes localpath was called first
  324. """
  325. raise NoMethodError("Missing implementation for url")
  326. def checkstatus(self, url, urldata, d):
  327. """
  328. Check the status of a URL
  329. Assumes localpath was called first
  330. """
  331. bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s could not be checked for status since no method exists." % url)
  332. return True
  333. def getSRCDate(urldata, d):
  334. """
  335. Return the SRC Date for the component
  336. d the bb.data module
  337. """
  338. if "srcdate" in urldata.parm:
  339. return urldata.parm['srcdate']
  340. pn = data.getVar("PN", d, 1)
  341. if pn:
  342. return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
  343. return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
  344. getSRCDate = staticmethod(getSRCDate)
  345. def srcrev_internal_helper(ud, d):
  346. """
  347. Return:
  348. a) a source revision if specified
  349. b) True if auto srcrev is in action
  350. c) False otherwise
  351. """
  352. if 'rev' in ud.parm:
  353. return ud.parm['rev']
  354. if 'tag' in ud.parm:
  355. return ud.parm['tag']
  356. rev = None
  357. if 'name' in ud.parm:
  358. pn = data.getVar("PN", d, 1)
  359. rev = data.getVar("SRCREV_pn-" + pn + "_" + ud.parm['name'], d, 1)
  360. if not rev:
  361. rev = data.getVar("SRCREV", d, 1)
  362. if rev == "INVALID":
  363. raise InvalidSRCREV("Please set SRCREV to a valid value")
  364. if not rev:
  365. return False
  366. if rev is "SRCREVINACTION":
  367. return True
  368. return rev
  369. srcrev_internal_helper = staticmethod(srcrev_internal_helper)
  370. def try_mirror(d, tarfn):
  371. """
  372. Try to use a mirrored version of the sources. We do this
  373. to avoid massive loads on foreign cvs and svn servers.
  374. This method will be used by the different fetcher
  375. implementations.
  376. d Is a bb.data instance
  377. tarfn is the name of the tarball
  378. """
  379. tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
  380. if os.access(tarpath, os.R_OK):
  381. bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
  382. return True
  383. pn = data.getVar('PN', d, True)
  384. src_tarball_stash = None
  385. if pn:
  386. src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
  387. for stash in src_tarball_stash:
  388. fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
  389. uri = stash + tarfn
  390. bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
  391. fetchcmd = fetchcmd.replace("${URI}", uri)
  392. ret = os.system(fetchcmd)
  393. if ret == 0:
  394. bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn)
  395. return True
  396. return False
  397. try_mirror = staticmethod(try_mirror)
  398. def verify_md5sum(ud, got_sum):
  399. """
  400. Verify the md5sum we wanted with the one we got
  401. """
  402. wanted_sum = None
  403. if 'md5sum' in ud.parm:
  404. wanted_sum = ud.parm['md5sum']
  405. if not wanted_sum:
  406. return True
  407. return wanted_sum == got_sum
  408. verify_md5sum = staticmethod(verify_md5sum)
  409. def write_md5sum(url, ud, d):
  410. md5data = bb.utils.md5_file(ud.localpath)
  411. # verify the md5sum
  412. if not Fetch.verify_md5sum(ud, md5data):
  413. raise MD5SumError(url)
  414. md5out = file(ud.md5, 'w')
  415. md5out.write(md5data)
  416. md5out.close()
  417. write_md5sum = staticmethod(write_md5sum)
  418. def latest_revision(self, url, ud, d):
  419. """
  420. Look in the cache for the latest revision, if not present ask the SCM.
  421. """
  422. if not hasattr(self, "_latest_revision"):
  423. raise ParameterError
  424. pd = persist_data.PersistData(d)
  425. key = self._revision_key(url, ud, d)
  426. rev = pd.getValue("BB_URI_HEADREVS", key)
  427. if rev != None:
  428. return str(rev)
  429. rev = self._latest_revision(url, ud, d)
  430. pd.setValue("BB_URI_HEADREVS", key, rev)
  431. return rev
  432. def sortable_revision(self, url, ud, d):
  433. """
  434. """
  435. if hasattr(self, "_sortable_revision"):
  436. return self._sortable_revision(url, ud, d)
  437. pd = persist_data.PersistData(d)
  438. key = self._revision_key(url, ud, d)
  439. latest_rev = self._build_revision(url, ud, d)
  440. last_rev = pd.getValue("BB_URI_LOCALCOUNT", key + "_rev")
  441. count = pd.getValue("BB_URI_LOCALCOUNT", key + "_count")
  442. if last_rev == latest_rev:
  443. return str(count + "+" + latest_rev)
  444. if count is None:
  445. count = "0"
  446. else:
  447. count = str(int(count) + 1)
  448. pd.setValue("BB_URI_LOCALCOUNT", key + "_rev", latest_rev)
  449. pd.setValue("BB_URI_LOCALCOUNT", key + "_count", count)
  450. return str(count + "+" + latest_rev)
  451. import cvs
  452. import git
  453. import local
  454. import svn
  455. import wget
  456. import svk
  457. import ssh
  458. import perforce
  459. import bzr
  460. import hg
  461. methods.append(local.Local())
  462. methods.append(wget.Wget())
  463. methods.append(svn.Svn())
  464. methods.append(git.Git())
  465. methods.append(cvs.Cvs())
  466. methods.append(svk.Svk())
  467. methods.append(ssh.SSH())
  468. methods.append(perforce.Perforce())
  469. methods.append(bzr.Bzr())
  470. methods.append(hg.Hg())