providers.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. # ex:ts=4:sw=4:sts=4:et
  2. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  3. #
  4. # Copyright (C) 2003, 2004 Chris Larson
  5. # Copyright (C) 2003, 2004 Phil Blundell
  6. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  7. # Copyright (C) 2005 Holger Hans Peter Freyther
  8. # Copyright (C) 2005 ROAD GmbH
  9. # Copyright (C) 2006 Richard Purdie
  10. #
  11. # This program is free software; you can redistribute it and/or modify
  12. # it under the terms of the GNU General Public License version 2 as
  13. # published by the Free Software Foundation.
  14. #
  15. # This program is distributed in the hope that it will be useful,
  16. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. # GNU General Public License for more details.
  19. #
  20. # You should have received a copy of the GNU General Public License along
  21. # with this program; if not, write to the Free Software Foundation, Inc.,
  22. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  23. import re
  24. import logging
  25. from bb import data, utils
  26. from collections import defaultdict
  27. import bb
  28. logger = logging.getLogger("BitBake.Provider")
  29. class NoProvider(bb.BBHandledException):
  30. """Exception raised when no provider of a build dependency can be found"""
  31. class NoRProvider(bb.BBHandledException):
  32. """Exception raised when no provider of a runtime dependency can be found"""
  33. class MultipleRProvider(bb.BBHandledException):
  34. """Exception raised when multiple providers of a runtime dependency can be found"""
  35. def findProviders(cfgData, dataCache, pkg_pn = None):
  36. """
  37. Convenience function to get latest and preferred providers in pkg_pn
  38. """
  39. if not pkg_pn:
  40. pkg_pn = dataCache.pkg_pn
  41. # Need to ensure data store is expanded
  42. localdata = data.createCopy(cfgData)
  43. bb.data.update_data(localdata)
  44. bb.data.expandKeys(localdata)
  45. preferred_versions = {}
  46. latest_versions = {}
  47. for pn in pkg_pn:
  48. (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
  49. preferred_versions[pn] = (pref_ver, pref_file)
  50. latest_versions[pn] = (last_ver, last_file)
  51. return (latest_versions, preferred_versions)
  52. def allProviders(dataCache):
  53. """
  54. Find all providers for each pn
  55. """
  56. all_providers = defaultdict(list)
  57. for (fn, pn) in dataCache.pkg_fn.items():
  58. ver = dataCache.pkg_pepvpr[fn]
  59. all_providers[pn].append((ver, fn))
  60. return all_providers
  61. def sortPriorities(pn, dataCache, pkg_pn = None):
  62. """
  63. Reorder pkg_pn by file priority and default preference
  64. """
  65. if not pkg_pn:
  66. pkg_pn = dataCache.pkg_pn
  67. files = pkg_pn[pn]
  68. priorities = {}
  69. for f in files:
  70. priority = dataCache.bbfile_priority[f]
  71. preference = dataCache.pkg_dp[f]
  72. if priority not in priorities:
  73. priorities[priority] = {}
  74. if preference not in priorities[priority]:
  75. priorities[priority][preference] = []
  76. priorities[priority][preference].append(f)
  77. tmp_pn = []
  78. for pri in sorted(priorities):
  79. tmp_pref = []
  80. for pref in sorted(priorities[pri]):
  81. tmp_pref.extend(priorities[pri][pref])
  82. tmp_pn = [tmp_pref] + tmp_pn
  83. return tmp_pn
  84. def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
  85. """
  86. Check if the version pe,pv,pr is the preferred one.
  87. If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
  88. """
  89. if (pr == preferred_r or preferred_r == None):
  90. if (pe == preferred_e or preferred_e == None):
  91. if preferred_v == pv:
  92. return True
  93. if preferred_v != None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
  94. return True
  95. return False
  96. def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
  97. """
  98. Find the first provider in pkg_pn with a PREFERRED_VERSION set.
  99. """
  100. preferred_file = None
  101. preferred_ver = None
  102. # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
  103. # hence we do this manually rather than use OVERRIDES
  104. preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn, True)
  105. if not preferred_v:
  106. preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn, True)
  107. if not preferred_v:
  108. preferred_v = cfgData.getVar("PREFERRED_VERSION", True)
  109. if preferred_v:
  110. m = re.match('(\d+:)*(.*)(_.*)*', preferred_v)
  111. if m:
  112. if m.group(1):
  113. preferred_e = m.group(1)[:-1]
  114. else:
  115. preferred_e = None
  116. preferred_v = m.group(2)
  117. if m.group(3):
  118. preferred_r = m.group(3)[1:]
  119. else:
  120. preferred_r = None
  121. else:
  122. preferred_e = None
  123. preferred_r = None
  124. for file_set in pkg_pn:
  125. for f in file_set:
  126. pe, pv, pr = dataCache.pkg_pepvpr[f]
  127. if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
  128. preferred_file = f
  129. preferred_ver = (pe, pv, pr)
  130. break
  131. if preferred_file:
  132. break;
  133. if preferred_r:
  134. pv_str = '%s-%s' % (preferred_v, preferred_r)
  135. else:
  136. pv_str = preferred_v
  137. if not (preferred_e is None):
  138. pv_str = '%s:%s' % (preferred_e, pv_str)
  139. itemstr = ""
  140. if item:
  141. itemstr = " (for item %s)" % item
  142. if preferred_file is None:
  143. logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
  144. available_vers = []
  145. for file_set in pkg_pn:
  146. for f in file_set:
  147. pe, pv, pr = dataCache.pkg_pepvpr[f]
  148. ver_str = pv
  149. if pe:
  150. ver_str = "%s:%s" % (pe, ver_str)
  151. if not ver_str in available_vers:
  152. available_vers.append(ver_str)
  153. if available_vers:
  154. available_vers.sort()
  155. logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
  156. else:
  157. logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
  158. return (preferred_ver, preferred_file)
  159. def findLatestProvider(pn, cfgData, dataCache, file_set):
  160. """
  161. Return the highest version of the providers in file_set.
  162. Take default preferences into account.
  163. """
  164. latest = None
  165. latest_p = 0
  166. latest_f = None
  167. for file_name in file_set:
  168. pe, pv, pr = dataCache.pkg_pepvpr[file_name]
  169. dp = dataCache.pkg_dp[file_name]
  170. if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
  171. latest = (pe, pv, pr)
  172. latest_f = file_name
  173. latest_p = dp
  174. return (latest, latest_f)
  175. def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
  176. """
  177. If there is a PREFERRED_VERSION, find the highest-priority bbfile
  178. providing that version. If not, find the latest version provided by
  179. an bbfile in the highest-priority set.
  180. """
  181. sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
  182. # Find the highest priority provider with a PREFERRED_VERSION set
  183. (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
  184. # Find the latest version of the highest priority provider
  185. (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
  186. if preferred_file is None:
  187. preferred_file = latest_f
  188. preferred_ver = latest
  189. return (latest, latest_f, preferred_ver, preferred_file)
  190. def _filterProviders(providers, item, cfgData, dataCache):
  191. """
  192. Take a list of providers and filter/reorder according to the
  193. environment variables
  194. """
  195. eligible = []
  196. preferred_versions = {}
  197. sortpkg_pn = {}
  198. # The order of providers depends on the order of the files on the disk
  199. # up to here. Sort pkg_pn to make dependency issues reproducible rather
  200. # than effectively random.
  201. providers.sort()
  202. # Collate providers by PN
  203. pkg_pn = {}
  204. for p in providers:
  205. pn = dataCache.pkg_fn[p]
  206. if pn not in pkg_pn:
  207. pkg_pn[pn] = []
  208. pkg_pn[pn].append(p)
  209. logger.debug(1, "providers for %s are: %s", item, list(pkg_pn.keys()))
  210. # First add PREFERRED_VERSIONS
  211. for pn in pkg_pn:
  212. sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
  213. preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
  214. if preferred_versions[pn][1]:
  215. eligible.append(preferred_versions[pn][1])
  216. # Now add latest versions
  217. for pn in sortpkg_pn:
  218. if pn in preferred_versions and preferred_versions[pn][1]:
  219. continue
  220. preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
  221. eligible.append(preferred_versions[pn][1])
  222. if len(eligible) == 0:
  223. logger.error("no eligible providers for %s", item)
  224. return 0
  225. # If pn == item, give it a slight default preference
  226. # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
  227. for p in providers:
  228. pn = dataCache.pkg_fn[p]
  229. if pn != item:
  230. continue
  231. (newvers, fn) = preferred_versions[pn]
  232. if not fn in eligible:
  233. continue
  234. eligible.remove(fn)
  235. eligible = [fn] + eligible
  236. return eligible
  237. def filterProviders(providers, item, cfgData, dataCache):
  238. """
  239. Take a list of providers and filter/reorder according to the
  240. environment variables
  241. Takes a "normal" target item
  242. """
  243. eligible = _filterProviders(providers, item, cfgData, dataCache)
  244. prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, True)
  245. if prefervar:
  246. dataCache.preferred[item] = prefervar
  247. foundUnique = False
  248. if item in dataCache.preferred:
  249. for p in eligible:
  250. pn = dataCache.pkg_fn[p]
  251. if dataCache.preferred[item] == pn:
  252. logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
  253. eligible.remove(p)
  254. eligible = [p] + eligible
  255. foundUnique = True
  256. break
  257. logger.debug(1, "sorted providers for %s are: %s", item, eligible)
  258. return eligible, foundUnique
  259. def filterProvidersRunTime(providers, item, cfgData, dataCache):
  260. """
  261. Take a list of providers and filter/reorder according to the
  262. environment variables
  263. Takes a "runtime" target item
  264. """
  265. eligible = _filterProviders(providers, item, cfgData, dataCache)
  266. # First try and match any PREFERRED_RPROVIDER entry
  267. prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item, True)
  268. foundUnique = False
  269. if prefervar:
  270. for p in eligible:
  271. pn = dataCache.pkg_fn[p]
  272. if prefervar == pn:
  273. logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
  274. eligible.remove(p)
  275. eligible = [p] + eligible
  276. foundUnique = True
  277. numberPreferred = 1
  278. break
  279. # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
  280. # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
  281. # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
  282. if not foundUnique:
  283. # Should use dataCache.preferred here?
  284. preferred = []
  285. preferred_vars = []
  286. pns = {}
  287. for p in eligible:
  288. pns[dataCache.pkg_fn[p]] = p
  289. for p in eligible:
  290. pn = dataCache.pkg_fn[p]
  291. provides = dataCache.pn_provides[pn]
  292. for provide in provides:
  293. prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, True)
  294. #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
  295. if prefervar in pns and pns[prefervar] not in preferred:
  296. var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
  297. logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
  298. preferred_vars.append(var)
  299. pref = pns[prefervar]
  300. eligible.remove(pref)
  301. eligible = [pref] + eligible
  302. preferred.append(pref)
  303. break
  304. numberPreferred = len(preferred)
  305. if numberPreferred > 1:
  306. logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
  307. logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
  308. return eligible, numberPreferred
  309. regexp_cache = {}
  310. def getRuntimeProviders(dataCache, rdepend):
  311. """
  312. Return any providers of runtime dependency
  313. """
  314. rproviders = []
  315. if rdepend in dataCache.rproviders:
  316. rproviders += dataCache.rproviders[rdepend]
  317. if rdepend in dataCache.packages:
  318. rproviders += dataCache.packages[rdepend]
  319. if rproviders:
  320. return rproviders
  321. # Only search dynamic packages if we can't find anything in other variables
  322. for pattern in dataCache.packages_dynamic:
  323. pattern = pattern.replace('+', "\+")
  324. if pattern in regexp_cache:
  325. regexp = regexp_cache[pattern]
  326. else:
  327. try:
  328. regexp = re.compile(pattern)
  329. except:
  330. logger.error("Error parsing regular expression '%s'", pattern)
  331. raise
  332. regexp_cache[pattern] = regexp
  333. if regexp.match(rdepend):
  334. rproviders += dataCache.packages_dynamic[pattern]
  335. logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
  336. return rproviders
  337. def buildWorldTargetList(dataCache):
  338. """
  339. Build package list for "bitbake world"
  340. """
  341. if dataCache.world_target:
  342. return
  343. logger.debug(1, "collating packages for \"world\"")
  344. for f in dataCache.possible_world:
  345. terminal = True
  346. pn = dataCache.pkg_fn[f]
  347. for p in dataCache.pn_provides[pn]:
  348. if p.startswith('virtual/'):
  349. logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
  350. terminal = False
  351. break
  352. for pf in dataCache.providers[p]:
  353. if dataCache.pkg_fn[pf] != pn:
  354. logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
  355. terminal = False
  356. break
  357. if terminal:
  358. dataCache.world_target.add(pn)