providers.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418
  1. #
  2. # Copyright (C) 2003, 2004 Chris Larson
  3. # Copyright (C) 2003, 2004 Phil Blundell
  4. # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
  5. # Copyright (C) 2005 Holger Hans Peter Freyther
  6. # Copyright (C) 2005 ROAD GmbH
  7. # Copyright (C) 2006 Richard Purdie
  8. #
  9. # SPDX-License-Identifier: GPL-2.0-only
  10. #
  11. import re
  12. import logging
  13. from bb import data, utils
  14. from collections import defaultdict
  15. import bb
  16. logger = logging.getLogger("BitBake.Provider")
  17. class NoProvider(bb.BBHandledException):
  18. """Exception raised when no provider of a build dependency can be found"""
  19. class NoRProvider(bb.BBHandledException):
  20. """Exception raised when no provider of a runtime dependency can be found"""
  21. class MultipleRProvider(bb.BBHandledException):
  22. """Exception raised when multiple providers of a runtime dependency can be found"""
  23. def findProviders(cfgData, dataCache, pkg_pn = None):
  24. """
  25. Convenience function to get latest and preferred providers in pkg_pn
  26. """
  27. if not pkg_pn:
  28. pkg_pn = dataCache.pkg_pn
  29. # Need to ensure data store is expanded
  30. localdata = data.createCopy(cfgData)
  31. bb.data.expandKeys(localdata)
  32. preferred_versions = {}
  33. latest_versions = {}
  34. for pn in pkg_pn:
  35. (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
  36. preferred_versions[pn] = (pref_ver, pref_file)
  37. latest_versions[pn] = (last_ver, last_file)
  38. return (latest_versions, preferred_versions)
  39. def allProviders(dataCache):
  40. """
  41. Find all providers for each pn
  42. """
  43. all_providers = defaultdict(list)
  44. for (fn, pn) in dataCache.pkg_fn.items():
  45. ver = dataCache.pkg_pepvpr[fn]
  46. all_providers[pn].append((ver, fn))
  47. return all_providers
  48. def sortPriorities(pn, dataCache, pkg_pn = None):
  49. """
  50. Reorder pkg_pn by file priority and default preference
  51. """
  52. if not pkg_pn:
  53. pkg_pn = dataCache.pkg_pn
  54. files = pkg_pn[pn]
  55. priorities = {}
  56. for f in files:
  57. priority = dataCache.bbfile_priority[f]
  58. preference = dataCache.pkg_dp[f]
  59. if priority not in priorities:
  60. priorities[priority] = {}
  61. if preference not in priorities[priority]:
  62. priorities[priority][preference] = []
  63. priorities[priority][preference].append(f)
  64. tmp_pn = []
  65. for pri in sorted(priorities):
  66. tmp_pref = []
  67. for pref in sorted(priorities[pri]):
  68. tmp_pref.extend(priorities[pri][pref])
  69. tmp_pn = [tmp_pref] + tmp_pn
  70. return tmp_pn
  71. def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
  72. """
  73. Check if the version pe,pv,pr is the preferred one.
  74. If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%'
  75. """
  76. if pr == preferred_r or preferred_r is None:
  77. if pe == preferred_e or preferred_e is None:
  78. if preferred_v == pv:
  79. return True
  80. if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]):
  81. return True
  82. return False
  83. def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
  84. """
  85. Find the first provider in pkg_pn with a PREFERRED_VERSION set.
  86. """
  87. preferred_file = None
  88. preferred_ver = None
  89. # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
  90. # hence we do this manually rather than use OVERRIDES
  91. preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
  92. if not preferred_v:
  93. preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
  94. if not preferred_v:
  95. preferred_v = cfgData.getVar("PREFERRED_VERSION")
  96. if preferred_v:
  97. m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
  98. if m:
  99. if m.group(1):
  100. preferred_e = m.group(1)[:-1]
  101. else:
  102. preferred_e = None
  103. preferred_v = m.group(2)
  104. if m.group(3):
  105. preferred_r = m.group(3)[1:]
  106. else:
  107. preferred_r = None
  108. else:
  109. preferred_e = None
  110. preferred_r = None
  111. for file_set in pkg_pn:
  112. for f in file_set:
  113. pe, pv, pr = dataCache.pkg_pepvpr[f]
  114. if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
  115. preferred_file = f
  116. preferred_ver = (pe, pv, pr)
  117. break
  118. if preferred_file:
  119. break;
  120. if preferred_r:
  121. pv_str = '%s-%s' % (preferred_v, preferred_r)
  122. else:
  123. pv_str = preferred_v
  124. if not (preferred_e is None):
  125. pv_str = '%s:%s' % (preferred_e, pv_str)
  126. itemstr = ""
  127. if item:
  128. itemstr = " (for item %s)" % item
  129. if preferred_file is None:
  130. logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
  131. available_vers = []
  132. for file_set in pkg_pn:
  133. for f in file_set:
  134. pe, pv, pr = dataCache.pkg_pepvpr[f]
  135. ver_str = pv
  136. if pe:
  137. ver_str = "%s:%s" % (pe, ver_str)
  138. if not ver_str in available_vers:
  139. available_vers.append(ver_str)
  140. if available_vers:
  141. available_vers.sort()
  142. logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
  143. else:
  144. logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
  145. return (preferred_ver, preferred_file)
  146. def findLatestProvider(pn, cfgData, dataCache, file_set):
  147. """
  148. Return the highest version of the providers in file_set.
  149. Take default preferences into account.
  150. """
  151. latest = None
  152. latest_p = 0
  153. latest_f = None
  154. for file_name in file_set:
  155. pe, pv, pr = dataCache.pkg_pepvpr[file_name]
  156. dp = dataCache.pkg_dp[file_name]
  157. if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p):
  158. latest = (pe, pv, pr)
  159. latest_f = file_name
  160. latest_p = dp
  161. return (latest, latest_f)
  162. def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
  163. """
  164. If there is a PREFERRED_VERSION, find the highest-priority bbfile
  165. providing that version. If not, find the latest version provided by
  166. an bbfile in the highest-priority set.
  167. """
  168. sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
  169. # Find the highest priority provider with a PREFERRED_VERSION set
  170. (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
  171. # Find the latest version of the highest priority provider
  172. (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
  173. if preferred_file is None:
  174. preferred_file = latest_f
  175. preferred_ver = latest
  176. return (latest, latest_f, preferred_ver, preferred_file)
  177. def _filterProviders(providers, item, cfgData, dataCache):
  178. """
  179. Take a list of providers and filter/reorder according to the
  180. environment variables
  181. """
  182. eligible = []
  183. preferred_versions = {}
  184. sortpkg_pn = {}
  185. # The order of providers depends on the order of the files on the disk
  186. # up to here. Sort pkg_pn to make dependency issues reproducible rather
  187. # than effectively random.
  188. providers.sort()
  189. # Collate providers by PN
  190. pkg_pn = {}
  191. for p in providers:
  192. pn = dataCache.pkg_fn[p]
  193. if pn not in pkg_pn:
  194. pkg_pn[pn] = []
  195. pkg_pn[pn].append(p)
  196. logger.debug(1, "providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
  197. # First add PREFERRED_VERSIONS
  198. for pn in sorted(pkg_pn):
  199. sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
  200. preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
  201. if preferred_versions[pn][1]:
  202. eligible.append(preferred_versions[pn][1])
  203. # Now add latest versions
  204. for pn in sorted(sortpkg_pn):
  205. if pn in preferred_versions and preferred_versions[pn][1]:
  206. continue
  207. preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
  208. eligible.append(preferred_versions[pn][1])
  209. if len(eligible) == 0:
  210. logger.error("no eligible providers for %s", item)
  211. return 0
  212. # If pn == item, give it a slight default preference
  213. # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
  214. for p in providers:
  215. pn = dataCache.pkg_fn[p]
  216. if pn != item:
  217. continue
  218. (newvers, fn) = preferred_versions[pn]
  219. if not fn in eligible:
  220. continue
  221. eligible.remove(fn)
  222. eligible = [fn] + eligible
  223. return eligible
  224. def filterProviders(providers, item, cfgData, dataCache):
  225. """
  226. Take a list of providers and filter/reorder according to the
  227. environment variables
  228. Takes a "normal" target item
  229. """
  230. eligible = _filterProviders(providers, item, cfgData, dataCache)
  231. prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item)
  232. if prefervar:
  233. dataCache.preferred[item] = prefervar
  234. foundUnique = False
  235. if item in dataCache.preferred:
  236. for p in eligible:
  237. pn = dataCache.pkg_fn[p]
  238. if dataCache.preferred[item] == pn:
  239. logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item)
  240. eligible.remove(p)
  241. eligible = [p] + eligible
  242. foundUnique = True
  243. break
  244. logger.debug(1, "sorted providers for %s are: %s", item, eligible)
  245. return eligible, foundUnique
  246. def filterProvidersRunTime(providers, item, cfgData, dataCache):
  247. """
  248. Take a list of providers and filter/reorder according to the
  249. environment variables
  250. Takes a "runtime" target item
  251. """
  252. eligible = _filterProviders(providers, item, cfgData, dataCache)
  253. # First try and match any PREFERRED_RPROVIDER entry
  254. prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item)
  255. foundUnique = False
  256. if prefervar:
  257. for p in eligible:
  258. pn = dataCache.pkg_fn[p]
  259. if prefervar == pn:
  260. logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item)
  261. eligible.remove(p)
  262. eligible = [p] + eligible
  263. foundUnique = True
  264. numberPreferred = 1
  265. break
  266. # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries
  267. # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set.
  268. # This is most useful for virtual/ entries rather than having a RPROVIDER per entry.
  269. if not foundUnique:
  270. # Should use dataCache.preferred here?
  271. preferred = []
  272. preferred_vars = []
  273. pns = {}
  274. for p in eligible:
  275. pns[dataCache.pkg_fn[p]] = p
  276. for p in eligible:
  277. pn = dataCache.pkg_fn[p]
  278. provides = dataCache.pn_provides[pn]
  279. for provide in provides:
  280. prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
  281. #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
  282. if prefervar in pns and pns[prefervar] not in preferred:
  283. var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
  284. logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
  285. preferred_vars.append(var)
  286. pref = pns[prefervar]
  287. eligible.remove(pref)
  288. eligible = [pref] + eligible
  289. preferred.append(pref)
  290. break
  291. numberPreferred = len(preferred)
  292. if numberPreferred > 1:
  293. logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
  294. logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
  295. return eligible, numberPreferred
  296. regexp_cache = {}
  297. def getRuntimeProviders(dataCache, rdepend):
  298. """
  299. Return any providers of runtime dependency
  300. """
  301. rproviders = []
  302. if rdepend in dataCache.rproviders:
  303. rproviders += dataCache.rproviders[rdepend]
  304. if rdepend in dataCache.packages:
  305. rproviders += dataCache.packages[rdepend]
  306. if rproviders:
  307. return rproviders
  308. # Only search dynamic packages if we can't find anything in other variables
  309. for pattern in dataCache.packages_dynamic:
  310. pattern = pattern.replace(r'+', r"\+")
  311. if pattern in regexp_cache:
  312. regexp = regexp_cache[pattern]
  313. else:
  314. try:
  315. regexp = re.compile(pattern)
  316. except:
  317. logger.error("Error parsing regular expression '%s'", pattern)
  318. raise
  319. regexp_cache[pattern] = regexp
  320. if regexp.match(rdepend):
  321. rproviders += dataCache.packages_dynamic[pattern]
  322. logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
  323. return rproviders
  324. def buildWorldTargetList(dataCache, task=None):
  325. """
  326. Build package list for "bitbake world"
  327. """
  328. if dataCache.world_target:
  329. return
  330. logger.debug(1, "collating packages for \"world\"")
  331. for f in dataCache.possible_world:
  332. terminal = True
  333. pn = dataCache.pkg_fn[f]
  334. if task and task not in dataCache.task_deps[f]['tasks']:
  335. logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
  336. terminal = False
  337. for p in dataCache.pn_provides[pn]:
  338. if p.startswith('virtual/'):
  339. logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
  340. terminal = False
  341. break
  342. for pf in dataCache.providers[p]:
  343. if dataCache.pkg_fn[pf] != pn:
  344. logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
  345. terminal = False
  346. break
  347. if terminal:
  348. dataCache.world_target.add(pn)