utils.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import subprocess
  5. import multiprocessing
  6. import traceback
  7. def read_file(filename):
  8. try:
  9. f = open( filename, "r" )
  10. except IOError as reason:
  11. return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
  12. else:
  13. data = f.read().strip()
  14. f.close()
  15. return data
  16. return None
  17. def ifelse(condition, iftrue = True, iffalse = False):
  18. if condition:
  19. return iftrue
  20. else:
  21. return iffalse
  22. def conditional(variable, checkvalue, truevalue, falsevalue, d):
  23. if d.getVar(variable) == checkvalue:
  24. return truevalue
  25. else:
  26. return falsevalue
  27. def vartrue(var, iftrue, iffalse, d):
  28. import oe.types
  29. if oe.types.boolean(d.getVar(var)):
  30. return iftrue
  31. else:
  32. return iffalse
  33. def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
  34. if float(d.getVar(variable)) <= float(checkvalue):
  35. return truevalue
  36. else:
  37. return falsevalue
  38. def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
  39. result = bb.utils.vercmp_string(d.getVar(variable), checkvalue)
  40. if result <= 0:
  41. return truevalue
  42. else:
  43. return falsevalue
  44. def both_contain(variable1, variable2, checkvalue, d):
  45. val1 = d.getVar(variable1)
  46. val2 = d.getVar(variable2)
  47. val1 = set(val1.split())
  48. val2 = set(val2.split())
  49. if isinstance(checkvalue, str):
  50. checkvalue = set(checkvalue.split())
  51. else:
  52. checkvalue = set(checkvalue)
  53. if checkvalue.issubset(val1) and checkvalue.issubset(val2):
  54. return " ".join(checkvalue)
  55. else:
  56. return ""
  57. def set_intersect(variable1, variable2, d):
  58. """
  59. Expand both variables, interpret them as lists of strings, and return the
  60. intersection as a flattened string.
  61. For example:
  62. s1 = "a b c"
  63. s2 = "b c d"
  64. s3 = set_intersect(s1, s2)
  65. => s3 = "b c"
  66. """
  67. val1 = set(d.getVar(variable1).split())
  68. val2 = set(d.getVar(variable2).split())
  69. return " ".join(val1 & val2)
  70. def prune_suffix(var, suffixes, d):
  71. # See if var ends with any of the suffixes listed and
  72. # remove it if found
  73. for suffix in suffixes:
  74. if suffix and var.endswith(suffix):
  75. var = var[:-len(suffix)]
  76. prefix = d.getVar("MLPREFIX")
  77. if prefix and var.startswith(prefix):
  78. var = var[len(prefix):]
  79. return var
  80. def str_filter(f, str, d):
  81. from re import match
  82. return " ".join([x for x in str.split() if match(f, x, 0)])
  83. def str_filter_out(f, str, d):
  84. from re import match
  85. return " ".join([x for x in str.split() if not match(f, x, 0)])
  86. def build_depends_string(depends, task):
  87. """Append a taskname to a string of dependencies as used by the [depends] flag"""
  88. return " ".join(dep + ":" + task for dep in depends.split())
  89. def inherits(d, *classes):
  90. """Return True if the metadata inherits any of the specified classes"""
  91. return any(bb.data.inherits_class(cls, d) for cls in classes)
  92. def features_backfill(var,d):
  93. # This construct allows the addition of new features to variable specified
  94. # as var
  95. # Example for var = "DISTRO_FEATURES"
  96. # This construct allows the addition of new features to DISTRO_FEATURES
  97. # that if not present would disable existing functionality, without
  98. # disturbing distributions that have already set DISTRO_FEATURES.
  99. # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should
  100. # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED
  101. features = (d.getVar(var) or "").split()
  102. backfill = (d.getVar(var+"_BACKFILL") or "").split()
  103. considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split()
  104. addfeatures = []
  105. for feature in backfill:
  106. if feature not in features and feature not in considered:
  107. addfeatures.append(feature)
  108. if addfeatures:
  109. d.appendVar(var, " " + " ".join(addfeatures))
  110. def all_distro_features(d, features, truevalue="1", falsevalue=""):
  111. """
  112. Returns truevalue if *all* given features are set in DISTRO_FEATURES,
  113. else falsevalue. The features can be given as single string or anything
  114. that can be turned into a set.
  115. This is a shorter, more flexible version of
  116. bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
  117. Without explicit true/false values it can be used directly where
  118. Python expects a boolean:
  119. if oe.utils.all_distro_features(d, "foo bar"):
  120. bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
  121. With just a truevalue, it can be used to include files that are meant to be
  122. used only when requested via DISTRO_FEATURES:
  123. require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
  124. """
  125. return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
  126. def any_distro_features(d, features, truevalue="1", falsevalue=""):
  127. """
  128. Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
  129. else falsevalue. The features can be given as single string or anything
  130. that can be turned into a set.
  131. This is a shorter, more flexible version of
  132. bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
  133. Without explicit true/false values it can be used directly where
  134. Python expects a boolean:
  135. if not oe.utils.any_distro_features(d, "foo bar"):
  136. bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
  137. With just a truevalue, it can be used to include files that are meant to be
  138. used only when requested via DISTRO_FEATURES:
  139. require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
  140. """
  141. return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
  142. def parallel_make(d, makeinst=False):
  143. """
  144. Return the integer value for the number of parallel threads to use when
  145. building, scraped out of PARALLEL_MAKE. If no parallelization option is
  146. found, returns None
  147. e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
  148. """
  149. if makeinst:
  150. pm = (d.getVar('PARALLEL_MAKEINST') or '').split()
  151. else:
  152. pm = (d.getVar('PARALLEL_MAKE') or '').split()
  153. # look for '-j' and throw other options (e.g. '-l') away
  154. while pm:
  155. opt = pm.pop(0)
  156. if opt == '-j':
  157. v = pm.pop(0)
  158. elif opt.startswith('-j'):
  159. v = opt[2:].strip()
  160. else:
  161. continue
  162. return int(v)
  163. return None
  164. def parallel_make_argument(d, fmt, limit=None, makeinst=False):
  165. """
  166. Helper utility to construct a parallel make argument from the number of
  167. parallel threads specified in PARALLEL_MAKE.
  168. Returns the input format string `fmt` where a single '%d' will be expanded
  169. with the number of parallel threads to use. If `limit` is specified, the
  170. number of parallel threads will be no larger than it. If no parallelization
  171. option is found in PARALLEL_MAKE, returns an empty string
  172. e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
  173. "-n 10"
  174. """
  175. v = parallel_make(d, makeinst)
  176. if v:
  177. if limit:
  178. v = min(limit, v)
  179. return fmt % v
  180. return ''
  181. def packages_filter_out_system(d):
  182. """
  183. Return a list of packages from PACKAGES with the "system" packages such as
  184. PN-dbg PN-doc PN-locale-eb-gb removed.
  185. """
  186. pn = d.getVar('PN')
  187. blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')]
  188. localepkg = pn + "-locale-"
  189. pkgs = []
  190. for pkg in d.getVar('PACKAGES').split():
  191. if pkg not in blacklist and localepkg not in pkg:
  192. pkgs.append(pkg)
  193. return pkgs
  194. def getstatusoutput(cmd):
  195. return subprocess.getstatusoutput(cmd)
  196. def trim_version(version, num_parts=2):
  197. """
  198. Return just the first <num_parts> of <version>, split by periods. For
  199. example, trim_version("1.2.3", 2) will return "1.2".
  200. """
  201. if type(version) is not str:
  202. raise TypeError("Version should be a string")
  203. if num_parts < 1:
  204. raise ValueError("Cannot split to parts < 1")
  205. parts = version.split(".")
  206. trimmed = ".".join(parts[:num_parts])
  207. return trimmed
  208. def cpu_count(at_least=1):
  209. cpus = len(os.sched_getaffinity(0))
  210. return max(cpus, at_least)
  211. def execute_pre_post_process(d, cmds):
  212. if cmds is None:
  213. return
  214. for cmd in cmds.strip().split(';'):
  215. cmd = cmd.strip()
  216. if cmd != '':
  217. bb.note("Executing %s ..." % cmd)
  218. bb.build.exec_func(cmd, d)
  219. # For each item in items, call the function 'target' with item as the first
  220. # argument, extraargs as the other arguments and handle any exceptions in the
  221. # parent thread
  222. def multiprocess_launch(target, items, d, extraargs=None):
  223. class ProcessLaunch(multiprocessing.Process):
  224. def __init__(self, *args, **kwargs):
  225. multiprocessing.Process.__init__(self, *args, **kwargs)
  226. self._pconn, self._cconn = multiprocessing.Pipe()
  227. self._exception = None
  228. self._result = None
  229. def run(self):
  230. try:
  231. ret = self._target(*self._args, **self._kwargs)
  232. self._cconn.send((None, ret))
  233. except Exception as e:
  234. tb = traceback.format_exc()
  235. self._cconn.send((e, tb))
  236. def update(self):
  237. if self._pconn.poll():
  238. (e, tb) = self._pconn.recv()
  239. if e is not None:
  240. self._exception = (e, tb)
  241. else:
  242. self._result = tb
  243. @property
  244. def exception(self):
  245. self.update()
  246. return self._exception
  247. @property
  248. def result(self):
  249. self.update()
  250. return self._result
  251. max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
  252. launched = []
  253. errors = []
  254. results = []
  255. items = list(items)
  256. while (items and not errors) or launched:
  257. if not errors and items and len(launched) < max_process:
  258. args = (items.pop(),)
  259. if extraargs is not None:
  260. args = args + extraargs
  261. p = ProcessLaunch(target=target, args=args)
  262. p.start()
  263. launched.append(p)
  264. for q in launched:
  265. # Have to manually call update() to avoid deadlocks. The pipe can be full and
  266. # transfer stalled until we try and read the results object but the subprocess won't exit
  267. # as it still has data to write (https://bugs.python.org/issue8426)
  268. q.update()
  269. # The finished processes are joined when calling is_alive()
  270. if not q.is_alive():
  271. if q.exception:
  272. errors.append(q.exception)
  273. if q.result:
  274. results.append(q.result)
  275. launched.remove(q)
  276. # Paranoia doesn't hurt
  277. for p in launched:
  278. p.join()
  279. if errors:
  280. msg = ""
  281. for (e, tb) in errors:
  282. if isinstance(e, subprocess.CalledProcessError) and e.output:
  283. msg = msg + str(e) + "\n"
  284. msg = msg + "Subprocess output:"
  285. msg = msg + e.output.decode("utf-8", errors="ignore")
  286. else:
  287. msg = msg + str(e) + ": " + str(tb) + "\n"
  288. bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg)
  289. return results
  290. def squashspaces(string):
  291. import re
  292. return re.sub(r"\s+", " ", string).strip()
  293. def format_pkg_list(pkg_dict, ret_format=None):
  294. output = []
  295. if ret_format == "arch":
  296. for pkg in sorted(pkg_dict):
  297. output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"]))
  298. elif ret_format == "file":
  299. for pkg in sorted(pkg_dict):
  300. output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"]))
  301. elif ret_format == "ver":
  302. for pkg in sorted(pkg_dict):
  303. output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
  304. elif ret_format == "deps":
  305. for pkg in sorted(pkg_dict):
  306. for dep in pkg_dict[pkg]["deps"]:
  307. output.append("%s|%s" % (pkg, dep))
  308. else:
  309. for pkg in sorted(pkg_dict):
  310. output.append(pkg)
  311. output_str = '\n'.join(output)
  312. if output_str:
  313. # make sure last line is newline terminated
  314. output_str += '\n'
  315. return output_str
  316. # Helper function to get the host compiler version
  317. # Do not assume the compiler is gcc
  318. def get_host_compiler_version(d, taskcontextonly=False):
  319. import re, subprocess
  320. if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
  321. return
  322. compiler = d.getVar("BUILD_CC")
  323. # Get rid of ccache since it is not present when parsing.
  324. if compiler.startswith('ccache '):
  325. compiler = compiler[7:]
  326. try:
  327. env = os.environ.copy()
  328. # datastore PATH does not contain session PATH as set by environment-setup-...
  329. # this breaks the install-buildtools use-case
  330. # env["PATH"] = d.getVar("PATH")
  331. output = subprocess.check_output("%s --version" % compiler, \
  332. shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
  333. except subprocess.CalledProcessError as e:
  334. bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
  335. match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
  336. if not match:
  337. bb.fatal("Can't get compiler version from %s --version output" % compiler)
  338. version = match.group(1)
  339. return compiler, version
  340. def host_gcc_version(d, taskcontextonly=False):
  341. import re, subprocess
  342. if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
  343. return
  344. compiler = d.getVar("BUILD_CC")
  345. # Get rid of ccache since it is not present when parsing.
  346. if compiler.startswith('ccache '):
  347. compiler = compiler[7:]
  348. try:
  349. env = os.environ.copy()
  350. env["PATH"] = d.getVar("PATH")
  351. output = subprocess.check_output("%s --version" % compiler, \
  352. shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
  353. except subprocess.CalledProcessError as e:
  354. bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
  355. match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
  356. if not match:
  357. bb.fatal("Can't get compiler version from %s --version output" % compiler)
  358. version = match.group(1)
  359. return "-%s" % version if version in ("4.8", "4.9") else ""
  360. def get_multilib_datastore(variant, d):
  361. localdata = bb.data.createCopy(d)
  362. if variant:
  363. overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant
  364. localdata.setVar("OVERRIDES", overrides)
  365. localdata.setVar("MLPREFIX", variant + "-")
  366. else:
  367. origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL")
  368. if origdefault:
  369. localdata.setVar("DEFAULTTUNE", origdefault)
  370. overrides = localdata.getVar("OVERRIDES", False).split(":")
  371. overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")])
  372. localdata.setVar("OVERRIDES", overrides)
  373. localdata.setVar("MLPREFIX", "")
  374. return localdata
  375. #
  376. # Python 2.7 doesn't have threaded pools (just multiprocessing)
  377. # so implement a version here
  378. #
  379. from queue import Queue
  380. from threading import Thread
  381. class ThreadedWorker(Thread):
  382. """Thread executing tasks from a given tasks queue"""
  383. def __init__(self, tasks, worker_init, worker_end):
  384. Thread.__init__(self)
  385. self.tasks = tasks
  386. self.daemon = True
  387. self.worker_init = worker_init
  388. self.worker_end = worker_end
  389. def run(self):
  390. from queue import Empty
  391. if self.worker_init is not None:
  392. self.worker_init(self)
  393. while True:
  394. try:
  395. func, args, kargs = self.tasks.get(block=False)
  396. except Empty:
  397. if self.worker_end is not None:
  398. self.worker_end(self)
  399. break
  400. try:
  401. func(self, *args, **kargs)
  402. except Exception as e:
  403. print(e)
  404. finally:
  405. self.tasks.task_done()
  406. class ThreadedPool:
  407. """Pool of threads consuming tasks from a queue"""
  408. def __init__(self, num_workers, num_tasks, worker_init=None,
  409. worker_end=None):
  410. self.tasks = Queue(num_tasks)
  411. self.workers = []
  412. for _ in range(num_workers):
  413. worker = ThreadedWorker(self.tasks, worker_init, worker_end)
  414. self.workers.append(worker)
  415. def start(self):
  416. for worker in self.workers:
  417. worker.start()
  418. def add_task(self, func, *args, **kargs):
  419. """Add a task to the queue"""
  420. self.tasks.put((func, args, kargs))
  421. def wait_completion(self):
  422. """Wait for completion of all the tasks in the queue"""
  423. self.tasks.join()
  424. for worker in self.workers:
  425. worker.join()
  426. def write_ld_so_conf(d):
  427. # Some utils like prelink may not have the correct target library paths
  428. # so write an ld.so.conf to help them
  429. ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf")
  430. if os.path.exists(ldsoconf):
  431. bb.utils.remove(ldsoconf)
  432. bb.utils.mkdirhier(os.path.dirname(ldsoconf))
  433. with open(ldsoconf, "w") as f:
  434. f.write(d.getVar("base_libdir") + '\n')
  435. f.write(d.getVar("libdir") + '\n')
  436. class ImageQAFailed(Exception):
  437. def __init__(self, description, name=None, logfile=None):
  438. self.description = description
  439. self.name = name
  440. self.logfile=logfile
  441. def __str__(self):
  442. msg = 'Function failed: %s' % self.name
  443. if self.description:
  444. msg = msg + ' (%s)' % self.description
  445. return msg
  446. def sh_quote(string):
  447. import shlex
  448. return shlex.quote(string)