misc.py 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. #
  2. # Copyright (c) 2013, Intel Corporation.
  3. #
  4. # SPDX-License-Identifier: GPL-2.0-only
  5. #
  6. # DESCRIPTION
  7. # This module provides a place to collect various wic-related utils
  8. # for the OpenEmbedded Image Tools.
  9. #
  10. # AUTHORS
  11. # Tom Zanussi <tom.zanussi (at] linux.intel.com>
  12. #
  13. """Miscellaneous functions."""
  14. import logging
  15. import os
  16. import re
  17. import subprocess
  18. from collections import defaultdict
  19. from distutils import spawn
  20. from wic import WicError
  21. logger = logging.getLogger('wic')
  22. # executable -> recipe pairs for exec_native_cmd
  23. NATIVE_RECIPES = {"bmaptool": "bmap-tools",
  24. "grub-mkimage": "grub-efi",
  25. "isohybrid": "syslinux",
  26. "mcopy": "mtools",
  27. "mdel" : "mtools",
  28. "mdeltree" : "mtools",
  29. "mdir" : "mtools",
  30. "mkdosfs": "dosfstools",
  31. "mkisofs": "cdrtools",
  32. "mkfs.btrfs": "btrfs-tools",
  33. "mkfs.ext2": "e2fsprogs",
  34. "mkfs.ext3": "e2fsprogs",
  35. "mkfs.ext4": "e2fsprogs",
  36. "mkfs.vfat": "dosfstools",
  37. "mksquashfs": "squashfs-tools",
  38. "mkswap": "util-linux",
  39. "mmd": "mtools",
  40. "parted": "parted",
  41. "sfdisk": "util-linux",
  42. "sgdisk": "gptfdisk",
  43. "syslinux": "syslinux",
  44. "tar": "tar"
  45. }
  46. def runtool(cmdln_or_args):
  47. """ wrapper for most of the subprocess calls
  48. input:
  49. cmdln_or_args: can be both args and cmdln str (shell=True)
  50. return:
  51. rc, output
  52. """
  53. if isinstance(cmdln_or_args, list):
  54. cmd = cmdln_or_args[0]
  55. shell = False
  56. else:
  57. import shlex
  58. cmd = shlex.split(cmdln_or_args)[0]
  59. shell = True
  60. sout = subprocess.PIPE
  61. serr = subprocess.STDOUT
  62. try:
  63. process = subprocess.Popen(cmdln_or_args, stdout=sout,
  64. stderr=serr, shell=shell)
  65. sout, serr = process.communicate()
  66. # combine stdout and stderr, filter None out and decode
  67. out = ''.join([out.decode('utf-8') for out in [sout, serr] if out])
  68. except OSError as err:
  69. if err.errno == 2:
  70. # [Errno 2] No such file or directory
  71. raise WicError('Cannot run command: %s, lost dependency?' % cmd)
  72. else:
  73. raise # relay
  74. return process.returncode, out
  75. def _exec_cmd(cmd_and_args, as_shell=False):
  76. """
  77. Execute command, catching stderr, stdout
  78. Need to execute as_shell if the command uses wildcards
  79. """
  80. logger.debug("_exec_cmd: %s", cmd_and_args)
  81. args = cmd_and_args.split()
  82. logger.debug(args)
  83. if as_shell:
  84. ret, out = runtool(cmd_and_args)
  85. else:
  86. ret, out = runtool(args)
  87. out = out.strip()
  88. if ret != 0:
  89. raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \
  90. (cmd_and_args, ret, out))
  91. logger.debug("_exec_cmd: output for %s (rc = %d): %s",
  92. cmd_and_args, ret, out)
  93. return ret, out
  94. def exec_cmd(cmd_and_args, as_shell=False):
  95. """
  96. Execute command, return output
  97. """
  98. return _exec_cmd(cmd_and_args, as_shell)[1]
  99. def find_executable(cmd, paths):
  100. recipe = cmd
  101. if recipe in NATIVE_RECIPES:
  102. recipe = NATIVE_RECIPES[recipe]
  103. provided = get_bitbake_var("ASSUME_PROVIDED")
  104. if provided and "%s-native" % recipe in provided:
  105. return True
  106. return spawn.find_executable(cmd, paths)
  107. def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
  108. """
  109. Execute native command, catching stderr, stdout
  110. Need to execute as_shell if the command uses wildcards
  111. Always need to execute native commands as_shell
  112. """
  113. # The reason -1 is used is because there may be "export" commands.
  114. args = cmd_and_args.split(';')[-1].split()
  115. logger.debug(args)
  116. if pseudo:
  117. cmd_and_args = pseudo + cmd_and_args
  118. native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin" % \
  119. (native_sysroot, native_sysroot,
  120. native_sysroot, native_sysroot)
  121. native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
  122. (native_paths, cmd_and_args)
  123. logger.debug("exec_native_cmd: %s", native_cmd_and_args)
  124. # If the command isn't in the native sysroot say we failed.
  125. if find_executable(args[0], native_paths):
  126. ret, out = _exec_cmd(native_cmd_and_args, True)
  127. else:
  128. ret = 127
  129. out = "can't find native executable %s in %s" % (args[0], native_paths)
  130. prog = args[0]
  131. # shell command-not-found
  132. if ret == 127 \
  133. or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog):
  134. msg = "A native program %s required to build the image "\
  135. "was not found (see details above).\n\n" % prog
  136. recipe = NATIVE_RECIPES.get(prog)
  137. if recipe:
  138. msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\
  139. "build it with 'bitbake wic-tools' and try again.\n" % recipe
  140. else:
  141. msg += "Wic failed to find a recipe to build native %s. Please "\
  142. "file a bug against wic.\n" % prog
  143. raise WicError(msg)
  144. return ret, out
  145. BOOTDD_EXTRA_SPACE = 16384
  146. class BitbakeVars(defaultdict):
  147. """
  148. Container for Bitbake variables.
  149. """
  150. def __init__(self):
  151. defaultdict.__init__(self, dict)
  152. # default_image and vars_dir attributes should be set from outside
  153. self.default_image = None
  154. self.vars_dir = None
  155. def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")):
  156. """
  157. Parse one line from bitbake -e output or from .env file.
  158. Put result key-value pair into the storage.
  159. """
  160. if "=" not in line:
  161. return
  162. match = matcher.match(line)
  163. if not match:
  164. return
  165. key, val = match.groups()
  166. self[image][key] = val.strip('"')
  167. def get_var(self, var, image=None, cache=True):
  168. """
  169. Get bitbake variable from 'bitbake -e' output or from .env file.
  170. This is a lazy method, i.e. it runs bitbake or parses file only when
  171. only when variable is requested. It also caches results.
  172. """
  173. if not image:
  174. image = self.default_image
  175. if image not in self:
  176. if image and self.vars_dir:
  177. fname = os.path.join(self.vars_dir, image + '.env')
  178. if os.path.isfile(fname):
  179. # parse .env file
  180. with open(fname) as varsfile:
  181. for line in varsfile:
  182. self._parse_line(line, image)
  183. else:
  184. print("Couldn't get bitbake variable from %s." % fname)
  185. print("File %s doesn't exist." % fname)
  186. return
  187. else:
  188. # Get bitbake -e output
  189. cmd = "bitbake -e"
  190. if image:
  191. cmd += " %s" % image
  192. log_level = logger.getEffectiveLevel()
  193. logger.setLevel(logging.INFO)
  194. ret, lines = _exec_cmd(cmd)
  195. logger.setLevel(log_level)
  196. if ret:
  197. logger.error("Couldn't get '%s' output.", cmd)
  198. logger.error("Bitbake failed with error:\n%s\n", lines)
  199. return
  200. # Parse bitbake -e output
  201. for line in lines.split('\n'):
  202. self._parse_line(line, image)
  203. # Make first image a default set of variables
  204. if cache:
  205. images = [key for key in self if key]
  206. if len(images) == 1:
  207. self[None] = self[image]
  208. result = self[image].get(var)
  209. if not cache:
  210. self.pop(image, None)
  211. return result
  212. # Create BB_VARS singleton
  213. BB_VARS = BitbakeVars()
  214. def get_bitbake_var(var, image=None, cache=True):
  215. """
  216. Provide old get_bitbake_var API by wrapping
  217. get_var method of BB_VARS singleton.
  218. """
  219. return BB_VARS.get_var(var, image, cache)