rootfs.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. from abc import ABCMeta, abstractmethod
  5. from oe.utils import execute_pre_post_process
  6. from oe.package_manager import *
  7. from oe.manifest import *
  8. import oe.path
  9. import shutil
  10. import os
  11. import subprocess
  12. import re
  13. from oe.package_manager.rpm.manifest import RpmManifest
  14. from oe.package_manager.ipk.manifest import OpkgManifest
  15. from oe.package_manager.deb.manifest import DpkgManifest
  16. from oe.package_manager.rpm import RpmPkgsList
  17. from oe.package_manager.ipk import OpkgPkgsList
  18. from oe.package_manager.deb import DpkgPkgsList
  19. class Rootfs(object, metaclass=ABCMeta):
  20. """
  21. This is an abstract class. Do not instantiate this directly.
  22. """
  23. def __init__(self, d, progress_reporter=None, logcatcher=None):
  24. self.d = d
  25. self.pm = None
  26. self.image_rootfs = self.d.getVar('IMAGE_ROOTFS')
  27. self.deploydir = self.d.getVar('IMGDEPLOYDIR')
  28. self.progress_reporter = progress_reporter
  29. self.logcatcher = logcatcher
  30. self.install_order = Manifest.INSTALL_ORDER
  31. @abstractmethod
  32. def _create(self):
  33. pass
  34. @abstractmethod
  35. def _get_delayed_postinsts(self):
  36. pass
  37. @abstractmethod
  38. def _save_postinsts(self):
  39. pass
  40. @abstractmethod
  41. def _log_check(self):
  42. pass
  43. def _log_check_common(self, type, match):
  44. # Ignore any lines containing log_check to avoid recursion, and ignore
  45. # lines beginning with a + since sh -x may emit code which isn't
  46. # actually executed, but may contain error messages
  47. excludes = [ 'log_check', r'^\+' ]
  48. if hasattr(self, 'log_check_expected_regexes'):
  49. excludes.extend(self.log_check_expected_regexes)
  50. # Insert custom log_check excludes
  51. excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x]
  52. excludes = [re.compile(x) for x in excludes]
  53. r = re.compile(match)
  54. log_path = self.d.expand("${T}/log.do_rootfs")
  55. messages = []
  56. with open(log_path, 'r') as log:
  57. for line in log:
  58. if self.logcatcher and self.logcatcher.contains(line.rstrip()):
  59. continue
  60. for ee in excludes:
  61. m = ee.search(line)
  62. if m:
  63. break
  64. if m:
  65. continue
  66. m = r.search(line)
  67. if m:
  68. messages.append('[log_check] %s' % line)
  69. if messages:
  70. if len(messages) == 1:
  71. msg = '1 %s message' % type
  72. else:
  73. msg = '%d %s messages' % (len(messages), type)
  74. msg = '[log_check] %s: found %s in the logfile:\n%s' % \
  75. (self.d.getVar('PN'), msg, ''.join(messages))
  76. if type == 'error':
  77. bb.fatal(msg)
  78. else:
  79. bb.warn(msg)
  80. def _log_check_warn(self):
  81. self._log_check_common('warning', '^(warn|Warn|WARNING:)')
  82. def _log_check_error(self):
  83. self._log_check_common('error', self.log_check_regex)
  84. def _insert_feed_uris(self):
  85. if bb.utils.contains("IMAGE_FEATURES", "package-management",
  86. True, False, self.d):
  87. self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "",
  88. self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "",
  89. self.d.getVar('PACKAGE_FEED_ARCHS'))
  90. """
  91. The _cleanup() method should be used to clean-up stuff that we don't really
  92. want to end up on target. For example, in the case of RPM, the DB locks.
  93. The method is called, once, at the end of create() method.
  94. """
  95. @abstractmethod
  96. def _cleanup(self):
  97. pass
  98. def _setup_dbg_rootfs(self, dirs):
  99. gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
  100. if gen_debugfs != '1':
  101. return
  102. bb.note(" Renaming the original rootfs...")
  103. try:
  104. shutil.rmtree(self.image_rootfs + '-orig')
  105. except:
  106. pass
  107. os.rename(self.image_rootfs, self.image_rootfs + '-orig')
  108. bb.note(" Creating debug rootfs...")
  109. bb.utils.mkdirhier(self.image_rootfs)
  110. bb.note(" Copying back package database...")
  111. for dir in dirs:
  112. if not os.path.isdir(self.image_rootfs + '-orig' + dir):
  113. continue
  114. bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir))
  115. shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True)
  116. # Copy files located in /usr/lib/debug or /usr/src/debug
  117. for dir in ["/usr/lib/debug", "/usr/src/debug"]:
  118. src = self.image_rootfs + '-orig' + dir
  119. if os.path.exists(src):
  120. dst = self.image_rootfs + dir
  121. bb.utils.mkdirhier(os.path.dirname(dst))
  122. shutil.copytree(src, dst)
  123. # Copy files with suffix '.debug' or located in '.debug' dir.
  124. for root, dirs, files in os.walk(self.image_rootfs + '-orig'):
  125. relative_dir = root[len(self.image_rootfs + '-orig'):]
  126. for f in files:
  127. if f.endswith('.debug') or '/.debug' in relative_dir:
  128. bb.utils.mkdirhier(self.image_rootfs + relative_dir)
  129. shutil.copy(os.path.join(root, f),
  130. self.image_rootfs + relative_dir)
  131. bb.note(" Install complementary '*-dbg' packages...")
  132. self.pm.install_complementary('*-dbg')
  133. if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
  134. bb.note(" Install complementary '*-src' packages...")
  135. self.pm.install_complementary('*-src')
  136. """
  137. Install additional debug packages. Possibility to install additional packages,
  138. which are not automatically installed as complementary package of
  139. standard one, e.g. debug package of static libraries.
  140. """
  141. extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
  142. if extra_debug_pkgs:
  143. bb.note(" Install extra debug packages...")
  144. self.pm.install(extra_debug_pkgs.split(), True)
  145. bb.note(" Rename debug rootfs...")
  146. try:
  147. shutil.rmtree(self.image_rootfs + '-dbg')
  148. except:
  149. pass
  150. os.rename(self.image_rootfs, self.image_rootfs + '-dbg')
  151. bb.note(" Restoreing original rootfs...")
  152. os.rename(self.image_rootfs + '-orig', self.image_rootfs)
  153. def _exec_shell_cmd(self, cmd):
  154. fakerootcmd = self.d.getVar('FAKEROOT')
  155. if fakerootcmd is not None:
  156. exec_cmd = [fakerootcmd, cmd]
  157. else:
  158. exec_cmd = cmd
  159. try:
  160. subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT)
  161. except subprocess.CalledProcessError as e:
  162. return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
  163. return None
  164. def create(self):
  165. bb.note("###### Generate rootfs #######")
  166. pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND")
  167. post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND")
  168. rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND')
  169. bb.utils.mkdirhier(self.image_rootfs)
  170. bb.utils.mkdirhier(self.deploydir)
  171. execute_pre_post_process(self.d, pre_process_cmds)
  172. if self.progress_reporter:
  173. self.progress_reporter.next_stage()
  174. # call the package manager dependent create method
  175. self._create()
  176. sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir')
  177. bb.utils.mkdirhier(sysconfdir)
  178. with open(sysconfdir + "/version", "w+") as ver:
  179. ver.write(self.d.getVar('BUILDNAME') + "\n")
  180. execute_pre_post_process(self.d, rootfs_post_install_cmds)
  181. self.pm.run_intercepts()
  182. execute_pre_post_process(self.d, post_process_cmds)
  183. if self.progress_reporter:
  184. self.progress_reporter.next_stage()
  185. if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
  186. True, False, self.d):
  187. delayed_postinsts = self._get_delayed_postinsts()
  188. if delayed_postinsts is not None:
  189. bb.fatal("The following packages could not be configured "
  190. "offline and rootfs is read-only: %s" %
  191. delayed_postinsts)
  192. if self.d.getVar('USE_DEVFS') != "1":
  193. self._create_devfs()
  194. self._uninstall_unneeded()
  195. if self.progress_reporter:
  196. self.progress_reporter.next_stage()
  197. self._insert_feed_uris()
  198. self._run_ldconfig()
  199. if self.d.getVar('USE_DEPMOD') != "0":
  200. self._generate_kernel_module_deps()
  201. self._cleanup()
  202. self._log_check()
  203. if self.progress_reporter:
  204. self.progress_reporter.next_stage()
  205. def _uninstall_unneeded(self):
  206. # Remove unneeded init script symlinks
  207. delayed_postinsts = self._get_delayed_postinsts()
  208. if delayed_postinsts is None:
  209. if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")):
  210. self._exec_shell_cmd(["update-rc.d", "-f", "-r",
  211. self.d.getVar('IMAGE_ROOTFS'),
  212. "run-postinsts", "remove"])
  213. image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
  214. True, False, self.d)
  215. image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE')
  216. if image_rorfs or image_rorfs_force == "1":
  217. # Remove components that we don't need if it's a read-only rootfs
  218. unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
  219. pkgs_installed = image_list_installed_packages(self.d)
  220. # Make sure update-alternatives is removed last. This is
  221. # because its database has to available while uninstalling
  222. # other packages, allowing alternative symlinks of packages
  223. # to be uninstalled or to be managed correctly otherwise.
  224. provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
  225. pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
  226. # update-alternatives provider is removed in its own remove()
  227. # call because all package managers do not guarantee the packages
  228. # are removed in the order they given in the list (which is
  229. # passed to the command line). The sorting done earlier is
  230. # utilized to implement the 2-stage removal.
  231. if len(pkgs_to_remove) > 1:
  232. self.pm.remove(pkgs_to_remove[:-1], False)
  233. if len(pkgs_to_remove) > 0:
  234. self.pm.remove([pkgs_to_remove[-1]], False)
  235. if delayed_postinsts:
  236. self._save_postinsts()
  237. if image_rorfs:
  238. bb.warn("There are post install scripts "
  239. "in a read-only rootfs")
  240. post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")
  241. execute_pre_post_process(self.d, post_uninstall_cmds)
  242. runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
  243. True, False, self.d)
  244. if not runtime_pkgmanage:
  245. # Remove the package manager data files
  246. self.pm.remove_packaging_data()
  247. def _run_ldconfig(self):
  248. if self.d.getVar('LDCONFIGDEPEND'):
  249. bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X")
  250. self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
  251. 'new', '-v', '-X'])
  252. def _check_for_kernel_modules(self, modules_dir):
  253. for root, dirs, files in os.walk(modules_dir, topdown=True):
  254. for name in files:
  255. found_ko = name.endswith(".ko")
  256. if found_ko:
  257. return found_ko
  258. return False
  259. def _generate_kernel_module_deps(self):
  260. modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
  261. # if we don't have any modules don't bother to do the depmod
  262. if not self._check_for_kernel_modules(modules_dir):
  263. bb.note("No Kernel Modules found, not running depmod")
  264. return
  265. kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod",
  266. 'kernel-abiversion')
  267. if not os.path.exists(kernel_abi_ver_file):
  268. bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
  269. kernel_ver = open(kernel_abi_ver_file).read().strip(' \n')
  270. versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
  271. bb.utils.mkdirhier(versioned_modules_dir)
  272. self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver])
  273. """
  274. Create devfs:
  275. * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
  276. * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
  277. for in the BBPATH
  278. If neither are specified then the default name of files/device_table-minimal.txt
  279. is searched for in the BBPATH (same as the old version.)
  280. """
  281. def _create_devfs(self):
  282. devtable_list = []
  283. devtable = self.d.getVar('IMAGE_DEVICE_TABLE')
  284. if devtable is not None:
  285. devtable_list.append(devtable)
  286. else:
  287. devtables = self.d.getVar('IMAGE_DEVICE_TABLES')
  288. if devtables is None:
  289. devtables = 'files/device_table-minimal.txt'
  290. for devtable in devtables.split():
  291. devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable))
  292. for devtable in devtable_list:
  293. self._exec_shell_cmd(["makedevs", "-r",
  294. self.image_rootfs, "-D", devtable])
  295. def get_class_for_type(imgtype):
  296. from oe.package_manager.rpm.rootfs import RpmRootfs
  297. from oe.package_manager.ipk.rootfs import OpkgRootfs
  298. from oe.package_manager.deb.rootfs import DpkgRootfs
  299. return {"rpm": RpmRootfs,
  300. "ipk": OpkgRootfs,
  301. "deb": DpkgRootfs}[imgtype]
  302. def variable_depends(d, manifest_dir=None):
  303. img_type = d.getVar('IMAGE_PKGTYPE')
  304. cls = get_class_for_type(img_type)
  305. return cls._depends_list()
  306. def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
  307. env_bkp = os.environ.copy()
  308. from oe.package_manager.rpm.rootfs import RpmRootfs
  309. from oe.package_manager.ipk.rootfs import OpkgRootfs
  310. from oe.package_manager.deb.rootfs import DpkgRootfs
  311. img_type = d.getVar('IMAGE_PKGTYPE')
  312. if img_type == "rpm":
  313. RpmRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
  314. elif img_type == "ipk":
  315. OpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
  316. elif img_type == "deb":
  317. DpkgRootfs(d, manifest_dir, progress_reporter, logcatcher).create()
  318. os.environ.clear()
  319. os.environ.update(env_bkp)
  320. def image_list_installed_packages(d, rootfs_dir=None):
  321. if not rootfs_dir:
  322. rootfs_dir = d.getVar('IMAGE_ROOTFS')
  323. img_type = d.getVar('IMAGE_PKGTYPE')
  324. if img_type == "rpm":
  325. return RpmPkgsList(d, rootfs_dir).list_pkgs()
  326. elif img_type == "ipk":
  327. return OpkgPkgsList(d, rootfs_dir, d.getVar("IPKGCONF_TARGET")).list_pkgs()
  328. elif img_type == "deb":
  329. return DpkgPkgsList(d, rootfs_dir).list_pkgs()
  330. if __name__ == "__main__":
  331. """
  332. We should be able to run this as a standalone script, from outside bitbake
  333. environment.
  334. """
  335. """
  336. TBD
  337. """