sdk.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. from abc import ABCMeta, abstractmethod
  5. from oe.utils import execute_pre_post_process
  6. from oe.manifest import *
  7. from oe.package_manager import *
  8. import os
  9. import shutil
  10. import glob
  11. import traceback
  12. class Sdk(object, metaclass=ABCMeta):
  13. def __init__(self, d, manifest_dir):
  14. self.d = d
  15. self.sdk_output = self.d.getVar('SDK_OUTPUT')
  16. self.sdk_native_path = self.d.getVar('SDKPATHNATIVE').strip('/')
  17. self.target_path = self.d.getVar('SDKTARGETSYSROOT').strip('/')
  18. self.sysconfdir = self.d.getVar('sysconfdir').strip('/')
  19. self.sdk_target_sysroot = os.path.join(self.sdk_output, self.target_path)
  20. self.sdk_host_sysroot = self.sdk_output
  21. if manifest_dir is None:
  22. self.manifest_dir = self.d.getVar("SDK_DIR")
  23. else:
  24. self.manifest_dir = manifest_dir
  25. self.remove(self.sdk_output, True)
  26. self.install_order = Manifest.INSTALL_ORDER
  27. @abstractmethod
  28. def _populate(self):
  29. pass
  30. def populate(self):
  31. self.mkdirhier(self.sdk_output)
  32. # call backend dependent implementation
  33. self._populate()
  34. # Don't ship any libGL in the SDK
  35. self.remove(os.path.join(self.sdk_output, self.sdk_native_path,
  36. self.d.getVar('libdir_nativesdk').strip('/'),
  37. "libGL*"))
  38. # Fix or remove broken .la files
  39. self.remove(os.path.join(self.sdk_output, self.sdk_native_path,
  40. self.d.getVar('libdir_nativesdk').strip('/'),
  41. "*.la"))
  42. # Link the ld.so.cache file into the hosts filesystem
  43. link_name = os.path.join(self.sdk_output, self.sdk_native_path,
  44. self.sysconfdir, "ld.so.cache")
  45. self.mkdirhier(os.path.dirname(link_name))
  46. os.symlink("/etc/ld.so.cache", link_name)
  47. execute_pre_post_process(self.d, self.d.getVar('SDK_POSTPROCESS_COMMAND'))
  48. def movefile(self, sourcefile, destdir):
  49. try:
  50. # FIXME: this check of movefile's return code to None should be
  51. # fixed within the function to use only exceptions to signal when
  52. # something goes wrong
  53. if (bb.utils.movefile(sourcefile, destdir) == None):
  54. raise OSError("moving %s to %s failed"
  55. %(sourcefile, destdir))
  56. #FIXME: using umbrella exc catching because bb.utils method raises it
  57. except Exception as e:
  58. bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
  59. bb.error("unable to place %s in final SDK location" % sourcefile)
  60. def mkdirhier(self, dirpath):
  61. try:
  62. bb.utils.mkdirhier(dirpath)
  63. except OSError as e:
  64. bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
  65. bb.fatal("cannot make dir for SDK: %s" % dirpath)
  66. def remove(self, path, recurse=False):
  67. try:
  68. bb.utils.remove(path, recurse)
  69. #FIXME: using umbrella exc catching because bb.utils method raises it
  70. except Exception as e:
  71. bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc())
  72. bb.warn("cannot remove SDK dir: %s" % path)
  73. def install_locales(self, pm):
  74. # This is only relevant for glibc
  75. if self.d.getVar("TCLIBC") != "glibc":
  76. return
  77. linguas = self.d.getVar("SDKIMAGE_LINGUAS")
  78. if linguas:
  79. import fnmatch
  80. # Install the binary locales
  81. if linguas == "all":
  82. pm.install_glob("nativesdk-glibc-binary-localedata-*.utf-8", sdk=True)
  83. else:
  84. pm.install(["nativesdk-glibc-binary-localedata-%s.utf-8" % \
  85. lang for lang in linguas.split()])
  86. # Generate a locale archive of them
  87. target_arch = self.d.getVar('SDK_ARCH')
  88. rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)
  89. localedir = oe.path.join(rootfs, self.d.getVar("libdir_nativesdk"), "locale")
  90. generate_locale_archive(self.d, rootfs, target_arch, localedir)
  91. # And now delete the binary locales
  92. pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8")
  93. pm.remove(pkgs)
  94. else:
  95. # No linguas so do nothing
  96. pass
  97. class RpmSdk(Sdk):
  98. def __init__(self, d, manifest_dir=None, rpm_workdir="oe-sdk-repo"):
  99. super(RpmSdk, self).__init__(d, manifest_dir)
  100. self.target_manifest = RpmManifest(d, self.manifest_dir,
  101. Manifest.MANIFEST_TYPE_SDK_TARGET)
  102. self.host_manifest = RpmManifest(d, self.manifest_dir,
  103. Manifest.MANIFEST_TYPE_SDK_HOST)
  104. rpm_repo_workdir = "oe-sdk-repo"
  105. if "sdk_ext" in d.getVar("BB_RUNTASK"):
  106. rpm_repo_workdir = "oe-sdk-ext-repo"
  107. self.target_pm = RpmPM(d,
  108. self.sdk_target_sysroot,
  109. self.d.getVar('TARGET_VENDOR'),
  110. 'target',
  111. rpm_repo_workdir=rpm_repo_workdir
  112. )
  113. self.host_pm = RpmPM(d,
  114. self.sdk_host_sysroot,
  115. self.d.getVar('SDK_VENDOR'),
  116. 'host',
  117. "SDK_PACKAGE_ARCHS",
  118. "SDK_OS",
  119. rpm_repo_workdir=rpm_repo_workdir
  120. )
  121. def _populate_sysroot(self, pm, manifest):
  122. pkgs_to_install = manifest.parse_initial_manifest()
  123. pm.create_configs()
  124. pm.write_index()
  125. pm.update()
  126. pkgs = []
  127. pkgs_attempt = []
  128. for pkg_type in pkgs_to_install:
  129. if pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY:
  130. pkgs_attempt += pkgs_to_install[pkg_type]
  131. else:
  132. pkgs += pkgs_to_install[pkg_type]
  133. pm.install(pkgs)
  134. pm.install(pkgs_attempt, True)
  135. def _populate(self):
  136. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
  137. bb.note("Installing TARGET packages")
  138. self._populate_sysroot(self.target_pm, self.target_manifest)
  139. self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY'))
  140. self.target_pm.run_intercepts(populate_sdk='target')
  141. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND"))
  142. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  143. self.target_pm.remove_packaging_data()
  144. bb.note("Installing NATIVESDK packages")
  145. self._populate_sysroot(self.host_pm, self.host_manifest)
  146. self.install_locales(self.host_pm)
  147. self.host_pm.run_intercepts(populate_sdk='host')
  148. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND"))
  149. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  150. self.host_pm.remove_packaging_data()
  151. # Move host RPM library data
  152. native_rpm_state_dir = os.path.join(self.sdk_output,
  153. self.sdk_native_path,
  154. self.d.getVar('localstatedir_nativesdk').strip('/'),
  155. "lib",
  156. "rpm"
  157. )
  158. self.mkdirhier(native_rpm_state_dir)
  159. for f in glob.glob(os.path.join(self.sdk_output,
  160. "var",
  161. "lib",
  162. "rpm",
  163. "*")):
  164. self.movefile(f, native_rpm_state_dir)
  165. self.remove(os.path.join(self.sdk_output, "var"), True)
  166. # Move host sysconfig data
  167. native_sysconf_dir = os.path.join(self.sdk_output,
  168. self.sdk_native_path,
  169. self.d.getVar('sysconfdir',
  170. True).strip('/'),
  171. )
  172. self.mkdirhier(native_sysconf_dir)
  173. for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")):
  174. self.movefile(f, native_sysconf_dir)
  175. for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")):
  176. self.movefile(f, native_sysconf_dir)
  177. self.remove(os.path.join(self.sdk_output, "etc"), True)
  178. class OpkgSdk(Sdk):
  179. def __init__(self, d, manifest_dir=None):
  180. super(OpkgSdk, self).__init__(d, manifest_dir)
  181. self.target_conf = self.d.getVar("IPKGCONF_TARGET")
  182. self.host_conf = self.d.getVar("IPKGCONF_SDK")
  183. self.target_manifest = OpkgManifest(d, self.manifest_dir,
  184. Manifest.MANIFEST_TYPE_SDK_TARGET)
  185. self.host_manifest = OpkgManifest(d, self.manifest_dir,
  186. Manifest.MANIFEST_TYPE_SDK_HOST)
  187. ipk_repo_workdir = "oe-sdk-repo"
  188. if "sdk_ext" in d.getVar("BB_RUNTASK"):
  189. ipk_repo_workdir = "oe-sdk-ext-repo"
  190. self.target_pm = OpkgPM(d, self.sdk_target_sysroot, self.target_conf,
  191. self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"),
  192. ipk_repo_workdir=ipk_repo_workdir)
  193. self.host_pm = OpkgPM(d, self.sdk_host_sysroot, self.host_conf,
  194. self.d.getVar("SDK_PACKAGE_ARCHS"),
  195. ipk_repo_workdir=ipk_repo_workdir)
  196. def _populate_sysroot(self, pm, manifest):
  197. pkgs_to_install = manifest.parse_initial_manifest()
  198. if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1":
  199. pm.write_index()
  200. pm.update()
  201. for pkg_type in self.install_order:
  202. if pkg_type in pkgs_to_install:
  203. pm.install(pkgs_to_install[pkg_type],
  204. [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
  205. def _populate(self):
  206. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
  207. bb.note("Installing TARGET packages")
  208. self._populate_sysroot(self.target_pm, self.target_manifest)
  209. self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY'))
  210. self.target_pm.run_intercepts(populate_sdk='target')
  211. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND"))
  212. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  213. self.target_pm.remove_packaging_data()
  214. bb.note("Installing NATIVESDK packages")
  215. self._populate_sysroot(self.host_pm, self.host_manifest)
  216. self.install_locales(self.host_pm)
  217. self.host_pm.run_intercepts(populate_sdk='host')
  218. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND"))
  219. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  220. self.host_pm.remove_packaging_data()
  221. target_sysconfdir = os.path.join(self.sdk_target_sysroot, self.sysconfdir)
  222. host_sysconfdir = os.path.join(self.sdk_host_sysroot, self.sysconfdir)
  223. self.mkdirhier(target_sysconfdir)
  224. shutil.copy(self.target_conf, target_sysconfdir)
  225. os.chmod(os.path.join(target_sysconfdir,
  226. os.path.basename(self.target_conf)), 0o644)
  227. self.mkdirhier(host_sysconfdir)
  228. shutil.copy(self.host_conf, host_sysconfdir)
  229. os.chmod(os.path.join(host_sysconfdir,
  230. os.path.basename(self.host_conf)), 0o644)
  231. native_opkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path,
  232. self.d.getVar('localstatedir_nativesdk').strip('/'),
  233. "lib", "opkg")
  234. self.mkdirhier(native_opkg_state_dir)
  235. for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "opkg", "*")):
  236. self.movefile(f, native_opkg_state_dir)
  237. self.remove(os.path.join(self.sdk_output, "var"), True)
  238. class DpkgSdk(Sdk):
  239. def __init__(self, d, manifest_dir=None):
  240. super(DpkgSdk, self).__init__(d, manifest_dir)
  241. self.target_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt")
  242. self.host_conf_dir = os.path.join(self.d.getVar("APTCONF_TARGET"), "apt-sdk")
  243. self.target_manifest = DpkgManifest(d, self.manifest_dir,
  244. Manifest.MANIFEST_TYPE_SDK_TARGET)
  245. self.host_manifest = DpkgManifest(d, self.manifest_dir,
  246. Manifest.MANIFEST_TYPE_SDK_HOST)
  247. deb_repo_workdir = "oe-sdk-repo"
  248. if "sdk_ext" in d.getVar("BB_RUNTASK"):
  249. deb_repo_workdir = "oe-sdk-ext-repo"
  250. self.target_pm = DpkgPM(d, self.sdk_target_sysroot,
  251. self.d.getVar("PACKAGE_ARCHS"),
  252. self.d.getVar("DPKG_ARCH"),
  253. self.target_conf_dir,
  254. deb_repo_workdir=deb_repo_workdir)
  255. self.host_pm = DpkgPM(d, self.sdk_host_sysroot,
  256. self.d.getVar("SDK_PACKAGE_ARCHS"),
  257. self.d.getVar("DEB_SDK_ARCH"),
  258. self.host_conf_dir,
  259. deb_repo_workdir=deb_repo_workdir)
  260. def _copy_apt_dir_to(self, dst_dir):
  261. staging_etcdir_native = self.d.getVar("STAGING_ETCDIR_NATIVE")
  262. self.remove(dst_dir, True)
  263. shutil.copytree(os.path.join(staging_etcdir_native, "apt"), dst_dir)
  264. def _populate_sysroot(self, pm, manifest):
  265. pkgs_to_install = manifest.parse_initial_manifest()
  266. pm.write_index()
  267. pm.update()
  268. for pkg_type in self.install_order:
  269. if pkg_type in pkgs_to_install:
  270. pm.install(pkgs_to_install[pkg_type],
  271. [False, True][pkg_type == Manifest.PKG_TYPE_ATTEMPT_ONLY])
  272. def _populate(self):
  273. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_PRE_TARGET_COMMAND"))
  274. bb.note("Installing TARGET packages")
  275. self._populate_sysroot(self.target_pm, self.target_manifest)
  276. self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY'))
  277. self.target_pm.run_intercepts(populate_sdk='target')
  278. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND"))
  279. self._copy_apt_dir_to(os.path.join(self.sdk_target_sysroot, "etc", "apt"))
  280. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  281. self.target_pm.remove_packaging_data()
  282. bb.note("Installing NATIVESDK packages")
  283. self._populate_sysroot(self.host_pm, self.host_manifest)
  284. self.install_locales(self.host_pm)
  285. self.host_pm.run_intercepts(populate_sdk='host')
  286. execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND"))
  287. self._copy_apt_dir_to(os.path.join(self.sdk_output, self.sdk_native_path,
  288. "etc", "apt"))
  289. if not bb.utils.contains("SDKIMAGE_FEATURES", "package-management", True, False, self.d):
  290. self.host_pm.remove_packaging_data()
  291. native_dpkg_state_dir = os.path.join(self.sdk_output, self.sdk_native_path,
  292. "var", "lib", "dpkg")
  293. self.mkdirhier(native_dpkg_state_dir)
  294. for f in glob.glob(os.path.join(self.sdk_output, "var", "lib", "dpkg", "*")):
  295. self.movefile(f, native_dpkg_state_dir)
  296. self.remove(os.path.join(self.sdk_output, "var"), True)
  297. def sdk_list_installed_packages(d, target, rootfs_dir=None):
  298. if rootfs_dir is None:
  299. sdk_output = d.getVar('SDK_OUTPUT')
  300. target_path = d.getVar('SDKTARGETSYSROOT').strip('/')
  301. rootfs_dir = [sdk_output, os.path.join(sdk_output, target_path)][target is True]
  302. img_type = d.getVar('IMAGE_PKGTYPE')
  303. if img_type == "rpm":
  304. arch_var = ["SDK_PACKAGE_ARCHS", None][target is True]
  305. os_var = ["SDK_OS", None][target is True]
  306. return RpmPkgsList(d, rootfs_dir).list_pkgs()
  307. elif img_type == "ipk":
  308. conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True]
  309. return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs()
  310. elif img_type == "deb":
  311. return DpkgPkgsList(d, rootfs_dir).list_pkgs()
  312. def populate_sdk(d, manifest_dir=None):
  313. env_bkp = os.environ.copy()
  314. img_type = d.getVar('IMAGE_PKGTYPE')
  315. if img_type == "rpm":
  316. RpmSdk(d, manifest_dir).populate()
  317. elif img_type == "ipk":
  318. OpkgSdk(d, manifest_dir).populate()
  319. elif img_type == "deb":
  320. DpkgSdk(d, manifest_dir).populate()
  321. os.environ.clear()
  322. os.environ.update(env_bkp)
  323. def get_extra_sdkinfo(sstate_dir):
  324. """
  325. This function is going to be used for generating the target and host manifest files packages of eSDK.
  326. """
  327. import math
  328. extra_info = {}
  329. extra_info['tasksizes'] = {}
  330. extra_info['filesizes'] = {}
  331. for root, _, files in os.walk(sstate_dir):
  332. for fn in files:
  333. if fn.endswith('.tgz'):
  334. fsize = int(math.ceil(float(os.path.getsize(os.path.join(root, fn))) / 1024))
  335. task = fn.rsplit(':',1)[1].split('_',1)[1].split(',')[0]
  336. origtotal = extra_info['tasksizes'].get(task, 0)
  337. extra_info['tasksizes'][task] = origtotal + fsize
  338. extra_info['filesizes'][fn] = fsize
  339. return extra_info
  340. if __name__ == "__main__":
  341. pass