manifest.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. from abc import ABCMeta, abstractmethod
  5. import os
  6. import re
  7. import bb
  8. class Manifest(object, metaclass=ABCMeta):
  9. """
  10. This is an abstract class. Do not instantiate this directly.
  11. """
  12. PKG_TYPE_MUST_INSTALL = "mip"
  13. PKG_TYPE_MULTILIB = "mlp"
  14. PKG_TYPE_LANGUAGE = "lgp"
  15. PKG_TYPE_ATTEMPT_ONLY = "aop"
  16. MANIFEST_TYPE_IMAGE = "image"
  17. MANIFEST_TYPE_SDK_HOST = "sdk_host"
  18. MANIFEST_TYPE_SDK_TARGET = "sdk_target"
  19. var_maps = {
  20. MANIFEST_TYPE_IMAGE: {
  21. "PACKAGE_INSTALL": PKG_TYPE_MUST_INSTALL,
  22. "PACKAGE_INSTALL_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY,
  23. "LINGUAS_INSTALL": PKG_TYPE_LANGUAGE
  24. },
  25. MANIFEST_TYPE_SDK_HOST: {
  26. "TOOLCHAIN_HOST_TASK": PKG_TYPE_MUST_INSTALL,
  27. "TOOLCHAIN_HOST_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY
  28. },
  29. MANIFEST_TYPE_SDK_TARGET: {
  30. "TOOLCHAIN_TARGET_TASK": PKG_TYPE_MUST_INSTALL,
  31. "TOOLCHAIN_TARGET_TASK_ATTEMPTONLY": PKG_TYPE_ATTEMPT_ONLY
  32. }
  33. }
  34. INSTALL_ORDER = [
  35. PKG_TYPE_LANGUAGE,
  36. PKG_TYPE_MUST_INSTALL,
  37. PKG_TYPE_ATTEMPT_ONLY,
  38. PKG_TYPE_MULTILIB
  39. ]
  40. initial_manifest_file_header = \
  41. "# This file was generated automatically and contains the packages\n" \
  42. "# passed on to the package manager in order to create the rootfs.\n\n" \
  43. "# Format:\n" \
  44. "# <package_type>,<package_name>\n" \
  45. "# where:\n" \
  46. "# <package_type> can be:\n" \
  47. "# 'mip' = must install package\n" \
  48. "# 'aop' = attempt only package\n" \
  49. "# 'mlp' = multilib package\n" \
  50. "# 'lgp' = language package\n\n"
  51. def __init__(self, d, manifest_dir=None, manifest_type=MANIFEST_TYPE_IMAGE):
  52. self.d = d
  53. self.manifest_type = manifest_type
  54. if manifest_dir is None:
  55. if manifest_type != self.MANIFEST_TYPE_IMAGE:
  56. self.manifest_dir = self.d.getVar('SDK_DIR')
  57. else:
  58. self.manifest_dir = self.d.getVar('WORKDIR')
  59. else:
  60. self.manifest_dir = manifest_dir
  61. bb.utils.mkdirhier(self.manifest_dir)
  62. self.initial_manifest = os.path.join(self.manifest_dir, "%s_initial_manifest" % manifest_type)
  63. self.final_manifest = os.path.join(self.manifest_dir, "%s_final_manifest" % manifest_type)
  64. self.full_manifest = os.path.join(self.manifest_dir, "%s_full_manifest" % manifest_type)
  65. # packages in the following vars will be split in 'must install' and
  66. # 'multilib'
  67. self.vars_to_split = ["PACKAGE_INSTALL",
  68. "TOOLCHAIN_HOST_TASK",
  69. "TOOLCHAIN_TARGET_TASK"]
  70. """
  71. This creates a standard initial manifest for core-image-(minimal|sato|sato-sdk).
  72. This will be used for testing until the class is implemented properly!
  73. """
  74. def _create_dummy_initial(self):
  75. image_rootfs = self.d.getVar('IMAGE_ROOTFS')
  76. pkg_list = dict()
  77. if image_rootfs.find("core-image-sato-sdk") > 0:
  78. pkg_list[self.PKG_TYPE_MUST_INSTALL] = \
  79. "packagegroup-core-x11-sato-games packagegroup-base-extended " \
  80. "packagegroup-core-x11-sato packagegroup-core-x11-base " \
  81. "packagegroup-core-sdk packagegroup-core-tools-debug " \
  82. "packagegroup-core-boot packagegroup-core-tools-testapps " \
  83. "packagegroup-core-eclipse-debug packagegroup-core-qt-demoapps " \
  84. "apt packagegroup-core-tools-profile psplash " \
  85. "packagegroup-core-standalone-sdk-target " \
  86. "packagegroup-core-ssh-openssh dpkg kernel-dev"
  87. pkg_list[self.PKG_TYPE_LANGUAGE] = \
  88. "locale-base-en-us locale-base-en-gb"
  89. elif image_rootfs.find("core-image-sato") > 0:
  90. pkg_list[self.PKG_TYPE_MUST_INSTALL] = \
  91. "packagegroup-core-ssh-dropbear packagegroup-core-x11-sato-games " \
  92. "packagegroup-core-x11-base psplash apt dpkg packagegroup-base-extended " \
  93. "packagegroup-core-x11-sato packagegroup-core-boot"
  94. pkg_list['lgp'] = \
  95. "locale-base-en-us locale-base-en-gb"
  96. elif image_rootfs.find("core-image-minimal") > 0:
  97. pkg_list[self.PKG_TYPE_MUST_INSTALL] = "packagegroup-core-boot"
  98. with open(self.initial_manifest, "w+") as manifest:
  99. manifest.write(self.initial_manifest_file_header)
  100. for pkg_type in pkg_list:
  101. for pkg in pkg_list[pkg_type].split():
  102. manifest.write("%s,%s\n" % (pkg_type, pkg))
  103. """
  104. This will create the initial manifest which will be used by Rootfs class to
  105. generate the rootfs
  106. """
  107. @abstractmethod
  108. def create_initial(self):
  109. pass
  110. """
  111. This creates the manifest after everything has been installed.
  112. """
  113. @abstractmethod
  114. def create_final(self):
  115. pass
  116. """
  117. This creates the manifest after the package in initial manifest has been
  118. dummy installed. It lists all *to be installed* packages. There is no real
  119. installation, just a test.
  120. """
  121. @abstractmethod
  122. def create_full(self, pm):
  123. pass
  124. """
  125. The following function parses an initial manifest and returns a dictionary
  126. object with the must install, attempt only, multilib and language packages.
  127. """
  128. def parse_initial_manifest(self):
  129. pkgs = dict()
  130. with open(self.initial_manifest) as manifest:
  131. for line in manifest.read().split('\n'):
  132. comment = re.match("^#.*", line)
  133. pattern = "^(%s|%s|%s|%s),(.*)$" % \
  134. (self.PKG_TYPE_MUST_INSTALL,
  135. self.PKG_TYPE_ATTEMPT_ONLY,
  136. self.PKG_TYPE_MULTILIB,
  137. self.PKG_TYPE_LANGUAGE)
  138. pkg = re.match(pattern, line)
  139. if comment is not None:
  140. continue
  141. if pkg is not None:
  142. pkg_type = pkg.group(1)
  143. pkg_name = pkg.group(2)
  144. if not pkg_type in pkgs:
  145. pkgs[pkg_type] = [pkg_name]
  146. else:
  147. pkgs[pkg_type].append(pkg_name)
  148. return pkgs
  149. '''
  150. This following function parses a full manifest and return a list
  151. object with packages.
  152. '''
  153. def parse_full_manifest(self):
  154. installed_pkgs = list()
  155. if not os.path.exists(self.full_manifest):
  156. bb.note('full manifest not exist')
  157. return installed_pkgs
  158. with open(self.full_manifest, 'r') as manifest:
  159. for pkg in manifest.read().split('\n'):
  160. installed_pkgs.append(pkg.strip())
  161. return installed_pkgs
  162. class RpmManifest(Manifest):
  163. """
  164. Returns a dictionary object with mip and mlp packages.
  165. """
  166. def _split_multilib(self, pkg_list):
  167. pkgs = dict()
  168. for pkg in pkg_list.split():
  169. pkg_type = self.PKG_TYPE_MUST_INSTALL
  170. ml_variants = self.d.getVar('MULTILIB_VARIANTS').split()
  171. for ml_variant in ml_variants:
  172. if pkg.startswith(ml_variant + '-'):
  173. pkg_type = self.PKG_TYPE_MULTILIB
  174. if not pkg_type in pkgs:
  175. pkgs[pkg_type] = pkg
  176. else:
  177. pkgs[pkg_type] += " " + pkg
  178. return pkgs
  179. def create_initial(self):
  180. pkgs = dict()
  181. with open(self.initial_manifest, "w+") as manifest:
  182. manifest.write(self.initial_manifest_file_header)
  183. for var in self.var_maps[self.manifest_type]:
  184. if var in self.vars_to_split:
  185. split_pkgs = self._split_multilib(self.d.getVar(var))
  186. if split_pkgs is not None:
  187. pkgs = dict(list(pkgs.items()) + list(split_pkgs.items()))
  188. else:
  189. pkg_list = self.d.getVar(var)
  190. if pkg_list is not None:
  191. pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var)
  192. for pkg_type in pkgs:
  193. for pkg in pkgs[pkg_type].split():
  194. manifest.write("%s,%s\n" % (pkg_type, pkg))
  195. def create_final(self):
  196. pass
  197. def create_full(self, pm):
  198. pass
  199. class OpkgManifest(Manifest):
  200. """
  201. Returns a dictionary object with mip and mlp packages.
  202. """
  203. def _split_multilib(self, pkg_list):
  204. pkgs = dict()
  205. for pkg in pkg_list.split():
  206. pkg_type = self.PKG_TYPE_MUST_INSTALL
  207. ml_variants = self.d.getVar('MULTILIB_VARIANTS').split()
  208. for ml_variant in ml_variants:
  209. if pkg.startswith(ml_variant + '-'):
  210. pkg_type = self.PKG_TYPE_MULTILIB
  211. if not pkg_type in pkgs:
  212. pkgs[pkg_type] = pkg
  213. else:
  214. pkgs[pkg_type] += " " + pkg
  215. return pkgs
  216. def create_initial(self):
  217. pkgs = dict()
  218. with open(self.initial_manifest, "w+") as manifest:
  219. manifest.write(self.initial_manifest_file_header)
  220. for var in self.var_maps[self.manifest_type]:
  221. if var in self.vars_to_split:
  222. split_pkgs = self._split_multilib(self.d.getVar(var))
  223. if split_pkgs is not None:
  224. pkgs = dict(list(pkgs.items()) + list(split_pkgs.items()))
  225. else:
  226. pkg_list = self.d.getVar(var)
  227. if pkg_list is not None:
  228. pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var)
  229. for pkg_type in sorted(pkgs):
  230. for pkg in sorted(pkgs[pkg_type].split()):
  231. manifest.write("%s,%s\n" % (pkg_type, pkg))
  232. def create_final(self):
  233. pass
  234. def create_full(self, pm):
  235. if not os.path.exists(self.initial_manifest):
  236. self.create_initial()
  237. initial_manifest = self.parse_initial_manifest()
  238. pkgs_to_install = list()
  239. for pkg_type in initial_manifest:
  240. pkgs_to_install += initial_manifest[pkg_type]
  241. if len(pkgs_to_install) == 0:
  242. return
  243. output = pm.dummy_install(pkgs_to_install)
  244. with open(self.full_manifest, 'w+') as manifest:
  245. pkg_re = re.compile('^Installing ([^ ]+) [^ ].*')
  246. for line in set(output.split('\n')):
  247. m = pkg_re.match(line)
  248. if m:
  249. manifest.write(m.group(1) + '\n')
  250. return
  251. class DpkgManifest(Manifest):
  252. def create_initial(self):
  253. with open(self.initial_manifest, "w+") as manifest:
  254. manifest.write(self.initial_manifest_file_header)
  255. for var in self.var_maps[self.manifest_type]:
  256. pkg_list = self.d.getVar(var)
  257. if pkg_list is None:
  258. continue
  259. for pkg in pkg_list.split():
  260. manifest.write("%s,%s\n" %
  261. (self.var_maps[self.manifest_type][var], pkg))
  262. def create_final(self):
  263. pass
  264. def create_full(self, pm):
  265. pass
  266. def create_manifest(d, final_manifest=False, manifest_dir=None,
  267. manifest_type=Manifest.MANIFEST_TYPE_IMAGE):
  268. manifest_map = {'rpm': RpmManifest,
  269. 'ipk': OpkgManifest,
  270. 'deb': DpkgManifest}
  271. manifest = manifest_map[d.getVar('IMAGE_PKGTYPE')](d, manifest_dir, manifest_type)
  272. if final_manifest:
  273. manifest.create_final()
  274. else:
  275. manifest.create_initial()
  276. if __name__ == "__main__":
  277. pass