cooker.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. # Copyright (C) 2016-2018 Wind River Systems, Inc.
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-only
  4. #
  5. # This program is free software; you can redistribute it and/or modify
  6. # it under the terms of the GNU General Public License version 2 as
  7. # published by the Free Software Foundation.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12. # See the GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program; if not, write to the Free Software
  16. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17. import logging
  18. import json
  19. from collections import OrderedDict, defaultdict
  20. from urllib.parse import unquote, urlparse
  21. import layerindexlib
  22. import layerindexlib.plugin
  23. logger = logging.getLogger('BitBake.layerindexlib.cooker')
  24. import bb.utils
  25. def plugin_init(plugins):
  26. return CookerPlugin()
  27. class CookerPlugin(layerindexlib.plugin.IndexPlugin):
  28. def __init__(self):
  29. self.type = "cooker"
  30. self.server_connection = None
  31. self.ui_module = None
  32. self.server = None
  33. def _run_command(self, command, path, default=None):
  34. try:
  35. result, _ = bb.process.run(command, cwd=path)
  36. result = result.strip()
  37. except bb.process.ExecutionError:
  38. result = default
  39. return result
  40. def _handle_git_remote(self, remote):
  41. if "://" not in remote:
  42. if ':' in remote:
  43. # This is assumed to be ssh
  44. remote = "ssh://" + remote
  45. else:
  46. # This is assumed to be a file path
  47. remote = "file://" + remote
  48. return remote
  49. def _get_bitbake_info(self):
  50. """Return a tuple of bitbake information"""
  51. # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
  52. bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
  53. bb_path = os.path.dirname(bb_path) # .../bitbake/lib/layerindex
  54. bb_path = os.path.dirname(bb_path) # .../bitbake/lib
  55. bb_path = os.path.dirname(bb_path) # .../bitbake
  56. bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
  57. bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
  58. bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
  59. for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
  60. remote = remotes.split("\t")[1].split(" ")[0]
  61. if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
  62. bb_remote = self._handle_git_remote(remote)
  63. break
  64. else:
  65. bb_remote = self._handle_git_remote(bb_path)
  66. return (bb_remote, bb_branch, bb_rev, bb_path)
  67. def _load_bblayers(self, branches=None):
  68. """Load the BBLAYERS and related collection information"""
  69. d = self.layerindex.data
  70. if not branches:
  71. raise LayerIndexFetchError("No branches specified for _load_bblayers!")
  72. index = layerindexlib.LayerIndexObj()
  73. branchId = 0
  74. index.branches = {}
  75. layerItemId = 0
  76. index.layerItems = {}
  77. layerBranchId = 0
  78. index.layerBranches = {}
  79. bblayers = d.getVar('BBLAYERS').split()
  80. if not bblayers:
  81. # It's blank! Nothing to process...
  82. return index
  83. collections = d.getVar('BBFILE_COLLECTIONS')
  84. layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', d)
  85. bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
  86. (_, bb_branch, _, _) = self._get_bitbake_info()
  87. for branch in branches:
  88. branchId += 1
  89. index.branches[branchId] = layerindexlib.Branch(index, None)
  90. index.branches[branchId].define_data(branchId, branch, bb_branch)
  91. for entry in collections.split():
  92. layerpath = entry
  93. if entry in bbfile_collections:
  94. layerpath = bbfile_collections[entry]
  95. layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
  96. layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
  97. layerurl = self._handle_git_remote(layerpath)
  98. layersubdir = ""
  99. layerrev = "<unknown>"
  100. layerbranch = "<unknown>"
  101. if os.path.isdir(layerpath):
  102. layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
  103. if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
  104. layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
  105. layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
  106. layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
  107. for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
  108. if not remotes:
  109. layerurl = self._handle_git_remote(layerpath)
  110. else:
  111. remote = remotes.split("\t")[1].split(" ")[0]
  112. if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
  113. layerurl = self._handle_git_remote(remote)
  114. break
  115. layerItemId += 1
  116. index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
  117. index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
  118. for branchId in index.branches:
  119. layerBranchId += 1
  120. index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
  121. index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
  122. vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
  123. return index
  124. def load_index(self, url, load):
  125. """
  126. Fetches layer information from a build configuration.
  127. The return value is a dictionary containing API,
  128. layer, branch, dependency, recipe, machine, distro, information.
  129. url type should be 'cooker'.
  130. url path is ignored
  131. """
  132. up = urlparse(url)
  133. if up.scheme != 'cooker':
  134. raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
  135. d = self.layerindex.data
  136. params = self.layerindex._parse_params(up.params)
  137. # Only reason to pass a branch is to emulate them...
  138. if 'branch' in params:
  139. branches = params['branch'].split(',')
  140. else:
  141. branches = ['HEAD']
  142. logger.debug(1, "Loading cooker data branches %s" % branches)
  143. index = self._load_bblayers(branches=branches)
  144. index.config = {}
  145. index.config['TYPE'] = self.type
  146. index.config['URL'] = url
  147. if 'desc' in params:
  148. index.config['DESCRIPTION'] = unquote(params['desc'])
  149. else:
  150. index.config['DESCRIPTION'] = 'local'
  151. if 'cache' in params:
  152. index.config['CACHE'] = params['cache']
  153. index.config['BRANCH'] = branches
  154. # ("layerDependencies", layerindexlib.LayerDependency)
  155. layerDependencyId = 0
  156. if "layerDependencies" in load:
  157. index.layerDependencies = {}
  158. for layerBranchId in index.layerBranches:
  159. branchName = index.layerBranches[layerBranchId].branch.name
  160. collection = index.layerBranches[layerBranchId].collection
  161. def add_dependency(layerDependencyId, index, deps, required):
  162. try:
  163. depDict = bb.utils.explode_dep_versions2(deps)
  164. except bb.utils.VersionStringException as vse:
  165. bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
  166. for dep, oplist in list(depDict.items()):
  167. # We need to search ourselves, so use the _ version...
  168. depLayerBranch = index.find_collection(dep, branches=[branchName])
  169. if not depLayerBranch:
  170. # Missing dependency?!
  171. logger.error('Missing dependency %s (%s)' % (dep, branchName))
  172. continue
  173. # We assume that the oplist matches...
  174. layerDependencyId += 1
  175. layerDependency = layerindexlib.LayerDependency(index, None)
  176. layerDependency.define_data(id=layerDependencyId,
  177. required=required, layerbranch=layerBranchId,
  178. dependency=depLayerBranch.layer_id)
  179. logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
  180. index.add_element("layerDependencies", [layerDependency])
  181. return layerDependencyId
  182. deps = d.getVar("LAYERDEPENDS_%s" % collection)
  183. if deps:
  184. layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
  185. deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
  186. if deps:
  187. layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
  188. # Need to load recipes here (requires cooker access)
  189. recipeId = 0
  190. ## TODO: NOT IMPLEMENTED
  191. # The code following this is an example of what needs to be
  192. # implemented. However, it does not work as-is.
  193. if False and 'recipes' in load:
  194. index.recipes = {}
  195. ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
  196. all_versions = self._run_command('allProviders')
  197. all_versions_list = defaultdict(list, all_versions)
  198. for pn in all_versions_list:
  199. for ((pe, pv, pr), fpath) in all_versions_list[pn]:
  200. realfn = bb.cache.virtualfn2realfn(fpath)
  201. filepath = os.path.dirname(realfn[0])
  202. filename = os.path.basename(realfn[0])
  203. # This is all HORRIBLY slow, and likely unnecessary
  204. #dscon = self._run_command('parseRecipeFile', fpath, False, [])
  205. #connector = myDataStoreConnector(self, dscon.dsindex)
  206. #recipe_data = bb.data.init()
  207. #recipe_data.setVar('_remote_data', connector)
  208. #summary = recipe_data.getVar('SUMMARY')
  209. #description = recipe_data.getVar('DESCRIPTION')
  210. #section = recipe_data.getVar('SECTION')
  211. #license = recipe_data.getVar('LICENSE')
  212. #homepage = recipe_data.getVar('HOMEPAGE')
  213. #bugtracker = recipe_data.getVar('BUGTRACKER')
  214. #provides = recipe_data.getVar('PROVIDES')
  215. layer = bb.utils.get_file_layer(realfn[0], self.config_data)
  216. depBranchId = collection_layerbranch[layer]
  217. recipeId += 1
  218. recipe = layerindexlib.Recipe(index, None)
  219. recipe.define_data(id=recipeId,
  220. filename=filename, filepath=filepath,
  221. pn=pn, pv=pv,
  222. summary=pn, description=pn, section='?',
  223. license='?', homepage='?', bugtracker='?',
  224. provides='?', bbclassextend='?', inherits='?',
  225. blacklisted='?', layerbranch=depBranchId)
  226. index = addElement("recipes", [recipe], index)
  227. # ("machines", layerindexlib.Machine)
  228. machineId = 0
  229. if 'machines' in load:
  230. index.machines = {}
  231. for layerBranchId in index.layerBranches:
  232. # load_bblayers uses the description to cache the actual path...
  233. machine_path = index.layerBranches[layerBranchId].layer.description
  234. machine_path = os.path.join(machine_path, 'conf/machine')
  235. if os.path.isdir(machine_path):
  236. for (dirpath, _, filenames) in os.walk(machine_path):
  237. # Ignore subdirs...
  238. if not dirpath.endswith('conf/machine'):
  239. continue
  240. for fname in filenames:
  241. if fname.endswith('.conf'):
  242. machineId += 1
  243. machine = layerindexlib.Machine(index, None)
  244. machine.define_data(id=machineId, name=fname[:-5],
  245. description=fname[:-5],
  246. layerbranch=index.layerBranches[layerBranchId])
  247. index.add_element("machines", [machine])
  248. # ("distros", layerindexlib.Distro)
  249. distroId = 0
  250. if 'distros' in load:
  251. index.distros = {}
  252. for layerBranchId in index.layerBranches:
  253. # load_bblayers uses the description to cache the actual path...
  254. distro_path = index.layerBranches[layerBranchId].layer.description
  255. distro_path = os.path.join(distro_path, 'conf/distro')
  256. if os.path.isdir(distro_path):
  257. for (dirpath, _, filenames) in os.walk(distro_path):
  258. # Ignore subdirs...
  259. if not dirpath.endswith('conf/distro'):
  260. continue
  261. for fname in filenames:
  262. if fname.endswith('.conf'):
  263. distroId += 1
  264. distro = layerindexlib.Distro(index, None)
  265. distro.define_data(id=distroId, name=fname[:-5],
  266. description=fname[:-5],
  267. layerbranch=index.layerBranches[layerBranchId])
  268. index.add_element("distros", [distro])
  269. return index