layerindex.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221
  1. #
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. #
  4. import layerindexlib
  5. import argparse
  6. import logging
  7. import os
  8. import subprocess
  9. from bblayers.action import ActionPlugin
  10. logger = logging.getLogger('bitbake-layers')
  11. def plugin_init(plugins):
  12. return LayerIndexPlugin()
  13. class LayerIndexPlugin(ActionPlugin):
  14. """Subcommands for interacting with the layer index.
  15. This class inherits ActionPlugin to get do_add_layer.
  16. """
  17. def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer, branch, shallow=False):
  18. layername = self.get_layer_name(url)
  19. if os.path.splitext(layername)[1] == '.git':
  20. layername = os.path.splitext(layername)[0]
  21. repodir = os.path.join(fetchdir, layername)
  22. layerdir = os.path.join(repodir, subdir)
  23. if not os.path.exists(repodir):
  24. if fetch_layer:
  25. cmd = ['git', 'clone']
  26. if shallow:
  27. cmd.extend(['--depth', '1'])
  28. if branch:
  29. cmd.extend(['-b' , branch])
  30. cmd.extend([url, repodir])
  31. result = subprocess.call(cmd)
  32. if result:
  33. logger.error("Failed to download %s (%s)" % (url, branch))
  34. return None, None, None
  35. else:
  36. return subdir, layername, layerdir
  37. else:
  38. logger.plain("Repository %s needs to be fetched" % url)
  39. return subdir, layername, layerdir
  40. elif os.path.exists(layerdir):
  41. return subdir, layername, layerdir
  42. else:
  43. logger.error("%s is not in %s" % (url, subdir))
  44. return None, None, None
  45. def do_layerindex_fetch(self, args):
  46. """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
  47. """
  48. def _construct_url(baseurls, branches):
  49. urls = []
  50. for baseurl in baseurls:
  51. if baseurl[-1] != '/':
  52. baseurl += '/'
  53. if not baseurl.startswith('cooker'):
  54. baseurl += "api/"
  55. if branches:
  56. baseurl += ";branch=%s" % ','.join(branches)
  57. urls.append(baseurl)
  58. return urls
  59. # Set the default...
  60. if args.branch:
  61. branches = [args.branch]
  62. else:
  63. branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
  64. logger.debug(1, 'Trying branches: %s' % branches)
  65. ignore_layers = []
  66. if args.ignore:
  67. ignore_layers.extend(args.ignore.split(','))
  68. # Load the cooker DB
  69. cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
  70. cookerIndex.load_layerindex('cooker://', load='layerDependencies')
  71. # Fast path, check if we already have what has been requested!
  72. (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
  73. if not args.show_only and not invalidnames:
  74. logger.plain("You already have the requested layer(s): %s" % args.layername)
  75. return 0
  76. # The information to show is already in the cookerIndex
  77. if invalidnames:
  78. # General URL to use to access the layer index
  79. # While there is ONE right now, we're expect users could enter several
  80. apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL').split()
  81. if not apiurl:
  82. logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
  83. return 1
  84. remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
  85. for remoteurl in _construct_url(apiurl, branches):
  86. logger.plain("Loading %s..." % remoteurl)
  87. remoteIndex.load_layerindex(remoteurl)
  88. if remoteIndex.is_empty():
  89. logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches))
  90. return 1
  91. lIndex = cookerIndex + remoteIndex
  92. (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
  93. if invalidnames:
  94. for invaluename in invalidnames:
  95. logger.error('Layer "%s" not found in layer index' % invaluename)
  96. return 1
  97. logger.plain("%s %s %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory"))
  98. logger.plain('=' * 125)
  99. for deplayerbranch in dependencies:
  100. layerBranch = dependencies[deplayerbranch][0]
  101. # TODO: Determine display behavior
  102. # This is the local content, uncomment to hide local
  103. # layers from the display.
  104. #if layerBranch.index.config['TYPE'] == 'cooker':
  105. # continue
  106. layerDeps = dependencies[deplayerbranch][1:]
  107. requiredby = []
  108. recommendedby = []
  109. for dep in layerDeps:
  110. if dep.required:
  111. requiredby.append(dep.layer.name)
  112. else:
  113. recommendedby.append(dep.layer.name)
  114. logger.plain('%s %s %s' % (("%s:%s:%s" %
  115. (layerBranch.index.config['DESCRIPTION'],
  116. layerBranch.branch.name,
  117. layerBranch.layer.name)).ljust(50),
  118. ("%s (%s)" % (layerBranch.layer.vcs_url,
  119. layerBranch.actual_branch)).ljust(55),
  120. layerBranch.vcs_subdir
  121. ))
  122. if requiredby:
  123. logger.plain(' required by: %s' % ' '.join(requiredby))
  124. if recommendedby:
  125. logger.plain(' recommended by: %s' % ' '.join(recommendedby))
  126. if dependencies:
  127. fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
  128. if not fetchdir:
  129. logger.error("Cannot get BBLAYERS_FETCH_DIR")
  130. return 1
  131. if not os.path.exists(fetchdir):
  132. os.makedirs(fetchdir)
  133. addlayers = []
  134. for deplayerbranch in dependencies:
  135. layerBranch = dependencies[deplayerbranch][0]
  136. if layerBranch.index.config['TYPE'] == 'cooker':
  137. # Anything loaded via cooker is already local, skip it
  138. continue
  139. subdir, name, layerdir = self.get_fetch_layer(fetchdir,
  140. layerBranch.layer.vcs_url,
  141. layerBranch.vcs_subdir,
  142. not args.show_only,
  143. layerBranch.actual_branch,
  144. args.shallow)
  145. if not name:
  146. # Error already shown
  147. return 1
  148. addlayers.append((subdir, name, layerdir))
  149. if not args.show_only:
  150. localargs = argparse.Namespace()
  151. localargs.layerdir = []
  152. localargs.force = args.force
  153. for subdir, name, layerdir in addlayers:
  154. if os.path.exists(layerdir):
  155. if subdir:
  156. logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
  157. else:
  158. logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir))
  159. localargs.layerdir.append(layerdir)
  160. else:
  161. break
  162. if localargs.layerdir:
  163. self.do_add_layer(localargs)
  164. def do_layerindex_show_depends(self, args):
  165. """Find layer dependencies from layer index.
  166. """
  167. args.show_only = True
  168. args.ignore = []
  169. self.do_layerindex_fetch(args)
  170. def register_commands(self, sp):
  171. parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch, parserecipes=False)
  172. parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
  173. parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
  174. parser_layerindex_fetch.add_argument('-s', '--shallow', help='do only shallow clones (--depth=1)', action='store_true')
  175. parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
  176. parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
  177. parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
  178. parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
  179. parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')