123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523 |
- #!/usr/bin/env python3
- # Script to extract information from image manifests
- #
- # Copyright (C) 2018 Intel Corporation
- # Copyright (C) 2021 Wind River Systems, Inc.
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import sys
- import os
- import argparse
- import logging
- import json
- import shutil
- import tempfile
- import tarfile
- from collections import OrderedDict
- scripts_path = os.path.dirname(__file__)
- lib_path = scripts_path + '/../lib'
- sys.path = sys.path + [lib_path]
- import scriptutils
- logger = scriptutils.logger_create(os.path.basename(__file__))
- import argparse_oe
- import scriptpath
- bitbakepath = scriptpath.add_bitbake_lib_path()
- if not bitbakepath:
- logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
- sys.exit(1)
- logger.debug('Using standard bitbake path %s' % bitbakepath)
- scriptpath.add_oe_lib_path()
- import bb.tinfoil
- import bb.utils
- import oe.utils
- import oe.recipeutils
- def get_pkg_list(manifest):
- pkglist = []
- with open(manifest, 'r') as f:
- for line in f:
- linesplit = line.split()
- if len(linesplit) == 3:
- # manifest file
- pkglist.append(linesplit[0])
- elif len(linesplit) == 1:
- # build dependency file
- pkglist.append(linesplit[0])
- return sorted(pkglist)
- def list_packages(args):
- pkglist = get_pkg_list(args.manifest)
- for pkg in pkglist:
- print('%s' % pkg)
- def pkg2recipe(tinfoil, pkg):
- if "-native" in pkg:
- logger.info('skipping %s' % pkg)
- return None
- pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
- pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
- logger.debug('pkgdatafile %s' % pkgdatafile)
- try:
- f = open(pkgdatafile, 'r')
- for line in f:
- if line.startswith('PN:'):
- recipe = line.split(':', 1)[1].strip()
- return recipe
- except Exception:
- logger.warning('%s is missing' % pkgdatafile)
- return None
- def get_recipe_list(manifest, tinfoil):
- pkglist = get_pkg_list(manifest)
- recipelist = []
- for pkg in pkglist:
- recipe = pkg2recipe(tinfoil,pkg)
- if recipe:
- if not recipe in recipelist:
- recipelist.append(recipe)
- return sorted(recipelist)
- def list_recipes(args):
- import bb.tinfoil
- with bb.tinfoil.Tinfoil() as tinfoil:
- tinfoil.logger.setLevel(logger.getEffectiveLevel())
- tinfoil.prepare(config_only=True)
- recipelist = get_recipe_list(args.manifest, tinfoil)
- for recipe in sorted(recipelist):
- print('%s' % recipe)
- def list_layers(args):
- def find_git_repo(pth):
- checkpth = pth
- while checkpth != os.sep:
- if os.path.exists(os.path.join(checkpth, '.git')):
- return checkpth
- checkpth = os.path.dirname(checkpth)
- return None
- def get_git_remote_branch(repodir):
- try:
- stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir)
- except bb.process.ExecutionError as e:
- stdout = None
- if stdout:
- return stdout.strip()
- else:
- return None
- def get_git_head_commit(repodir):
- try:
- stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir)
- except bb.process.ExecutionError as e:
- stdout = None
- if stdout:
- return stdout.strip()
- else:
- return None
- def get_git_repo_url(repodir, remote='origin'):
- import bb.process
- # Try to get upstream repo location from origin remote
- try:
- stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir)
- except bb.process.ExecutionError as e:
- stdout = None
- if stdout:
- for line in stdout.splitlines():
- splitline = line.split()
- if len(splitline) > 1:
- if splitline[0] == remote and scriptutils.is_src_url(splitline[1]):
- return splitline[1]
- return None
- with bb.tinfoil.Tinfoil() as tinfoil:
- tinfoil.logger.setLevel(logger.getEffectiveLevel())
- tinfoil.prepare(config_only=False)
- layers = OrderedDict()
- for layerdir in tinfoil.config_data.getVar('BBLAYERS').split():
- layerdata = OrderedDict()
- layername = os.path.basename(layerdir)
- logger.debug('layername %s, layerdir %s' % (layername, layerdir))
- if layername in layers:
- logger.warning('layername %s is not unique in configuration' % layername)
- layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir)
- logger.debug('trying layername %s' % layername)
- if layername in layers:
- logger.error('Layer name %s is not unique in configuration' % layername)
- sys.exit(2)
- repodir = find_git_repo(layerdir)
- if repodir:
- remotebranch = get_git_remote_branch(repodir)
- remote = 'origin'
- if remotebranch and '/' in remotebranch:
- rbsplit = remotebranch.split('/', 1)
- layerdata['actual_branch'] = rbsplit[1]
- remote = rbsplit[0]
- layerdata['vcs_url'] = get_git_repo_url(repodir, remote)
- if os.path.abspath(repodir) != os.path.abspath(layerdir):
- layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir)
- commit = get_git_head_commit(repodir)
- if commit:
- layerdata['vcs_commit'] = commit
- layers[layername] = layerdata
- json.dump(layers, args.output, indent=2)
- def get_recipe(args):
- with bb.tinfoil.Tinfoil() as tinfoil:
- tinfoil.logger.setLevel(logger.getEffectiveLevel())
- tinfoil.prepare(config_only=True)
- recipe = pkg2recipe(tinfoil, args.package)
- print(' %s package provided by %s' % (args.package, recipe))
- def pkg_dependencies(args):
- def get_recipe_info(tinfoil, recipe):
- try:
- info = tinfoil.get_recipe_info(recipe)
- except Exception:
- logger.error('Failed to get recipe info for: %s' % recipe)
- sys.exit(1)
- if not info:
- logger.warning('No recipe info found for: %s' % recipe)
- sys.exit(1)
- append_files = tinfoil.get_file_appends(info.fn)
- appends = True
- data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
- data.pn = info.pn
- data.pv = info.pv
- return data
- def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order):
- spaces = ' ' * order
- data = recipe_info[rn]
- if args.native:
- logger.debug('%s- %s' % (spaces, data.pn))
- elif "-native" not in data.pn:
- if "cross" not in data.pn:
- logger.debug('%s- %s' % (spaces, data.pn))
- depends = []
- for dep in data.depends:
- if dep not in assume_provided:
- depends.append(dep)
- # First find all dependencies not in package list.
- for dep in depends:
- if dep not in packages:
- packages.append(dep)
- dep_data = get_recipe_info(tinfoil, dep)
- # Do this once now to reduce the number of bitbake calls.
- dep_data.depends = dep_data.getVar('DEPENDS').split()
- recipe_info[dep] = dep_data
- # Then recursively analyze all of the dependencies for the current recipe.
- for dep in depends:
- find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
- with bb.tinfoil.Tinfoil() as tinfoil:
- tinfoil.logger.setLevel(logger.getEffectiveLevel())
- tinfoil.prepare()
- assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
- logger.debug('assumed provided:')
- for ap in sorted(assume_provided):
- logger.debug(' - %s' % ap)
- recipe = pkg2recipe(tinfoil, args.package)
- data = get_recipe_info(tinfoil, recipe)
- data.depends = []
- depends = data.getVar('DEPENDS').split()
- for dep in depends:
- if dep not in assume_provided:
- data.depends.append(dep)
- recipe_info = dict([(recipe, data)])
- packages = []
- find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1)
- print('\nThe following packages are required to build %s' % recipe)
- for p in sorted(packages):
- data = recipe_info[p]
- if "-native" not in data.pn:
- if "cross" not in data.pn:
- print(" %s (%s)" % (data.pn,p))
- if args.native:
- print('\nThe following native packages are required to build %s' % recipe)
- for p in sorted(packages):
- data = recipe_info[p]
- if "-native" in data.pn:
- print(" %s(%s)" % (data.pn,p))
- if "cross" in data.pn:
- print(" %s(%s)" % (data.pn,p))
- def default_config():
- vlist = OrderedDict()
- vlist['PV'] = 'yes'
- vlist['SUMMARY'] = 'no'
- vlist['DESCRIPTION'] = 'no'
- vlist['SECTION'] = 'no'
- vlist['LICENSE'] = 'yes'
- vlist['HOMEPAGE'] = 'no'
- vlist['BUGTRACKER'] = 'no'
- vlist['PROVIDES'] = 'no'
- vlist['BBCLASSEXTEND'] = 'no'
- vlist['DEPENDS'] = 'no'
- vlist['PACKAGECONFIG'] = 'no'
- vlist['SRC_URI'] = 'yes'
- vlist['SRCREV'] = 'yes'
- vlist['EXTRA_OECONF'] = 'no'
- vlist['EXTRA_OESCONS'] = 'no'
- vlist['EXTRA_OECMAKE'] = 'no'
- vlist['EXTRA_OEMESON'] = 'no'
- clist = OrderedDict()
- clist['variables'] = vlist
- clist['filepath'] = 'no'
- clist['sha256sum'] = 'no'
- clist['layerdir'] = 'no'
- clist['layer'] = 'no'
- clist['inherits'] = 'no'
- clist['source_urls'] = 'no'
- clist['packageconfig_opts'] = 'no'
- clist['patches'] = 'no'
- clist['packagedir'] = 'no'
- return clist
- def dump_config(args):
- config = default_config()
- f = open('default_config.json', 'w')
- json.dump(config, f, indent=2)
- logger.info('Default config list dumped to default_config.json')
- def export_manifest_info(args):
- def handle_value(value):
- if value:
- return oe.utils.squashspaces(value)
- else:
- return value
- if args.config:
- logger.debug('config: %s' % args.config)
- f = open(args.config, 'r')
- config = json.load(f, object_pairs_hook=OrderedDict)
- else:
- config = default_config()
- if logger.isEnabledFor(logging.DEBUG):
- print('Configuration:')
- json.dump(config, sys.stdout, indent=2)
- print('')
- tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-')
- logger.debug('tmp dir: %s' % tmpoutdir)
- # export manifest
- shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest"))
- with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
- tinfoil.logger.setLevel(logger.getEffectiveLevel())
- tinfoil.prepare(config_only=False)
- pkglist = get_pkg_list(args.manifest)
- # export pkg list
- f = open(os.path.join(tmpoutdir, "pkgs"), 'w')
- for pkg in pkglist:
- f.write('%s\n' % pkg)
- f.close()
- recipelist = []
- for pkg in pkglist:
- recipe = pkg2recipe(tinfoil,pkg)
- if recipe:
- if not recipe in recipelist:
- recipelist.append(recipe)
- recipelist.sort()
- # export recipe list
- f = open(os.path.join(tmpoutdir, "recipes"), 'w')
- for recipe in recipelist:
- f.write('%s\n' % recipe)
- f.close()
- try:
- rvalues = OrderedDict()
- for pn in sorted(recipelist):
- logger.debug('Package: %s' % pn)
- rd = tinfoil.parse_recipe(pn)
- rvalues[pn] = OrderedDict()
- for varname in config['variables']:
- if config['variables'][varname] == 'yes':
- rvalues[pn][varname] = handle_value(rd.getVar(varname))
- fpth = rd.getVar('FILE')
- layerdir = oe.recipeutils.find_layerdir(fpth)
- if config['filepath'] == 'yes':
- rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir)
- if config['sha256sum'] == 'yes':
- rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth)
- if config['layerdir'] == 'yes':
- rvalues[pn]['layerdir'] = layerdir
- if config['layer'] == 'yes':
- rvalues[pn]['layer'] = os.path.basename(layerdir)
- if config['inherits'] == 'yes':
- gr = set(tinfoil.config_data.getVar("__inherit_cache") or [])
- lr = set(rd.getVar("__inherit_cache") or [])
- rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr})
- if config['source_urls'] == 'yes':
- rvalues[pn]['source_urls'] = []
- for url in (rd.getVar('SRC_URI') or '').split():
- if not url.startswith('file://'):
- url = url.split(';')[0]
- rvalues[pn]['source_urls'].append(url)
- if config['packageconfig_opts'] == 'yes':
- rvalues[pn]['packageconfig_opts'] = OrderedDict()
- for key in rd.getVarFlags('PACKAGECONFIG').keys():
- if key == 'doc':
- continue
- rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key, True)
- if config['patches'] == 'yes':
- patches = oe.recipeutils.get_recipe_patches(rd)
- rvalues[pn]['patches'] = []
- if patches:
- recipeoutdir = os.path.join(tmpoutdir, pn, 'patches')
- bb.utils.mkdirhier(recipeoutdir)
- for patch in patches:
- # Patches may be in other layers too
- patchlayerdir = oe.recipeutils.find_layerdir(patch)
- # patchlayerdir will be None for remote patches, which we ignore
- # (since currently they are considered as part of sources)
- if patchlayerdir:
- rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir)))
- shutil.copy(patch, recipeoutdir)
- if config['packagedir'] == 'yes':
- pn_dir = os.path.join(tmpoutdir, pn)
- bb.utils.mkdirhier(pn_dir)
- f = open(os.path.join(pn_dir, 'recipe.json'), 'w')
- json.dump(rvalues[pn], f, indent=2)
- f.close()
- with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f:
- json.dump(rvalues, f, indent=2)
- if args.output:
- outname = os.path.basename(args.output)
- else:
- outname = os.path.splitext(os.path.basename(args.manifest))[0]
- if outname.endswith('.tar.gz'):
- outname = outname[:-7]
- elif outname.endswith('.tgz'):
- outname = outname[:-4]
- tarfn = outname
- if tarfn.endswith(os.sep):
- tarfn = tarfn[:-1]
- if not tarfn.endswith(('.tar.gz', '.tgz')):
- tarfn += '.tar.gz'
- with open(tarfn, 'wb') as f:
- with tarfile.open(None, "w:gz", f) as tar:
- tar.add(tmpoutdir, outname)
- finally:
- shutil.rmtree(tmpoutdir)
- def main():
- parser = argparse_oe.ArgumentParser(description="Image manifest utility",
- epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
- parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
- parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
- subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
- subparsers.required = True
- # get recipe info
- parser_get_recipes = subparsers.add_parser('recipe-info',
- help='Get recipe info',
- description='Get recipe information for a package')
- parser_get_recipes.add_argument('package', help='Package name')
- parser_get_recipes.set_defaults(func=get_recipe)
- # list runtime dependencies
- parser_pkg_dep = subparsers.add_parser('list-depends',
- help='List dependencies',
- description='List dependencies required to build the package')
- parser_pkg_dep.add_argument('--native', help='also print native and cross packages', action='store_true')
- parser_pkg_dep.add_argument('package', help='Package name')
- parser_pkg_dep.set_defaults(func=pkg_dependencies)
- # list recipes
- parser_recipes = subparsers.add_parser('list-recipes',
- help='List recipes producing packages within an image',
- description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata')
- parser_recipes.add_argument('manifest', help='Manifest file')
- parser_recipes.set_defaults(func=list_recipes)
- # list packages
- parser_packages = subparsers.add_parser('list-packages',
- help='List packages within an image',
- description='Lists packages that went into an image, using the manifest')
- parser_packages.add_argument('manifest', help='Manifest file')
- parser_packages.set_defaults(func=list_packages)
- # list layers
- parser_layers = subparsers.add_parser('list-layers',
- help='List included layers',
- description='Lists included layers')
- parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified',
- default=sys.stdout, type=argparse.FileType('w'))
- parser_layers.set_defaults(func=list_layers)
- # dump default configuration file
- parser_dconfig = subparsers.add_parser('dump-config',
- help='Dump default config',
- description='Dump default config to default_config.json')
- parser_dconfig.set_defaults(func=dump_config)
- # export recipe info for packages in manifest
- parser_export = subparsers.add_parser('manifest-info',
- help='Export recipe info for a manifest',
- description='Export recipe information using the manifest')
- parser_export.add_argument('-c', '--config', help='load config from json file')
- parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified')
- parser_export.add_argument('manifest', help='Manifest file')
- parser_export.set_defaults(func=export_manifest_info)
- args = parser.parse_args()
- if args.debug:
- logger.setLevel(logging.DEBUG)
- logger.debug("Debug Enabled")
- elif args.quiet:
- logger.setLevel(logging.ERROR)
- ret = args.func(args)
- return ret
- if __name__ == "__main__":
- try:
- ret = main()
- except Exception:
- ret = 1
- import traceback
- traceback.print_exc()
- sys.exit(ret)
|