123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392 |
- # Copyright (C) 2016-2018 Wind River Systems, Inc.
- #
- # SPDX-License-Identifier: GPL-2.0-only
- #
- import logging
- import json
- import os
- from urllib.parse import unquote
- from urllib.parse import urlparse
- import bb
- import layerindexlib
- import layerindexlib.plugin
- logger = logging.getLogger('BitBake.layerindexlib.restapi')
- def plugin_init(plugins):
- return RestApiPlugin()
- class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
- def __init__(self):
- self.type = "restapi"
- def load_index(self, url, load):
- """
- Fetches layer information from a local or remote layer index.
- The return value is a LayerIndexObj.
- url is the url to the rest api of the layer index, such as:
- http://layers.openembedded.org/layerindex/api/
- Or a local file...
- """
- up = urlparse(url)
- if up.scheme == 'file':
- return self.load_index_file(up, url, load)
- if up.scheme == 'http' or up.scheme == 'https':
- return self.load_index_web(up, url, load)
- raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
- def load_index_file(self, up, url, load):
- """
- Fetches layer information from a local file or directory.
- The return value is a LayerIndexObj.
- ud is the parsed url to the local file or directory.
- """
- if not os.path.exists(up.path):
- raise FileNotFoundError(up.path)
- index = layerindexlib.LayerIndexObj()
- index.config = {}
- index.config['TYPE'] = self.type
- index.config['URL'] = url
- params = self.layerindex._parse_params(up.params)
- if 'desc' in params:
- index.config['DESCRIPTION'] = unquote(params['desc'])
- else:
- index.config['DESCRIPTION'] = up.path
- if 'cache' in params:
- index.config['CACHE'] = params['cache']
- if 'branch' in params:
- branches = params['branch'].split(',')
- index.config['BRANCH'] = branches
- else:
- branches = ['*']
- def load_cache(path, index, branches=[]):
- logger.debug(1, 'Loading json file %s' % path)
- with open(path, 'rt', encoding='utf-8') as f:
- pindex = json.load(f)
- # Filter the branches on loaded files...
- newpBranch = []
- for branch in branches:
- if branch != '*':
- if 'branches' in pindex:
- for br in pindex['branches']:
- if br['name'] == branch:
- newpBranch.append(br)
- else:
- if 'branches' in pindex:
- for br in pindex['branches']:
- newpBranch.append(br)
- if newpBranch:
- index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
- else:
- logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
- # No matching branches.. return nothing...
- return
- for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
- ("layerBranches", layerindexlib.LayerBranch),
- ("layerDependencies", layerindexlib.LayerDependency),
- ("recipes", layerindexlib.Recipe),
- ("machines", layerindexlib.Machine),
- ("distros", layerindexlib.Distro)]:
- if lName in pindex:
- index.add_raw_element(lName, lType, pindex[lName])
- if not os.path.isdir(up.path):
- load_cache(up.path, index, branches)
- return index
- logger.debug(1, 'Loading from dir %s...' % (up.path))
- for (dirpath, _, filenames) in os.walk(up.path):
- for filename in filenames:
- if not filename.endswith('.json'):
- continue
- fpath = os.path.join(dirpath, filename)
- load_cache(fpath, index, branches)
- return index
- def load_index_web(self, up, url, load):
- """
- Fetches layer information from a remote layer index.
- The return value is a LayerIndexObj.
- ud is the parsed url to the rest api of the layer index, such as:
- http://layers.openembedded.org/layerindex/api/
- """
- def _get_json_response(apiurl=None, username=None, password=None, retry=True):
- assert apiurl is not None
- logger.debug(1, "fetching %s" % apiurl)
- up = urlparse(apiurl)
- username=up.username
- password=up.password
- # Strip username/password and params
- if up.port:
- up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
- else:
- up_stripped = up._replace(params="", netloc=up.hostname)
- res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
- try:
- parsed = json.loads(res.read().decode('utf-8'))
- except ConnectionResetError:
- if retry:
- logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
- parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
- logger.debug(1, "%s: retry successful.")
- else:
- raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
- return parsed
- index = layerindexlib.LayerIndexObj()
- index.config = {}
- index.config['TYPE'] = self.type
- index.config['URL'] = url
- params = self.layerindex._parse_params(up.params)
- if 'desc' in params:
- index.config['DESCRIPTION'] = unquote(params['desc'])
- else:
- index.config['DESCRIPTION'] = up.hostname
- if 'cache' in params:
- index.config['CACHE'] = params['cache']
- if 'branch' in params:
- branches = params['branch'].split(',')
- index.config['BRANCH'] = branches
- else:
- branches = ['*']
- try:
- index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
- except Exception as e:
- raise layerindexlib.LayerIndexFetchError(url, e)
- # Local raw index set...
- pindex = {}
- # Load all the requested branches at the same time time,
- # a special branch of '*' means load all branches
- filter = ""
- if "*" not in branches:
- filter = "?filter=name:%s" % "OR".join(branches)
- logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
- # The link won't include username/password, so pull it from the original url
- pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
- username=up.username, password=up.password)
- if not pindex['branches']:
- logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
- return index
- index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
- # Load all of the layerItems (these can not be easily filtered)
- logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
- # The link won't include username/password, so pull it from the original url
- pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
- username=up.username, password=up.password)
- if not pindex['layerItems']:
- logger.debug(1, "No layers were found at url %s." % (url))
- return index
- index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
- # From this point on load the contents for each branch. Otherwise we
- # could run into a timeout.
- for branch in index.branches:
- filter = "?filter=branch__name:%s" % index.branches[branch].name
- logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
- # The link won't include username/password, so pull it from the original url
- pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
- username=up.username, password=up.password)
- if not pindex['layerBranches']:
- logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
- return index
- index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
- # Load the rest, they all have a similar format
- # Note: the layer index has a few more items, we can add them if necessary
- # in the future.
- filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
- for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
- ("recipes", layerindexlib.Recipe),
- ("machines", layerindexlib.Machine),
- ("distros", layerindexlib.Distro)]:
- if lName not in load:
- continue
- logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
- # The link won't include username/password, so pull it from the original url
- pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
- username=up.username, password=up.password)
- index.add_raw_element(lName, lType, pindex[lName])
- return index
- def store_index(self, url, index):
- """
- Store layer information into a local file/dir.
- The return value is a dictionary containing API,
- layer, branch, dependency, recipe, machine, distro, information.
- ud is a parsed url to a directory or file. If the path is a
- directory, we will split the files into one file per layer.
- If the path is to a file (exists or not) the entire DB will be
- dumped into that one file.
- """
- up = urlparse(url)
- if up.scheme != 'file':
- raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
- logger.debug(1, "Storing to %s..." % up.path)
- try:
- layerbranches = index.layerBranches
- except KeyError:
- logger.error('No layerBranches to write.')
- return
- def filter_item(layerbranchid, objects):
- filtered = []
- for obj in getattr(index, objects, None):
- try:
- if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
- filtered.append(getattr(index, objects)[obj]._data)
- except AttributeError:
- logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
- # No simple filter method, just include it...
- try:
- filtered.append(getattr(index, objects)[obj]._data)
- except AttributeError:
- logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
- filtered.append(obj)
- return filtered
- # Write out to a single file.
- # Filter out unnecessary items, then sort as we write for determinism
- if not os.path.isdir(up.path):
- pindex = {}
- pindex['branches'] = []
- pindex['layerItems'] = []
- pindex['layerBranches'] = []
- for layerbranchid in layerbranches:
- if layerbranches[layerbranchid].branch._data not in pindex['branches']:
- pindex['branches'].append(layerbranches[layerbranchid].branch._data)
- if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
- pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
- if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
- pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
- for entry in index._index:
- # Skip local items, apilinks and items already processed
- if entry in index.config['local'] or \
- entry == 'apilinks' or \
- entry == 'branches' or \
- entry == 'layerBranches' or \
- entry == 'layerItems':
- continue
- if entry not in pindex:
- pindex[entry] = []
- pindex[entry].extend(filter_item(layerbranchid, entry))
- bb.debug(1, 'Writing index to %s' % up.path)
- with open(up.path, 'wt') as f:
- json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
- return
- # Write out to a directory one file per layerBranch
- # Prepare all layer related items, to create a minimal file.
- # We have to sort the entries as we write so they are deterministic
- for layerbranchid in layerbranches:
- pindex = {}
- for entry in index._index:
- # Skip local items, apilinks and items already processed
- if entry in index.config['local'] or \
- entry == 'apilinks' or \
- entry == 'branches' or \
- entry == 'layerBranches' or \
- entry == 'layerItems':
- continue
- pindex[entry] = filter_item(layerbranchid, entry)
- # Add the layer we're processing as the first one...
- pindex['branches'] = [layerbranches[layerbranchid].branch._data]
- pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
- pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
- # We also need to include the layerbranch for any dependencies...
- for layerdep in pindex['layerDependencies']:
- layerdependency = layerindexlib.LayerDependency(index, layerdep)
- layeritem = layerdependency.dependency
- layerbranch = layerdependency.dependency_layerBranch
- # We need to avoid duplicates...
- if layeritem._data not in pindex['layerItems']:
- pindex['layerItems'].append(layeritem._data)
- if layerbranch._data not in pindex['layerBranches']:
- pindex['layerBranches'].append(layerbranch._data)
- # apply mirroring adjustments here....
- fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
- fname = fname.translate(str.maketrans('/ ', '__'))
- fpath = os.path.join(up.path, fname)
- bb.debug(1, 'Writing index to %s' % fpath + '.json')
- with open(fpath + '.json', 'wt') as f:
- json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
|