restapi.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398
  1. # Copyright (C) 2016-2018 Wind River Systems, Inc.
  2. #
  3. # This program is free software; you can redistribute it and/or modify
  4. # it under the terms of the GNU General Public License version 2 as
  5. # published by the Free Software Foundation.
  6. #
  7. # This program is distributed in the hope that it will be useful,
  8. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  10. # See the GNU General Public License for more details.
  11. #
  12. # You should have received a copy of the GNU General Public License
  13. # along with this program; if not, write to the Free Software
  14. # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  15. import logging
  16. import json
  17. from urllib.parse import unquote
  18. from urllib.parse import urlparse
  19. import layerindexlib
  20. import layerindexlib.plugin
  21. logger = logging.getLogger('BitBake.layerindexlib.restapi')
  22. def plugin_init(plugins):
  23. return RestApiPlugin()
  24. class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
  25. def __init__(self):
  26. self.type = "restapi"
  27. def load_index(self, url, load):
  28. """
  29. Fetches layer information from a local or remote layer index.
  30. The return value is a LayerIndexObj.
  31. url is the url to the rest api of the layer index, such as:
  32. http://layers.openembedded.org/layerindex/api/
  33. Or a local file...
  34. """
  35. up = urlparse(url)
  36. if up.scheme == 'file':
  37. return self.load_index_file(up, url, load)
  38. if up.scheme == 'http' or up.scheme == 'https':
  39. return self.load_index_web(up, url, load)
  40. raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
  41. def load_index_file(self, up, url, load):
  42. """
  43. Fetches layer information from a local file or directory.
  44. The return value is a LayerIndexObj.
  45. ud is the parsed url to the local file or directory.
  46. """
  47. if not os.path.exists(up.path):
  48. raise FileNotFoundError(up.path)
  49. index = layerindexlib.LayerIndexObj()
  50. index.config = {}
  51. index.config['TYPE'] = self.type
  52. index.config['URL'] = url
  53. params = self.layerindex._parse_params(up.params)
  54. if 'desc' in params:
  55. index.config['DESCRIPTION'] = unquote(params['desc'])
  56. else:
  57. index.config['DESCRIPTION'] = up.path
  58. if 'cache' in params:
  59. index.config['CACHE'] = params['cache']
  60. if 'branch' in params:
  61. branches = params['branch'].split(',')
  62. index.config['BRANCH'] = branches
  63. else:
  64. branches = ['*']
  65. def load_cache(path, index, branches=[]):
  66. logger.debug(1, 'Loading json file %s' % path)
  67. with open(path, 'rt', encoding='utf-8') as f:
  68. pindex = json.load(f)
  69. # Filter the branches on loaded files...
  70. newpBranch = []
  71. for branch in branches:
  72. if branch != '*':
  73. if 'branches' in pindex:
  74. for br in pindex['branches']:
  75. if br['name'] == branch:
  76. newpBranch.append(br)
  77. else:
  78. if 'branches' in pindex:
  79. for br in pindex['branches']:
  80. newpBranch.append(br)
  81. if newpBranch:
  82. index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
  83. else:
  84. logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
  85. # No matching branches.. return nothing...
  86. return
  87. for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
  88. ("layerBranches", layerindexlib.LayerBranch),
  89. ("layerDependencies", layerindexlib.LayerDependency),
  90. ("recipes", layerindexlib.Recipe),
  91. ("machines", layerindexlib.Machine),
  92. ("distros", layerindexlib.Distro)]:
  93. if lName in pindex:
  94. index.add_raw_element(lName, lType, pindex[lName])
  95. if not os.path.isdir(up.path):
  96. load_cache(up.path, index, branches)
  97. return index
  98. logger.debug(1, 'Loading from dir %s...' % (up.path))
  99. for (dirpath, _, filenames) in os.walk(up.path):
  100. for filename in filenames:
  101. if not filename.endswith('.json'):
  102. continue
  103. fpath = os.path.join(dirpath, filename)
  104. load_cache(fpath, index, branches)
  105. return index
  106. def load_index_web(self, up, url, load):
  107. """
  108. Fetches layer information from a remote layer index.
  109. The return value is a LayerIndexObj.
  110. ud is the parsed url to the rest api of the layer index, such as:
  111. http://layers.openembedded.org/layerindex/api/
  112. """
  113. def _get_json_response(apiurl=None, username=None, password=None, retry=True):
  114. assert apiurl is not None
  115. logger.debug(1, "fetching %s" % apiurl)
  116. up = urlparse(apiurl)
  117. username=up.username
  118. password=up.password
  119. # Strip username/password and params
  120. if up.port:
  121. up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
  122. else:
  123. up_stripped = up._replace(params="", netloc=up.hostname)
  124. res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
  125. try:
  126. parsed = json.loads(res.read().decode('utf-8'))
  127. except ConnectionResetError:
  128. if retry:
  129. logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
  130. parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
  131. logger.debug(1, "%s: retry successful.")
  132. else:
  133. raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
  134. return parsed
  135. index = layerindexlib.LayerIndexObj()
  136. index.config = {}
  137. index.config['TYPE'] = self.type
  138. index.config['URL'] = url
  139. params = self.layerindex._parse_params(up.params)
  140. if 'desc' in params:
  141. index.config['DESCRIPTION'] = unquote(params['desc'])
  142. else:
  143. index.config['DESCRIPTION'] = up.hostname
  144. if 'cache' in params:
  145. index.config['CACHE'] = params['cache']
  146. if 'branch' in params:
  147. branches = params['branch'].split(',')
  148. index.config['BRANCH'] = branches
  149. else:
  150. branches = ['*']
  151. try:
  152. index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
  153. except Exception as e:
  154. raise layerindexlib.LayerIndexFetchError(url, e)
  155. # Local raw index set...
  156. pindex = {}
  157. # Load all the requested branches at the same time time,
  158. # a special branch of '*' means load all branches
  159. filter = ""
  160. if "*" not in branches:
  161. filter = "?filter=name:%s" % "OR".join(branches)
  162. logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
  163. # The link won't include username/password, so pull it from the original url
  164. pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
  165. username=up.username, password=up.password)
  166. if not pindex['branches']:
  167. logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
  168. return index
  169. index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
  170. # Load all of the layerItems (these can not be easily filtered)
  171. logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
  172. # The link won't include username/password, so pull it from the original url
  173. pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
  174. username=up.username, password=up.password)
  175. if not pindex['layerItems']:
  176. logger.debug(1, "No layers were found at url %s." % (url))
  177. return index
  178. index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
  179. # From this point on load the contents for each branch. Otherwise we
  180. # could run into a timeout.
  181. for branch in index.branches:
  182. filter = "?filter=branch__name:%s" % index.branches[branch].name
  183. logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
  184. # The link won't include username/password, so pull it from the original url
  185. pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
  186. username=up.username, password=up.password)
  187. if not pindex['layerBranches']:
  188. logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
  189. return index
  190. index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
  191. # Load the rest, they all have a similar format
  192. # Note: the layer index has a few more items, we can add them if necessary
  193. # in the future.
  194. filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
  195. for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
  196. ("recipes", layerindexlib.Recipe),
  197. ("machines", layerindexlib.Machine),
  198. ("distros", layerindexlib.Distro)]:
  199. if lName not in load:
  200. continue
  201. logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
  202. # The link won't include username/password, so pull it from the original url
  203. pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
  204. username=up.username, password=up.password)
  205. index.add_raw_element(lName, lType, pindex[lName])
  206. return index
  207. def store_index(self, url, index):
  208. """
  209. Store layer information into a local file/dir.
  210. The return value is a dictionary containing API,
  211. layer, branch, dependency, recipe, machine, distro, information.
  212. ud is a parsed url to a directory or file. If the path is a
  213. directory, we will split the files into one file per layer.
  214. If the path is to a file (exists or not) the entire DB will be
  215. dumped into that one file.
  216. """
  217. up = urlparse(url)
  218. if up.scheme != 'file':
  219. raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
  220. logger.debug(1, "Storing to %s..." % up.path)
  221. try:
  222. layerbranches = index.layerBranches
  223. except KeyError:
  224. logger.error('No layerBranches to write.')
  225. return
  226. def filter_item(layerbranchid, objects):
  227. filtered = []
  228. for obj in getattr(index, objects, None):
  229. try:
  230. if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
  231. filtered.append(getattr(index, objects)[obj]._data)
  232. except AttributeError:
  233. logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
  234. # No simple filter method, just include it...
  235. try:
  236. filtered.append(getattr(index, objects)[obj]._data)
  237. except AttributeError:
  238. logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
  239. filtered.append(obj)
  240. return filtered
  241. # Write out to a single file.
  242. # Filter out unnecessary items, then sort as we write for determinism
  243. if not os.path.isdir(up.path):
  244. pindex = {}
  245. pindex['branches'] = []
  246. pindex['layerItems'] = []
  247. pindex['layerBranches'] = []
  248. for layerbranchid in layerbranches:
  249. if layerbranches[layerbranchid].branch._data not in pindex['branches']:
  250. pindex['branches'].append(layerbranches[layerbranchid].branch._data)
  251. if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
  252. pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
  253. if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
  254. pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
  255. for entry in index._index:
  256. # Skip local items, apilinks and items already processed
  257. if entry in index.config['local'] or \
  258. entry == 'apilinks' or \
  259. entry == 'branches' or \
  260. entry == 'layerBranches' or \
  261. entry == 'layerItems':
  262. continue
  263. if entry not in pindex:
  264. pindex[entry] = []
  265. pindex[entry].extend(filter_item(layerbranchid, entry))
  266. bb.debug(1, 'Writing index to %s' % up.path)
  267. with open(up.path, 'wt') as f:
  268. json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
  269. return
  270. # Write out to a directory one file per layerBranch
  271. # Prepare all layer related items, to create a minimal file.
  272. # We have to sort the entries as we write so they are deterministic
  273. for layerbranchid in layerbranches:
  274. pindex = {}
  275. for entry in index._index:
  276. # Skip local items, apilinks and items already processed
  277. if entry in index.config['local'] or \
  278. entry == 'apilinks' or \
  279. entry == 'branches' or \
  280. entry == 'layerBranches' or \
  281. entry == 'layerItems':
  282. continue
  283. pindex[entry] = filter_item(layerbranchid, entry)
  284. # Add the layer we're processing as the first one...
  285. pindex['branches'] = [layerbranches[layerbranchid].branch._data]
  286. pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
  287. pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
  288. # We also need to include the layerbranch for any dependencies...
  289. for layerdep in pindex['layerDependencies']:
  290. layerdependency = layerindexlib.LayerDependency(index, layerdep)
  291. layeritem = layerdependency.dependency
  292. layerbranch = layerdependency.dependency_layerBranch
  293. # We need to avoid duplicates...
  294. if layeritem._data not in pindex['layerItems']:
  295. pindex['layerItems'].append(layeritem._data)
  296. if layerbranch._data not in pindex['layerBranches']:
  297. pindex['layerBranches'].append(layerbranch._data)
  298. # apply mirroring adjustments here....
  299. fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
  300. fname = fname.translate(str.maketrans('/ ', '__'))
  301. fpath = os.path.join(up.path, fname)
  302. bb.debug(1, 'Writing index to %s' % fpath + '.json')
  303. with open(fpath + '.json', 'wt') as f:
  304. json.dump(layerindexlib.sort_entry(pindex), f, indent=4)