__init__.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353
  1. # Copyright (C) 2016-2018 Wind River Systems, Inc.
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-only
  4. #
  5. import datetime
  6. import logging
  7. import imp
  8. from collections import OrderedDict
  9. from layerindexlib.plugin import LayerIndexPluginUrlError
  10. logger = logging.getLogger('BitBake.layerindexlib')
  11. # Exceptions
  12. class LayerIndexException(Exception):
  13. '''LayerIndex Generic Exception'''
  14. def __init__(self, message):
  15. self.msg = message
  16. Exception.__init__(self, message)
  17. def __str__(self):
  18. return self.msg
  19. class LayerIndexUrlError(LayerIndexException):
  20. '''Exception raised when unable to access a URL for some reason'''
  21. def __init__(self, url, message=""):
  22. if message:
  23. msg = "Unable to access layerindex url %s: %s" % (url, message)
  24. else:
  25. msg = "Unable to access layerindex url %s" % url
  26. self.url = url
  27. LayerIndexException.__init__(self, msg)
  28. class LayerIndexFetchError(LayerIndexException):
  29. '''General layerindex fetcher exception when something fails'''
  30. def __init__(self, url, message=""):
  31. if message:
  32. msg = "Unable to fetch layerindex url %s: %s" % (url, message)
  33. else:
  34. msg = "Unable to fetch layerindex url %s" % url
  35. self.url = url
  36. LayerIndexException.__init__(self, msg)
  37. # Interface to the overall layerindex system
  38. # the layer may contain one or more individual indexes
  39. class LayerIndex():
  40. def __init__(self, d):
  41. if not d:
  42. raise LayerIndexException("Must be initialized with bb.data.")
  43. self.data = d
  44. # List of LayerIndexObj
  45. self.indexes = []
  46. self.plugins = []
  47. import bb.utils
  48. bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__))
  49. for plugin in self.plugins:
  50. if hasattr(plugin, 'init'):
  51. plugin.init(self)
  52. def __add__(self, other):
  53. newIndex = LayerIndex(self.data)
  54. if self.__class__ != newIndex.__class__ or \
  55. other.__class__ != newIndex.__class__:
  56. raise TypeException("Can not add different types.")
  57. for indexEnt in self.indexes:
  58. newIndex.indexes.append(indexEnt)
  59. for indexEnt in other.indexes:
  60. newIndex.indexes.append(indexEnt)
  61. return newIndex
  62. def _parse_params(self, params):
  63. '''Take a parameter list, return a dictionary of parameters.
  64. Expected to be called from the data of urllib.parse.urlparse(url).params
  65. If there are two conflicting parameters, last in wins...
  66. '''
  67. param_dict = {}
  68. for param in params.split(';'):
  69. if not param:
  70. continue
  71. item = param.split('=', 1)
  72. logger.debug(1, item)
  73. param_dict[item[0]] = item[1]
  74. return param_dict
  75. def _fetch_url(self, url, username=None, password=None, debuglevel=0):
  76. '''Fetch data from a specific URL.
  77. Fetch something from a specific URL. This is specifically designed to
  78. fetch data from a layerindex-web instance, but may be useful for other
  79. raw fetch actions.
  80. It is not designed to be used to fetch recipe sources or similar. the
  81. regular fetcher class should used for that.
  82. It is the responsibility of the caller to check BB_NO_NETWORK and related
  83. BB_ALLOWED_NETWORKS.
  84. '''
  85. if not url:
  86. raise LayerIndexUrlError(url, "empty url")
  87. import urllib
  88. from urllib.request import urlopen, Request
  89. from urllib.parse import urlparse
  90. up = urlparse(url)
  91. if username:
  92. logger.debug(1, "Configuring authentication for %s..." % url)
  93. password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
  94. password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
  95. handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
  96. opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel))
  97. else:
  98. opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel))
  99. urllib.request.install_opener(opener)
  100. logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
  101. try:
  102. res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
  103. except urllib.error.HTTPError as e:
  104. logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
  105. logger.debug(1, " Requested: %s" % (url))
  106. logger.debug(1, " Actual: %s" % (e.geturl()))
  107. if e.code == 404:
  108. logger.debug(1, "Request not found.")
  109. raise LayerIndexFetchError(url, e)
  110. else:
  111. logger.debug(1, "Headers:\n%s" % (e.headers))
  112. raise LayerIndexFetchError(url, e)
  113. except OSError as e:
  114. error = 0
  115. reason = ""
  116. # Process base OSError first...
  117. if hasattr(e, 'errno'):
  118. error = e.errno
  119. reason = e.strerror
  120. # Process gaierror (socket error) subclass if available.
  121. if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'):
  122. error = e.reason.errno
  123. reason = e.reason.strerror
  124. if error == -2:
  125. raise LayerIndexFetchError(url, "%s: %s" % (e, reason))
  126. if error and error != 0:
  127. raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason))
  128. else:
  129. raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
  130. finally:
  131. logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
  132. return res
  133. def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False):
  134. '''Load the layerindex.
  135. indexURI - An index to load. (Use multiple calls to load multiple indexes)
  136. reload - If reload is True, then any previously loaded indexes will be forgotten.
  137. load - List of elements to load. Default loads all items.
  138. Note: plugs may ignore this.
  139. The format of the indexURI:
  140. <url>;branch=<branch>;cache=<cache>;desc=<description>
  141. Note: the 'branch' parameter if set can select multiple branches by using
  142. comma, such as 'branch=master,morty,pyro'. However, many operations only look
  143. at the -first- branch specified!
  144. The cache value may be undefined, in this case a network failure will
  145. result in an error, otherwise the system will look for a file of the cache
  146. name and load that instead.
  147. For example:
  148. http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
  149. cooker://
  150. '''
  151. if reload:
  152. self.indexes = []
  153. logger.debug(1, 'Loading: %s' % indexURI)
  154. if not self.plugins:
  155. raise LayerIndexException("No LayerIndex Plugins available")
  156. for plugin in self.plugins:
  157. # Check if the plugin was initialized
  158. logger.debug(1, 'Trying %s' % plugin.__class__)
  159. if not hasattr(plugin, 'type') or not plugin.type:
  160. continue
  161. try:
  162. # TODO: Implement 'cache', for when the network is not available
  163. indexEnt = plugin.load_index(indexURI, load)
  164. break
  165. except LayerIndexPluginUrlError as e:
  166. logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
  167. except NotImplementedError:
  168. pass
  169. else:
  170. logger.debug(1, "No plugins support %s" % indexURI)
  171. raise LayerIndexException("No plugins support %s" % indexURI)
  172. # Mark CONFIG data as something we've added...
  173. indexEnt.config['local'] = []
  174. indexEnt.config['local'].append('config')
  175. # No longer permit changes..
  176. indexEnt.lockData()
  177. self.indexes.append(indexEnt)
  178. def store_layerindex(self, indexURI, index=None):
  179. '''Store one layerindex
  180. Typically this will be used to create a local cache file of a remote index.
  181. file://<path>;branch=<branch>
  182. We can write out in either the restapi or django formats. The split option
  183. will write out the individual elements split by layer and related components.
  184. '''
  185. if not index:
  186. logger.warning('No index to write, nothing to do.')
  187. return
  188. if not self.plugins:
  189. raise LayerIndexException("No LayerIndex Plugins available")
  190. for plugin in self.plugins:
  191. # Check if the plugin was initialized
  192. logger.debug(1, 'Trying %s' % plugin.__class__)
  193. if not hasattr(plugin, 'type') or not plugin.type:
  194. continue
  195. try:
  196. plugin.store_index(indexURI, index)
  197. break
  198. except LayerIndexPluginUrlError as e:
  199. logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
  200. except NotImplementedError:
  201. logger.debug(1, "Store not implemented in %s" % plugin.type)
  202. pass
  203. else:
  204. logger.debug(1, "No plugins support %s" % url)
  205. raise LayerIndexException("No plugins support %s" % url)
  206. def is_empty(self):
  207. '''Return True or False if the index has any usable data.
  208. We check the indexes entries to see if they have a branch set, as well as
  209. layerBranches set. If not, they are effectively blank.'''
  210. found = False
  211. for index in self.indexes:
  212. if index.__bool__():
  213. found = True
  214. break
  215. return not found
  216. def find_vcs_url(self, vcs_url, branch=None):
  217. '''Return the first layerBranch with the given vcs_url
  218. If a branch has not been specified, we will iterate over the branches in
  219. the default configuration until the first vcs_url/branch match.'''
  220. for index in self.indexes:
  221. logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
  222. layerBranch = index.find_vcs_url(vcs_url, [branch])
  223. if layerBranch:
  224. return layerBranch
  225. return None
  226. def find_collection(self, collection, version=None, branch=None):
  227. '''Return the first layerBranch with the given collection name
  228. If a branch has not been specified, we will iterate over the branches in
  229. the default configuration until the first collection/branch match.'''
  230. logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
  231. if branch:
  232. branches = [branch]
  233. else:
  234. branches = None
  235. for index in self.indexes:
  236. logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
  237. layerBranch = index.find_collection(collection, version, branches)
  238. if layerBranch:
  239. return layerBranch
  240. else:
  241. logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
  242. return None
  243. def find_layerbranch(self, name, branch=None):
  244. '''Return the layerBranch item for a given name and branch
  245. If a branch has not been specified, we will iterate over the branches in
  246. the default configuration until the first name/branch match.'''
  247. if branch:
  248. branches = [branch]
  249. else:
  250. branches = None
  251. for index in self.indexes:
  252. layerBranch = index.find_layerbranch(name, branches)
  253. if layerBranch:
  254. return layerBranch
  255. return None
  256. def find_dependencies(self, names=None, layerbranches=None, ignores=None):
  257. '''Return a tuple of all dependencies and valid items for the list of (layer) names
  258. The dependency scanning happens depth-first. The returned
  259. dependencies should be in the best order to define bblayers.
  260. names - list of layer names (searching layerItems)
  261. branches - when specified (with names) only this list of branches are evaluated
  262. layerbranches - list of layerbranches to resolve dependencies
  263. ignores - list of layer names to ignore
  264. return: (dependencies, invalid)
  265. dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
  266. invalid = [ LayerItem.name1, LayerItem.name2, ... ]
  267. '''
  268. invalid = []
  269. # Convert name/branch to layerbranches
  270. if layerbranches is None:
  271. layerbranches = []
  272. for name in names:
  273. if ignores and name in ignores:
  274. continue
  275. for index in self.indexes:
  276. layerbranch = index.find_layerbranch(name)
  277. if not layerbranch:
  278. # Not in this index, hopefully it's in another...
  279. continue
  280. layerbranches.append(layerbranch)
  281. break
  282. else:
  283. invalid.append(name)
  284. def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
  285. for layerbranch in layerbranches:
  286. if ignores and layerbranch.layer.name in ignores:
  287. continue
  288. # Get a list of dependencies and then recursively process them
  289. for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
  290. deplayerbranch = layerdependency.dependency_layerBranch
  291. if ignores and deplayerbranch.layer.name in ignores:
  292. continue
  293. # This little block is why we can't re-use the LayerIndexObj version,
  294. # we must be able to satisfy each dependencies across layer indexes and
  295. # use the layer index order for priority. (r stands for replacement below)
  296. # If this is the primary index, we can fast path and skip this
  297. if deplayerbranch.index != self.indexes[0]:
  298. # Is there an entry in a prior index for this collection/version?
  299. rdeplayerbranch = self.find_collection(
  300. collection=deplayerbranch.collection,
  301. version=deplayerbranch.version
  302. )
  303. if rdeplayerbranch != deplayerbranch:
  304. logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
  305. (deplayerbranch.index.config['DESCRIPTION'],
  306. deplayerbranch.branch.name,
  307. deplayerbranch.layer.name,
  308. rdeplayerbranch.index.config['DESCRIPTION'],
  309. rdeplayerbranch.branch.name,
  310. rdeplayerbranch.layer.name))
  311. deplayerbranch = rdeplayerbranch
  312. # New dependency, we need to resolve it now... depth-first
  313. if deplayerbranch.layer.name not in dependencies:
  314. (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
  315. if deplayerbranch.layer.name not in dependencies:
  316. dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
  317. else:
  318. if layerdependency not in dependencies[deplayerbranch.layer.name]:
  319. dependencies[deplayerbranch.layer.name].append(layerdependency)
  320. return (dependencies, invalid)
  321. # OK, resolve this one...
  322. dependencies = OrderedDict()
  323. (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
  324. for layerbranch in layerbranches:
  325. if layerbranch.layer.name not in dependencies:
  326. dependencies[layerbranch.layer.name] = [layerbranch]
  327. return (dependencies, invalid)
  328. def list_obj(self, object):
  329. '''Print via the plain logger object information
  330. This function is used to implement debugging and provide the user info.
  331. '''
  332. for lix in self.indexes:
  333. if not hasattr(lix, object):
  334. continue
  335. logger.plain ('')
  336. logger.plain ('Index: %s' % lix.config['DESCRIPTION'])
  337. output = []
  338. if object == 'branches':
  339. logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch')))
  340. logger.plain ('{:-^80}'.format(""))
  341. for branchid in lix.branches:
  342. output.append('%s %s %s' % (
  343. '{:26}'.format(lix.branches[branchid].name),
  344. '{:34}'.format(lix.branches[branchid].short_description),
  345. '{:22}'.format(lix.branches[branchid].bitbake_branch)
  346. ))
  347. for line in sorted(output):
  348. logger.plain (line)
  349. continue
  350. if object == 'layerItems':
  351. logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description')))
  352. logger.plain ('{:-^80}'.format(""))
  353. for layerid in lix.layerItems:
  354. output.append('%s %s' % (
  355. '{:26}'.format(lix.layerItems[layerid].name),
  356. '{:34}'.format(lix.layerItems[layerid].summary)
  357. ))
  358. for line in sorted(output):
  359. logger.plain (line)
  360. continue
  361. if object == 'layerBranches':
  362. logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version')))
  363. logger.plain ('{:-^80}'.format(""))
  364. for layerbranchid in lix.layerBranches:
  365. output.append('%s %s %s' % (
  366. '{:26}'.format(lix.layerBranches[layerbranchid].layer.name),
  367. '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary),
  368. '{:19}'.format("%s:%s" %
  369. (lix.layerBranches[layerbranchid].collection,
  370. lix.layerBranches[layerbranchid].version)
  371. )
  372. ))
  373. for line in sorted(output):
  374. logger.plain (line)
  375. continue
  376. if object == 'layerDependencies':
  377. logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer')))
  378. logger.plain ('{:-^80}'.format(""))
  379. for layerDependency in lix.layerDependencies:
  380. if not lix.layerDependencies[layerDependency].dependency_layerBranch:
  381. continue
  382. output.append('%s %s %s %s' % (
  383. '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name),
  384. '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name),
  385. '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'),
  386. '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name)
  387. ))
  388. for line in sorted(output):
  389. logger.plain (line)
  390. continue
  391. if object == 'recipes':
  392. logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer'))
  393. logger.plain ('{:-^80}'.format(""))
  394. output = []
  395. for recipe in lix.recipes:
  396. output.append('%s %s %s' % (
  397. '{:30}'.format(lix.recipes[recipe].pn),
  398. '{:30}'.format(lix.recipes[recipe].pv),
  399. lix.recipes[recipe].layer.name
  400. ))
  401. for line in sorted(output):
  402. logger.plain (line)
  403. continue
  404. if object == 'machines':
  405. logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer')))
  406. logger.plain ('{:-^80}'.format(""))
  407. for machine in lix.machines:
  408. output.append('%s %s %s' % (
  409. '{:24}'.format(lix.machines[machine].name),
  410. '{:34}'.format(lix.machines[machine].description)[:34],
  411. '{:19}'.format(lix.machines[machine].layerbranch.layer.name)
  412. ))
  413. for line in sorted(output):
  414. logger.plain (line)
  415. continue
  416. if object == 'distros':
  417. logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer')))
  418. logger.plain ('{:-^80}'.format(""))
  419. for distro in lix.distros:
  420. output.append('%s %s %s' % (
  421. '{:24}'.format(lix.distros[distro].name),
  422. '{:34}'.format(lix.distros[distro].description)[:34],
  423. '{:19}'.format(lix.distros[distro].layerbranch.layer.name)
  424. ))
  425. for line in sorted(output):
  426. logger.plain (line)
  427. continue
  428. logger.plain ('')
  429. # This class holds a single layer index instance
  430. # The LayerIndexObj is made up of dictionary of elements, such as:
  431. # index['config'] - configuration data for this index
  432. # index['branches'] - dictionary of Branch objects, by id number
  433. # index['layerItems'] - dictionary of layerItem objects, by id number
  434. # ...etc... (See: http://layers.openembedded.org/layerindex/api/)
  435. #
  436. # The class needs to manage the 'index' entries and allow easily adding
  437. # of new items, as well as simply loading of the items.
  438. class LayerIndexObj():
  439. def __init__(self):
  440. super().__setattr__('_index', {})
  441. super().__setattr__('_lock', False)
  442. def __bool__(self):
  443. '''False if the index is effectively empty
  444. We check the index to see if it has a branch set, as well as
  445. layerbranches set. If not, it is effectively blank.'''
  446. if not bool(self._index):
  447. return False
  448. try:
  449. if self.branches and self.layerBranches:
  450. return True
  451. except AttributeError:
  452. pass
  453. return False
  454. def __getattr__(self, name):
  455. if name.startswith('_'):
  456. return super().__getattribute__(name)
  457. if name not in self._index:
  458. raise AttributeError('%s not in index datastore' % name)
  459. return self._index[name]
  460. def __setattr__(self, name, value):
  461. if self.isLocked():
  462. raise TypeError("Can not set attribute '%s': index is locked" % name)
  463. if name.startswith('_'):
  464. super().__setattr__(name, value)
  465. return
  466. self._index[name] = value
  467. def __delattr__(self, name):
  468. if self.isLocked():
  469. raise TypeError("Can not delete attribute '%s': index is locked" % name)
  470. if name.startswith('_'):
  471. super().__delattr__(name)
  472. self._index.pop(name)
  473. def lockData(self):
  474. '''Lock data object (make it readonly)'''
  475. super().__setattr__("_lock", True)
  476. def unlockData(self):
  477. '''unlock data object (make it readonly)'''
  478. super().__setattr__("_lock", False)
  479. # When the data is unlocked, we have to clear the caches, as
  480. # modification is allowed!
  481. del(self._layerBranches_layerId_branchId)
  482. del(self._layerDependencies_layerBranchId)
  483. del(self._layerBranches_vcsUrl)
  484. def isLocked(self):
  485. '''Is this object locked (readonly)?'''
  486. return self._lock
  487. def add_element(self, indexname, objs):
  488. '''Add a layer index object to index.<indexname>'''
  489. if indexname not in self._index:
  490. self._index[indexname] = {}
  491. for obj in objs:
  492. if obj.id in self._index[indexname]:
  493. if self._index[indexname][obj.id] == obj:
  494. continue
  495. raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id))
  496. self._index[indexname][obj.id] = obj
  497. def add_raw_element(self, indexname, objtype, rawobjs):
  498. '''Convert a raw layer index data item to a layer index item object and add to the index'''
  499. objs = []
  500. for entry in rawobjs:
  501. objs.append(objtype(self, entry))
  502. self.add_element(indexname, objs)
  503. # Quick lookup table for searching layerId and branchID combos
  504. @property
  505. def layerBranches_layerId_branchId(self):
  506. def createCache(self):
  507. cache = {}
  508. for layerbranchid in self.layerBranches:
  509. layerbranch = self.layerBranches[layerbranchid]
  510. cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch
  511. return cache
  512. if self.isLocked():
  513. cache = getattr(self, '_layerBranches_layerId_branchId', None)
  514. else:
  515. cache = None
  516. if not cache:
  517. cache = createCache(self)
  518. if self.isLocked():
  519. super().__setattr__('_layerBranches_layerId_branchId', cache)
  520. return cache
  521. # Quick lookup table for finding all dependencies of a layerBranch
  522. @property
  523. def layerDependencies_layerBranchId(self):
  524. def createCache(self):
  525. cache = {}
  526. # This ensures empty lists for all branchids
  527. for layerbranchid in self.layerBranches:
  528. cache[layerbranchid] = []
  529. for layerdependencyid in self.layerDependencies:
  530. layerdependency = self.layerDependencies[layerdependencyid]
  531. cache[layerdependency.layerbranch_id].append(layerdependency)
  532. return cache
  533. if self.isLocked():
  534. cache = getattr(self, '_layerDependencies_layerBranchId', None)
  535. else:
  536. cache = None
  537. if not cache:
  538. cache = createCache(self)
  539. if self.isLocked():
  540. super().__setattr__('_layerDependencies_layerBranchId', cache)
  541. return cache
  542. # Quick lookup table for finding all instances of a vcs_url
  543. @property
  544. def layerBranches_vcsUrl(self):
  545. def createCache(self):
  546. cache = {}
  547. for layerbranchid in self.layerBranches:
  548. layerbranch = self.layerBranches[layerbranchid]
  549. if layerbranch.layer.vcs_url not in cache:
  550. cache[layerbranch.layer.vcs_url] = [layerbranch]
  551. else:
  552. cache[layerbranch.layer.vcs_url].append(layerbranch)
  553. return cache
  554. if self.isLocked():
  555. cache = getattr(self, '_layerBranches_vcsUrl', None)
  556. else:
  557. cache = None
  558. if not cache:
  559. cache = createCache(self)
  560. if self.isLocked():
  561. super().__setattr__('_layerBranches_vcsUrl', cache)
  562. return cache
  563. def find_vcs_url(self, vcs_url, branches=None):
  564. ''''Return the first layerBranch with the given vcs_url
  565. If a list of branches has not been specified, we will iterate on
  566. all branches until the first vcs_url is found.'''
  567. if not self.__bool__():
  568. return None
  569. for layerbranch in self.layerBranches_vcsUrl:
  570. if branches and layerbranch.branch.name not in branches:
  571. continue
  572. return layerbranch
  573. return None
  574. def find_collection(self, collection, version=None, branches=None):
  575. '''Return the first layerBranch with the given collection name
  576. If a list of branches has not been specified, we will iterate on
  577. all branches until the first collection is found.'''
  578. if not self.__bool__():
  579. return None
  580. for layerbranchid in self.layerBranches:
  581. layerbranch = self.layerBranches[layerbranchid]
  582. if branches and layerbranch.branch.name not in branches:
  583. continue
  584. if layerbranch.collection == collection and \
  585. (version is None or version == layerbranch.version):
  586. return layerbranch
  587. return None
  588. def find_layerbranch(self, name, branches=None):
  589. '''Return the first layerbranch whose layer name matches
  590. If a list of branches has not been specified, we will iterate on
  591. all branches until the first layer with that name is found.'''
  592. if not self.__bool__():
  593. return None
  594. for layerbranchid in self.layerBranches:
  595. layerbranch = self.layerBranches[layerbranchid]
  596. if branches and layerbranch.branch.name not in branches:
  597. continue
  598. if layerbranch.layer.name == name:
  599. return layerbranch
  600. return None
  601. def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None):
  602. '''Return a tuple of all dependencies and valid items for the list of (layer) names
  603. The dependency scanning happens depth-first. The returned
  604. dependencies should be in the best order to define bblayers.
  605. names - list of layer names (searching layerItems)
  606. branches - when specified (with names) only this list of branches are evaluated
  607. layerBranches - list of layerBranches to resolve dependencies
  608. ignores - list of layer names to ignore
  609. return: (dependencies, invalid)
  610. dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
  611. invalid = [ LayerItem.name1, LayerItem.name2, ... ]'''
  612. invalid = []
  613. # Convert name/branch to layerBranches
  614. if layerbranches is None:
  615. layerbranches = []
  616. for name in names:
  617. if ignores and name in ignores:
  618. continue
  619. layerbranch = self.find_layerbranch(name, branches)
  620. if not layerbranch:
  621. invalid.append(name)
  622. else:
  623. layerbranches.append(layerbranch)
  624. for layerbranch in layerbranches:
  625. if layerbranch.index != self:
  626. raise LayerIndexException("Can not resolve dependencies across indexes with this class function!")
  627. def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
  628. for layerbranch in layerbranches:
  629. if ignores and layerBranch.layer.name in ignores:
  630. continue
  631. for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]:
  632. deplayerbranch = layerDependency.dependency_layerBranch
  633. if ignores and deplayerbranch.layer.name in ignores:
  634. continue
  635. # New dependency, we need to resolve it now... depth-first
  636. if deplayerbranch.layer.name not in dependencies:
  637. (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
  638. if deplayerbranch.layer.name not in dependencies:
  639. dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
  640. else:
  641. if layerdependency not in dependencies[deplayerbranch.layer.name]:
  642. dependencies[deplayerbranch.layer.name].append(layerdependency)
  643. return (dependencies, invalid)
  644. # OK, resolve this one...
  645. dependencies = OrderedDict()
  646. (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
  647. # Is this item already in the list, if not add it
  648. for layerbranch in layerbranches:
  649. if layerbranch.layer.name not in dependencies:
  650. dependencies[layerbranch.layer.name] = [layerbranch]
  651. return (dependencies, invalid)
  652. # Define a basic LayerIndexItemObj. This object forms the basis for all other
  653. # objects. The raw Layer Index data is stored in the _data element, but we
  654. # do not want users to access data directly. So wrap this and protect it
  655. # from direct manipulation.
  656. #
  657. # It is up to the insantiators of the objects to fill them out, and once done
  658. # lock the objects to prevent further accidently manipulation.
  659. #
  660. # Using the getattr, setattr and properties we can access and manipulate
  661. # the data within the data element.
  662. class LayerIndexItemObj():
  663. def __init__(self, index, data=None, lock=False):
  664. if data is None:
  665. data = {}
  666. if type(data) != type(dict()):
  667. raise TypeError('data (%s) is not a dict' % type(data))
  668. super().__setattr__('_lock', lock)
  669. super().__setattr__('index', index)
  670. super().__setattr__('_data', data)
  671. def __eq__(self, other):
  672. if self.__class__ != other.__class__:
  673. return False
  674. res=(self._data == other._data)
  675. return res
  676. def __bool__(self):
  677. return bool(self._data)
  678. def __getattr__(self, name):
  679. # These are internal to THIS class, and not part of data
  680. if name == "index" or name.startswith('_'):
  681. return super().__getattribute__(name)
  682. if name not in self._data:
  683. raise AttributeError('%s not in datastore' % name)
  684. return self._data[name]
  685. def _setattr(self, name, value, prop=True):
  686. '''__setattr__ like function, but with control over property object behavior'''
  687. if self.isLocked():
  688. raise TypeError("Can not set attribute '%s': Object data is locked" % name)
  689. if name.startswith('_'):
  690. super().__setattr__(name, value)
  691. return
  692. # Since __setattr__ runs before properties, we need to check if
  693. # there is a setter property and then execute it
  694. # ... or return self._data[name]
  695. propertyobj = getattr(self.__class__, name, None)
  696. if prop and isinstance(propertyobj, property):
  697. if propertyobj.fset:
  698. propertyobj.fset(self, value)
  699. else:
  700. raise AttributeError('Attribute %s is readonly, and may not be set' % name)
  701. else:
  702. self._data[name] = value
  703. def __setattr__(self, name, value):
  704. self._setattr(name, value, prop=True)
  705. def _delattr(self, name, prop=True):
  706. # Since __delattr__ runs before properties, we need to check if
  707. # there is a deleter property and then execute it
  708. # ... or we pop it ourselves..
  709. propertyobj = getattr(self.__class__, name, None)
  710. if prop and isinstance(propertyobj, property):
  711. if propertyobj.fdel:
  712. propertyobj.fdel(self)
  713. else:
  714. raise AttributeError('Attribute %s is readonly, and may not be deleted' % name)
  715. else:
  716. self._data.pop(name)
  717. def __delattr__(self, name):
  718. self._delattr(name, prop=True)
  719. def lockData(self):
  720. '''Lock data object (make it readonly)'''
  721. super().__setattr__("_lock", True)
  722. def unlockData(self):
  723. '''unlock data object (make it readonly)'''
  724. super().__setattr__("_lock", False)
  725. def isLocked(self):
  726. '''Is this object locked (readonly)?'''
  727. return self._lock
  728. # Branch object
  729. class Branch(LayerIndexItemObj):
  730. def define_data(self, id, name, bitbake_branch,
  731. short_description=None, sort_priority=1,
  732. updates_enabled=True, updated=None,
  733. update_environment=None):
  734. self.id = id
  735. self.name = name
  736. self.bitbake_branch = bitbake_branch
  737. self.short_description = short_description or name
  738. self.sort_priority = sort_priority
  739. self.updates_enabled = updates_enabled
  740. self.updated = updated or datetime.datetime.today().isoformat()
  741. self.update_environment = update_environment
  742. @property
  743. def name(self):
  744. return self.__getattr__('name')
  745. @name.setter
  746. def name(self, value):
  747. self._data['name'] = value
  748. if self.bitbake_branch == value:
  749. self.bitbake_branch = ""
  750. @name.deleter
  751. def name(self):
  752. self._delattr('name', prop=False)
  753. @property
  754. def bitbake_branch(self):
  755. try:
  756. return self.__getattr__('bitbake_branch')
  757. except AttributeError:
  758. return self.name
  759. @bitbake_branch.setter
  760. def bitbake_branch(self, value):
  761. if self.name == value:
  762. self._data['bitbake_branch'] = ""
  763. else:
  764. self._data['bitbake_branch'] = value
  765. @bitbake_branch.deleter
  766. def bitbake_branch(self):
  767. self._delattr('bitbake_branch', prop=False)
  768. class LayerItem(LayerIndexItemObj):
  769. def define_data(self, id, name, status='P',
  770. layer_type='A', summary=None,
  771. description=None,
  772. vcs_url=None, vcs_web_url=None,
  773. vcs_web_tree_base_url=None,
  774. vcs_web_file_base_url=None,
  775. usage_url=None,
  776. mailing_list_url=None,
  777. index_preference=1,
  778. classic=False,
  779. updated=None):
  780. self.id = id
  781. self.name = name
  782. self.status = status
  783. self.layer_type = layer_type
  784. self.summary = summary or name
  785. self.description = description or summary or name
  786. self.vcs_url = vcs_url
  787. self.vcs_web_url = vcs_web_url
  788. self.vcs_web_tree_base_url = vcs_web_tree_base_url
  789. self.vcs_web_file_base_url = vcs_web_file_base_url
  790. self.index_preference = index_preference
  791. self.classic = classic
  792. self.updated = updated or datetime.datetime.today().isoformat()
  793. class LayerBranch(LayerIndexItemObj):
  794. def define_data(self, id, collection, version, layer, branch,
  795. vcs_subdir="", vcs_last_fetch=None,
  796. vcs_last_rev=None, vcs_last_commit=None,
  797. actual_branch="",
  798. updated=None):
  799. self.id = id
  800. self.collection = collection
  801. self.version = version
  802. if isinstance(layer, LayerItem):
  803. self.layer = layer
  804. else:
  805. self.layer_id = layer
  806. if isinstance(branch, Branch):
  807. self.branch = branch
  808. else:
  809. self.branch_id = branch
  810. self.vcs_subdir = vcs_subdir
  811. self.vcs_last_fetch = vcs_last_fetch
  812. self.vcs_last_rev = vcs_last_rev
  813. self.vcs_last_commit = vcs_last_commit
  814. self.actual_branch = actual_branch
  815. self.updated = updated or datetime.datetime.today().isoformat()
  816. # This is a little odd, the _data attribute is 'layer', but it's really
  817. # referring to the layer id.. so lets adjust this to make it useful
  818. @property
  819. def layer_id(self):
  820. return self.__getattr__('layer')
  821. @layer_id.setter
  822. def layer_id(self, value):
  823. self._setattr('layer', value, prop=False)
  824. @layer_id.deleter
  825. def layer_id(self):
  826. self._delattr('layer', prop=False)
  827. @property
  828. def layer(self):
  829. try:
  830. return self.index.layerItems[self.layer_id]
  831. except KeyError:
  832. raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id)
  833. except IndexError:
  834. raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id)
  835. @layer.setter
  836. def layer(self, value):
  837. if not isinstance(value, LayerItem):
  838. raise TypeError('value is not a LayerItem')
  839. if self.index != value.index:
  840. raise AttributeError('Object and value do not share the same index and thus key set.')
  841. self.layer_id = value.id
  842. @layer.deleter
  843. def layer(self):
  844. del self.layer_id
  845. @property
  846. def branch_id(self):
  847. return self.__getattr__('branch')
  848. @branch_id.setter
  849. def branch_id(self, value):
  850. self._setattr('branch', value, prop=False)
  851. @branch_id.deleter
  852. def branch_id(self):
  853. self._delattr('branch', prop=False)
  854. @property
  855. def branch(self):
  856. try:
  857. logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
  858. return self.index.branches[self.branch_id]
  859. except KeyError:
  860. raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
  861. except IndexError:
  862. raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id)
  863. @branch.setter
  864. def branch(self, value):
  865. if not isinstance(value, LayerItem):
  866. raise TypeError('value is not a LayerItem')
  867. if self.index != value.index:
  868. raise AttributeError('Object and value do not share the same index and thus key set.')
  869. self.branch_id = value.id
  870. @branch.deleter
  871. def branch(self):
  872. del self.branch_id
  873. @property
  874. def actual_branch(self):
  875. if self.__getattr__('actual_branch'):
  876. return self.__getattr__('actual_branch')
  877. else:
  878. return self.branch.name
  879. @actual_branch.setter
  880. def actual_branch(self, value):
  881. logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
  882. if value != self.branch.name:
  883. self._setattr('actual_branch', value, prop=False)
  884. else:
  885. self._setattr('actual_branch', '', prop=False)
  886. @actual_branch.deleter
  887. def actual_branch(self):
  888. self._delattr('actual_branch', prop=False)
  889. # Extend LayerIndexItemObj with common LayerBranch manipulations
  890. # All of the remaining LayerIndex objects refer to layerbranch, and it is
  891. # up to the user to follow that back through the LayerBranch object into
  892. # the layer object to get various attributes. So add an intermediate set
  893. # of attributes that can easily get us the layerbranch as well as layer.
  894. class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
  895. @property
  896. def layerbranch_id(self):
  897. return self.__getattr__('layerbranch')
  898. @layerbranch_id.setter
  899. def layerbranch_id(self, value):
  900. self._setattr('layerbranch', value, prop=False)
  901. @layerbranch_id.deleter
  902. def layerbranch_id(self):
  903. self._delattr('layerbranch', prop=False)
  904. @property
  905. def layerbranch(self):
  906. try:
  907. return self.index.layerBranches[self.layerbranch_id]
  908. except KeyError:
  909. raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id)
  910. except IndexError:
  911. raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id)
  912. @layerbranch.setter
  913. def layerbranch(self, value):
  914. if not isinstance(value, LayerBranch):
  915. raise TypeError('value (%s) is not a layerBranch' % type(value))
  916. if self.index != value.index:
  917. raise AttributeError('Object and value do not share the same index and thus key set.')
  918. self.layerbranch_id = value.id
  919. @layerbranch.deleter
  920. def layerbranch(self):
  921. del self.layerbranch_id
  922. @property
  923. def layer_id(self):
  924. return self.layerbranch.layer_id
  925. # Doesn't make sense to set or delete layer_id
  926. @property
  927. def layer(self):
  928. return self.layerbranch.layer
  929. # Doesn't make sense to set or delete layer
  930. class LayerDependency(LayerIndexItemObj_LayerBranch):
  931. def define_data(self, id, layerbranch, dependency, required=True):
  932. self.id = id
  933. if isinstance(layerbranch, LayerBranch):
  934. self.layerbranch = layerbranch
  935. else:
  936. self.layerbranch_id = layerbranch
  937. if isinstance(dependency, LayerDependency):
  938. self.dependency = dependency
  939. else:
  940. self.dependency_id = dependency
  941. self.required = required
  942. @property
  943. def dependency_id(self):
  944. return self.__getattr__('dependency')
  945. @dependency_id.setter
  946. def dependency_id(self, value):
  947. self._setattr('dependency', value, prop=False)
  948. @dependency_id.deleter
  949. def dependency_id(self):
  950. self._delattr('dependency', prop=False)
  951. @property
  952. def dependency(self):
  953. try:
  954. return self.index.layerItems[self.dependency_id]
  955. except KeyError:
  956. raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id)
  957. except IndexError:
  958. raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id)
  959. @dependency.setter
  960. def dependency(self, value):
  961. if not isinstance(value, LayerDependency):
  962. raise TypeError('value (%s) is not a dependency' % type(value))
  963. if self.index != value.index:
  964. raise AttributeError('Object and value do not share the same index and thus key set.')
  965. self.dependency_id = value.id
  966. @dependency.deleter
  967. def dependency(self):
  968. self._delattr('dependency', prop=False)
  969. @property
  970. def dependency_layerBranch(self):
  971. layerid = self.dependency_id
  972. branchid = self.layerbranch.branch_id
  973. try:
  974. return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)]
  975. except IndexError:
  976. # layerBranches_layerId_branchId -- but not layerId:branchId
  977. raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid))
  978. except KeyError:
  979. raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid))
  980. # dependency_layerBranch doesn't make sense to set or del
  981. class Recipe(LayerIndexItemObj_LayerBranch):
  982. def define_data(self, id,
  983. filename, filepath, pn, pv, layerbranch,
  984. summary="", description="", section="", license="",
  985. homepage="", bugtracker="", provides="", bbclassextend="",
  986. inherits="", blacklisted="", updated=None):
  987. self.id = id
  988. self.filename = filename
  989. self.filepath = filepath
  990. self.pn = pn
  991. self.pv = pv
  992. self.summary = summary
  993. self.description = description
  994. self.section = section
  995. self.license = license
  996. self.homepage = homepage
  997. self.bugtracker = bugtracker
  998. self.provides = provides
  999. self.bbclassextend = bbclassextend
  1000. self.inherits = inherits
  1001. self.updated = updated or datetime.datetime.today().isoformat()
  1002. self.blacklisted = blacklisted
  1003. if isinstance(layerbranch, LayerBranch):
  1004. self.layerbranch = layerbranch
  1005. else:
  1006. self.layerbranch_id = layerbranch
  1007. @property
  1008. def fullpath(self):
  1009. return os.path.join(self.filepath, self.filename)
  1010. # Set would need to understand how to split it
  1011. # del would we del both parts?
  1012. @property
  1013. def inherits(self):
  1014. if 'inherits' not in self._data:
  1015. # Older indexes may not have this, so emulate it
  1016. if '-image-' in self.pn:
  1017. return 'image'
  1018. return self.__getattr__('inherits')
  1019. @inherits.setter
  1020. def inherits(self, value):
  1021. return self._setattr('inherits', value, prop=False)
  1022. @inherits.deleter
  1023. def inherits(self):
  1024. return self._delattr('inherits', prop=False)
  1025. class Machine(LayerIndexItemObj_LayerBranch):
  1026. def define_data(self, id,
  1027. name, description, layerbranch,
  1028. updated=None):
  1029. self.id = id
  1030. self.name = name
  1031. self.description = description
  1032. if isinstance(layerbranch, LayerBranch):
  1033. self.layerbranch = layerbranch
  1034. else:
  1035. self.layerbranch_id = layerbranch
  1036. self.updated = updated or datetime.datetime.today().isoformat()
  1037. class Distro(LayerIndexItemObj_LayerBranch):
  1038. def define_data(self, id,
  1039. name, description, layerbranch,
  1040. updated=None):
  1041. self.id = id
  1042. self.name = name
  1043. self.description = description
  1044. if isinstance(layerbranch, LayerBranch):
  1045. self.layerbranch = layerbranch
  1046. else:
  1047. self.layerbranch_id = layerbranch
  1048. self.updated = updated or datetime.datetime.today().isoformat()
  1049. # When performing certain actions, we may need to sort the data.
  1050. # This will allow us to keep it consistent from run to run.
  1051. def sort_entry(item):
  1052. newitem = item
  1053. try:
  1054. if type(newitem) == type(dict()):
  1055. newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0]))
  1056. for index in newitem:
  1057. newitem[index] = sort_entry(newitem[index])
  1058. elif type(newitem) == type(list()):
  1059. newitem.sort(key=lambda obj: obj['id'])
  1060. for index, _ in enumerate(newitem):
  1061. newitem[index] = sort_entry(newitem[index])
  1062. except:
  1063. logger.error('Sort failed for item %s' % type(item))
  1064. pass
  1065. return newitem