recipeutils.py 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114
  1. # Utility functions for reading and modifying recipes
  2. #
  3. # Some code borrowed from the OE layer index
  4. #
  5. # Copyright (C) 2013-2017 Intel Corporation
  6. #
  7. # SPDX-License-Identifier: GPL-2.0-only
  8. #
  9. import sys
  10. import os
  11. import os.path
  12. import tempfile
  13. import textwrap
  14. import difflib
  15. from . import utils
  16. import shutil
  17. import re
  18. import fnmatch
  19. import glob
  20. import bb.tinfoil
  21. from collections import OrderedDict, defaultdict
  22. from bb.utils import vercmp_string
  23. # Help us to find places to insert values
  24. recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()']
  25. # Variables that sometimes are a bit long but shouldn't be wrapped
  26. nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]']
  27. list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
  28. meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
  29. def simplify_history(history, d):
  30. """
  31. Eliminate any irrelevant events from a variable history
  32. """
  33. ret_history = []
  34. has_set = False
  35. # Go backwards through the history and remove any immediate operations
  36. # before the most recent set
  37. for event in reversed(history):
  38. if 'flag' in event or not 'file' in event:
  39. continue
  40. if event['op'] == 'set':
  41. if has_set:
  42. continue
  43. has_set = True
  44. elif event['op'] in ('append', 'prepend', 'postdot', 'predot'):
  45. # Reminder: "append" and "prepend" mean += and =+ respectively, NOT _append / _prepend
  46. if has_set:
  47. continue
  48. ret_history.insert(0, event)
  49. return ret_history
  50. def get_var_files(fn, varlist, d):
  51. """Find the file in which each of a list of variables is set.
  52. Note: requires variable history to be enabled when parsing.
  53. """
  54. varfiles = {}
  55. for v in varlist:
  56. files = []
  57. if '[' in v:
  58. varsplit = v.split('[')
  59. varflag = varsplit[1].split(']')[0]
  60. history = d.varhistory.variable(varsplit[0])
  61. for event in history:
  62. if 'file' in event and event.get('flag', '') == varflag:
  63. files.append(event['file'])
  64. else:
  65. history = d.varhistory.variable(v)
  66. for event in history:
  67. if 'file' in event and not 'flag' in event:
  68. files.append(event['file'])
  69. if files:
  70. actualfile = files[-1]
  71. else:
  72. actualfile = None
  73. varfiles[v] = actualfile
  74. return varfiles
  75. def split_var_value(value, assignment=True):
  76. """
  77. Split a space-separated variable's value into a list of items,
  78. taking into account that some of the items might be made up of
  79. expressions containing spaces that should not be split.
  80. Parameters:
  81. value:
  82. The string value to split
  83. assignment:
  84. True to assume that the value represents an assignment
  85. statement, False otherwise. If True, and an assignment
  86. statement is passed in the first item in
  87. the returned list will be the part of the assignment
  88. statement up to and including the opening quote character,
  89. and the last item will be the closing quote.
  90. """
  91. inexpr = 0
  92. lastchar = None
  93. out = []
  94. buf = ''
  95. for char in value:
  96. if char == '{':
  97. if lastchar == '$':
  98. inexpr += 1
  99. elif char == '}':
  100. inexpr -= 1
  101. elif assignment and char in '"\'' and inexpr == 0:
  102. if buf:
  103. out.append(buf)
  104. out.append(char)
  105. char = ''
  106. buf = ''
  107. elif char.isspace() and inexpr == 0:
  108. char = ''
  109. if buf:
  110. out.append(buf)
  111. buf = ''
  112. buf += char
  113. lastchar = char
  114. if buf:
  115. out.append(buf)
  116. # Join together assignment statement and opening quote
  117. outlist = out
  118. if assignment:
  119. assigfound = False
  120. for idx, item in enumerate(out):
  121. if '=' in item:
  122. assigfound = True
  123. if assigfound:
  124. if '"' in item or "'" in item:
  125. outlist = [' '.join(out[:idx+1])]
  126. outlist.extend(out[idx+1:])
  127. break
  128. return outlist
  129. def patch_recipe_lines(fromlines, values, trailing_newline=True):
  130. """Update or insert variable values into lines from a recipe.
  131. Note that some manual inspection/intervention may be required
  132. since this cannot handle all situations.
  133. """
  134. import bb.utils
  135. if trailing_newline:
  136. newline = '\n'
  137. else:
  138. newline = ''
  139. nowrap_vars_res = []
  140. for item in nowrap_vars:
  141. nowrap_vars_res.append(re.compile('^%s$' % item))
  142. recipe_progression_res = []
  143. recipe_progression_restrs = []
  144. for item in recipe_progression:
  145. if item.endswith('()'):
  146. key = item[:-2]
  147. else:
  148. key = item
  149. restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key
  150. if item.endswith('()'):
  151. recipe_progression_restrs.append(restr + '()')
  152. else:
  153. recipe_progression_restrs.append(restr)
  154. recipe_progression_res.append(re.compile('^%s$' % restr))
  155. def get_recipe_pos(variable):
  156. for i, p in enumerate(recipe_progression_res):
  157. if p.match(variable):
  158. return i
  159. return -1
  160. remainingnames = {}
  161. for k in values.keys():
  162. remainingnames[k] = get_recipe_pos(k)
  163. remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
  164. modifying = False
  165. def outputvalue(name, lines, rewindcomments=False):
  166. if values[name] is None:
  167. return
  168. if isinstance(values[name], tuple):
  169. op, value = values[name]
  170. if op == '+=' and value.strip() == '':
  171. return
  172. else:
  173. value = values[name]
  174. op = '='
  175. rawtext = '%s %s "%s"%s' % (name, op, value, newline)
  176. addlines = []
  177. nowrap = False
  178. for nowrap_re in nowrap_vars_res:
  179. if nowrap_re.match(name):
  180. nowrap = True
  181. break
  182. if nowrap:
  183. addlines.append(rawtext)
  184. elif name in list_vars:
  185. splitvalue = split_var_value(value, assignment=False)
  186. if len(splitvalue) > 1:
  187. linesplit = ' \\\n' + (' ' * (len(name) + 4))
  188. addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline))
  189. else:
  190. addlines.append(rawtext)
  191. else:
  192. wrapped = textwrap.wrap(rawtext)
  193. for wrapline in wrapped[:-1]:
  194. addlines.append('%s \\%s' % (wrapline, newline))
  195. addlines.append('%s%s' % (wrapped[-1], newline))
  196. # Split on newlines - this isn't strictly necessary if you are only
  197. # going to write the output to disk, but if you want to compare it
  198. # (as patch_recipe_file() will do if patch=True) then it's important.
  199. addlines = [line for l in addlines for line in l.splitlines(True)]
  200. if rewindcomments:
  201. # Ensure we insert the lines before any leading comments
  202. # (that we'd want to ensure remain leading the next value)
  203. for i, ln in reversed(list(enumerate(lines))):
  204. if not ln.startswith('#'):
  205. lines[i+1:i+1] = addlines
  206. break
  207. else:
  208. lines.extend(addlines)
  209. else:
  210. lines.extend(addlines)
  211. existingnames = []
  212. def patch_recipe_varfunc(varname, origvalue, op, newlines):
  213. if modifying:
  214. # Insert anything that should come before this variable
  215. pos = get_recipe_pos(varname)
  216. for k in list(remainingnames):
  217. if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames:
  218. outputvalue(k, newlines, rewindcomments=True)
  219. del remainingnames[k]
  220. # Now change this variable, if it needs to be changed
  221. if varname in existingnames and op in ['+=', '=', '=+']:
  222. if varname in remainingnames:
  223. outputvalue(varname, newlines)
  224. del remainingnames[varname]
  225. return None, None, 0, True
  226. else:
  227. if varname in values:
  228. existingnames.append(varname)
  229. return origvalue, None, 0, True
  230. # First run - establish which values we want to set are already in the file
  231. varlist = [re.escape(item) for item in values.keys()]
  232. bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc)
  233. # Second run - actually set everything
  234. modifying = True
  235. varlist.extend(recipe_progression_restrs)
  236. changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True)
  237. if remainingnames:
  238. if tolines and tolines[-1].strip() != '':
  239. tolines.append('\n')
  240. for k in remainingnames.keys():
  241. outputvalue(k, tolines)
  242. return changed, tolines
  243. def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
  244. """Update or insert variable values into a recipe file (assuming you
  245. have already identified the exact file you want to update.)
  246. Note that some manual inspection/intervention may be required
  247. since this cannot handle all situations.
  248. """
  249. with open(fn, 'r') as f:
  250. fromlines = f.readlines()
  251. _, tolines = patch_recipe_lines(fromlines, values)
  252. if redirect_output:
  253. with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
  254. f.writelines(tolines)
  255. return None
  256. elif patch:
  257. relfn = os.path.relpath(fn, relpath)
  258. diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
  259. return diff
  260. else:
  261. with open(fn, 'w') as f:
  262. f.writelines(tolines)
  263. return None
  264. def localise_file_vars(fn, varfiles, varlist):
  265. """Given a list of variables and variable history (fetched with get_var_files())
  266. find where each variable should be set/changed. This handles for example where a
  267. recipe includes an inc file where variables might be changed - in most cases
  268. we want to update the inc file when changing the variable value rather than adding
  269. it to the recipe itself.
  270. """
  271. fndir = os.path.dirname(fn) + os.sep
  272. first_meta_file = None
  273. for v in meta_vars:
  274. f = varfiles.get(v, None)
  275. if f:
  276. actualdir = os.path.dirname(f) + os.sep
  277. if actualdir.startswith(fndir):
  278. first_meta_file = f
  279. break
  280. filevars = defaultdict(list)
  281. for v in varlist:
  282. f = varfiles[v]
  283. # Only return files that are in the same directory as the recipe or in some directory below there
  284. # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
  285. # in if we were going to set a value specific to this recipe)
  286. if f:
  287. actualfile = f
  288. else:
  289. # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
  290. if first_meta_file:
  291. actualfile = first_meta_file
  292. else:
  293. actualfile = fn
  294. actualdir = os.path.dirname(actualfile) + os.sep
  295. if not actualdir.startswith(fndir):
  296. actualfile = fn
  297. filevars[actualfile].append(v)
  298. return filevars
  299. def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
  300. """Modify a list of variable values in the specified recipe. Handles inc files if
  301. used by the recipe.
  302. """
  303. overrides = d.getVar('OVERRIDES').split(':')
  304. def override_applicable(hevent):
  305. op = hevent['op']
  306. if '[' in op:
  307. opoverrides = op.split('[')[1].split(']')[0].split('_')
  308. for opoverride in opoverrides:
  309. if not opoverride in overrides:
  310. return False
  311. return True
  312. varlist = varvalues.keys()
  313. fn = os.path.abspath(fn)
  314. varfiles = get_var_files(fn, varlist, d)
  315. locs = localise_file_vars(fn, varfiles, varlist)
  316. patches = []
  317. for f,v in locs.items():
  318. vals = {k: varvalues[k] for k in v}
  319. f = os.path.abspath(f)
  320. if f == fn:
  321. extravals = {}
  322. for var, value in vals.items():
  323. if var in list_vars:
  324. history = simplify_history(d.varhistory.variable(var), d)
  325. recipe_set = False
  326. for event in history:
  327. if os.path.abspath(event['file']) == fn:
  328. if event['op'] == 'set':
  329. recipe_set = True
  330. if not recipe_set:
  331. for event in history:
  332. if event['op'].startswith('_remove'):
  333. continue
  334. if not override_applicable(event):
  335. continue
  336. newvalue = value.replace(event['detail'], '')
  337. if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith('_'):
  338. op = event['op'].replace('[', '_').replace(']', '')
  339. extravals[var + op] = None
  340. value = newvalue
  341. vals[var] = ('+=', value)
  342. vals.update(extravals)
  343. patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
  344. if patch:
  345. patches.append(patchdata)
  346. if patch:
  347. return patches
  348. else:
  349. return None
  350. def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False):
  351. """Copy (local) recipe files, including both files included via include/require,
  352. and files referred to in the SRC_URI variable."""
  353. import bb.fetch2
  354. import oe.path
  355. # FIXME need a warning if the unexpanded SRC_URI value contains variable references
  356. uri_values = []
  357. localpaths = []
  358. def fetch_urls(rdata):
  359. # Collect the local paths from SRC_URI
  360. srcuri = rdata.getVar('SRC_URI') or ""
  361. if srcuri not in uri_values:
  362. fetch = bb.fetch2.Fetch(srcuri.split(), rdata)
  363. if download:
  364. fetch.download()
  365. for pth in fetch.localpaths():
  366. if pth not in localpaths:
  367. localpaths.append(pth)
  368. uri_values.append(srcuri)
  369. fetch_urls(d)
  370. if all_variants:
  371. # Get files for other variants e.g. in the case of a SRC_URI_append
  372. localdata = bb.data.createCopy(d)
  373. variants = (localdata.getVar('BBCLASSEXTEND') or '').split()
  374. if variants:
  375. # Ensure we handle class-target if we're dealing with one of the variants
  376. variants.append('target')
  377. for variant in variants:
  378. localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant)
  379. fetch_urls(localdata)
  380. # Copy local files to target directory and gather any remote files
  381. bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep
  382. remotes = []
  383. copied = []
  384. # Need to do this in two steps since we want to check against the absolute path
  385. includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)]
  386. # We also check this below, but we don't want any items in this list being considered remotes
  387. includes = [path for path in includes if path.startswith(bb_dir)]
  388. for path in localpaths + includes:
  389. # Only import files that are under the meta directory
  390. if path.startswith(bb_dir):
  391. if not whole_dir:
  392. relpath = os.path.relpath(path, bb_dir)
  393. subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
  394. if not os.path.exists(subdir):
  395. os.makedirs(subdir)
  396. shutil.copy2(path, os.path.join(tgt_dir, relpath))
  397. copied.append(relpath)
  398. else:
  399. remotes.append(path)
  400. # Simply copy whole meta dir, if requested
  401. if whole_dir:
  402. shutil.copytree(bb_dir, tgt_dir)
  403. return copied, remotes
  404. def get_recipe_local_files(d, patches=False, archives=False):
  405. """Get a list of local files in SRC_URI within a recipe."""
  406. import oe.patch
  407. uris = (d.getVar('SRC_URI') or "").split()
  408. fetch = bb.fetch2.Fetch(uris, d)
  409. # FIXME this list should be factored out somewhere else (such as the
  410. # fetcher) though note that this only encompasses actual container formats
  411. # i.e. that can contain multiple files as opposed to those that only
  412. # contain a compressed stream (i.e. .tar.gz as opposed to just .gz)
  413. archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z']
  414. ret = {}
  415. for uri in uris:
  416. if fetch.ud[uri].type == 'file':
  417. if (not patches and
  418. oe.patch.patch_path(uri, fetch, '', expand=False)):
  419. continue
  420. # Skip files that are referenced by absolute path
  421. fname = fetch.ud[uri].basepath
  422. if os.path.isabs(fname):
  423. continue
  424. # Handle subdir=
  425. subdir = fetch.ud[uri].parm.get('subdir', '')
  426. if subdir:
  427. if os.path.isabs(subdir):
  428. continue
  429. fname = os.path.join(subdir, fname)
  430. localpath = fetch.localpath(uri)
  431. if not archives:
  432. # Ignore archives that will be unpacked
  433. if localpath.endswith(tuple(archive_exts)):
  434. unpack = fetch.ud[uri].parm.get('unpack', True)
  435. if unpack:
  436. continue
  437. if os.path.isdir(localpath):
  438. for root, dirs, files in os.walk(localpath):
  439. for fname in files:
  440. fileabspath = os.path.join(root,fname)
  441. srcdir = os.path.dirname(localpath)
  442. ret[os.path.relpath(fileabspath,srcdir)] = fileabspath
  443. else:
  444. ret[fname] = localpath
  445. return ret
  446. def get_recipe_patches(d):
  447. """Get a list of the patches included in SRC_URI within a recipe."""
  448. import oe.patch
  449. patches = oe.patch.src_patches(d, expand=False)
  450. patchfiles = []
  451. for patch in patches:
  452. _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
  453. patchfiles.append(local)
  454. return patchfiles
  455. def get_recipe_patched_files(d):
  456. """
  457. Get the list of patches for a recipe along with the files each patch modifies.
  458. Params:
  459. d: the datastore for the recipe
  460. Returns:
  461. a dict mapping patch file path to a list of tuples of changed files and
  462. change mode ('A' for add, 'D' for delete or 'M' for modify)
  463. """
  464. import oe.patch
  465. patches = oe.patch.src_patches(d, expand=False)
  466. patchedfiles = {}
  467. for patch in patches:
  468. _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
  469. striplevel = int(parm['striplevel'])
  470. patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', '')))
  471. return patchedfiles
  472. def validate_pn(pn):
  473. """Perform validation on a recipe name (PN) for a new recipe."""
  474. reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
  475. if not re.match('^[0-9a-z-.+]+$', pn):
  476. return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn
  477. elif pn in reserved_names:
  478. return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
  479. elif pn.startswith('pn-'):
  480. return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
  481. elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')):
  482. return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn
  483. return ''
  484. def get_bbfile_path(d, destdir, extrapathhint=None):
  485. """
  486. Determine the correct path for a recipe within a layer
  487. Parameters:
  488. d: Recipe-specific datastore
  489. destdir: destination directory. Can be the path to the base of the layer or a
  490. partial path somewhere within the layer.
  491. extrapathhint: a path relative to the base of the layer to try
  492. """
  493. import bb.cookerdata
  494. destdir = os.path.abspath(destdir)
  495. destlayerdir = find_layerdir(destdir)
  496. # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
  497. confdata = d.createCopy()
  498. confdata.setVar('BBFILES', '')
  499. confdata.setVar('LAYERDIR', destlayerdir)
  500. destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
  501. confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
  502. pn = d.getVar('PN')
  503. bbfilespecs = (confdata.getVar('BBFILES') or '').split()
  504. if destdir == destlayerdir:
  505. for bbfilespec in bbfilespecs:
  506. if not bbfilespec.endswith('.bbappend'):
  507. for match in glob.glob(bbfilespec):
  508. splitext = os.path.splitext(os.path.basename(match))
  509. if splitext[1] == '.bb':
  510. mpn = splitext[0].split('_')[0]
  511. if mpn == pn:
  512. return os.path.dirname(match)
  513. # Try to make up a path that matches BBFILES
  514. # this is a little crude, but better than nothing
  515. bpn = d.getVar('BPN')
  516. recipefn = os.path.basename(d.getVar('FILE'))
  517. pathoptions = [destdir]
  518. if extrapathhint:
  519. pathoptions.append(os.path.join(destdir, extrapathhint))
  520. if destdir == destlayerdir:
  521. pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
  522. pathoptions.append(os.path.join(destdir, 'recipes', bpn))
  523. pathoptions.append(os.path.join(destdir, bpn))
  524. elif not destdir.endswith(('/' + pn, '/' + bpn)):
  525. pathoptions.append(os.path.join(destdir, bpn))
  526. closepath = ''
  527. for pathoption in pathoptions:
  528. bbfilepath = os.path.join(pathoption, 'test.bb')
  529. for bbfilespec in bbfilespecs:
  530. if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
  531. return pathoption
  532. return None
  533. def get_bbappend_path(d, destlayerdir, wildcardver=False):
  534. """Determine how a bbappend for a recipe should be named and located within another layer"""
  535. import bb.cookerdata
  536. destlayerdir = os.path.abspath(destlayerdir)
  537. recipefile = d.getVar('FILE')
  538. recipefn = os.path.splitext(os.path.basename(recipefile))[0]
  539. if wildcardver and '_' in recipefn:
  540. recipefn = recipefn.split('_', 1)[0] + '_%'
  541. appendfn = recipefn + '.bbappend'
  542. # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
  543. confdata = d.createCopy()
  544. confdata.setVar('BBFILES', '')
  545. confdata.setVar('LAYERDIR', destlayerdir)
  546. destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
  547. confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
  548. origlayerdir = find_layerdir(recipefile)
  549. if not origlayerdir:
  550. return (None, False)
  551. # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
  552. appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
  553. closepath = ''
  554. pathok = True
  555. for bbfilespec in confdata.getVar('BBFILES').split():
  556. if fnmatch.fnmatchcase(appendpath, bbfilespec):
  557. # Our append path works, we're done
  558. break
  559. elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
  560. # Try to find the longest matching path
  561. if len(bbfilespec) > len(closepath):
  562. closepath = bbfilespec
  563. else:
  564. # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
  565. if closepath:
  566. # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
  567. # Now we just need to substitute out any wildcards
  568. appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
  569. if 'recipes-*' in appendsubdir:
  570. # Try to copy this part from the original recipe path
  571. res = re.search('/recipes-[^/]+/', recipefile)
  572. if res:
  573. appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
  574. # This is crude, but we have to do something
  575. appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
  576. appendsubdir = appendsubdir.replace('?', 'a')
  577. appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
  578. else:
  579. pathok = False
  580. return (appendpath, pathok)
  581. def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
  582. """
  583. Writes a bbappend file for a recipe
  584. Parameters:
  585. rd: data dictionary for the recipe
  586. destlayerdir: base directory of the layer to place the bbappend in
  587. (subdirectory path from there will be determined automatically)
  588. srcfiles: dict of source files to add to SRC_URI, where the value
  589. is the full path to the file to be added, and the value is the
  590. original filename as it would appear in SRC_URI or None if it
  591. isn't already present. You may pass None for this parameter if
  592. you simply want to specify your own content via the extralines
  593. parameter.
  594. install: dict mapping entries in srcfiles to a tuple of two elements:
  595. install path (*without* ${D} prefix) and permission value (as a
  596. string, e.g. '0644').
  597. wildcardver: True to use a % wildcard in the bbappend filename, or
  598. False to make the bbappend specific to the recipe version.
  599. machine:
  600. If specified, make the changes in the bbappend specific to this
  601. machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
  602. to be added to the bbappend.
  603. extralines:
  604. Extra lines to add to the bbappend. This may be a dict of name
  605. value pairs, or simply a list of the lines.
  606. removevalues:
  607. Variable values to remove - a dict of names/values.
  608. redirect_output:
  609. If specified, redirects writing the output file to the
  610. specified directory (for dry-run purposes)
  611. """
  612. if not removevalues:
  613. removevalues = {}
  614. # Determine how the bbappend should be named
  615. appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
  616. if not appendpath:
  617. bb.error('Unable to determine layer directory containing %s' % recipefile)
  618. return (None, None)
  619. if not pathok:
  620. bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
  621. appenddir = os.path.dirname(appendpath)
  622. if not redirect_output:
  623. bb.utils.mkdirhier(appenddir)
  624. # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
  625. layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
  626. if not os.path.abspath(destlayerdir) in layerdirs:
  627. bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
  628. bbappendlines = []
  629. if extralines:
  630. if isinstance(extralines, dict):
  631. for name, value in extralines.items():
  632. bbappendlines.append((name, '=', value))
  633. else:
  634. # Do our best to split it
  635. for line in extralines:
  636. if line[-1] == '\n':
  637. line = line[:-1]
  638. splitline = line.split(None, 2)
  639. if len(splitline) == 3:
  640. bbappendlines.append(tuple(splitline))
  641. else:
  642. raise Exception('Invalid extralines value passed')
  643. def popline(varname):
  644. for i in range(0, len(bbappendlines)):
  645. if bbappendlines[i][0] == varname:
  646. line = bbappendlines.pop(i)
  647. return line
  648. return None
  649. def appendline(varname, op, value):
  650. for i in range(0, len(bbappendlines)):
  651. item = bbappendlines[i]
  652. if item[0] == varname:
  653. bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
  654. break
  655. else:
  656. bbappendlines.append((varname, op, value))
  657. destsubdir = rd.getVar('PN')
  658. if srcfiles:
  659. bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:'))
  660. appendoverride = ''
  661. if machine:
  662. bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
  663. appendoverride = '_%s' % machine
  664. copyfiles = {}
  665. if srcfiles:
  666. instfunclines = []
  667. for newfile, origsrcfile in srcfiles.items():
  668. srcfile = origsrcfile
  669. srcurientry = None
  670. if not srcfile:
  671. srcfile = os.path.basename(newfile)
  672. srcurientry = 'file://%s' % srcfile
  673. # Double-check it's not there already
  674. # FIXME do we care if the entry is added by another bbappend that might go away?
  675. if not srcurientry in rd.getVar('SRC_URI').split():
  676. if machine:
  677. appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry)
  678. else:
  679. appendline('SRC_URI', '+=', srcurientry)
  680. copyfiles[newfile] = srcfile
  681. if install:
  682. institem = install.pop(newfile, None)
  683. if institem:
  684. (destpath, perms) = institem
  685. instdestpath = replace_dir_vars(destpath, rd)
  686. instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
  687. if not instdirline in instfunclines:
  688. instfunclines.append(instdirline)
  689. instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
  690. if instfunclines:
  691. bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
  692. if redirect_output:
  693. bb.note('Writing append file %s (dry-run)' % appendpath)
  694. outfile = os.path.join(redirect_output, os.path.basename(appendpath))
  695. # Only take a copy if the file isn't already there (this function may be called
  696. # multiple times per operation when we're handling overrides)
  697. if os.path.exists(appendpath) and not os.path.exists(outfile):
  698. shutil.copy2(appendpath, outfile)
  699. else:
  700. bb.note('Writing append file %s' % appendpath)
  701. outfile = appendpath
  702. if os.path.exists(outfile):
  703. # Work around lack of nonlocal in python 2
  704. extvars = {'destsubdir': destsubdir}
  705. def appendfile_varfunc(varname, origvalue, op, newlines):
  706. if varname == 'FILESEXTRAPATHS_prepend':
  707. if origvalue.startswith('${THISDIR}/'):
  708. popline('FILESEXTRAPATHS_prepend')
  709. extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
  710. elif varname == 'PACKAGE_ARCH':
  711. if machine:
  712. popline('PACKAGE_ARCH')
  713. return (machine, None, 4, False)
  714. elif varname.startswith('do_install_append'):
  715. func = popline(varname)
  716. if func:
  717. instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
  718. for line in func[2]:
  719. if not line in instfunclines:
  720. instfunclines.append(line)
  721. return (instfunclines, None, 4, False)
  722. else:
  723. splitval = split_var_value(origvalue, assignment=False)
  724. changed = False
  725. removevar = varname
  726. if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]:
  727. removevar = 'SRC_URI'
  728. line = popline(varname)
  729. if line:
  730. if line[2] not in splitval:
  731. splitval.append(line[2])
  732. changed = True
  733. else:
  734. line = popline(varname)
  735. if line:
  736. splitval = [line[2]]
  737. changed = True
  738. if removevar in removevalues:
  739. remove = removevalues[removevar]
  740. if isinstance(remove, str):
  741. if remove in splitval:
  742. splitval.remove(remove)
  743. changed = True
  744. else:
  745. for removeitem in remove:
  746. if removeitem in splitval:
  747. splitval.remove(removeitem)
  748. changed = True
  749. if changed:
  750. newvalue = splitval
  751. if len(newvalue) == 1:
  752. # Ensure it's written out as one line
  753. if '_append' in varname:
  754. newvalue = ' ' + newvalue[0]
  755. else:
  756. newvalue = newvalue[0]
  757. if not newvalue and (op in ['+=', '.='] or '_append' in varname):
  758. # There's no point appending nothing
  759. newvalue = None
  760. if varname.endswith('()'):
  761. indent = 4
  762. else:
  763. indent = -1
  764. return (newvalue, None, indent, True)
  765. return (origvalue, None, 4, False)
  766. varnames = [item[0] for item in bbappendlines]
  767. if removevalues:
  768. varnames.extend(list(removevalues.keys()))
  769. with open(outfile, 'r') as f:
  770. (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
  771. destsubdir = extvars['destsubdir']
  772. else:
  773. updated = False
  774. newlines = []
  775. if bbappendlines:
  776. for line in bbappendlines:
  777. if line[0].endswith('()'):
  778. newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
  779. else:
  780. newlines.append('%s %s "%s"\n\n' % line)
  781. updated = True
  782. if updated:
  783. with open(outfile, 'w') as f:
  784. f.writelines(newlines)
  785. if copyfiles:
  786. if machine:
  787. destsubdir = os.path.join(destsubdir, machine)
  788. if redirect_output:
  789. outdir = redirect_output
  790. else:
  791. outdir = appenddir
  792. for newfile, srcfile in copyfiles.items():
  793. filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
  794. if os.path.abspath(newfile) != os.path.abspath(filedest):
  795. if newfile.startswith(tempfile.gettempdir()):
  796. newfiledisp = os.path.basename(newfile)
  797. else:
  798. newfiledisp = newfile
  799. if redirect_output:
  800. bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
  801. else:
  802. bb.note('Copying %s to %s' % (newfiledisp, filedest))
  803. bb.utils.mkdirhier(os.path.dirname(filedest))
  804. shutil.copyfile(newfile, filedest)
  805. return (appendpath, os.path.join(appenddir, destsubdir))
  806. def find_layerdir(fn):
  807. """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
  808. pth = os.path.abspath(fn)
  809. layerdir = ''
  810. while pth:
  811. if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
  812. layerdir = pth
  813. break
  814. pth = os.path.dirname(pth)
  815. if pth == '/':
  816. return None
  817. return layerdir
  818. def replace_dir_vars(path, d):
  819. """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
  820. dirvars = {}
  821. # Sort by length so we get the variables we're interested in first
  822. for var in sorted(list(d.keys()), key=len):
  823. if var.endswith('dir') and var.lower() == var:
  824. value = d.getVar(var)
  825. if value.startswith('/') and not '\n' in value and value not in dirvars:
  826. dirvars[value] = var
  827. for dirpath in sorted(list(dirvars.keys()), reverse=True):
  828. path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
  829. return path
  830. def get_recipe_pv_without_srcpv(pv, uri_type):
  831. """
  832. Get PV without SRCPV common in SCM's for now only
  833. support git.
  834. Returns tuple with pv, prefix and suffix.
  835. """
  836. pfx = ''
  837. sfx = ''
  838. if uri_type == 'git':
  839. git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>[^\+]*)((?P<sfx>\+(git)?r?(AUTOINC\+))(?P<rev>.*))?")
  840. m = git_regex.match(pv)
  841. if m:
  842. pv = m.group('ver')
  843. pfx = m.group('pfx')
  844. sfx = m.group('sfx')
  845. else:
  846. regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)")
  847. m = regex.match(pv)
  848. if m:
  849. pv = m.group('ver')
  850. pfx = m.group('pfx')
  851. return (pv, pfx, sfx)
  852. def get_recipe_upstream_version(rd):
  853. """
  854. Get upstream version of recipe using bb.fetch2 methods with support for
  855. http, https, ftp and git.
  856. bb.fetch2 exceptions can be raised,
  857. FetchError when don't have network access or upstream site don't response.
  858. NoMethodError when uri latest_versionstring method isn't implemented.
  859. Returns a dictonary with version, repository revision, current_version, type and datetime.
  860. Type can be A for Automatic, M for Manual and U for Unknown.
  861. """
  862. from bb.fetch2 import decodeurl
  863. from datetime import datetime
  864. ru = {}
  865. ru['current_version'] = rd.getVar('PV')
  866. ru['version'] = ''
  867. ru['type'] = 'U'
  868. ru['datetime'] = ''
  869. ru['revision'] = ''
  870. # XXX: If don't have SRC_URI means that don't have upstream sources so
  871. # returns the current recipe version, so that upstream version check
  872. # declares a match.
  873. src_uris = rd.getVar('SRC_URI')
  874. if not src_uris:
  875. ru['version'] = ru['current_version']
  876. ru['type'] = 'M'
  877. ru['datetime'] = datetime.now()
  878. return ru
  879. # XXX: we suppose that the first entry points to the upstream sources
  880. src_uri = src_uris.split()[0]
  881. uri_type, _, _, _, _, _ = decodeurl(src_uri)
  882. (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type)
  883. ru['current_version'] = pv
  884. manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION")
  885. if manual_upstream_version:
  886. # manual tracking of upstream version.
  887. ru['version'] = manual_upstream_version
  888. ru['type'] = 'M'
  889. manual_upstream_date = rd.getVar("CHECK_DATE")
  890. if manual_upstream_date:
  891. date = datetime.strptime(manual_upstream_date, "%b %d, %Y")
  892. else:
  893. date = datetime.now()
  894. ru['datetime'] = date
  895. elif uri_type == "file":
  896. # files are always up-to-date
  897. ru['version'] = pv
  898. ru['type'] = 'A'
  899. ru['datetime'] = datetime.now()
  900. else:
  901. ud = bb.fetch2.FetchData(src_uri, rd)
  902. if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1":
  903. revision = ud.method.latest_revision(ud, rd, 'default')
  904. upversion = pv
  905. if revision != rd.getVar("SRCREV"):
  906. upversion = upversion + "-new-commits-available"
  907. else:
  908. pupver = ud.method.latest_versionstring(ud, rd)
  909. (upversion, revision) = pupver
  910. if upversion:
  911. ru['version'] = upversion
  912. ru['type'] = 'A'
  913. if revision:
  914. ru['revision'] = revision
  915. ru['datetime'] = datetime.now()
  916. return ru
  917. def _get_recipe_upgrade_status(data):
  918. uv = get_recipe_upstream_version(data)
  919. pn = data.getVar('PN')
  920. cur_ver = uv['current_version']
  921. upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
  922. if not uv['version']:
  923. status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
  924. else:
  925. cmp = vercmp_string(uv['current_version'], uv['version'])
  926. if cmp == -1:
  927. status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
  928. elif cmp == 0:
  929. status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
  930. else:
  931. status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
  932. next_ver = uv['version'] if uv['version'] else "N/A"
  933. revision = uv['revision'] if uv['revision'] else "N/A"
  934. maintainer = data.getVar('RECIPE_MAINTAINER')
  935. no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')
  936. return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)
  937. def get_recipe_upgrade_status(recipes=None):
  938. pkgs_list = []
  939. data_copy_list = []
  940. copy_vars = ('SRC_URI',
  941. 'PV',
  942. 'GITDIR',
  943. 'DL_DIR',
  944. 'PN',
  945. 'CACHE',
  946. 'PERSISTENT_DIR',
  947. 'BB_URI_HEADREVS',
  948. 'UPSTREAM_CHECK_COMMITS',
  949. 'UPSTREAM_CHECK_GITTAGREGEX',
  950. 'UPSTREAM_CHECK_REGEX',
  951. 'UPSTREAM_CHECK_URI',
  952. 'UPSTREAM_VERSION_UNKNOWN',
  953. 'RECIPE_MAINTAINER',
  954. 'RECIPE_NO_UPDATE_REASON',
  955. 'RECIPE_UPSTREAM_VERSION',
  956. 'RECIPE_UPSTREAM_DATE',
  957. 'CHECK_DATE',
  958. )
  959. with bb.tinfoil.Tinfoil() as tinfoil:
  960. tinfoil.prepare(config_only=False)
  961. if not recipes:
  962. recipes = tinfoil.all_recipe_files(variants=False)
  963. for fn in recipes:
  964. try:
  965. if fn.startswith("/"):
  966. data = tinfoil.parse_recipe_file(fn)
  967. else:
  968. data = tinfoil.parse_recipe(fn)
  969. except bb.providers.NoProvider:
  970. bb.note(" No provider for %s" % fn)
  971. continue
  972. unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
  973. if unreliable == "1":
  974. bb.note(" Skip package %s as upstream check unreliable" % pn)
  975. continue
  976. data_copy = bb.data.init()
  977. for var in copy_vars:
  978. data_copy.setVar(var, data.getVar(var))
  979. for k in data:
  980. if k.startswith('SRCREV'):
  981. data_copy.setVar(k, data.getVar(k))
  982. data_copy_list.append(data_copy)
  983. from concurrent.futures import ProcessPoolExecutor
  984. with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
  985. pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)
  986. return pkgs_list