oe-pkgdata-util 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525
  1. #!/usr/bin/env python
  2. # OpenEmbedded pkgdata utility
  3. #
  4. # Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
  5. #
  6. # Copyright 2012-2015 Intel Corporation
  7. #
  8. # This program is free software; you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License version 2 as
  10. # published by the Free Software Foundation.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License along
  18. # with this program; if not, write to the Free Software Foundation, Inc.,
  19. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20. #
  21. import sys
  22. import os
  23. import os.path
  24. import fnmatch
  25. import re
  26. import argparse
  27. import logging
  28. from collections import defaultdict, OrderedDict
  29. scripts_path = os.path.dirname(os.path.realpath(__file__))
  30. lib_path = scripts_path + '/lib'
  31. sys.path = sys.path + [lib_path]
  32. import scriptutils
  33. import argparse_oe
  34. logger = scriptutils.logger_create('pkgdatautil')
  35. def tinfoil_init():
  36. import bb.tinfoil
  37. import logging
  38. tinfoil = bb.tinfoil.Tinfoil()
  39. tinfoil.prepare(True)
  40. tinfoil.logger.setLevel(logging.WARNING)
  41. return tinfoil
  42. def glob(args):
  43. # Handle both multiple arguments and multiple values within an arg (old syntax)
  44. globs = []
  45. for globitem in args.glob:
  46. globs.extend(globitem.split())
  47. if not os.path.exists(args.pkglistfile):
  48. logger.error('Unable to find package list file %s' % args.pkglistfile)
  49. sys.exit(1)
  50. skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-"
  51. if args.exclude:
  52. skipval += "|" + args.exclude
  53. skipregex = re.compile(skipval)
  54. skippedpkgs = set()
  55. mappedpkgs = set()
  56. with open(args.pkglistfile, 'r') as f:
  57. for line in f:
  58. fields = line.rstrip().split()
  59. if not fields:
  60. continue
  61. pkg = fields[0]
  62. # We don't care about other args (used to need the package architecture but the
  63. # new pkgdata structure avoids the need for that)
  64. # Skip packages for which there is no point applying globs
  65. if skipregex.search(pkg):
  66. logger.debug("%s -> !!" % pkg)
  67. skippedpkgs.add(pkg)
  68. continue
  69. # Skip packages that already match the globs, so if e.g. a dev package
  70. # is already installed and thus in the list, we don't process it any further
  71. # Most of these will be caught by skipregex already, but just in case...
  72. already = False
  73. for g in globs:
  74. if fnmatch.fnmatchcase(pkg, g):
  75. already = True
  76. break
  77. if already:
  78. skippedpkgs.add(pkg)
  79. logger.debug("%s -> !" % pkg)
  80. continue
  81. # Define some functions
  82. def revpkgdata(pkgn):
  83. return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn)
  84. def fwdpkgdata(pkgn):
  85. return os.path.join(args.pkgdata_dir, "runtime", pkgn)
  86. def readpn(pkgdata_file):
  87. pn = ""
  88. with open(pkgdata_file, 'r') as f:
  89. for line in f:
  90. if line.startswith("PN:"):
  91. pn = line.split(': ')[1].rstrip()
  92. return pn
  93. def readrenamed(pkgdata_file):
  94. renamed = ""
  95. pn = os.path.basename(pkgdata_file)
  96. with open(pkgdata_file, 'r') as f:
  97. for line in f:
  98. if line.startswith("PKG_%s:" % pn):
  99. renamed = line.split(': ')[1].rstrip()
  100. return renamed
  101. # Main processing loop
  102. for g in globs:
  103. mappedpkg = ""
  104. # First just try substitution (i.e. packagename -> packagename-dev)
  105. newpkg = g.replace("*", pkg)
  106. revlink = revpkgdata(newpkg)
  107. if os.path.exists(revlink):
  108. mappedpkg = os.path.basename(os.readlink(revlink))
  109. fwdfile = fwdpkgdata(mappedpkg)
  110. if os.path.exists(fwdfile):
  111. mappedpkg = readrenamed(fwdfile)
  112. if not os.path.exists(fwdfile + ".packaged"):
  113. mappedpkg = ""
  114. else:
  115. revlink = revpkgdata(pkg)
  116. if os.path.exists(revlink):
  117. # Check if we can map after undoing the package renaming (by resolving the symlink)
  118. origpkg = os.path.basename(os.readlink(revlink))
  119. newpkg = g.replace("*", origpkg)
  120. fwdfile = fwdpkgdata(newpkg)
  121. if os.path.exists(fwdfile):
  122. mappedpkg = readrenamed(fwdfile)
  123. else:
  124. # That didn't work, so now get the PN, substitute that, then map in the other direction
  125. pn = readpn(revlink)
  126. newpkg = g.replace("*", pn)
  127. fwdfile = fwdpkgdata(newpkg)
  128. if os.path.exists(fwdfile):
  129. mappedpkg = readrenamed(fwdfile)
  130. if not os.path.exists(fwdfile + ".packaged"):
  131. mappedpkg = ""
  132. else:
  133. # Package doesn't even exist...
  134. logger.debug("%s is not a valid package!" % (pkg))
  135. break
  136. if mappedpkg:
  137. logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg))
  138. mappedpkgs.add(mappedpkg)
  139. else:
  140. logger.debug("%s (%s) -> ?" % (pkg, g))
  141. logger.debug("------")
  142. print("\n".join(mappedpkgs - skippedpkgs))
  143. def read_value(args):
  144. # Handle both multiple arguments and multiple values within an arg (old syntax)
  145. packages = []
  146. if args.file:
  147. with open(args.file, 'r') as f:
  148. for line in f:
  149. splitline = line.split()
  150. if splitline:
  151. packages.append(splitline[0])
  152. else:
  153. for pkgitem in args.pkg:
  154. packages.extend(pkgitem.split())
  155. if not packages:
  156. logger.error("No packages specified")
  157. sys.exit(1)
  158. def readvar(pkgdata_file, valuename):
  159. val = ""
  160. with open(pkgdata_file, 'r') as f:
  161. for line in f:
  162. if line.startswith(valuename + ":"):
  163. val = line.split(': ', 1)[1].rstrip()
  164. return val
  165. logger.debug("read-value('%s', '%s' '%s'" % (args.pkgdata_dir, args.valuename, packages))
  166. for package in packages:
  167. pkg_split = package.split('_')
  168. pkg_name = pkg_split[0]
  169. logger.debug("package: '%s'" % pkg_name)
  170. revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name)
  171. logger.debug(revlink)
  172. if os.path.exists(revlink):
  173. mappedpkg = os.path.basename(os.readlink(revlink))
  174. qvar = args.valuename
  175. if qvar == "PKGSIZE":
  176. # append packagename
  177. qvar = "%s_%s" % (args.valuename, mappedpkg)
  178. # PKGSIZE is now in bytes, but we we want it in KB
  179. pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024
  180. value = "%d" % pkgsize
  181. else:
  182. value = readvar(revlink, qvar)
  183. if args.prefix_name:
  184. print('%s %s' % (pkg_name, value))
  185. else:
  186. print(value)
  187. def lookup_pkglist(pkgs, pkgdata_dir, reverse):
  188. if reverse:
  189. mappings = OrderedDict()
  190. for pkg in pkgs:
  191. revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg)
  192. logger.debug(revlink)
  193. if os.path.exists(revlink):
  194. mappings[pkg] = os.path.basename(os.readlink(revlink))
  195. else:
  196. mappings = defaultdict(list)
  197. for pkg in pkgs:
  198. pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg)
  199. if os.path.exists(pkgfile):
  200. with open(pkgfile, 'r') as f:
  201. for line in f:
  202. fields = line.rstrip().split(': ')
  203. if fields[0] == 'PKG_%s' % pkg:
  204. mappings[pkg].append(fields[1])
  205. break
  206. return mappings
  207. def lookup_pkg(args):
  208. # Handle both multiple arguments and multiple values within an arg (old syntax)
  209. pkgs = []
  210. for pkgitem in args.pkg:
  211. pkgs.extend(pkgitem.split())
  212. mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse)
  213. if len(mappings) < len(pkgs):
  214. missing = list(set(pkgs) - set(mappings.keys()))
  215. logger.error("The following packages could not be found: %s" % ', '.join(missing))
  216. sys.exit(1)
  217. if args.reverse:
  218. items = mappings.values()
  219. else:
  220. items = []
  221. for pkg in pkgs:
  222. items.extend(mappings.get(pkg, []))
  223. print('\n'.join(items))
  224. def lookup_recipe(args):
  225. # Handle both multiple arguments and multiple values within an arg (old syntax)
  226. pkgs = []
  227. for pkgitem in args.pkg:
  228. pkgs.extend(pkgitem.split())
  229. mappings = defaultdict(list)
  230. for pkg in pkgs:
  231. pkgfile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg)
  232. if os.path.exists(pkgfile):
  233. with open(pkgfile, 'r') as f:
  234. for line in f:
  235. fields = line.rstrip().split(': ')
  236. if fields[0] == 'PN':
  237. mappings[pkg].append(fields[1])
  238. break
  239. if len(mappings) < len(pkgs):
  240. missing = list(set(pkgs) - set(mappings.keys()))
  241. logger.error("The following packages could not be found: %s" % ', '.join(missing))
  242. sys.exit(1)
  243. items = []
  244. for pkg in pkgs:
  245. items.extend(mappings.get(pkg, []))
  246. print('\n'.join(items))
  247. def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged):
  248. recipedatafile = os.path.join(pkgdata_dir, recipe)
  249. if not os.path.exists(recipedatafile):
  250. logger.error("Unable to find packaged recipe with name %s" % recipe)
  251. sys.exit(1)
  252. packages = []
  253. with open(recipedatafile, 'r') as f:
  254. for line in f:
  255. fields = line.rstrip().split(': ')
  256. if fields[0] == 'PACKAGES':
  257. packages = fields[1].split()
  258. break
  259. if not unpackaged:
  260. pkglist = []
  261. for pkg in packages:
  262. if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
  263. pkglist.append(pkg)
  264. return pkglist
  265. else:
  266. return packages
  267. def list_pkgs(args):
  268. found = False
  269. def matchpkg(pkg):
  270. if args.pkgspec:
  271. matched = False
  272. for pkgspec in args.pkgspec:
  273. if fnmatch.fnmatchcase(pkg, pkgspec):
  274. matched = True
  275. break
  276. if not matched:
  277. return False
  278. if not args.unpackaged:
  279. if args.runtime:
  280. revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
  281. if os.path.exists(revlink):
  282. # We're unlikely to get here if the package was not packaged, but just in case
  283. # we add the symlinks for unpackaged files in the future
  284. mappedpkg = os.path.basename(os.readlink(revlink))
  285. if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)):
  286. return False
  287. else:
  288. return False
  289. else:
  290. if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
  291. return False
  292. return True
  293. if args.recipe:
  294. packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
  295. if args.runtime:
  296. pkglist = []
  297. runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False)
  298. for rtpkgs in runtime_pkgs.values():
  299. pkglist.extend(rtpkgs)
  300. else:
  301. pkglist = packages
  302. for pkg in pkglist:
  303. if matchpkg(pkg):
  304. found = True
  305. print("%s" % pkg)
  306. else:
  307. if args.runtime:
  308. searchdir = 'runtime-reverse'
  309. else:
  310. searchdir = 'runtime'
  311. for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)):
  312. for fn in files:
  313. if fn.endswith('.packaged'):
  314. continue
  315. if matchpkg(fn):
  316. found = True
  317. print("%s" % fn)
  318. if not found:
  319. if args.pkgspec:
  320. logger.error("Unable to find any package matching %s" % args.pkgspec)
  321. else:
  322. logger.error("No packages found")
  323. sys.exit(1)
  324. def list_pkg_files(args):
  325. import json
  326. if args.recipe:
  327. if args.pkg:
  328. logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified")
  329. sys.exit(1)
  330. recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
  331. if args.runtime:
  332. pkglist = []
  333. runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False)
  334. for rtpkgs in runtime_pkgs.values():
  335. pkglist.extend(rtpkgs)
  336. else:
  337. pkglist = recipepkglist
  338. else:
  339. if not args.pkg:
  340. logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified")
  341. sys.exit(1)
  342. pkglist = args.pkg
  343. for pkg in sorted(pkglist):
  344. print("%s:" % pkg)
  345. if args.runtime:
  346. pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
  347. if not os.path.exists(pkgdatafile):
  348. if args.recipe:
  349. # This package was empty and thus never packaged, ignore
  350. continue
  351. logger.error("Unable to find any built runtime package named %s" % pkg)
  352. sys.exit(1)
  353. else:
  354. pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg)
  355. if not os.path.exists(pkgdatafile):
  356. logger.error("Unable to find any built recipe-space package named %s" % pkg)
  357. sys.exit(1)
  358. with open(pkgdatafile, 'r') as f:
  359. found = False
  360. for line in f:
  361. if line.startswith('FILES_INFO:'):
  362. found = True
  363. val = line.split(':', 1)[1].strip()
  364. dictval = json.loads(val)
  365. for fullpth in sorted(dictval):
  366. print("\t%s" % fullpth)
  367. break
  368. if not found:
  369. logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile)
  370. sys.exit(1)
  371. def find_path(args):
  372. import json
  373. found = False
  374. for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')):
  375. for fn in files:
  376. with open(os.path.join(root,fn)) as f:
  377. for line in f:
  378. if line.startswith('FILES_INFO:'):
  379. val = line.split(':', 1)[1].strip()
  380. dictval = json.loads(val)
  381. for fullpth in dictval.keys():
  382. if fnmatch.fnmatchcase(fullpth, args.targetpath):
  383. found = True
  384. print("%s: %s" % (fn, fullpth))
  385. break
  386. if not found:
  387. logger.error("Unable to find any package producing path %s" % args.targetpath)
  388. sys.exit(1)
  389. def main():
  390. parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package",
  391. epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
  392. parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
  393. parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)')
  394. subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
  395. parser_lookup_pkg = subparsers.add_parser('lookup-pkg',
  396. help='Translate between recipe-space package names and runtime package names',
  397. description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.')
  398. parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up')
  399. parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true')
  400. parser_lookup_pkg.set_defaults(func=lookup_pkg)
  401. parser_list_pkgs = subparsers.add_parser('list-pkgs',
  402. help='List packages',
  403. description='Lists packages that have been built')
  404. parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)')
  405. parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true')
  406. parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe')
  407. parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true')
  408. parser_list_pkgs.set_defaults(func=list_pkgs)
  409. parser_list_pkg_files = subparsers.add_parser('list-pkg-files',
  410. help='List files within a package',
  411. description='Lists files included in one or more packages')
  412. parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)')
  413. parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true')
  414. parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe')
  415. parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true')
  416. parser_list_pkg_files.set_defaults(func=list_pkg_files)
  417. parser_lookup_recipe = subparsers.add_parser('lookup-recipe',
  418. help='Find recipe producing one or more packages',
  419. description='Looks up the specified runtime package(s) to see which recipe they were produced by')
  420. parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up')
  421. parser_lookup_recipe.set_defaults(func=lookup_recipe)
  422. parser_find_path = subparsers.add_parser('find-path',
  423. help='Find package providing a target path',
  424. description='Finds the recipe-space package providing the specified target path')
  425. parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)')
  426. parser_find_path.set_defaults(func=find_path)
  427. parser_read_value = subparsers.add_parser('read-value',
  428. help='Read any pkgdata value for one or more packages',
  429. description='Reads the named value from the pkgdata files for the specified packages')
  430. parser_read_value.add_argument('valuename', help='Name of the value to look up')
  431. parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up')
  432. parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)')
  433. parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true')
  434. parser_read_value.set_defaults(func=read_value)
  435. parser_glob = subparsers.add_parser('glob',
  436. help='Expand package name glob expression',
  437. description='Expands one or more glob expressions over the packages listed in pkglistfile')
  438. parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)')
  439. parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev')
  440. parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation')
  441. parser_glob.set_defaults(func=glob)
  442. args = parser.parse_args()
  443. if args.debug:
  444. logger.setLevel(logging.DEBUG)
  445. if not args.pkgdata_dir:
  446. import scriptpath
  447. bitbakepath = scriptpath.add_bitbake_lib_path()
  448. if not bitbakepath:
  449. logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
  450. sys.exit(1)
  451. logger.debug('Found bitbake path: %s' % bitbakepath)
  452. tinfoil = tinfoil_init()
  453. args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
  454. logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir)
  455. if not args.pkgdata_dir:
  456. logger.error('Unable to determine pkgdata directory from PKGDATA_DIR')
  457. sys.exit(1)
  458. if not os.path.exists(args.pkgdata_dir):
  459. logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir)
  460. sys.exit(1)
  461. ret = args.func(args)
  462. return ret
  463. if __name__ == "__main__":
  464. main()