gen-manual-lists.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. ## gen-manual-lists.py
  2. ##
  3. ## This script generates the following Buildroot manual appendices:
  4. ## - the package tables (one for the target, the other for host tools);
  5. ## - the deprecated items.
  6. ##
  7. ## Author(s):
  8. ## - Samuel Martin <s.martin49@gmail.com>
  9. ##
  10. ## Copyright (C) 2013 Samuel Martin
  11. ##
  12. ## This program is free software; you can redistribute it and/or modify
  13. ## it under the terms of the GNU General Public License as published by
  14. ## the Free Software Foundation; either version 2 of the License, or
  15. ## (at your option) any later version.
  16. ##
  17. ## This program is distributed in the hope that it will be useful,
  18. ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  19. ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20. ## GNU General Public License for more details.
  21. ##
  22. ## You should have received a copy of the GNU General Public License
  23. ## along with this program; if not, write to the Free Software
  24. ## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  25. ##
  26. from __future__ import print_function
  27. from __future__ import unicode_literals
  28. import os
  29. import re
  30. import sys
  31. import datetime
  32. from argparse import ArgumentParser
  33. try:
  34. import kconfiglib
  35. except ImportError:
  36. message = """
  37. Could not find the module 'kconfiglib' in the PYTHONPATH:
  38. """
  39. message += "\n".join([" {0}".format(path) for path in sys.path])
  40. message += """
  41. Make sure the Kconfiglib directory is in the PYTHONPATH, then relaunch the
  42. script.
  43. You can get kconfiglib from:
  44. https://github.com/ulfalizer/Kconfiglib
  45. """
  46. sys.stderr.write(message)
  47. raise
  48. def get_symbol_subset(root, filter_func):
  49. """ Return a generator of kconfig items.
  50. :param root_item: Root item of the generated subset of items
  51. :param filter_func: Filter function
  52. """
  53. if hasattr(root, "get_items"):
  54. get_items = root.get_items
  55. elif hasattr(root, "get_top_level_items"):
  56. get_items = root.get_top_level_items
  57. else:
  58. message = "The symbol does not contain any subset of symbols"
  59. raise Exception(message)
  60. for item in get_items():
  61. if item.is_symbol():
  62. if not filter_func(item):
  63. continue
  64. yield item
  65. elif item.is_menu() or item.is_choice():
  66. for i in get_symbol_subset(item, filter_func):
  67. yield i
  68. def get_symbol_parents(item, root=None, enable_choice=False):
  69. """ Return the list of the item's parents. The last item of the list is
  70. the closest parent, the first the furthest.
  71. :param item: Item from which the parent list is generated
  72. :param root: Root item stopping the search (not included in the
  73. parent list)
  74. :param enable_choice: Flag enabling choices to appear in the parent list
  75. """
  76. parent = item.get_parent()
  77. parents = []
  78. while parent and parent != root:
  79. if parent.is_menu():
  80. parents.append(parent.get_title())
  81. elif enable_choice and parent.is_choice():
  82. parents.append(parent.get_prompts()[0])
  83. parent = parent.get_parent()
  84. if isinstance(root, kconfiglib.Menu) or \
  85. (enable_choice and isinstance(root, kconfiglib.Choice)):
  86. parents.append("") # Dummy empty parent to get a leading arrow ->
  87. parents.reverse()
  88. return parents
  89. def format_asciidoc_table(root, get_label_func, filter_func=lambda x: True,
  90. format_func=lambda x: x,
  91. enable_choice=False, sorted=True,
  92. item_label=None):
  93. """ Return the asciidoc formatted table of the items and their location.
  94. :param root: Root item of the item subset
  95. :param get_label_func: Item's label getter function
  96. :param filter_func: Filter function to apply on the item subset
  97. :param format_func: Function to format a symbol and the table header
  98. :param enable_choice: Enable choices to appear as part of the item's
  99. location
  100. :param sorted: Flag to alphabetically sort the table
  101. """
  102. lines = []
  103. for item in get_symbol_subset(root, filter_func):
  104. lines.append(format_func(what="symbol", symbol=item, root=root,
  105. get_label_func=get_label_func,
  106. enable_choice=enable_choice))
  107. if sorted:
  108. lines.sort(key=lambda x: x.lower())
  109. table = ":halign: center\n\n"
  110. width, columns = format_func(what="layout")
  111. table = "[width=\"{0}\",cols=\"{1}\",options=\"header\"]\n".format(width, columns)
  112. table += "|===================================================\n"
  113. table += format_func(what="header", header=item_label, root=root)
  114. table += "\n" + "".join(lines) + "\n"
  115. table += "|===================================================\n"
  116. return table
  117. class Buildroot:
  118. """ Buildroot configuration object.
  119. """
  120. root_config = "Config.in"
  121. package_dirname = "package"
  122. package_prefixes = ["BR2_PACKAGE_", "BR2_PACKAGE_HOST_"]
  123. re_pkg_prefix = re.compile(r"^(" + "|".join(package_prefixes) + ").*")
  124. deprecated_symbol = "BR2_DEPRECATED"
  125. list_in = """\
  126. //
  127. // Automatically generated list for Buildroot manual.
  128. //
  129. {table}
  130. """
  131. list_info = {
  132. 'target-packages': {
  133. 'filename': "package-list",
  134. 'root_menu': "Target packages",
  135. 'filter': "_is_real_package",
  136. 'format': "_format_symbol_prompt_location",
  137. 'sorted': True,
  138. },
  139. 'host-packages': {
  140. 'filename': "host-package-list",
  141. 'root_menu': "Host utilities",
  142. 'filter': "_is_real_package",
  143. 'format': "_format_symbol_prompt",
  144. 'sorted': True,
  145. },
  146. 'virtual-packages': {
  147. 'filename': "virtual-package-list",
  148. 'root_menu': "Target packages",
  149. 'filter': "_is_virtual_package",
  150. 'format': "_format_symbol_virtual",
  151. 'sorted': True,
  152. },
  153. 'deprecated': {
  154. 'filename': "deprecated-list",
  155. 'root_menu': None,
  156. 'filter': "_is_deprecated_feature",
  157. 'format': "_format_symbol_prompt_location",
  158. 'sorted': False,
  159. },
  160. }
  161. def __init__(self):
  162. self.base_dir = os.environ.get("TOPDIR")
  163. self.output_dir = os.environ.get("O")
  164. self.package_dir = os.path.join(self.base_dir, self.package_dirname)
  165. self.config = kconfiglib.Config(os.path.join(self.base_dir,
  166. self.root_config),
  167. self.base_dir)
  168. self._deprecated = self.config.get_symbol(self.deprecated_symbol)
  169. self.gen_date = datetime.datetime.utcnow()
  170. self.br_version_full = os.environ.get("BR2_VERSION_FULL")
  171. if self.br_version_full and self.br_version_full.endswith("-git"):
  172. self.br_version_full = self.br_version_full[:-4]
  173. if not self.br_version_full:
  174. self.br_version_full = "undefined"
  175. def _get_package_symbols(self, package_name):
  176. """ Return a tuple containing the target and host package symbol.
  177. """
  178. symbols = re.sub("[-+.]", "_", package_name)
  179. symbols = symbols.upper()
  180. symbols = tuple([prefix + symbols for prefix in self.package_prefixes])
  181. return symbols
  182. def _is_deprecated(self, symbol):
  183. """ Return True if the symbol is marked as deprecated, otherwise False.
  184. """
  185. # This also catches BR2_DEPRECATED_SINCE_xxxx_xx
  186. return bool([ symbol for x in symbol.get_referenced_symbols()
  187. if x.get_name().startswith(self._deprecated.get_name()) ])
  188. def _is_package(self, symbol, type='real'):
  189. """ Return True if the symbol is a package or a host package, otherwise
  190. False.
  191. :param symbol: The symbol to check
  192. :param type: Limit to 'real' or 'virtual' types of packages,
  193. with 'real' being the default.
  194. Note: only 'real' is (implictly) handled for now
  195. """
  196. if not symbol.is_symbol():
  197. return False
  198. if type == 'real' and not symbol.get_prompts():
  199. return False
  200. if type == 'virtual' and symbol.get_prompts():
  201. return False
  202. if not self.re_pkg_prefix.match(symbol.get_name()):
  203. return False
  204. pkg_name = self._get_pkg_name(symbol)
  205. pattern = "^(HOST_)?" + pkg_name + "$"
  206. pattern = re.sub("_", ".", pattern)
  207. pattern = re.compile(pattern, re.IGNORECASE)
  208. # Here, we cannot just check for the location of the Config.in because
  209. # of the "virtual" package.
  210. #
  211. # So, to check that a symbol is a package (not a package option or
  212. # anything else), we check for the existence of the package *.mk file.
  213. #
  214. # By the way, to actually check for a package, we should grep all *.mk
  215. # files for the following regex:
  216. # "\$\(eval \$\((host-)?(generic|autotools|cmake)-package\)\)"
  217. #
  218. # Implementation details:
  219. #
  220. # * The package list is generated from the *.mk file existence, the
  221. # first time this function is called. Despite the memory consumption,
  222. # this list is stored because the execution time of this script is
  223. # noticeably shorter than rescanning the package sub-tree for each
  224. # symbol.
  225. if not hasattr(self, "_package_list"):
  226. pkg_list = []
  227. for _, _, files in os.walk(self.package_dir):
  228. for file_ in (f for f in files if f.endswith(".mk")):
  229. pkg_list.append(re.sub(r"(.*?)\.mk", r"\1", file_))
  230. setattr(self, "_package_list", pkg_list)
  231. for pkg in getattr(self, "_package_list"):
  232. if type == 'real':
  233. if pattern.match(pkg) and not self._exists_virt_symbol(pkg):
  234. return True
  235. if type == 'virtual':
  236. if pattern.match('has_' + pkg):
  237. return True
  238. return False
  239. def _is_real_package(self, symbol):
  240. return self._is_package(symbol, 'real')
  241. def _is_virtual_package(self, symbol):
  242. return self._is_package(symbol, 'virtual')
  243. def _is_deprecated_feature(self, symbol):
  244. return symbol.get_prompts() and self._is_deprecated(symbol)
  245. def _exists_virt_symbol(self, pkg_name):
  246. """ Return True if a symbol exists that defines the package as
  247. a virtual package, False otherwise
  248. :param pkg_name: The name of the package, for which to check if
  249. a symbol exists defining it as a virtual package
  250. """
  251. virt_pattern = "BR2_PACKAGE_HAS_" + pkg_name + "$"
  252. virt_pattern = re.sub("_", ".", virt_pattern)
  253. virt_pattern = re.compile(virt_pattern, re.IGNORECASE)
  254. for sym in self.config:
  255. if virt_pattern.match(sym.get_name()):
  256. return True
  257. return False
  258. def _get_pkg_name(self, symbol):
  259. """ Return the package name of the specified symbol.
  260. :param symbol: The symbol to get the package name of
  261. """
  262. return re.sub("BR2_PACKAGE_(HOST_)?(.*)", r"\2", symbol.get_name())
  263. def _get_symbol_label(self, symbol, mark_deprecated=True):
  264. """ Return the label (a.k.a. prompt text) of the symbol.
  265. :param symbol: The symbol
  266. :param mark_deprecated: Append a 'deprecated' to the label
  267. """
  268. label = symbol.get_prompts()[0]
  269. if self._is_deprecated(symbol) and mark_deprecated:
  270. label += " *(deprecated)*"
  271. return label
  272. def _format_symbol_prompt(self, what=None, symbol=None, root=None,
  273. enable_choice=False, header=None,
  274. get_label_func=lambda x: x):
  275. if what == "layout":
  276. return ( "30%", "^1" )
  277. if what == "header":
  278. return "| {0:<40}\n".format(header)
  279. if what == "symbol":
  280. return "| {0:<40}\n".format(get_label_func(symbol))
  281. message = "Invalid argument 'what': '%s'\n" % str(what)
  282. message += "Allowed values are: 'layout', 'header' and 'symbol'"
  283. raise Exception(message)
  284. def _format_symbol_prompt_location(self, what=None, symbol=None, root=None,
  285. enable_choice=False, header=None,
  286. get_label_func=lambda x: x):
  287. if what == "layout":
  288. return ( "100%", "^1,4" )
  289. if what == "header":
  290. if hasattr(root, "get_title"):
  291. loc_label = get_symbol_parents(root, None, enable_choice=enable_choice)
  292. loc_label += [root.get_title(), "..."]
  293. else:
  294. loc_label = ["Location"]
  295. return "| {0:<40} <| {1}\n".format(header, " -> ".join(loc_label))
  296. if what == "symbol":
  297. parents = get_symbol_parents(symbol, root, enable_choice)
  298. return "| {0:<40} <| {1}\n".format(get_label_func(symbol),
  299. " -> ".join(parents))
  300. message = "Invalid argument 'what': '%s'\n" % str(what)
  301. message += "Allowed values are: 'layout', 'header' and 'symbol'"
  302. raise Exception(message)
  303. def _format_symbol_virtual(self, what=None, symbol=None, root=None,
  304. enable_choice=False, header=None,
  305. get_label_func=lambda x: "?"):
  306. def _symbol_is_legacy(symbol):
  307. selects = [ s.get_name() for s in symbol.get_selected_symbols() ]
  308. return ("BR2_LEGACY" in selects)
  309. def _get_parent_package(sym):
  310. if self._is_real_package(sym):
  311. return None
  312. # Trim the symbol name from its last component (separated with
  313. # underscores), until we either find a symbol which is a real
  314. # package, or until we have no component (i.e. just 'BR2')
  315. name = sym.get_name()
  316. while name != "BR2":
  317. name = name.rsplit("_", 1)[0]
  318. s = self.config.get_symbol(name)
  319. if s is None:
  320. continue
  321. if self._is_real_package(s):
  322. return s
  323. return None
  324. def _get_providers(symbol):
  325. providers = list()
  326. for sym in self.config:
  327. if not sym.is_symbol():
  328. continue
  329. if _symbol_is_legacy(sym):
  330. continue
  331. selects = sym.get_selected_symbols()
  332. if not selects:
  333. continue
  334. for s in selects:
  335. if s == symbol:
  336. if sym.get_prompts():
  337. l = self._get_symbol_label(sym,False)
  338. parent_pkg = _get_parent_package(sym)
  339. if parent_pkg is not None:
  340. l = self._get_symbol_label(parent_pkg, False) \
  341. + " (w/ " + l + ")"
  342. providers.append(l)
  343. else:
  344. providers.extend(_get_providers(sym))
  345. return providers
  346. if what == "layout":
  347. return ( "100%", "^1,4,4" )
  348. if what == "header":
  349. return "| {0:<20} <| {1:<32} <| Providers\n".format("Virtual packages", "Symbols")
  350. if what == "symbol":
  351. pkg = re.sub(r"^BR2_PACKAGE_HAS_(.+)$", r"\1", symbol.get_name())
  352. providers = _get_providers(symbol)
  353. return "| {0:<20} <| {1:<32} <| {2}\n".format(pkg.lower(),
  354. '+' + symbol.get_name() + '+',
  355. ", ".join(providers))
  356. message = "Invalid argument 'what': '%s'\n" % str(what)
  357. message += "Allowed values are: 'layout', 'header' and 'symbol'"
  358. raise Exception(message)
  359. def print_list(self, list_type, enable_choice=True, enable_deprecated=True,
  360. dry_run=False, output=None):
  361. """ Print the requested list. If not dry run, then the list is
  362. automatically written in its own file.
  363. :param list_type: The list type to be generated
  364. :param enable_choice: Flag enabling choices to appear in the list
  365. :param enable_deprecated: Flag enabling deprecated items to appear in
  366. the package lists
  367. :param dry_run: Dry run (print the list in stdout instead of
  368. writing the list file
  369. """
  370. def _get_menu(title):
  371. """ Return the first symbol menu matching the given title.
  372. """
  373. menus = self.config.get_menus()
  374. menu = [m for m in menus if m.get_title().lower() == title.lower()]
  375. if not menu:
  376. message = "No such menu: '{0}'".format(title)
  377. raise Exception(message)
  378. return menu[0]
  379. list_config = self.list_info[list_type]
  380. root_title = list_config.get('root_menu')
  381. if root_title:
  382. root_item = _get_menu(root_title)
  383. else:
  384. root_item = self.config
  385. filter_ = getattr(self, list_config.get('filter'))
  386. filter_func = lambda x: filter_(x)
  387. format_func = getattr(self, list_config.get('format'))
  388. if not enable_deprecated and list_type != "deprecated":
  389. filter_func = lambda x: filter_(x) and not self._is_deprecated(x)
  390. mark_depr = list_type != "deprecated"
  391. get_label = lambda x: self._get_symbol_label(x, mark_depr)
  392. item_label = "Features" if list_type == "deprecated" else "Packages"
  393. table = format_asciidoc_table(root_item, get_label,
  394. filter_func=filter_func,
  395. format_func=format_func,
  396. enable_choice=enable_choice,
  397. sorted=list_config.get('sorted'),
  398. item_label=item_label)
  399. content = self.list_in.format(table=table)
  400. if dry_run:
  401. print(content)
  402. return
  403. if not output:
  404. output_dir = self.output_dir
  405. if not output_dir:
  406. print("Warning: Undefined output directory.")
  407. print("\tUse source directory as output location.")
  408. output_dir = self.base_dir
  409. output = os.path.join(output_dir,
  410. list_config.get('filename') + ".txt")
  411. if not os.path.exists(os.path.dirname(output)):
  412. os.makedirs(os.path.dirname(output))
  413. print("Writing the {0} list in:\n\t{1}".format(list_type, output))
  414. with open(output, 'w') as fout:
  415. fout.write(content)
  416. if __name__ == '__main__':
  417. list_types = ['target-packages', 'host-packages', 'virtual-packages', 'deprecated']
  418. parser = ArgumentParser()
  419. parser.add_argument("list_type", nargs="?", choices=list_types,
  420. help="""\
  421. Generate the given list (generate all lists if unspecified)""")
  422. parser.add_argument("-n", "--dry-run", dest="dry_run", action='store_true',
  423. help="Output the generated list to stdout")
  424. parser.add_argument("--output-target", dest="output_target",
  425. help="Output target package file")
  426. parser.add_argument("--output-host", dest="output_host",
  427. help="Output host package file")
  428. parser.add_argument("--output-virtual", dest="output_virtual",
  429. help="Output virtual package file")
  430. parser.add_argument("--output-deprecated", dest="output_deprecated",
  431. help="Output deprecated file")
  432. args = parser.parse_args()
  433. lists = [args.list_type] if args.list_type else list_types
  434. buildroot = Buildroot()
  435. for list_name in lists:
  436. output = getattr(args, "output_" + list_name.split("-", 1)[0])
  437. buildroot.print_list(list_name, dry_run=args.dry_run, output=output)