checkkconfigsymbols.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. #!/usr/bin/env python3
  2. # SPDX-License-Identifier: GPL-2.0-only
  3. """Find Kconfig symbols that are referenced but not defined."""
  4. # (c) 2014-2017 Valentin Rothberg <valentinrothberg@gmail.com>
  5. # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
  6. #
  7. import argparse
  8. import difflib
  9. import os
  10. import re
  11. import signal
  12. import subprocess
  13. import sys
  14. from multiprocessing import Pool, cpu_count
  15. # regex expressions
  16. OPERATORS = r"&|\(|\)|\||\!"
  17. SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}"
  18. DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*"
  19. EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+"
  20. DEFAULT = r"default\s+.*?(?:if\s.+){,1}"
  21. STMT = r"^\s*(?:if|select|imply|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR
  22. SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")"
  23. # regex objects
  24. REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
  25. REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)')
  26. REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL)
  27. REGEX_KCONFIG_DEF = re.compile(DEF)
  28. REGEX_KCONFIG_EXPR = re.compile(EXPR)
  29. REGEX_KCONFIG_STMT = re.compile(STMT)
  30. REGEX_KCONFIG_HELP = re.compile(r"^\s+help\s*$")
  31. REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$")
  32. REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")
  33. REGEX_QUOTES = re.compile("(\"(.*?)\")")
  34. def parse_options():
  35. """The user interface of this module."""
  36. usage = "Run this tool to detect Kconfig symbols that are referenced but " \
  37. "not defined in Kconfig. If no option is specified, " \
  38. "checkkconfigsymbols defaults to check your current tree. " \
  39. "Please note that specifying commits will 'git reset --hard\' " \
  40. "your current tree! You may save uncommitted changes to avoid " \
  41. "losing data."
  42. parser = argparse.ArgumentParser(description=usage)
  43. parser.add_argument('-c', '--commit', dest='commit', action='store',
  44. default="",
  45. help="check if the specified commit (hash) introduces "
  46. "undefined Kconfig symbols")
  47. parser.add_argument('-d', '--diff', dest='diff', action='store',
  48. default="",
  49. help="diff undefined symbols between two commits "
  50. "(e.g., -d commmit1..commit2)")
  51. parser.add_argument('-f', '--find', dest='find', action='store_true',
  52. default=False,
  53. help="find and show commits that may cause symbols to be "
  54. "missing (required to run with --diff)")
  55. parser.add_argument('-i', '--ignore', dest='ignore', action='store',
  56. default="",
  57. help="ignore files matching this Python regex "
  58. "(e.g., -i '.*defconfig')")
  59. parser.add_argument('-s', '--sim', dest='sim', action='store', default="",
  60. help="print a list of max. 10 string-similar symbols")
  61. parser.add_argument('--force', dest='force', action='store_true',
  62. default=False,
  63. help="reset current Git tree even when it's dirty")
  64. parser.add_argument('--no-color', dest='color', action='store_false',
  65. default=True,
  66. help="don't print colored output (default when not "
  67. "outputting to a terminal)")
  68. args = parser.parse_args()
  69. if args.commit and args.diff:
  70. sys.exit("Please specify only one option at once.")
  71. if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff):
  72. sys.exit("Please specify valid input in the following format: "
  73. "\'commit1..commit2\'")
  74. if args.commit or args.diff:
  75. if not args.force and tree_is_dirty():
  76. sys.exit("The current Git tree is dirty (see 'git status'). "
  77. "Running this script may\ndelete important data since it "
  78. "calls 'git reset --hard' for some performance\nreasons. "
  79. " Please run this script in a clean Git tree or pass "
  80. "'--force' if you\nwant to ignore this warning and "
  81. "continue.")
  82. if args.commit:
  83. args.find = False
  84. if args.ignore:
  85. try:
  86. re.match(args.ignore, "this/is/just/a/test.c")
  87. except:
  88. sys.exit("Please specify a valid Python regex.")
  89. return args
  90. def main():
  91. """Main function of this module."""
  92. args = parse_options()
  93. global COLOR
  94. COLOR = args.color and sys.stdout.isatty()
  95. if args.sim and not args.commit and not args.diff:
  96. sims = find_sims(args.sim, args.ignore)
  97. if sims:
  98. print("%s: %s" % (yel("Similar symbols"), ', '.join(sims)))
  99. else:
  100. print("%s: no similar symbols found" % yel("Similar symbols"))
  101. sys.exit(0)
  102. # dictionary of (un)defined symbols
  103. defined = {}
  104. undefined = {}
  105. if args.commit or args.diff:
  106. head = get_head()
  107. # get commit range
  108. commit_a = None
  109. commit_b = None
  110. if args.commit:
  111. commit_a = args.commit + "~"
  112. commit_b = args.commit
  113. elif args.diff:
  114. split = args.diff.split("..")
  115. commit_a = split[0]
  116. commit_b = split[1]
  117. undefined_a = {}
  118. undefined_b = {}
  119. # get undefined items before the commit
  120. reset(commit_a)
  121. undefined_a, _ = check_symbols(args.ignore)
  122. # get undefined items for the commit
  123. reset(commit_b)
  124. undefined_b, defined = check_symbols(args.ignore)
  125. # report cases that are present for the commit but not before
  126. for symbol in sorted(undefined_b):
  127. # symbol has not been undefined before
  128. if symbol not in undefined_a:
  129. files = sorted(undefined_b.get(symbol))
  130. undefined[symbol] = files
  131. # check if there are new files that reference the undefined symbol
  132. else:
  133. files = sorted(undefined_b.get(symbol) -
  134. undefined_a.get(symbol))
  135. if files:
  136. undefined[symbol] = files
  137. # reset to head
  138. reset(head)
  139. # default to check the entire tree
  140. else:
  141. undefined, defined = check_symbols(args.ignore)
  142. # now print the output
  143. for symbol in sorted(undefined):
  144. print(red(symbol))
  145. files = sorted(undefined.get(symbol))
  146. print("%s: %s" % (yel("Referencing files"), ", ".join(files)))
  147. sims = find_sims(symbol, args.ignore, defined)
  148. sims_out = yel("Similar symbols")
  149. if sims:
  150. print("%s: %s" % (sims_out, ', '.join(sims)))
  151. else:
  152. print("%s: %s" % (sims_out, "no similar symbols found"))
  153. if args.find:
  154. print("%s:" % yel("Commits changing symbol"))
  155. commits = find_commits(symbol, args.diff)
  156. if commits:
  157. for commit in commits:
  158. commit = commit.split(" ", 1)
  159. print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1]))
  160. else:
  161. print("\t- no commit found")
  162. print() # new line
  163. def reset(commit):
  164. """Reset current git tree to %commit."""
  165. execute(["git", "reset", "--hard", commit])
  166. def yel(string):
  167. """
  168. Color %string yellow.
  169. """
  170. return "\033[33m%s\033[0m" % string if COLOR else string
  171. def red(string):
  172. """
  173. Color %string red.
  174. """
  175. return "\033[31m%s\033[0m" % string if COLOR else string
  176. def execute(cmd):
  177. """Execute %cmd and return stdout. Exit in case of error."""
  178. try:
  179. stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False)
  180. stdout = stdout.decode(errors='replace')
  181. except subprocess.CalledProcessError as fail:
  182. exit(fail)
  183. return stdout
  184. def find_commits(symbol, diff):
  185. """Find commits changing %symbol in the given range of %diff."""
  186. commits = execute(["git", "log", "--pretty=oneline",
  187. "--abbrev-commit", "-G",
  188. symbol, diff])
  189. return [x for x in commits.split("\n") if x]
  190. def tree_is_dirty():
  191. """Return true if the current working tree is dirty (i.e., if any file has
  192. been added, deleted, modified, renamed or copied but not committed)."""
  193. stdout = execute(["git", "status", "--porcelain"])
  194. for line in stdout:
  195. if re.findall(r"[URMADC]{1}", line[:2]):
  196. return True
  197. return False
  198. def get_head():
  199. """Return commit hash of current HEAD."""
  200. stdout = execute(["git", "rev-parse", "HEAD"])
  201. return stdout.strip('\n')
  202. def partition(lst, size):
  203. """Partition list @lst into eveni-sized lists of size @size."""
  204. return [lst[i::size] for i in range(size)]
  205. def init_worker():
  206. """Set signal handler to ignore SIGINT."""
  207. signal.signal(signal.SIGINT, signal.SIG_IGN)
  208. def find_sims(symbol, ignore, defined=[]):
  209. """Return a list of max. ten Kconfig symbols that are string-similar to
  210. @symbol."""
  211. if defined:
  212. return difflib.get_close_matches(symbol, set(defined), 10)
  213. pool = Pool(cpu_count(), init_worker)
  214. kfiles = []
  215. for gitfile in get_files():
  216. if REGEX_FILE_KCONFIG.match(gitfile):
  217. kfiles.append(gitfile)
  218. arglist = []
  219. for part in partition(kfiles, cpu_count()):
  220. arglist.append((part, ignore))
  221. for res in pool.map(parse_kconfig_files, arglist):
  222. defined.extend(res[0])
  223. return difflib.get_close_matches(symbol, set(defined), 10)
  224. def get_files():
  225. """Return a list of all files in the current git directory."""
  226. # use 'git ls-files' to get the worklist
  227. stdout = execute(["git", "ls-files"])
  228. if len(stdout) > 0 and stdout[-1] == "\n":
  229. stdout = stdout[:-1]
  230. files = []
  231. for gitfile in stdout.rsplit("\n"):
  232. if ".git" in gitfile or "ChangeLog" in gitfile or \
  233. ".log" in gitfile or os.path.isdir(gitfile) or \
  234. gitfile.startswith("tools/"):
  235. continue
  236. files.append(gitfile)
  237. return files
  238. def check_symbols(ignore):
  239. """Find undefined Kconfig symbols and return a dict with the symbol as key
  240. and a list of referencing files as value. Files matching %ignore are not
  241. checked for undefined symbols."""
  242. pool = Pool(cpu_count(), init_worker)
  243. try:
  244. return check_symbols_helper(pool, ignore)
  245. except KeyboardInterrupt:
  246. pool.terminate()
  247. pool.join()
  248. sys.exit(1)
  249. def check_symbols_helper(pool, ignore):
  250. """Helper method for check_symbols(). Used to catch keyboard interrupts in
  251. check_symbols() in order to properly terminate running worker processes."""
  252. source_files = []
  253. kconfig_files = []
  254. defined_symbols = []
  255. referenced_symbols = dict() # {file: [symbols]}
  256. for gitfile in get_files():
  257. if REGEX_FILE_KCONFIG.match(gitfile):
  258. kconfig_files.append(gitfile)
  259. else:
  260. if ignore and not re.match(ignore, gitfile):
  261. continue
  262. # add source files that do not match the ignore pattern
  263. source_files.append(gitfile)
  264. # parse source files
  265. arglist = partition(source_files, cpu_count())
  266. for res in pool.map(parse_source_files, arglist):
  267. referenced_symbols.update(res)
  268. # parse kconfig files
  269. arglist = []
  270. for part in partition(kconfig_files, cpu_count()):
  271. arglist.append((part, ignore))
  272. for res in pool.map(parse_kconfig_files, arglist):
  273. defined_symbols.extend(res[0])
  274. referenced_symbols.update(res[1])
  275. defined_symbols = set(defined_symbols)
  276. # inverse mapping of referenced_symbols to dict(symbol: [files])
  277. inv_map = dict()
  278. for _file, symbols in referenced_symbols.items():
  279. for symbol in symbols:
  280. inv_map[symbol] = inv_map.get(symbol, set())
  281. inv_map[symbol].add(_file)
  282. referenced_symbols = inv_map
  283. undefined = {} # {symbol: [files]}
  284. for symbol in sorted(referenced_symbols):
  285. # filter some false positives
  286. if symbol == "FOO" or symbol == "BAR" or \
  287. symbol == "FOO_BAR" or symbol == "XXX":
  288. continue
  289. if symbol not in defined_symbols:
  290. if symbol.endswith("_MODULE"):
  291. # avoid false positives for kernel modules
  292. if symbol[:-len("_MODULE")] in defined_symbols:
  293. continue
  294. undefined[symbol] = referenced_symbols.get(symbol)
  295. return undefined, defined_symbols
  296. def parse_source_files(source_files):
  297. """Parse each source file in @source_files and return dictionary with source
  298. files as keys and lists of references Kconfig symbols as values."""
  299. referenced_symbols = dict()
  300. for sfile in source_files:
  301. referenced_symbols[sfile] = parse_source_file(sfile)
  302. return referenced_symbols
  303. def parse_source_file(sfile):
  304. """Parse @sfile and return a list of referenced Kconfig symbols."""
  305. lines = []
  306. references = []
  307. if not os.path.exists(sfile):
  308. return references
  309. with open(sfile, "r", encoding='utf-8', errors='replace') as stream:
  310. lines = stream.readlines()
  311. for line in lines:
  312. if "CONFIG_" not in line:
  313. continue
  314. symbols = REGEX_SOURCE_SYMBOL.findall(line)
  315. for symbol in symbols:
  316. if not REGEX_FILTER_SYMBOLS.search(symbol):
  317. continue
  318. references.append(symbol)
  319. return references
  320. def get_symbols_in_line(line):
  321. """Return mentioned Kconfig symbols in @line."""
  322. return REGEX_SYMBOL.findall(line)
  323. def parse_kconfig_files(args):
  324. """Parse kconfig files and return tuple of defined and references Kconfig
  325. symbols. Note, @args is a tuple of a list of files and the @ignore
  326. pattern."""
  327. kconfig_files = args[0]
  328. ignore = args[1]
  329. defined_symbols = []
  330. referenced_symbols = dict()
  331. for kfile in kconfig_files:
  332. defined, references = parse_kconfig_file(kfile)
  333. defined_symbols.extend(defined)
  334. if ignore and re.match(ignore, kfile):
  335. # do not collect references for files that match the ignore pattern
  336. continue
  337. referenced_symbols[kfile] = references
  338. return (defined_symbols, referenced_symbols)
  339. def parse_kconfig_file(kfile):
  340. """Parse @kfile and update symbol definitions and references."""
  341. lines = []
  342. defined = []
  343. references = []
  344. skip = False
  345. if not os.path.exists(kfile):
  346. return defined, references
  347. with open(kfile, "r", encoding='utf-8', errors='replace') as stream:
  348. lines = stream.readlines()
  349. for i in range(len(lines)):
  350. line = lines[i]
  351. line = line.strip('\n')
  352. line = line.split("#")[0] # ignore comments
  353. if REGEX_KCONFIG_DEF.match(line):
  354. symbol_def = REGEX_KCONFIG_DEF.findall(line)
  355. defined.append(symbol_def[0])
  356. skip = False
  357. elif REGEX_KCONFIG_HELP.match(line):
  358. skip = True
  359. elif skip:
  360. # ignore content of help messages
  361. pass
  362. elif REGEX_KCONFIG_STMT.match(line):
  363. line = REGEX_QUOTES.sub("", line)
  364. symbols = get_symbols_in_line(line)
  365. # multi-line statements
  366. while line.endswith("\\"):
  367. i += 1
  368. line = lines[i]
  369. line = line.strip('\n')
  370. symbols.extend(get_symbols_in_line(line))
  371. for symbol in set(symbols):
  372. if REGEX_NUMERIC.match(symbol):
  373. # ignore numeric values
  374. continue
  375. references.append(symbol)
  376. return defined, references
  377. if __name__ == "__main__":
  378. main()