verify-bashisms 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. #!/usr/bin/env python3
  2. #
  3. # SPDX-License-Identifier: GPL-2.0-only
  4. #
  5. import sys, os, subprocess, re, shutil
  6. whitelist = (
  7. # type is supported by dash
  8. 'if type systemctl >/dev/null 2>/dev/null; then',
  9. 'if type systemd-tmpfiles >/dev/null 2>/dev/null; then',
  10. 'type update-rc.d >/dev/null 2>/dev/null; then',
  11. 'command -v',
  12. # HOSTNAME is set locally
  13. 'buildhistory_single_commit "$CMDLINE" "$HOSTNAME"',
  14. # False-positive, match is a grep not shell expression
  15. 'grep "^$groupname:[^:]*:[^:]*:\\([^,]*,\\)*$username\\(,[^,]*\\)*"',
  16. # TODO verify dash's '. script args' behaviour
  17. '. $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE'
  18. )
  19. def is_whitelisted(s):
  20. for w in whitelist:
  21. if w in s:
  22. return True
  23. return False
  24. SCRIPT_LINENO_RE = re.compile(r' line (\d+) ')
  25. BASHISM_WARNING = re.compile(r'^(possible bashism in.*)$', re.MULTILINE)
  26. def process(filename, function, lineno, script):
  27. import tempfile
  28. if not script.startswith("#!"):
  29. script = "#! /bin/sh\n" + script
  30. fn = tempfile.NamedTemporaryFile(mode="w+t")
  31. fn.write(script)
  32. fn.flush()
  33. try:
  34. subprocess.check_output(("checkbashisms.pl", fn.name), universal_newlines=True, stderr=subprocess.STDOUT)
  35. # No bashisms, so just return
  36. return
  37. except subprocess.CalledProcessError as e:
  38. # TODO check exit code is 1
  39. # Replace the temporary filename with the function and split it
  40. output = e.output.replace(fn.name, function)
  41. if not output or not output.startswith('possible bashism'):
  42. # Probably starts with or contains only warnings. Dump verbatim
  43. # with one space indention. Can't do the splitting and whitelist
  44. # checking below.
  45. return '\n'.join([filename,
  46. ' Unexpected output from checkbashisms.pl'] +
  47. [' ' + x for x in output.splitlines()])
  48. # We know that the first line matches and that therefore the first
  49. # list entry will be empty - skip it.
  50. output = BASHISM_WARNING.split(output)[1:]
  51. # Turn the output into a single string like this:
  52. # /.../foobar.bb
  53. # possible bashism in updatercd_postrm line 2 (type):
  54. # if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then
  55. # ...
  56. # ...
  57. result = []
  58. # Check the results against the whitelist
  59. for message, source in zip(output[0::2], output[1::2]):
  60. if not is_whitelisted(source):
  61. if lineno is not None:
  62. message = SCRIPT_LINENO_RE.sub(lambda m: ' line %d ' % (int(m.group(1)) + int(lineno) - 1),
  63. message)
  64. result.append(' ' + message.strip())
  65. result.extend([' %s' % x for x in source.splitlines()])
  66. if result:
  67. result.insert(0, filename)
  68. return '\n'.join(result)
  69. else:
  70. return None
  71. def get_tinfoil():
  72. scripts_path = os.path.dirname(os.path.realpath(__file__))
  73. lib_path = scripts_path + '/lib'
  74. sys.path = sys.path + [lib_path]
  75. import scriptpath
  76. scriptpath.add_bitbake_lib_path()
  77. import bb.tinfoil
  78. tinfoil = bb.tinfoil.Tinfoil()
  79. tinfoil.prepare()
  80. # tinfoil.logger.setLevel(logging.WARNING)
  81. return tinfoil
  82. if __name__=='__main__':
  83. import argparse, shutil
  84. parser = argparse.ArgumentParser(description='Bashim detector for shell fragments in recipes.')
  85. parser.add_argument("recipes", metavar="RECIPE", nargs="*", help="recipes to check (if not specified, all will be checked)")
  86. parser.add_argument("--verbose", default=False, action="store_true")
  87. args = parser.parse_args()
  88. if shutil.which("checkbashisms.pl") is None:
  89. print("Cannot find checkbashisms.pl on $PATH, get it from https://anonscm.debian.org/cgit/collab-maint/devscripts.git/plain/scripts/checkbashisms.pl")
  90. sys.exit(1)
  91. # The order of defining the worker function,
  92. # initializing the pool and connecting to the
  93. # bitbake server is crucial, don't change it.
  94. def func(item):
  95. (filename, key, lineno), script = item
  96. if args.verbose:
  97. print("Scanning %s:%s" % (filename, key))
  98. return process(filename, key, lineno, script)
  99. import multiprocessing
  100. pool = multiprocessing.Pool()
  101. tinfoil = get_tinfoil()
  102. # This is only the default configuration and should iterate over
  103. # recipecaches to handle multiconfig environments
  104. pkg_pn = tinfoil.cooker.recipecaches[""].pkg_pn
  105. if args.recipes:
  106. initial_pns = args.recipes
  107. else:
  108. initial_pns = sorted(pkg_pn)
  109. pns = set()
  110. scripts = {}
  111. print("Generating scripts...")
  112. for pn in initial_pns:
  113. for fn in pkg_pn[pn]:
  114. # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
  115. # (at least in general - there is some risk that the variants contain different scripts)
  116. realfn, _, _ = bb.cache.virtualfn2realfn(fn)
  117. if realfn not in pns:
  118. pns.add(realfn)
  119. data = tinfoil.parse_recipe_file(realfn)
  120. for key in data.keys():
  121. if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"):
  122. script = data.getVar(key, False)
  123. if script:
  124. filename = data.getVarFlag(key, "filename")
  125. lineno = data.getVarFlag(key, "lineno")
  126. # There's no point in checking a function multiple
  127. # times just because different recipes include it.
  128. # We identify unique scripts by file, name, and (just in case)
  129. # line number.
  130. attributes = (filename or realfn, key, lineno)
  131. scripts.setdefault(attributes, script)
  132. print("Scanning scripts...\n")
  133. for result in pool.imap(func, scripts.items()):
  134. if result:
  135. print(result)
  136. tinfoil.shutdown()