gen_compile_commands.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237
  1. #!/usr/bin/env python3
  2. # SPDX-License-Identifier: GPL-2.0
  3. #
  4. # Copyright (C) Google LLC, 2018
  5. #
  6. # Author: Tom Roeder <tmroeder@google.com>
  7. #
  8. """A tool for generating compile_commands.json in the Linux kernel."""
  9. import argparse
  10. import json
  11. import logging
  12. import os
  13. import re
  14. import subprocess
  15. import sys
  16. _DEFAULT_OUTPUT = 'compile_commands.json'
  17. _DEFAULT_LOG_LEVEL = 'WARNING'
  18. _FILENAME_PATTERN = r'^\..*\.cmd$'
  19. _LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
  20. _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
  21. def parse_arguments():
  22. """Sets up and parses command-line arguments.
  23. Returns:
  24. log_level: A logging level to filter log output.
  25. directory: The work directory where the objects were built.
  26. ar: Command used for parsing .a archives.
  27. output: Where to write the compile-commands JSON file.
  28. paths: The list of files/directories to handle to find .cmd files.
  29. """
  30. usage = 'Creates a compile_commands.json database from kernel .cmd files'
  31. parser = argparse.ArgumentParser(description=usage)
  32. directory_help = ('specify the output directory used for the kernel build '
  33. '(defaults to the working directory)')
  34. parser.add_argument('-d', '--directory', type=str, default='.',
  35. help=directory_help)
  36. output_help = ('path to the output command database (defaults to ' +
  37. _DEFAULT_OUTPUT + ')')
  38. parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
  39. help=output_help)
  40. log_level_help = ('the level of log messages to produce (defaults to ' +
  41. _DEFAULT_LOG_LEVEL + ')')
  42. parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
  43. default=_DEFAULT_LOG_LEVEL, help=log_level_help)
  44. ar_help = 'command used for parsing .a archives'
  45. parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
  46. paths_help = ('directories to search or files to parse '
  47. '(files should be *.o, *.a, or modules.order). '
  48. 'If nothing is specified, the current directory is searched')
  49. parser.add_argument('paths', type=str, nargs='*', help=paths_help)
  50. args = parser.parse_args()
  51. return (args.log_level,
  52. os.path.abspath(args.directory),
  53. args.output,
  54. args.ar,
  55. args.paths if len(args.paths) > 0 else [args.directory])
  56. def cmdfiles_in_dir(directory):
  57. """Generate the iterator of .cmd files found under the directory.
  58. Walk under the given directory, and yield every .cmd file found.
  59. Args:
  60. directory: The directory to search for .cmd files.
  61. Yields:
  62. The path to a .cmd file.
  63. """
  64. filename_matcher = re.compile(_FILENAME_PATTERN)
  65. for dirpath, _, filenames in os.walk(directory):
  66. for filename in filenames:
  67. if filename_matcher.match(filename):
  68. yield os.path.join(dirpath, filename)
  69. def to_cmdfile(path):
  70. """Return the path of .cmd file used for the given build artifact
  71. Args:
  72. Path: file path
  73. Returns:
  74. The path to .cmd file
  75. """
  76. dir, base = os.path.split(path)
  77. return os.path.join(dir, '.' + base + '.cmd')
  78. def cmdfiles_for_o(obj):
  79. """Generate the iterator of .cmd files associated with the object
  80. Yield the .cmd file used to build the given object
  81. Args:
  82. obj: The object path
  83. Yields:
  84. The path to .cmd file
  85. """
  86. yield to_cmdfile(obj)
  87. def cmdfiles_for_a(archive, ar):
  88. """Generate the iterator of .cmd files associated with the archive.
  89. Parse the given archive, and yield every .cmd file used to build it.
  90. Args:
  91. archive: The archive to parse
  92. Yields:
  93. The path to every .cmd file found
  94. """
  95. for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
  96. yield to_cmdfile(obj)
  97. def cmdfiles_for_modorder(modorder):
  98. """Generate the iterator of .cmd files associated with the modules.order.
  99. Parse the given modules.order, and yield every .cmd file used to build the
  100. contained modules.
  101. Args:
  102. modorder: The modules.order file to parse
  103. Yields:
  104. The path to every .cmd file found
  105. """
  106. with open(modorder) as f:
  107. for line in f:
  108. ko = line.rstrip()
  109. base, ext = os.path.splitext(ko)
  110. if ext != '.ko':
  111. sys.exit('{}: module path must end with .ko'.format(ko))
  112. mod = base + '.mod'
  113. # The first line of *.mod lists the objects that compose the module.
  114. with open(mod) as m:
  115. for obj in m.readline().split():
  116. yield to_cmdfile(obj)
  117. def process_line(root_directory, command_prefix, file_path):
  118. """Extracts information from a .cmd line and creates an entry from it.
  119. Args:
  120. root_directory: The directory that was searched for .cmd files. Usually
  121. used directly in the "directory" entry in compile_commands.json.
  122. command_prefix: The extracted command line, up to the last element.
  123. file_path: The .c file from the end of the extracted command.
  124. Usually relative to root_directory, but sometimes absolute.
  125. Returns:
  126. An entry to append to compile_commands.
  127. Raises:
  128. ValueError: Could not find the extracted file based on file_path and
  129. root_directory or file_directory.
  130. """
  131. # The .cmd files are intended to be included directly by Make, so they
  132. # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
  133. # kernel version). The compile_commands.json file is not interepreted
  134. # by Make, so this code replaces the escaped version with '#'.
  135. prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
  136. # Use os.path.abspath() to normalize the path resolving '.' and '..' .
  137. abs_path = os.path.abspath(os.path.join(root_directory, file_path))
  138. if not os.path.exists(abs_path):
  139. raise ValueError('File %s not found' % abs_path)
  140. return {
  141. 'directory': root_directory,
  142. 'file': abs_path,
  143. 'command': prefix + file_path,
  144. }
  145. def main():
  146. """Walks through the directory and finds and parses .cmd files."""
  147. log_level, directory, output, ar, paths = parse_arguments()
  148. level = getattr(logging, log_level)
  149. logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
  150. line_matcher = re.compile(_LINE_PATTERN)
  151. compile_commands = []
  152. for path in paths:
  153. # If 'path' is a directory, handle all .cmd files under it.
  154. # Otherwise, handle .cmd files associated with the file.
  155. # Most of built-in objects are linked via archives (built-in.a or lib.a)
  156. # but some objects are linked to vmlinux directly.
  157. # Modules are listed in modules.order.
  158. if os.path.isdir(path):
  159. cmdfiles = cmdfiles_in_dir(path)
  160. elif path.endswith('.o'):
  161. cmdfiles = cmdfiles_for_o(path)
  162. elif path.endswith('.a'):
  163. cmdfiles = cmdfiles_for_a(path, ar)
  164. elif path.endswith('modules.order'):
  165. cmdfiles = cmdfiles_for_modorder(path)
  166. else:
  167. sys.exit('{}: unknown file type'.format(path))
  168. for cmdfile in cmdfiles:
  169. with open(cmdfile, 'rt') as f:
  170. result = line_matcher.match(f.readline())
  171. if result:
  172. try:
  173. entry = process_line(directory, result.group(1),
  174. result.group(2))
  175. compile_commands.append(entry)
  176. except ValueError as err:
  177. logging.info('Could not add line from %s: %s',
  178. cmdfile, err)
  179. with open(output, 'wt') as f:
  180. json.dump(compile_commands, f, indent=2, sort_keys=True)
  181. if __name__ == '__main__':
  182. main()