merge_lib.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. # Copyright 2019 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Functions for interacting with llvm-profdata"""
  5. import logging
  6. import multiprocessing
  7. import os
  8. import re
  9. import shutil
  10. import subprocess
  11. import sys
  12. _DIR_SOURCE_ROOT = os.path.normpath(
  13. os.path.join(os.path.dirname(__file__), '..', '..', '..'))
  14. _JAVA_PATH = os.path.join(_DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current',
  15. 'bin', 'java')
  16. logging.basicConfig(
  17. format='[%(asctime)s %(levelname)s] %(message)s', level=logging.DEBUG)
  18. def _call_profdata_tool(profile_input_file_paths,
  19. profile_output_file_path,
  20. profdata_tool_path,
  21. sparse=False):
  22. """Calls the llvm-profdata tool.
  23. Args:
  24. profile_input_file_paths: A list of relative paths to the files that
  25. are to be merged.
  26. profile_output_file_path: The path to the merged file to write.
  27. profdata_tool_path: The path to the llvm-profdata executable.
  28. sparse (bool): flag to indicate whether to run llvm-profdata with --sparse.
  29. Doc: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge
  30. Returns:
  31. A list of paths to profiles that had to be excluded to get the merge to
  32. succeed, suspected of being corrupted or malformed.
  33. Raises:
  34. CalledProcessError: An error occurred merging profiles.
  35. """
  36. try:
  37. subprocess_cmd = [
  38. profdata_tool_path, 'merge', '-o', profile_output_file_path,
  39. ]
  40. if sparse:
  41. subprocess_cmd += ['-sparse=true',]
  42. subprocess_cmd.extend(profile_input_file_paths)
  43. logging.info('profdata command: %r', subprocess_cmd)
  44. # Redirecting stderr is required because when error happens, llvm-profdata
  45. # writes the error output to stderr and our error handling logic relies on
  46. # that output.
  47. subprocess.check_call(subprocess_cmd, stderr=subprocess.STDOUT)
  48. except subprocess.CalledProcessError as error:
  49. logging.error('Failed to merge profiles, return code (%d), output: %r' %
  50. (error.returncode, error.output))
  51. raise error
  52. logging.info('Profile data is created as: "%r".', profile_output_file_path)
  53. return []
  54. def _get_profile_paths(input_dir,
  55. input_extension,
  56. input_filename_pattern='.*'):
  57. """Finds all the profiles in the given directory (recursively)."""
  58. paths = []
  59. for dir_path, _sub_dirs, file_names in os.walk(input_dir):
  60. paths.extend([
  61. # Normalize to POSIX style paths for consistent results.
  62. os.path.join(dir_path, fn).replace('\\', '/')
  63. for fn in file_names
  64. if fn.endswith(input_extension) and re.search(input_filename_pattern,fn)
  65. ])
  66. return paths
  67. def _validate_and_convert_profraws(profraw_files,
  68. profdata_tool_path,
  69. sparse=False):
  70. """Validates and converts profraws to profdatas.
  71. For each given .profraw file in the input, this method first validates it by
  72. trying to convert it to an indexed .profdata file, and if the validation and
  73. conversion succeeds, the generated .profdata file will be included in the
  74. output, otherwise, won't.
  75. This method is mainly used to filter out invalid profraw files.
  76. Args:
  77. profraw_files: A list of .profraw paths.
  78. profdata_tool_path: The path to the llvm-profdata executable.
  79. sparse (bool): flag to indicate whether to run llvm-profdata with --sparse.
  80. Doc: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge
  81. Returns:
  82. A tulple:
  83. A list of converted .profdata files of *valid* profraw files.
  84. A list of *invalid* profraw files.
  85. A list of profraw files that have counter overflows.
  86. """
  87. for profraw_file in profraw_files:
  88. if not profraw_file.endswith('.profraw'):
  89. raise RuntimeError('%r is expected to be a .profraw file.' % profraw_file)
  90. cpu_count = multiprocessing.cpu_count()
  91. counts = max(10, cpu_count - 5) # Use 10+ processes, but leave 5 cpu cores.
  92. if sys.platform == 'win32':
  93. # TODO(crbug.com/1190269) - we can't use more than 56 child processes on
  94. # Windows or Python3 may hang.
  95. counts = min(counts, 56)
  96. pool = multiprocessing.Pool(counts)
  97. output_profdata_files = multiprocessing.Manager().list()
  98. invalid_profraw_files = multiprocessing.Manager().list()
  99. counter_overflows = multiprocessing.Manager().list()
  100. for profraw_file in profraw_files:
  101. pool.apply_async(
  102. _validate_and_convert_profraw,
  103. (profraw_file, output_profdata_files, invalid_profraw_files,
  104. counter_overflows, profdata_tool_path, sparse))
  105. pool.close()
  106. pool.join()
  107. # Remove inputs, as they won't be needed and they can be pretty large.
  108. for input_file in profraw_files:
  109. os.remove(input_file)
  110. return list(output_profdata_files), list(invalid_profraw_files), list(
  111. counter_overflows)
  112. def _validate_and_convert_profraw(profraw_file, output_profdata_files,
  113. invalid_profraw_files, counter_overflows,
  114. profdata_tool_path, sparse=False):
  115. output_profdata_file = profraw_file.replace('.profraw', '.profdata')
  116. subprocess_cmd = [
  117. profdata_tool_path,
  118. 'merge',
  119. '-o',
  120. output_profdata_file,
  121. ]
  122. if sparse:
  123. subprocess_cmd.append('--sparse')
  124. subprocess_cmd.append(profraw_file)
  125. logging.info('profdata command: %r', subprocess_cmd)
  126. profile_valid = False
  127. counter_overflow = False
  128. validation_output = None
  129. # 1. Determine if the profile is valid.
  130. try:
  131. # Redirecting stderr is required because when error happens, llvm-profdata
  132. # writes the error output to stderr and our error handling logic relies on
  133. # that output.
  134. validation_output = subprocess.check_output(
  135. subprocess_cmd, stderr=subprocess.STDOUT)
  136. if 'Counter overflow' in validation_output:
  137. counter_overflow = True
  138. else:
  139. profile_valid = True
  140. except subprocess.CalledProcessError as error:
  141. logging.warning('Validating and converting %r to %r failed with output: %r',
  142. profraw_file, output_profdata_file, error.output)
  143. validation_output = error.output
  144. # 2. Add the profile to the appropriate list(s).
  145. if profile_valid:
  146. output_profdata_files.append(output_profdata_file)
  147. else:
  148. invalid_profraw_files.append(profraw_file)
  149. if counter_overflow:
  150. counter_overflows.append(profraw_file)
  151. # 3. Log appropriate message
  152. if not profile_valid:
  153. template = 'Bad profile: %r, output: %r'
  154. if counter_overflow:
  155. template = 'Counter overflow: %r, output: %r'
  156. logging.warning(template, profraw_file, validation_output)
  157. # 4. Delete profdata for invalid profiles if present.
  158. if os.path.exists(output_profdata_file):
  159. # The output file may be created before llvm-profdata determines the
  160. # input is invalid. Delete it so that it does not leak and affect other
  161. # merge scripts.
  162. os.remove(output_profdata_file)
  163. def merge_java_exec_files(input_dir, output_path, jacococli_path):
  164. """Merges generated .exec files to output_path.
  165. Args:
  166. input_dir (str): The path to traverse to find input files.
  167. output_path (str): Where to write the merged .exec file.
  168. jacococli_path: The path to jacococli.jar.
  169. Raises:
  170. CalledProcessError: merge command failed.
  171. """
  172. exec_input_file_paths = _get_profile_paths(input_dir, '.exec')
  173. if not exec_input_file_paths:
  174. logging.info('No exec file found under %s', input_dir)
  175. return
  176. cmd = [_JAVA_PATH, '-jar', jacococli_path, 'merge']
  177. cmd.extend(exec_input_file_paths)
  178. cmd.extend(['--destfile', output_path])
  179. subprocess.check_call(cmd, stderr=subprocess.STDOUT)
  180. def merge_profiles(input_dir,
  181. output_file,
  182. input_extension,
  183. profdata_tool_path,
  184. input_filename_pattern='.*',
  185. sparse=False,
  186. skip_validation=False):
  187. """Merges the profiles produced by the shards using llvm-profdata.
  188. Args:
  189. input_dir (str): The path to traverse to find input profiles.
  190. output_file (str): Where to write the merged profile.
  191. input_extension (str): File extension to look for in the input_dir.
  192. e.g. '.profdata' or '.profraw'
  193. profdata_tool_path: The path to the llvm-profdata executable.
  194. input_filename_pattern (str): The regex pattern of input filename. Should be
  195. a valid regex pattern if present.
  196. sparse (bool): flag to indicate whether to run llvm-profdata with --sparse.
  197. Doc: https://llvm.org/docs/CommandGuide/llvm-profdata.html#profdata-merge
  198. skip_validation (bool): flag to skip the _validate_and_convert_profraws
  199. invocation. only applicable when input_extension is .profraw.
  200. Returns:
  201. The list of profiles that had to be excluded to get the merge to
  202. succeed and a list of profiles that had a counter overflow.
  203. """
  204. profile_input_file_paths = _get_profile_paths(input_dir,
  205. input_extension,
  206. input_filename_pattern)
  207. invalid_profraw_files = []
  208. counter_overflows = []
  209. if skip_validation:
  210. logging.warning('--skip-validation has been enabled. Skipping conversion '
  211. 'to ensure that profiles are valid.')
  212. if input_extension == '.profraw' and not skip_validation:
  213. profile_input_file_paths, invalid_profraw_files, counter_overflows = (
  214. _validate_and_convert_profraws(profile_input_file_paths,
  215. profdata_tool_path,
  216. sparse=sparse))
  217. logging.info((
  218. 'List of invalid .profraw files that failed to validate and convert: %r'
  219. ), invalid_profraw_files)
  220. if counter_overflows:
  221. logging.warning('There were %d profiles with counter overflows',
  222. len(counter_overflows))
  223. # The list of input files could be empty in the following scenarios:
  224. # 1. The test target is pure Python scripts test which doesn't execute any
  225. # C/C++ binaries, such as devtools_type_check.
  226. # 2. The test target executes binary and does dumps coverage profile data
  227. # files, however, all of them turned out to be invalid.
  228. if not profile_input_file_paths:
  229. logging.info('There is no valid profraw/profdata files to merge, skip '
  230. 'invoking profdata tools.')
  231. return invalid_profraw_files, counter_overflows
  232. invalid_profdata_files = _call_profdata_tool(
  233. profile_input_file_paths=profile_input_file_paths,
  234. profile_output_file_path=output_file,
  235. profdata_tool_path=profdata_tool_path,
  236. sparse=sparse)
  237. # Remove inputs when merging profraws as they won't be needed and they can be
  238. # pretty large. If the inputs are profdata files, do not remove them as they
  239. # might be used again for multiple test types coverage.
  240. if input_extension == '.profraw':
  241. for input_file in profile_input_file_paths:
  242. os.remove(input_file)
  243. return invalid_profraw_files + invalid_profdata_files, counter_overflows
  244. # We want to retry shards that contain one or more profiles that cannot be
  245. # merged (typically due to corruption described in crbug.com/937521).
  246. def get_shards_to_retry(bad_profiles):
  247. bad_shard_ids = set()
  248. def is_task_id(s):
  249. # Swarming task ids are 16 hex chars. The pythonic way to validate this is
  250. # to cast to int and catch a value error.
  251. try:
  252. assert len(s) == 16, 'Swarming task IDs are expected be of length 16'
  253. _int_id = int(s, 16)
  254. return True
  255. except (AssertionError, ValueError):
  256. return False
  257. for profile in bad_profiles:
  258. # E.g. /b/s/w/ir/tmp/t/tmpSvBRii/44b643576cf39f10/profraw/default-1.profraw
  259. _base_path, task_id, _profraw, _filename = os.path.normpath(profile).rsplit(
  260. os.path.sep, 3)
  261. # Since we are getting a task_id from a file path, which is less than ideal,
  262. # do some checking to at least verify that the snippet looks like a valid
  263. # task id.
  264. assert is_task_id(task_id)
  265. bad_shard_ids.add(task_id)
  266. return bad_shard_ids