generate_def_files.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. #!/usr/bin/env python3
  2. """Script to generate Chromium's Abseil .def files at roll time.
  3. This script generates //third_party/abseil-app/absl/symbols_*.def at Abseil
  4. roll time.
  5. Since Abseil doesn't export symbols, Chromium is forced to consider all
  6. Abseil's symbols as publicly visible. On POSIX it is possible to use
  7. -fvisibility=default but on Windows a .def file with all the symbols
  8. is needed.
  9. Unless you are on a Windows machine, you need to set up your Chromium
  10. checkout for cross-compilation by following the instructions at
  11. https://chromium.googlesource.com/chromium/src.git/+/main/docs/win_cross.md.
  12. If you are on Windows, you may need to tweak this script to run, e.g. by
  13. changing "gn" to "gn.bat", changing "llvm-nm" to the name of your copy of
  14. llvm-nm, etc.
  15. """
  16. import fnmatch
  17. import logging
  18. import os
  19. import re
  20. import subprocess
  21. import sys
  22. import tempfile
  23. import time
  24. # Matches a mangled symbol that has 'absl' in it, this should be a good
  25. # enough heuristic to select Abseil symbols to list in the .def file.
  26. ABSL_SYM_RE = re.compile(r'0* [BT] (?P<symbol>(\?+)[^\?].*absl.*)')
  27. if sys.platform == 'win32':
  28. # Typical dumpbin /symbol lines look like this:
  29. # 04B 0000000C SECT14 notype Static | ?$S1@?1??SetCurrent
  30. # ThreadIdentity@base_internal@absl@@YAXPAUThreadIdentity@12@P6AXPAX@Z@Z@4IA
  31. # (unsigned int `void __cdecl absl::base_internal::SetCurrentThreadIdentity...
  32. # We need to start on "| ?" and end on the first " (" (stopping on space would
  33. # also work).
  34. # This regex is identical inside the () characters except for the ? after .*,
  35. # which is needed to prevent greedily grabbing the undecorated version of the
  36. # symbols.
  37. ABSL_SYM_RE = '.*External \| (?P<symbol>(\?+)[^\?].*?absl.*?) \(.*'
  38. # Typical exported symbols in dumpbin /directives look like:
  39. # /EXPORT:?kHexChar@numbers_internal@absl@@3QBDB,DATA
  40. ABSL_EXPORTED_RE = '.*/EXPORT:(.*),.*'
  41. def _DebugOrRelease(is_debug):
  42. return 'dbg' if is_debug else 'rel'
  43. def _GenerateDefFile(cpu, is_debug, extra_gn_args=[], suffix=None):
  44. """Generates a .def file for the absl component build on the specified CPU."""
  45. if extra_gn_args:
  46. assert suffix != None, 'suffix is needed when extra_gn_args is used'
  47. flavor = _DebugOrRelease(is_debug)
  48. gn_args = [
  49. 'ffmpeg_branding = "Chrome"',
  50. 'is_component_build = true',
  51. 'is_debug = {}'.format(str(is_debug).lower()),
  52. 'proprietary_codecs = true',
  53. 'symbol_level = 0',
  54. 'target_cpu = "{}"'.format(cpu),
  55. 'target_os = "win"',
  56. ]
  57. gn_args.extend(extra_gn_args)
  58. gn = 'gn'
  59. autoninja = 'autoninja'
  60. symbol_dumper = ['third_party/llvm-build/Release+Asserts/bin/llvm-nm']
  61. if sys.platform == 'win32':
  62. gn = 'gn.bat'
  63. autoninja = 'autoninja.bat'
  64. symbol_dumper = ['dumpbin', '/symbols']
  65. import shutil
  66. if not shutil.which('dumpbin'):
  67. logging.error('dumpbin not found. Run tools\win\setenv.bat.')
  68. exit(1)
  69. with tempfile.TemporaryDirectory() as out_dir:
  70. logging.info('[%s - %s] Creating tmp out dir in %s', cpu, flavor, out_dir)
  71. subprocess.check_call([gn, 'gen', out_dir, '--args=' + ' '.join(gn_args)],
  72. cwd=os.getcwd())
  73. logging.info('[%s - %s] gn gen completed', cpu, flavor)
  74. subprocess.check_call(
  75. [autoninja, '-C', out_dir, 'third_party/abseil-cpp:absl_component_deps'],
  76. cwd=os.getcwd())
  77. logging.info('[%s - %s] autoninja completed', cpu, flavor)
  78. obj_files = []
  79. for root, _dirnames, filenames in os.walk(
  80. os.path.join(out_dir, 'obj', 'third_party', 'abseil-cpp')):
  81. matched_files = fnmatch.filter(filenames, '*.obj')
  82. obj_files.extend((os.path.join(root, f) for f in matched_files))
  83. logging.info('[%s - %s] Found %d object files.', cpu, flavor, len(obj_files))
  84. absl_symbols = set()
  85. dll_exports = set()
  86. if sys.platform == 'win32':
  87. for f in obj_files:
  88. # Track all of the functions exported with __declspec(dllexport) and
  89. # don't list them in the .def file - double-exports are not allowed. The
  90. # error is "lld-link: error: duplicate /export option".
  91. exports_out = subprocess.check_output(['dumpbin', '/directives', f], cwd=os.getcwd())
  92. for line in exports_out.splitlines():
  93. line = line.decode('utf-8')
  94. match = re.match(ABSL_EXPORTED_RE, line)
  95. if match:
  96. dll_exports.add(match.groups()[0])
  97. for f in obj_files:
  98. stdout = subprocess.check_output(symbol_dumper + [f], cwd=os.getcwd())
  99. for line in stdout.splitlines():
  100. try:
  101. line = line.decode('utf-8')
  102. except UnicodeDecodeError:
  103. # Due to a dumpbin bug there are sometimes invalid utf-8 characters in
  104. # the output. This only happens on an unimportant line so it can
  105. # safely and silently be skipped.
  106. # https://developercommunity.visualstudio.com/content/problem/1091330/dumpbin-symbols-produces-randomly-wrong-output-on.html
  107. continue
  108. match = re.match(ABSL_SYM_RE, line)
  109. if match:
  110. symbol = match.group('symbol')
  111. assert symbol.count(' ') == 0, ('Regex matched too much, probably got '
  112. 'undecorated name as well')
  113. # Avoid getting names exported with dllexport, to avoid
  114. # "lld-link: error: duplicate /export option" on symbols such as:
  115. # ?kHexChar@numbers_internal@absl@@3QBDB
  116. if symbol in dll_exports:
  117. continue
  118. # Avoid to export deleting dtors since they trigger
  119. # "lld-link: error: export of deleting dtor" linker errors, see
  120. # crbug.com/1201277.
  121. if symbol.startswith('??_G'):
  122. continue
  123. absl_symbols.add(symbol)
  124. logging.info('[%s - %s] Found %d absl symbols.', cpu, flavor, len(absl_symbols))
  125. if extra_gn_args:
  126. def_file = os.path.join('third_party', 'abseil-cpp',
  127. 'symbols_{}_{}_{}.def'.format(cpu, flavor, suffix))
  128. else:
  129. def_file = os.path.join('third_party', 'abseil-cpp',
  130. 'symbols_{}_{}.def'.format(cpu, flavor))
  131. with open(def_file, 'w', newline='') as f:
  132. f.write('EXPORTS\n')
  133. for s in sorted(absl_symbols):
  134. f.write(' {}\n'.format(s))
  135. # Hack, it looks like there is a race in the directory cleanup.
  136. time.sleep(10)
  137. logging.info('[%s - %s] .def file successfully generated.', cpu, flavor)
  138. if __name__ == '__main__':
  139. logging.getLogger().setLevel(logging.INFO)
  140. if sys.version_info.major == 2:
  141. logging.error('This script requires Python 3.')
  142. exit(1)
  143. if not os.getcwd().endswith('src') or not os.path.exists('chrome/browser'):
  144. logging.error('Run this script from a chromium/src/ directory.')
  145. exit(1)
  146. _GenerateDefFile('x86', True)
  147. _GenerateDefFile('x86', False)
  148. _GenerateDefFile('x64', True)
  149. _GenerateDefFile('x64', False)
  150. _GenerateDefFile('x64', False, ['is_asan = true'], 'asan')
  151. _GenerateDefFile('arm64', True)
  152. _GenerateDefFile('arm64', False)