ml.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. #!/usr/bin/env python
  2. # Copyright 2018 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Wraps ml.exe or ml64.exe and postprocesses the output to be deterministic.
  6. Sets timestamp in .obj file to 0, hence incompatible with link.exe /incremental.
  7. Use by prefixing the ml(64).exe invocation with this script:
  8. python ml.py ml.exe [args...]"""
  9. import array
  10. import collections
  11. import struct
  12. import subprocess
  13. import sys
  14. class Struct(object):
  15. """A thin wrapper around the struct module that returns a namedtuple"""
  16. def __init__(self, name, *args):
  17. """Pass the name of the return type, and then an interleaved list of
  18. format strings as used by the struct module and of field names."""
  19. self.fmt = '<' + ''.join(args[0::2])
  20. self.type = collections.namedtuple(name, args[1::2])
  21. def pack_into(self, buffer, offset, data):
  22. return struct.pack_into(self.fmt, buffer, offset, *data)
  23. def unpack_from(self, buffer, offset=0):
  24. return self.type(*struct.unpack_from(self.fmt, buffer, offset))
  25. def size(self):
  26. return struct.calcsize(self.fmt)
  27. def Subtract(nt, **kwargs):
  28. """Subtract(nt, f=2) returns a new namedtuple with 2 subtracted from nt.f"""
  29. return nt._replace(**{k: getattr(nt, k) - v for k, v in kwargs.items()})
  30. def MakeDeterministic(objdata):
  31. # Takes data produced by ml(64).exe (without any special flags) and
  32. # 1. Sets the timestamp to 0
  33. # 2. Strips the .debug$S section (which contains an unwanted absolute path)
  34. # This makes several assumptions about ml's output:
  35. # - Section data is in the same order as the corresponding section headers:
  36. # section headers preceding the .debug$S section header have their data
  37. # preceding the .debug$S section data; likewise for section headers
  38. # following the .debug$S section.
  39. # - The .debug$S section contains only the absolute path to the obj file and
  40. # nothing else, in particular there's only a single entry in the symbol
  41. # table referring to the .debug$S section.
  42. # - There are no COFF line number entries.
  43. # - There's no IMAGE_SYM_CLASS_CLR_TOKEN symbol.
  44. # These seem to hold in practice; if they stop holding this script needs to
  45. # become smarter.
  46. objdata = array.array('b', objdata) # Writable, e.g. via struct.pack_into.
  47. # Read coff header.
  48. COFFHEADER = Struct('COFFHEADER', 'H', 'Machine', 'H', 'NumberOfSections',
  49. 'I', 'TimeDateStamp', 'I', 'PointerToSymbolTable', 'I',
  50. 'NumberOfSymbols', 'H', 'SizeOfOptionalHeader', 'H',
  51. 'Characteristics')
  52. coff_header = COFFHEADER.unpack_from(objdata)
  53. assert coff_header.SizeOfOptionalHeader == 0 # Only set for binaries.
  54. # Read section headers following coff header.
  55. SECTIONHEADER = Struct('SECTIONHEADER', '8s', 'Name', 'I', 'VirtualSize', 'I',
  56. 'VirtualAddress', 'I', 'SizeOfRawData', 'I',
  57. 'PointerToRawData', 'I', 'PointerToRelocations', 'I',
  58. 'PointerToLineNumbers', 'H', 'NumberOfRelocations',
  59. 'H', 'NumberOfLineNumbers', 'I', 'Characteristics')
  60. section_headers = []
  61. debug_section_index = -1
  62. for i in range(0, coff_header.NumberOfSections):
  63. section_header = SECTIONHEADER.unpack_from(objdata,
  64. offset=COFFHEADER.size() +
  65. i * SECTIONHEADER.size())
  66. assert not section_header[0].startswith(b'/') # Support short names only.
  67. section_headers.append(section_header)
  68. if section_header.Name == b'.debug$S':
  69. assert debug_section_index == -1
  70. debug_section_index = i
  71. assert debug_section_index != -1
  72. data_start = COFFHEADER.size() + len(section_headers) * SECTIONHEADER.size()
  73. # Verify the .debug$S section looks like we expect.
  74. assert section_headers[debug_section_index].Name == b'.debug$S'
  75. assert section_headers[debug_section_index].VirtualSize == 0
  76. assert section_headers[debug_section_index].VirtualAddress == 0
  77. debug_size = section_headers[debug_section_index].SizeOfRawData
  78. debug_offset = section_headers[debug_section_index].PointerToRawData
  79. assert section_headers[debug_section_index].PointerToRelocations == 0
  80. assert section_headers[debug_section_index].PointerToLineNumbers == 0
  81. assert section_headers[debug_section_index].NumberOfRelocations == 0
  82. assert section_headers[debug_section_index].NumberOfLineNumbers == 0
  83. # Make sure sections in front of .debug$S have their data preceding it.
  84. for header in section_headers[:debug_section_index]:
  85. assert header.PointerToRawData < debug_offset
  86. assert header.PointerToRelocations < debug_offset
  87. assert header.PointerToLineNumbers < debug_offset
  88. # Make sure sections after of .debug$S have their data following it.
  89. for header in section_headers[debug_section_index + 1:]:
  90. # Make sure the .debug$S data is at the very end of section data:
  91. assert header.PointerToRawData > debug_offset
  92. assert header.PointerToRelocations == 0
  93. assert header.PointerToLineNumbers == 0
  94. # Make sure the first non-empty section's data starts right after the section
  95. # headers.
  96. for section_header in section_headers:
  97. if section_header.PointerToRawData == 0:
  98. assert section_header.PointerToRelocations == 0
  99. assert section_header.PointerToLineNumbers == 0
  100. continue
  101. assert section_header.PointerToRawData == data_start
  102. break
  103. # Make sure the symbol table (and hence, string table) appear after the last
  104. # section:
  105. assert (
  106. coff_header.PointerToSymbolTable >=
  107. section_headers[-1].PointerToRawData + section_headers[-1].SizeOfRawData)
  108. # The symbol table contains a symbol for the no-longer-present .debug$S
  109. # section. If we leave it there, lld-link will complain:
  110. #
  111. # lld-link: error: .debug$S should not refer to non-existent section 5
  112. #
  113. # so we need to remove that symbol table entry as well. This shifts symbol
  114. # entries around and we need to update symbol table indices in:
  115. # - relocations
  116. # - line number records (never present)
  117. # - one aux symbol entry (IMAGE_SYM_CLASS_CLR_TOKEN; not present in ml output)
  118. SYM = Struct(
  119. 'SYM',
  120. '8s',
  121. 'Name',
  122. 'I',
  123. 'Value',
  124. 'h',
  125. 'SectionNumber', # Note: Signed!
  126. 'H',
  127. 'Type',
  128. 'B',
  129. 'StorageClass',
  130. 'B',
  131. 'NumberOfAuxSymbols')
  132. i = 0
  133. debug_sym = -1
  134. while i < coff_header.NumberOfSymbols:
  135. sym_offset = coff_header.PointerToSymbolTable + i * SYM.size()
  136. sym = SYM.unpack_from(objdata, sym_offset)
  137. # 107 is IMAGE_SYM_CLASS_CLR_TOKEN, which has aux entry "CLR Token
  138. # Definition", which contains a symbol index. Check it's never present.
  139. assert sym.StorageClass != 107
  140. # Note: sym.SectionNumber is 1-based, debug_section_index is 0-based.
  141. if sym.SectionNumber - 1 == debug_section_index:
  142. assert debug_sym == -1, 'more than one .debug$S symbol found'
  143. debug_sym = i
  144. # Make sure the .debug$S symbol looks like we expect.
  145. # In particular, it should have exactly one aux symbol.
  146. assert sym.Name == b'.debug$S'
  147. assert sym.Value == 0
  148. assert sym.Type == 0
  149. assert sym.StorageClass == 3
  150. assert sym.NumberOfAuxSymbols == 1
  151. elif sym.SectionNumber > debug_section_index:
  152. sym = Subtract(sym, SectionNumber=1)
  153. SYM.pack_into(objdata, sym_offset, sym)
  154. i += 1 + sym.NumberOfAuxSymbols
  155. assert debug_sym != -1, '.debug$S symbol not found'
  156. # Note: Usually the .debug$S section is the last, but for files saying
  157. # `includelib foo.lib`, like safe_terminate_process.asm in 32-bit builds,
  158. # this isn't true: .drectve is after .debug$S.
  159. # Update symbol table indices in relocations.
  160. # There are a few processor types that have one or two relocation types
  161. # where SymbolTableIndex has a different meaning, but not for x86.
  162. REL = Struct('REL', 'I', 'VirtualAddress', 'I', 'SymbolTableIndex', 'H',
  163. 'Type')
  164. for header in section_headers[0:debug_section_index]:
  165. for j in range(0, header.NumberOfRelocations):
  166. rel_offset = header.PointerToRelocations + j * REL.size()
  167. rel = REL.unpack_from(objdata, rel_offset)
  168. assert rel.SymbolTableIndex != debug_sym
  169. if rel.SymbolTableIndex > debug_sym:
  170. rel = Subtract(rel, SymbolTableIndex=2)
  171. REL.pack_into(objdata, rel_offset, rel)
  172. # Update symbol table indices in line numbers -- just check they don't exist.
  173. for header in section_headers:
  174. assert header.NumberOfLineNumbers == 0
  175. # Now that all indices are updated, remove the symbol table entry referring to
  176. # .debug$S and its aux entry.
  177. del objdata[coff_header.PointerToSymbolTable +
  178. debug_sym * SYM.size():coff_header.PointerToSymbolTable +
  179. (debug_sym + 2) * SYM.size()]
  180. # Now we know that it's safe to write out the input data, with just the
  181. # timestamp overwritten to 0, the last section header cut out (and the
  182. # offsets of all other section headers decremented by the size of that
  183. # one section header), and the last section's data cut out. The symbol
  184. # table offset needs to be reduced by one section header and the size of
  185. # the missing section.
  186. # (The COFF spec only requires on-disk sections to be aligned in image files,
  187. # for obj files it's not required. If that wasn't the case, deleting slices
  188. # if data would not generally be safe.)
  189. # Update section offsets and remove .debug$S section data.
  190. for i in range(0, debug_section_index):
  191. header = section_headers[i]
  192. if header.SizeOfRawData:
  193. header = Subtract(header, PointerToRawData=SECTIONHEADER.size())
  194. if header.NumberOfRelocations:
  195. header = Subtract(header, PointerToRelocations=SECTIONHEADER.size())
  196. if header.NumberOfLineNumbers:
  197. header = Subtract(header, PointerToLineNumbers=SECTIONHEADER.size())
  198. SECTIONHEADER.pack_into(objdata,
  199. COFFHEADER.size() + i * SECTIONHEADER.size(),
  200. header)
  201. for i in range(debug_section_index + 1, len(section_headers)):
  202. header = section_headers[i]
  203. shift = SECTIONHEADER.size() + debug_size
  204. if header.SizeOfRawData:
  205. header = Subtract(header, PointerToRawData=shift)
  206. if header.NumberOfRelocations:
  207. header = Subtract(header, PointerToRelocations=shift)
  208. if header.NumberOfLineNumbers:
  209. header = Subtract(header, PointerToLineNumbers=shift)
  210. SECTIONHEADER.pack_into(objdata,
  211. COFFHEADER.size() + i * SECTIONHEADER.size(),
  212. header)
  213. del objdata[debug_offset:debug_offset + debug_size]
  214. # Finally, remove .debug$S section header and update coff header.
  215. coff_header = coff_header._replace(TimeDateStamp=0)
  216. coff_header = Subtract(coff_header,
  217. NumberOfSections=1,
  218. PointerToSymbolTable=SECTIONHEADER.size() + debug_size,
  219. NumberOfSymbols=2)
  220. COFFHEADER.pack_into(objdata, 0, coff_header)
  221. del objdata[COFFHEADER.size() +
  222. debug_section_index * SECTIONHEADER.size():COFFHEADER.size() +
  223. (debug_section_index + 1) * SECTIONHEADER.size()]
  224. # All done!
  225. if sys.version_info.major == 2:
  226. return objdata.tostring()
  227. else:
  228. return objdata.tobytes()
  229. def main():
  230. ml_result = subprocess.call(sys.argv[1:])
  231. if ml_result != 0:
  232. return ml_result
  233. objfile = None
  234. for i in range(1, len(sys.argv)):
  235. if sys.argv[i].startswith('/Fo'):
  236. objfile = sys.argv[i][len('/Fo'):]
  237. assert objfile, 'failed to find ml output'
  238. with open(objfile, 'rb') as f:
  239. objdata = f.read()
  240. objdata = MakeDeterministic(objdata)
  241. with open(objfile, 'wb') as f:
  242. f.write(objdata)
  243. if __name__ == '__main__':
  244. sys.exit(main())