123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286 |
- #!/usr/bin/env python
- # Copyright 2018 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Wraps ml.exe or ml64.exe and postprocesses the output to be deterministic.
- Sets timestamp in .obj file to 0, hence incompatible with link.exe /incremental.
- Use by prefixing the ml(64).exe invocation with this script:
- python ml.py ml.exe [args...]"""
- import array
- import collections
- import struct
- import subprocess
- import sys
- class Struct(object):
- """A thin wrapper around the struct module that returns a namedtuple"""
- def __init__(self, name, *args):
- """Pass the name of the return type, and then an interleaved list of
- format strings as used by the struct module and of field names."""
- self.fmt = '<' + ''.join(args[0::2])
- self.type = collections.namedtuple(name, args[1::2])
- def pack_into(self, buffer, offset, data):
- return struct.pack_into(self.fmt, buffer, offset, *data)
- def unpack_from(self, buffer, offset=0):
- return self.type(*struct.unpack_from(self.fmt, buffer, offset))
- def size(self):
- return struct.calcsize(self.fmt)
- def Subtract(nt, **kwargs):
- """Subtract(nt, f=2) returns a new namedtuple with 2 subtracted from nt.f"""
- return nt._replace(**{k: getattr(nt, k) - v for k, v in kwargs.items()})
- def MakeDeterministic(objdata):
- # Takes data produced by ml(64).exe (without any special flags) and
- # 1. Sets the timestamp to 0
- # 2. Strips the .debug$S section (which contains an unwanted absolute path)
- # This makes several assumptions about ml's output:
- # - Section data is in the same order as the corresponding section headers:
- # section headers preceding the .debug$S section header have their data
- # preceding the .debug$S section data; likewise for section headers
- # following the .debug$S section.
- # - The .debug$S section contains only the absolute path to the obj file and
- # nothing else, in particular there's only a single entry in the symbol
- # table referring to the .debug$S section.
- # - There are no COFF line number entries.
- # - There's no IMAGE_SYM_CLASS_CLR_TOKEN symbol.
- # These seem to hold in practice; if they stop holding this script needs to
- # become smarter.
- objdata = array.array('b', objdata) # Writable, e.g. via struct.pack_into.
- # Read coff header.
- COFFHEADER = Struct('COFFHEADER', 'H', 'Machine', 'H', 'NumberOfSections',
- 'I', 'TimeDateStamp', 'I', 'PointerToSymbolTable', 'I',
- 'NumberOfSymbols', 'H', 'SizeOfOptionalHeader', 'H',
- 'Characteristics')
- coff_header = COFFHEADER.unpack_from(objdata)
- assert coff_header.SizeOfOptionalHeader == 0 # Only set for binaries.
- # Read section headers following coff header.
- SECTIONHEADER = Struct('SECTIONHEADER', '8s', 'Name', 'I', 'VirtualSize', 'I',
- 'VirtualAddress', 'I', 'SizeOfRawData', 'I',
- 'PointerToRawData', 'I', 'PointerToRelocations', 'I',
- 'PointerToLineNumbers', 'H', 'NumberOfRelocations',
- 'H', 'NumberOfLineNumbers', 'I', 'Characteristics')
- section_headers = []
- debug_section_index = -1
- for i in range(0, coff_header.NumberOfSections):
- section_header = SECTIONHEADER.unpack_from(objdata,
- offset=COFFHEADER.size() +
- i * SECTIONHEADER.size())
- assert not section_header[0].startswith(b'/') # Support short names only.
- section_headers.append(section_header)
- if section_header.Name == b'.debug$S':
- assert debug_section_index == -1
- debug_section_index = i
- assert debug_section_index != -1
- data_start = COFFHEADER.size() + len(section_headers) * SECTIONHEADER.size()
- # Verify the .debug$S section looks like we expect.
- assert section_headers[debug_section_index].Name == b'.debug$S'
- assert section_headers[debug_section_index].VirtualSize == 0
- assert section_headers[debug_section_index].VirtualAddress == 0
- debug_size = section_headers[debug_section_index].SizeOfRawData
- debug_offset = section_headers[debug_section_index].PointerToRawData
- assert section_headers[debug_section_index].PointerToRelocations == 0
- assert section_headers[debug_section_index].PointerToLineNumbers == 0
- assert section_headers[debug_section_index].NumberOfRelocations == 0
- assert section_headers[debug_section_index].NumberOfLineNumbers == 0
- # Make sure sections in front of .debug$S have their data preceding it.
- for header in section_headers[:debug_section_index]:
- assert header.PointerToRawData < debug_offset
- assert header.PointerToRelocations < debug_offset
- assert header.PointerToLineNumbers < debug_offset
- # Make sure sections after of .debug$S have their data following it.
- for header in section_headers[debug_section_index + 1:]:
- # Make sure the .debug$S data is at the very end of section data:
- assert header.PointerToRawData > debug_offset
- assert header.PointerToRelocations == 0
- assert header.PointerToLineNumbers == 0
- # Make sure the first non-empty section's data starts right after the section
- # headers.
- for section_header in section_headers:
- if section_header.PointerToRawData == 0:
- assert section_header.PointerToRelocations == 0
- assert section_header.PointerToLineNumbers == 0
- continue
- assert section_header.PointerToRawData == data_start
- break
- # Make sure the symbol table (and hence, string table) appear after the last
- # section:
- assert (
- coff_header.PointerToSymbolTable >=
- section_headers[-1].PointerToRawData + section_headers[-1].SizeOfRawData)
- # The symbol table contains a symbol for the no-longer-present .debug$S
- # section. If we leave it there, lld-link will complain:
- #
- # lld-link: error: .debug$S should not refer to non-existent section 5
- #
- # so we need to remove that symbol table entry as well. This shifts symbol
- # entries around and we need to update symbol table indices in:
- # - relocations
- # - line number records (never present)
- # - one aux symbol entry (IMAGE_SYM_CLASS_CLR_TOKEN; not present in ml output)
- SYM = Struct(
- 'SYM',
- '8s',
- 'Name',
- 'I',
- 'Value',
- 'h',
- 'SectionNumber', # Note: Signed!
- 'H',
- 'Type',
- 'B',
- 'StorageClass',
- 'B',
- 'NumberOfAuxSymbols')
- i = 0
- debug_sym = -1
- while i < coff_header.NumberOfSymbols:
- sym_offset = coff_header.PointerToSymbolTable + i * SYM.size()
- sym = SYM.unpack_from(objdata, sym_offset)
- # 107 is IMAGE_SYM_CLASS_CLR_TOKEN, which has aux entry "CLR Token
- # Definition", which contains a symbol index. Check it's never present.
- assert sym.StorageClass != 107
- # Note: sym.SectionNumber is 1-based, debug_section_index is 0-based.
- if sym.SectionNumber - 1 == debug_section_index:
- assert debug_sym == -1, 'more than one .debug$S symbol found'
- debug_sym = i
- # Make sure the .debug$S symbol looks like we expect.
- # In particular, it should have exactly one aux symbol.
- assert sym.Name == b'.debug$S'
- assert sym.Value == 0
- assert sym.Type == 0
- assert sym.StorageClass == 3
- assert sym.NumberOfAuxSymbols == 1
- elif sym.SectionNumber > debug_section_index:
- sym = Subtract(sym, SectionNumber=1)
- SYM.pack_into(objdata, sym_offset, sym)
- i += 1 + sym.NumberOfAuxSymbols
- assert debug_sym != -1, '.debug$S symbol not found'
- # Note: Usually the .debug$S section is the last, but for files saying
- # `includelib foo.lib`, like safe_terminate_process.asm in 32-bit builds,
- # this isn't true: .drectve is after .debug$S.
- # Update symbol table indices in relocations.
- # There are a few processor types that have one or two relocation types
- # where SymbolTableIndex has a different meaning, but not for x86.
- REL = Struct('REL', 'I', 'VirtualAddress', 'I', 'SymbolTableIndex', 'H',
- 'Type')
- for header in section_headers[0:debug_section_index]:
- for j in range(0, header.NumberOfRelocations):
- rel_offset = header.PointerToRelocations + j * REL.size()
- rel = REL.unpack_from(objdata, rel_offset)
- assert rel.SymbolTableIndex != debug_sym
- if rel.SymbolTableIndex > debug_sym:
- rel = Subtract(rel, SymbolTableIndex=2)
- REL.pack_into(objdata, rel_offset, rel)
- # Update symbol table indices in line numbers -- just check they don't exist.
- for header in section_headers:
- assert header.NumberOfLineNumbers == 0
- # Now that all indices are updated, remove the symbol table entry referring to
- # .debug$S and its aux entry.
- del objdata[coff_header.PointerToSymbolTable +
- debug_sym * SYM.size():coff_header.PointerToSymbolTable +
- (debug_sym + 2) * SYM.size()]
- # Now we know that it's safe to write out the input data, with just the
- # timestamp overwritten to 0, the last section header cut out (and the
- # offsets of all other section headers decremented by the size of that
- # one section header), and the last section's data cut out. The symbol
- # table offset needs to be reduced by one section header and the size of
- # the missing section.
- # (The COFF spec only requires on-disk sections to be aligned in image files,
- # for obj files it's not required. If that wasn't the case, deleting slices
- # if data would not generally be safe.)
- # Update section offsets and remove .debug$S section data.
- for i in range(0, debug_section_index):
- header = section_headers[i]
- if header.SizeOfRawData:
- header = Subtract(header, PointerToRawData=SECTIONHEADER.size())
- if header.NumberOfRelocations:
- header = Subtract(header, PointerToRelocations=SECTIONHEADER.size())
- if header.NumberOfLineNumbers:
- header = Subtract(header, PointerToLineNumbers=SECTIONHEADER.size())
- SECTIONHEADER.pack_into(objdata,
- COFFHEADER.size() + i * SECTIONHEADER.size(),
- header)
- for i in range(debug_section_index + 1, len(section_headers)):
- header = section_headers[i]
- shift = SECTIONHEADER.size() + debug_size
- if header.SizeOfRawData:
- header = Subtract(header, PointerToRawData=shift)
- if header.NumberOfRelocations:
- header = Subtract(header, PointerToRelocations=shift)
- if header.NumberOfLineNumbers:
- header = Subtract(header, PointerToLineNumbers=shift)
- SECTIONHEADER.pack_into(objdata,
- COFFHEADER.size() + i * SECTIONHEADER.size(),
- header)
- del objdata[debug_offset:debug_offset + debug_size]
- # Finally, remove .debug$S section header and update coff header.
- coff_header = coff_header._replace(TimeDateStamp=0)
- coff_header = Subtract(coff_header,
- NumberOfSections=1,
- PointerToSymbolTable=SECTIONHEADER.size() + debug_size,
- NumberOfSymbols=2)
- COFFHEADER.pack_into(objdata, 0, coff_header)
- del objdata[COFFHEADER.size() +
- debug_section_index * SECTIONHEADER.size():COFFHEADER.size() +
- (debug_section_index + 1) * SECTIONHEADER.size()]
- # All done!
- if sys.version_info.major == 2:
- return objdata.tostring()
- else:
- return objdata.tobytes()
- def main():
- ml_result = subprocess.call(sys.argv[1:])
- if ml_result != 0:
- return ml_result
- objfile = None
- for i in range(1, len(sys.argv)):
- if sys.argv[i].startswith('/Fo'):
- objfile = sys.argv[i][len('/Fo'):]
- assert objfile, 'failed to find ml output'
- with open(objfile, 'rb') as f:
- objdata = f.read()
- objdata = MakeDeterministic(objdata)
- with open(objfile, 'wb') as f:
- f.write(objdata)
- if __name__ == '__main__':
- sys.exit(main())
|