123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745 |
- # SPDX-License-Identifier: GPL-2.0+
- # Copyright (c) 2018 Google, Inc
- # Written by Simon Glass <sjg@chromium.org>
- """Entry-type module for sections (groups of entries)
- Sections are entries which can contain other entries. This allows hierarchical
- images to be created.
- """
- from collections import OrderedDict
- import concurrent.futures
- import re
- import sys
- from binman.entry import Entry
- from binman import state
- from dtoc import fdt_util
- from patman import tools
- from patman import tout
- from patman.tools import ToHexSize
- class Entry_section(Entry):
- """Entry that contains other entries
- Properties / Entry arguments: (see binman README for more information):
- pad-byte: Pad byte to use when padding
- sort-by-offset: True if entries should be sorted by offset, False if
- they must be in-order in the device tree description
- end-at-4gb: Used to build an x86 ROM which ends at 4GB (2^32)
- skip-at-start: Number of bytes before the first entry starts. These
- effectively adjust the starting offset of entries. For example,
- if this is 16, then the first entry would start at 16. An entry
- with offset = 20 would in fact be written at offset 4 in the image
- file, since the first 16 bytes are skipped when writing.
- name-prefix: Adds a prefix to the name of every entry in the section
- when writing out the map
- align_default: Default alignment for this section, if no alignment is
- given in the entry
- Properties:
- allow_missing: True if this section permits external blobs to be
- missing their contents. The second will produce an image but of
- course it will not work.
- Since a section is also an entry, it inherits all the properies of entries
- too.
- A section is an entry which can contain other entries, thus allowing
- hierarchical images to be created. See 'Sections and hierarchical images'
- in the binman README for more information.
- """
- def __init__(self, section, etype, node, test=False):
- if not test:
- super().__init__(section, etype, node)
- self._entries = OrderedDict()
- self._pad_byte = 0
- self._sort = False
- self._skip_at_start = None
- self._end_4gb = False
- def ReadNode(self):
- """Read properties from the section node"""
- super().ReadNode()
- self._pad_byte = fdt_util.GetInt(self._node, 'pad-byte', 0)
- self._sort = fdt_util.GetBool(self._node, 'sort-by-offset')
- self._end_4gb = fdt_util.GetBool(self._node, 'end-at-4gb')
- self._skip_at_start = fdt_util.GetInt(self._node, 'skip-at-start')
- if self._end_4gb:
- if not self.size:
- self.Raise("Section size must be provided when using end-at-4gb")
- if self._skip_at_start is not None:
- self.Raise("Provide either 'end-at-4gb' or 'skip-at-start'")
- else:
- self._skip_at_start = 0x100000000 - self.size
- else:
- if self._skip_at_start is None:
- self._skip_at_start = 0
- self._name_prefix = fdt_util.GetString(self._node, 'name-prefix')
- self.align_default = fdt_util.GetInt(self._node, 'align-default', 0)
- filename = fdt_util.GetString(self._node, 'filename')
- if filename:
- self._filename = filename
- self._ReadEntries()
- def _ReadEntries(self):
- for node in self._node.subnodes:
- if node.name.startswith('hash') or node.name.startswith('signature'):
- continue
- entry = Entry.Create(self, node,
- expanded=self.GetImage().use_expanded)
- entry.ReadNode()
- entry.SetPrefix(self._name_prefix)
- self._entries[node.name] = entry
- def _Raise(self, msg):
- """Raises an error for this section
- Args:
- msg: Error message to use in the raise string
- Raises:
- ValueError()
- """
- raise ValueError("Section '%s': %s" % (self._node.path, msg))
- def GetFdts(self):
- fdts = {}
- for entry in self._entries.values():
- fdts.update(entry.GetFdts())
- return fdts
- def ProcessFdt(self, fdt):
- """Allow entries to adjust the device tree
- Some entries need to adjust the device tree for their purposes. This
- may involve adding or deleting properties.
- """
- todo = self._entries.values()
- for passnum in range(3):
- next_todo = []
- for entry in todo:
- if not entry.ProcessFdt(fdt):
- next_todo.append(entry)
- todo = next_todo
- if not todo:
- break
- if todo:
- self.Raise('Internal error: Could not complete processing of Fdt: remaining %s' %
- todo)
- return True
- def ExpandEntries(self):
- super().ExpandEntries()
- for entry in self._entries.values():
- entry.ExpandEntries()
- def AddMissingProperties(self, have_image_pos):
- """Add new properties to the device tree as needed for this entry"""
- super().AddMissingProperties(have_image_pos)
- if self.compress != 'none':
- have_image_pos = False
- for entry in self._entries.values():
- entry.AddMissingProperties(have_image_pos)
- def ObtainContents(self):
- return self.GetEntryContents()
- def GetPaddedDataForEntry(self, entry, entry_data):
- """Get the data for an entry including any padding
- Gets the entry data and uses the section pad-byte value to add padding
- before and after as defined by the pad-before and pad-after properties.
- This does not consider alignment.
- Args:
- entry: Entry to check
- Returns:
- Contents of the entry along with any pad bytes before and
- after it (bytes)
- """
- pad_byte = (entry._pad_byte if isinstance(entry, Entry_section)
- else self._pad_byte)
- data = b''
- # Handle padding before the entry
- if entry.pad_before:
- data += tools.GetBytes(self._pad_byte, entry.pad_before)
- # Add in the actual entry data
- data += entry_data
- # Handle padding after the entry
- if entry.pad_after:
- data += tools.GetBytes(self._pad_byte, entry.pad_after)
- if entry.size:
- data += tools.GetBytes(pad_byte, entry.size - len(data))
- self.Detail('GetPaddedDataForEntry: size %s' % ToHexSize(self.data))
- return data
- def _BuildSectionData(self, required):
- """Build the contents of a section
- This places all entries at the right place, dealing with padding before
- and after entries. It does not do padding for the section itself (the
- pad-before and pad-after properties in the section items) since that is
- handled by the parent section.
- Args:
- required: True if the data must be present, False if it is OK to
- return None
- Returns:
- Contents of the section (bytes)
- """
- section_data = b''
- for entry in self._entries.values():
- entry_data = entry.GetData(required)
- if not required and entry_data is None:
- return None
- data = self.GetPaddedDataForEntry(entry, entry_data)
- # Handle empty space before the entry
- pad = (entry.offset or 0) - self._skip_at_start - len(section_data)
- if pad > 0:
- section_data += tools.GetBytes(self._pad_byte, pad)
- # Add in the actual entry data
- section_data += data
- self.Detail('GetData: %d entries, total size %#x' %
- (len(self._entries), len(section_data)))
- return self.CompressData(section_data)
- def GetPaddedData(self, data=None):
- """Get the data for a section including any padding
- Gets the section data and uses the parent section's pad-byte value to
- add padding before and after as defined by the pad-before and pad-after
- properties. If this is a top-level section (i.e. an image), this is the
- same as GetData(), since padding is not supported.
- This does not consider alignment.
- Returns:
- Contents of the section along with any pad bytes before and
- after it (bytes)
- """
- section = self.section or self
- if data is None:
- data = self.GetData()
- return section.GetPaddedDataForEntry(self, data)
- def GetData(self, required=True):
- """Get the contents of an entry
- This builds the contents of the section, stores this as the contents of
- the section and returns it
- Args:
- required: True if the data must be present, False if it is OK to
- return None
- Returns:
- bytes content of the section, made up for all all of its subentries.
- This excludes any padding. If the section is compressed, the
- compressed data is returned
- """
- data = self._BuildSectionData(required)
- if data is None:
- return None
- self.SetContents(data)
- return data
- def GetOffsets(self):
- """Handle entries that want to set the offset/size of other entries
- This calls each entry's GetOffsets() method. If it returns a list
- of entries to update, it updates them.
- """
- self.GetEntryOffsets()
- return {}
- def ResetForPack(self):
- """Reset offset/size fields so that packing can be done again"""
- super().ResetForPack()
- for entry in self._entries.values():
- entry.ResetForPack()
- def Pack(self, offset):
- """Pack all entries into the section"""
- self._PackEntries()
- if self._sort:
- self._SortEntries()
- self._ExpandEntries()
- data = self._BuildSectionData(True)
- self.SetContents(data)
- self.CheckSize()
- offset = super().Pack(offset)
- self.CheckEntries()
- return offset
- def _PackEntries(self):
- """Pack all entries into the section"""
- offset = self._skip_at_start
- for entry in self._entries.values():
- offset = entry.Pack(offset)
- return offset
- def _ExpandEntries(self):
- """Expand any entries that are permitted to"""
- exp_entry = None
- for entry in self._entries.values():
- if exp_entry:
- exp_entry.ExpandToLimit(entry.offset)
- exp_entry = None
- if entry.expand_size:
- exp_entry = entry
- if exp_entry:
- exp_entry.ExpandToLimit(self.size)
- def _SortEntries(self):
- """Sort entries by offset"""
- entries = sorted(self._entries.values(), key=lambda entry: entry.offset)
- self._entries.clear()
- for entry in entries:
- self._entries[entry._node.name] = entry
- def CheckEntries(self):
- """Check that entries do not overlap or extend outside the section"""
- max_size = self.size if self.uncomp_size is None else self.uncomp_size
- offset = 0
- prev_name = 'None'
- for entry in self._entries.values():
- entry.CheckEntries()
- if (entry.offset < self._skip_at_start or
- entry.offset + entry.size > self._skip_at_start +
- max_size):
- entry.Raise('Offset %#x (%d) size %#x (%d) is outside the '
- "section '%s' starting at %#x (%d) "
- 'of size %#x (%d)' %
- (entry.offset, entry.offset, entry.size, entry.size,
- self._node.path, self._skip_at_start,
- self._skip_at_start, max_size, max_size))
- if entry.offset < offset and entry.size:
- entry.Raise("Offset %#x (%d) overlaps with previous entry '%s' "
- "ending at %#x (%d)" %
- (entry.offset, entry.offset, prev_name, offset, offset))
- offset = entry.offset + entry.size
- prev_name = entry.GetPath()
- def WriteSymbols(self, section):
- """Write symbol values into binary files for access at run time"""
- for entry in self._entries.values():
- entry.WriteSymbols(self)
- def SetCalculatedProperties(self):
- super().SetCalculatedProperties()
- for entry in self._entries.values():
- entry.SetCalculatedProperties()
- def SetImagePos(self, image_pos):
- super().SetImagePos(image_pos)
- if self.compress == 'none':
- for entry in self._entries.values():
- entry.SetImagePos(image_pos + self.offset)
- def ProcessContents(self):
- sizes_ok_base = super(Entry_section, self).ProcessContents()
- sizes_ok = True
- for entry in self._entries.values():
- if not entry.ProcessContents():
- sizes_ok = False
- return sizes_ok and sizes_ok_base
- def WriteMap(self, fd, indent):
- """Write a map of the section to a .map file
- Args:
- fd: File to write the map to
- """
- Entry.WriteMapLine(fd, indent, self.name, self.offset or 0,
- self.size, self.image_pos)
- for entry in self._entries.values():
- entry.WriteMap(fd, indent + 1)
- def GetEntries(self):
- return self._entries
- def GetContentsByPhandle(self, phandle, source_entry, required):
- """Get the data contents of an entry specified by a phandle
- This uses a phandle to look up a node and and find the entry
- associated with it. Then it returns the contents of that entry.
- The node must be a direct subnode of this section.
- Args:
- phandle: Phandle to look up (integer)
- source_entry: Entry containing that phandle (used for error
- reporting)
- required: True if the data must be present, False if it is OK to
- return None
- Returns:
- data from associated entry (as a string), or None if not found
- """
- node = self._node.GetFdt().LookupPhandle(phandle)
- if not node:
- source_entry.Raise("Cannot find node for phandle %d" % phandle)
- for entry in self._entries.values():
- if entry._node == node:
- return entry.GetData(required)
- source_entry.Raise("Cannot find entry for node '%s'" % node.name)
- def LookupSymbol(self, sym_name, optional, msg, base_addr, entries=None):
- """Look up a symbol in an ELF file
- Looks up a symbol in an ELF file. Only entry types which come from an
- ELF image can be used by this function.
- At present the only entry properties supported are:
- offset
- image_pos - 'base_addr' is added if this is not an end-at-4gb image
- size
- Args:
- sym_name: Symbol name in the ELF file to look up in the format
- _binman_<entry>_prop_<property> where <entry> is the name of
- the entry and <property> is the property to find (e.g.
- _binman_u_boot_prop_offset). As a special case, you can append
- _any to <entry> to have it search for any matching entry. E.g.
- _binman_u_boot_any_prop_offset will match entries called u-boot,
- u-boot-img and u-boot-nodtb)
- optional: True if the symbol is optional. If False this function
- will raise if the symbol is not found
- msg: Message to display if an error occurs
- base_addr: Base address of image. This is added to the returned
- image_pos in most cases so that the returned position indicates
- where the targetted entry/binary has actually been loaded. But
- if end-at-4gb is used, this is not done, since the binary is
- already assumed to be linked to the ROM position and using
- execute-in-place (XIP).
- Returns:
- Value that should be assigned to that symbol, or None if it was
- optional and not found
- Raises:
- ValueError if the symbol is invalid or not found, or references a
- property which is not supported
- """
- m = re.match(r'^_binman_(\w+)_prop_(\w+)$', sym_name)
- if not m:
- raise ValueError("%s: Symbol '%s' has invalid format" %
- (msg, sym_name))
- entry_name, prop_name = m.groups()
- entry_name = entry_name.replace('_', '-')
- if not entries:
- entries = self._entries
- entry = entries.get(entry_name)
- if not entry:
- if entry_name.endswith('-any'):
- root = entry_name[:-4]
- for name in entries:
- if name.startswith(root):
- rest = name[len(root):]
- if rest in ['', '-img', '-nodtb']:
- entry = entries[name]
- if not entry:
- err = ("%s: Entry '%s' not found in list (%s)" %
- (msg, entry_name, ','.join(entries.keys())))
- if optional:
- print('Warning: %s' % err, file=sys.stderr)
- return None
- raise ValueError(err)
- if prop_name == 'offset':
- return entry.offset
- elif prop_name == 'image_pos':
- value = entry.image_pos
- if not self.GetImage()._end_4gb:
- value += base_addr
- return value
- if prop_name == 'size':
- return entry.size
- else:
- raise ValueError("%s: No such property '%s'" % (msg, prop_name))
- def GetRootSkipAtStart(self):
- """Get the skip-at-start value for the top-level section
- This is used to find out the starting offset for root section that
- contains this section. If this is a top-level section then it returns
- the skip-at-start offset for this section.
- This is used to get the absolute position of section within the image.
- Returns:
- Integer skip-at-start value for the root section containing this
- section
- """
- if self.section:
- return self.section.GetRootSkipAtStart()
- return self._skip_at_start
- def GetStartOffset(self):
- """Get the start offset for this section
- Returns:
- The first available offset in this section (typically 0)
- """
- return self._skip_at_start
- def GetImageSize(self):
- """Get the size of the image containing this section
- Returns:
- Image size as an integer number of bytes, which may be None if the
- image size is dynamic and its sections have not yet been packed
- """
- return self.GetImage().size
- def FindEntryType(self, etype):
- """Find an entry type in the section
- Args:
- etype: Entry type to find
- Returns:
- entry matching that type, or None if not found
- """
- for entry in self._entries.values():
- if entry.etype == etype:
- return entry
- return None
- def GetEntryContents(self):
- """Call ObtainContents() for each entry in the section
- """
- def _CheckDone(entry):
- if not entry.ObtainContents():
- next_todo.append(entry)
- return entry
- todo = self._entries.values()
- for passnum in range(3):
- threads = state.GetThreads()
- next_todo = []
- if threads == 0:
- for entry in todo:
- _CheckDone(entry)
- else:
- with concurrent.futures.ThreadPoolExecutor(
- max_workers=threads) as executor:
- future_to_data = {
- entry: executor.submit(_CheckDone, entry)
- for entry in todo}
- timeout = 60
- if self.GetImage().test_section_timeout:
- timeout = 0
- done, not_done = concurrent.futures.wait(
- future_to_data.values(), timeout=timeout)
- # Make sure we check the result, so any exceptions are
- # generated. Check the results in entry order, since tests
- # may expect earlier entries to fail first.
- for entry in todo:
- job = future_to_data[entry]
- job.result()
- if not_done:
- self.Raise('Timed out obtaining contents')
- todo = next_todo
- if not todo:
- break
- if todo:
- self.Raise('Internal error: Could not complete processing of contents: remaining %s' %
- todo)
- return True
- def _SetEntryOffsetSize(self, name, offset, size):
- """Set the offset and size of an entry
- Args:
- name: Entry name to update
- offset: New offset, or None to leave alone
- size: New size, or None to leave alone
- """
- entry = self._entries.get(name)
- if not entry:
- self._Raise("Unable to set offset/size for unknown entry '%s'" %
- name)
- entry.SetOffsetSize(self._skip_at_start + offset if offset is not None
- else None, size)
- def GetEntryOffsets(self):
- """Handle entries that want to set the offset/size of other entries
- This calls each entry's GetOffsets() method. If it returns a list
- of entries to update, it updates them.
- """
- for entry in self._entries.values():
- offset_dict = entry.GetOffsets()
- for name, info in offset_dict.items():
- self._SetEntryOffsetSize(name, *info)
- def CheckSize(self):
- contents_size = len(self.data)
- size = self.size
- if not size:
- data = self.GetPaddedData(self.data)
- size = len(data)
- size = tools.Align(size, self.align_size)
- if self.size and contents_size > self.size:
- self._Raise("contents size %#x (%d) exceeds section size %#x (%d)" %
- (contents_size, contents_size, self.size, self.size))
- if not self.size:
- self.size = size
- if self.size != tools.Align(self.size, self.align_size):
- self._Raise("Size %#x (%d) does not match align-size %#x (%d)" %
- (self.size, self.size, self.align_size,
- self.align_size))
- return size
- def ListEntries(self, entries, indent):
- """List the files in the section"""
- Entry.AddEntryInfo(entries, indent, self.name, 'section', self.size,
- self.image_pos, None, self.offset, self)
- for entry in self._entries.values():
- entry.ListEntries(entries, indent + 1)
- def LoadData(self, decomp=True):
- for entry in self._entries.values():
- entry.LoadData(decomp)
- self.Detail('Loaded data')
- def GetImage(self):
- """Get the image containing this section
- Note that a top-level section is actually an Image, so this function may
- return self.
- Returns:
- Image object containing this section
- """
- if not self.section:
- return self
- return self.section.GetImage()
- def GetSort(self):
- """Check if the entries in this section will be sorted
- Returns:
- True if to be sorted, False if entries will be left in the order
- they appear in the device tree
- """
- return self._sort
- def ReadData(self, decomp=True):
- tout.Info("ReadData path='%s'" % self.GetPath())
- parent_data = self.section.ReadData(True)
- offset = self.offset - self.section._skip_at_start
- data = parent_data[offset:offset + self.size]
- tout.Info(
- '%s: Reading data from offset %#x-%#x (real %#x), size %#x, got %#x' %
- (self.GetPath(), self.offset, self.offset + self.size, offset,
- self.size, len(data)))
- return data
- def ReadChildData(self, child, decomp=True):
- tout.Debug("ReadChildData for child '%s'" % child.GetPath())
- parent_data = self.ReadData(True)
- offset = child.offset - self._skip_at_start
- tout.Debug("Extract for child '%s': offset %#x, skip_at_start %#x, result %#x" %
- (child.GetPath(), child.offset, self._skip_at_start, offset))
- data = parent_data[offset:offset + child.size]
- if decomp:
- indata = data
- data = tools.Decompress(indata, child.compress)
- if child.uncomp_size:
- tout.Info("%s: Decompressing data size %#x with algo '%s' to data size %#x" %
- (child.GetPath(), len(indata), child.compress,
- len(data)))
- return data
- def WriteChildData(self, child):
- return True
- def SetAllowMissing(self, allow_missing):
- """Set whether a section allows missing external blobs
- Args:
- allow_missing: True if allowed, False if not allowed
- """
- self.allow_missing = allow_missing
- for entry in self._entries.values():
- entry.SetAllowMissing(allow_missing)
- def CheckMissing(self, missing_list):
- """Check if any entries in this section have missing external blobs
- If there are missing blobs, the entries are added to the list
- Args:
- missing_list: List of Entry objects to be added to
- """
- for entry in self._entries.values():
- entry.CheckMissing(missing_list)
- def _CollectEntries(self, entries, entries_by_name, add_entry):
- """Collect all the entries in an section
- This builds up a dict of entries in this section and all subsections.
- Entries are indexed by path and by name.
- Since all paths are unique, entries will not have any conflicts. However
- entries_by_name make have conflicts if two entries have the same name
- (e.g. with different parent sections). In this case, an entry at a
- higher level in the hierarchy will win over a lower-level entry.
- Args:
- entries: dict to put entries:
- key: entry path
- value: Entry object
- entries_by_name: dict to put entries
- key: entry name
- value: Entry object
- add_entry: Entry to add
- """
- entries[add_entry.GetPath()] = add_entry
- to_add = add_entry.GetEntries()
- if to_add:
- for entry in to_add.values():
- entries[entry.GetPath()] = entry
- for entry in to_add.values():
- self._CollectEntries(entries, entries_by_name, entry)
- entries_by_name[add_entry.name] = add_entry
- def MissingArgs(self, entry, missing):
- """Report a missing argument, if enabled
- For entries which require arguments, this reports an error if some are
- missing. If missing entries are being ignored (e.g. because we read the
- entry from an image rather than creating it), this function does
- nothing.
- Args:
- missing: List of missing properties / entry args, each a string
- """
- if not self._ignore_missing:
- entry.Raise('Missing required properties/entry args: %s' %
- (', '.join(missing)))
|