diff options
Diffstat (limited to 'tools')
71 files changed, 1724 insertions, 338 deletions
diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 771645380e..935839c433 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -480,8 +480,8 @@ image-pos: for each entry. This makes it easy to find out exactly where the entry ended up in the image, regardless of parent sections, etc. -expand-size: - Expand the size of this entry to fit available space. This space is only +extend-size: + Extend the size of this entry to fit available space. This space is only limited by the size of the image/section and the position of the next entry. @@ -1375,18 +1375,20 @@ Some entry types deal with data obtained from others. For example, }; This shows mkimage being passed a file consisting of SPL and U-Boot proper. It -is create by calling `Entry.collect_contents_to_file()`. Note that in this case, -the data is passed to mkimage for processing but does not appear separately in -the image. It may not appear at all, depending on what mkimage does. The -contents of the `mkimage` entry are entirely dependent on the processing done -by the entry, with the provided subnodes (`u-boot-spl` and `u-boot`) simply -providing the input data for that processing. +is created by calling `Entry.collect_contents_to_file()`. Note that in this +case, the data is passed to mkimage for processing but does not appear +separately in the image. It may not appear at all, depending on what mkimage +does. The contents of the `mkimage` entry are entirely dependent on the +processing done by the entry, with the provided subnodes (`u-boot-spl` and +`u-boot`) simply providing the input data for that processing. Note that `Entry.collect_contents_to_file()` simply concatenates the data from the different entries together, with no control over alignment, etc. Another approach is to subclass `Entry_section` so that those features become available, such as `size` and `pad-byte`. Then the contents of the entry can be obtained by -calling `BuildSectionData()`. +calling `super().BuildSectionData()` in the entry's BuildSectionData() +implementation to get the input data, then write it to a file and process it +however is desired. There are other ways to obtain data also, depending on the situation. If the entry type is simply signing data which exists elsewhere in the image, then @@ -1396,6 +1398,7 @@ is used by `Entry_vblock`, for example:: u_boot: u-boot { }; + vblock { content = <&u_boot &dtb>; keyblock = "firmware.keyblock"; @@ -1440,9 +1443,11 @@ The `soc-fw` node is a `blob-ext` (i.e. it reads in a named binary file) whereas a known blob type. When adding new entry types you are encouraged to use subnodes to provide the -data for processing, unless the `content` approach is more suitable. Ad-hoc -properties and other methods of obtaining data are discouraged, since it adds to -confusion for users. +data for processing, unless the `content` approach is more suitable. Consider +whether the input entries are contained within (or consumed by) the entry, vs +just being 'referenced' by the entry. In the latter case, the `content` approach +makes more sense. Ad-hoc properties and other methods of obtaining data are +discouraged, since it adds to confusion for users. History / Credits ----------------- @@ -1495,7 +1500,8 @@ Some ideas: - Figure out how to make Fdt support changing the node order, so that Node.AddSubnode() can support adding a node before another, existing node. Perhaps it should completely regenerate the flat tree? - +- Put faked files into a separate subdir and remove them on start-up, to avoid + seeing them as 'real' files on a subsequent run -- Simon Glass <sjg@chromium.org> diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 0626b850f4..1d1ca43993 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -7,7 +7,7 @@ import argparse from argparse import ArgumentParser -import state +from binman import state def make_extract_parser(subparsers): """make_extract_parser: Make a subparser for the 'extract' command diff --git a/tools/binman/control.py b/tools/binman/control.py index a179f78129..d4c8dc8920 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -20,6 +20,10 @@ from binman import elf from patman import command from patman import tout +# These are imported if needed since they import libfdt +state = None +Image = None + # List of images we plan to create # Make this global so that it can be referenced from tests images = OrderedDict() @@ -41,6 +45,8 @@ def _ReadImageDesc(binman_node, use_expanded): Returns: OrderedDict of Image objects, each of which describes an image """ + # For Image() + # pylint: disable=E1102 images = OrderedDict() if 'multiple-images' in binman_node.props: for node in binman_node.subnodes: @@ -501,7 +507,7 @@ def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded): # entry offsets remain the same. for image in images.values(): image.CollectBintools() - image.ExpandEntries() + image.gen_entries() if update_fdt: image.AddMissingProperties(True) image.ProcessFdt(dtb) diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 5e7d6ae7b9..afa05e58fd 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -85,6 +85,57 @@ def GetSymbols(fname, patterns): # Sort dict by address return OrderedDict(sorted(syms.items(), key=lambda x: x[1].address)) +def _GetFileOffset(elf, addr): + """Get the file offset for an address + + Args: + elf (ELFFile): ELF file to check + addr (int): Address to search for + + Returns + int: Offset of that address in the ELF file, or None if not valid + """ + for seg in elf.iter_segments(): + seg_end = seg['p_vaddr'] + seg['p_filesz'] + if seg.header['p_type'] == 'PT_LOAD': + if addr >= seg['p_vaddr'] and addr < seg_end: + return addr - seg['p_vaddr'] + seg['p_offset'] + +def GetFileOffset(fname, addr): + """Get the file offset for an address + + Args: + fname (str): Filename of ELF file to check + addr (int): Address to search for + + Returns + int: Offset of that address in the ELF file, or None if not valid + """ + if not ELF_TOOLS: + raise ValueError("Python: No module named 'elftools'") + with open(fname, 'rb') as fd: + elf = ELFFile(fd) + return _GetFileOffset(elf, addr) + +def GetSymbolFromAddress(fname, addr): + """Get the symbol at a particular address + + Args: + fname (str): Filename of ELF file to check + addr (int): Address to search for + + Returns: + str: Symbol name, or None if no symbol at that address + """ + if not ELF_TOOLS: + raise ValueError("Python: No module named 'elftools'") + with open(fname, 'rb') as fd: + elf = ELFFile(fd) + syms = GetSymbols(fname, None) + for name, sym in syms.items(): + if sym.address == addr: + return name + def GetSymbolFileOffset(fname, patterns): """Get the symbols from an ELF file @@ -97,15 +148,8 @@ def GetSymbolFileOffset(fname, patterns): key: Name of symbol value: Hex value of symbol """ - def _GetFileOffset(elf, addr): - for seg in elf.iter_segments(): - seg_end = seg['p_vaddr'] + seg['p_filesz'] - if seg.header['p_type'] == 'PT_LOAD': - if addr >= seg['p_vaddr'] and addr < seg_end: - return addr - seg['p_vaddr'] + seg['p_offset'] - if not ELF_TOOLS: - raise ValueError('Python elftools package is not available') + raise ValueError("Python: No module named 'elftools'") syms = {} with open(fname, 'rb') as fd: @@ -371,7 +415,7 @@ def UpdateFile(infile, outfile, start_sym, end_sym, insert): tools.write_file(outfile, newdata) tout.info('Written to offset %#x' % syms[start_sym].offset) -def read_segments(data): +def read_loadable_segments(data): """Read segments from an ELF file Args: @@ -389,7 +433,7 @@ def read_segments(data): ValueError: elftools is not available """ if not ELF_TOOLS: - raise ValueError('Python elftools package is not available') + raise ValueError("Python: No module named 'elftools'") with io.BytesIO(data) as inf: try: elf = ELFFile(inf) diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index a67915bda6..02bc108374 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -116,7 +116,7 @@ class TestElf(unittest.TestCase): entry = FakeEntry(10) section = FakeSection() with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols('missing-file', entry, section) + elf.LookupAndWriteSymbols('missing-file', entry, section) self.assertIn("Filename 'missing-file' not found in input path", str(e.exception)) @@ -126,7 +126,7 @@ class TestElf(unittest.TestCase): section = FakeSection() elf_fname = self.ElfTestFile('u_boot_binman_syms') with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) + elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertIn('entry_path has offset 4 (size 8) but the contents size ' 'is a', str(e.exception)) @@ -139,8 +139,7 @@ class TestElf(unittest.TestCase): entry = FakeEntry(10) section = FakeSection() elf_fname = self.ElfTestFile('u_boot_binman_syms_bad') - self.assertEqual(elf.LookupAndWriteSymbols(elf_fname, entry, section), - None) + elf.LookupAndWriteSymbols(elf_fname, entry, section) def testBadSymbolSize(self): """Test that an attempt to use an 8-bit symbol are detected @@ -152,7 +151,7 @@ class TestElf(unittest.TestCase): section = FakeSection() elf_fname =self.ElfTestFile('u_boot_binman_syms_size') with self.assertRaises(ValueError) as e: - syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) + elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertIn('has size 1: only 4 and 8 are supported', str(e.exception)) @@ -165,7 +164,7 @@ class TestElf(unittest.TestCase): entry = FakeEntry(24) section = FakeSection(sym_value=None) elf_fname = self.ElfTestFile('u_boot_binman_syms') - syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) + elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertEqual(tools.get_bytes(255, 20) + tools.get_bytes(ord('a'), 4), entry.data) @@ -177,7 +176,7 @@ class TestElf(unittest.TestCase): section = FakeSection() elf_fname = self.ElfTestFile('u_boot_binman_syms') with test_util.capture_sys_output() as (stdout, stderr): - syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) + elf.LookupAndWriteSymbols(elf_fname, entry, section) self.assertTrue(len(stdout.getvalue()) > 0) finally: tout.init(tout.WARNING) @@ -244,7 +243,7 @@ class TestElf(unittest.TestCase): fname = self.ElfTestFile('embed_data') with self.assertRaises(ValueError) as e: elf.GetSymbolFileOffset(fname, ['embed_start', 'embed_end']) - self.assertIn('Python elftools package is not available', + self.assertIn("Python: No module named 'elftools'", str(e.exception)) finally: elf.ELF_TOOLS = old_val @@ -258,33 +257,81 @@ class TestElf(unittest.TestCase): offset = elf.GetSymbolFileOffset(fname, ['missing_sym']) self.assertEqual({}, offset) - def test_read_segments(self): - """Test for read_segments()""" + def test_read_loadable_segments(self): + """Test for read_loadable_segments()""" if not elf.ELF_TOOLS: self.skipTest('Python elftools not available') fname = self.ElfTestFile('embed_data') - segments, entry = elf.read_segments(tools.read_file(fname)) + segments, entry = elf.read_loadable_segments(tools.read_file(fname)) def test_read_segments_fail(self): - """Test for read_segments() without elftools""" + """Test for read_loadable_segments() without elftools""" try: old_val = elf.ELF_TOOLS elf.ELF_TOOLS = False fname = self.ElfTestFile('embed_data') with self.assertRaises(ValueError) as e: - elf.read_segments(tools.read_file(fname)) - self.assertIn('Python elftools package is not available', + elf.read_loadable_segments(tools.read_file(fname)) + self.assertIn("Python: No module named 'elftools'", str(e.exception)) finally: elf.ELF_TOOLS = old_val def test_read_segments_bad_data(self): - """Test for read_segments() with an invalid ELF file""" + """Test for read_loadable_segments() with an invalid ELF file""" fname = self.ElfTestFile('embed_data') with self.assertRaises(ValueError) as e: - elf.read_segments(tools.get_bytes(100, 100)) + elf.read_loadable_segments(tools.get_bytes(100, 100)) self.assertIn('Magic number does not match', str(e.exception)) + def test_get_file_offset(self): + """Test GetFileOffset() gives the correct file offset for a symbol""" + fname = self.ElfTestFile('embed_data') + syms = elf.GetSymbols(fname, ['embed']) + addr = syms['embed'].address + offset = elf.GetFileOffset(fname, addr) + data = tools.read_file(fname) + + # Just use the first 4 bytes and assume it is little endian + embed_data = data[offset:offset + 4] + embed_value = struct.unpack('<I', embed_data)[0] + self.assertEqual(0x1234, embed_value) + + def test_get_file_offset_fail(self): + """Test calling GetFileOffset() without elftools""" + try: + old_val = elf.ELF_TOOLS + elf.ELF_TOOLS = False + fname = self.ElfTestFile('embed_data') + with self.assertRaises(ValueError) as e: + elf.GetFileOffset(fname, 0) + self.assertIn("Python: No module named 'elftools'", + str(e.exception)) + finally: + elf.ELF_TOOLS = old_val + + def test_get_symbol_from_address(self): + """Test GetSymbolFromAddress()""" + fname = self.ElfTestFile('elf_sections') + sym_name = 'calculate' + syms = elf.GetSymbols(fname, [sym_name]) + addr = syms[sym_name].address + sym = elf.GetSymbolFromAddress(fname, addr) + self.assertEqual(sym_name, sym) + + def test_get_symbol_from_address_fail(self): + """Test calling GetSymbolFromAddress() without elftools""" + try: + old_val = elf.ELF_TOOLS + elf.ELF_TOOLS = False + fname = self.ElfTestFile('embed_data') + with self.assertRaises(ValueError) as e: + elf.GetSymbolFromAddress(fname, 0x1000) + self.assertIn("Python: No module named 'elftools'", + str(e.exception)) + finally: + elf.ELF_TOOLS = old_val + if __name__ == '__main__': unittest.main() diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 484cde5c80..ae4305c99e 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -612,6 +612,9 @@ gen-fdt-nodes Generate FDT nodes as above. This is the default if there is no `fit,operation` property. +split-elf + Split an ELF file into a separate node for each segment. + Generating nodes from an FDT list (gen-fdt-nodes) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -655,6 +658,149 @@ for each of your two files. Note that if no devicetree files are provided (with '-a of-list' as above) then no nodes will be generated. +Generating nodes from an ELF file (split-elf) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This uses the node as a template to generate multiple nodes. The following +special properties are available: + +split-elf + Split an ELF file into a separate node for each segment. This uses the + node as a template to generate multiple nodes. The following special + properties are available: + + fit,load + Generates a `load = <...>` property with the load address of the + segment + + fit,entry + Generates a `entry = <...>` property with the entry address of the + ELF. This is only produced for the first entry + + fit,data + Generates a `data = <...>` property with the contents of the segment + + fit,loadables + Generates a `loadable = <...>` property with a list of the generated + nodes (including all nodes if this operation is used multiple times) + + +Here is an example showing ATF, TEE and a device tree all combined:: + + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list = "of-list"; + + images { + u-boot { + description = "U-Boot (64-bit)"; + type = "standalone"; + os = "U-Boot"; + arch = "arm64"; + compression = "none"; + load = <CONFIG_SYS_TEXT_BASE>; + u-boot-nodtb { + }; + }; + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + }; + @atf-SEQ { + fit,operation = "split-elf"; + description = "ARM Trusted Firmware"; + type = "firmware"; + arch = "arm64"; + os = "arm-trusted-firmware"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + atf-bl31 { + }; + }; + + @tee-SEQ { + fit,operation = "split-elf"; + description = "TEE"; + type = "tee"; + arch = "arm64"; + os = "tee"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + tee-os { + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + @config-SEQ { + description = "conf-NAME.dtb"; + fdt = "fdt-SEQ"; + firmware = "u-boot"; + fit,loadables; + }; + }; + }; + +If ATF-BL31 is available, this generates a node for each segment in the +ELF file, for example:: + + images { + atf-1 { + data = <...contents of first segment...>; + data-offset = <0x00000000>; + entry = <0x00040000>; + load = <0x00040000>; + compression = "none"; + os = "arm-trusted-firmware"; + arch = "arm64"; + type = "firmware"; + description = "ARM Trusted Firmware"; + }; + atf-2 { + data = <...contents of second segment...>; + load = <0xff3b0000>; + compression = "none"; + os = "arm-trusted-firmware"; + arch = "arm64"; + type = "firmware"; + description = "ARM Trusted Firmware"; + }; + }; + +The same applies for OP-TEE if that is available. + +If each binary is not available, the relevant template node (@atf-SEQ or +@tee-SEQ) is removed from the output. + +This also generates a `config-xxx` node for each device tree in `of-list`. +Note that the U-Boot build system uses `-a of-list=$(CONFIG_OF_LIST)` +so you can use `CONFIG_OF_LIST` to define that list. In this example it is +set up for `firefly-rk3399` with a single device tree and the default set +with `-a default-dt=$(CONFIG_DEFAULT_DEVICE_TREE)`, so the resulting output +is:: + + configurations { + default = "config-1"; + config-1 { + loadables = "atf-1", "atf-2", "atf-3", "tee-1", "tee-2"; + description = "rk3399-firefly.dtb"; + fdt = "fdt-1"; + firmware = "u-boot"; + }; + }; + +U-Boot SPL can then load the firmware (U-Boot proper) and all the loadables +(ATF and TEE), then proceed with the boot. + Entry: fmap: An entry which contains an Fmap section @@ -1009,6 +1155,44 @@ placed at offset 'RESET_VECTOR_ADDRESS - 0xffc'. +Entry: pre-load: Pre load image header +-------------------------------------- + +Properties / Entry arguments: + - key-path: Path of the directory that store key (provided by the environment variable KEY_PATH) + - content: List of phandles to entries to sign + - algo-name: Hash and signature algo to use for the signature + - padding-name: Name of the padding (pkcs-1.5 or pss) + - key-name: Filename of the private key to sign + - header-size: Total size of the header + - version: Version of the header + +This entry creates a pre-load header that contains a global +image signature. + +For example, this creates an image with a pre-load header and a binary:: + + binman { + image2 { + filename = "sandbox.bin"; + + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + padding-name = "pss"; + key-name = "private.pem"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "sandbox.itb"; + }; + }; + }; + + + Entry: scp: System Control Processor (SCP) firmware blob -------------------------------------------------------- diff --git a/tools/binman/entry.py b/tools/binman/entry.py index bf68a85b24..18a7a35105 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -19,6 +19,8 @@ from patman import tout modules = {} +# This is imported if needed +state = None # An argument which can be passed to entries on the command line, in lieu of # device-tree properties. @@ -104,7 +106,7 @@ class Entry(object): self.pad_after = 0 self.offset_unset = False self.image_pos = None - self.expand_size = False + self.extend_size = False self.compress = 'none' self.missing = False self.faked = False @@ -233,6 +235,8 @@ class Entry(object): """ if 'pos' in self._node.props: self.Raise("Please use 'offset' instead of 'pos'") + if 'expand-size' in self._node.props: + self.Raise("Please use 'extend-size' instead of 'expand-size'") self.offset = fdt_util.GetInt(self._node, 'offset') self.size = fdt_util.GetInt(self._node, 'size') self.orig_offset = fdt_util.GetInt(self._node, 'orig-offset') @@ -260,7 +264,7 @@ class Entry(object): self.align_size) self.align_end = fdt_util.GetInt(self._node, 'align-end') self.offset_unset = fdt_util.GetBool(self._node, 'offset-unset') - self.expand_size = fdt_util.GetBool(self._node, 'expand-size') + self.extend_size = fdt_util.GetBool(self._node, 'extend-size') self.missing_msg = fdt_util.GetString(self._node, 'missing-msg') # This is only supported by blobs and sections at present @@ -282,8 +286,8 @@ class Entry(object): """ return {} - def ExpandEntries(self): - """Expand out entries which produce other entries + def gen_entries(self): + """Allow entries to generate other entries Some entries generate subnodes automatically, from which sub-entries are then created. This method allows those to be added to the binman @@ -413,9 +417,13 @@ class Entry(object): self.SetContents(data) return size_ok - def ObtainContents(self): + def ObtainContents(self, skip_entry=None, fake_size=0): """Figure out the contents of an entry. + Args: + skip_entry (Entry): Entry to skip when obtaining section contents + fake_size (int): Size of fake file to create if needed + Returns: True if the contents were found, False if another call is needed after the other entries are processed. @@ -772,8 +780,8 @@ features to produce new behaviours. name = '%s.%s' % (node.name, name) return name - def ExpandToLimit(self, limit): - """Expand an entry so that it ends at the given offset limit""" + def extend_to_limit(self, limit): + """Extend an entry so that it ends at the given offset limit""" if self.offset + self.size < limit: self.size = limit - self.offset # Request the contents again, since changing the size requires that @@ -986,24 +994,28 @@ features to produce new behaviours. if self.missing: missing_list.append(self) - def check_fake_fname(self, fname): + def check_fake_fname(self, fname, size=0): """If the file is missing and the entry allows fake blobs, fake it Sets self.faked to True if faked Args: fname (str): Filename to check + size (int): Size of fake file to create Returns: - fname (str): Filename of faked file + tuple: + fname (str): Filename of faked file + bool: True if the blob was faked, False if not """ if self.allow_fake and not pathlib.Path(fname).is_file(): outfname = tools.get_output_filename(os.path.basename(fname)) with open(outfname, "wb") as out: - out.truncate(1024) + out.truncate(size) self.faked = True - return outfname - return fname + tout.info(f"Entry '{self._node.path}': Faked file '{outfname}'") + return outfname, True + return fname, False def CheckFakedBlobs(self, faked_blobs_list): """Check if any entries in this section have faked external blobs @@ -1097,11 +1109,11 @@ features to produce new behaviours. """ pass - def AddBintools(self, tools): + def AddBintools(self, btools): """Add the bintools used by this entry type Args: - tools (dict of Bintool): + btools (dict of Bintool): """ pass @@ -1124,28 +1136,29 @@ features to produce new behaviours. """ self.update_hash = update_hash - def collect_contents_to_file(self, entries, prefix): + def collect_contents_to_file(self, entries, prefix, fake_size=0): """Put the contents of a list of entries into a file Args: entries (list of Entry): Entries to collect prefix (str): Filename prefix of file to write to + fake_size (int): Size of fake file to create if needed If any entry does not have contents yet, this function returns False for the data. Returns: Tuple: - bytes: Concatenated data from all the entries (or False) - str: Filename of file written (or False if no data) - str: Unique portion of filename (or False if no data) + bytes: Concatenated data from all the entries (or None) + str: Filename of file written (or None if no data) + str: Unique portion of filename (or None if no data) """ data = b'' for entry in entries: # First get the input data and put it in a file. If not available, # try later. - if not entry.ObtainContents(): - return False, False, False + if not entry.ObtainContents(fake_size=fake_size): + return None, None, None data += entry.GetData() uniq = self.GetUniqueName() fname = tools.get_output_filename(f'{prefix}.{uniq}') diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index 7ed9b262bb..1d60076be1 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -5,6 +5,7 @@ # Test for the Entry class import collections +import importlib import os import sys import unittest @@ -32,11 +33,7 @@ class TestEntry(unittest.TestCase): def _ReloadEntry(self): global entry if entry: - if sys.version_info[0] >= 3: - import importlib - importlib.reload(entry) - else: - reload(entry) + importlib.reload(entry) else: from binman import entry diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 0800c25899..5089de3642 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -82,7 +82,7 @@ class Entry__testing(Entry): self.return_contents = True self.contents = b'aa' - def ObtainContents(self): + def ObtainContents(self, fake_size=0): if self.return_unknown_contents or not self.return_contents: return False if self.return_contents_later: diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 25ec5d26c9..ceaefb07b7 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -35,16 +35,18 @@ class Entry_blob(Entry): super().__init__(section, etype, node) self._filename = fdt_util.GetString(self._node, 'filename', self.etype) - def ObtainContents(self): + def ObtainContents(self, fake_size=0): self._filename = self.GetDefaultFilename() self._pathname = tools.get_input_filename(self._filename, self.external and self.section.GetAllowMissing()) # Allow the file to be missing if not self._pathname: - self._pathname = self.check_fake_fname(self._filename) - self.SetContents(b'') + self._pathname, faked = self.check_fake_fname(self._filename, + fake_size) self.missing = True - return True + if not faked: + self.SetContents(b'') + return True self.ReadBlobContents() return True diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index 3ce7511f6f..4159e3032a 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -8,6 +8,9 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob +# This is imported if needed +state = None + class Entry_blob_dtb(Entry_blob): """A blob that holds a device tree diff --git a/tools/binman/etype/blob_ext_list.py b/tools/binman/etype/blob_ext_list.py index 76ad32a1ee..f00202e9eb 100644 --- a/tools/binman/etype/blob_ext_list.py +++ b/tools/binman/etype/blob_ext_list.py @@ -37,7 +37,7 @@ class Entry_blob_ext_list(Entry_blob): missing = False pathnames = [] for fname in self._filenames: - fname = self.check_fake_fname(fname) + fname, _ = self.check_fake_fname(fname) pathname = tools.get_input_filename( fname, self.external and self.section.GetAllowMissing()) # Allow the file to be missing diff --git a/tools/binman/etype/blob_phase.py b/tools/binman/etype/blob_phase.py index ed25e467a1..b937158756 100644 --- a/tools/binman/etype/blob_phase.py +++ b/tools/binman/etype/blob_phase.py @@ -7,6 +7,9 @@ from binman.etype.section import Entry_section +# This is imported if needed +state = None + class Entry_blob_phase(Entry_section): """Section that holds a phase binary @@ -39,7 +42,7 @@ class Entry_blob_phase(Entry_section): self.dtb_file = dtb_file self.bss_pad = bss_pad - def ExpandEntries(self): + def gen_entries(self): """Create the subnodes""" names = [self.root_fname + '-nodtb', self.root_fname + '-dtb'] if self.bss_pad: diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index cc1fbdf4b5..4a1837f26c 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -12,6 +12,9 @@ from binman.cbfs_util import CbfsWriter from binman.entry import Entry from dtoc import fdt_util +# This is imported if needed +state = None + class Entry_cbfs(Entry): """Coreboot Filesystem (CBFS) diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index 76e8dbe818..33c9d039a9 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -15,6 +15,11 @@ from patman import tout FDTMAP_MAGIC = b'_FDTMAP_' FDTMAP_HDR_LEN = 16 +# These is imported if needed +Fdt = None +libfdt = None +state = None + def LocateFdtmap(data): """Search an image for an fdt map diff --git a/tools/binman/etype/files.py b/tools/binman/etype/files.py index 0650a69c55..2081bc727b 100644 --- a/tools/binman/etype/files.py +++ b/tools/binman/etype/files.py @@ -13,6 +13,8 @@ from binman.etype.section import Entry_section from dtoc import fdt_util from patman import tools +# This is imported if needed +state = None class Entry_files(Entry_section): """A set of files arranged in a section @@ -46,7 +48,7 @@ class Entry_files(Entry_section): self._require_matches = fdt_util.GetBool(self._node, 'require-matches') - def ExpandEntries(self): + def gen_entries(self): files = tools.get_input_filename_glob(self._pattern) if self._require_matches and not files: self.Raise("Pattern '%s' matched no files" % self._pattern) diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index 2d4c5f6545..e0407715d8 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -2,22 +2,23 @@ # Copyright (c) 2016 Google, Inc # Written by Simon Glass <sjg@chromium.org> # -# Entry-type module for producing a FIT -# -from collections import defaultdict, OrderedDict +"""Entry-type module for producing a FIT""" + import libfdt from binman.entry import Entry, EntryArg from binman.etype.section import Entry_section +from binman import elf from dtoc import fdt_util from dtoc.fdt import Fdt from patman import tools # Supported operations, with the fit,operation property -OP_GEN_FDT_NODES = range(1) +OP_GEN_FDT_NODES, OP_SPLIT_ELF = range(2) OPERATIONS = { 'gen-fdt-nodes': OP_GEN_FDT_NODES, + 'split-elf': OP_SPLIT_ELF, } class Entry_fit(Entry_section): @@ -113,6 +114,9 @@ class Entry_fit(Entry_section): Generate FDT nodes as above. This is the default if there is no `fit,operation` property. + split-elf + Split an ELF file into a separate node for each segment. + Generating nodes from an FDT list (gen-fdt-nodes) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -155,6 +159,149 @@ class Entry_fit(Entry_section): Note that if no devicetree files are provided (with '-a of-list' as above) then no nodes will be generated. + + Generating nodes from an ELF file (split-elf) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This uses the node as a template to generate multiple nodes. The following + special properties are available: + + split-elf + Split an ELF file into a separate node for each segment. This uses the + node as a template to generate multiple nodes. The following special + properties are available: + + fit,load + Generates a `load = <...>` property with the load address of the + segment + + fit,entry + Generates a `entry = <...>` property with the entry address of the + ELF. This is only produced for the first entry + + fit,data + Generates a `data = <...>` property with the contents of the segment + + fit,loadables + Generates a `loadable = <...>` property with a list of the generated + nodes (including all nodes if this operation is used multiple times) + + + Here is an example showing ATF, TEE and a device tree all combined:: + + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list = "of-list"; + + images { + u-boot { + description = "U-Boot (64-bit)"; + type = "standalone"; + os = "U-Boot"; + arch = "arm64"; + compression = "none"; + load = <CONFIG_SYS_TEXT_BASE>; + u-boot-nodtb { + }; + }; + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + }; + @atf-SEQ { + fit,operation = "split-elf"; + description = "ARM Trusted Firmware"; + type = "firmware"; + arch = "arm64"; + os = "arm-trusted-firmware"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + atf-bl31 { + }; + }; + + @tee-SEQ { + fit,operation = "split-elf"; + description = "TEE"; + type = "tee"; + arch = "arm64"; + os = "tee"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + tee-os { + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + @config-SEQ { + description = "conf-NAME.dtb"; + fdt = "fdt-SEQ"; + firmware = "u-boot"; + fit,loadables; + }; + }; + }; + + If ATF-BL31 is available, this generates a node for each segment in the + ELF file, for example:: + + images { + atf-1 { + data = <...contents of first segment...>; + data-offset = <0x00000000>; + entry = <0x00040000>; + load = <0x00040000>; + compression = "none"; + os = "arm-trusted-firmware"; + arch = "arm64"; + type = "firmware"; + description = "ARM Trusted Firmware"; + }; + atf-2 { + data = <...contents of second segment...>; + load = <0xff3b0000>; + compression = "none"; + os = "arm-trusted-firmware"; + arch = "arm64"; + type = "firmware"; + description = "ARM Trusted Firmware"; + }; + }; + + The same applies for OP-TEE if that is available. + + If each binary is not available, the relevant template node (@atf-SEQ or + @tee-SEQ) is removed from the output. + + This also generates a `config-xxx` node for each device tree in `of-list`. + Note that the U-Boot build system uses `-a of-list=$(CONFIG_OF_LIST)` + so you can use `CONFIG_OF_LIST` to define that list. In this example it is + set up for `firefly-rk3399` with a single device tree and the default set + with `-a default-dt=$(CONFIG_DEFAULT_DEVICE_TREE)`, so the resulting output + is:: + + configurations { + default = "config-1"; + config-1 { + loadables = "atf-1", "atf-2", "atf-3", "tee-1", "tee-2"; + description = "rk3399-firefly.dtb"; + fdt = "fdt-1"; + firmware = "u-boot"; + }; + }; + + U-Boot SPL can then load the firmware (U-Boot proper) and all the loadables + (ATF and TEE), then proceed with the boot. """ def __init__(self, section, etype, node): """ @@ -164,16 +311,25 @@ class Entry_fit(Entry_section): key: relative path to entry Node (from the base of the FIT) value: Entry_section object comprising the contents of this node + _priv_entries: Internal copy of _entries which includes 'generator' + entries which are used to create the FIT, but should not be + processed as real entries. This is set up once we have the + entries + _loadables: List of generated split-elf nodes, each a node name """ super().__init__(section, etype, node) self._fit = None self._fit_props = {} + self._fdts = None + self.mkimage = None + self._priv_entries = {} + self._loadables = [] + def ReadNode(self): + super().ReadNode() for pname, prop in self._node.props.items(): if pname.startswith('fit,'): self._fit_props[pname] = prop - - self._fdts = None self._fit_list_prop = self._fit_props.get('fit,fdt-list') if self._fit_list_prop: fdts, = self.GetEntryArgsOrProps( @@ -182,17 +338,12 @@ class Entry_fit(Entry_section): self._fdts = fdts.split() self._fit_default_dt = self.GetEntryArgsOrProps([EntryArg('default-dt', str)])[0] - self.mkimage = None - - def ReadNode(self): - self.ReadEntries() - super().ReadNode() - def _get_operation(self, subnode): + def _get_operation(self, base_node, node): """Get the operation referenced by a subnode Args: - subnode (Node): Subnode (of the FIT) to check + node (Node): Subnode (of the FIT) to check Returns: int: Operation to perform @@ -200,15 +351,107 @@ class Entry_fit(Entry_section): Raises: ValueError: Invalid operation name """ - oper_name = subnode.props.get('fit,operation') + oper_name = node.props.get('fit,operation') if not oper_name: return OP_GEN_FDT_NODES oper = OPERATIONS.get(oper_name.value) - if not oper: - self.Raise(f"Unknown operation '{oper_name.value}'") + if oper is None: + self._raise_subnode(node, f"Unknown operation '{oper_name.value}'") return oper def ReadEntries(self): + def _add_entries(base_node, depth, node): + """Add entries for any nodes that need them + + Args: + base_node: Base Node of the FIT (with 'description' property) + depth: Current node depth (0 is the base 'fit' node) + node: Current node to process + + Here we only need to provide binman entries which are used to define + the 'data' for each image. We create an entry_Section for each. + """ + rel_path = node.path[len(base_node.path):] + in_images = rel_path.startswith('/images') + has_images = depth == 2 and in_images + if has_images: + # This node is a FIT subimage node (e.g. "/images/kernel") + # containing content nodes. We collect the subimage nodes and + # section entries for them here to merge the content subnodes + # together and put the merged contents in the subimage node's + # 'data' property later. + entry = Entry.Create(self.section, node, etype='section') + entry.ReadNode() + # The hash subnodes here are for mkimage, not binman. + entry.SetUpdateHash(False) + self._entries[rel_path] = entry + + for subnode in node.subnodes: + _add_entries(base_node, depth + 1, subnode) + + _add_entries(self._node, 0, self._node) + + # Keep a copy of all entries, including generator entries, since these + # removed from self._entries later. + self._priv_entries = dict(self._entries) + + def BuildSectionData(self, required): + """Build FIT entry contents + + This adds the 'data' properties to the input ITB (Image-tree Binary) + then runs mkimage to process it. + + Args: + required (bool): True if the data must be present, False if it is OK + to return None + + Returns: + bytes: Contents of the section + """ + data = self._build_input() + uniq = self.GetUniqueName() + input_fname = tools.get_output_filename(f'{uniq}.itb') + output_fname = tools.get_output_filename(f'{uniq}.fit') + tools.write_file(input_fname, data) + tools.write_file(output_fname, data) + + args = {} + ext_offset = self._fit_props.get('fit,external-offset') + if ext_offset is not None: + args = { + 'external': True, + 'pad': fdt_util.fdt32_to_cpu(ext_offset.value) + } + if self.mkimage.run(reset_timestamp=True, output_fname=output_fname, + **args) is None: + # Bintool is missing; just use empty data as the output + self.record_missing_bintool(self.mkimage) + return tools.get_bytes(0, 1024) + + return tools.read_file(output_fname) + + def _raise_subnode(self, node, msg): + """Raise an error with a paticular FIT subnode + + Args: + node (Node): FIT subnode containing the error + msg (str): Message to report + + Raises: + ValueError, as requested + """ + rel_path = node.path[len(self._node.path) + 1:] + self.Raise(f"subnode '{rel_path}': {msg}") + + def _build_input(self): + """Finish the FIT by adding the 'data' properties to it + + Arguments: + fdt: FIT to update + + Returns: + bytes: New fdt contents + """ def _process_prop(pname, prop): """Process special properties @@ -229,16 +472,22 @@ class Entry_fit(Entry_section): if not self._fit_default_dt: self.Raise("Generated 'default' node requires default-dt entry argument") if self._fit_default_dt not in self._fdts: - self.Raise("default-dt entry argument '%s' not found in fdt list: %s" % - (self._fit_default_dt, - ', '.join(self._fdts))) + self.Raise( + f"default-dt entry argument '{self._fit_default_dt}' " + f"not found in fdt list: {', '.join(self._fdts)}") seq = self._fdts.index(self._fit_default_dt) val = val[1:].replace('DEFAULT-SEQ', str(seq + 1)) fsw.property_string(pname, val) return + elif pname.startswith('fit,'): + # Ignore these, which are commands for binman to process + return + elif pname in ['offset', 'size', 'image-pos']: + # Don't add binman's calculated properties + return fsw.property(pname, prop.bytes) - def _scan_gen_fdt_nodes(subnode, depth, in_images): + def _gen_fdt_nodes(base_node, node, depth, in_images): """Generate FDT nodes This creates one node for each member of self._fdts using the @@ -248,7 +497,7 @@ class Entry_fit(Entry_section): first. Args: - subnode (None): Generator node to process + node (None): Generator node to process depth: Current node depth (0 is the base 'fit' node) in_images: True if this is inside the 'images' node, so that 'data' properties should be generated @@ -256,28 +505,77 @@ class Entry_fit(Entry_section): if self._fdts: # Generate nodes for each FDT for seq, fdt_fname in enumerate(self._fdts): - node_name = subnode.name[1:].replace('SEQ', str(seq + 1)) + node_name = node.name[1:].replace('SEQ', str(seq + 1)) fname = tools.get_input_filename(fdt_fname + '.dtb') with fsw.add_node(node_name): - for pname, prop in subnode.props.items(): - val = prop.bytes.replace( - b'NAME', tools.to_bytes(fdt_fname)) - val = val.replace( - b'SEQ', tools.to_bytes(str(seq + 1))) - fsw.property(pname, val) + for pname, prop in node.props.items(): + if pname == 'fit,loadables': + val = '\0'.join(self._loadables) + '\0' + fsw.property('loadables', val.encode('utf-8')) + elif pname == 'fit,operation': + pass + elif pname.startswith('fit,'): + self._raise_subnode( + node, f"Unknown directive '{pname}'") + else: + val = prop.bytes.replace( + b'NAME', tools.to_bytes(fdt_fname)) + val = val.replace( + b'SEQ', tools.to_bytes(str(seq + 1))) + fsw.property(pname, val) # Add data for 'images' nodes (but not 'config') if depth == 1 and in_images: fsw.property('data', tools.read_file(fname)) + + for subnode in node.subnodes: + with fsw.add_node(subnode.name): + _add_node(node, depth + 1, subnode) else: if self._fdts is None: if self._fit_list_prop: - self.Raise("Generator node requires '%s' entry argument" % - self._fit_list_prop.value) + self.Raise('Generator node requires ' + f"'{self._fit_list_prop.value}' entry argument") else: self.Raise("Generator node requires 'fit,fdt-list' property") - def _scan_node(subnode, depth, in_images): + def _gen_split_elf(base_node, node, elf_data, missing): + """Add nodes for the ELF file, one per group of contiguous segments + + Args: + base_node (Node): Template node from the binman definition + node (Node): Node to replace (in the FIT being built) + data (bytes): ELF-format data to process (may be empty) + missing (bool): True if any of the data is missing + + """ + # If any pieces are missing, skip this. The missing entries will + # show an error + if not missing: + try: + segments, entry = elf.read_loadable_segments(elf_data) + except ValueError as exc: + self._raise_subnode(node, + f'Failed to read ELF file: {str(exc)}') + for (seq, start, data) in segments: + node_name = node.name[1:].replace('SEQ', str(seq + 1)) + with fsw.add_node(node_name): + loadables.append(node_name) + for pname, prop in node.props.items(): + if not pname.startswith('fit,'): + fsw.property(pname, prop.bytes) + elif pname == 'fit,load': + fsw.property_u32('load', start) + elif pname == 'fit,entry': + if seq == 0: + fsw.property_u32('entry', entry) + elif pname == 'fit,data': + fsw.property('data', bytes(data)) + elif pname != 'fit,operation': + self._raise_subnode( + node, f"Unknown directive '{pname}'") + + def _gen_node(base_node, node, depth, in_images, entry): """Generate nodes from a template This creates one node for each member of self._fdts using the @@ -287,124 +585,93 @@ class Entry_fit(Entry_section): first. Args: - subnode (None): Generator node to process - depth: Current node depth (0 is the base 'fit' node) - in_images: True if this is inside the 'images' node, so that - 'data' properties should be generated + base_node (Node): Base Node of the FIT (with 'description' + property) + node (Node): Generator node to process + depth (int): Current node depth (0 is the base 'fit' node) + in_images (bool): True if this is inside the 'images' node, so + that 'data' properties should be generated """ - oper = self._get_operation(subnode) + oper = self._get_operation(base_node, node) if oper == OP_GEN_FDT_NODES: - _scan_gen_fdt_nodes(subnode, depth, in_images) - - def _AddNode(base_node, depth, node): - """Add a node to the FIT + _gen_fdt_nodes(base_node, node, depth, in_images) + elif oper == OP_SPLIT_ELF: + # Entry_section.ObtainContents() either returns True or + # raises an exception. + data = None + missing_list = [] + entry.ObtainContents() + entry.Pack(0) + data = entry.GetData() + entry.CheckMissing(missing_list) + + _gen_split_elf(base_node, node, data, bool(missing_list)) + + def _add_node(base_node, depth, node): + """Add nodes to the output FIT Args: - base_node: Base Node of the FIT (with 'description' property) - depth: Current node depth (0 is the base 'fit' node) - node: Current node to process + base_node (Node): Base Node of the FIT (with 'description' + property) + depth (int): Current node depth (0 is the base 'fit' node) + node (Node): Current node to process There are two cases to deal with: - hash and signature nodes which become part of the FIT - binman entries which are used to define the 'data' for each - image + image, so don't appear in the FIT """ + # Copy over all the relevant properties for pname, prop in node.props.items(): - if not pname.startswith('fit,'): - _process_prop(pname, prop) + _process_prop(pname, prop) rel_path = node.path[len(base_node.path):] in_images = rel_path.startswith('/images') + has_images = depth == 2 and in_images if has_images: - # This node is a FIT subimage node (e.g. "/images/kernel") - # containing content nodes. We collect the subimage nodes and - # section entries for them here to merge the content subnodes - # together and put the merged contents in the subimage node's - # 'data' property later. - entry = Entry.Create(self.section, node, etype='section') - entry.ReadNode() - # The hash subnodes here are for mkimage, not binman. - entry.SetUpdateHash(False) - self._entries[rel_path] = entry + entry = self._priv_entries[rel_path] + data = entry.GetData() + fsw.property('data', bytes(data)) for subnode in node.subnodes: + subnode_path = f'{rel_path}/{subnode.name}' if has_images and not (subnode.name.startswith('hash') or subnode.name.startswith('signature')): # This subnode is a content node not meant to appear in # the FIT (e.g. "/images/kernel/u-boot"), so don't call - # fsw.add_node() or _AddNode() for it. + # fsw.add_node() or _add_node() for it. pass elif self.GetImage().generate and subnode.name.startswith('@'): - _scan_node(subnode, depth, in_images) + entry = self._priv_entries.get(subnode_path) + _gen_node(base_node, subnode, depth, in_images, entry) + # This is a generator (template) entry, so remove it from + # the list of entries used by PackEntries(), etc. Otherwise + # it will appear in the binman output + to_remove.append(subnode_path) else: with fsw.add_node(subnode.name): - _AddNode(base_node, depth + 1, subnode) + _add_node(base_node, depth + 1, subnode) # Build a new tree with all nodes and properties starting from the # entry node fsw = libfdt.FdtSw() fsw.finish_reservemap() + to_remove = [] + loadables = [] with fsw.add_node(''): - _AddNode(self._node, 0, self._node) + _add_node(self._node, 0, self._node) + self._loadables = loadables fdt = fsw.as_fdt() + # Remove generator entries from the main list + for path in to_remove: + if path in self._entries: + del self._entries[path] + # Pack this new FDT and scan it so we can add the data later fdt.pack() - self._fdt = Fdt.FromData(fdt.as_bytearray()) - self._fdt.Scan() - - def BuildSectionData(self, required): - """Build FIT entry contents - - This adds the 'data' properties to the input ITB (Image-tree Binary) - then runs mkimage to process it. - - Args: - required: True if the data must be present, False if it is OK to - return None - - Returns: - Contents of the section (bytes) - """ - data = self._BuildInput(self._fdt) - uniq = self.GetUniqueName() - input_fname = tools.get_output_filename('%s.itb' % uniq) - output_fname = tools.get_output_filename('%s.fit' % uniq) - tools.write_file(input_fname, data) - tools.write_file(output_fname, data) - - args = {} - ext_offset = self._fit_props.get('fit,external-offset') - if ext_offset is not None: - args = { - 'external': True, - 'pad': fdt_util.fdt32_to_cpu(ext_offset.value) - } - if self.mkimage.run(reset_timestamp=True, output_fname=output_fname, - **args) is None: - # Bintool is missing; just use empty data as the output - self.record_missing_bintool(self.mkimage) - return tools.get_bytes(0, 1024) - - return tools.read_file(output_fname) - - def _BuildInput(self, fdt): - """Finish the FIT by adding the 'data' properties to it - - Arguments: - fdt: FIT to update - - Returns: - New fdt contents (bytes) - """ - for path, section in self._entries.items(): - node = fdt.GetNode(path) - data = section.GetData() - node.AddData('data', data) - - fdt.Sync(auto_resize=True) - data = fdt.GetContents() + data = fdt.as_bytearray() return data def SetImagePos(self, image_pos): @@ -414,7 +681,7 @@ class Entry_fit(Entry_section): according to where they ended up in the packed FIT file. Args: - image_pos: Position of this entry in the image + image_pos (int): Position of this entry in the image """ super().SetImagePos(image_pos) @@ -453,11 +720,18 @@ class Entry_fit(Entry_section): # This should never happen else: # pragma: no cover - self.Raise("%s: missing data properties" % (path)) + self.Raise(f'{path}: missing data properties') section.SetOffsetSize(offset, size) section.SetImagePos(self.image_pos) - def AddBintools(self, tools): - super().AddBintools(tools) - self.mkimage = self.AddBintool(tools, 'mkimage') + def AddBintools(self, btools): + super().AddBintools(btools) + self.mkimage = self.AddBintool(btools, 'mkimage') + + def CheckMissing(self, missing_list): + # We must use our private entry list for this since generator notes + # which are removed from self._entries will otherwise not show up as + # missing + for entry in self._priv_entries.values(): + entry.CheckMissing(missing_list) diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index e32fae27ce..7394e4e5d3 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -99,5 +99,5 @@ class Entry_gbb(Entry): return True - def AddBintools(self, tools): - self.futility = self.AddBintool(tools, 'futility') + def AddBintools(self, btools): + self.futility = self.AddBintool(btools, 'futility') diff --git a/tools/binman/etype/intel_ifwi.py b/tools/binman/etype/intel_ifwi.py index 46bdf116e6..4fa7d636fe 100644 --- a/tools/binman/etype/intel_ifwi.py +++ b/tools/binman/etype/intel_ifwi.py @@ -143,5 +143,5 @@ class Entry_intel_ifwi(Entry_blob_ext): for entry in self._ifwi_entries.values(): entry.WriteSymbols(self) - def AddBintools(self, tools): - self.ifwitool = self.AddBintool(tools, 'ifwitool') + def AddBintools(self, btools): + self.ifwitool = self.AddBintool(btools, 'ifwitool') diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index 8e74ebb9c0..5f6def2287 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -51,9 +51,10 @@ class Entry_mkimage(Entry): self.ReadEntries() def ObtainContents(self): + # Use a non-zero size for any fake files to keep mkimage happy data, input_fname, uniq = self.collect_contents_to_file( - self._mkimage_entries.values(), 'mkimage') - if data is False: + self._mkimage_entries.values(), 'mkimage', 1024) + if data is None: return False output_fname = tools.get_output_filename('mkimage-out.%s' % uniq) if self.mkimage.run_cmd('-d', input_fname, *self._args, @@ -73,6 +74,16 @@ class Entry_mkimage(Entry): entry.ReadNode() self._mkimage_entries[entry.name] = entry + def SetAllowMissing(self, allow_missing): + """Set whether a section allows missing external blobs + + Args: + allow_missing: True if allowed, False if not allowed + """ + self.allow_missing = allow_missing + for entry in self._mkimage_entries.values(): + entry.SetAllowMissing(allow_missing) + def SetAllowFakeBlob(self, allow_fake): """Set whether the sub nodes allows to create a fake blob @@ -93,5 +104,5 @@ class Entry_mkimage(Entry): for entry in self._mkimage_entries.values(): entry.CheckFakedBlobs(faked_blobs_list) - def AddBintools(self, tools): - self.mkimage = self.AddBintool(tools, 'mkimage') + def AddBintools(self, btools): + self.mkimage = self.AddBintool(btools, 'mkimage') diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py new file mode 100644 index 0000000000..245ee75525 --- /dev/null +++ b/tools/binman/etype/pre_load.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2022 Softathome +# Written by Philippe Reynes <philippe.reynes@softathome.com> +# +# Entry-type for the global header +# + +import os +import struct +from dtoc import fdt_util +from patman import tools + +from binman.entry import Entry +from binman.etype.collection import Entry_collection +from binman.entry import EntryArg + +from Cryptodome.Hash import SHA256, SHA384, SHA512 +from Cryptodome.PublicKey import RSA +from Cryptodome.Signature import pkcs1_15 +from Cryptodome.Signature import pss + +PRE_LOAD_MAGIC = b'UBSH' + +RSAS = { + 'rsa1024': 1024 / 8, + 'rsa2048': 2048 / 8, + 'rsa4096': 4096 / 8 +} + +SHAS = { + 'sha256': SHA256, + 'sha384': SHA384, + 'sha512': SHA512 +} + +class Entry_pre_load(Entry_collection): + """Pre load image header + + Properties / Entry arguments: + - pre-load-key-path: Path of the directory that store key (provided by the environment variable PRE_LOAD_KEY_PATH) + - content: List of phandles to entries to sign + - algo-name: Hash and signature algo to use for the signature + - padding-name: Name of the padding (pkcs-1.5 or pss) + - key-name: Filename of the private key to sign + - header-size: Total size of the header + - version: Version of the header + + This entry creates a pre-load header that contains a global + image signature. + + For example, this creates an image with a pre-load header and a binary:: + + binman { + image2 { + filename = "sandbox.bin"; + + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + padding-name = "pss"; + key-name = "private.pem"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "sandbox.itb"; + }; + }; + }; + """ + + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.algo_name = fdt_util.GetString(self._node, 'algo-name') + self.padding_name = fdt_util.GetString(self._node, 'padding-name') + self.key_name = fdt_util.GetString(self._node, 'key-name') + self.header_size = fdt_util.GetInt(self._node, 'header-size') + self.version = fdt_util.GetInt(self._node, 'version') + + def ReadNode(self): + super().ReadNode() + self.key_path, = self.GetEntryArgsOrProps([EntryArg('pre-load-key-path', str)]) + if self.key_path is None: + self.key_path = '' + + def _CreateHeader(self): + """Create a pre load header""" + hash_name, sign_name = self.algo_name.split(',') + padding_name = self.padding_name + key_name = os.path.join(self.key_path, self.key_name) + + # Check hash and signature name/type + if hash_name not in SHAS: + self.Raise(hash_name + " is not supported") + if sign_name not in RSAS: + self.Raise(sign_name + " is not supported") + + # Read the key + with open(key_name, 'rb') as pem: + key = RSA.import_key(pem.read()) + + # Check if the key has the expected size + if key.size_in_bytes() != RSAS[sign_name]: + self.Raise("The key " + self.key_name + " don't have the expected size") + + # Compute the hash + hash_image = SHAS[hash_name].new() + hash_image.update(self.image) + + # Compute the signature + if padding_name is None: + padding_name = "pkcs-1.5" + if padding_name == "pss": + salt_len = key.size_in_bytes() - hash_image.digest_size - 2 + padding = pss + padding_args = {'salt_bytes': salt_len} + elif padding_name == "pkcs-1.5": + padding = pkcs1_15 + padding_args = {} + else: + self.Raise(padding_name + " is not supported") + + sig = padding.new(key, **padding_args).sign(hash_image) + + hash_sig = SHA256.new() + hash_sig.update(sig) + + version = self.version + header_size = self.header_size + image_size = len(self.image) + ofs_img_sig = 64 + len(sig) + flags = 0 + reserved0 = 0 + reserved1 = 0 + + first_header = struct.pack('>4sIIIIIII32s', PRE_LOAD_MAGIC, + version, header_size, image_size, + ofs_img_sig, flags, reserved0, + reserved1, hash_sig.digest()) + + hash_first_header = SHAS[hash_name].new() + hash_first_header.update(first_header) + sig_first_header = padding.new(key, **padding_args).sign(hash_first_header) + + data = first_header + sig_first_header + sig + pad = bytearray(self.header_size - len(data)) + + return data + pad + + def ObtainContents(self): + """Obtain a placeholder for the header contents""" + # wait that the image is available + self.image = self.GetContents(False) + if self.image is None: + return False + self.SetContents(self._CreateHeader()) + return True + + def ProcessContents(self): + data = self._CreateHeader() + return self.ProcessContentsUpdate(data) diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 25159074ba..ccac658c18 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -163,6 +163,7 @@ class Entry_section(Entry): self._sort = False self._skip_at_start = None self._end_4gb = False + self._ignore_missing = False def ReadNode(self): """Read properties from the section node""" @@ -233,10 +234,10 @@ class Entry_section(Entry): todo) return True - def ExpandEntries(self): - super().ExpandEntries() + def gen_entries(self): + super().gen_entries() for entry in self._entries.values(): - entry.ExpandEntries() + entry.gen_entries() def AddMissingProperties(self, have_image_pos): """Add new properties to the device tree as needed for this entry""" @@ -246,7 +247,7 @@ class Entry_section(Entry): for entry in self._entries.values(): entry.AddMissingProperties(have_image_pos) - def ObtainContents(self, skip_entry=None): + def ObtainContents(self, fake_size=0, skip_entry=None): return self.GetEntryContents(skip_entry=skip_entry) def GetPaddedDataForEntry(self, entry, entry_data): @@ -385,7 +386,7 @@ class Entry_section(Entry): self._PackEntries() if self._sort: self._SortEntries() - self._ExpandEntries() + self._extend_entries() data = self.BuildSectionData(True) self.SetContents(data) @@ -403,17 +404,17 @@ class Entry_section(Entry): offset = entry.Pack(offset) return offset - def _ExpandEntries(self): - """Expand any entries that are permitted to""" + def _extend_entries(self): + """Extend any entries that are permitted to""" exp_entry = None for entry in self._entries.values(): if exp_entry: - exp_entry.ExpandToLimit(entry.offset) + exp_entry.extend_to_limit(entry.offset) exp_entry = None - if entry.expand_size: + if entry.extend_size: exp_entry = entry if exp_entry: - exp_entry.ExpandToLimit(self.size) + exp_entry.extend_to_limit(self.size) def _SortEntries(self): """Sort entries by offset""" @@ -894,6 +895,6 @@ class Entry_section(Entry): for entry in self._entries.values(): entry.CheckAltFormats(alt_formats) - def AddBintools(self, tools): + def AddBintools(self, btools): for entry in self._entries.values(): - entry.AddBintools(tools) + entry.AddBintools(btools) diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 554b3b2e00..047d310cdf 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -9,6 +9,9 @@ from binman.entry import Entry from binman.etype.blob_dtb import Entry_blob_dtb from patman import tools +# This is imported if needed +state = None + class Entry_u_boot_dtb_with_ucode(Entry_blob_dtb): """A U-Boot device tree file, with the microcode removed diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index a1de98290b..065b6ed2f6 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -97,5 +97,5 @@ class Entry_vblock(Entry_collection): data = self.GetVblock(True) return self.ProcessContentsUpdate(data) - def AddBintools(self, tools): - self.futility = self.AddBintool(tools, 'futility') + def AddBintools(self, btools): + self.futility = self.AddBintool(btools, 'futility') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 8f00db6945..4ce181a066 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -91,6 +91,9 @@ SCP_DATA = b'scp' TEST_FDT1_DATA = b'fdt1' TEST_FDT2_DATA = b'test-fdt2' ENV_DATA = b'var1=1\nvar2="2"' +PRE_LOAD_MAGIC = b'UBSH' +PRE_LOAD_VERSION = 0x11223344.to_bytes(4, 'big') +PRE_LOAD_HDR_SIZE = 0x00001000.to_bytes(4, 'big') # Subdirectory of the input dir to use to put test FDTs TEST_FDT_SUBDIR = 'fdts' @@ -202,6 +205,13 @@ class TestFunctional(unittest.TestCase): TestFunctional._MakeInputFile('env.txt', ENV_DATA) + # ELF file with two sections in different parts of memory, used for both + # ATF and OP_TEE + TestFunctional._MakeInputFile('bl31.elf', + tools.read_file(cls.ElfTestFile('elf_sections'))) + TestFunctional._MakeInputFile('tee.elf', + tools.read_file(cls.ElfTestFile('elf_sections'))) + cls.have_lz4 = comp_util.HAVE_LZ4 @classmethod @@ -991,7 +1001,7 @@ class TestFunctional(unittest.TestCase): self.assertIn("Section '/binman': Size 0x7 (7) does not match " "align-size 0x8 (8)", str(e.exception)) - def testPackAlignPowerOf2(self): + def testPackAlignPowerOf2Inv(self): """Test that invalid image alignment is detected""" with self.assertRaises(ValueError) as e: self._DoTestFile('020_pack_inv_image_align_power2.dts') @@ -2028,9 +2038,9 @@ class TestFunctional(unittest.TestCase): self.assertIn("Node '/binman/files': Missing 'pattern' property", str(e.exception)) - def testExpandSize(self): - """Test an expanding entry""" - data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts', + def testExtendSize(self): + """Test an extending entry""" + data, _, map_data, _ = self._DoReadFileDtb('088_extend_size.dts', map=True) expect = (tools.get_bytes(ord('a'), 8) + U_BOOT_DATA + MRC_DATA + tools.get_bytes(ord('b'), 1) + U_BOOT_DATA + @@ -2050,11 +2060,11 @@ class TestFunctional(unittest.TestCase): 00000020 00000020 00000008 fill2 ''', map_data) - def testExpandSizeBad(self): - """Test an expanding entry which fails to provide contents""" + def testExtendSizeBad(self): + """Test an extending entry which fails to provide contents""" with test_util.capture_sys_output() as (stdout, stderr): with self.assertRaises(ValueError) as e: - self._DoReadFileDtb('089_expand_size_bad.dts', map=True) + self._DoReadFileDtb('089_extend_size_bad.dts', map=True) self.assertIn("Node '/binman/_testing': Cannot obtain contents when " 'expanding entry', str(e.exception)) @@ -2487,22 +2497,22 @@ class TestFunctional(unittest.TestCase): str(e.exception)) def testEntryExpand(self): - """Test expanding an entry after it is packed""" - data = self._DoReadFile('121_entry_expand.dts') + """Test extending an entry after it is packed""" + data = self._DoReadFile('121_entry_extend.dts') self.assertEqual(b'aaa', data[:3]) self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)]) self.assertEqual(b'aaa', data[-3:]) - def testEntryExpandBad(self): - """Test expanding an entry after it is packed, twice""" + def testEntryExtendBad(self): + """Test extending an entry after it is packed, twice""" with self.assertRaises(ValueError) as e: - self._DoReadFile('122_entry_expand_twice.dts') + self._DoReadFile('122_entry_extend_twice.dts') self.assertIn("Image '/binman': Entries changed size after packing", str(e.exception)) - def testEntryExpandSection(self): - """Test expanding an entry within a section after it is packed""" - data = self._DoReadFile('123_entry_expand_section.dts') + def testEntryExtendSection(self): + """Test extending an entry within a section after it is packed""" + data = self._DoReadFile('123_entry_extend_section.dts') self.assertEqual(b'aaa', data[:3]) self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)]) self.assertEqual(b'aaa', data[-3:]) @@ -3714,7 +3724,7 @@ class TestFunctional(unittest.TestCase): err = stderr.getvalue() self.assertRegex(err, "Image 'main-section'.*missing.*: intel-ifwi") - def testPackOverlap(self): + def testPackOverlapZero(self): """Test that zero-size overlapping regions are ignored""" self._DoTestFile('160_pack_overlap_zero.dts') @@ -3780,6 +3790,7 @@ class TestFunctional(unittest.TestCase): dtb.Scan() props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS) + self.maxDiff = None self.assertEqual({ 'image-pos': 0, 'offset': 0, @@ -3793,19 +3804,19 @@ class TestFunctional(unittest.TestCase): 'fit:offset': 4, 'fit:size': 1840, - 'fit/images/kernel:image-pos': 160, - 'fit/images/kernel:offset': 156, + 'fit/images/kernel:image-pos': 304, + 'fit/images/kernel:offset': 300, 'fit/images/kernel:size': 4, - 'fit/images/kernel/u-boot:image-pos': 160, + 'fit/images/kernel/u-boot:image-pos': 304, 'fit/images/kernel/u-boot:offset': 0, 'fit/images/kernel/u-boot:size': 4, - 'fit/images/fdt-1:image-pos': 456, - 'fit/images/fdt-1:offset': 452, + 'fit/images/fdt-1:image-pos': 552, + 'fit/images/fdt-1:offset': 548, 'fit/images/fdt-1:size': 6, - 'fit/images/fdt-1/u-boot-spl-dtb:image-pos': 456, + 'fit/images/fdt-1/u-boot-spl-dtb:image-pos': 552, 'fit/images/fdt-1/u-boot-spl-dtb:offset': 0, 'fit/images/fdt-1/u-boot-spl-dtb:size': 6, @@ -4029,6 +4040,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(expected_data, fnode.props['data'].bytes) self.assertEqual('fdt-test-fdt%d.dtb' % seq, fnode.props['description'].value) + self.assertEqual(fnode.subnodes[0].name, 'hash') def _CheckConfig(seq, expected_data): """Check the configuration nodes @@ -4095,13 +4107,6 @@ class TestFunctional(unittest.TestCase): self.assertIn("Generator node requires 'fit,fdt-list' property", str(e.exception)) - def testFitFdtEmptyList(self): - """Test handling of an empty 'of-list' entry arg""" - entry_args = { - 'of-list': '', - } - data = self._DoReadFileDtb('170_fit_fdt.dts', entry_args=entry_args)[0] - def testFitFdtMissing(self): """Test handling of a missing 'default-dt' entry arg""" entry_args = { @@ -4986,16 +4991,6 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual('rot-cert', fent.fip_type) self.assertEqual(b'aa', fent.data) - def testFipOther(self): - """Basic FIP with something that isn't a external blob""" - data = self._DoReadFile('204_fip_other.dts') - hdr, fents = fip_util.decode_fip(data) - - self.assertEqual(2, len(fents)) - fent = fents[1] - self.assertEqual('rot-cert', fent.fip_type) - self.assertEqual(b'aa', fent.data) - def testFipNoType(self): """FIP with an entry of an unknown type""" with self.assertRaises(ValueError) as e: @@ -5318,9 +5313,215 @@ fdt fdtmap Extract the devicetree blob from the fdtmap """Check handling of an FDT map when the section cannot be found""" with self.assertRaises(ValueError) as exc: self._DoReadFileDtb('224_fit_bad_oper.dts') - self.assertIn("Node '/binman/fit': Unknown operation 'unknown'", + self.assertIn("Node '/binman/fit': subnode 'images/@fdt-SEQ': Unknown operation 'unknown'", str(exc.exception)) + def test_uses_expand_size(self): + """Test that the 'expand-size' property cannot be used anymore""" + with self.assertRaises(ValueError) as e: + data = self._DoReadFile('225_expand_size_bad.dts') + self.assertIn( + "Node '/binman/u-boot': Please use 'extend-size' instead of 'expand-size'", + str(e.exception)) + + def testMkimageMissingBlob(self): + """Test using mkimage to build an image""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('229_mkimage_missing.dts', allow_missing=True, + allow_fake_blobs=True) + err = stderr.getvalue() + self.assertRegex( + err, + "Image '.*' has faked external blobs and is non-functional: .*") + + def testFitSplitElf(self): + """Test an image with an FIT with an split-elf operation""" + entry_args = { + 'of-list': 'test-fdt1 test-fdt2', + 'default-dt': 'test-fdt2', + 'atf-bl31-path': 'bl31.elf', + 'tee-os-path': 'tee.elf', + } + test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) + data = self._DoReadFileDtb( + '226_fit_split_elf.dts', + entry_args=entry_args, + extra_indirs=[test_subdir])[0] + + self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):]) + fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)] + + base_keys = {'description', 'type', 'arch', 'os', 'compression', + 'data', 'load'} + dtb = fdt.Fdt.FromData(fit_data) + dtb.Scan() + + elf_data = tools.read_file(os.path.join(self._indir, 'bl31.elf')) + segments, entry = elf.read_loadable_segments(elf_data) + + # We assume there are two segments + self.assertEquals(2, len(segments)) + + atf1 = dtb.GetNode('/images/atf-1') + _, start, data = segments[0] + self.assertEqual(base_keys | {'entry'}, atf1.props.keys()) + self.assertEqual(entry, + fdt_util.fdt32_to_cpu(atf1.props['entry'].value)) + self.assertEqual(start, + fdt_util.fdt32_to_cpu(atf1.props['load'].value)) + self.assertEqual(data, atf1.props['data'].bytes) + + atf2 = dtb.GetNode('/images/atf-2') + self.assertEqual(base_keys, atf2.props.keys()) + _, start, data = segments[1] + self.assertEqual(start, + fdt_util.fdt32_to_cpu(atf2.props['load'].value)) + self.assertEqual(data, atf2.props['data'].bytes) + + conf = dtb.GetNode('/configurations') + self.assertEqual({'default'}, conf.props.keys()) + + for subnode in conf.subnodes: + self.assertEqual({'description', 'fdt', 'loadables'}, + subnode.props.keys()) + self.assertEqual( + ['atf-1', 'atf-2', 'tee-1', 'tee-2'], + fdt_util.GetStringList(subnode, 'loadables')) + + def _check_bad_fit(self, dts): + """Check a bad FIT + + This runs with the given dts and returns the assertion raised + + Args: + dts (str): dts filename to use + + Returns: + str: Assertion string raised + """ + entry_args = { + 'of-list': 'test-fdt1 test-fdt2', + 'default-dt': 'test-fdt2', + 'atf-bl31-path': 'bl31.elf', + 'tee-os-path': 'tee.elf', + } + test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) + with self.assertRaises(ValueError) as exc: + self._DoReadFileDtb(dts, entry_args=entry_args, + extra_indirs=[test_subdir])[0] + return str(exc.exception) + + def testFitSplitElfBadElf(self): + """Test a FIT split-elf operation with an invalid ELF file""" + TestFunctional._MakeInputFile('bad.elf', tools.get_bytes(100, 100)) + entry_args = { + 'of-list': 'test-fdt1 test-fdt2', + 'default-dt': 'test-fdt2', + 'atf-bl31-path': 'bad.elf', + 'tee-os-path': 'tee.elf', + } + test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) + with self.assertRaises(ValueError) as exc: + self._DoReadFileDtb( + '226_fit_split_elf.dts', + entry_args=entry_args, + extra_indirs=[test_subdir])[0] + self.assertIn( + "Node '/binman/fit': subnode 'images/@atf-SEQ': Failed to read ELF file: Magic number does not match", + str(exc.exception)) + + def testFitSplitElfBadDirective(self): + """Test a FIT split-elf invalid fit,xxx directive in an image node""" + err = self._check_bad_fit('227_fit_bad_dir.dts') + self.assertIn( + "Node '/binman/fit': subnode 'images/@atf-SEQ': Unknown directive 'fit,something'", + err) + + def testFitSplitElfBadDirectiveConfig(self): + """Test a FIT split-elf with invalid fit,xxx directive in config""" + err = self._check_bad_fit('228_fit_bad_dir_config.dts') + self.assertEqual( + "Node '/binman/fit': subnode 'configurations/@config-SEQ': Unknown directive 'fit,config'", + err) + + def checkFitSplitElf(self, **kwargs): + """Test an split-elf FIT with a missing ELF file""" + entry_args = { + 'of-list': 'test-fdt1 test-fdt2', + 'default-dt': 'test-fdt2', + 'atf-bl31-path': 'bl31.elf', + 'tee-os-path': 'missing.elf', + } + test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile( + '226_fit_split_elf.dts', entry_args=entry_args, + extra_indirs=[test_subdir], **kwargs) + err = stderr.getvalue() + return err + + def testFitSplitElfMissing(self): + """Test an split-elf FIT with a missing ELF file""" + err = self.checkFitSplitElf(allow_missing=True) + self.assertRegex( + err, + "Image '.*' is missing external blobs and is non-functional: .*") + + def testFitSplitElfFaked(self): + """Test an split-elf FIT with faked ELF file""" + err = self.checkFitSplitElf(allow_missing=True, allow_fake_blobs=True) + self.assertRegex( + err, + "Image '.*' is missing external blobs and is non-functional: .*") + + def testPreLoad(self): + """Test an image with a pre-load header""" + entry_args = { + 'pre-load-key-path': '.', + } + data, _, _, _ = self._DoReadFileDtb('225_pre_load.dts', + entry_args=entry_args) + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) + self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) + self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + data = self._DoReadFile('225_pre_load.dts') + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) + self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) + self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + + def testPreLoadPkcs(self): + """Test an image with a pre-load header with padding pkcs""" + data = self._DoReadFile('226_pre_load_pkcs.dts') + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) + self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) + self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + + def testPreLoadPss(self): + """Test an image with a pre-load header with padding pss""" + data = self._DoReadFile('227_pre_load_pss.dts') + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) + self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) + self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + + def testPreLoadInvalidPadding(self): + """Test an image with a pre-load header with an invalid padding""" + with self.assertRaises(ValueError) as e: + data = self._DoReadFile('228_pre_load_invalid_padding.dts') + + def testPreLoadInvalidSha(self): + """Test an image with a pre-load header with an invalid hash""" + with self.assertRaises(ValueError) as e: + data = self._DoReadFile('229_pre_load_invalid_sha.dts') + + def testPreLoadInvalidAlgo(self): + """Test an image with a pre-load header with an invalid algo""" + with self.assertRaises(ValueError) as e: + data = self._DoReadFile('230_pre_load_invalid_algo.dts') + + def testPreLoadInvalidKey(self): + """Test an image with a pre-load header with an invalid key""" + with self.assertRaises(ValueError) as e: + data = self._DoReadFile('231_pre_load_invalid_key.dts') if __name__ == "__main__": unittest.main() diff --git a/tools/binman/main.py b/tools/binman/main.py index ab25b48b5f..9392b59adb 100755 --- a/tools/binman/main.py +++ b/tools/binman/main.py @@ -41,6 +41,7 @@ from patman import test_util # Bring in the libfdt module sys.path.insert(2, 'scripts/dtc/pylibfdt') sys.path.insert(2, os.path.join(srctree, 'scripts/dtc/pylibfdt')) +sys.path.insert(2, os.path.join(srctree, 'build-sandbox/scripts/dtc/pylibfdt')) sys.path.insert(2, os.path.join(srctree, 'build-sandbox_spl/scripts/dtc/pylibfdt')) # When running under python-coverage on Ubuntu 16.04, the dist-packages diff --git a/tools/binman/test/088_expand_size.dts b/tools/binman/test/088_extend_size.dts index c8a01308ec..f352699e37 100644 --- a/tools/binman/test/088_expand_size.dts +++ b/tools/binman/test/088_extend_size.dts @@ -5,7 +5,7 @@ binman { size = <40>; fill { - expand-size; + extend-size; fill-byte = [61]; size = <0>; }; @@ -13,7 +13,7 @@ offset = <8>; }; section { - expand-size; + extend-size; pad-byte = <0x62>; intel-mrc { }; @@ -25,7 +25,7 @@ section2 { type = "section"; fill { - expand-size; + extend-size; fill-byte = [63]; size = <0>; }; @@ -35,7 +35,7 @@ }; fill2 { type = "fill"; - expand-size; + extend-size; fill-byte = [64]; size = <0>; }; diff --git a/tools/binman/test/089_expand_size_bad.dts b/tools/binman/test/089_extend_size_bad.dts index edc0e5cf68..edc60e43fd 100644 --- a/tools/binman/test/089_expand_size_bad.dts +++ b/tools/binman/test/089_extend_size_bad.dts @@ -4,7 +4,7 @@ / { binman { _testing { - expand-size; + extend-size; return-contents-once; }; u-boot { diff --git a/tools/binman/test/121_entry_expand.dts b/tools/binman/test/121_entry_extend.dts index ebb7816db9..ebb7816db9 100644 --- a/tools/binman/test/121_entry_expand.dts +++ b/tools/binman/test/121_entry_extend.dts diff --git a/tools/binman/test/122_entry_expand_twice.dts b/tools/binman/test/122_entry_extend_twice.dts index 258cf859f4..258cf859f4 100644 --- a/tools/binman/test/122_entry_expand_twice.dts +++ b/tools/binman/test/122_entry_extend_twice.dts diff --git a/tools/binman/test/123_entry_expand_section.dts b/tools/binman/test/123_entry_extend_section.dts index 046f723434..046f723434 100644 --- a/tools/binman/test/123_entry_expand_section.dts +++ b/tools/binman/test/123_entry_extend_section.dts diff --git a/tools/binman/test/170_fit_fdt.dts b/tools/binman/test/170_fit_fdt.dts index 99d710c57e..0197ffd159 100644 --- a/tools/binman/test/170_fit_fdt.dts +++ b/tools/binman/test/170_fit_fdt.dts @@ -36,6 +36,9 @@ description = "fdt-NAME.dtb"; type = "flat_dt"; compression = "none"; + hash { + algo = "sha256"; + }; }; }; diff --git a/tools/binman/test/224_fit_bad_oper.dts b/tools/binman/test/224_fit_bad_oper.dts index cee801e2ea..8a8014ea33 100644 --- a/tools/binman/test/224_fit_bad_oper.dts +++ b/tools/binman/test/224_fit_bad_oper.dts @@ -21,7 +21,5 @@ }; }; }; - fdtmap { - }; }; }; diff --git a/tools/binman/test/225_dev.key b/tools/binman/test/225_dev.key new file mode 100644 index 0000000000..b36bad2cfb --- /dev/null +++ b/tools/binman/test/225_dev.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDYngNWUvXYRXX/ +WEUI7k164fcpv1srXz+u+5Y3Yhouw3kPs+ffvYyHAPfjF7aUIAgezKk/4o7AvsxE +Rdih3T+0deAd/q/yuqN4Adzt6ImnsO/EqdtYl3Yh+Vck9xWhLd3SAw1++GfSmNMT +gxlcc/z6z+bIh2tJNtPtRSNNHMmvYYOkBmkfwcjbMXD+fe4vBwYjVrIize+l7Yuv +1qN2nFlq56pFi8Lj5vOvFyNhZHRvwcpWdUdkx39beNUfwrGhgewOeWngTcY75n7S +FY45TBR1G2PR90CQvyDinCi9Mm0u5s+1WASQWPblovfD6CPbHQu4GZm+FAs7yUvr +hA7VCyNxAgMBAAECggEAUbq0uaJNfc8faTtNuMPo2d9eGRNI+8FRTt0/3R+Xj2NT +TvhrGUD0P4++96Df012OkshXZ3I8uD6E5ZGQ3emTeqwq5kZM7oE64jGZwO3G2k1o ++cO4reFfwgvItHrBX3HlyrI6KljhG1Vr9mW1cOuWXK+KfMiTUylrpo86dYLSGeg3 +7ZlsOPArr4eof/A0iPryQZX6X5POf7k/e9qRFYsOkoRQO8pBL3J4rIKwBl3uBN3K ++FY40vCkd8JyTo2DNfHeIe1XYA9fG2ahjD2qMsw10TUsRRMd5yhonEcJ7VzGzy8m +MnuMDAr7CwbbLkKi4UfZUl6YDkojqerwLOrxikBqkQKBgQD6sS6asDgwiq5MtstE +4/PxMrVEsCdkrU+jjQN749qIt/41a6lbp0Pr6aUKKKGs0QbcnCtlpp7qmhvymBcW +hlqxk2wokKMChv4WLXjZS3DGcOdMglc81y2F+252bToN8vwUfm6DPp9/GKtejA0a +GP57GeHxoVO7vfDX1F/vZRogRQKBgQDdNCLWOlGWvnKjfgNZHgX+Ou6ZgTSAzy+/ +hRsZPlY5nwO5iD7YkIKvqBdOmfyjlUpHWk2uAcT9pfgzYygvyBRaoQhAYBGkHItt +slaMxnLd+09wWufoCbgJvFn+wVQxBLcA5PXB98ws0Dq8ZYuo6AOuoRivsSO4lblK +MW0guBJXPQKBgQDGjf0ukbH/aGfC5Oi8SJvWhuYhYC/jQo2YKUEAKCjXLnuOThZW +PHXEbUrFcAcVfH0l0B9jJIQrpiHKlAF9Wq6MhQoeWuhxQQAQCrXzzRemZJgd9gIo +cvlgbBNCgyJ/F9vmU3kuRDRJkv1wJhbee7tbPtXA7pkGUttl5pSRZI87zQKBgQC/ +0ZkwCox72xTQP9MpcYai6nnDta5Q0NnIC+Xu4wakmwcA2WweIlqhdnMXnyLcu/YY +n+9iqHgpuMXd0eukW62C1cexA13o4TPrYU36b5BmfKprdPlLVzo3fxTPfNjEVSFY +7jNLC9YLOlrkym3sf53Jzjr5B/RA+d0ewHOwfs6wxQKBgFSyfjx5wtdHK4fO+Z1+ +q3bxouZryM/4CiPCFuw4+aZmRHPmufuNCvfXdF+IH8dM0E9ObwKZAe/aMP/Y+Abx +Wz9Vm4CP6g7k3DU3INEygyjmIQQDKQ9lFdDnsP9ESzrPbaGxZhc4x2lo7qmeW1BR +/RuiAofleFkT4s+EhLrfE/v5 +-----END PRIVATE KEY----- diff --git a/tools/binman/test/225_expand_size_bad.dts b/tools/binman/test/225_expand_size_bad.dts new file mode 100644 index 0000000000..d4ad9a6a1a --- /dev/null +++ b/tools/binman/test/225_expand_size_bad.dts @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + u-boot { + expand-size; + }; + }; +}; diff --git a/tools/binman/test/225_pre_load.dts b/tools/binman/test/225_pre_load.dts new file mode 100644 index 0000000000..c1ffe1a2ff --- /dev/null +++ b/tools/binman/test/225_pre_load.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <0x11223344>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/226_fit_split_elf.dts b/tools/binman/test/226_fit_split_elf.dts new file mode 100644 index 0000000000..fab15338b2 --- /dev/null +++ b/tools/binman/test/226_fit_split_elf.dts @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list = "of-list"; + + images { + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + }; + atf: @atf-SEQ { + fit,operation = "split-elf"; + description = "ARM Trusted Firmware"; + type = "firmware"; + arch = "arm64"; + os = "arm-trusted-firmware"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + atf-bl31 { + }; + }; + + @tee-SEQ { + fit,operation = "split-elf"; + description = "TEE"; + type = "tee"; + arch = "arm64"; + os = "tee"; + compression = "none"; + fit,load; + fit,entry; + fit,data; + + tee-os { + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + config: @config-SEQ { + description = "conf-NAME.dtb"; + fdt = "fdt-SEQ"; + fit,loadables; + }; + }; + }; + + u-boot-nodtb { + }; + }; +}; diff --git a/tools/binman/test/226_pre_load_pkcs.dts b/tools/binman/test/226_pre_load_pkcs.dts new file mode 100644 index 0000000000..3db0a37f77 --- /dev/null +++ b/tools/binman/test/226_pre_load_pkcs.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + padding-name = "pkcs-1.5"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <0x11223344>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/227_fit_bad_dir.dts b/tools/binman/test/227_fit_bad_dir.dts new file mode 100644 index 0000000000..51f4816c4c --- /dev/null +++ b/tools/binman/test/227_fit_bad_dir.dts @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +#include "226_fit_split_elf.dts" + +&atf { + fit,something = "bad"; +}; diff --git a/tools/binman/test/227_pre_load_pss.dts b/tools/binman/test/227_pre_load_pss.dts new file mode 100644 index 0000000000..b1b01d5ad5 --- /dev/null +++ b/tools/binman/test/227_pre_load_pss.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + padding-name = "pss"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <0x11223344>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/228_fit_bad_dir_config.dts b/tools/binman/test/228_fit_bad_dir_config.dts new file mode 100644 index 0000000000..825a346c3e --- /dev/null +++ b/tools/binman/test/228_fit_bad_dir_config.dts @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +#include "226_fit_split_elf.dts" + +&config { + fit,config = "bad"; +}; diff --git a/tools/binman/test/228_pre_load_invalid_padding.dts b/tools/binman/test/228_pre_load_invalid_padding.dts new file mode 100644 index 0000000000..84fe289183 --- /dev/null +++ b/tools/binman/test/228_pre_load_invalid_padding.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa2048"; + padding-name = "padding"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/229_mkimage_missing.dts b/tools/binman/test/229_mkimage_missing.dts new file mode 100644 index 0000000000..54a5a6c571 --- /dev/null +++ b/tools/binman/test/229_mkimage_missing.dts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + mkimage { + args = "-n test -T script"; + + blob-ext { + filename = "missing.bin"; + }; + }; + }; +}; diff --git a/tools/binman/test/229_pre_load_invalid_sha.dts b/tools/binman/test/229_pre_load_invalid_sha.dts new file mode 100644 index 0000000000..a2b6725c89 --- /dev/null +++ b/tools/binman/test/229_pre_load_invalid_sha.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha2560,rsa2048"; + padding-name = "pkcs-1.5"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/230_pre_load_invalid_algo.dts b/tools/binman/test/230_pre_load_invalid_algo.dts new file mode 100644 index 0000000000..34c8d34f15 --- /dev/null +++ b/tools/binman/test/230_pre_load_invalid_algo.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa20480"; + padding-name = "pkcs-1.5"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/binman/test/231_pre_load_invalid_key.dts b/tools/binman/test/231_pre_load_invalid_key.dts new file mode 100644 index 0000000000..08d5a75ddf --- /dev/null +++ b/tools/binman/test/231_pre_load_invalid_key.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + pre-load { + content = <&image>; + algo-name = "sha256,rsa4096"; + padding-name = "pkcs-1.5"; + key-name = "tools/binman/test/225_dev.key"; + header-size = <4096>; + version = <1>; + }; + + image: blob-ext { + filename = "refcode.bin"; + }; + }; +}; diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index 0b7208da37..e634bbb279 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -74,6 +74,7 @@ def CreateBuildmanConfigFile(config_fname): print('''[toolchain] # name = path # e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux +other = / [toolchain-prefix] # name = path to prefix diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 754642d4a6..ecbfa3e361 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1763,7 +1763,7 @@ class Builder: if self.num_threads: self.queue.put(job) else: - results = self._single_builder.RunJob(job) + self._single_builder.RunJob(job) if self.num_threads: term = threading.Thread(target=self.queue.join) diff --git a/tools/buildman/cfgutil.py b/tools/buildman/cfgutil.py index 4eba50868f..ab74a8ef06 100644 --- a/tools/buildman/cfgutil.py +++ b/tools/buildman/cfgutil.py @@ -123,10 +123,10 @@ def adjust_cfg_file(fname, adjust_cfg): C=val to set the value of C (val must have quotes if C is a string Kconfig) """ - lines = tools.ReadFile(fname, binary=False).splitlines() + lines = tools.read_file(fname, binary=False).splitlines() out_lines = adjust_cfg_lines(lines, adjust_cfg) out = '\n'.join(out_lines) + '\n' - tools.WriteFile(fname, out, binary=False) + tools.write_file(fname, out, binary=False) def convert_list_to_dict(adjust_cfg_list): """Convert a list of config changes into the dict used by adjust_cfg_file() @@ -219,7 +219,7 @@ def check_cfg_file(fname, adjust_cfg): Returns: str: None if OK, else an error string listing the problems """ - lines = tools.ReadFile(fname, binary=False).splitlines() + lines = tools.read_file(fname, binary=False).splitlines() bad_cfgs = check_cfg_lines(lines, adjust_cfg) if bad_cfgs: out = [f'{cfg:20} {line}' for cfg, line in bad_cfgs] diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6fcceb0ea5..fbf6706644 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -524,12 +524,6 @@ class TestFunctional(unittest.TestCase): # Each commit has a config and make self.assertEqual(self._make_calls, len(boards) * self._commits * 2) - def testForceReconfigure(self): - """The -f flag should force a rebuild""" - self._RunControl('-b', TEST_BRANCH, '-C', '-o', self._output_dir) - # Each commit has a config and make - self.assertEqual(self._make_calls, len(boards) * self._commits * 2) - def testMrproper(self): """The -f flag should force a rebuild""" self._RunControl('-b', TEST_BRANCH, '-m', '-o', self._output_dir) diff --git a/tools/buildman/kconfiglib.py b/tools/buildman/kconfiglib.py index c67895ced6..b9f3756755 100644 --- a/tools/buildman/kconfiglib.py +++ b/tools/buildman/kconfiglib.py @@ -556,6 +556,7 @@ from os.path import dirname, exists, expandvars, islink, join, realpath VERSION = (14, 1, 0) +# pylint: disable=E1101 # File layout: # diff --git a/tools/buildman/main.py b/tools/buildman/main.py index 01271061e6..3b6af24080 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -30,8 +30,8 @@ from patman import terminal from patman import test_util def RunTests(skip_net_tests, verboose, args): - import func_test - import test + from buildman import func_test + from buildman import test import doctest result = unittest.TestResult() diff --git a/tools/concurrencytest/__init__.py b/tools/concurrencytest/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/concurrencytest/__init__.py diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 7a3f5cbade..c51e343013 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -2,7 +2,7 @@ # This Dockerfile is used to build an image containing basic stuff to be used # to build U-Boot and run our test suites. -FROM ubuntu:focal-20220113 +FROM ubuntu:focal-20220302 MAINTAINER Tom Rini <trini@konsulko.com> LABEL Description=" This image is for building U-Boot inside a container" @@ -48,6 +48,7 @@ RUN apt-get update && apt-get install -y \ dosfstools \ e2fsprogs \ efitools \ + erofs-utils \ expect \ fakeroot \ flex \ diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index c16909a876..d933972918 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -516,9 +516,7 @@ class Node: Returns: Prop added """ - out = b'' - for string in val: - out += bytes(string, 'utf-8') + b'\0' + out = b'\0'.join(bytes(s, 'utf-8') for s in val) + b'\0' if val else b'' return self.AddData(prop_name, out) def AddInt(self, prop_name, val): diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index c82e7747aa..d7c38ad1e0 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -158,6 +158,8 @@ def GetString(node, propname, default=None): if not prop: return default value = prop.value + if not prop.bytes: + return '' if isinstance(value, list): raise ValueError("Node '%s' property '%s' has list value: expecting " "a single string" % (node.name, propname)) @@ -179,6 +181,8 @@ def GetStringList(node, propname, default=None): if not prop: return default value = prop.value + if not prop.bytes: + return [] if not isinstance(value, list): strval = GetString(node, propname) return [strval] @@ -192,8 +196,12 @@ def GetArgs(node, propname): value = GetStringList(node, propname) else: value = [] - lists = [v.split() for v in value] - args = [x for l in lists for x in l] + if not value: + args = [] + elif len(value) == 1: + args = value[0].split() + else: + args = value return args def GetBool(node, propname, default=False): diff --git a/tools/dtoc/test/dtoc_test_simple.dts b/tools/dtoc/test/dtoc_test_simple.dts index 2d321fb034..aef07efeae 100644 --- a/tools/dtoc/test/dtoc_test_simple.dts +++ b/tools/dtoc/test/dtoc_test_simple.dts @@ -63,5 +63,7 @@ orig-node { orig = <1 23 4>; args = "-n first", "second", "-p", "123,456", "-x"; + args2 = "a space", "there"; + args3 = "-n first second -p 123,456 -x"; }; }; diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index ea88954923..c81bcc9c32 100755 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -16,13 +16,13 @@ import os import struct import unittest -from dtb_platdata import Ftype -from dtb_platdata import get_value -from dtb_platdata import tab_to from dtoc import dtb_platdata from dtoc import fdt from dtoc import fdt_util from dtoc import src_scan +from dtoc.dtb_platdata import Ftype +from dtoc.dtb_platdata import get_value +from dtoc.dtb_platdata import tab_to from dtoc.src_scan import conv_name_to_c from dtoc.src_scan import get_compat_name from patman import test_util diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index 576d65b97e..914ed6aed5 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -25,7 +25,7 @@ sys.path.insert(2, os.path.join(our_path, from dtoc import fdt from dtoc import fdt_util from dtoc.fdt_util import fdt32_to_cpu, fdt64_to_cpu -from fdt import Type, BytesToValue +from dtoc.fdt import Type, BytesToValue import libfdt from patman import command from patman import test_util @@ -119,9 +119,9 @@ class TestFdt(unittest.TestCase): """Test that packing a device tree works""" self.dtb.Pack() - def testGetFdt(self): + def testGetFdtRaw(self): """Tetst that we can access the raw device-tree data""" - self.assertTrue(isinstance(self.dtb.GetContents(), bytearray)) + self.assertTrue(isinstance(self.dtb.GetContents(), bytes)) def testGetProps(self): """Tests obtaining a list of properties""" @@ -550,6 +550,12 @@ class TestProp(unittest.TestCase): data = self.fdt.getprop(self.node.Offset(), 'stringlist') self.assertEqual(b'123\x00456\0', data) + val = [] + self.node.AddStringList('stringlist', val) + self.dtb.Sync(auto_resize=True) + data = self.fdt.getprop(self.node.Offset(), 'stringlist') + self.assertEqual(b'', data) + def test_delete_node(self): """Test deleting a node""" old_offset = self.fdt.path_offset('/spl-test') @@ -637,6 +643,7 @@ class TestFdtUtil(unittest.TestCase): self.assertEqual('message', fdt_util.GetString(self.node, 'stringval')) self.assertEqual('test', fdt_util.GetString(self.node, 'missing', 'test')) + self.assertEqual('', fdt_util.GetString(self.node, 'boolval')) with self.assertRaises(ValueError) as e: self.assertEqual(3, fdt_util.GetString(self.node, 'stringarray')) @@ -651,6 +658,7 @@ class TestFdtUtil(unittest.TestCase): fdt_util.GetStringList(self.node, 'stringarray')) self.assertEqual(['test'], fdt_util.GetStringList(self.node, 'missing', ['test'])) + self.assertEqual([], fdt_util.GetStringList(self.node, 'boolval')) def testGetArgs(self): node = self.dtb.GetNode('/orig-node') @@ -659,8 +667,12 @@ class TestFdtUtil(unittest.TestCase): ['multi-word', 'message'], fdt_util.GetArgs(self.node, 'stringarray')) self.assertEqual([], fdt_util.GetArgs(self.node, 'boolval')) - self.assertEqual(['-n', 'first', 'second', '-p', '123,456', '-x'], + self.assertEqual(['-n first', 'second', '-p', '123,456', '-x'], fdt_util.GetArgs(node, 'args')) + self.assertEqual(['a space', 'there'], + fdt_util.GetArgs(node, 'args2')) + self.assertEqual(['-n', 'first', 'second', '-p', '123,456', '-x'], + fdt_util.GetArgs(node, 'args3')) with self.assertRaises(ValueError) as exc: fdt_util.GetArgs(self.node, 'missing') self.assertIn( diff --git a/tools/fit_image.c b/tools/fit_image.c index 15f7c82d61..1884a2eb0b 100644 --- a/tools/fit_image.c +++ b/tools/fit_image.c @@ -59,6 +59,9 @@ static int fit_add_file_data(struct image_tool_params *params, size_t size_inc, ret = fit_set_timestamp(ptr, 0, time); } + if (!ret) + ret = fit_pre_load_data(params->keydir, dest_blob, ptr); + if (!ret) { ret = fit_cipher_data(params->keydir, dest_blob, ptr, params->comment, diff --git a/tools/image-host.c b/tools/image-host.c index eaeb76545c..ab6f756cf1 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -14,6 +14,11 @@ #include <image.h> #include <version.h> +#include <openssl/pem.h> +#include <openssl/evp.h> + +#define IMAGE_PRE_LOAD_PATH "/image/pre-load/sig" + /** * fit_set_hash_value - set hash value in requested has node * @fit: pointer to the FIT format image header @@ -1111,6 +1116,115 @@ static int fit_config_add_verification_data(const char *keydir, return 0; } +/* + * 0) open file (open) + * 1) read certificate (PEM_read_X509) + * 2) get public key (X509_get_pubkey) + * 3) provide der format (d2i_RSAPublicKey) + */ +static int read_pub_key(const char *keydir, const void *name, + unsigned char **pubkey, int *pubkey_len) +{ + char path[1024]; + EVP_PKEY *key = NULL; + X509 *cert; + FILE *f; + int ret; + + memset(path, 0, 1024); + snprintf(path, sizeof(path), "%s/%s.crt", keydir, (char *)name); + + /* Open certificate file */ + f = fopen(path, "r"); + if (!f) { + fprintf(stderr, "Couldn't open RSA certificate: '%s': %s\n", + path, strerror(errno)); + return -EACCES; + } + + /* Read the certificate */ + cert = NULL; + if (!PEM_read_X509(f, &cert, NULL, NULL)) { + printf("Couldn't read certificate"); + ret = -EINVAL; + goto err_cert; + } + + /* Get the public key from the certificate. */ + key = X509_get_pubkey(cert); + if (!key) { + printf("Couldn't read public key\n"); + ret = -EINVAL; + goto err_pubkey; + } + + /* Get DER form */ + ret = i2d_PublicKey(key, pubkey); + if (ret < 0) { + printf("Couldn't get DER form\n"); + ret = -EINVAL; + goto err_pubkey; + } + + *pubkey_len = ret; + ret = 0; + +err_pubkey: + X509_free(cert); +err_cert: + fclose(f); + return ret; +} + +int fit_pre_load_data(const char *keydir, void *keydest, void *fit) +{ + int pre_load_noffset; + const void *algo_name; + const void *key_name; + unsigned char *pubkey = NULL; + int ret, pubkey_len; + + if (!keydir || !keydest || !fit) + return 0; + + /* Search node pre-load sig */ + pre_load_noffset = fdt_path_offset(keydest, IMAGE_PRE_LOAD_PATH); + if (pre_load_noffset < 0) { + ret = 0; + goto out; + } + + algo_name = fdt_getprop(keydest, pre_load_noffset, "algo-name", NULL); + key_name = fdt_getprop(keydest, pre_load_noffset, "key-name", NULL); + + /* Check that all mandatory properties are present */ + if (!algo_name || !key_name) { + if (!algo_name) + printf("The property algo-name is missing in the node %s\n", + IMAGE_PRE_LOAD_PATH); + if (!key_name) + printf("The property key-name is missing in the node %s\n", + IMAGE_PRE_LOAD_PATH); + ret = -ENODATA; + goto out; + } + + /* Read public key */ + ret = read_pub_key(keydir, key_name, &pubkey, &pubkey_len); + if (ret < 0) + goto out; + + /* Add the public key to the device tree */ + ret = fdt_setprop(keydest, pre_load_noffset, "public-key", + pubkey, pubkey_len); + if (ret) + printf("Can't set public-key in node %s (ret = %d)\n", + IMAGE_PRE_LOAD_PATH, ret); + + out: + return ret; +} + int fit_cipher_data(const char *keydir, void *keydest, void *fit, const char *comment, int require_keys, const char *engine_id, const char *cmdname) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index cff1e30658..84bc875fff 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -94,17 +94,6 @@ SIZES = { RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig') ### helper functions ### -def remove_defconfig(defc): - """Drop the _defconfig suffix on a string - - Args: - defc (str): String to convert - - Returns: - str: string with the '_defconfig' suffix removed - """ - return RE_REMOVE_DEFCONFIG.match(defc)[1] - def check_top_directory(): """Exit if we are not at the top of source directory.""" for fname in 'README', 'Licenses': @@ -339,7 +328,7 @@ def read_file(fname, as_lines=True, skip_unicode=False): return inf.read() except UnicodeDecodeError as e: if not skip_unicode: - raises + raise print("Failed on file %s': %s" % (fname, e)) return None @@ -790,9 +779,6 @@ class KconfigParser: actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ % value log_color = COLOR_LIGHT_PURPLE - elif action == ACTION_SPL_NOT_EXIST: - actlog = 'SPL is not enabled for this defconfig. Skip.' - log_color = COLOR_PURPLE else: sys.exit('Internal Error. This should not happen.') @@ -1621,8 +1607,7 @@ def defconfig_matches(configs, re_match): bool: True if any CONFIG matches the regex """ for cfg in configs: - m_cfg = re_match.match(cfg) - if m_cfg and m_cfg.span()[1] == len(cfg): + if re_match.fullmatch(cfg): return True return False @@ -1671,7 +1656,7 @@ def do_find_config(config_list): print(f"Error: Not in Kconfig: %s" % ' '.join(adhoc)) else: print(f'{len(out)} matches') - print(' '.join([remove_defconfig(item) for item in out])) + print(' '.join(item.split('_defconfig')[0] for item in out)) def prefix_config(cfg): diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index dd792efee0..70ba561c26 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -125,7 +125,7 @@ def check_patch_parse(checkpatch_output, verbose=False): Returns: namedtuple containing: ok: False=failure, True=ok - problems: List of problems, each a dict: + problems (list of problems): each a dict: 'type'; error or warning 'msg': text message 'file' : filename @@ -252,6 +252,8 @@ def check_patches(verbose, args): if (len(result.problems) != result.errors + result.warnings + result.checks): print("Internal error: some problems lost") + # Python seems to get confused by this + # pylint: disable=E1133 for item in result.problems: sys.stderr.write( get_warning_msg(col, item.get('type', '<unknown>'), diff --git a/tools/patman/command.py b/tools/patman/command.py index 24358784f2..92c453b5c1 100644 --- a/tools/patman/command.py +++ b/tools/patman/command.py @@ -17,13 +17,6 @@ class CommandResult: return_code: Return code from command exception: Exception received, or None if all ok """ - def __init__(self): - self.stdout = None - self.stderr = None - self.combined = None - self.return_code = None - self.exception = None - def __init__(self, stdout='', stderr='', combined='', return_code=0, exception=None): self.stdout = stdout @@ -72,6 +65,7 @@ def run_pipe(pipe_list, infile=None, outfile=None, """ if test_result: if hasattr(test_result, '__call__'): + # pylint: disable=E1102 result = test_result(pipe_list=pipe_list) if result: return result diff --git a/tools/patman/commit.py b/tools/patman/commit.py index c331a3b122..9537de43d3 100644 --- a/tools/patman/commit.py +++ b/tools/patman/commit.py @@ -28,10 +28,11 @@ class Commit: key: rtag type (e.g. 'Reviewed-by') value: Set of people who gave that rtag, each a name/email string warn: List of warnings for this commit, each a str + patch (str): Filename of the patch file for this commit """ def __init__(self, hash): self.hash = hash - self.subject = None + self.subject = '' self.tags = [] self.changes = {} self.cc_list = [] @@ -40,6 +41,7 @@ class Commit: self.change_id = None self.rtags = collections.defaultdict(set) self.warn = [] + self.patch = '' def __str__(self): return self.subject diff --git a/tools/patman/cros_subprocess.py b/tools/patman/cros_subprocess.py index f1b26087cf..cd614f38a6 100644 --- a/tools/patman/cros_subprocess.py +++ b/tools/patman/cros_subprocess.py @@ -113,7 +113,7 @@ class Popen(subprocess.Popen): return b'' return data - def communicate_filter(self, output): + def communicate_filter(self, output, input_buf=''): """Interact with process: Read data from stdout and stderr. This method runs until end-of-file is reached, then waits for the @@ -166,7 +166,7 @@ class Popen(subprocess.Popen): # Flush stdio buffer. This might block, if the user has # been writing to .stdin in an uncontrolled fashion. self.stdin.flush() - if input: + if input_buf: write_set.append(self.stdin) else: self.stdin.close() @@ -195,10 +195,10 @@ class Popen(subprocess.Popen): # When select has indicated that the file is writable, # we can write up to PIPE_BUF bytes without risk # blocking. POSIX defines PIPE_BUF >= 512 - chunk = input[input_offset : input_offset + 512] + chunk = input_buf[input_offset : input_offset + 512] bytes_written = os.write(self.stdin.fileno(), chunk) input_offset += bytes_written - if input_offset >= len(input): + if input_offset >= len(input_buf): self.stdin.close() write_set.remove(self.stdin) @@ -240,16 +240,6 @@ class Popen(subprocess.Popen): stderr = self.convert_data(stderr) combined = self.convert_data(combined) - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - self.wait() return (stdout, stderr, combined) diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index 59ee90c344..7b92bc67be 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -341,6 +341,8 @@ Changes in v2: tools.write_file(path, text, binary=False) index = self.repo.index index.add(fname) + # pylint doesn't seem to find this + # pylint: disable=E1101 author = pygit2.Signature('Test user', 'test@email.com') committer = author tree = index.write_tree() @@ -363,6 +365,8 @@ Changes in v2: self.repo = repo new_tree = repo.TreeBuilder().write() + # pylint doesn't seem to find this + # pylint: disable=E1101 author = pygit2.Signature('Test user', 'test@email.com') committer = author _ = repo.create_commit('HEAD', author, committer, 'Created master', @@ -414,6 +418,8 @@ better than before''') first_target = repo.revparse_single('HEAD') target = repo.revparse_single('HEAD~2') + # pylint doesn't seem to find this + # pylint: disable=E1101 repo.reset(target.oid, pygit2.GIT_CHECKOUT_FORCE) self.make_commit_with_file('video: Some video improvements', ''' Fix up the video so that @@ -459,6 +465,8 @@ complicated as possible''') """Test creating patches from a branch""" repo = self.make_git_tree() target = repo.lookup_reference('refs/heads/first') + # pylint doesn't seem to find this + # pylint: disable=E1101 self.repo.checkout(target, strategy=pygit2.GIT_CHECKOUT_FORCE) control.setup() try: @@ -615,6 +623,8 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c """Test CountCommitsToBranch when there is no upstream""" repo = self.make_git_tree() target = repo.lookup_reference('refs/heads/base') + # pylint doesn't seem to find this + # pylint: disable=E1101 self.repo.checkout(target, strategy=pygit2.GIT_CHECKOUT_FORCE) # Check that it can detect the current branch diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index 9b32fd4790..fb6a6036f3 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -597,7 +597,7 @@ class PatchStream: if 'prefix' in self.series: parts.append(self.series['prefix']) if 'postfix' in self.series: - parts.append(self.serties['postfix']) + parts.append(self.series['postfix']) if 'version' in self.series: parts.append("v%s" % self.series['version']) diff --git a/tools/patman/series.py b/tools/patman/series.py index 891f278534..3075378ac1 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -122,8 +122,7 @@ class Series(dict): cc_list = list(self._generated_cc[commit.patch]) for email in sorted(set(cc_list) - to_set - cc_set): if email == None: - email = col.build(col.YELLOW, "<alias '%s' not found>" - % tag) + email = col.build(col.YELLOW, '<alias not found>') if email: print(' Cc: ', email) print diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 014bb376d8..7c2b5c196c 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -200,12 +200,12 @@ def CreatePatmanConfigFile(gitutil, config_fname): """ name = gitutil.get_default_user_name() if name == None: - name = raw_input("Enter name: ") + name = input("Enter name: ") email = gitutil.get_default_user_email() if email == None: - email = raw_input("Enter email: ") + email = input("Enter email: ") try: f = open(config_fname, 'w') diff --git a/tools/patman/tools.py b/tools/patman/tools.py index 5e4d4ac05c..2ac814d476 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -62,8 +62,8 @@ def prepare_output_dir(dirname, preserve=False): try: os.makedirs(outdir) except OSError as err: - raise CmdError("Cannot make output directory '%s': '%s'" % - (outdir, err.strerror)) + raise ValueError( + f"Cannot make output directory 'outdir': 'err.strerror'") tout.debug("Using output directory '%s'" % outdir) else: outdir = tempfile.mkdtemp(prefix='binman.') @@ -160,7 +160,7 @@ def get_input_filename_glob(pattern): A list of matching files in all input directories """ if not indir: - return glob.glob(fname) + return glob.glob(pattern) files = [] for dirname in indir: pathname = os.path.join(dirname, pattern) @@ -201,7 +201,7 @@ def path_has_file(path_spec, fname): return True return False -def get_host_compile_tool(name): +def get_host_compile_tool(env, name): """Get the host-specific version for a compile tool This checks the environment variables that specify which version of @@ -356,7 +356,7 @@ def run_result(name, *args, **kwargs): name, extra_args = get_target_compile_tool(name) args = tuple(extra_args) + args elif for_host: - name, extra_args = get_host_compile_tool(name) + name, extra_args = get_host_compile_tool(env, name) args = tuple(extra_args) + args name = os.path.expanduser(name) # Expand paths containing ~ all_args = (name,) + args |