binman: Support adding files

In some cases it is useful to add a group of files to the image and be
able to access them at run-time. Of course it is possible to generate
the binman config file with a set of blobs each with a filename. But for
convenience, add an entry type which can do this.

Add required support (for adding nodes and string properties) into the
state module.

Signed-off-by: Simon Glass <sjg@chromium.org>

17 files changed, 225 insertions, 0 deletions

diff --git a/tools/binman/README.entries b/tools/binman/README.entries
index 2cf7dc0338..3afc560052 100644
--- a/tools/binman/README.entries
+++ b/tools/binman/README.entries
@@ -71,6 +71,21 @@ updating the EC on startup via software sync.
 
 
 
+Entry: files: Entry containing a set of files
+---------------------------------------------
+
+Properties / Entry arguments:
+    - pattern: Filename pattern to match the files to include
+    - compress: Compression algorithm to use:
+        none: No compression
+        lz4: Use lz4 compression (via 'lz4' command-line utility)
+
+This entry reads a number of files and places each in a separate sub-entry
+within this entry. To access these you need to enable device-tree updates
+at run-time so you can obtain the file positions.
+
+
+
 Entry: fill: An entry which is filled to a particular byte value
 ----------------------------------------------------------------
 
diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py
index 1c37d84e99..4bf206878d 100644
--- a/tools/binman/bsection.py
+++ b/tools/binman/bsection.py
@@ -103,6 +103,10 @@ class Section(object):
     def SetOffset(self, offset):
         self._offset = offset
 
+    def ExpandEntries(self):
+        for entry in self._entries.values():
+            entry.ExpandEntries()
+
     def AddMissingProperties(self):
         """Add new properties to the device tree as needed for this entry"""
         for prop in ['offset', 'size', 'image-pos']:
diff --git a/tools/binman/control.py b/tools/binman/control.py
index e326456a4b..caa194c899 100644
--- a/tools/binman/control.py
+++ b/tools/binman/control.py
@@ -146,6 +146,7 @@ def Binman(options, args):
             # without changing the device-tree size, thus ensuring that our
             # entry offsets remain the same.
             for image in images.values():
+                image.ExpandEntries()
                 if options.update_fdt:
                     image.AddMissingProperties()
                 image.ProcessFdt(dtb)
diff --git a/tools/binman/entry.py b/tools/binman/entry.py
index f922107523..7316ad43b5 100644
--- a/tools/binman/entry.py
+++ b/tools/binman/entry.py
@@ -179,6 +179,9 @@ class Entry(object):
             return Set([fname])
         return Set()
 
+    def ExpandEntries(self):
+        pass
+
     def AddMissingProperties(self):
         """Add new properties to the device tree as needed for this entry"""
         for prop in ['offset', 'size', 'image-pos']:
diff --git a/tools/binman/etype/files.py b/tools/binman/etype/files.py
new file mode 100644
index 0000000000..99f2f2f67f
--- /dev/null
+++ b/tools/binman/etype/files.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2018 Google, Inc
+# Written by Simon Glass <sjg@chromium.org>
+#
+# Entry-type module for a set of files which are placed in individual
+# sub-entries
+#
+
+import glob
+import os
+
+from section import Entry_section
+import fdt_util
+import state
+import tools
+
+import bsection
+
+class Entry_files(Entry_section):
+    """Entry containing a set of files
+
+    Properties / Entry arguments:
+        - pattern: Filename pattern to match the files to include
+        - compress: Compression algorithm to use:
+            none: No compression
+            lz4: Use lz4 compression (via 'lz4' command-line utility)
+
+    This entry reads a number of files and places each in a separate sub-entry
+    within this entry. To access these you need to enable device-tree updates
+    at run-time so you can obtain the file positions.
+    """
+    def __init__(self, section, etype, node):
+        Entry_section.__init__(self, section, etype, node)
+        self._pattern = fdt_util.GetString(self._node, 'pattern')
+        if not self._pattern:
+            self.Raise("Missing 'pattern' property")
+        self._compress = fdt_util.GetString(self._node, 'compress', 'none')
+        self._require_matches = fdt_util.GetBool(self._node,
+                                                'require-matches')
+
+    def ExpandEntries(self):
+        files = tools.GetInputFilenameGlob(self._pattern)
+        if self._require_matches and not files:
+            self.Raise("Pattern '%s' matched no files" % self._pattern)
+        for fname in files:
+            if not os.path.isfile(fname):
+                continue
+            name = os.path.basename(fname)
+            subnode = self._node.FindNode(name)
+            if not subnode:
+                subnode = state.AddSubnode(self._node, name)
+            state.AddString(subnode, 'type', 'blob')
+            state.AddString(subnode, 'filename', fname)
+            state.AddString(subnode, 'compress', self._compress)
+
+        # Read entries again, now that we have some
+        self._section._ReadEntries()
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index 1c3c46fc23..e919e702b5 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -54,6 +54,8 @@ CROS_EC_RW_DATA       = 'ecrw'
 GBB_DATA              = 'gbbd'
 BMPBLK_DATA           = 'bmp'
 VBLOCK_DATA           = 'vblk'
+FILES_DATA            = ("sorry I'm late\nOh, don't bother apologising, I'm " +
+                         "sorry you're alive\n")
 COMPRESS_DATA         = 'data to compress'
 
 
@@ -117,6 +119,9 @@ class TestFunctional(unittest.TestCase):
         with open(self.TestFile('descriptor.bin')) as fd:
             TestFunctional._MakeInputFile('descriptor.bin', fd.read())
 
+        shutil.copytree(self.TestFile('files'),
+                        os.path.join(self._indir, 'files'))
+
         TestFunctional._MakeInputFile('compress', COMPRESS_DATA)
 
     @classmethod
@@ -1536,6 +1541,44 @@ class TestFunctional(unittest.TestCase):
             }
         self.assertEqual(expected, props)
 
+    def testFiles(self):
+        """Test bringing in multiple files"""
+        data = self._DoReadFile('84_files.dts')
+        self.assertEqual(FILES_DATA, data)
+
+    def testFilesCompress(self):
+        """Test bringing in multiple files and compressing them"""
+        data = self._DoReadFile('85_files_compress.dts')
+
+        image = control.images['image']
+        entries = image.GetEntries()
+        files = entries['files']
+        entries = files._section._entries
+
+        orig = ''
+        for i in range(1, 3):
+            key = '%d.dat' % i
+            start = entries[key].image_pos
+            len = entries[key].size
+            chunk = data[start:start + len]
+            orig += self._decompress(chunk)
+
+        self.assertEqual(FILES_DATA, orig)
+
+    def testFilesMissing(self):
+        """Test missing files"""
+        with self.assertRaises(ValueError) as e:
+            data = self._DoReadFile('86_files_none.dts')
+        self.assertIn("Node '/binman/files': Pattern \'files/*.none\' matched "
+                      'no files', str(e.exception))
+
+    def testFilesNoPattern(self):
+        """Test missing files"""
+        with self.assertRaises(ValueError) as e:
+            data = self._DoReadFile('87_files_no_pattern.dts')
+        self.assertIn("Node '/binman/files': Missing 'pattern' property",
+                      str(e.exception))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tools/binman/image.py b/tools/binman/image.py
index bfb2229779..4b922b51c4 100644
--- a/tools/binman/image.py
+++ b/tools/binman/image.py
@@ -58,6 +58,15 @@ class Image:
         """Get the set of device tree files used by this image"""
         return self._section.GetFdtSet()
 
+    def ExpandEntries(self):
+        """Expand out any entries which have calculated sub-entries
+
+        Some entries are expanded out at runtime, e.g. 'files', which produces
+        a section containing a list of files. Process these entries so that
+        this information is added to the device tree.
+        """
+        self._section.ExpandEntries()
+
     def AddMissingProperties(self):
         """Add properties that are not present in the device tree
 
diff --git a/tools/binman/state.py b/tools/binman/state.py
index 09ead442a6..2f8c0863a8 100644
--- a/tools/binman/state.py
+++ b/tools/binman/state.py
@@ -189,6 +189,33 @@ def AddZeroProp(node, prop):
     for n in GetUpdateNodes(node):
         n.AddZeroProp(prop)
 
+def AddSubnode(node, name):
+    """Add a new subnode to a node in affected device trees
+
+    Args:
+        node: Node to add to
+        name: name of node to add
+
+    Returns:
+        New subnode that was created in main tree
+    """
+    first = None
+    for n in GetUpdateNodes(node):
+        subnode = n.AddSubnode(name)
+        if not first:
+            first = subnode
+    return first
+
+def AddString(node, prop, value):
+    """Add a new string property to affected device trees
+
+    Args:
+        prop_name: Name of property
+        value: String value (which will be \0-terminated in the DT)
+    """
+    for n in GetUpdateNodes(node):
+        n.AddString(prop, value)
+
 def SetInt(node, prop, value):
     """Update an integer property in affected device trees with an integer value
 
diff --git a/tools/binman/test/84_files.dts b/tools/binman/test/84_files.dts
new file mode 100644
index 0000000000..83ddb78f8e
--- /dev/null
+++ b/tools/binman/test/84_files.dts
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+/ {
+	binman {
+		files {
+			pattern = "files/*.dat";
+			compress = "none";
+		};
+	};
+};
diff --git a/tools/binman/test/85_files_compress.dts b/tools/binman/test/85_files_compress.dts
new file mode 100644
index 0000000000..847b398bf2
--- /dev/null
+++ b/tools/binman/test/85_files_compress.dts
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+/ {
+	binman {
+		files {
+			pattern = "files/*.dat";
+			compress = "lz4";
+		};
+	};
+};
diff --git a/tools/binman/test/86_files_none.dts b/tools/binman/test/86_files_none.dts
new file mode 100644
index 0000000000..34bd92f224
--- /dev/null
+++ b/tools/binman/test/86_files_none.dts
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+/ {
+	binman {
+		files {
+			pattern = "files/*.none";
+			compress = "none";
+			require-matches;
+		};
+	};
+};
diff --git a/tools/binman/test/87_files_no_pattern.dts b/tools/binman/test/87_files_no_pattern.dts
new file mode 100644
index 0000000000..0cb5b469cb
--- /dev/null
+++ b/tools/binman/test/87_files_no_pattern.dts
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+/ {
+	binman {
+		files {
+			compress = "none";
+			require-matches;
+		};
+	};
+};
diff --git a/tools/binman/test/files/1.dat b/tools/binman/test/files/1.dat
new file mode 100644
index 0000000000..a952470617
--- /dev/null
+++ b/tools/binman/test/files/1.dat
@@ -0,0 +1 @@
+sorry I'm late
diff --git a/tools/binman/test/files/2.dat b/tools/binman/test/files/2.dat
new file mode 100644
index 0000000000..687ea52730
--- /dev/null
+++ b/tools/binman/test/files/2.dat
@@ -0,0 +1 @@
+Oh, don't bother apologising, I'm sorry you're alive
diff --git a/tools/binman/test/files/ignored_dir.dat/ignore b/tools/binman/test/files/ignored_dir.dat/ignore
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/binman/test/files/ignored_dir.dat/ignore
diff --git a/tools/binman/test/files/not-this-one b/tools/binman/test/files/not-this-one
new file mode 100644
index 0000000000..e71c2250f9
--- /dev/null
+++ b/tools/binman/test/files/not-this-one
@@ -0,0 +1 @@
+this does not have a .dat extenion
diff --git a/tools/patman/tools.py b/tools/patman/tools.py
index 0870bccca0..1c9bf4e810 100644
--- a/tools/patman/tools.py
+++ b/tools/patman/tools.py
@@ -4,6 +4,7 @@
 #
 
 import command
+import glob
 import os
 import shutil
 import tempfile
@@ -123,6 +124,23 @@ def GetInputFilename(fname):
     raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" %
                      (fname, ','.join(indir), os.getcwd()))
 
+def GetInputFilenameGlob(pattern):
+    """Return a list of filenames for use as input.
+
+    Args:
+        pattern: Filename pattern to search for
+
+    Returns:
+        A list of matching files in all input directories
+    """
+    if not indir:
+        return glob.glob(fname)
+    files = []
+    for dirname in indir:
+        pathname = os.path.join(dirname, pattern)
+        files += glob.glob(pathname)
+    return sorted(files)
+
 def Align(pos, align):
     if align:
         mask = align - 1