rpm: support generating compressed patches

This patch adds the support to generate compressed patches. User can
define the patch file size limit after which patches gzipped.

Re-writes the write_patch() function(s) and now uses the same-and-only
write_patch() for buildpackage-rpm and pq-rpm.

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>

5 files changed, 76 insertions, 83 deletions

diff --git a/gbp-rpm.conf b/gbp-rpm.conf
index 2db9156f..0510fb0a 100644
--- a/gbp-rpm.conf
+++ b/gbp-rpm.conf
@@ -26,6 +26,8 @@
 #spec-file = gbp.spec
 # Don't create patches from commits whose message matches regex
 #patch-export-ignore-regex = ignore-autopatch-gen
+# Compress auto-generated patches
+#patch-export-compress=100k
 # Export patches with numbering in filenames
 #patch-numbers = False
 
diff --git a/gbp/config.py b/gbp/config.py
index 091a194b..3b10722f 100644
--- a/gbp/config.py
+++ b/gbp/config.py
@@ -526,6 +526,7 @@ class GbpOptionParserRpm(GbpOptionParser):
                        'rpmbuild-buildrootdir'  : 'BUILDROOT',
                        'patch-export'           : 'False',
                        'patch-export-ignore-regex'  : '^GBP: patch-export-ignore',
+                       'patch-export-compress'  : '0',
                        'pristine-tarball-name'  : 'auto',
                      } )
 
@@ -547,6 +548,8 @@ class GbpOptionParserRpm(GbpOptionParser):
                         "Create patches between upstream and export-treeish, default is '%(patch-export)s'",
                    'patch-export-ignore-regex':
                         "Don't create patches from commits whose message matches given regex, default is '%(patch-export-ignore-regex)s'",
+                   'patch-export-compress':
+                        "Compress (auto-generated) patches larger than given number of bytes, 0 never compresses, default is '%(patch-export-compress)s'",
                    'pristine-tarball-name':
                         "Filename to record to pristine-tar, set to 'auto' to not mangle the file name, default is '%(pristine-tarball-name)s'",
                  } )
diff --git a/gbp/rpm/__init__.py b/gbp/rpm/__init__.py
index 5fe2a9f9..b6a608b5 100644
--- a/gbp/rpm/__init__.py
+++ b/gbp/rpm/__init__.py
@@ -535,4 +535,33 @@ def guess_spec_repo(repo, branch, packaging_dir):
     raise NoSpecError, "Searching spec from other branch not implemented yet"
 
 
+def string_to_int(val_str):
+    """
+    Convert string of possible unit identifier to int.
+
+    @param val_str: value to be converted
+    @type val_str: C{str}
+    @return: value as integer
+    @rtype: C{int}
+
+    >>> string_to_int("1234")
+    1234
+    >>> string_to_int("123k")
+    125952
+    >>> string_to_int("1234K")
+    1263616
+    >>> string_to_int("1M")
+    1048576
+    """
+    units = {'k': 1024,
+             'm': 1024**2,
+             'g': 1024**3,
+             't': 1024**4}
+
+    if val_str[-1].lower() in units:
+        return int(val_str[:-1]) * units[val_str[-1].lower()]
+    else:
+        return int(val_str)
+
+
 # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·:
diff --git a/gbp/scripts/buildpackage_rpm.py b/gbp/scripts/buildpackage_rpm.py
index e0376d73..8e4124eb 100755
--- a/gbp/scripts/buildpackage_rpm.py
+++ b/gbp/scripts/buildpackage_rpm.py
@@ -26,6 +26,7 @@ import tempfile
 import shutil
 import re
 import datetime
+import gzip
 import gbp.rpm as rpm
 from gbp.rpm.policy import RpmPkgPolicy
 from gbp.command_wrappers import (Command,
@@ -41,6 +42,7 @@ from gbp.scripts.common.buildpackage import (index_name, wc_names,
                                              git_archive_single, dump_tree,
                                              write_wc, drop_index)
 from gbp.pkg import (compressor_opts, compressor_aliases)
+from gbp.scripts.pq_rpm import write_patch
 
 def git_archive(repo, spec, output_dir, treeish, comp_level, with_submodules):
     "create a compressed orig tarball in output_dir using git_archive"
@@ -186,67 +188,6 @@ def git_archive_build_orig(repo, spec, output_dir, options):
     return upstream_tree
 
 
-def write_patch(patch, options):
-    """Write the patch exported by 'git-format-patch' to it's final location
-       (as specified in the commit)"""
-    oldname = os.path.basename(patch)
-    newname = oldname
-    tmpname = patch + ".gbp"
-    old = file(patch, 'r')
-    tmp = file(tmpname, 'w')
-    in_patch = False
-    topic = None
-
-    # Skip first line (from <sha1>)
-    old.readline()
-    for line in old:
-        if in_patch:
-            if line == '-- \n':
-                # Found final signature, we're done:
-                tmp.write(line)
-                break
-        else:
-            if re.match(options.patch_export_ignore_regex, line):
-                gbp.log.debug("Ignoring patch %s, matches ignore-regex" % patch)
-                old.close()
-                tmp.close()
-                os.unlink(patch)
-                os.unlink(tmpname)
-                return
-            if line.lower().startswith("gbp-pq-topic: "):
-                topic = line.split(" ",1)[1].strip()
-                gbp.log.debug("Topic %s found for %s" % (topic, patch))
-                continue
-            elif (line.startswith("diff --git a/") or
-                  line.startswith("---")):
-                in_patch = True
-        tmp.write(line)
-
-    tmp.close()
-    old.close()
-
-    if not options.patch_numbers:
-        patch_re = re.compile("[0-9]+-(?P<name>.+)")
-        m = patch_re.match(oldname)
-        if m:
-            newname = m.group('name')
-
-    if topic:
-        topicdir = os.path.join(os.path.dirname(patch), topic)
-    else:
-        topicdir = os.path.dirname(patch)
-
-    if not os.path.isdir(topicdir):
-        os.makedirs(topicdir, 0755)
-
-    os.unlink(patch)
-    dstname = os.path.join(topicdir, newname)
-    gbp.log.debug("Moving %s to %s" % (tmpname, dstname))
-    shutil.move(tmpname, dstname)
-
-    return dstname
-
-
 def gen_patches(repo, spec, totree, options):
     """Generate patches"""
     upstream_tree = get_upstream_tree(repo, spec, options)
@@ -280,7 +221,11 @@ def gen_patches(repo, spec, totree, options):
     if patches:
         gbp.log.info("Regenerating patch series in '%s'." % spec.specdir)
         for patch in patches:
-            patch_file = write_patch(patch, options)
+            patch_file = write_patch(patch,
+                                     spec.specdir,
+                                     options.patch_numbers,
+                                     options.patch_export_compress,
+                                     options.patch_export_ignore_regex)
             if patch_file != None:
                 filenames.append(os.path.basename(patch_file))
 
@@ -449,9 +394,12 @@ def parse_args(argv, prefix):
                       help="only export packaging files, don't build")
     export_group.add_boolean_config_file_option("patch-export", dest="patch_export")
     export_group.add_config_file_option("patch-export-ignore-regex", dest="patch_export_ignore_regex")
+    export_group.add_config_file_option("patch-export-compress", dest="patch_export_compress")
     export_group.add_boolean_config_file_option(option_name="patch-numbers", dest="patch_numbers")
     options, args = parser.parse_args(args)
 
+    options.patch_export_compress = rpm.string_to_int(options.patch_export_compress)
+
     gbp.log.setup(options.color, options.verbose)
     if options.retag:
         if not options.tag and not options.tag_only:
diff --git a/gbp/scripts/pq_rpm.py b/gbp/scripts/pq_rpm.py
index 7a2980ec..497e52b6 100755
--- a/gbp/scripts/pq_rpm.py
+++ b/gbp/scripts/pq_rpm.py
@@ -23,6 +23,7 @@ import os
 import shutil
 import sys
 import tempfile
+import re
 import gzip
 from gbp.config import (GbpOptionParserRpm, GbpOptionGroup)
 from gbp.rpm.git import (GitRepositoryError, RpmGitRepository)
@@ -33,49 +34,55 @@ from gbp.errors import GbpError
 import gbp.log
 from gbp.patch_series import (PatchSeries, Patch)
 from gbp.pkg import parse_archive_filename
-from gbp.rpm import (SpecFile, guess_spec)
+from gbp.rpm import (SpecFile, guess_spec, string_to_int)
 from gbp.scripts.common.pq import (is_pq_branch, pq_branch_name, pq_branch_base,
                                    switch_to_pq_branch, apply_single_patch,
                                    apply_and_commit_patch, drop_pq)
 
-def write_patch(patch, patch_dir, options):
+def write_patch(patch, out_dir, patch_numbers=True, compress_size=0, ignore_regex=None):
     """Write the patch exported by 'git-format-patch' to it's final location
        (as specified in the commit)"""
     oldname = os.path.basename(patch)
     newname = oldname
     tmpname = patch + ".gbp"
     old = file(patch, 'r')
-    tmp = file(tmpname, 'w')
-    in_patch = False
-    topic = None
 
-    # Skip first line (From <sha1>)
+    # Compress if patch file is larger than "threshold" value
+    if compress_size and os.path.getsize(patch) > compress_size:
+        tmp = gzip.open(tmpname, 'w')
+        newname += '.gz'
+    else:
+        tmp = file(tmpname, 'w')
+
+    # Skip the first From <sha>... line
     old.readline()
     for line in old:
-        if line.lower().startswith("gbp-pq-topic: "):
-            topic = line.split(" ",1)[1].strip()
-            gbp.log.debug("Topic %s found for %s" % (topic, patch))
-            continue
+        if ignore_regex and re.match(ignore_regex, line):
+                gbp.log.debug("Ignoring patch %s, matches ignore-regex" % patch)
+                old.close()
+                tmp.close()
+                os.unlink(patch)
+                os.unlink(tmpname)
+                return
+        elif (line.startswith("diff --git a/") or
+              line.startswith("---")):
+              tmp.write(line)
+              break;
         tmp.write(line)
+
+    # Write the rest of the file
+    tmp.writelines(old)
     tmp.close()
     old.close()
 
-    if not options.patch_numbers:
+    if not patch_numbers:
         patch_re = re.compile("[0-9]+-(?P<name>.+)")
         m = patch_re.match(oldname)
         if m:
             newname = m.group('name')
 
-    if topic:
-        topicdir = os.path.join(patch_dir, topic)
-    else:
-        topicdir = patch_dir
-
-    if not os.path.isdir(topicdir):
-        os.makedirs(topicdir, 0755)
-
     os.unlink(patch)
-    dstname = os.path.join(topicdir, newname)
+    dstname = os.path.join(out_dir, newname)
     gbp.log.debug("Moving %s to %s" % (tmpname, dstname))
     shutil.move(tmpname, dstname)
 
@@ -137,7 +144,9 @@ def export_patches(repo, branch, options):
     if patches:
         gbp.log.debug("Regenerating patch queue in '%s'." % spec.specdir)
         for patch in patches:
-            filenames.append(os.path.basename(write_patch(patch, spec.specdir, options)))
+            patch_file = write_patch(patch, spec.specdir, options.patch_numbers, options.patch_export_compress)
+            if patch_file != None:
+                filenames.append(os.path.basename(patch_file))
 
         spec.update_patches(filenames)
         spec.write_spec_file()
@@ -355,9 +364,11 @@ def main(argv):
     parser.add_config_file_option(option_name="pq-branch", dest="pq_branch")
     parser.add_option("--export-rev", action="store", dest="export_rev", default="",
                       help="Export patches from treeish object TREEISH instead of head of patch-queue branch", metavar="TREEISH")
+    parser.add_config_file_option("patch-export-compress", dest="patch_export_compress")
 
     (options, args) = parser.parse_args(argv)
     gbp.log.setup(options.color, options.verbose)
+    options.patch_export_compress = string_to_int(options.patch_export_compress)
 
     if len(args) < 2:
         gbp.log.err("No action given.")