From 7025024ae9af3b873427524ddd94e7c7183c2a9e Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 7 Jun 2012 17:20:21 +0300 Subject: rpm: support generating compressed patches This patch adds the support to generate compressed patches. User can define the patch file size limit after which patches gzipped. Re-writes the write_patch() function(s) and now uses the same-and-only write_patch() for buildpackage-rpm and pq-rpm. Signed-off-by: Markus Lehtonen --- gbp-rpm.conf | 2 ++ gbp/config.py | 3 ++ gbp/rpm/__init__.py | 29 +++++++++++++++++ gbp/scripts/buildpackage_rpm.py | 72 ++++++----------------------------------- gbp/scripts/pq_rpm.py | 53 ++++++++++++++++++------------ 5 files changed, 76 insertions(+), 83 deletions(-) diff --git a/gbp-rpm.conf b/gbp-rpm.conf index 2db9156f..0510fb0a 100644 --- a/gbp-rpm.conf +++ b/gbp-rpm.conf @@ -26,6 +26,8 @@ #spec-file = gbp.spec # Don't create patches from commits whose message matches regex #patch-export-ignore-regex = ignore-autopatch-gen +# Compress auto-generated patches +#patch-export-compress=100k # Export patches with numbering in filenames #patch-numbers = False diff --git a/gbp/config.py b/gbp/config.py index 091a194b..3b10722f 100644 --- a/gbp/config.py +++ b/gbp/config.py @@ -526,6 +526,7 @@ class GbpOptionParserRpm(GbpOptionParser): 'rpmbuild-buildrootdir' : 'BUILDROOT', 'patch-export' : 'False', 'patch-export-ignore-regex' : '^GBP: patch-export-ignore', + 'patch-export-compress' : '0', 'pristine-tarball-name' : 'auto', } ) @@ -547,6 +548,8 @@ class GbpOptionParserRpm(GbpOptionParser): "Create patches between upstream and export-treeish, default is '%(patch-export)s'", 'patch-export-ignore-regex': "Don't create patches from commits whose message matches given regex, default is '%(patch-export-ignore-regex)s'", + 'patch-export-compress': + "Compress (auto-generated) patches larger than given number of bytes, 0 never compresses, default is '%(patch-export-compress)s'", 'pristine-tarball-name': "Filename to record to pristine-tar, set to 'auto' to not mangle the file name, default is '%(pristine-tarball-name)s'", } ) diff --git a/gbp/rpm/__init__.py b/gbp/rpm/__init__.py index 5fe2a9f9..b6a608b5 100644 --- a/gbp/rpm/__init__.py +++ b/gbp/rpm/__init__.py @@ -535,4 +535,33 @@ def guess_spec_repo(repo, branch, packaging_dir): raise NoSpecError, "Searching spec from other branch not implemented yet" +def string_to_int(val_str): + """ + Convert string of possible unit identifier to int. + + @param val_str: value to be converted + @type val_str: C{str} + @return: value as integer + @rtype: C{int} + + >>> string_to_int("1234") + 1234 + >>> string_to_int("123k") + 125952 + >>> string_to_int("1234K") + 1263616 + >>> string_to_int("1M") + 1048576 + """ + units = {'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + 't': 1024**4} + + if val_str[-1].lower() in units: + return int(val_str[:-1]) * units[val_str[-1].lower()] + else: + return int(val_str) + + # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/buildpackage_rpm.py b/gbp/scripts/buildpackage_rpm.py index e0376d73..8e4124eb 100755 --- a/gbp/scripts/buildpackage_rpm.py +++ b/gbp/scripts/buildpackage_rpm.py @@ -26,6 +26,7 @@ import tempfile import shutil import re import datetime +import gzip import gbp.rpm as rpm from gbp.rpm.policy import RpmPkgPolicy from gbp.command_wrappers import (Command, @@ -41,6 +42,7 @@ from gbp.scripts.common.buildpackage import (index_name, wc_names, git_archive_single, dump_tree, write_wc, drop_index) from gbp.pkg import (compressor_opts, compressor_aliases) +from gbp.scripts.pq_rpm import write_patch def git_archive(repo, spec, output_dir, treeish, comp_level, with_submodules): "create a compressed orig tarball in output_dir using git_archive" @@ -186,67 +188,6 @@ def git_archive_build_orig(repo, spec, output_dir, options): return upstream_tree -def write_patch(patch, options): - """Write the patch exported by 'git-format-patch' to it's final location - (as specified in the commit)""" - oldname = os.path.basename(patch) - newname = oldname - tmpname = patch + ".gbp" - old = file(patch, 'r') - tmp = file(tmpname, 'w') - in_patch = False - topic = None - - # Skip first line (from ) - old.readline() - for line in old: - if in_patch: - if line == '-- \n': - # Found final signature, we're done: - tmp.write(line) - break - else: - if re.match(options.patch_export_ignore_regex, line): - gbp.log.debug("Ignoring patch %s, matches ignore-regex" % patch) - old.close() - tmp.close() - os.unlink(patch) - os.unlink(tmpname) - return - if line.lower().startswith("gbp-pq-topic: "): - topic = line.split(" ",1)[1].strip() - gbp.log.debug("Topic %s found for %s" % (topic, patch)) - continue - elif (line.startswith("diff --git a/") or - line.startswith("---")): - in_patch = True - tmp.write(line) - - tmp.close() - old.close() - - if not options.patch_numbers: - patch_re = re.compile("[0-9]+-(?P.+)") - m = patch_re.match(oldname) - if m: - newname = m.group('name') - - if topic: - topicdir = os.path.join(os.path.dirname(patch), topic) - else: - topicdir = os.path.dirname(patch) - - if not os.path.isdir(topicdir): - os.makedirs(topicdir, 0755) - - os.unlink(patch) - dstname = os.path.join(topicdir, newname) - gbp.log.debug("Moving %s to %s" % (tmpname, dstname)) - shutil.move(tmpname, dstname) - - return dstname - - def gen_patches(repo, spec, totree, options): """Generate patches""" upstream_tree = get_upstream_tree(repo, spec, options) @@ -280,7 +221,11 @@ def gen_patches(repo, spec, totree, options): if patches: gbp.log.info("Regenerating patch series in '%s'." % spec.specdir) for patch in patches: - patch_file = write_patch(patch, options) + patch_file = write_patch(patch, + spec.specdir, + options.patch_numbers, + options.patch_export_compress, + options.patch_export_ignore_regex) if patch_file != None: filenames.append(os.path.basename(patch_file)) @@ -449,9 +394,12 @@ def parse_args(argv, prefix): help="only export packaging files, don't build") export_group.add_boolean_config_file_option("patch-export", dest="patch_export") export_group.add_config_file_option("patch-export-ignore-regex", dest="patch_export_ignore_regex") + export_group.add_config_file_option("patch-export-compress", dest="patch_export_compress") export_group.add_boolean_config_file_option(option_name="patch-numbers", dest="patch_numbers") options, args = parser.parse_args(args) + options.patch_export_compress = rpm.string_to_int(options.patch_export_compress) + gbp.log.setup(options.color, options.verbose) if options.retag: if not options.tag and not options.tag_only: diff --git a/gbp/scripts/pq_rpm.py b/gbp/scripts/pq_rpm.py index 7a2980ec..497e52b6 100755 --- a/gbp/scripts/pq_rpm.py +++ b/gbp/scripts/pq_rpm.py @@ -23,6 +23,7 @@ import os import shutil import sys import tempfile +import re import gzip from gbp.config import (GbpOptionParserRpm, GbpOptionGroup) from gbp.rpm.git import (GitRepositoryError, RpmGitRepository) @@ -33,49 +34,55 @@ from gbp.errors import GbpError import gbp.log from gbp.patch_series import (PatchSeries, Patch) from gbp.pkg import parse_archive_filename -from gbp.rpm import (SpecFile, guess_spec) +from gbp.rpm import (SpecFile, guess_spec, string_to_int) from gbp.scripts.common.pq import (is_pq_branch, pq_branch_name, pq_branch_base, switch_to_pq_branch, apply_single_patch, apply_and_commit_patch, drop_pq) -def write_patch(patch, patch_dir, options): +def write_patch(patch, out_dir, patch_numbers=True, compress_size=0, ignore_regex=None): """Write the patch exported by 'git-format-patch' to it's final location (as specified in the commit)""" oldname = os.path.basename(patch) newname = oldname tmpname = patch + ".gbp" old = file(patch, 'r') - tmp = file(tmpname, 'w') - in_patch = False - topic = None - # Skip first line (From ) + # Compress if patch file is larger than "threshold" value + if compress_size and os.path.getsize(patch) > compress_size: + tmp = gzip.open(tmpname, 'w') + newname += '.gz' + else: + tmp = file(tmpname, 'w') + + # Skip the first From ... line old.readline() for line in old: - if line.lower().startswith("gbp-pq-topic: "): - topic = line.split(" ",1)[1].strip() - gbp.log.debug("Topic %s found for %s" % (topic, patch)) - continue + if ignore_regex and re.match(ignore_regex, line): + gbp.log.debug("Ignoring patch %s, matches ignore-regex" % patch) + old.close() + tmp.close() + os.unlink(patch) + os.unlink(tmpname) + return + elif (line.startswith("diff --git a/") or + line.startswith("---")): + tmp.write(line) + break; tmp.write(line) + + # Write the rest of the file + tmp.writelines(old) tmp.close() old.close() - if not options.patch_numbers: + if not patch_numbers: patch_re = re.compile("[0-9]+-(?P.+)") m = patch_re.match(oldname) if m: newname = m.group('name') - if topic: - topicdir = os.path.join(patch_dir, topic) - else: - topicdir = patch_dir - - if not os.path.isdir(topicdir): - os.makedirs(topicdir, 0755) - os.unlink(patch) - dstname = os.path.join(topicdir, newname) + dstname = os.path.join(out_dir, newname) gbp.log.debug("Moving %s to %s" % (tmpname, dstname)) shutil.move(tmpname, dstname) @@ -137,7 +144,9 @@ def export_patches(repo, branch, options): if patches: gbp.log.debug("Regenerating patch queue in '%s'." % spec.specdir) for patch in patches: - filenames.append(os.path.basename(write_patch(patch, spec.specdir, options))) + patch_file = write_patch(patch, spec.specdir, options.patch_numbers, options.patch_export_compress) + if patch_file != None: + filenames.append(os.path.basename(patch_file)) spec.update_patches(filenames) spec.write_spec_file() @@ -355,9 +364,11 @@ def main(argv): parser.add_config_file_option(option_name="pq-branch", dest="pq_branch") parser.add_option("--export-rev", action="store", dest="export_rev", default="", help="Export patches from treeish object TREEISH instead of head of patch-queue branch", metavar="TREEISH") + parser.add_config_file_option("patch-export-compress", dest="patch_export_compress") (options, args) = parser.parse_args(argv) gbp.log.setup(options.color, options.verbose) + options.patch_export_compress = string_to_int(options.patch_export_compress) if len(args) < 2: gbp.log.err("No action given.") -- cgit v1.2.3