Introduce git-rpm-ch tool

Initial version of the git-rpm-ch tool which is intended for maintaining
RPM changelogs. Supports both spec files and separate "OBS style"
changelog files.

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>

4 files changed, 565 insertions, 0 deletions

diff --git a/gbp-rpm.conf b/gbp-rpm.conf
index 27f80257..f9ebd9e6 100644
--- a/gbp-rpm.conf
+++ b/gbp-rpm.conf
@@ -132,3 +132,12 @@
 [gbp-create-remote-repo]
 # Disable remote branch tracking
 #track = False
+
+# Options only affecting git-rpm-changelog
+[git-rpm-changelog]
+# Changelog filename, relative to the git topdir
+#changelog-file = git-buildpackage.changelog
+# Format string for the revision part of the changelog header
+#changelog-revision = %(tagname)s
+# Preferred editor
+#editor-cmd = vim
diff --git a/gbp/config.py b/gbp/config.py
index 07569a88..6f251a46 100644
--- a/gbp/config.py
+++ b/gbp/config.py
@@ -620,6 +620,10 @@ class GbpOptionParserRpm(GbpOptionParser):
             'merge'                     : 'False',
             'pristine-tarball-name'     : 'auto',
             'orig-prefix'               : 'auto',
+            'changelog-file'            : 'auto',
+            'changelog-revision'        : '',
+            'spawn-editor'              : 'always',
+            'editor-cmd'                : 'vim',
                     })
 
     help = dict(GbpOptionParser.help)
@@ -679,6 +683,17 @@ class GbpOptionParserRpm(GbpOptionParser):
             'orig-prefix':
                 "Prefix (dir) to be used when generating/importing tarballs, "
                 "default is '%(orig-prefix)s'",
+            'changelog-file':
+                "Changelog file to be used, default is '%(changelog-file)s'",
+            'changelog-revision':
+                "Format string for the revision field in the changelog header. "
+                "If empty or not defined the default from packaging policy is "
+                "used.",
+            'editor-cmd':
+                "Editor command to use",
+            'git-author':
+                "Use name and email from git-config for the changelog header, "
+                "default is '%(git-author)s'",
                  })
 
 # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·:
diff --git a/gbp/rpm/policy.py b/gbp/rpm/policy.py
index 6ccb1196..82d6abee 100644
--- a/gbp/rpm/policy.py
+++ b/gbp/rpm/policy.py
@@ -17,7 +17,9 @@
 """Default packaging policy for RPM"""
 
 import re
+
 from gbp.pkg import PkgPolicy, parse_archive_filename
+from gbp.scripts.common.pq import parse_gbp_commands
 
 class RpmPkgPolicy(PkgPolicy):
     """Packaging policy for RPM"""
@@ -158,3 +160,113 @@ class RpmPkgPolicy(PkgPolicy):
         header_format = "* %(time)s %(name)s <%(email)s> %(revision)s"
         header_time_format = "%a %b %d %Y"
         header_rev_format = "%(version)s"
+
+
+    class ChangelogEntryFormatter(object):
+        """Helper class for generating changelog entries from git commits"""
+
+        # Maximum length for a changelog entry line
+        max_entry_line_length = 76
+        # Bug tracking system related meta tags recognized from git commit msg
+        bts_meta_tags = ("Close", "Closes", "Fixes", "Fix")
+        # Regexp for matching bug tracking system ids (e.g. "bgo#123")
+        bug_id_re = r'[A-Za-z0-9#_\-]+'
+
+        @classmethod
+        def _parse_bts_tags(cls, lines, meta_tags):
+            """
+            Parse and filter out bug tracking system related meta tags from
+            commit message.
+
+            @param lines: commit message
+            @type lines: C{list} of C{str}
+            @param meta_tags: meta tags to look for
+            @type meta_tags: C{tuple} of C{str}
+            @return: bts-ids per meta tag and the non-mathced lines
+            @rtype: (C{dict}, C{list} of C{str})
+            """
+            tags = {}
+            other_lines = []
+            bts_re = re.compile(r'^(?P<tag>%s):\s*(?P<ids>.*)' %
+                                ('|'.join(meta_tags)), re.I)
+            bug_id_re = re.compile(cls.bug_id_re)
+            for line in lines:
+                match = bts_re.match(line)
+                if match:
+                    tag = match.group('tag')
+                    ids_str = match.group('ids')
+                    bug_ids = [bug_id.strip() for bug_id in
+                                bug_id_re.findall(ids_str)]
+                    if tag in tags:
+                        tags[tag] += bug_ids
+                    else:
+                        tags[tag] = bug_ids
+                else:
+                    other_lines.append(line)
+            return (tags, other_lines)
+
+        @classmethod
+        def _extra_filter(cls, lines, ignore_re):
+            """
+            Filter out specific lines from the commit message.
+
+            @param lines: commit message
+            @type lines: C{list} of C{str}
+            @param ignore_re: regexp for matching ignored lines
+            @type ignore_re: C{str}
+            @return: filtered commit message
+            @rtype: C{list} of C{str}
+            """
+            if ignore_re:
+                match = re.compile(ignore_re)
+                return [line for line in lines if not match.match(line)]
+            else:
+                return lines
+
+        @classmethod
+        def compose(cls, commit_info, **kwargs):
+            """
+            Generate a changelog entry from a git commit.
+
+            @param commit_info: info about the commit
+            @type commit_info: C{commit_info} object from
+                L{gbp.git.repository.GitRepository.get_commit_info()}.
+            @param kwargs: additional arguments to the compose() method,
+                currently we recognize 'full', 'id_len' and 'ignore_re'
+            @type kwargs: C{dict}
+            @return: formatted changelog entry
+            @rtype: C{list} of C{str}
+            """
+            # Parse and filter out gbp command meta-tags
+            cmds, body = parse_gbp_commands(commit_info, 'gbp-rpm-ch',
+                                            ('ignore', 'short', 'full'), ())
+            if 'ignore' in cmds:
+                return None
+
+            # Parse and filter out bts-related meta-tags
+            bts_tags, body = cls._parse_bts_tags(body, cls.bts_meta_tags)
+
+            # Additional filtering
+            body = cls._extra_filter(body, kwargs['ignore_re'])
+
+            # Generate changelog entry
+            subject = commit_info['subject']
+            commitid = commit_info['id']
+            if kwargs['id_len']:
+                text = ["- [%s] %s" % (commitid[0:kwargs['id_len']], subject)]
+            else:
+                text = ["- %s" % subject]
+
+            # Add all non-filtered-out lines from commit message, unless 'short'
+            if (kwargs['full'] or 'full' in cmds) and not 'short' in cmds:
+                # Add all non-blank body lines.
+                text.extend(["  " + line for line in body if line.strip()])
+
+            # Add bts tags and ids in the end
+            for tag, ids in bts_tags.iteritems():
+                bts_msg = " (%s: %s)" % (tag, ', '.join(ids))
+                if len(text[-1]) + len(bts_msg) >= cls.max_entry_line_length:
+                    text.append(" ")
+                text[-1] += bts_msg
+
+            return text
diff --git a/gbp/scripts/rpm_ch.py b/gbp/scripts/rpm_ch.py
new file mode 100755
index 00000000..7f66f624
--- /dev/null
+++ b/gbp/scripts/rpm_ch.py
@@ -0,0 +1,429 @@
+# vim: set fileencoding=utf-8 :
+#
+# (C) 2007, 2008, 2009, 2010, 2013 Guido Guenther <agx@sigxcpu.org>
+# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com>
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+"""Generate RPM changelog entries from git commit messages"""
+
+import ConfigParser
+from datetime import datetime
+import os.path
+import pwd
+import re
+import sys
+import socket
+
+import gbp.command_wrappers as gbpc
+import gbp.log
+from gbp.config import GbpOptionParserRpm, GbpOptionGroup
+from gbp.errors import GbpError
+from gbp.rpm import guess_spec, NoSpecError, SpecFile
+from gbp.rpm.changelog import Changelog, ChangelogParser, ChangelogError
+from gbp.rpm.git import GitRepositoryError, RpmGitRepository
+from gbp.rpm.policy import RpmPkgPolicy
+
+
+ChangelogEntryFormatter = RpmPkgPolicy.ChangelogEntryFormatter
+
+
+class ChangelogFile(object):
+    """Container for changelog file, whether it be a standalone changelog
+       or a spec file"""
+
+    def __init__(self, file_path):
+        parser = ChangelogParser(RpmPkgPolicy)
+
+        if os.path.splitext(file_path)[1] == '.spec':
+            gbp.log.debug("Using spec file '%s' as changelog" % file_path)
+            self._file = SpecFile(file_path)
+            self.changelog = parser.raw_parse_string(self._file.get_changelog())
+        else:
+            self._file = os.path.abspath(file_path)
+            if not os.path.exists(file_path):
+                gbp.log.info("Changelog '%s' not found, creating new "
+                             "changelog file" % file_path)
+                self.changelog = Changelog(RpmPkgPolicy)
+            else:
+                gbp.log.debug("Using changelog file '%s'" % file_path)
+                self.changelog = parser.raw_parse_file(self._file)
+
+        # Parse topmost section and try to determine the start commit
+        if self.changelog.sections:
+            self.changelog.sections[0] = parser.parse_section(
+                    self.changelog.sections[0])
+
+    def write(self):
+        """Write changelog file to disk"""
+        if isinstance(self._file, SpecFile):
+            self._file.set_changelog(str(self.changelog))
+            self._file.write_spec_file()
+        else:
+            with open(self._file, 'w') as fobj:
+                fobj.write(str(self.changelog))
+
+    @property
+    def path(self):
+        """File path"""
+        if isinstance(self._file, SpecFile):
+            return self._file.specpath
+        else:
+            return self._file
+
+def load_customizations(customization_file):
+    """Load user defined customizations file"""
+    # Load customization file
+    if not customization_file:
+        return
+    customizations = {}
+    try:
+        execfile(customization_file, customizations, customizations)
+    except Exception as err:
+        raise GbpError("Failed to load customization file: %s" % err)
+
+    # Set customization classes / functions
+    global ChangelogEntryFormatter
+    if 'ChangelogEntryFormatter' in customizations:
+        ChangelogEntryFormatter = customizations.get('ChangelogEntryFormatter')
+
+
+def determine_editor(options):
+    """Determine text editor"""
+
+    # Check if we need to spawn an editor
+    states = ['always']
+    if options.release:
+        states.append('release')
+    if options.spawn_editor not in states:
+        return None
+
+    # Determine the correct editor
+    if options.editor_cmd:
+        return options.editor_cmd
+    elif 'EDITOR' in os.environ:
+        return os.environ['EDITOR']
+    else:
+        return 'vi'
+
+
+def check_branch(repo, options):
+    """Check the current git branch"""
+    branch = repo.get_branch()
+    if options.packaging_branch != branch and not options.ignore_branch:
+        gbp.log.err("You are not on branch '%s' but on '%s'" %
+                    (options.packaging_branch, branch))
+        raise GbpError("Use --ignore-branch to ignore or "
+                       "--packaging-branch to set the branch name.")
+
+
+def parse_spec_file(repo, options):
+    """Find and parse spec file"""
+    if options.spec_file != 'auto':
+        spec_path = os.path.join(repo.path, options.spec_file)
+        spec = SpecFile(spec_path)
+    else:
+        spec = guess_spec(os.path.join(repo.path, options.packaging_dir),
+                          True, os.path.basename(repo.path) + '.spec')
+    options.packaging_dir = spec.specdir
+    return spec
+
+
+def parse_changelog_file(repo, spec, options):
+    """Find and parse changelog file"""
+    changes_file_name = os.path.splitext(spec.specfile)[0] + '.changes'
+    changes_file_path = os.path.join(options.packaging_dir, changes_file_name)
+
+    # Determine changelog file path
+    if options.changelog_file == "SPEC":
+        changelog_path = spec.specpath
+    elif options.changelog_file == "CHANGES":
+        changelog_path = changes_file_path
+    elif options.changelog_file == 'auto':
+        if os.path.exists(changes_file_path):
+            changelog_path = changes_file_path
+        else:
+            changelog_path = spec.specpath
+    else:
+        changelog_path = os.path.join(repo.path, options.changelog_file)
+
+    return ChangelogFile(changelog_path)
+
+
+def guess_commit(section, repo, options):
+    """Guess the last commit documented in a changelog header"""
+
+    if not section:
+        return None
+    header = section.header
+
+    # Try to parse the fields from the header revision
+    rev_re = '^%s$' % re.sub(r'%\((\S+?)\)s', r'(?P<\1>\S+)',
+                             options.changelog_revision)
+    match = re.match(rev_re, header['revision'], re.I)
+    fields = match.groupdict() if match else {}
+
+    # First, try to find tag-name, if present
+    if 'tagname' in fields:
+        gbp.log.debug("Trying to find tagname %s" % fields['tagname'])
+        try:
+            return repo.rev_parse("%s^0" % fields['tagname'])
+        except GitRepositoryError:
+            gbp.log.warn("Changelog points to tagname '%s' which is not found "
+                         "in the git repository" % fields['tagname'])
+
+    # Next, try to find packaging tag matching the version
+    tag_str_fields = {'vendor': options.vendor}
+    if 'version' in fields:
+        gbp.log.debug("Trying to find packaging tag for version '%s'" %
+                      fields['version'])
+        full_version = fields['version']
+        tag_str_fields.update(RpmPkgPolicy.split_full_version(full_version))
+    elif 'upstreamversion' in fields:
+        gbp.log.debug("Trying to find packaging tag for version '%s'" %
+                      fields['upstreamversion'])
+        tag_str_fields['upstreamversion'] = fields['upstreamversion']
+        if 'release' in fields:
+            tag_str_fields['release'] = fields['release']
+    commit = repo.find_version(options.packaging_tag,
+                                   tag_str_fields)
+    if commit:
+        return commit
+    else:
+        gbp.log.info("Couldn't find packaging tag for version %s" %
+                     header['revision'])
+
+    # As a last resort we look at the timestamp
+    timestamp = header['time'].isoformat()
+    last = repo.get_commits(num=1, options="--until='%s'" % timestamp)
+    if last:
+        gbp.log.info("Using commit (%s) before the last changelog timestamp "
+                     "(%s)" % (last, timestamp))
+        return last[0]
+    return None
+
+
+def get_start_commit(changelog, repo, options):
+    """Get the start commit from which to generate new entries"""
+    if options.since:
+        since = options.since
+    else:
+        if changelog.sections:
+            since = guess_commit(changelog.sections[0], repo, options)
+        else:
+            since = None
+        if not since:
+            raise GbpError("Couldn't determine starting point from "
+                           "changelog, please use the '--since' option")
+        gbp.log.info("Continuing from commit '%s'" % since)
+    return since
+
+
+def get_author(repo, use_git_config):
+    """Get author and email from git configuration"""
+    author = email = None
+
+    if use_git_config:
+        modifier = repo.get_author_info()
+        author = modifier.name
+        email = modifier.email
+
+    passwd_data = pwd.getpwuid(os.getuid())
+    if not author:
+        # On some distros (Ubuntu, at least) the gecos field has it's own
+        # internal structure of comma-separated fields
+        author = passwd_data.pw_gecos.split(',')[0].strip()
+        if not author:
+            author = passwd_data.pw_name
+    if not email:
+        if 'EMAIL' in os.environ:
+            email = os.environ['EMAIL']
+        else:
+            email = "%s@%s" % (passwd_data.pw_name, socket.getfqdn())
+
+    return author, email
+
+
+def entries_from_commits(changelog, repo, commits, options):
+    """Generate a list of formatted changelog entries from a list of commits"""
+    entries = []
+    for commit in commits:
+        info = repo.get_commit_info(commit)
+        entry_text = ChangelogEntryFormatter.compose(info, full=options.full,
+                        ignore_re=options.ignore_regex, id_len=options.idlen)
+        if entry_text:
+            entries.append(changelog.create_entry(author=info['author'].name,
+                                                  text=entry_text))
+    return entries
+
+
+def update_changelog(changelog, entries, repo, spec, options):
+    """Update the changelog with a range of commits"""
+    # Get info for section header
+    now = datetime.now()
+    name, email = get_author(repo, options.git_author)
+    rev_str_fields = dict(spec.version,
+                version=RpmPkgPolicy.compose_full_version(spec.version),
+                vendor=options.vendor,
+                tagname=repo.describe('HEAD', longfmt=True, always=True))
+    try:
+        revision = options.changelog_revision % rev_str_fields
+    except KeyError as err:
+        raise GbpError("Unable to construct revision field: unknown key "
+                "%s, only %s are accepted" % (err, rev_str_fields.keys()))
+
+    # Add a new changelog section if new release or an empty changelog
+    if options.release or not changelog.sections:
+        top_section = changelog.add_section(time=now, name=name,
+                                            email=email, revision=revision)
+    else:
+        # Re-use already parsed top section
+        top_section = changelog.sections[0]
+        top_section.set_header(time=now, name=name,
+                               email=email, revision=revision)
+
+    # Add new entries to the topmost section
+    for entry in entries:
+        top_section.append_entry(entry)
+
+
+def parse_args(argv):
+    """Parse command line and config file options"""
+    try:
+        parser = GbpOptionParserRpm(command=os.path.basename(argv[0]),
+                                    prefix='', usage='%prog [options] paths')
+    except ConfigParser.ParsingError as err:
+        gbp.log.error('invalid config file: %s' % err)
+        return None, None
+
+    range_grp = GbpOptionGroup(parser, "commit range options",
+                    "which commits to add to the changelog")
+    format_grp = GbpOptionGroup(parser, "changelog entry formatting",
+                    "how to format the changelog entries")
+    naming_grp = GbpOptionGroup(parser, "naming",
+                    "branch names, tag formats, directory and file naming")
+    parser.add_option_group(range_grp)
+    parser.add_option_group(format_grp)
+    parser.add_option_group(naming_grp)
+
+    # Non-grouped options
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+                    help="verbose command execution")
+    parser.add_config_file_option(option_name="color", dest="color",
+                    type='tristate')
+    parser.add_config_file_option(option_name="color-scheme",
+                    dest="color_scheme")
+    parser.add_config_file_option(option_name="vendor", action="store",
+                    dest="vendor")
+    parser.add_config_file_option(option_name="git-log", dest="git_log",
+                    help="options to pass to git-log, default is '%(git-log)s'")
+    parser.add_boolean_config_file_option(option_name="ignore-branch",
+                    dest="ignore_branch")
+    parser.add_config_file_option(option_name="customizations",
+                    dest="customization_file",
+                    help="Load Python code from CUSTOMIZATION_FILE. At the "
+                    "moment, the only useful thing the code can do is define a "
+                    "custom ChangelogEntryFormatter class.")
+
+    # Naming group options
+    naming_grp.add_config_file_option(option_name="packaging-branch",
+                    dest="packaging_branch")
+    naming_grp.add_config_file_option(option_name="packaging-tag",
+                    dest="packaging_tag")
+    naming_grp.add_config_file_option(option_name="packaging-dir",
+                    dest="packaging_dir")
+    naming_grp.add_config_file_option(option_name="changelog-file",
+                    dest="changelog_file")
+    naming_grp.add_config_file_option(option_name="spec-file", dest="spec_file")
+    # Range group options
+    range_grp.add_option("-s", "--since", dest="since",
+                    help="commit to start from (e.g. HEAD^^^, release/0.1.2)")
+    # Formatting group options
+    format_grp.add_option("--no-release", action="store_false", default=True,
+                    dest="release",
+                    help="no release, just update the last changelog section")
+    format_grp.add_boolean_config_file_option(option_name="git-author",
+                    dest="git_author")
+    format_grp.add_boolean_config_file_option(option_name="full", dest="full")
+    format_grp.add_config_file_option(option_name="id-length", dest="idlen",
+                    help="include N digits of the commit id in the changelog "
+                         "entry, default is '%(id-length)s'",
+                    type="int", metavar="N")
+    format_grp.add_config_file_option(option_name="ignore-regex",
+                    dest="ignore_regex",
+                    help="Ignore lines in commit message matching regex, "
+                         "default is '%(ignore-regex)s'")
+    format_grp.add_config_file_option(option_name="changelog-revision",
+                    dest="changelog_revision")
+    format_grp.add_config_file_option(option_name="spawn-editor",
+                    dest="spawn_editor")
+    format_grp.add_config_file_option(option_name="editor-cmd",
+                    dest="editor_cmd")
+
+    options, args = parser.parse_args(argv[1:])
+    if not options.changelog_revision:
+        options.changelog_revision = RpmPkgPolicy.Changelog.header_rev_format
+
+    gbp.log.setup(options.color, options.verbose, options.color_scheme)
+
+    return options, args
+
+def main(argv):
+    """Script main function"""
+    options, args = parse_args(argv)
+    if not options:
+        return 1
+
+    try:
+        load_customizations(options.customization_file)
+        editor_cmd = determine_editor(options)
+
+        repo = RpmGitRepository('.')
+        check_branch(repo, options)
+
+        # Find and parse spec file
+        spec = parse_spec_file(repo, options)
+
+        # Find and parse changelog file
+        ch_file = parse_changelog_file(repo, spec, options)
+        since = get_start_commit(ch_file.changelog, repo, options)
+
+        # Get range of commits from where to generate changes
+        if args:
+            gbp.log.info("Only looking for changes in '%s'" % ", ".join(args))
+        commits = repo.get_commits(since=since, until='HEAD', paths=args,
+                                   options=options.git_log.split(" "))
+        commits.reverse()
+        if not commits:
+            gbp.log.info("No changes detected from %s to %s." % (since, 'HEAD'))
+
+        # Do the actual update
+        entries = entries_from_commits(ch_file.changelog, repo, commits,
+                                       options)
+        update_changelog(ch_file.changelog, entries, repo, spec, options)
+
+        # Write to file
+        ch_file.write()
+
+        if editor_cmd:
+            gbpc.Command(editor_cmd, [ch_file.path])()
+
+    except (GbpError, GitRepositoryError, ChangelogError, NoSpecError) as err:
+        if len(err.__str__()):
+            gbp.log.err(err)
+        return 1
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))