diff options
Diffstat (limited to 'gbp')
| -rw-r--r-- | gbp/bb/__init__.py | 501 | ||||
| -rw-r--r-- | gbp/config.py | 230 | ||||
| -rw-r--r-- | gbp/rpm/__init__.py | 1 | ||||
| -rw-r--r-- | gbp/scripts/buildpackage_bb.py | 536 | ||||
| -rwxr-xr-x | gbp/scripts/buildpackage_rpm.py | 2 | ||||
| -rwxr-xr-x | gbp/scripts/clone_bb.py | 174 | ||||
| -rwxr-xr-x | gbp/scripts/import_bb.py | 419 | ||||
| -rw-r--r-- | gbp/scripts/import_orig.py | 6 | ||||
| -rwxr-xr-x | gbp/scripts/pq_bb.py | 427 | ||||
| -rwxr-xr-x | gbp/scripts/pq_rpm.py | 9 | ||||
| -rwxr-xr-x | gbp/scripts/submit_bb.py | 138 |
11 files changed, 2344 insertions, 99 deletions
diff --git a/gbp/bb/__init__.py b/gbp/bb/__init__.py new file mode 100644 index 00000000..1efeb221 --- /dev/null +++ b/gbp/bb/__init__.py @@ -0,0 +1,501 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +"""Bitbake helper functionality""" + +import os +import re +import shutil +import string +import subprocess +import sys +import tempfile +from collections import defaultdict + +import gbp.log +from gbp.errors import GbpError +from gbp.git.repository import GitRepository, GitRepositoryError +from gbp.scripts.common.buildpackage import dump_tree + +bb = None + +# pylint: disable=bad-continuation + + +def import_bb(): + """Import bitbake lib""" + bb_bin = subprocess.Popen(['which', 'bitbake'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()[0] + if bb_bin: + bb_lib_path = os.path.dirname(bb_bin) + '/../lib' + sys.path.insert(0, bb_lib_path) + try: + return __import__('bb') + except ImportError: + print "ERROR: Unable to find bitbake/lib, try initializing build " \ + "environment with the 'oe-init-build-env' script\n" + # Return None instead of raising (ImportError) so that building of + # this package succeeds in Debian. Otherwise dpkg-buildpackage fails + # because of an import error in epydoc. + return None + +def init_tinfoil(config_only=False, tracking=False): + """Initialize the Bitbake tinfoil module""" + import bb.tinfoil + try: + tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) + except (SystemExit, bb.BBHandledException): + raise GbpError("Failed to initialize tinfoil") + tinfoil.prepare(config_only=config_only) + return tinfoil + + +def pkg_version(data): + """Get package version as a dict""" + return {'upstreamversion': data.getVar('PV', True), + 'release': data.getVar('PR', True), + 'version': data.getVar('PV', True) + '-' + data.getVar('PR', True)} + + +class BBFile(object): + """Class representing .bb meta data""" + var_ops = r'\+=|=\+|\?=|\?\?=|:=|=' + vardef_re = re.compile( + r'(^(?P<name>\w+)\s*(?P<op>%s)\s*)(?P<value>\S.*)' % var_ops) + + + def __init__(self, path, cfg_data=None): + self.bb_file = os.path.basename(path) + self.bb_dir = os.path.abspath(os.path.dirname(path)) + + self._pkg_data = None + self._variables = {} + self.includes = [] + self.localfiles = [] + + if cfg_data is not None: + self.parse_bb(path, cfg_data) + else: + self.naive_parse_bb(path) + + @property + def version(self): + """Get version information as a dict""" + return {'upstreamversion': self.getVar('PV', True), + 'release': self.getVar('PR', True)} + + @property + def bb_path(self): + """Full path of the bb file""" + return os.path.join(self.bb_dir, self.bb_file) + + def parse_bb(self, path, cfg_data): + """Parse bb meta file""" + self._pkg_data = bb.cache.Cache.loadDataFull(path, [], cfg_data) + + # Determine local packaging files + uris = (self.getVar('SRC_URI', True) or "").split() + fetcher = bb.fetch2.Fetch(uris, self._pkg_data) + bb_dir = os.path.dirname(self.getVar('FILE')) + # Also check for file existence as fetcher incorrecly returns some + # non-existent .bbclass files under the recipe directory + self.includes = [path for path in self.getVar('BBINCLUDED').split() if + path.startswith(bb_dir) and os.path.exists(path)] + self.localfiles = [path for path in fetcher.localpaths() if + path.startswith(bb_dir)] + + def naive_parse_bb(self, path): + """Naive parsing of standalone recipes""" + # Some variable defaults + # e.g. take package name and version directly from recipe file name + self._variables['FILE'] = os.path.abspath(path) + fn_base, _fn_ext = os.path.splitext(os.path.basename(path)) + split_base = fn_base.rsplit('_', 1) + if len(split_base) == 2: + self._variables['PN'] = split_base[0] + self._variables['PV'] = split_base[1] + else: + self._variables['PN'] = fn_base + self._variables['PV'] = '1.0' + self._variables['PR'] = 'r0' + + def var_parse_cb(lines): + """Callback function for parsing variables""" + unwrapped = self.unwrap_lines(lines) + match = self.vardef_re.match(unwrapped) + if match: + var = match.groupdict() + value = self.unquote_val(var['value']) + + if (var['name'] not in self._variables or + var['op'] in ('=', ':=')): + self._variables[var['name']] = value + elif var['op'] in ('+=', '=+'): + self._variables[var['name']] += ' ' + value + else: + splitted = unwrapped.split(None, 1) + if (len(splitted) > 1 and + splitted[0] in ('include', 'require')): + inc_fname = splitted[1].strip() + inc_path = os.path.join(os.path.dirname(path), + inc_fname) + self.includes.append(os.path.abspath(inc_path)) + return lines + self.parse_file(inc_path, var_parse_cb) + return lines + + # Parse variables from file + self.parse_file(path, var_parse_cb) + + # Find local files + filedirs = [self.getVar('PN') + '-' + self.getVar('PV'), + self.getVar('PN'), 'files'] + uris = (self.getVar('SRC_URI') or "").split() + for uri_str in uris: + uri = bb.fetch2.URI(uri_str) + if uri.scheme == 'file': + found = False + for path in [os.path.join(self.bb_dir, dirn, uri.path) for dirn + in filedirs]: + if os.path.exists(path): + self.localfiles.append(path) + found = True + break + if not found: + gbp.log.warn("Seemingly local file '%s' not found under " + "'%s'" % (uri_str, self.bb_dir)) + + def _expand_single(self, match): + """Expand single occurrence of a variable reference""" + if match.group(1) in self._variables: + return self._variables[match.group(1)] + return match.group(0) + + def expand_val(self, val, rec=0): + """Expand variable""" + expanded = re.sub(r'\${(\w+)}', self._expand_single, val) + if expanded == val: + return expanded + elif rec < 20: + return self.expand_val(expanded, rec +1) + else: + raise GbpError("Too many recursions when expanding variable value") + + def getVar(self, var, expand=True): + """Get variable""" + if self._pkg_data: + return self._pkg_data.getVar(var, expand) + elif var in self._variables: + if expand: + return self.expand_val(self._variables[var]) + else: + return self._variables[var] + return None + + @staticmethod + def unquote_val(val): + """Unquote / strip variable value""" + return val.strip(string.whitespace + r'"\'\\') + + @staticmethod + def unwrap_lines(lines): + """Return a joined string of multiple lines""" + return ''.join([re.sub(r'\\\s*$', '', line) for line in lines]) + + @staticmethod + def var_to_str(var, values, oper='+='): + """Create a well formatted string buffer containing a multiline variable + assignment""" + indent = ' ' * (len(var) + 2 + len(oper)) + linebuf = ['%s %s "%s \\\n' % (var, oper, values[0])] + for val in values[1:]: + linebuf.append(indent + ' ' + val + '\\\n') + linebuf.append(indent + '"\n') + return linebuf + + @staticmethod + def parse_file(filepath, cb_func): + """Parse recipe""" + ret_buf = [] + with open(filepath) as fobj: + multiline = [] + for line in fobj.readlines(): + stripped = line.rstrip() + if not multiline: + if not stripped.endswith('\\'): + ret_buf.extend(cb_func([line])) + else: + multiline = [line] + else: + multiline.append(line) + if not stripped.endswith('\\'): + ret_buf.extend(cb_func(multiline)) + multiline = [] + return ret_buf + + @staticmethod + def set_var_val(filepath, var, val): + """Set variable value in a recipe""" + class _Setter(object): + """Class for handling variable injections""" + def __init__(self): + self.was_set = False + + def set_cb(self, lines): + """Parser callback for setting variable value""" + unwrapped = BBFile.unwrap_lines(lines) + match = BBFile.vardef_re.match(unwrapped) + if match and match.group('name') == var: + if not self.was_set: + self.was_set = True + print "Setting value %s = %s" % (var, val) + return ['%s = "%s"\n' % (var, val)] + else: + return [] + return lines + + # Parse file and set values + setter = _Setter() + linebuf = BBFile.parse_file(filepath, setter.set_cb) + + # Write file + with open(filepath, 'w') as fobj: + if not setter.was_set: + fobj.write('%s = "%s"\n') + fobj.writelines(linebuf) + + @staticmethod + def substitute_var_val(filepath, var, pattern, repl): + """Update variable in a recipe""" + def subst_cb(lines): + """Parser callback for substituting variable values""" + unwrapped = BBFile.unwrap_lines(lines) + match = BBFile.vardef_re.match(unwrapped) + if match and match.group('name') == var: + filtered = [] + for line in lines: + line = re.sub(pattern, repl, line) + # Drop empty lines + if not re.match(r'\s*\\\s*', line): + filtered.append(line) + return filtered + return lines + + # Parse file and substitute values + linebuf = BBFile.parse_file(filepath, subst_cb) + + # Write file + with open(filepath, 'w') as fobj: + fobj.writelines(linebuf) + + @staticmethod + def append_var_val(filepath, var, new_vals): + """Update variable in a recipe""" + if not new_vals: + return + + class _Finder(object): + """Class for recording definitions of variables""" + def __init__(self): + self.line_ind = 0 + self.last_occurrence = -1 + + def find_last_occurrence_cb(self, lines): + """Get the point of insertion for the variable""" + unwrapped = BBFile.unwrap_lines(lines) + match = BBFile.vardef_re.match(unwrapped) + if match and match.group('name') == var: + self.last_occurrence = self.line_ind + len(lines) - 1 + self.line_ind += len(lines) + return lines + + finder = _Finder() + linebuf = BBFile.parse_file(filepath, finder.find_last_occurrence_cb) + + # Prepare for appending values + quote = None + if finder.last_occurrence >= 0: + last_line = linebuf[finder.last_occurrence].rstrip() + # Guess indentation + match = BBFile.vardef_re.match(last_line) + if match: + indent = ' ' * (len(match.group(1)) + 1) + else: + indent = re.match(r'(\s*)', last_line).group(1) + + # Guess point of insertion for new values and mangle the last line + if re.match(r'^\s*$', last_line[:-1]): + # Insert before the last line if it's an empty line (with a + # quotation character only) + insert_ind = finder.last_occurrence + indent += ' ' + else: + # Else, remove the quotation character and append after the + # last line + quote = last_line[-1] + last_line = last_line[:-1] + ' \\\n' + linebuf[finder.last_occurrence] = last_line + insert_ind = finder.last_occurrence + 1 + else: + indent = ' ' * (len(var) + 4) + + # Write file + with open(filepath, 'w') as fobj: + if finder.last_occurrence > -1: + fobj.writelines(linebuf[:insert_ind]) + for val in new_vals: + fobj.write(indent + val + ' \\\n') + if quote: + fobj.write(indent + quote + '\n') + fobj.writelines(linebuf[insert_ind:]) + else: + fobj.writelines(BBFile.var_to_str(var, new_vals, '+=')) + fobj.writelines(linebuf) + +def guess_bb_file(file_list, bbappend): + """Guess bb recipe from a list of filenames""" + recipes = [] + file_exts = ['.bb'] if not bbappend else ['.bb', '.bbappend'] + for ext in file_exts: + for filepath in file_list: + if filepath.endswith(ext): + gbp.log.debug("Found bb recipe file %s" % filepath) + recipes.append(filepath) + if len(recipes) == 0: + raise GbpError("No recipes found.") + return sorted(recipes)[-1] + +def bb_from_repo(cfg_data, repo, treeish, bb_path): + """Get and parse a bb recipe from a Git treeish""" + try: + tmpdir = tempfile.mkdtemp(prefix='gbp-bb_') + # Dump whole bb directory + dump_tree(repo, tmpdir, '%s:%s' % (treeish, os.path.dirname(bb_path)), + False) + fpath = os.path.join(tmpdir, os.path.basename(bb_path)) + return BBFile(fpath, cfg_data) + except GitRepositoryError as err: + raise GbpError("Git error: %s" % err) + finally: + shutil.rmtree(tmpdir) + +def guess_bb_path_from_fs(topdir, recursive=True, bbappend=False): + """Guess a bitbake recipe file""" + file_list = [] + if not topdir: + topdir = '.' + for root, dirs, files in os.walk(topdir): + file_list.extend([os.path.join(root, fname) for fname in files]) + if not recursive: + del dirs[:] + # Skip .git dir in any case + if '.git' in dirs: + dirs.remove('.git') + return guess_bb_file(file_list, bbappend) + +def guess_bb_path_from_repo(repo, treeish=None, topdir='', recursive=True, + bbappend=False): + """Guess a bitbake recipe path from a git repository""" + topdir = topdir.rstrip('/') + ('/') if topdir else '' + # Search from working copy + if not treeish: + abspath = guess_bb_path_from_fs(os.path.join(repo.path, topdir), + recursive, bbappend) + return os.path.relpath(abspath, repo.path) + + # Search from treeish + try: + file_list = [nam for (mod, typ, sha, nam) in + repo.list_tree(treeish, recursive, topdir) if typ == 'blob'] + except GitRepositoryError as err: + raise GbpError("Failed to search bb recipe from treeish %s, " + "Git error: %s" % (treeish, err)) + return guess_bb_file(file_list, bbappend) + +def guess_bb_path(options, repo, treeish=None, bbappend=False): + """Guess recipe path, relative to repo rootdir""" + bb_path = options.bb_file + if options.bb_file: + if not treeish: + path = os.path.join(repo.path, bb_path) + if not os.path.exists(path): + raise GbpError("'%s' does not exist" % bb_path) + else: + try: + repo.show("%s:%s" % (treeish, bb_path)) + except GbpError as err: + raise GbpError(str(err)) + else: + bb_path = guess_bb_path_from_repo(repo, treeish, options.meta_dir, + bbappend=bbappend) + return bb_path + +def parse_bb(cfg_data, options, repo, treeish=None, bbappend=False): + """Find and parse a bb recipe from a repository""" + try: + bb_path = guess_bb_path(options, repo, treeish, bbappend=bbappend) + gbp.log.debug("Using recipe '%s'" % bb_path) + options.meta_dir = os.path.dirname(bb_path) + if treeish: + pkg_data = bb_from_repo(cfg_data, repo, treeish, bb_path) + else: + full_path = os.path.join(repo.path, bb_path) + pkg_data = BBFile(full_path, cfg_data) + except GbpError as err: + raise GbpError("Can't parse bb recipe: %s" % err) + return pkg_data + + +def guess_pkg_from_dir(pkg_dir, tinfoil): + """Guess a package from a directory in configured bitbake environment""" + abspath = os.path.abspath(pkg_dir) + layer_dirs = tinfoil.config_data.getVar('BBLAYERS').split() + gbp.log.debug("Checking if %s is in %s" % (abspath, layer_dirs)) + layer_dir = '' + for path in layer_dirs: + if abspath.startswith(path): + layer_dir = path + if not layer_dir: + raise GbpError("%s not under configured layers" % abspath) + + bb_files = [path for path in tinfoil.cooker_data.pkg_fn + if os.path.dirname(path) == abspath] + if len(bb_files): + bb_file = bb_files[-1] + gbp.log.debug("Found %d recipes in %s, choosing %s" % + (len(bb_files), pkg_dir, os.path.basename(bb_file))) + else: + raise GbpError("No recipes found in %s" % pkg_dir) + return bb_file + +def guess_pkg(tinfoil, pkg): + """Guess package (recipe) from configured bitbake environment""" + if pkg in tinfoil.cooker_data.pkg_pn: + pkg_bb = tinfoil.cooker_data.pkg_pn[pkg][0] + elif not os.path.isdir(pkg): + abspath = os.path.abspath(pkg) + if abspath in tinfoil.cooker_data.pkg_fn: + pkg_bb = abspath + else: + raise GbpError("Package %s not found in any configured layer" % pkg) + elif os.path.exists(pkg): + pkg_bb = guess_pkg_from_dir(pkg, tinfoil) + else: + raise GbpError("Unable to find %s" % pkg) + return pkg_bb + + +# Initialize module +bb = import_bb() diff --git a/gbp/config.py b/gbp/config.py index 9fd31b73..267400ce 100644 --- a/gbp/config.py +++ b/gbp/config.py @@ -589,101 +589,149 @@ class GbpOptionParserRpm(GbpOptionParser): Handles commandline options and parsing of config files for rpm tools """ defaults = dict(GbpOptionParser.defaults) + defaults.update({ + 'tmp-dir' : '/var/tmp/gbp/', + 'vendor' : 'vendor', + 'native' : 'auto', + 'builder' : 'rpmbuild', + 'cleaner' : '/bin/true', + 'merge' : 'False', + 'packaging-dir' : '', + 'packaging-branch' : 'master', + 'packaging-tag' : '%(vendor)s/%(version)s', + 'upstream-tag' : 'upstream/%(upstreamversion)s', + 'pq-branch' : 'development/%(branch)s', + 'import-files' : ['.gbp.conf', + 'debian/gbp.conf'], + 'spec-file' : 'auto', + + 'export-dir' : '../rpmbuild', + 'ignore-untracked' : 'False', + 'rpmbuild-builddir' : 'BUILD', + 'rpmbuild-rpmdir' : 'RPMS', + 'rpmbuild-sourcedir' : 'SOURCES', + 'rpmbuild-specdir' : 'SPECS', + 'rpmbuild-srpmdir' : 'SRPMS', + 'rpmbuild-buildrootdir' : 'BUILDROOT', + 'patch-export' : 'False', + 'patch-export-ignore-path' : '', + 'patch-export-compress' : '0', + 'patch-export-squash-until' : '', + 'pristine-tarball-name' : 'auto', + 'orig-prefix' : 'auto', + 'patch-import' : 'True', + 'spec-vcs-tag' : '', + + + 'changelog-file' : 'auto', + 'changelog-revision' : '', + 'spawn-editor' : 'always', + 'editor-cmd' : 'vim', + 'meta-bts' : '(Close|Closes|Fixes|Fix)', + }) + + help = dict(GbpOptionParser.help) + help.update({ + + 'vendor': + "Distribution vendor name, default is '%(vendor)s'", + 'native': + "Treat this package as native, default is '%(native)s'", + 'packaging-branch': + "Branch the packaging is being maintained on, rpm counterpart " + "of the 'debian-branch' option, default is " + "'%(packaging-branch)s'", + 'packaging-dir': + "Subdir for RPM packaging files, default is '%(packaging-dir)s'", + 'packaging-tag': + "Format string for packaging tags, RPM counterpart of the " + "'debian-tag' option, default is '%(packaging-tag)s'", + 'pq-branch': + "format string for the patch-queue branch name, default is '%(pq-branch)s'", + 'import-files': + "Comma-separated list of additional file(s) to import from " + "packaging branch. These will appear as one monolithic patch " + "in the pq/development branch. Default is %(import-files)s", + 'spec-file': + "Spec file to use, 'auto' makes gbp to guess, other values " + "make the packaging-dir option to be ignored, default is " + "'%(spec-file)s'", + + 'ignore-untracked': + "build with untracked files in the source tree, default is " + "'%(ignore-untracked)s'", + 'patch-export': + "Create patches between upstream and export-treeish, default " + "is '%(patch-export)s'", + 'patch-export-compress': + "Compress (auto-generated) patches larger than given number of " + "bytes, 0 never compresses, default is " + "'%(patch-export-compress)s'", + 'patch-export-ignore-path': + "Exclude changes to path(s) matching regex, default is " + "'%(patch-export-ignore-path)s'", + 'patch-export-squash-until': + "Squash commits (from upstream) until given tree-ish into one " + "big diff, format is '<commit_ish>[:<filename_base>]'. " + "Default is '%(patch-export-squash-until)s'", + 'patch-import': + "Import patches to the packaging branch, default is " + "'%(patch-import)s'", + 'spec-vcs-tag': + "Set/update the 'VCS:' tag in the spec file, empty value " + "removes the tag entirely, default is '%(spec-vcs-tag)s'", + 'pristine-tarball-name': + "Filename to record to pristine-tar, set to 'auto' to not " + "mangle the file name, default is '%(pristine-tarball-name)s'", + 'orig-prefix': + "Prefix (dir) to be used when generating/importing tarballs, " + "default is '%(orig-prefix)s'", + 'changelog-file': + "Changelog file to be used, default is '%(changelog-file)s'", + 'changelog-revision': + "Format string for the revision field in the changelog header. " + "If empty or not defined the default from packaging policy is " + "used.", + 'editor-cmd': + "Editor command to use", + 'git-author': + "Use name and email from git-config for the changelog header, " + "default is '%(git-author)s'", + 'meta-bts': + "Meta tags for the bts commands, default is '%(meta-bts)s'", + 'tmp-dir': + "Base directory under which temporary directories are " + "created, default is '%(tmp-dir)s'", + }) + +class GbpOptionParserBB(GbpOptionParserRpm): + """Commandline and config file option parser for the -bb commands""" + defaults = dict(GbpOptionParserRpm.defaults) defaults.update( { - 'vendor' : 'vendor', - 'native' : 'auto', - 'builder' : 'rpmbuild', - 'cleaner' : '/bin/true', - 'merge' : 'False', - 'packaging-dir' : '', - 'packaging-tag' : '%(vendor)s/%(version)s', - 'upstream-tag' : 'upstream/%(upstreamversion)s', - 'pq-branch' : 'development/%(branch)s', - 'import-files' : ['.gbp.conf', - 'debian/gbp.conf'], - 'spec-file' : 'auto', - 'export-dir' : '../rpmbuild', - 'ignore-untracked' : 'False', - 'rpmbuild-builddir' : 'BUILD', - 'rpmbuild-rpmdir' : 'RPMS', - 'rpmbuild-sourcedir' : 'SOURCES', - 'rpmbuild-specdir' : 'SPECS', - 'rpmbuild-srpmdir' : 'SRPMS', - 'rpmbuild-buildrootdir' : 'BUILDROOT', - 'patch-export' : 'False', - 'patch-export-ignore-path' : '', - 'patch-export-compress' : '0', - 'patch-export-squash-until' : '', - 'pristine-tarball-name' : 'auto', - 'orig-prefix' : 'auto', - 'patch-import' : 'True', - 'spec-vcs-tag' : '', - 'changelog-file' : 'auto', - 'changelog-revision' : '', - 'spawn-editor' : 'always', - 'editor-cmd' : 'vim', - 'meta-bts' : '(Close|Closes|Fixes|Fix)', + 'builder' : 'bitbake', + 'export-dir' : '', + 'meta-dir' : '', + 'bb-file' : '', + 'bb-vcs-info' : '', + 'submit-tag' : 'submit/%(target)s/%(nowtime)s', + 'target' : 'tizen', } ) - help = dict(GbpOptionParser.help) + help = dict(GbpOptionParserRpm.help) help.update( { - 'vendor': - "Distribution vendor name", - 'native': - "Treat this package as native, default is '%(native)s'", - 'packaging-dir': - "subdir where packaging files are stored, default is '%(packaging-dir)s'", - 'packaging-tag': - "format string for packaging tags, rpm counterpart of the 'debian-tag' option, default is '%(packaging-tag)s'", - 'pq-branch': - "format string for the patch-queue branch name, default is '%(pq-branch)s'", - 'import-files': - ("Comma-separated list of additional file(s) to import " - "from packaging branch. These will appear as one " - "monolithic patch in the pq/development branch. " - "Default is %(import-files)s"), - 'spec-file': - "Spec file to use, 'auto' makes gbp to guess, other values make the packaging-dir option to be ignored, default is '%(spec-file)s'", - 'ignore-untracked': - "build with untracked files in the source tree, default is '%(ignore-untracked)s'", - 'patch-export': - "Create patches between upstream and export-treeish, default is '%(patch-export)s'", - 'patch-export-ignore-path': - ("Exclude changes to path(s) matching regex, default " - "is '%(patch-export-ignore-path)s'"), - 'patch-export-compress': - "Compress (auto-generated) patches larger than given number of bytes, 0 never compresses, default is '%(patch-export-compress)s'", - 'patch-export-squash-until': - ("Squash commits (from upstream) until given tree-ish " - "into one big diff, format is " - "'<commit_ish>[:<filename_base>]'. " - "Default is '%(patch-export-squash-until)s'"), - 'pristine-tarball-name': - "Filename to record to pristine-tar, set to 'auto' to not mangle the file name, default is '%(pristine-tarball-name)s'", - 'orig-prefix': - "Prefix (dir) to be used when generating/importing tarballs, default is '%(orig-prefix)s'", - 'patch-import': - "Import patches to the packaging branch, default is '%(patch-import)s'", - 'spec-vcs-tag': - ("Set/update the 'VCS:' tag in the spec file, empty " - "value removes the tag entirely, default is " - "'%(spec-vcs-tag)s'"), - 'changelog-file': - ("Changelog file to be used, default is " - "'%(changelog-file)s'"), - 'changelog-revision': - ("Format string for the revision field in the " - "changelog header. If empty or not defined the " - "default from packaging policy is used."), - 'editor-cmd': - "Editor command to use", - 'git-author': - ("Use name and email from git-config for the changelog " - "header, default is '%(git-author)s'"), - 'meta-bts': - ("Meta tags for the bts commands, default is " - "'%(meta-bts)s'"), + 'meta-dir': + "Subdir where bitbake meta files are stored, default " + "is '%(meta-dir)s'", + 'bb-file': + "Bitbake recipe file to build", + 'bb-vcs-info': + "Format string for the VCS information automatically " + "set in the recipe file, default is '%(bb-vcs-info)s'", + 'submit-tag': + "Submit tag format, default is '%(submit-tag)s'", + 'target': + "Submit target used in submit tag, default is " + "'%(target)s'", } ) - # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/rpm/__init__.py b/gbp/rpm/__init__.py index 4c5c22bf..d5fa1ee8 100644 --- a/gbp/rpm/__init__.py +++ b/gbp/rpm/__init__.py @@ -901,5 +901,4 @@ def string_to_int(val_str): else: return int(val_str) - # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/buildpackage_bb.py b/gbp/scripts/buildpackage_bb.py new file mode 100644 index 00000000..95c66368 --- /dev/null +++ b/gbp/scripts/buildpackage_bb.py @@ -0,0 +1,536 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +"""Build an RPM package out of a Git repo with Bitbake meta data""" + +import ConfigParser +import os, os.path +import sys +import shutil +import tempfile + +import gbp.rpm as rpm +from gbp.rpm.policy import RpmPkgPolicy +from gbp.command_wrappers import Command, RunAtCommand, CommandExecFailed +from gbp.config import GbpOptionParserBB, GbpOptionGroup +from gbp.rpm.git import (GitRepositoryError, RpmGitRepository) +from gbp.errors import GbpError +import gbp.log +import gbp.notifications +from gbp.scripts.common.buildpackage import (index_name, wc_names, dump_tree, + drop_index) +from gbp.scripts.buildpackage_rpm import (disable_hooks, get_tree, + get_current_branch, get_upstream_tree, get_vcs_info, + packaging_tag_name, create_packaging_tag, GbpAutoGenerateError) +from gbp.scripts.import_bb import recursive_copy +from gbp.scripts.pq_bb import update_patch_series +from gbp.scripts.common.pq import is_pq_branch, pq_branch_base +from gbp.bb import (bb, init_tinfoil, guess_bb_path, BBFile, bb_from_repo, + pkg_version, parse_bb) + +# pylint: disable=bad-continuation + + +def guess_export_params(repo, options): + """Get commit and tree from where to export packaging and patches""" + tree = None + branch = None + if options.export in wc_names.keys() + [index_name, 'HEAD']: + branch = get_current_branch(repo) + elif options.export in repo.get_local_branches(): + branch = options.export + if branch: + if is_pq_branch(branch, options): + packaging_branch = pq_branch_base(branch, options) + if repo.has_branch(packaging_branch): + gbp.log.info("It seems you're building a development/patch-" + "queue branch. Export target changed to '%s' and " + "patch-export enabled!" % packaging_branch) + options.patch_export = True + if not options.patch_export_rev: + options.patch_export_rev = options.export + options.export = packaging_branch + else: + gbp.log.warn("It seems you're building a development/patch-" + "queue branch. No corresponding packaging branch " + "found. Build may fail!") + if tree is None: + tree = get_tree(repo, options.export) + + # Get recipe path + bb_path = guess_bb_path(options, repo, tree, bbappend=True) + # Adjust meta-dir accordingly + options.meta_dir = os.path.dirname(bb_path) + + # Filter out changes in recipe directory + if options.patch_export: + relpath = os.path.relpath(os.path.abspath(options.meta_dir), repo.path) + if relpath != '.': + gbp.log.info("Auto-excluding changes under meta-dir (%s/)" % + relpath) + if options.patch_export_ignore_path: + options.patch_export_ignore_path += '|' + relpath + '/*' + else: + options.patch_export_ignore_path = relpath + '/*' + return tree + +def guess_export_dir(options, tinfoil, repo, treeish): + """Guess export directory""" + if not tinfoil: + gbp.log.err("Bitbake build environment (bb.tinfoil) not initialized, " + "unable to guess export directory") + gbp.log.err("Please use --git-export-dir or try initializing bitbake " + "build environment with the 'oe-init-build-env' script") + raise GbpError + + gbp.log.info('Guessing export directory') + tinfoil.parseRecipes() + + # Parse recipe + bb_path = guess_bb_path(options, repo, treeish, bbappend=True) + #cfg_data = bb.data.createCopy(tinfoil.config_data) + #bbfile = bb_from_repo(cfg_data, repo, treeish, bb_path) + # Use naive parsing, at least for now as the file might be .bbappend + bbfile = bb_from_repo(None, repo, treeish, bb_path) + + pkg_name = bbfile.getVar('PN', True) + bb_name = os.path.basename(bb_path) + if bb_name.endswith('.bb'): + for name in tinfoil.cooker_data.pkg_fn: + if os.path.basename(name) == bb_name and os.path.isabs(name): + gbp.log.debug("Found matching recipe filename: %s" % name) + return os.path.dirname(name) + else: + for name, appends in tinfoil.cooker.collection.appendlist.iteritems(): + print name, appends + if name.rsplit('_', 1)[0] == pkg_name: + gbp.log.debug("Found %s from appends" % name) + for append_name in appends: + if os.path.basename(append_name) == bb_name: + gbp.log.debug("Found matching recipe filename: %s" % + append_name) + return os.path.dirname(append_name) + export_dir = os.path.dirname(appends[-1]) + gbp.log.debug("Using existing appends directory %s" % + export_dir) + return export_dir + if pkg_name in tinfoil.cooker_data.pkg_pn: + export_dir = os.path.dirname(tinfoil.cooker_data.pkg_pn[pkg_name][-1]) + gbp.log.debug("Using existing package directory %s" % export_dir) + return export_dir + else: + pkg_ver = bbfile.getVar('PV', True) + raise GbpError("Package %s-%s not found under any configured layer, " + "please use --git-export-dir to define the export " + "directory" % (pkg_name, pkg_ver)) + +def export_patches(repo, bbfile, export_treeish, options): + """Generate patches and update recipe""" + try: + if bbfile.getVar('SRCREV', True): + upstream_tree = bbfile.getVar('SRCREV', True) + else: + upstream_version = bbfile.getVar('PV', True) + upstream_tree = get_upstream_tree(repo, upstream_version, options) + update_patch_series(repo, bbfile, upstream_tree, export_treeish, + options) + except (GitRepositoryError, GbpError) as err: + raise GbpAutoGenerateError(str(err)) + + +def is_native(repo, options): + """Determine whether a package is native or non-native""" + if options.native.is_auto(): + if repo.has_branch(options.upstream_branch): + return False + # Check remotes, too + for remote_branch in repo.get_remote_branches(): + remote, branch = remote_branch.split('/', 1) + if branch == options.upstream_branch: + gbp.log.debug("Found upstream branch '%s' from remote '%s'" % + (remote, branch)) + return False + return True + + return options.native.is_on() + + +def setup_builder(options, builder_args): + """Setup everything to use git-pbuilder""" + # TODO: placeholder for Bitbake: implement or remove entirely + pass + +def bb_get_local_files(bbfile, tgt_dir, whole_dir=False): + """Get (local) packaging files""" + if not whole_dir: + for path in bbfile.localfiles + bbfile.includes + [bbfile.bb_path]: + relpath = os.path.relpath(path, bbfile.bb_dir) + subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) + if not os.path.exists(subdir): + os.makedirs(subdir) + shutil.copy2(path, os.path.join(tgt_dir, relpath)) + else: + # Simply copy whole meta dir, if requested + recursive_copy(bbfile.bb_dir, tgt_dir) + +def dump_meta(cfg_data, options, repo, treeish, dump_dir): + """Parse and dump meta information from a treeish""" + tmpdir = tempfile.mkdtemp(prefix='gbp-bb_') + try: + bb_path = guess_bb_path(options, repo, treeish, bbappend=True) + # Dump whole meta directory + dump_tree(repo, tmpdir, '%s:%s' % (treeish, os.path.dirname(bb_path)), + False) + # Parse recipe + full_path = os.path.join(tmpdir, os.path.basename(bb_path)) + bbfile = BBFile(full_path, cfg_data) + bb_get_local_files(bbfile, dump_dir) + except GitRepositoryError as err: + raise GbpError("Git error: %s" % err) + finally: + shutil.rmtree(tmpdir) + + # Re-parse recipe from final location + full_path = os.path.abspath(os.path.join(dump_dir, + os.path.basename(bb_path))) + return BBFile(full_path, cfg_data) + + +def build_parser(name, prefix=None, git_treeish=None): + """Create command line parser""" + try: + parser = GbpOptionParserBB(command=os.path.basename(name), + prefix=prefix, git_treeish=git_treeish) + except ConfigParser.ParsingError, err: + gbp.log.err(err) + return None + + tag_group = GbpOptionGroup(parser, "tag options", + "options related to git tag creation") + branch_group = GbpOptionGroup(parser, "branch options", + "branch layout options") + cmd_group = GbpOptionGroup(parser, "external command options", + "how and when to invoke external commands and hooks") + orig_group = GbpOptionGroup(parser, "orig tarball options", + "options related to the creation of the orig tarball") + export_group = GbpOptionGroup(parser, "export build-tree options", + "alternative build tree related options") + parser.add_option_group(tag_group) + parser.add_option_group(orig_group) + parser.add_option_group(branch_group) + parser.add_option_group(cmd_group) + parser.add_option_group(export_group) + + parser.add_boolean_config_file_option(option_name = "ignore-untracked", + dest="ignore_untracked") + parser.add_boolean_config_file_option(option_name = "ignore-new", + dest="ignore_new") + parser.add_option("--git-verbose", action="store_true", dest="verbose", + help="verbose command execution") + parser.add_config_file_option(option_name="tmp-dir", dest="tmp_dir") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_config_file_option(option_name="notify", dest="notify", + type='tristate') + parser.add_config_file_option(option_name="vendor", action="store", + dest="vendor") + parser.add_config_file_option(option_name="native", dest="native", + type='tristate') + tag_group.add_option("--git-tag", action="store_true", dest="tag", + help="create a tag after a successful build") + tag_group.add_option("--git-tag-only", action="store_true", dest="tag_only", + help="don't build, only tag and run the posttag hook") + tag_group.add_option("--git-retag", action="store_true", dest="retag", + help="don't fail if the tag already exists") + tag_group.add_boolean_config_file_option(option_name="sign-tags", + dest="sign_tags") + tag_group.add_config_file_option(option_name="keyid", dest="keyid") + tag_group.add_config_file_option(option_name="packaging-tag", + dest="packaging_tag") + tag_group.add_config_file_option(option_name="upstream-tag", + dest="upstream_tag") + orig_group.add_config_file_option(option_name="upstream-tree", + dest="upstream_tree") + branch_group.add_config_file_option(option_name="upstream-branch", + dest="upstream_branch") + branch_group.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch") + branch_group.add_config_file_option(option_name="pq-branch", + dest="pq_branch") + branch_group.add_boolean_config_file_option(option_name = "ignore-branch", + dest="ignore_branch") + cmd_group.add_config_file_option(option_name="builder", dest="builder", + help="command to build the package, default is " + "'%(builder)s'") + cmd_group.add_config_file_option(option_name="cleaner", dest="cleaner", + help="command to clean the working copy, default is " + "'%(cleaner)s'") + cmd_group.add_config_file_option(option_name="prebuild", dest="prebuild", + help="command to run before a build, default is " + "'%(prebuild)s'") + cmd_group.add_config_file_option(option_name="postexport", + dest="postexport", + help="command to run after exporting the source tree, " + "default is '%(postexport)s'") + cmd_group.add_config_file_option(option_name="postbuild", dest="postbuild", + help="hook run after a successful build, default is " + "'%(postbuild)s'") + cmd_group.add_config_file_option(option_name="posttag", dest="posttag", + help="hook run after a successful tag operation, default " + "is '%(posttag)s'") + cmd_group.add_boolean_config_file_option(option_name="hooks", dest="hooks") + export_group.add_option("--git-no-build", action="store_true", + dest="no_build", + help="Don't run builder or the associated hooks") + export_group.add_config_file_option(option_name="export-dir", + dest="export_dir", type="path", + help="Build topdir, also export the sources under " + "EXPORT_DIR, default is '%(export-dir)s'") + export_group.add_config_file_option("export", dest="export", + help="export treeish object TREEISH, default is " + "'%(export)s'", metavar="TREEISH") + export_group.add_config_file_option(option_name="meta-dir", + dest="meta_dir") + export_group.add_config_file_option(option_name="bb-file", dest="bb_file") + export_group.add_boolean_config_file_option("patch-export", + dest="patch_export") + export_group.add_option("--git-patch-export-rev", dest="patch_export_rev", + metavar="TREEISH", + help="[experimental] Export patches from treeish object " + "TREEISH") + export_group.add_config_file_option("patch-export-ignore-path", + dest="patch_export_ignore_path") + export_group.add_config_file_option("patch-export-compress", + dest="patch_export_compress") + export_group.add_config_file_option("patch-export-squash-until", + dest="patch_export_squash_until") + export_group.add_boolean_config_file_option(option_name="patch-numbers", + dest="patch_numbers") + export_group.add_config_file_option("bb-vcs-info", dest="bb_vcs_info") + return parser + +def parse_args(argv, prefix, git_treeish=None): + """Parse config and command line arguments""" + args = [arg for arg in argv[1:] if arg.find('--%s' % prefix) == 0] + builder_args = [arg for arg in argv[1:] if arg.find('--%s' % prefix) == -1] + + # We handle these although they don't have a --git- prefix + for arg in ["--help", "-h", "--version"]: + if arg in builder_args: + args.append(arg) + + parser = build_parser(argv[0], prefix=prefix, git_treeish=git_treeish) + if not parser: + return None, None, None + options, args = parser.parse_args(args) + + options.patch_export_compress = rpm.string_to_int( + options.patch_export_compress) + + gbp.log.setup(options.color, options.verbose, options.color_scheme) + if not options.hooks: + disable_hooks(options) + if options.retag: + if not options.tag and not options.tag_only: + gbp.log.err("'--%sretag' needs either '--%stag' or '--%stag-only'" % + (prefix, prefix, prefix)) + return None, None, None + + return options, args, builder_args + + +def main(argv): + """Entry point for git-buildpackage-bb""" + retval = 0 + prefix = "git-" + bbfile = None + dump_dir = None + + if not bb: + return 1 + + options, gbp_args, builder_args = parse_args(argv, prefix) + if not options: + return 1 + + try: + repo = RpmGitRepository(os.path.curdir) + except GitRepositoryError: + gbp.log.err("%s is not a git repository" % (os.path.abspath('.'))) + return 1 + + # Determine tree-ish to be exported + try: + tree = get_tree(repo, options.export) + except GbpError as err: + gbp.log.err('Failed to determine export treeish: %s' % err) + return 1 + # Re-parse config options with using the per-tree config file(s) from the + # exported tree-ish + options, gbp_args, builder_args = parse_args(argv, prefix, tree) + + branch = get_current_branch(repo) + + try: + tinfoil = init_tinfoil(config_only=True) + #bb_cfg_data = bb.data.createCopy(tinfoil.config_data) + except GbpError: + tinfoil = None + + # Use naive parsing because repository might only have .bb file + gbp.log.info("Using naive standalone parsing of recipes in package repo.") + bb_cfg_data = None + + try: + tree = guess_export_params(repo, options) + + Command(options.cleaner, shell=True)() + if not options.ignore_new: + (ret, out) = repo.is_clean(options.ignore_untracked) + if not ret: + gbp.log.err("You have uncommitted changes in your source tree:") + gbp.log.err(out) + raise GbpError("Use --git-ignore-new or --git-ignore-untracked " + "to ignore.") + + if not options.ignore_new and not options.ignore_branch: + if branch != options.packaging_branch: + gbp.log.err("You are not on branch '%s' but on '%s'" % + (options.packaging_branch, branch)) + raise GbpError("Use --git-ignore-branch to ignore or " + "--git-packaging-branch to set the branch name.") + + if not options.tag_only: + # Dump/parse meta to export dir + if options.export_dir: + export_dir = os.path.abspath(options.export_dir) + else: + export_dir = guess_export_dir(options, tinfoil, repo, tree) + gbp.log.info("Dumping meta from tree '%s' to '%s'" % + (options.export, export_dir)) + bbfile = dump_meta(bb_cfg_data, options, repo, tree, + export_dir) + + # Setup builder opts + setup_builder(options, builder_args) + + if is_native(repo, options) and bbfile.getVar('SRCREV') == 'HEAD': + # Update SRCREV for native packages that are exported from + # pristine repository + BBFile.set_var_val(bbfile.bb_path, 'SRCREV', + repo.rev_parse(tree)) + + # TODO: Re-design the handling of native packages. Updating + # SRCREV must probably be more explicit + if options.patch_export: + # Generate patches, if requested + if options.patch_export_rev: + patch_tree = get_tree(repo, options.patch_export_rev) + else: + patch_tree = tree + export_patches(repo, bbfile, patch_tree, options) + + # Run postexport hook + if options.postexport: + RunAtCommand(options.postexport, shell=True, + extra_env={'GBP_GIT_DIR': repo.git_dir, + 'GBP_TMP_DIR': export_dir} + )(dir=export_dir) + # Do actual build + if not options.no_build: + if options.prebuild: + RunAtCommand(options.prebuild, shell=True, + extra_env={'GBP_GIT_DIR': repo.git_dir, + 'GBP_BUILD_DIR': export_dir} + )(dir=export_dir) + + # Unlock cooker so that we are able to run external bitbake + if options.builder == 'bitbake' and tinfoil: + bb.utils.unlockfile(tinfoil.cooker.lock) + + # Finally build the package: + bb_path = bbfile.getVar('FILE', True) + builder_args.extend(['-b', bb_path]) + RunAtCommand(options.builder, builder_args, shell=True, + extra_env={'GBP_BUILD_DIR': export_dir})() + + if options.postbuild: + Command(options.postbuild, shell=True, + extra_env={'GBP_BUILD_DIR': export_dir})() + else: + # Tag-only: we just need to parse the meta + bbfile = parse_bb(bb_cfg_data, options, repo, tree) + + # Tag (note: tags the exported version) + if options.tag or options.tag_only: + version = pkg_version(bbfile) + gbp.log.info("Tagging %s" % + RpmPkgPolicy.compose_full_version(version)) + commit_info = repo.get_commit_info(tree) + tag = packaging_tag_name(repo, version, commit_info, options) + if options.retag and repo.has_tag(tag): + repo.delete_tag(tag) + create_packaging_tag(repo, tag, commit=tree, version=version, + options=options) + vcs_info = get_vcs_info(repo, tag) + if options.posttag: + sha = repo.rev_parse("%s^{}" % tag) + Command(options.posttag, shell=True, + extra_env={'GBP_TAG': tag, + 'GBP_BRANCH': branch, + 'GBP_SHA1': sha})() + else: + vcs_info = get_vcs_info(repo, tree) + # TODO: Put VCS information to recipe + if options.bb_vcs_info: + raise GbpError("Injecting VCS info into recipe not yet supported") + + except CommandExecFailed: + retval = 1 + except GitRepositoryError as err: + gbp.log.err("Git command failed: %s" % err) + retval = 1 + except GbpAutoGenerateError as err: + if len(err.__str__()): + gbp.log.err(err) + retval = 2 + except GbpError, err: + if len(err.__str__()): + gbp.log.err(err) + retval = 1 + finally: + drop_index(repo) + if dump_dir and os.path.exists(dump_dir): + shutil.rmtree(dump_dir) + + if not options.tag_only: + if bbfile and options.notify: + summary = "GBP buildpackage-bb %s" % \ + ["failed", "successful"][not retval] + message = ("Build of %s %s %s" % (bbfile.getVar('PN', True), + RpmPkgPolicy.compose_full_version(pkg_version(bbfile)), + ["failed", "succeeded"][not retval])) + if not gbp.notifications.notify(summary, message, options.notify): + gbp.log.err("Failed to send notification") + retval = 1 + + return retval + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/gbp/scripts/buildpackage_rpm.py b/gbp/scripts/buildpackage_rpm.py index 8fb0b2a4..56db5e13 100755 --- a/gbp/scripts/buildpackage_rpm.py +++ b/gbp/scripts/buildpackage_rpm.py @@ -621,7 +621,7 @@ def main(argv): # Get/build the orig tarball if is_native(repo, options): - if spec.orig_src and not options.no_create_orig: + if spec.orig_src: # Just build source archive from the exported tree gbp.log.info("Creating (native) source archive %s from '%s'" % (spec.orig_src['filename'], tree)) if spec.orig_src['compression']: diff --git a/gbp/scripts/clone_bb.py b/gbp/scripts/clone_bb.py new file mode 100755 index 00000000..a7e9c9f4 --- /dev/null +++ b/gbp/scripts/clone_bb.py @@ -0,0 +1,174 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2009,2010 Guido Guenther <agx@sigxcpu.org> +# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# inspired by dom-git-checkout +# +"""Clone a package Git repository from a bitbake-based distro""" + +import ConfigParser +import re +import sys +import os, os.path + +from gbp.config import (GbpOptionParser, GbpOptionGroup) +from gbp.git import GitRepositoryError +from gbp.errors import GbpError +import gbp.log +from gbp.rpm.git import RpmGitRepository as GitRepository +from gbp.bb import bb, init_tinfoil, guess_pkg + +# pylint: disable=bad-continuation + + +def guess_remote(tinfoil, source): + """Guess the remote repository URL""" + # Try to determine if a remote URL is referenced + if re.match(r'[a-z]{3,5}://', source) or re.match(r'\S+@\S+', source): + return source, None + + # Get remote repo from recipe + recipe = guess_pkg(tinfoil, source) + appends = tinfoil.cooker.collection.get_file_appends(recipe) + gbp.log.info("Using %s with appends %s" % (recipe, appends)) + pkg_data = bb.cache.Cache.loadDataFull(recipe, appends, tinfoil.config_data) + uri = pkg_data.getVar('GBP_PACKAGING_REPO', True) + if not uri: + raise GbpError("GBP_PACKAGING_REPO not defined in recipe. Unable to " + "determine remote repo") + rev = pkg_data.getVar('GBP_PACKAGING_REV', True) + return uri, rev + + +def build_parser(name): + """Create command line argument parser""" + try: + parser = GbpOptionParser(command=os.path.basename(name), prefix='', + usage='%prog [options] repository - clone a ' + 'remote per-package repository') + except ConfigParser.ParsingError as err: + gbp.log.err(err) + return None + + branch_group = GbpOptionGroup(parser, "branch options", + "branch tracking and layout options") + parser.add_option_group(branch_group) + + branch_group.add_option("--all", action="store_true", dest="all", + help="track all branches, not only packaging and upstream") + branch_group.add_config_file_option(option_name="upstream-branch", + dest="upstream_branch") + branch_group.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch") + branch_group.add_option("--depth", action="store", dest="depth", default=0, + help="git history depth (for creating shallow clones)") + + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + help="verbose command execution") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + return parser + + +def parse_args (argv): + """Parse command line arguments""" + parser = build_parser(argv[0]) + if not parser: + return None, None + + (options, args) = parser.parse_args(argv) + gbp.log.setup(options.color, options.verbose, options.color_scheme) + return (options, args) + + +def main(argv): + """Entry point for gbp-clone-bb""" + retval = 0 + + if not bb: + return 1 + + (options, args) = parse_args(argv) + if not options: + return 1 + + if len(args) < 2: + gbp.log.err("Need a package or repository to clone.") + return 1 + + # Determine target dir + clone_to = os.path.curdir + auto_name = False + if len(args) < 3: + if 'BUILDDIR' in os.environ: + clone_to = os.path.join(os.environ['BUILDDIR'], 'devel') + auto_name = True + else: + clone_to = args[2] + + try: + tinfoil = init_tinfoil() + + source, revision = guess_remote(tinfoil, args[1]) + + gbp.log.info("Cloning from %s..." % source) + repo = GitRepository.clone(clone_to, source, options.depth, + auto_name=auto_name) + os.chdir(repo.path) + + # Reparse the config files of the cloned repository so we pick up the + # branch information from there: + (options, args) = parse_args(argv) + + # Track all branches: + if options.all: + remotes = repo.get_remote_branches() + for remote in remotes: + local = remote.replace("origin/", "", 1) + if not repo.has_branch(local) and local != "HEAD": + repo.create_branch(local, remote) + else: # only track gbp's default branches + branches = [ options.packaging_branch, options.upstream_branch ] + gbp.log.debug('Will track branches: %s' % branches) + for branch in branches: + remote = 'origin/%s' % branch + if repo.has_branch(remote, remote=True) and \ + not repo.has_branch(branch): + repo.create_branch(branch, remote) + + gbp.log.info("Successfully cloned into %s" % clone_to) + if (revision and repo.rev_parse('HEAD') != + repo.rev_parse('%s^0' % revision)): + gbp.log.info("Checking out revision %s" % revision) + repo.set_branch(revision) + + except GitRepositoryError as err: + gbp.log.err("Git command failed: %s" % err) + retval = 1 + except GbpError as err: + if len(err.__str__()): + gbp.log.err(err) + retval = 1 + + return retval + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/import_bb.py b/gbp/scripts/import_bb.py new file mode 100755 index 00000000..d0aeae1c --- /dev/null +++ b/gbp/scripts/import_bb.py @@ -0,0 +1,419 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +"""Import an RPM package in Bitbake format""" + +import ConfigParser +import sys +import os +import shutil + +import gbp.tmpfile as tempfile +import gbp.command_wrappers as gbpc +import gbp.log +from gbp.rpm import RpmUpstreamSource +from gbp.rpm.policy import RpmPkgPolicy +from gbp.rpm.git import RpmGitRepository, GitRepositoryError +from gbp.config import (GbpOptionParserBB, GbpOptionGroup, + no_upstream_branch_msg) +from gbp.errors import GbpError +from gbp.pkg import parse_archive_filename +from gbp.scripts.import_srpm import move_tag_stamp, force_to_branch_head +from gbp.bb import bb, init_tinfoil, pkg_version, guess_pkg + +# pylint: disable=bad-continuation + +NO_PACKAGING_BRANCH_MSG = """ +Repository does not have branch '%s' for meta/packaging files. +You need to reate it or use --packaging-branch to specify it. +""" + +class SkipImport(Exception): + """Nothing imported""" + pass + +def set_bare_repo_options(options): + """Modify options for import into a bare repository""" + if options.pristine_tar: + gbp.log.info("Bare repository: setting %s option '--no-pristine-tar'") + options.pristine_tar = False + + +def build_parser(name): + """Create command line parser""" + try: + parser = GbpOptionParserBB(command=os.path.basename(name), + prefix='', + usage='%prog [options] /path/to/package' + '.src.rpm') + except ConfigParser.ParsingError, err: + gbp.log.err(err) + return None + + import_group = GbpOptionGroup(parser, "import options", + "pristine-tar and filtering") + tag_group = GbpOptionGroup(parser, "tag options", + "options related to git tag creation") + branch_group = GbpOptionGroup(parser, "version and branch naming options", + "version number and branch layout options") + + for group in [import_group, branch_group, tag_group ]: + parser.add_option_group(group) + + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + default=False, help="verbose command execution") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_config_file_option(option_name="tmp-dir", dest="tmp_dir") + parser.add_config_file_option(option_name="vendor", action="store", + dest="vendor") + branch_group.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch") + branch_group.add_config_file_option(option_name="upstream-branch", + dest="upstream_branch") + branch_group.add_option("--upstream-vcs-tag", dest="vcs_tag", + help="Upstream VCS tag on top of which to import " + "the orig sources") + branch_group.add_boolean_config_file_option( + option_name="create-missing-branches", + dest="create_missing_branches") + + tag_group.add_boolean_config_file_option(option_name="sign-tags", + dest="sign_tags") + tag_group.add_config_file_option(option_name="keyid", + dest="keyid") + tag_group.add_config_file_option(option_name="packaging-tag", + dest="packaging_tag") + tag_group.add_config_file_option(option_name="upstream-tag", + dest="upstream_tag") + + import_group.add_config_file_option(option_name="filter", + dest="filters", action="append") + import_group.add_boolean_config_file_option(option_name="pristine-tar", + dest="pristine_tar") + import_group.add_option("--allow-same-version", action="store_true", + dest="allow_same_version", default=False, + help="allow to import already imported version") + import_group.add_config_file_option(option_name="meta-dir", + dest="meta_dir") + return parser + +def parse_args(argv): + """Parse commandline arguments""" + parser = build_parser(argv[0]) + if not parser: + return None, None + + (options, args) = parser.parse_args(argv[1:]) + gbp.log.setup(options.color, options.verbose, options.color_scheme) + return options, args + + +def init_repo(path): + """Check and initialize Git repository""" + try: + repo = RpmGitRepository(path) + clean, out = repo.is_clean() + if not clean and not repo.is_empty(): + gbp.log.err("Repository has uncommitted changes, commit " + "these first:") + gbp.log.err(out) + raise GbpError + except GitRepositoryError: + gbp.log.info("No git repository found, creating one in %s" % path) + repo = RpmGitRepository.create(path) + return repo + +def recursive_copy(src, dst): + """Recursive copy, overwriting files and preserving symlinks""" + # Remove existing destinations, if needed + if os.path.isfile(dst) or os.path.islink(dst): + os.unlink(dst) + elif (os.path.isfile(src) or os.path.islink(src)) and os.path.isdir(dst): + # Remove dst dir if src is a file + shutil.rmtree(dst) + + try: + if os.path.islink(src): + os.symlink(os.readlink(src), dst) + elif os.path.isdir(src): + if not os.path.exists(dst): + os.makedirs(dst) + for fname in os.listdir(src): + recursive_copy(os.path.join(src, fname), + os.path.join(dst, fname)) + else: + shutil.copy2(src, dst) + except (IOError, OSError) as err: + raise GbpError("Error while copying '%s' to '%s': %s" % (src, dst, err)) + +def guess_upstream_source(pkg_data, remotes): + """Guess the primary upstream source archive.""" + orig = None + name = pkg_data.getVar('PN', True) + + for fetch_data in remotes: + if fetch_data.type == 'git': + orig = fetch_data + else: + path = fetch_data.localpath + fname = os.path.basename(path) + fn_base, archive_fmt, _ = parse_archive_filename(fname) + if fn_base.startswith(name) and archive_fmt: + # Take an archive that starts with pkg name + orig = fetch_data + # otherwise we take the first archive + elif not orig and archive_fmt: + orig = fetch_data + # else don't accept + return orig + +def bb_get_files(pkg_data, tgt_dir, whole_dir=False, download=True): + """Get (local) packaging files""" + uris = (pkg_data.getVar('SRC_URI', True) or "").split() + try: + fetch = bb.fetch2.Fetch(uris, pkg_data) + if download: + gbp.log.info("Fetching sources...") + fetch.download() + except bb.fetch2.BBFetchException as err: + raise GbpError("Failed to fetch packaging files: %s" % err) + + # Copy local files to target directory + bb_dir = os.path.dirname(pkg_data.getVar('FILE', True)) + remote = [] + local = [path for path in pkg_data.getVar('BBINCLUDED', True).split() if + path.startswith(bb_dir) and os.path.exists(path)] + for url in fetch.urls: + path = fetch.localpath(url) + if path.startswith(bb_dir): + if not whole_dir: + gbp.log.debug("Found local meta file '%s'" % path) + local.append(path) + else: + gbp.log.debug("Found remote file '%s'" % path) + remote.append(fetch.ud[url]) + + if whole_dir: + # Simply copy whole meta dir, if requested + recursive_copy(bb_dir, tgt_dir) + else: + for path in local: + relpath = os.path.relpath(path, bb_dir) + subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) + if not os.path.exists(subdir): + os.makedirs(subdir) + shutil.copy2(path, os.path.join(tgt_dir, relpath)) + + return remote + +def import_upstream_archive(repo, pkg_data, fetch_data, dirs, options): + """Import upstream sources from archive""" + # Unpack orig source archive + path = fetch_data.localpath + sources = RpmUpstreamSource(path) + sources = sources.unpack(dirs['origsrc'], options.filters) + + tag_str_fields = dict(pkg_version(pkg_data), vendor=options.vendor.lower()) + tag = repo.version_to_tag(options.upstream_tag, tag_str_fields) + if not repo.has_tag(tag): + gbp.log.info("Tag %s not found, importing upstream sources" % tag) + branch = options.upstream_branch + + msg = "Upstream version %s" % tag_str_fields['upstreamversion'] + if options.vcs_tag: + parents = [repo.rev_parse("%s^{}" % options.vcs_tag)] + else: + parents = None + commit = repo.commit_dir(sources.unpacked, "Imported %s" % msg, + branch, other_parents=parents, + create_missing_branch=options.create_missing_branches) + repo.create_tag(name=tag, msg=msg, commit=commit, + sign=options.sign_tags, keyid=options.keyid) + + if options.pristine_tar: + archive_fmt = parse_archive_filename(path)[1] + if archive_fmt == 'tar': + repo.pristine_tar.commit(path, 'refs/heads/%s' % branch) + else: + gbp.log.warn('Ignoring pristine-tar, %s archives ' + 'not supported' % archive_fmt) + return repo.rev_parse('%s^0' % tag) + +def import_upstream_git(repo, fetch_data, options): + """Import upstream sources from Git""" + # Fetch from local cached repo + for branch in fetch_data.branches.values(): + repo.fetch(repo=fetch_data.localpath, refspec=branch) + + commit = fetch_data.revision + repo.update_ref('refs/heads/' + options.upstream_branch, commit) + return commit + +def import_upstream_sources(repo, pkg_data, remotes, dirs, options): + """Import upstream sources to Git""" + fetch_data = guess_upstream_source(pkg_data, remotes) + if fetch_data: + gbp.log.debug("Using upstream source '%s'" % fetch_data.url) + else: + gbp.log.info("No orig source archive imported") + return + + if not repo.has_branch(options.upstream_branch): + if options.create_missing_branches: + gbp.log.info("Will create missing branch '%s'" % + options.upstream_branch) + else: + gbp.log.err(no_upstream_branch_msg % options.upstream_branch + "\n" + "Also check the --create-missing-branches option.") + raise GbpError + + if fetch_data.type == 'git': + return import_upstream_git(repo, fetch_data, options) + else: + return import_upstream_archive(repo, pkg_data, fetch_data, dirs, + options) + + +def main(argv): + """Main function of the gbp import-bb script""" + dirs = dict(top=os.path.abspath(os.curdir)) + ret = 0 + skipped = False + + if not bb: + return 1 + + options, args = parse_args(argv) + + if len(args) == 0 or len(args) > 2: + gbp.log.err("Need to give exactly one package to import. Try --help.") + return 1 + + try: + dirs['tmp_base'] = tempfile.mkdtemp(dir=options.tmp_dir, + prefix='import-bb') + tinfoil = init_tinfoil() + pkg_bb = guess_pkg(tinfoil, args[0]) + dirs['src'] = os.path.abspath(os.path.dirname(pkg_bb)) + gbp.log.info("Importing '%s' from '%s'" % + (os.path.basename(pkg_bb), dirs['src'])) + + pkg_data = bb.cache.Cache.loadDataFull(pkg_bb, [], tinfoil.config_data) + + # Determine target repo dir + target_dir = '' + if len(args) == 2: + target_dir = args[1] + else: + if 'BUILDDIR' in os.environ: + target_dir = os.path.join(os.environ['BUILDDIR'], 'devel') + target_dir = os.path.join(target_dir, pkg_data.getVar('PN', True)) + + # Check the Git repository state + repo = init_repo(target_dir) + if repo.bare: + set_bare_repo_options(options) + if repo.is_empty(): + options.create_missing_branches = True + os.chdir(repo.path) + + # Create more tempdirs + dirs['origsrc'] = tempfile.mkdtemp(dir=dirs['tmp_base'], + prefix='origsrc_') + dirs['packaging_base'] = tempfile.mkdtemp(dir=dirs['tmp_base'], + prefix='packaging_') + dirs['packaging'] = os.path.join(dirs['packaging_base'], + options.meta_dir) + + # Copy (local) packaging files to tmp dir + remote_srcs = bb_get_files(pkg_data, dirs['packaging']) + + version_dict = pkg_version(pkg_data) + tag_str_fields = dict(version_dict, vendor=options.vendor.lower()) + ver_str = RpmPkgPolicy.compose_full_version(version_dict) + + # Check if the same version of the package is already imported + if repo.find_version(options.packaging_tag, tag_str_fields): + gbp.log.warn("Version %s already imported." % ver_str) + if options.allow_same_version: + gbp.log.info("Moving tag of version '%s' since import forced" % + ver_str) + move_tag_stamp(repo, options.packaging_tag, tag_str_fields) + else: + raise SkipImport + + # Import upstream sources + import_upstream_sources(repo, pkg_data, remote_srcs, dirs, options) + + # Import packaging files + gbp.log.info("Importing local meta/packaging files") + branch = options.packaging_branch + if not repo.has_branch(branch): + if options.create_missing_branches: + gbp.log.info("Will create missing branch '%s'" % branch) + else: + gbp.log.err(NO_PACKAGING_BRANCH_MSG % branch + "\n" + "Also check the --create-missing-branches " + "option.") + raise GbpError + + tag = repo.version_to_tag(options.packaging_tag, tag_str_fields) + msg = "%s release %s" % (options.vendor, ver_str) + + commit = repo.commit_dir(dirs['packaging_base'], + "Imported %s" % msg, + branch, + create_missing_branch=options.create_missing_branches) + + # Create packaging tag + repo.create_tag(name=tag, + msg=msg, + commit=commit, + sign=options.sign_tags, + keyid=options.keyid) + + force_to_branch_head(repo, options.packaging_branch) + + except KeyboardInterrupt: + ret = 1 + gbp.log.err("Interrupted. Aborting.") + except gbpc.CommandExecFailed: + ret = 1 + except GitRepositoryError as err: + gbp.log.err("Git command failed: %s" % err) + ret = 1 + except GbpError as err: + if len(err.__str__()): + gbp.log.err(err) + ret = 1 + except SkipImport: + skipped = True + finally: + os.chdir(dirs['top']) + gbpc.RemoveTree(dirs['tmp_base'])() + + if not ret and not skipped: + gbp.log.info("Version '%s' imported under '%s'" % + (ver_str, repo.path)) + return ret + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py index 9d183b6d..56b1d9dc 100644 --- a/gbp/scripts/import_orig.py +++ b/gbp/scripts/import_orig.py @@ -270,12 +270,6 @@ def main(argv): source, pkg_name, version, prepare_pristine, options.filters, options.filter_pristine_tar, None, tmpdir) - # Prepare sources for importing - pristine_name = pristine_tarball_name(source, pkg_name, version) - prepare_pristine = pristine_name if options.pristine_tar else None - unpacked_orig, pristine_orig = prepare_sources( - source, pkg_name, version, prepare_pristine, options.filters, - options.filter_pristine_tar, None, tmpdir) # Don't mess up our repo with git metadata from an upstream tarball try: if os.path.isdir(os.path.join(unpacked_orig, '.git/')): diff --git a/gbp/scripts/pq_bb.py b/gbp/scripts/pq_bb.py new file mode 100755 index 00000000..c8247bc8 --- /dev/null +++ b/gbp/scripts/pq_bb.py @@ -0,0 +1,427 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2011 Guido Günther <agx@sigxcpu.org> +# (C) 2012 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +"""manage patches in a patch queue""" + +import ConfigParser +import errno +import os +import shutil +import sys + +import gbp.tmpfile as tempfile +from gbp.config import GbpOptionParserBB +from gbp.rpm.git import GitRepositoryError, RpmGitRepository +from gbp.command_wrappers import GitCommand, CommandExecFailed +from gbp.errors import GbpError +import gbp.log +from gbp.patch_series import PatchSeries, Patch +from gbp.rpm import string_to_int +from gbp.scripts.common.pq import (is_pq_branch, pq_branch_name, pq_branch_base, + apply_and_commit_patch, drop_pq) +from gbp.scripts.pq_rpm import (generate_patches, safe_patches, + import_extra_files) +from gbp.bb import bb, init_tinfoil, parse_bb, pkg_version + +# pylint: disable=bad-continuation + +USAGE_STRING = \ +"""%prog [options] action - maintain patches on a patch queue branch +tions: +export Export the patch queue / devel branch associated to the + current branch into a patch series in and update the recipe file +import Create a patch queue / devel branch from recipe file + and patches in current dir. +rebase Switch to patch queue / devel branch associated to the current + branch and rebase against upstream. +drop Drop (delete) the patch queue /devel branch associated to + the current branch. +apply Apply a patch +switch Switch to patch-queue branch and vice versa.""" + + +def rm_patch_files(bbfile): + """Delete the patch files listed in the pkg meta data.""" + unlinked = set() + + # Go through local files + for path in bbfile.localfiles: + if path.endswith('.patch'): + gbp.log.debug("Removing patch '%s'" % path) + unlinked.add(os.path.basename(path)) + try: + os.unlink(path) + except OSError as err: + if err.errno != errno.ENOENT: + raise GbpError("Failed to remove patch: %s" % err) + else: + gbp.log.debug("Patch %s does not exist." % path) + else: + gbp.log.debug("Unlink skipping non-local/non-patch file %s" % path) + uris = (bbfile.getVar('SRC_URI', False) or "").split() + return [uri for uri in uris if os.path.basename(uri) not in unlinked] + + +def update_patch_series(repo, bbfile, start, end, options): + """Export patches to packaging directory and update recipe file""" + squash = options.patch_export_squash_until.split(':', 1) + if len(squash) == 1: + squash.append(None) + else: + squash[1] += '.diff' + + # Unlink old (local) patch files and generate new patches + rm_patch_files(bbfile) + + # Guess patch subdir + bb_dir = os.path.dirname(bbfile.getVar('FILE', True)) + pkg_name = bbfile.getVar('PN', True) + pkg_ver = bbfile.getVar('PV', True) + subdir = pkg_name + '-' + pkg_ver + if not os.path.isdir(os.path.join(bb_dir, subdir)): + if os.path.isdir(os.path.join(bb_dir, pkg_name)): + subdir = pkg_name + elif os.path.isdir(os.path.join(bb_dir, 'files')): + subdir = 'files' + tgt_dir = os.path.join(bb_dir, subdir) + + patches, _commands = generate_patches(repo, start, squash, end, + tgt_dir, options) + # TODO: implement commands processing (e.g. topic) + new_uris = ['file://' + patch for patch in patches] + bbfile.substitute_var_val(bbfile.bb_path, 'SRC_URI', r'file://\S+.\.patch', + '') + bbfile.append_var_val(bbfile.bb_path, 'SRC_URI', new_uris) + return patches + +def var_to_str(var, value): + """Create a well formatted string buffer for a variable assignment""" + indent = ' ' * (len(var) + 3) + linebuf = ['%s = "%s \\\n' % (var, value[0])] + for val in value[1:]: + linebuf.append(indent + ' ' + val + '\\\n') + linebuf.append(indent + '"\n') + return linebuf + + +def find_upstream_commit(repo, bbfile, upstream_tag): + """Find commit corresponding upstream version""" + src_rev = bbfile.getVar('SRCREV', True) + if src_rev and src_rev != 'INVALID': + return bbfile.getVar('SRCREV', True) + + # Find tag + upstreamversion = bbfile.getVar('PV', True) + tag_str_fields = {'upstreamversion': upstreamversion, + 'vendor': 'Upstream'} + upstream_commit = repo.find_version(upstream_tag, tag_str_fields) + if not upstream_commit: + raise GbpError("Couldn't find upstream version %s" % upstreamversion) + return upstream_commit + + +def export_patches(cfg, repo, options): + """Export patches from the pq branch into a packaging branch""" + current = repo.get_branch() + if is_pq_branch(current, options): + base = pq_branch_base(current, options) + gbp.log.info("On branch '%s', switching to '%s'" % (current, base)) + repo.set_branch(base) + bbfile = parse_bb(cfg, options, repo) + pq_branch = current + else: + bbfile = parse_bb(cfg, options, repo) + pq_branch = pq_branch_name(current, options, pkg_version(bbfile)) + upstream_commit = find_upstream_commit(repo, bbfile, options.upstream_tag) + + export_treeish = options.export_rev if options.export_rev else pq_branch + + update_patch_series(repo, bbfile, upstream_commit, export_treeish, options) + + bb_dir = os.path.dirname(bbfile.getVar('FILE', True)) + GitCommand('status')(['--', bb_dir]) + + +def bb_to_patch_series(bbfile): + """Get all local patches as a series""" + series = PatchSeries() + for path in bbfile.localfiles: + if path.endswith('.patch'): + series.append(Patch(path)) + return series + + +def import_bb_patches(cfg, repo, options): + """Apply a series of patches in a recipe to branch onto a pq branch""" + current = repo.get_branch() + + if is_pq_branch(current, options): + base = pq_branch_base(current, options) + raise GbpError("Already on a patch-queue branch '%s' - doing " + "nothing." % current) + else: + bbfile = parse_bb(cfg, options, repo) + base = current + upstream_commit = find_upstream_commit(repo, bbfile, options.upstream_tag) + pq_branch = pq_branch_name(base, options, pkg_version(bbfile)) + + # Create pq-branch + if repo.has_branch(pq_branch) and not options.force: + raise GbpError("Patch-queue branch '%s' already exists. " + "Try 'rebase' instead." % pq_branch) + try: + if repo.get_branch() == pq_branch: + repo.force_head(upstream_commit, hard=True) + else: + repo.create_branch(pq_branch, upstream_commit, force=True) + except GitRepositoryError as err: + raise GbpError("Cannot create patch-queue branch '%s': %s" % + (pq_branch, err)) + + # Put patches in a safe place + in_queue = bb_to_patch_series(bbfile) + queue = safe_patches(in_queue, options.tmp_dir) + # Do import + try: + gbp.log.info("Switching to branch '%s'" % pq_branch) + repo.set_branch(pq_branch) + import_extra_files(repo, base, options.import_files) + + if not queue: + return + gbp.log.info("Trying to apply patches from branch '%s' onto '%s'" % + (base, upstream_commit)) + for patch in queue: + gbp.log.debug("Applying %s" % patch.path) + apply_and_commit_patch(repo, patch, fallback_author=None) + except (GbpError, GitRepositoryError) as err: + gbp.log.err('Import failed: %s' % err) + repo.force_head('HEAD', hard=True) + repo.set_branch(base) + repo.delete_branch(pq_branch) + raise + + recipe_fn = os.path.basename(bbfile.getVar('FILE', True)) + gbp.log.info("Patches listed in '%s' imported on '%s'" % (recipe_fn, + pq_branch)) + + +def rebase_pq(cfg, repo, options): + """Rebase pq branch on the correct upstream version""" + current = repo.get_branch() + if is_pq_branch(current, options): + base = pq_branch_base(current, options) + bbfile = parse_bb(cfg, options, repo, base) + else: + base = current + bbfile = parse_bb(cfg, options, repo) + upstream_commit = find_upstream_commit(repo, bbfile, options.upstream_tag) + + switch_to_pq_branch(cfg, repo, base, options) + GitCommand("rebase")([upstream_commit]) + + +def switch_pq(cfg, repo, options): + """Switch to patch-queue branch if on base branch and vice versa""" + current = repo.get_branch() + if is_pq_branch(current, options): + base = pq_branch_base(current, options) + gbp.log.info("Switching to branch '%s'" % base) + repo.checkout(base) + else: + switch_to_pq_branch(cfg, repo, current, options) + + +def drop_pq_bb(cfg, repo, options): + """Remove pq branch""" + current = repo.get_branch() + if is_pq_branch(current, options): + base = pq_branch_base(current, options) + bbfile = parse_bb(cfg, options, repo, base) + else: + bbfile = parse_bb(cfg, options, repo) + drop_pq(repo, current, options, pkg_version(bbfile)) + + +def switch_to_pq_branch(cfg, repo, branch, options): + """ + Switch to patch-queue branch if not already there, create it if it + doesn't exist yet + """ + if is_pq_branch(branch, options): + return + + bbfile = parse_bb(cfg, options, repo, branch) + pq_branch = pq_branch_name(branch, options, pkg_version(bbfile)) + if not repo.has_branch(pq_branch): + raise GbpError("Branch '%s' does not exist" % pq_branch) + + gbp.log.info("Switching to branch '%s'" % pq_branch) + repo.set_branch(pq_branch) + +def apply_single_patch(cfg, repo, patchfile, options): + """Apply a single patch onto the pq branch""" + current = repo.get_branch() + if not is_pq_branch(current, options): + switch_to_pq_branch(cfg, repo, current, options) + patch = Patch(patchfile) + apply_and_commit_patch(repo, patch, fallback_author=None) + +def opt_split_cb(option, opt_str, value, parser): + """Split option string into a list""" + setattr(parser.values, option.dest, value.split(',')) + +def build_parser(name): + """Create command line argument parser""" + try: + parser = GbpOptionParserBB(command=os.path.basename(name), + prefix='', usage=USAGE_STRING) + except ConfigParser.ParsingError as err: + gbp.log.err(err) + return None + + parser.add_boolean_config_file_option(option_name="patch-numbers", + dest="patch_numbers") + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + default=False, help="Verbose command execution") + parser.add_option("--force", dest="force", action="store_true", + default=False, + help="In case of import even import if the branch already exists") + parser.add_config_file_option(option_name="vendor", action="store", + dest="vendor") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_config_file_option(option_name="tmp-dir", dest="tmp_dir") + parser.add_config_file_option(option_name="upstream-tag", + dest="upstream_tag") + parser.add_config_file_option(option_name="bb-file", dest="bb_file") + parser.add_config_file_option(option_name="meta-dir", + dest="meta_dir") + parser.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch", + help="Branch the packaging is being maintained on. Only relevant " + "if a invariable/single pq-branch is defined, in which case " + "this is used as the 'base' branch. Default is " + "'%(packaging-branch)s'") + parser.add_config_file_option(option_name="pq-branch", dest="pq_branch") + parser.add_config_file_option(option_name="import-files", + dest="import_files", type="string", action="callback", + callback=opt_split_cb) + parser.add_option("--export-rev", action="store", dest="export_rev", + default="", + help="Export patches from treeish object TREEISH instead of head " + "of patch-queue branch", metavar="TREEISH") + parser.add_config_file_option("patch-export-compress", + dest="patch_export_compress") + parser.add_config_file_option("patch-export-squash-until", + dest="patch_export_squash_until") + parser.add_config_file_option("patch-export-ignore-path", + dest="patch_export_ignore_path") + return parser + +def parse_args(argv): + """Parse command line arguments""" + parser = build_parser(argv[0]) + if not parser: + return None, None + + options, args = parser.parse_args(argv) + gbp.log.setup(options.color, options.verbose, options.color_scheme) + options.patch_export_compress = string_to_int(options.patch_export_compress) + + return options, args + + +def main(argv): + """Main function for the gbp pq-rpm command""" + retval = 0 + + if not bb: + return 1 + + options, args = parse_args(argv) + if not options: + return 1 + + if len(args) < 2: + gbp.log.err("No action given.") + return 1 + else: + action = args[1] + + if args[1] in ["export", "import", "rebase", "drop", "switch"]: + pass + elif args[1] in ["apply"]: + if len(args) != 3: + gbp.log.err("No patch name given.") + return 1 + else: + patchfile = args[2] + else: + gbp.log.err("Unknown action '%s'." % args[1]) + return 1 + + try: + repo = RpmGitRepository(os.path.curdir) + except GitRepositoryError: + gbp.log.err("%s is not a git repository" % (os.path.abspath('.'))) + return 1 + + if os.path.abspath('.') != repo.path: + gbp.log.warn("Switching to topdir before running commands") + os.chdir(repo.path) + + try: + # Initialize BitBake + tinfoil = init_tinfoil(config_only=True, tracking=True) + bb_cfg_data = bb.data.createCopy(tinfoil.config_data) + + # Create base temporary directory for this run + options.tmp_dir = tempfile.mkdtemp(dir=options.tmp_dir, + prefix='gbp-pq-bb_') + if action == "export": + export_patches(bb_cfg_data, repo, options) + elif action == "import": + import_bb_patches(bb_cfg_data, repo, options) + elif action == "drop": + drop_pq_bb(bb_cfg_data, repo, options) + elif action == "rebase": + rebase_pq(bb_cfg_data, repo, options) + elif action == "apply": + apply_single_patch(bb_cfg_data, repo, patchfile, options) + elif action == "switch": + switch_pq(bb_cfg_data, repo, options) + except CommandExecFailed: + retval = 1 + except GitRepositoryError as err: + gbp.log.err("Git command failed: %s" % err) + retval = 1 + except GbpError, err: + if len(err.__str__()): + gbp.log.err(err) + retval = 1 + finally: + shutil.rmtree(options.tmp_dir, ignore_errors=True) + + return retval + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + diff --git a/gbp/scripts/pq_rpm.py b/gbp/scripts/pq_rpm.py index e19218a9..32ffadee 100755 --- a/gbp/scripts/pq_rpm.py +++ b/gbp/scripts/pq_rpm.py @@ -26,6 +26,7 @@ import sys import re import gzip import bz2 +import tarfile import subprocess import gbp.tmpfile as tempfile from gbp.config import GbpOptionParserRpm @@ -300,6 +301,9 @@ def safe_patches(queue, tmpdir_base): gbp.log.debug("Uncompressing '%s'" % os.path.basename(patch.path)) src = uncompressors[comp](patch.path, 'r') dst_name = os.path.join(tmpdir, os.path.basename(base)) + if _archive_fmt: + tar_name = dst_name + dst_name += '.tar' elif comp: raise GbpError("Unsupported patch compression '%s', giving up" % comp) @@ -311,6 +315,11 @@ def safe_patches(queue, tmpdir_base): dst.writelines(src) src.close() dst.close() + if _archive_fmt: + t = tarfile.open(dst_name, 'r:') + t.extractall(path = tmpdir) + t.close() + dst_name = tar_name safequeue.append(patch) safequeue[-1].path = dst_name diff --git a/gbp/scripts/submit_bb.py b/gbp/scripts/submit_bb.py new file mode 100755 index 00000000..7edece5d --- /dev/null +++ b/gbp/scripts/submit_bb.py @@ -0,0 +1,138 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2014 Intel Corporation <markus.lehtonen@linux.intel.com> +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +"""Create and push submit tag""" + +import ConfigParser +import os +import sys +from datetime import datetime + +import gbp.log +from gbp.config import GbpOptionParserBB +from gbp.errors import GbpError +from gbp.format import format_msg +from gbp.git import GitRepository, GitRepositoryError + +# pylint: disable=bad-continuation + + +def guess_remote(repo, options): + """Guess remote where to push""" + if options.remote: + return options.remote + + remotes = repo.get_remotes() + if not remotes: + raise GbpError("Local repo has no remotes configured. Please add one " + "or use --remote to define the remote where to push.") + elif len(remotes) == 1: + return remotes.keys()[0] + else: + raise GbpError("Local repo has multiple remotes (%s). Don't know which " + "one to choose. Use --remote to define where to push." % + ', '.join(remotes.keys())) + + +def build_parser(name): + """Build command line parser""" + usage_str = "%prog [options] - create and push submit tag" + try: + parser = GbpOptionParserBB(command=os.path.basename(name), prefix='', + usage=usage_str) + except ConfigParser.ParsingError as err: + gbp.log.err(err) + return None + + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + help="verbose command execution") + parser.add_config_file_option(option_name="color", dest="color", + type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_option("-m", "--message", dest="message", help="tag message") + parser.add_option("-c", "--commit", dest="commit", help="commit to submit", + default='HEAD') + parser.add_option("-r", "--remote", dest="remote", + help="remote where to push") + parser.add_config_file_option(option_name="submit-tag", dest="submit_tag") + parser.add_config_file_option(option_name="target", dest="target") + parser.add_boolean_config_file_option(option_name="sign-tags", + dest="sign_tags") + parser.add_config_file_option(option_name="keyid", dest="keyid") + + return parser + + +def parse_args(argv): + """Parse command line arguments""" + parser = build_parser(argv[0]) + if not parser: + return None, None + options, args = parser.parse_args(argv) + + gbp.log.setup(options.color, options.verbose, options.color_scheme) + + return (options, args) + + +def main(argv): + """Entry point for gbp-submit-bb""" + retval = 0 + + options, _args = parse_args(argv) + if not options: + return 1 + + try: + repo = GitRepository(os.path.curdir) + except GitRepositoryError: + gbp.log.err("The command must be run under a Git repository") + return 1 + + try: + remote = guess_remote(repo, options) + + tag_fields = {'nowtime': datetime.now().strftime('%Y%m%d.%H%M%S'), + 'target': options.target} + tag_name = format_msg(options.submit_tag, tag_fields) + + gbp.log.info("Tagging %s" % tag_name) + repo.create_tag(tag_name, msg=options.message, commit=options.commit, + sign=options.sign_tags, keyid=options.keyid, + annotate=True) + + gbp.log.info("Pushing to remote %s" % remote) + try: + repo.push_tag(remote, tag_name) + except GitRepositoryError as err: + gbp.log.err(err) + gbp.log.info("Removing tag %s" % tag_name) + repo.delete_tag(tag_name) + raise GbpError("Git push failed!") + + except (GbpError, GitRepositoryError) as err: + if len(err.__str__()): + gbp.log.err(err) + retval = 1 + + return retval + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + |