From 9bff3ddfdc256d760f64497baa11196f919d9b62 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 28 Jun 2012 11:01:25 +0300 Subject: rpm: bigger refactor of import-srpm Import source rpms in the same way as unpacked sources: that is, first unpack the srpm and then use the same code for importing. Also, tries to make tmpdir handling a bit better: all temporary directories are created under one "base tmpdir". Also, get rid of the orig tarball guessing and unpackging in SrcRpmFile class in gbp.rpm. Signed-off-by: Markus Lehtonen --- gbp/rpm/__init__.py | 41 +---- gbp/scripts/import_srpm.py | 392 +++++++++++++++++++++------------------------ 2 files changed, 190 insertions(+), 243 deletions(-) (limited to 'gbp') diff --git a/gbp/rpm/__init__.py b/gbp/rpm/__init__.py index 9dbb1e6c..ddf29961 100644 --- a/gbp/rpm/__init__.py +++ b/gbp/rpm/__init__.py @@ -72,7 +72,6 @@ class SrcRpmFile(object): self.rpmhdr = rpm.ts(vsflags=ts_vsflags).hdrFromFdno(srpmfp.fileno()) srpmfp.close() self.srpmfile = os.path.abspath(srpmfile) - (self.orig_file, self.orig_base, self.orig_archive_fmt, self.orig_comp) = self.guess_orig_file() def _get_version(self): """ @@ -106,49 +105,15 @@ class SrcRpmFile(object): return self.rpmhdr[rpm.RPMTAG_PACKAGER] packager = property(_get_packager) - def unpack(self, dest_dir, srctarballdir=None): + def unpack(self, dest_dir): """ - Unpack the source rpm to tmpdir, move source tarball to srctallbardir. - Leave the cleanup to the caller in case of an error + Unpack the source rpm to tmpdir. + Leave the cleanup to the caller in case of an error. """ gbpc.RunAtCommand('rpm2cpio', [self.srpmfile, '|', 'cpio', '-id'], shell=True)(dir=dest_dir) - # Unpack source tarball - if self.orig_file: - orig_tarball = os.path.join(dest_dir, self.orig_file) - if srctarballdir: - if os.path.isdir(srctarballdir): - shutil.move(orig_tarball, srctarballdir) - else: - raise GbpError, "Src tarball destination dir not found or not a directory" - else: - gbp.log.warn("Failed to detect source tarball. Import may be incorrect") - #raise GbpError, "Failed to detect source tarball" - - def guess_orig_file(self): - """ - Try to guess the name of the primary upstream/source archive - returns a tuple with full file path, filename base, archive format and - compression method. - """ - full_path, base, archive_fmt, comp = None, None, None, None - - for _full_path in self.rpmhdr[rpm.RPMTAG_SOURCE]: - filename = os.path.basename(_full_path) - _base, _archive_fmt, _comp = parse_archive_filename(filename) - if RpmPkgPolicy.is_valid_orig_archive(filename): - if filename.startswith(self.name): - # Take the first archive that starts with pkg name - full_path, base, archive_fmt, comp = _full_path, _base, _archive_fmt, _comp - break - # otherwise we take the first archive - elif not full_path: - full_path, base, archive_fmt, comp = _full_path, _base, _archive_fmt, _comp - # else don't accept - return (full_path, base, archive_fmt, comp) - class SpecFile(object): """Class for parsing/modifying spec files""" diff --git a/gbp/scripts/import_srpm.py b/gbp/scripts/import_srpm.py index 498b3d34..fb5babf2 100755 --- a/gbp/scripts/import_srpm.py +++ b/gbp/scripts/import_srpm.py @@ -55,7 +55,7 @@ def download_source(pkg, dirs): else: mode = 'yumdownloader' - dirs['download'] = os.path.abspath(tempfile.mkdtemp()) + dirs['download'] = tempfile.mkdtemp(prefix='download', dir=dirs['tmp_base']) gbp.log.info("Downloading '%s' using '%s'..." % (pkg, mode)) if mode == 'yumdownloader': gbpc.RunAtCommand('yumdownloader', @@ -164,229 +164,214 @@ def parse_args(argv): def main(argv): dirs = dict(top=os.path.abspath(os.curdir)) - needs_repo = False + ret = 0 skipped = False parents = None options, args = parse_args(argv) + if len(args) != 1: + gbp.log.err("Need to give exactly one package to import. Try --help.") + return 1 + try: + dirs['tmp_base'] = os.path.abspath(tempfile.mkdtemp()) + except GbpError as err: + gbp.log.err(err) + return 1 try: - if len(args) != 1: - gbp.log.err("Need to give exactly one package to import. Try --help.") - raise GbpError + srpm = args[0] + if options.download: + srpm = download_source(srpm, dirs) + + # Real srpm, we need to unpack, first + if not os.path.isdir(srpm) and not srpm.endswith(".spec"): + src = parse_srpm(srpm) + dirs['pkgextract'] = tempfile.mkdtemp(prefix='pkgextract', dir=dirs['tmp_base']) + gbp.log.info("Extracting src rpm to '%s'" % dirs['pkgextract']) + src.unpack(dirs['pkgextract']) + srpm = dirs['pkgextract'] + + # Find and parse spec file + if os.path.isdir(srpm): + gbp.log.debug("Trying to import an unpacked srpm from '%s'" % srpm) + dirs['src'] = os.path.abspath(srpm) + spec = parse_spec(guess_spec(srpm, True)) else: - pkg = args[0] - if options.download: - srpm = download_source(pkg, dirs=dirs) - else: - srpm = pkg + gbp.log.debug("Trying to import an srpm from '%s' with spec file '%s'" % (os.path.dirname(srpm), srpm)) + dirs['src'] = os.path.abspath(os.path.dirname(srpm)) + spec = parse_spec(srpm) - if os.path.isdir(pkg) or pkg.endswith(".spec"): - if os.path.isdir(pkg): - gbp.log.debug("Trying to import an unpacked srpm from '%s'" % pkg) - dirs['src'] = os.path.abspath(pkg) - spec = parse_spec(guess_spec(pkg, True)) - else: - gbp.log.debug("Trying to import an srpm from '%s' with spec file '%s'" % (os.path.dirname(pkg), pkg)) - dirs['src'] = os.path.abspath(os.path.dirname(pkg)) - spec = parse_spec(pkg) - - pkgname = spec.name - pkgver = spec.version - upstream_version = spec.upstreamversion - packager = spec.packager - unpacked = True - else: - gbp.log.debug("Trying to import an source rpm '%s'" % srpm) - dirs['src'] = os.path.abspath(os.path.dirname(pkg)) - src = parse_srpm(srpm) - pkgname = src.name - pkgver = src.version - upstream_version = src.upstreamversion - packager = src.packager - unpacked = False - - try: - repo = RpmGitRepository('.') - is_empty = repo.is_empty() - - (clean, out) = repo.is_clean() - if not clean and not is_empty: - gbp.log.err("Repository has uncommitted changes, commit these first: ") - raise GbpError, out - - except GitRepositoryError: - # no repo found, create one - needs_repo = True - is_empty = True - - if needs_repo: - gbp.log.info("No git repository found, creating one.") - repo = RpmGitRepository.create(pkgname) - os.chdir(repo.path) - - if repo.bare: - set_bare_repo_options(options) - - dirs['pkgextract'] = os.path.abspath(tempfile.mkdtemp(dir='..')) - dirs['pkgextract-packaging'] = os.path.join(dirs['pkgextract'], options.packaging_dir) - try: - os.mkdir(dirs['pkgextract-packaging']) - except OSError, (e, emsg): - if e == errno.EEXIST: - pass - dirs['srctarball'] = os.path.abspath(tempfile.mkdtemp(dir='..')) - dirs['srcunpack'] = os.path.abspath(tempfile.mkdtemp(dir='..')) - - orig_tarball = None - if unpacked: - files = [os.path.basename(patch['filename']) for patch in spec.patches.itervalues()] - for num, src in spec.sources.iteritems(): - if num != spec.orig_src_num: - files.append(src['filename']) - files.append(spec.specfile) - for fname in files: - fpath = os.path.join(dirs['src'], fname) - if os.path.exists(fpath): - shutil.copy2(fpath, dirs['pkgextract-packaging']) - else: - gbp.log.err("File '%s' listed in spec not found" % fname) - raise GbpError - if spec.orig_src: - orig_tarball=os.path.join(dirs['src'], spec.orig_src['filename']) - else: - gbp.log.info("Extracting src rpm...") - src.unpack(dirs['pkgextract-packaging'], dirs['srctarball']) - if src.orig_file: - orig_tarball = os.path.join(dirs['srctarball'], src.orig_file) - - if orig_tarball: - upstream = RpmUpstreamSource(orig_tarball) - upstream.unpack(dirs['srcunpack'], options.filters) + # Check the repository state + try: + repo = RpmGitRepository('.') + is_empty = repo.is_empty() + + (clean, out) = repo.is_clean() + if not clean and not is_empty: + gbp.log.err("Repository has uncommitted changes, commit these first: ") + raise GbpError, out + + except GitRepositoryError: + gbp.log.info("No git repository found, creating one.") + is_empty = True + repo = RpmGitRepository.create(spec.name) + os.chdir(repo.path) + + if repo.bare: + set_bare_repo_options(options) + + # Create more tempdirs + dirs['origsrc'] = tempfile.mkdtemp(prefix='origsrc', dir=dirs['tmp_base']) + dirs['packaging_base'] = tempfile.mkdtemp(prefix='packaging', dir=dirs['tmp_base']) + dirs['packaging'] = os.path.join(dirs['packaging_base'], options.packaging_dir) + try: + os.mkdir(dirs['packaging']) + except OSError, (e, emsg): + if e != errno.EEXIST: + raise + + # Need to copy files to the packaging directory given by caller + files = [os.path.basename(patch['filename']) for patch in spec.patches.itervalues()] + for num, src in spec.sources.iteritems(): + if num != spec.orig_src_num: + files.append(src['filename']) + files.append(spec.specfile) + for fname in files: + fpath = os.path.join(dirs['src'], fname) + if os.path.exists(fpath): + shutil.copy2(fpath, dirs['packaging']) else: - upstream = None + gbp.log.err("File '%s' listed in spec not found" % fname) + raise GbpError + + # Unpack orig source archive + if spec.orig_src: + orig_tarball=os.path.join(dirs['src'], spec.orig_src['filename']) + upstream = RpmUpstreamSource(orig_tarball) + upstream.unpack(dirs['origsrc'], options.filters) + else: + upstream = None - format = [(options.upstream_tag, "Upstream"), (options.packaging_tag, options.vendor)][options.native] - tag_str_fields = dict(pkgver, vendor=options.vendor) - tag = repo.version_to_tag(format[0], tag_str_fields) + format = [(options.upstream_tag, "Upstream"), (options.packaging_tag, options.vendor)][options.native] + tag_str_fields = dict(spec.version, vendor=options.vendor) + tag = repo.version_to_tag(format[0], tag_str_fields) - if repo.find_version(options.packaging_tag, tag_str_fields): - gbp.log.warn("Version %s already imported." % RpmPkgPolicy.compose_full_version(pkgver)) - if options.allow_same_version: - gbp.log.info("Moving tag of version '%s' since import forced" % RpmPkgPolicy.compose_full_version(pkgver)) - move_tag_stamp(repo, options.packaging_tag, tag_str_fields) - else: - raise SkipImport - - if is_empty: - options.create_missing_branches = True - - # Determine author and committer info, currently same info is used - # for both upstream sources and packaging files - author=None - if packager: - match = re.match('(?P.*[^ ])\s*<(?P\S*)>', packager.strip()) - if match: - author=GitModifier(match.group('name'), match.group('email')) - if not author: - author=GitModifier() - gbp.log.debug("Couldn't determine packager info") - committer = committer_from_author(author, options) - - # Import upstream sources - if upstream: - upstream_commit = repo.find_version(format[0], tag_str_fields) - if not upstream_commit: - gbp.log.info("Tag %s not found, importing %s tarball" % (tag, format[1])) - - branch = [options.upstream_branch, - options.packaging_branch][options.native] - if not repo.has_branch(branch): - if options.create_missing_branches: - gbp.log.info("Will create missing branch '%s'" % branch) - else: - gbp.log.err(no_upstream_branch_msg % branch + - "\nAlso check the --create-missing-branches option.") - raise GbpError - - msg = "%s version %s" % (format[1], upstream_version) - upstream_commit = repo.commit_dir(upstream.unpacked, - "Imported %s" % msg, - branch, - author=author, - committer=committer, - create_missing_branch=options.create_missing_branches) - repo.create_tag(name=tag, - msg=msg, - commit=upstream_commit, - sign=options.sign_tags, - keyid=options.keyid) - - if not options.native: - if options.pristine_tar: - repo.pristine_tar.commit(orig_tarball, 'refs/heads/%s' % options.upstream_branch) - parents = [ options.upstream_branch ] + if repo.find_version(options.packaging_tag, tag_str_fields): + gbp.log.warn("Version %s already imported." % RpmPkgPolicy.compose_full_version(spec.version)) + if options.allow_same_version: + gbp.log.info("Moving tag of version '%s' since import forced" % RpmPkgPolicy.compose_full_version(spec.version)) + move_tag_stamp(repo, options.packaging_tag, tag_str_fields) else: - gbp.log.info("No source tarball imported") - - if not options.native or not upstream: - # Import packaging files - gbp.log.info("Importing packaging files...") - branch = options.packaging_branch + raise SkipImport + + if is_empty: + options.create_missing_branches = True + + # Determine author and committer info, currently same info is used + # for both upstream sources and packaging files + author=None + if spec.packager: + match = re.match('(?P.*[^ ])\s*<(?P\S*)>', spec.packager.strip()) + if match: + author=GitModifier(match.group('name'), match.group('email')) + if not author: + author=GitModifier() + gbp.log.debug("Couldn't determine packager info") + committer = committer_from_author(author, options) + + # Import upstream sources + if upstream: + upstream_commit = repo.find_version(format[0], tag_str_fields) + if not upstream_commit: + gbp.log.info("Tag %s not found, importing %s upstream sources" % (tag, format[1])) + + branch = [options.upstream_branch, + options.packaging_branch][options.native] if not repo.has_branch(branch): if options.create_missing_branches: gbp.log.info("Will create missing branch '%s'" % branch) else: - gbp.log.err(no_packaging_branch_msg % branch + - "\nAlso check the --create-missing-branches option.") + gbp.log.err(no_upstream_branch_msg % branch + + "\nAlso check the --create-missing-branches option.") raise GbpError - tag_str_fields = dict(pkgver, vendor=options.vendor) - tag = repo.version_to_tag(options.packaging_tag, tag_str_fields) - msg = "%s release %s" % (options.vendor, RpmPkgPolicy.compose_full_version(pkgver)) - - if options.orphan_packaging or not upstream: - parents = [] - commit = repo.commit_dir(dirs['pkgextract'], - "Imported %s" % msg, - branch, - author=author, - committer=committer, - create_missing_branch=options.create_missing_branches) - else: - # Copy packaging files to the unpacked sources dir - try: - pkgsubdir = os.path.join(upstream.unpacked, options.packaging_dir) - os.mkdir(pkgsubdir) - except OSError, (e, emsg): - if e == errno.EEXIST: - pass - else: - raise - for f in os.listdir(dirs['pkgextract-packaging']): - shutil.copy2(os.path.join(dirs['pkgextract-packaging'], f), - pkgsubdir) - commit = repo.commit_dir(upstream.unpacked, - "Imported %s" % msg, - branch, - other_parents=[upstream_commit], - author=author, - committer=committer, - create_missing_branch=options.create_missing_branches) - + msg = "%s version %s" % (format[1], spec.upstreamversion) + upstream_commit = repo.commit_dir(upstream.unpacked, + "Imported %s" % msg, + branch, + author=author, + committer=committer, + create_missing_branch=options.create_missing_branches) repo.create_tag(name=tag, msg=msg, - commit=commit, + commit=upstream_commit, sign=options.sign_tags, keyid=options.keyid) - if repo.get_branch() == options.packaging_branch: - # Update HEAD if we modified the checked out branch - repo.force_head(options.packaging_branch, hard=True) - # Checkout packaging branch - repo.set_branch(options.packaging_branch) - + if not options.native: + if options.pristine_tar: + repo.pristine_tar.commit(orig_tarball, 'refs/heads/%s' % options.upstream_branch) + parents = [ options.upstream_branch ] + else: + gbp.log.info("No orig source archive imported") + + # Import packaging files. For native packages we assume that also + # packaging files are found in the source tarball + if not options.native or not upstream: + gbp.log.info("Importing packaging files") + branch = options.packaging_branch + if not repo.has_branch(branch): + if options.create_missing_branches: + gbp.log.info("Will create missing branch '%s'" % branch) + else: + gbp.log.err(no_packaging_branch_msg % branch + + "\nAlso check the --create-missing-branches option.") + raise GbpError + + tag_str_fields = dict(spec.version, vendor=options.vendor) + tag = repo.version_to_tag(options.packaging_tag, tag_str_fields) + msg = "%s release %s" % (options.vendor, RpmPkgPolicy.compose_full_version(spec.version)) + + if options.orphan_packaging or not upstream: + parents = [] + commit = repo.commit_dir(dirs['packaging_base'], + "Imported %s" % msg, + branch, + author=author, + committer=committer, + create_missing_branch=options.create_missing_branches) + else: + # Copy packaging files to the unpacked sources dir + try: + pkgsubdir = os.path.join(upstream.unpacked, options.packaging_dir) + os.mkdir(pkgsubdir) + except OSError, (e, emsg): + if e != errno.EEXIST: + raise + for f in os.listdir(dirs['packaging']): + shutil.copy2(os.path.join(dirs['packaging'], f), + pkgsubdir) + commit = repo.commit_dir(upstream.unpacked, + "Imported %s" % msg, + branch, + other_parents=[upstream_commit], + author=author, + committer=committer, + create_missing_branch=options.create_missing_branches) + + # Create packaging tag + repo.create_tag(name=tag, + msg=msg, + commit=commit, + sign=options.sign_tags, + keyid=options.keyid) + + if repo.get_branch() == options.packaging_branch: + # Update HEAD if we modified the checked out branch + repo.force_head(options.packaging_branch, hard=True) + # Checkout packaging branch + repo.set_branch(options.packaging_branch) except KeyboardInterrupt: ret = 1 @@ -407,13 +392,10 @@ def main(argv): skipped = True finally: os.chdir(dirs['top']) - - for d in [ 'pkgextract', 'srctarball', 'srcunpack', 'download' ]: - if dirs.has_key(d): - gbpc.RemoveTree(dirs[d])() + gbpc.RemoveTree(dirs['tmp_base'])() if not ret and not skipped: - gbp.log.info("Version '%s' imported under '%s'" % (RpmPkgPolicy.compose_full_version(pkgver), pkgname)) + gbp.log.info("Version '%s' imported under '%s'" % (RpmPkgPolicy.compose_full_version(spec.version), spec.name)) return ret if __name__ == '__main__': -- cgit v1.2.3