#!/usr/bin/python -t # primary functions and glue for generating the repository metadata # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University # Portions Copyright 2009 Red Hat, Inc - # written by seth vidal skvidal at fedoraproject.org import os import sys import re from optparse import OptionParser import time import createrepo from createrepo import MDError from createrepo.utils import errorprint, _ import yum.misc def parse_args(args, conf): """ Parse the command line args. return a config object. Sanity check all the things being passed in. """ _def = yum.misc._default_checksums[0] _avail = yum.misc._available_checksums parser = OptionParser(version = "createrepo %s" % createrepo.__version__) # query options parser.add_option("-q", "--quiet", default=False, action="store_true", help="output nothing except for serious errors") parser.add_option("-v", "--verbose", default=False, action="store_true", help="output more debugging info.") parser.add_option("--profile", default=False, action="store_true", help="output timing/profile info.") parser.add_option("-x", "--excludes", default=[], action="append", help="files to exclude") parser.add_option("--basedir", default=os.getcwd(), help="basedir for path to directories") parser.add_option("-u", "--baseurl", default=None, help="baseurl to append on all files") parser.add_option("-g", "--groupfile", default=None, help="path to groupfile to include in metadata") parser.add_option("-s", "--checksum", default=_def, dest='sumtype', help="specify the checksum type to use (default: %s)" % _def) parser.add_option("-p", "--pretty", default=False, action="store_true", help="make sure all xml generated is formatted") parser.add_option("-c", "--cachedir", default=None, help="set path to cache dir") parser.add_option("-C", "--checkts", default=False, action="store_true", help="check timestamps on files vs the metadata to see " \ "if we need to update") parser.add_option("-d", "--database", default=True, action="store_true", help="create sqlite database files: now default, see --no-database to disable") parser.add_option("--no-database", default=False, dest="nodatabase", action="store_true", help="do not create sqlite dbs of metadata") # temporarily disabled #parser.add_option("--database-only", default=False, action="store_true", # dest='database_only', # help="Only make the sqlite databases - does not work with --update, yet") parser.add_option("--update", default=False, action="store_true", help="use the existing repodata to speed up creation of new") parser.add_option("--update-md-path", default=None, dest='update_md_path', help="use the existing repodata for --update from this path") parser.add_option("--skip-stat", dest='skip_stat', default=False, help="skip the stat() call on a --update, assumes if the file" \ "name is the same then the file is still the same " \ "(only use this if you're fairly trusting or gullible)", action="store_true") parser.add_option("--split", default=False, action="store_true", help="generate split media") parser.add_option("-i", "--pkglist", default=None, help="use only the files listed in this file from the " \ "directory specified") parser.add_option("-n", "--includepkg", default=[], action="append", help="add this pkg to the list - can be specified multiple times") parser.add_option("-o", "--outputdir", default=None, help=" = optional directory to output to") parser.add_option("-S", "--skip-symlinks", dest="skip_symlinks", default=False, action="store_true", help="ignore symlinks of packages") parser.add_option("--changelog-limit", dest="changelog_limit", default=None, help="only import the last N changelog entries") parser.add_option("--unique-md-filenames", dest="unique_md_filenames", help="include the file's checksum in the filename, helps with proxies", default=True, action="store_true") parser.add_option("--simple-md-filenames", dest="simple_md_filenames", help="do not include the file's checksum in the filename, helps with proxies", default=False, action="store_true") parser.add_option("--distro", default=[], action="append", help="distro tag and optional cpeid: --distro" "'cpeid,textname'") parser.add_option("--content", default=[], dest='content_tags', action="append", help="tags for the content in the repository") parser.add_option("--repo", default=[], dest='repo_tags', action="append", help="tags to describe the repository itself") parser.add_option("--revision", default=None, help="user-specified revision for this repository") parser.add_option("--deltas", default=False, action="store_true", help="create delta rpms and metadata") parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", action="append", help="paths to look for older pkgs to delta against") parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', help="the number of older versions to make deltas against") parser.add_option("--read-pkgs-list", default=None, dest='read_pkgs_list', help="output the paths to the pkgs actually read useful with --update") parser.add_option("--max-delta-rpm-size", default=100000000, dest='max_delta_rpm_size', type='int', help="max size of an rpm that to run deltarpm against (in bytes)") parser.add_option("--workers", default=1, dest='workers', type='int', help="number of workers to spawn to read rpms") (opts, argsleft) = parser.parse_args(args) if len(argsleft) > 1 and not opts.split: errorprint(_('Error: Only one directory allowed per run.')) parser.print_usage() sys.exit(1) elif len(argsleft) == 0: errorprint(_('Error: Must specify a directory to index.')) parser.print_usage() sys.exit(1) else: directories = argsleft if opts.sumtype == 'sha1': errorprint(_('Warning: It is more compatible to use sha instead of sha1')) if opts.sumtype != 'sha' and opts.sumtype not in _avail: errorprint(_('Error: Checksum %s not available (sha, %s)') % (opts.sumtype, ", ".join(sorted(_avail)))) sys.exit(1) if opts.split and opts.checkts: errorprint(_('--split and --checkts options are mutually exclusive')) sys.exit(1) if opts.simple_md_filenames: opts.unique_md_filenames = False if opts.nodatabase: opts.database = False # let's switch over to using the conf object - put all the opts into it for opt in parser.option_list: if opt.dest is None: # this is fairly silly continue # if it's not set, take the default from the base class if getattr(opts, opt.dest) is None: continue setattr(conf, opt.dest, getattr(opts, opt.dest)) directory = directories[0] conf.directory = directory conf.directories = directories # distro tag parsing for spec in opts.distro: if spec.find(',') == -1: conf.distro_tags.append((None, spec)) else: splitspec = spec.split(',') conf.distro_tags.append((splitspec[0], splitspec[1])) lst = [] if conf.pkglist: pfo = open(conf.pkglist, 'r') for line in pfo.readlines(): line = line.strip() if re.match('^\s*\#.*', line) or re.match('^\s*$', line): continue lst.append(line) pfo.close() conf.pkglist = lst if conf.includepkg: conf.pkglist.extend(conf.includepkg) if conf.changelog_limit: # make sure it is an int, not a string conf.changelog_limit = int(conf.changelog_limit) return conf class MDCallBack(object): """cli callback object for createrepo""" def __init__(self): self.__show_progress = os.isatty(1) def errorlog(self, thing): """error log output""" print >> sys.stderr, thing def log(self, thing): """log output""" print thing def progress(self, item, current, total): """progress bar""" if not self.__show_progress: return beg = "%*d/%d - " % (len(str(total)), current, total) left = 80 - len(beg) sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) sys.stdout.flush() def main(args): """createrepo from cli main flow""" start_st = time.time() conf = createrepo.MetaDataConfig() conf = parse_args(args, conf) if conf.profile: print ('start time: %0.3f' % (time.time() - start_st)) mid_st = time.time() try: if conf.split: mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) else: mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) if mdgen.checkTimeStamps(): if mdgen.conf.verbose: print _('repo is up to date') sys.exit(0) if conf.profile: print ('mid time: %0.3f' % (time.time() - mid_st)) pm_st = time.time() mdgen.doPkgMetadata() if conf.profile: print ('pm time: %0.3f' % (time.time() - pm_st)) rm_st = time.time() mdgen.doRepoMetadata() if conf.profile: print ('rm time: %0.3f' % (time.time() - rm_st)) fm_st = time.time() mdgen.doFinalMove() if conf.profile: print ('fm time: %0.3f' % (time.time() - fm_st)) except MDError, errormsg: errorprint(_('%s') % errormsg) sys.exit(1) if __name__ == "__main__": if len(sys.argv) > 1: if sys.argv[1] == 'profile': import hotshot p = hotshot.Profile(os.path.expanduser("~/createrepo.prof")) p.run('main(sys.argv[2:])') p.close() else: main(sys.argv[1:]) else: main(sys.argv[1:])