diff options
| -rw-r--r-- | doc/RELEASE_NOTES | 218 | ||||
| -rw-r--r-- | etc/mic.conf.in | 9 | ||||
| -rw-r--r-- | mic/bootstrap.py | 65 | ||||
| -rw-r--r-- | mic/chroot.py | 22 | ||||
| -rw-r--r-- | mic/conf.py | 65 | ||||
| -rw-r--r-- | mic/creator.py | 9 | ||||
| -rw-r--r-- | mic/imager/baseimager.py | 2 | ||||
| -rw-r--r-- | mic/imager/loop.py | 5 | ||||
| -rw-r--r-- | mic/imager/raw.py | 13 | ||||
| -rw-r--r-- | mic/kickstart/custom_commands/partition.py | 5 | ||||
| -rw-r--r-- | mic/msger.py | 2 | ||||
| -rw-r--r-- | mic/rt_util.py | 35 | ||||
| -rw-r--r-- | mic/utils/fs_related.py | 74 | ||||
| -rw-r--r-- | mic/utils/misc.py | 16 | ||||
| -rw-r--r-- | mic/utils/rpmmisc.py | 18 | ||||
| -rw-r--r-- | packaging/mic.spec | 1 | ||||
| -rw-r--r-- | plugins/backend/yumpkgmgr.py | 9 | ||||
| -rwxr-xr-x | plugins/backend/zypppkgmgr.py | 2 | ||||
| -rw-r--r-- | plugins/imager/fs_plugin.py | 2 | ||||
| -rw-r--r-- | plugins/imager/livecd_plugin.py | 2 | ||||
| -rw-r--r-- | plugins/imager/liveusb_plugin.py | 2 | ||||
| -rw-r--r-- | plugins/imager/loop_plugin.py | 2 | ||||
| -rw-r--r-- | plugins/imager/raw_plugin.py | 2 | ||||
| -rw-r--r-- | setup.py | 3 |
24 files changed, 434 insertions, 149 deletions
diff --git a/doc/RELEASE_NOTES b/doc/RELEASE_NOTES new file mode 100644 index 0000000..b966eb5 --- /dev/null +++ b/doc/RELEASE_NOTES @@ -0,0 +1,218 @@ +Release 0.15 - Tue Dec 13 2012 +=========================================================== + * adapt new mechanism for bootstrap mode + - create 'mic-bootstrap-x86-arm' by obs build + - publish 'mic-bootstrap-x86-arm' into server repo + - use 'mic-bootstrap-x86-arm' as bootstrap env for x86 and arm image + * filesystem parameter support for ext[234] fs + - sample in ks: part / --size 1000 --fstype=ext3 --extoptions="-I 256" + - other filesystem will ignore option '--extoptions' + * update the documentation and man page + * multiple bootstrap path to support multi-instance mic + * use 'pkgmgr=auto' to select available backend in conf file + * improve loop device creation algorithm and create loop device by 'mknod' + * bug fix: + - bear unexpected checksum type when getting metadata + - avoid traceback when loopback is NoneType + - lseek limit to 2G in 32bit env + - split out username and password in zypp repo file + - use rpm real path instead of 'cp' if it's local repo + - fix local repo unavailable in bootstrap + - fix traceback when failed to unmap kpartx device + - fix timestamp incorrect issue in logfile + +Release 0.14.2 - Wed Nov 14 2012 +=========================================================== + * support dracut for live image + * update bmap version to 1.1 + +Release 0.14.1 - Fri Oct 15 2012 +=========================================================== + * support bmap file for ivi flashing tool + * just warning in chroot when not Tizen/MeeGo chroot dir + * fix logfile lost in bootstrap mode + * clean mounts in bootstrap when exiting + * bug fix: + - fix https proxy issue in yum backend + - avoid traceback when loop instance is NoneType + +Release 0.14 - Thu Aug 02 2012 +=========================================================== + * use cached metadata when checksum is not changed + * skip non-fatal error in ks file and prompt user to handle + * prompt user to handle when failed to apply img configure + * replace hard name with device uuid in etc/fstab + * enhance extlinux cfg file for symbolic kernel like IVI + * support label assign for raw image + * bug fix: + - fix live image create failure when label assigned + - avoid traceback when converting unsupported type + - fix mic --version ugly output + +Release 0.13 - Wed Jul 12 2012 +=========================================================== + * create logfile as default when --release specifid + * use 'gzip' and 'bzip2' to pack image instead of python + * automatically detect path of 'env' for chroot + * record version and os info in build log and logfile + * bug fix: + - fix popup message in ubuntus + - fix unicode issue for logfile + - better fix for 'chroot raw' issue + +Release 0.12 - Wed Jun 20 2012 +=========================================================== + * use default value when @BUILD_ID@ and @ARCH@ not specified + * enhance proxy support in attachment retrieve + * add new --shrink opt for loop image to control img shrinking + * avoid invalid literal for loop device generation + * relocate and refactor selinux_check func + * remove prefix for make install + * bug fix: + - fix compres image in raw image + - fix src pkgs download failed issue + - fix convert failed issue + +Release 0.11 - Fri Jun 08 2012 +=========================================================== + * support new subcmd 'auto' to handle magic line in ks + * enhance the handle of authentication url and https proxy + * support packing images together and support compressed file format + * reset LD_PRELOAD for chroot env + * centralized interface to check existing images + * avoid live image creating when using multi-partitions + * resolve the depends of python-urlgrabber + * bug fix: + - fix logfile context lost issue + - fix attachment package url handling + - fix mic ch raw failed issue + +Release 0.10 - Tue May 15 2012 +=========================================================== + * container support using '%attachment' section in ks + * add --compress-to option to support zip format in loop image + * auto-detect config and plugindir to meet virtualenv and customized install + * remove all hardcoded info in setup.py and use sys.prefix for installing + * tolerate some OS errors in the image configurations stage + * extra patch: + - fix zypp missing password when using username passwd + - some fixes to enhance authentication url + - refine repostr structure to fix comma issue in baseurl + +Release 0.9 - Fri Apr 13 2012 +=========================================================== + * support pre-install package with zypp backend + * sync /etc/mic/mic.conf to bootstrap + * enhance sorting for version comparsion in zypp + * rewrite chroot tar image using xml format mount point file + * fix the incorrect number showing in fs src pkgs download + * remove tests directory for dist in Makefile + * fix liveusb parted mkpart failure, revert mbr size expand in raw + * cleanup /tmp/repolic* dir in the EULA checking + +Release 0.8 - Mon Mar 26 2012 +=========================================================== + * partition alignment support + * remove bootloader option 'quiet vga' for raw + * update dist files in git source + * update unittest, add cases for chroot, msger, runner + * add 40 system test case for help + * rewrite loop device allocation mechanism + +Release 0.7 - Fri Mar 02 2012 +=========================================================== + * zypp backend: fixed a fatal issue of unreleasable loop devs + * zypp backend: more friendly output message + * backend: share cached rpm files between yum and zypp + * enhancement for multiple partition loop format + * make msger to accept Unicode string + * fixed a regression of compress option for FS format + * fixed issues in openSUSE12.1 + * new written man page + +Release 0.6 - Thu Feb 16 2012 +=========================================================== + * give hint when converted image existed + * conf.py: proxy scheme check + * space check before copy image + * zypp: abort with error msg for repo resolver issues + * runner.py refinement + * ks file syntax check for '%post' without '%end' + * support more compression formats than only bzip2 + * fix msg NoneType issue, causing exit after install + * bootstrap: + - catch creator error when retrieving bootstrap metadata + - correct matching .metadata file in bootstrap + +Release 0.5 - Mon Feb 06 2012 +=========================================================== + * Rewrite the algorithm of checking free space for download and install + * Add --shell option for convert to recreate image modified by internal shell + * Add -s option for chroot to unpack image + * Introduce --copy-kernel option for creator + * Remove the hardcoded default args for bootloader + * Disable logstderr and flush message buffer in disable_logstderr + * Deal with yum.conf inside yum backend by itself + * Bug fix: + - Fix rpmdb error in yum and zypp to avoid bad file descriptor message + - Fix MANIFEST syntax to be compliant with md5sum + - Correct dependencies for mic in bootstrap + +Release 0.4 - Fri Jan 06 2012 +=========================================================== + * Support bootstrap mode, run with '--runtime=bootstrap' + * Full support for taring-to output, use 'mic ch x.tar' + * Break dependency between backend and baseimage + * Check valid repos in ks file + * Space check update and catch no space exception + * Fix no prompt when cv and ch no existed image + * Fix NoneType 'createopts' when convert + * Fix no existed local_pkgs_path + +Release 0.3 - Mon Dec 26 2011 +=========================================================== + * Unit test support, run 'make test' + * Enable proxy support in config file + * Refine configmgr and pluginmgr + * Support multi instance with different cache dir + * Add 47 system test case + * Improve md5sum generation + * Add repo option --ssl_verify + * Add option --name_prefix + * Reformatted code according to PEP08 + * Backport from mic2: + - Add priority and cost option for repos + - Reinstroduced compress-disk-image option + +Release 0.2 - Tue Nov 29 2011 +=========================================================== + * Support btrfs and ext4 fstype for creator, convertor, and chroot + * Append distfiles and Makefile + * Check arch type from repo data + * Set rpm dbpath to fix 'rpm -qa' issue + * Fix chroot issue caused by image size + * Improve setup.py and make it compatible with python 2.5 + * Disable ca check for https + * Change default output dir name to ./mic-output + * untrack mic/__version__.py + * Fix some minor issues + +Release 0.1 - Thu Oct 27 2011 +=========================================================== + * Support three subcommand: create, convert, chroot + * Support five image types: fs, loop, raw, livecd, liveusb + * Support two package manager backend: yum and zypp + * Support the following global command line options: + - --verbose + - --debug + * Creator subcommand support the following command line options: + - --logfile=LOGFILE + - -c CONFIG, --config=CONFIG + - -k CACHEDIR, --cachedir=CACHEDIR + - -o OUTDIR, --outdir=OUTDIR + - -A ARCH, --arch=ARCH + - --release=RID + - --record-pkgs=RECORD_PKGS + - --pkgmgr=PKGMGR + - --local-pkgs-path=LOCAL_PKGS_PATH + diff --git a/etc/mic.conf.in b/etc/mic.conf.in index b79603e..3d41eff 100644 --- a/etc/mic.conf.in +++ b/etc/mic.conf.in @@ -9,7 +9,7 @@ plugin_dir = @PREFIX@/lib/mic/plugins tmpdir= /var/tmp/mic cachedir= /var/tmp/mic/cache outdir= ./mic-output -bootstrapdir= /var/tmp/mic/bootstrap +runtime=bootstrap pkgmgr = auto @@ -29,8 +29,7 @@ pkgmgr = auto [chroot] ; settings for chroot subcommand -[bootstrap1] -name=micbootstrap -bootstrap=http://download.tizen.org/tools/micbootstrap/ -; bootstrap_proxy=http://proxy.yourcompany.com:port +[bootstrap] +rootdir=/var/tmp/mic-bootstrap +packages=mic-bootstrap-x86-arm diff --git a/mic/bootstrap.py b/mic/bootstrap.py index 83ce0bb..6f00d66 100644 --- a/mic/bootstrap.py +++ b/mic/bootstrap.py @@ -57,6 +57,7 @@ class MiniBackend(object): self.optionals = [] self.preins = {} self.postins = {} + self.scriptlets = False def __del__(self): try: @@ -94,6 +95,9 @@ class MiniBackend(object): if self.arch.startswith("arm"): misc.setup_qemu_emulator(self.rootdir, self.arch) + if not self.scriptlets: + return + for pkg in self.preins.keys(): prog, script = self.preins[pkg] self.run_pkg_script(pkg, prog, script, '0') @@ -167,7 +171,7 @@ class MiniBackend(object): class Bootstrap(object): def __init__(self, rootdir, distro, arch=None): - self.rootdir = rootdir + self.rootdir = misc.mkdtemp(dir=rootdir, prefix=distro) self.distro = distro self.arch = arch self.logfile = None @@ -183,37 +187,61 @@ class Bootstrap(object): os.makedirs(self.rootdir) return self.rootdir - def _path(self, pth): - return os.path.join(self.rootdir, pth.lstrip('/')) + def dirsetup(self, rootdir=None): + _path = lambda pth: os.path.join(rootdir, pth.lstrip('/')) - def create(self, repomd, pkglist, optlist=[]): - try: - pkgmgr = MiniBackend(self.get_rootdir()) - pkgmgr.arch = self.arch - pkgmgr.repomd = repomd - pkgmgr.optionals = optlist - map(pkgmgr.selectPackage, pkglist + optlist) - pkgmgr.runInstall() + if not rootdir: + rootdir = self.rootdir - # make /tmp path - tmpdir = self._path('/tmp') + try: + # make /tmp and /etc path + tmpdir = _path('/tmp') if not os.path.exists(tmpdir): os.makedirs(tmpdir) + etcdir = _path('/etc') + if not os.path.exists(etcdir): + os.makedirs(etcdir) # touch distro file - tzdist = self._path('/etc/%s-release' % self.distro) + tzdist = _path('/etc/%s-release' % self.distro) if not os.path.exists(tzdist): with open(tzdist, 'w') as wf: wf.write("bootstrap") + except: + pass + + def create(self, repomd, pkglist, optlist=[]): + try: + pkgmgr = MiniBackend(self.get_rootdir()) + pkgmgr.arch = self.arch + pkgmgr.repomd = repomd + pkgmgr.optionals = optlist + map(pkgmgr.selectPackage, pkglist + optlist) + pkgmgr.runInstall() except (OSError, IOError, errors.CreatorError), err: raise errors.BootstrapError("%s" % err) - def run(self, cmd, chdir, bindmounts=None): + except: + raise + + def run(self, cmd, chdir, rootdir=None, bindmounts=None): def mychroot(): - os.chroot(self.rootdir) + os.chroot(rootdir) os.chdir(chdir) + def sync_timesetting(rootdir): + try: + # sync time and zone info to bootstrap + if os.path.exists(rootdir + "/etc/localtime"): + os.unlink(rootdir + "/etc/localtime") + shutil.copyfile("/etc/localtime", rootdir + "/etc/localtime") + except: + pass + + if not rootdir: + rootdir = self.rootdir + if isinstance(cmd, list): shell = False else: @@ -226,14 +254,15 @@ class Bootstrap(object): gloablmounts = None try: proxy.set_proxy_environ() - gloablmounts = setup_chrootenv(self.rootdir, bindmounts) + gloablmounts = setup_chrootenv(rootdir, bindmounts, False) + sync_timesetting(rootdir) retcode = subprocess.call(cmd, preexec_fn=mychroot, env=env, shell=shell) except (OSError, IOError), err: raise RuntimeError(err) finally: if self.logfile: msger.log(file(self.logfile).read()) - cleanup_chrootenv(self.rootdir, bindmounts, gloablmounts) + cleanup_chrootenv(rootdir, bindmounts, gloablmounts) proxy.unset_proxy_environ() return retcode diff --git a/mic/chroot.py b/mic/chroot.py index f4f5942..ed867df 100644 --- a/mic/chroot.py +++ b/mic/chroot.py @@ -98,10 +98,10 @@ def cleanup_mounts(chrootdir): return 0 -def setup_chrootenv(chrootdir, bindmounts = None): +def setup_chrootenv(chrootdir, bindmounts = None, mountparent = True): global chroot_lockfd, chroot_lock - def get_bind_mounts(chrootdir, bindmounts): + def get_bind_mounts(chrootdir, bindmounts, mountparent = True): chrootmounts = [] if bindmounts in ("", None): bindmounts = "" @@ -115,6 +115,11 @@ def setup_chrootenv(chrootdir, bindmounts = None): if len(srcdst) == 1: srcdst.append("none") + # if some bindmount is not existed, but it's created inside + # chroot, this is not expected + if not os.path.exists(srcdst[0]): + os.makedirs(srcdst[0]) + if not os.path.isdir(srcdst[0]): continue @@ -141,10 +146,11 @@ def setup_chrootenv(chrootdir, bindmounts = None): chrootdir, None)) - chrootmounts.append(fs_related.BindChrootMount("/", - chrootdir, - "/parentroot", - "ro")) + if mountparent: + chrootmounts.append(fs_related.BindChrootMount("/", + chrootdir, + "/parentroot", + "ro")) for kernel in os.listdir("/lib/modules"): chrootmounts.append(fs_related.BindChrootMount( @@ -166,7 +172,7 @@ def setup_chrootenv(chrootdir, bindmounts = None): except: pass - globalmounts = get_bind_mounts(chrootdir, bindmounts) + globalmounts = get_bind_mounts(chrootdir, bindmounts, mountparent) bind_mount(globalmounts) setup_resolv(chrootdir) @@ -237,7 +243,7 @@ def cleanup_chrootenv(chrootdir, bindmounts = None, globalmounts = []): if not fs_related.my_fuser(chroot_lock): tmpdir = chrootdir + "/parentroot" - if len(os.listdir(tmpdir)) == 0: + if os.path.exists(tmpdir) and len(os.listdir(tmpdir)) == 0: shutil.rmtree(tmpdir, ignore_errors = True) cleanup_resolv(chrootdir) diff --git a/mic/conf.py b/mic/conf.py index d0d96ce..82d9049 100644 --- a/mic/conf.py +++ b/mic/conf.py @@ -61,6 +61,8 @@ class ConfigMgr(object): "no_proxy": None, "copy_kernel": False, "repourl": {}, + "localrepos": [], # save localrepos + "runtime": "bootstrap", }, 'chroot': { "saveto": None, @@ -69,10 +71,8 @@ class ConfigMgr(object): "shell": False, }, 'bootstrap': { - "enable": False, - "distros": [], # supported distros - "distro_name": None, "rootdir": '/var/tmp/mic-bootstrap', + "packages": [], }, } @@ -94,10 +94,6 @@ class ConfigMgr(object): # initial options from siteconf self._siteconf = siteconf - # set bootstrap from bootstrap.conf - bsconf = os.path.join(os.path.dirname(siteconf), 'bootstrap.conf') - self._parse_bootstrap(bsconf) - if ksconf: self._ksconf = ksconf @@ -146,7 +142,7 @@ class ConfigMgr(object): # append common section items to other sections for section in self.DEFAULTS.keys(): - if section != "common" and not section.startswith('bootstrap'): + if section != "common": getattr(self, section).update(self.common) # check and normalize the scheme of proxy url @@ -163,6 +159,16 @@ class ConfigMgr(object): proxy.set_proxies(self.create['proxy'], self.create['no_proxy']) + # bootstrap option handling + self.set_runtime(self.create['runtime']) + if isinstance(self.bootstrap['packages'], basestring): + packages = self.bootstrap['packages'].replace('\n', ' ') + if packages.find(',') != -1: + packages = packages.split(',') + else: + packages = packages.split() + self.bootstrap['packages'] = packages + def _parse_kickstart(self, ksconf=None): if not ksconf: return @@ -185,6 +191,12 @@ class ConfigMgr(object): if not ksrepos: raise errors.KsError('no valid repos found in ks file') + for repo in ksrepos: + if 'baseurl' in repo and repo['baseurl'].startswith("file:"): + repourl = repo['baseurl'].replace('file:', '') + repourl = "/%s" % repourl.lstrip('/') + self.create['localrepos'].append(repourl) + self.create['repomd'] = misc.get_metadata_from_repos( ksrepos, self.create['cachedir']) @@ -211,37 +223,12 @@ class ConfigMgr(object): misc.selinux_check(self.create['arch'], [p.fstype for p in ks.handler.partition.partitions]) + def set_runtime(self, runtime): + if runtime not in ("bootstrap", "native"): + msger.error("Invalid runtime mode: %s" % runtime) - def _parse_bootstrap(self, bsconf): - if not bsconf or not os.path.exists(bsconf): - self.bootstrap['enable'] = False - return - - parser = ConfigParser.SafeConfigParser() - parser.read(bsconf) - - for section in parser.sections(): - if section == "main": - self.bootstrap.update(dict(parser.items(section))) - else: - self.bootstrap['distros'].append(section) - distro = section.lower() - self.bootstrap[distro] = dict(parser.items(section)) - for item in ('optional', 'packages'): - if not item in self.bootstrap[distro]: - continue - pks = self.bootstrap[distro][item].replace('\n', ' ') - self.bootstrap[distro][item] = pks.split() - - # update bootstrap options - if self.bootstrap['enable'] not in (True, False): - try: - self.bootstrap['enable'] = parser.getboolean('main', 'enable') - except: - self.bootstrap['enable'] = False - if not self.bootstrap['distro_name']: - self.bootstrap['distro_name'] = self.common['distro_name'] - if self.bootstrap['distro_name'] not in self.bootstrap['distros']: - self.bootstrap['enable'] = False + if misc.get_distro()[0] in ("tizen", "Tizen"): + runtime = "native" + self.create['runtime'] = runtime configmgr = ConfigMgr() diff --git a/mic/creator.py b/mic/creator.py index be92d01..0c336d6 100644 --- a/mic/creator.py +++ b/mic/creator.py @@ -92,8 +92,7 @@ class Creator(cmdln.Cmdln): help='Path for local pkgs(rpms) to be installed') optparser.add_option('', '--runtime', type='string', dest='runtime', default=None, - #help='Specify runtime mode, avaiable: bootstrap') - help=SUPPRESS_HELP) + help='Specify runtime mode, avaiable: bootstrap, native') # --taring-to is alias to --pack-to optparser.add_option('', '--taring-to', type='string', dest='pack_to', default=None, @@ -161,11 +160,11 @@ class Creator(cmdln.Cmdln): if self.options.logfile: logfile_abs_path = abspath(self.options.logfile) - if not os.path.exists(os.path.dirname(logfile_abs_path)): - os.makedirs(os.path.dirname(logfile_abs_path)) if os.path.isdir(logfile_abs_path): raise errors.Usage("logfile's path %s should be file" % self.options.logfile) + if not os.path.exists(os.path.dirname(logfile_abs_path)): + os.makedirs(os.path.dirname(logfile_abs_path)) msger.set_interactive(False) msger.set_logfile(logfile_abs_path) configmgr.create['logfile'] = self.options.logfile @@ -213,7 +212,7 @@ class Creator(cmdln.Cmdln): configmgr.create['pkgmgr'] = self.options.pkgmgr if self.options.runtime: - configmgr.create['runtime'] = self.options.runtime + configmgr.set_runtime(self.options.runtime) if self.options.pack_to is not None: configmgr.create['pack_to'] = self.options.pack_to diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index 92bdc86..919420b 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -111,7 +111,7 @@ class BaseImageCreator(object): if ext not in misc.pack_formats: self.pack_to += ".tar" - self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe", "passwd"] + self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"] # Output image file names self.outimage = [] diff --git a/mic/imager/loop.py b/mic/imager/loop.py index 27dd677..70a79a6 100644 --- a/mic/imager/loop.py +++ b/mic/imager/loop.py @@ -150,6 +150,7 @@ class LoopImageCreator(BaseImageCreator): 'name': imgname, 'size': part.size or 4096L * 1024 * 1024, 'fstype': part.fstype or 'ext3', + 'extopts': part.extopts or None, 'loop': None, # to be created in _mount_instroot }) self._instloops = allloops @@ -314,6 +315,7 @@ class LoopImageCreator(BaseImageCreator): "name": imgname, "size": self.__image_size or 4096L, "fstype": self.__fstype or "ext3", + "extopts": part.extopts or None, "loop": None }) @@ -342,6 +344,9 @@ class LoopImageCreator(BaseImageCreator): self._blocksize, loop['label']) + if fstype in ("ext2", "ext3", "ext4"): + loop['loop'].extopts = loop['extopts'] + try: msger.verbose('Mounting image "%s" on "%s"' % (imgname, mp)) fs.makedirs(mp) diff --git a/mic/imager/raw.py b/mic/imager/raw.py index b4d06a9..0ea3b08 100644 --- a/mic/imager/raw.py +++ b/mic/imager/raw.py @@ -18,7 +18,6 @@ import os import stat import shutil -import hashlib from fcntl import ioctl from struct import pack, unpack from itertools import groupby @@ -83,8 +82,13 @@ class RawImageCreator(BaseImageCreator): p = p1 break + if p['uuid'] is None: + device = "/dev/%s%-d" % (p['disk'], p['num']) + else: + device = "UUID=%s" % p['uuid'] + s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % { - 'device': "UUID=%s" % p['uuid'], + 'device': device, 'mountpoint': p['mountpoint'], 'fstype': p['fstype'], 'fsopts': "defaults,noatime" if not p['fsopts'] else p['fsopts']} @@ -374,7 +378,10 @@ class RawImageCreator(BaseImageCreator): def _unmount_instroot(self): if not self.__instloop is None: - self.__instloop.cleanup() + try: + self.__instloop.cleanup() + except MountError, err: + msger.warning("%s" % err) def _resparse(self, size = None): return self.__instloop.resparse(size) diff --git a/mic/kickstart/custom_commands/partition.py b/mic/kickstart/custom_commands/partition.py index 7b78319..1e458e5 100644 --- a/mic/kickstart/custom_commands/partition.py +++ b/mic/kickstart/custom_commands/partition.py @@ -25,6 +25,7 @@ class MeeGo_PartData(FC4_PartData): FC4_PartData.__init__(self, *args, **kwargs) self.deleteRemovedAttrs() self.align = kwargs.get("align", None) + self.extopts = kwargs.get("extopts", None) def _getArgsAsStr(self): retval = FC4_PartData._getArgsAsStr(self) @@ -32,6 +33,8 @@ class MeeGo_PartData(FC4_PartData): if self.align: retval += " --align" + if self.extopts: + retval += " --extoptions=%s" % self.extopts return retval class MeeGo_Partition(FC4_Partition): @@ -44,4 +47,6 @@ class MeeGo_Partition(FC4_Partition): # the partition is aligned to start from 8096 byte boundary. op.add_option("--align", type="int", action="store", dest="align", default=None) + op.add_option("--extoptions", type="string", action="store", dest="extopts", + default=None) return op diff --git a/mic/msger.py b/mic/msger.py index 258fb03..9afc85b 100644 --- a/mic/msger.py +++ b/mic/msger.py @@ -87,7 +87,7 @@ def _general_print(head, color, msg = None, stream = None, level = 'normal'): # append normal msg to LOG save_msg = msg.strip() if msg else None if save_msg: - timestr = time.strftime("[%m/%d %H:%M:%S] ", time.localtime()) + timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime()) LOG_CONTENT += timestr + save_msg + '\n' if errormsg: diff --git a/mic/rt_util.py b/mic/rt_util.py index 918c6d5..6dfbbb2 100644 --- a/mic/rt_util.py +++ b/mic/rt_util.py @@ -47,32 +47,36 @@ def bootstrap_mic(argv=None): cropts = configmgr.create bsopts = configmgr.bootstrap distro = bsopts['distro_name'].lower() - if distro not in bsopts['distros'] or \ - 'packages' not in bsopts[distro]: - msger.info("Use native running for distro don't support bootstrap") - return rootdir = bsopts['rootdir'] - pkglist = bsopts[distro]['packages'] + pkglist = bsopts['packages'] cwd = os.getcwd() # create bootstrap and run mic in bootstrap bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch']) bsenv.logfile = cropts['logfile'] + # rootdir is regenerated as a temp dir + rootdir = bsenv.rootdir - if 'optional' in bsopts[distro]: - optlist = bsopts[distro]['optional'] + if 'optional' in bsopts: + optlist = bsopts['optional'] else: optlist = [] try: msger.info("Creating %s bootstrap ..." % distro) bsenv.create(cropts['repomd'], pkglist, optlist) + + # bootstrap is relocated under "bootstrap" + if os.path.exists(os.path.join(rootdir, "bootstrap")): + rootdir = os.path.join(rootdir, "bootstrap") + + bsenv.dirsetup(rootdir) sync_mic(rootdir) msger.info("Start mic in bootstrap: %s\n" % rootdir) bindmounts = get_bindmounts(cropts) - ret = bsenv.run(argv, cwd, bindmounts) + ret = bsenv.run(argv, cwd, rootdir, bindmounts) except errors.BootstrapError, err: msger.warning('\n%s' % err) @@ -100,6 +104,9 @@ def get_bindmounts(cropts): configmgr._ksconf, ] + for lrepo in cropts['localrepos']: + binddirs.append(lrepo) + bindlist = map(expath, filter(None, binddirs)) bindlist += map(os.path.dirname, map(expath, filter(None, bindfiles))) bindlist = sorted(set(bindlist)) @@ -170,18 +177,6 @@ def sync_mic(bootstrap, binpth = '/usr/bin/mic', except (OSError, IOError), err: raise errors.BootstrapError(err) - # clean stuff: - # bootstrap.conf, disable bootstrap mode inside bootstrap - clrpaths = [ - '/etc/mic/bootstrap.conf', - ] - - for pth in clrpaths: - try: - os.unlink(_path(pth)) - except: - pass - # auto select backend conf_str = file(_path(conf)).read() conf_str = re.sub("pkgmgr\s*=\s*.*", "pkgmgr=auto", conf_str) diff --git a/mic/utils/fs_related.py b/mic/utils/fs_related.py index 4c2bd27..8a678d5 100644 --- a/mic/utils/fs_related.py +++ b/mic/utils/fs_related.py @@ -314,12 +314,14 @@ class SparseLoopbackDisk(LoopbackDisk): else: fd = os.open(self.lofile, flags) + if size <= 0: + size = 1 try: - os.lseek(fd, size, os.SEEK_SET) + os.ftruncate(fd, size) except: # may be limited by 2G in 32bit env - os.lseek(fd, 2**31L - 1, os.SEEK_SET) - os.write(fd, '\x00') + os.ftruncate(fd, 2**31L) + os.close(fd) def truncate(self, size = None): @@ -427,6 +429,7 @@ class ExtDiskMount(DiskMount): self.uuid = None self.skipformat = skipformat self.fsopts = fsopts + self.extopts = None self.dumpe2fs = find_binary_path("dumpe2fs") self.tune2fs = find_binary_path("tune2fs") @@ -443,18 +446,31 @@ class ExtDiskMount(DiskMount): return msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) - rc = runner.show([self.mkfscmd, - "-F", "-L", self.fslabel, - "-m", "1", "-b", str(self.blocksize), - self.disk.device]) # str(self.disk.size / self.blocksize)]) + cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b", + str(self.blocksize)] + if self.extopts: + cmdlist.extend(self.extopts.split()) + cmdlist.extend([self.disk.device]) + + rc, errout = runner.runtool(cmdlist, catch=2) if rc != 0: - raise MountError("Error creating %s filesystem on disk %s" % (self.fstype, self.disk.device)) + raise MountError("Error creating %s filesystem on disk %s:\n%s" % + (self.fstype, self.disk.device, errout)) - out = runner.outs([self.dumpe2fs, '-h', self.disk.device]) + if not self.extopts: + msger.debug("Tuning filesystem on %s" % self.disk.device) + runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) - self.uuid = self.__parse_field(out, "Filesystem UUID") - msger.debug("Tuning filesystem on %s" % self.disk.device) - runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) + rc, errout = runner.runtool([self.dumpe2fs, '-h', self.disk.device], + catch=2) + if rc != 0: + raise MountError("Error dumpe2fs %s filesystem on disk %s:\n%s" % + (self.fstype, self.disk.device, errout)) + # FIXME: specify uuid in mkfs parameter + try: + self.uuid = self.__parse_field(out, "Filesystem UUID") + except: + self.uuid = None def __resize_filesystem(self, size = None): current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] @@ -878,11 +894,10 @@ class LoopDevice(object): else: self.created = True return - try: - os.mknod(self.device, - 0664 | stat.S_IFBLK, - os.makedev(7, self.loopid)) - except: + + mknod = find_binary_path('mknod') + rc = runner.show([mknod, '-m664', self.device, 'b', '7', self.loopid]) + if rc != 0: raise MountError("Failed to create device %s" % self.device) else: self.created = True @@ -922,15 +937,20 @@ def get_loop_device(losetupcmd, lofile): fp = open("/var/lock/__mic_loopdev.lock", 'w') fcntl.flock(fp, fcntl.LOCK_EX) try: - devinst = LoopDevice() - devinst.create() + rc, out = runner.runtool([losetupcmd, "--find"]) + if rc == 0: + loopdev = out.split()[0] + else: + devinst = LoopDevice() + devinst.create() + loopdev = devinst.device + rc = runner.show([losetupcmd, loopdev, lofile]) + if rc != 0: + raise MountError("Failed to setup loop device for '%s'" % lofile) + except MountError, err: + raise CreatorError("%s" % str(err)) except: - rc, out = runner.runtool([losetupcmd, "-f"]) - if rc != 0: - raise MountError("1-Failed to allocate loop device for '%s'" % lofile) - loopdev = out.split()[0] - else: - loopdev = devinst.device + raise finally: try: fcntl.flock(fp, fcntl.LOCK_UN) @@ -939,10 +959,6 @@ def get_loop_device(losetupcmd, lofile): except: pass - rc = runner.show([losetupcmd, loopdev, lofile]) - if rc != 0: - raise MountError("2-Failed to allocate loop device for '%s'" % lofile) - return loopdev diff --git a/mic/utils/misc.py b/mic/utils/misc.py index 5e11f5c..1eec13a 100644 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -28,7 +28,15 @@ import rpmmisc import hashlib import sqlite3 as sqlite -from xml.etree import cElementTree +try: + import sqlite3 as sqlite +except ImportError: + import sqlite + +try: + from xml.etree import cElementTree +except ImportError: + import cElementTree xmlparse = cElementTree.parse from errors import * @@ -231,8 +239,8 @@ def calc_hashes(file_path, hash_names, start = 0, end = None): ending file offset to calculate the has functions for. The 'hash_names' argument is a list of hash names to calculate. Returns the the list of calculated hash values in the hexadecimal form in the same order - as 'hash_names'. """ - + as 'hash_names'. + """ if end == None: end = os.path.getsize(file_path) @@ -283,7 +291,7 @@ def normalize_ksfile(ksconf, release, arch): ksc = f.read() if "@ARCH@" in ksc or "@BUILD_ID@" in ksc: - msger.info("Substitute macro variable @BUILD_ID@/@ARCH in ks: %s" % ksconf) + msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf) ksc = ksc.replace("@ARCH@", arch) ksc = ksc.replace("@BUILD_ID@", release) fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf), dir="/tmp/") diff --git a/mic/utils/rpmmisc.py b/mic/utils/rpmmisc.py index 0bbb689..5cd7b46 100644 --- a/mic/utils/rpmmisc.py +++ b/mic/utils/rpmmisc.py @@ -35,10 +35,15 @@ def myurlgrab(url, filename, proxies, progress_obj = None): progress_obj = TextProgress() if url.startswith("file:/"): - file = url.replace("file:", "") - if not os.path.exists(file): - raise CreatorError("URLGrabber error: can't find file %s" % file) - runner.show(['cp', "-f", file, filename]) + filepath = "/%s" % url.replace("file:", "").lstrip('/') + if not os.path.exists(filepath): + raise CreatorError("URLGrabber error: can't find file %s" % url) + if url.endswith('.rpm'): + return filepath + else: + # untouch repometadata in source path + runner.show(['cp', '-f', filepath, filename]) + else: try: filename = g.urlgrab(url = url, filename = filename, @@ -182,8 +187,9 @@ class RPMInstallCallback: handle = self._makeHandle(hdr) fd = os.open(rpmloc, os.O_RDONLY) self.callbackfilehandles[handle]=fd - self.total_installed += 1 - self.installed_pkg_names.append(hdr['name']) + if hdr['name'] not in self.installed_pkg_names: + self.installed_pkg_names.append(hdr['name']) + self.total_installed += 1 return fd else: self._localprint("No header - huh?") diff --git a/packaging/mic.spec b/packaging/mic.spec index a8a1485..b3a299e 100644 --- a/packaging/mic.spec +++ b/packaging/mic.spec @@ -96,6 +96,7 @@ install -m644 mic.1 %{buildroot}/%{_prefix}/share/man/man1 %files %defattr(-,root,root,-) %doc README.rst +%doc doc/RELEASE_NOTES %{_mandir}/man1/* %dir %{_sysconfdir}/%{name} %config(noreplace) %{_sysconfdir}/%{name}/%{name}.conf diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index 90f9c77..33e354e 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -68,9 +68,12 @@ class MyYumRepository(yum.yumRepo.YumRepository): m2c_connection = None if not self.sslverify: - import M2Crypto - m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck - M2Crypto.SSL.Connection.clientPostConnectionCheck = None + try: + import M2Crypto + m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck + M2Crypto.SSL.Connection.clientPostConnectionCheck = None + except ImportError, err: + raise CreatorError("%s, please try to install python-m2crypto" % str(err)) proxy = None if url: diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index 7e76089..9c6126a 100755 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -678,6 +678,8 @@ class Zypp(BackendPlugin): if not self.ts: self.__initialize_transaction() + # clean rpm lock + self._cleanupRpmdbLocks(self.instroot) # Set filters probfilter = 0 for flag in self.probFilterFlags: diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index 47df1bb..6801def 100644 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -49,7 +49,7 @@ class FsPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/livecd_plugin.py b/plugins/imager/livecd_plugin.py index 26c2d11..5782e13 100644 --- a/plugins/imager/livecd_plugin.py +++ b/plugins/imager/livecd_plugin.py @@ -45,7 +45,7 @@ class LiveCDPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/liveusb_plugin.py b/plugins/imager/liveusb_plugin.py index babcfa6..6a12834 100644 --- a/plugins/imager/liveusb_plugin.py +++ b/plugins/imager/liveusb_plugin.py @@ -47,7 +47,7 @@ class LiveUSBPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index 7e3141f..8872206 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -54,7 +54,7 @@ class LoopPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index 5c8be60..bcc3f55 100644 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -56,7 +56,7 @@ class RawPlugin(ImagerPlugin): creatoropts = configmgr.create ksconf = args[0] - if configmgr.bootstrap['enable']: + if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() @@ -78,7 +78,6 @@ else: etc_prefix = os.path.join(prefix, 'etc') conffile = 'etc/mic.conf' -bsconf = 'etc/bootstrap.conf' if os.path.isfile('%s/mic/mic.conf' % etc_prefix): conffile += '.new' @@ -102,7 +101,7 @@ try: packages = PACKAGES, data_files = [("%s/lib/mic/plugins/imager" % prefix, IMAGER_PLUGINS), ("%s/lib/mic/plugins/backend" % prefix, BACKEND_PLUGINS), - ("%s/mic" % etc_prefix, [conffile, bsconf])] + ("%s/mic" % etc_prefix, [conffile])] ) finally: # remove dynamic file distfiles/mic.conf |