diff options
| author | SoonKyu Park <sk7.park@samsung.com> | 2016-09-23 10:35:49 +0900 |
|---|---|---|
| committer | SoonKyu Park <sk7.park@samsung.com> | 2016-09-23 10:35:49 +0900 |
| commit | c46ae15366c67ee9ecabd417f1773f454e724aa7 (patch) | |
| tree | 12329309b8067983625c63213f658a5ec8e05ba0 | |
| parent | 4915f9a5a1da5eaa20c58e798a2ac5de0ef7eeb0 (diff) | |
| parent | 4161678b3bf9d2eb36fdcb82c820bce02f6ba5bc (diff) | |
| download | mic-tizen_0.27.1.tar.gz mic-tizen_0.27.1.tar.bz2 mic-tizen_0.27.1.zip | |
Merge release-0.27.1 from 'tools/mic'submit/tizen/20160923.060613accepted/tizen/common/20160923.160804tizen_0.27.1
Change-Id: Ifba625f3902104e6a21e8340f5b292f0190057d7
79 files changed, 5796 insertions, 6158 deletions
diff --git a/.gbp.conf b/.gbp.conf new file mode 100644 index 0000000..ed62168 --- /dev/null +++ b/.gbp.conf @@ -0,0 +1,6 @@ +[DEFAULT] +# Vendor/Distro name +vendor=Tizen +# Subdir for RPM packaging data +packaging-dir=packaging +packaging-tag=%(upstreamversion)s @@ -1,22 +1,21 @@ +# Contributors of MIC -PRIMARY AUTHORS are: - - * Anas Nashif <anas.nashif@intel.com> - * Gui Chen <gui.chen@intel.com> - * Jian-feng Ding <jian-feng.ding@intel.com> - * Zhang Qiang <qiang.z.zhang@intel.com> - -ADDITIONAL CONTRIBUTORS include: - - * Artem Bityutskiy <artem.bityutskiy@intel.com> - * Ed Bartosh <eduard.bartosh@intel.com> - * Huanhuan Li <huanhuanx.li@intel.com> - * Huaxu Wan <huaxu.wan@intel.com> - * Huang Hao <hao.h.huang@intel.com> - * Marko Saukko <marko.saukko@cybercom.com> - * Markus Lehtonen <markus.lehtonen@linux.intel.com> - * Patrick McCarty <patrick.mccarty@linux.intel.com> - * Shuangquan Zhou <shuangquan.zhou@intel.com> - * jobol <jobol@nonadev.net> - * yanqingx.li <yanqingx.li@intel.com> + * Anas Nashif <anas.nashif@intel.com> + * Gui Chen <gui.chen@intel.com> + * Jian-feng Ding <jian-feng.ding@intel.com> + * Zhang Qiang <qiang.z.zhang@intel.com> + * Artem Bityutskiy <artem.bityutskiy@intel.com> + * Ed Bartosh <eduard.bartosh@intel.com> + * Huanhuan Li <huanhuanx.li@intel.com> + * Huaxu Wan <huaxu.wan@intel.com> + * Huang Hao <hao.h.huang@intel.com> + * Lukasz Stelmach <l.stelmach@samsung.com> + * Marko Saukko <marko.saukko@cybercom.com> + * Markus Lehtonen <markus.lehtonen@linux.intel.com> + * Patrick McCarty <patrick.mccarty@linux.intel.com> + * Shuangquan Zhou <shuangquan.zhou@intel.com> + * Sun Lihong <lihongx.sun@intel.com> + * jobol <jobol@nonadev.net> + * yanqingx.li <yanqingx.li@intel.com> + * Yeongil Jang <yg0577.jang@samsung.com> @@ -1,10 +1,130 @@ +Release 0.27.1 - Wed May 25 2016 - Jianzhong Fang <jz.fang@samsung.com> +===================================================================== + * new distribution support: Ubuntu 16.04, Fedora 23 + * add raw image format support + * bug fix: + - Remove BmapCreate and Filemap source code from MIC (#DEVT-151) + +Release 0.27 - Mon Mar 28 2016 - Jianzhong Fang <jz.fang@samsung.com> +===================================================================== + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, openSUSE 13.2 + * generate manifest file to describe image information + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum (#DEVT-224) + * drop mic-native support (#DEVT-248) + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support (#DEVT-243, #DEVT-263) + * use argparse module to parse the cmd line (#DEVT-52) + + + * bug fix: + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure + - fix qemu arm and arm64 issues + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply (#DEVT-254) + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive + +Release 0.24 - Tue Mar 11 2014 - Gui Chen <gui.chen@intel.com> +===================================================================== + * enhance to handle password with special characters + * change python-zypp require to python-zypp-tizen + * add --repo comand option for local build + * add --user and --password option for %repo directive of ks file + * clean up some codes relevant to EULA agreement + * add hostname showing in log + * bug fix: + - fix chroot failed by space in image not enough + - fix obsolete packages incorrect handling + - fix yum backend failed to cache packages + - fix bare ip in no_proxy not working + - fix repeated log showing with yum backend + - fix loop device latency timing + - fix zypp failed to download by changing 'cachedir' + - fix 'mkfs' not working caused by mic-bootstrap install failed + +Release 0.23 - Fri Dec 12 2013 - Gui Chen <gui.chen@intel.com> +===================================================================== + * new distribution support: Ubuntu 13.10 and OpenSUSE 13.1 + * split requirements to subpackage 'mic-native' to reduce mic's dependencies + * support arm64 architecture image creation in native mode + * new option '--interactive'/'--non-interactive' to enable/disable interaction + * new option '--uuid' for 'part' in ks file to set filesystem uuid + * export more variables related to installer framework for loop format + * bug fix: + - fix bootstrap handling if bootstrap package failed + - fix 'mapper_device' key error + - fix detailed error messages missing in mounting + - fix version comparing issue of urlgrabber in Fedora + +Release 0.22 - Thu Oct 24 2013 - Gui Chen <gui.chen@intel.com> +===================================================================== + * use __version__ variable instead of VERSION file + * refactor msger module to ulitize logging module + * refine error class module + * improve installation in virtualenv + * add bash completion support + * add zsh completion support + * export mapper device related to installer framework + * update BmapCreate to the latest version + * bug fix: + - fix customized plugin_dir not work in bootstrap + - fix packing process exit on Ubuntu + - fix loop device alloaction failed on openSUSE + - fix incorrect number showing during installing + - set owner of cacheidr/outdir to SUDO_USER + - correct project url in setup.py + - fix mic not work when mic.conf disappear + +Release 0.21 - Mon Aug 26 2013 - Gui Chen <gui.chen@intel.com> +===================================================================== + * new distribution support: Fedora 19 + * refactor chroot module to correct the logic + * add an alias for installerfw - installerfw_plugins + * remove fuser dependency to avoid some unmount issue + * enable proxy setting with authentication + * don't get proxy info from /etc/sysconfig/proxy ever + * kill processes inside chroot after post script running + * reload device mapper using 'dmsetup' utility + * bug fix: + - fix bootloader options omitted + - warn failed boot partition flags set + - fix wrong file descriptor issue + - fix some requires + +Release 0.20 - Mon Jul 01 2013 - Gui Chen <gui.chen@intel.com> +===================================================================== + * new distribution support: CentOS 6 + * drop image creation if checked packages not present in image + * introduce 'installerfw' command in kickstart to customize configuration + * improve output message of post scripts + * bug fix: + - fix rpm not support 'VCS' tag traceback + Release 0.19 - Thu May 16 2013 - Gui Chen <gui.chen@intel.com> ===================================================================== - - new distribution support: Ubuntu 13.04 and openSUSE 12.3 - - introduce '--part-type' to handle GPT partition - - copy bmap creation from bmap-tools - - update some depends and fix depends issue - - bug fix: + * new distribution support: Ubuntu 13.04 and openSUSE 12.3 + * introduce '--part-type' to handle GPT partition + * copy bmap creation from bmap-tools + * update some depends and fix depends issue + * bug fix: - fix bug autologinuser always set - fix symlink bind mount left issue - fix '/var/lock' non-existent throw traceback @@ -1,33 +1,28 @@ PYTHON ?= python -VERSION = $(shell cat VERSION) -TAGVER = $(shell cat VERSION | sed -e "s/\([0-9\.]*\).*/\1/") +VERSION = $(shell sed -ne 's/__version__\s*=\s*"\(.*\)"/\1/p ' mic/__init__.py) +TAGVER = $(shell git describe --abbrev=0 --tags) PKGNAME = mic -ifeq ($(VERSION), $(TAGVER)) - TAG = $(TAGVER) -else - TAG = "HEAD" -endif - - all: build build: $(PYTHON) setup.py build -dist-common: man - git archive --format=tar --prefix=$(PKGNAME)-$(TAGVER)/ $(TAG) | tar xpf - - git show $(TAG) --oneline | head -1 > $(PKGNAME)-$(TAGVER)/commit-id - rm -rf $(PKGNAME)-$(TAGVER)/tests +_archive: man + git archive --format=tar --prefix=$(PKGNAME)-$(VER)/ $(TAG) | tar xpf - + git show $(TAG) --oneline | head -1 > $(PKGNAME)-$(VER)/commit-id + rm -rf $(PKGNAME)-$(VER)/tests + tar zcpf $(PKGNAME)_$(VER).tar.gz $(PKGNAME)-$(VER) + rm -rf $(PKGNAME)-$(VER) -dist-bz2: dist-common - tar jcpf $(PKGNAME)-$(TAGVER).tar.bz2 $(PKGNAME)-$(TAGVER) - rm -rf $(PKGNAME)-$(TAGVER) +dist: VER=$(VERSION) +dist: TAG='HEAD' +dist: _archive -dist-gz: dist-common - tar zcpf $(PKGNAME)-$(TAGVER).tar.gz $(PKGNAME)-$(TAGVER) - rm -rf $(PKGNAME)-$(TAGVER) +release: VER=$(TAGVER) +release: TAG=$(TAGVER) +release: _archive man: rst2man doc/man.rst > doc/mic.1 @@ -39,14 +34,10 @@ develop: build $(PYTHON) setup.py develop test: - cd tests/ && $(PYTHON) suite.py + cd tests/ && $(PYTHON) suite.py clean: rm -f *.tar.gz - rm -f *.tar.bz2 - rm -f mic/__version__.* - rm -f tools/*.py[co] - rm -f mic.1 + rm -f doc/mic.1 rm -rf *.egg-info - rm -rf build/ - rm -rf dist/ + rm -rf build/ dist/ @@ -1,32 +1,44 @@ - mic - Mic Image Creator -======================== +MIC - Mic the Image Creator +=========================== Overview -------- -MIC means Image Creator and it's used to create images for Tizen. The tool offers three major functions: +MIC means "Mic the Image Creator" and it's used to create images for Tizen. +The tool offers three major functions: - image creation -- image conversion -- chrooting into an image +- image conversion bwtween two different formats +- chrooting into an image -The tool is derived mainly from MIC2, which is used to create MeeGo images. With great improvements on many features, it has become clear, friendly, and flexible. It provides a python plugin mechanism for developers, to expand image type or image options, and even to hook. +With the MIC tool, users can create different types of images for different +verticals, including live CD images, live USB images, raw images for KVM, +loop images for IVI platforms, and fs images for chrooting. Also, users can +work in a chroot environment, based on an existing live image using MIC's +enhanced chrooting. Besides, MIC enables transforming an image to another +image format, a very useful function for those sensitive to image format. -With the MIC tool, users can create different types of images for different verticals, including live CD images, live USB images, raw images for KVM, loop images for IVI platforms, and fs images for chrooting. Also, users can work in a chroot environment, based on an existing live image using MIC's enhanced chrooting. Besides, MIC enables transforming an image to another image format, a very useful function for those sensitive to image format. +It can also provide plugin mechanism for developers to expand image type or +image options, and even to hook. Resource -------- - * SITE: https://www.tizen.org/ * REPO: https://download.tizen.org/tools/ * DOCS: https://source.tizen.org/documentation/reference/mic-image-creator - * CODE: https://github.com/01org/mic + * CODE: https://review.tizen.org/gerrit/tools/mic + https://github.com/01org/mic * BUGS: https://bugs.tizen.org/jira * HELP: general@lists.tizen.org + License ------- -MIC is Open Source and is distributed under the GPLv2 License. Please see the COPYING file included with this software +MIC is Open Source and is distributed under the GPLv2 License. +Please see the COPYING file included with this software + Contacts -------- -Contact with us at JIRA: https://bugs.tizen.org/jira, or directly at github.com: https://github.com/01org/mic +When you found a bug, you can file this bug in our official bug tracker: +https://bugs.tizen.org/jira + diff --git a/VERSION b/VERSION deleted file mode 100644 index caa4836..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.19 diff --git a/debian/changelog b/debian/changelog index 33f263e..3924533 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,127 @@ +mic (0.27.1) unstable; urgency=low + * new distribution support: Ubuntu 16.04, Fedora 23 + * add raw image format support + * bug fix: + - Remove BmapCreate and Filemap source code from MIC (#DEVT-151) + + -- Jianzhong Fang <jz.fang@samsung.com> Wed, 25 May 2016 15:00:00 +0800 + +mic (0.27) unstable; urgency=low + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, openSUSE 13.2 + * generate manifest file to describe image information + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum (#DEVT-224) + * drop mic-native support (#DEVT-248) + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support (#DEVT-243, #DEVT-263) + * use argparse module to parse the cmd line (#DEVT-52) + * bug fix: + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure + - fix qemu arm and arm64 issues + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply (#DEVT-254) + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive + + -- Jianzhong Fang <jz.fang@samsung.com> Mon, 28 Mar 2016 19:50:35 +0800 + +mic (0.24) unstable; urgency=low + * enhance to handle password with special characters + * change python-zypp require to python-zypp-tizen + * add --repo comand option for local build + * add --user and --password option for %repo directive of ks file + * clean up some codes relevant to EULA agreement + * add hostname showing in log + * bug fix: + - fix chroot failed by space in image not enough + - fix obsolete packages incorrect handling + - fix yum backend failed to cache packages + - fix bare ip in no_proxy not working + - fix repeated log showing with yum backend + - fix loop device latency timing + - fix zypp failed to download by changing 'cachedir' + - fix 'mkfs' not working caused by mic-bootstrap install failed + + -- Gui Chen <gui.chen@intel.com> Tue, 11 Mar 2014 17:25:35 +0800 + +mic (0.23-1) unstable; urgency=low + * new distribution support: Ubuntu 13.10 and OpenSUSE 13.1 + * split requirements to subpackage 'mic-native' to reduce mic's dependencies + * support arm64 architecture image creation in native mode + * new option '--interactive'/'--non-interactive' to enable/disable interaction + * new option '--uuid' for 'part' in ks file to set filesystem uuid + * export more variables related to installer framework for loop format + * bug fix: + - fix bootstrap handling if bootstrap package failed + - fix 'mapper_device' key error + - fix detailed error messages missing in mounting + - fix version comparing issue of urlgrabber in Fedora + + -- Gui Chen <gui.chen@intel.com> Fri, 12 Dec 2013 17:25:35 +0800 + +mic (0.22-1) unstable; urgency=low + * use __version__ variable instead of VERSION file + * refactor msger module to ulitize logging module + * refine error class module + * improve installation in virtualenv + * add bash completion support + * add zsh completion support + * export mapper device related to installer framework + * update BmapCreate to the latest version + * bug fix: + - fix customized plugin_dir not work in bootstrap + - fix packing process exit on Ubuntu + - fix loop device alloaction failed on openSUSE + - fix incorrect number showing during installing + - set owner of cacheidr/outdir to SUDO_USER + - correct project url in setup.py + - fix mic not work when mic.conf disappear + + -- Gui Chen <gui.chen@intel.com> Thu, 24 Oct 2013 17:25:35 +0800 + +mic (0.21-1) unstable; urgency=low + * new distribution support: Fedora 19 + * refactor chroot module to correct the logic + * add an alias for installerfw - installerfw_plugins + * remove fuser dependency to avoid some unmount issue + * enable proxy setting with authentication + * don't get proxy info from /etc/sysconfig/proxy ever + * kill processes inside chroot after post script running + * bug fix: + - fix bootloader options omitted + - raise when incorrectly set partition flags 'legacy_boot' + - fix wrong file descriptor issue + - fix some requires + + -- Gui Chen <gui.chen@intel.com> Mon, 26 Aug 2013 17:25:35 +0800 + +mic (0.20-1) unstable; urgency=low + * new distribution support: CentOS 6 + * drop image creation if checked packages not present in image + * introduce 'installerfw' command in kickstart to customize configuration + * improve output message of post scripts + * bug fix: + - fix rpm not support 'VCS' tag traceback + + -- Gui Chen <gui.chen@intel.com> Mon, 01 Jul 2013 17:25:35 +0800 + mic (0.19-1) unstable; urgency=low * new distribution support: Ubuntu 13.04 and openSUSE 12.3 * introduce '--part-type' to handle GPT partition diff --git a/debian/control b/debian/control index 86be578..8d921f5 100644 --- a/debian/control +++ b/debian/control @@ -8,29 +8,13 @@ Homepage: http://www.tizen.org Package: mic Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, ${dist:Depends}, +Depends: ${misc:Depends}, ${python:Depends}, + rpm, python-rpm, - bzip2, - dmsetup, - dosfstools, - e2fsprogs (>= 1.41), - isomd5sum, - genisoimage, - kpartx, - parted, - psmisc, - squashfs-tools (>= 4.0), - yum (>= 3.2), - syslinux (>= 2:4.05), - extlinux (>= 2:4.05), - libzypp, - tizen-python-zypp-0.5.14, - python-m2crypto, python-urlgrabber, -Recommends: - binfmt-support, - btrfs-tools, - udisks | hal + cpio, + bzip2, + gzip Conflicts: mic2 Description: image creator for Linux distributions @@ -39,3 +23,4 @@ Description: image creator for Linux distributions is used to create images with different types; subcommand convert is used to convert an image to a specified type; subcommand chroot is used to chroot into an image. + diff --git a/debian/docs b/debian/docs index a1320b1..678115e 100644 --- a/debian/docs +++ b/debian/docs @@ -1 +1,2 @@ README.rst +doc/RELEASE_NOTES diff --git a/debian/mic.install b/debian/mic.install new file mode 100644 index 0000000..adadd7f --- /dev/null +++ b/debian/mic.install @@ -0,0 +1,4 @@ +debian/tmp/usr/bin/mic /usr/bin +debian/tmp/usr/lib/* /usr/lib +debian/tmp/usr/share/* /usr/share +debian/tmp/etc/* /etc diff --git a/debian/rules b/debian/rules index ebdca12..41a13a9 100755 --- a/debian/rules +++ b/debian/rules @@ -3,13 +3,6 @@ # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 -ifeq ($(shell cat /etc/issue | cut -d' ' -f2), 11.10) - SUBSTVARS = -Vdist:Depends="qemu-arm-static" -else - SUBSTVARS = -Vdist:Depends="qemu-user-static" -endif - - build: build-stamp build-stamp: dh_testdir @@ -31,9 +24,15 @@ install: build dh_installdirs # Installing package - mkdir -p $(CURDIR)/debian/mic $(CURDIR)/debian/mic/usr/bin $(CURDIR)/debian/mic/usr/share/man/man1 - install -m644 doc/mic.1 $(CURDIR)/debian/mic/usr/share/man/man1 - python setup.py install --root=$(CURDIR)/debian/mic + mkdir -p $(CURDIR)/debian/tmp/ + mkdir -p $(CURDIR)/debian/tmp/usr/bin + mkdir -p $(CURDIR)/debian/tmp/usr/share/man/man1 + mkdir -p $(CURDIR)/debian/tmp/etc/bash_completion.d + mkdir -p $(CURDIR)/debian/tmp/etc/zsh_completion.d + install -m644 doc/mic.1 $(CURDIR)/debian/tmp/usr/share/man/man1 + install -m755 etc/bash_completion.d/mic.sh $(CURDIR)/debian/tmp/etc/bash_completion.d/ + install -m755 etc/zsh_completion.d/_mic $(CURDIR)/debian/tmp/etc/zsh_completion.d/_mic + python setup.py install --root=$(CURDIR)/debian/tmp --prefix=/usr binary-indep: build install dh_testdir @@ -49,7 +48,7 @@ binary-indep: build install dh_pysupport dh_installdeb dh_shlibdeps - dh_gencontrol -- $(SUBSTVARS) + dh_gencontrol dh_md5sums dh_builddeb diff --git a/doc/FAQ.rst b/doc/FAQ.rst new file mode 100644 index 0000000..e52baa3 --- /dev/null +++ b/doc/FAQ.rst @@ -0,0 +1,58 @@ +FAQ +=== + +Q: When creating an image, MIC shows: + "Error <creator>: URLGrabber error: http://.../.../repos/oss/ia32/packages/repodata/repomd.xml" + +A: Perhaps your network has some issues, or your proxy doesn't work. + Try another proxy or find out the network issue. + +Q: MIC complains "ERROR: found 1 resolver problem, abort!" + +A: This is not an issue of MIC, it's caused by the repo you used. + Make sure the packages in the repo you used have proper dependencies. + +Q: I used '-A i586' to create an i586 image, but it showed + "nothing provided ....". What's wrong with it? + +A: Use '-A i686'. i586 is lower than i686, so many packages will be missing + from the installation. + +Q: Error shows: "uninstallable providers: somepackageA" + +A: It's caused by the missing package in the repo. To find it out, modify the + "%packages" section with only one item 'somepackageA' in kickstart file, + then you can root cause what's the missing dependency. + +Q: MIC shows in the log: + "file /usr/share/whatever conflicts between attempted installs of somepackageA and somepackageB" + +A: There are conflicts between some packages in the repo you used, but this + is not an issue with MIC. Please make sure you are using a proper repo. + +Q: Error shows: Command 'modprobe' is not available. + +A: In some distributions, when you use sudo, the PATH variable will be changed + and you will lose some important paths. Run 'export PATH=/sbin:$PATH' + before running MIC. + +Q: MIC lost some packages which are specified in '--includepkgs'/'--excludepkgs' + +A: Assume you want to include/exclude some packages in one repo, you will use + '--includepkgs'/'--excludepkgs' option in the according repo command line, + but you should list these packages to %packages section too, otherwise they + will not take any effect. + +Q: How does mic select packages? And how to set the priority of a repo? + +A: In general, mic will select a higher version if two or more available in + all repos, if the version is the same, a higher release number is + prefferred. But if you assign a priority to one repo, mic will prefer to + select packages from the repo with higher priority, even in case a higher + version is available in the repo with a lower priority. Actually the + default priority for a repo is 99, the range of a repo priority is 1~99, + the larger number has the lower priority. + An example is given: + "repo --name=base --baseurl=http://whateverurl --prioirity=1" + + diff --git a/doc/KNOWN_ISSUES b/doc/KNOWN_ISSUES index 1f58731..cf53eac 100644 --- a/doc/KNOWN_ISSUES +++ b/doc/KNOWN_ISSUES @@ -1,11 +1,13 @@ Known Issues ============ -Bug of latest "syslinux" package --------------------------------- -In some new Linux distributions, the "syslinux" package in their official -software repositories is the version 4.04. It will cause segment fault for -a fatal bug, and mic will failed with syslinux installation errors. +ARM64 support +------------- +Currently mic only supports aarch64 (ARM64) in native running mode because +of current enabling status in Tizen repository. And the following packages +need to be upgraded by using Tizen-tools repository: + +* libzypp +* python-zypp +* qemu-arm-static -The solution is to install the patched "syslinux" package in MeeGo or Tizen's -tools repos, until the official released one being fixed. diff --git a/doc/RELEASE_NOTES b/doc/RELEASE_NOTES index 8c9c8f0..c6e1bcb 100644 --- a/doc/RELEASE_NOTES +++ b/doc/RELEASE_NOTES @@ -1,3 +1,400 @@ +MIC Image Creator 0.27.1 Release Notes +====================================== +Released May 25 2016 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Enhancements +-------------------------- + * new distribution support: Ubuntu 16.04, Fedora 23 + * add raw image format support + +Bug Fixes +--------- + * Remove BmapCreate and Filemap source code from MIC (#DEVT-151) + +MIC Image Creator 0.27 Release Notes +====================================== +Released Mar 28 2016 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Enhancements +-------------------------- + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, openSUSE 13.2 + * generate manifest file to describe image information + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum (#DEVT-224) + * drop mic-native support (#DEVT-248) + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support (#DEVT-243, #DEVT-263) + * use argparse module to parse the cmd line (#DEVT-52) + +Bug Fixes +--------- + * fix logfile incomplete in release option + * fix config file disappear in bootstrap + * fix aarch64 bin_format + * fix pylint + * fix real path of device mapper causing initrd failure + * fix qemu arm and arm64 issues + * fix AttributeError in zypp backend + * fix 'python-xml' depends used by cElements + * fix xml requirements + * fix logfile not in release of '--release' when creation failed + * fix copyright missing issue + * fix syslinux installation path issue in Arch Linux + * fix priority option of ks file not apply (#DEVT-254) + * fix need to check loop device after excute 'losetup --find' + * fix check scriptlet error file on /tmp/.postscript/error/ + * fix broken tar archive + +MIC Image Creator 0.26 Release Notes +====================================== +Released Oct 14 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * add new arch MIPS support + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum + * enhance VCS info in yum backend + +Bug Fixes +--------- + * fix real path of device mapper causing initrd failure + * fix qemu arm and arm64 issues + * fix AttributeError in zypp backend + * fix 'python-xml' depends used by cElements + +MIC Image Creator 0.25.2 Release Notes +====================================== +Released Jun 11 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * fix AttributeError in zypp backend + +MIC Image Creator 0.25.1 Release Notes +====================================== +Released May 28 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * revert bind mount config file to instroot + * fix xml requirements + +MIC Image Creator 0.25 Release Notes +==================================== +Released May 23 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * new distribution support: Ubuntu 14.04 and Debian 7.0 + * generate manifest file to describe image information + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * support lzop compress + +Bug Fixes +--------- + * fix logfile incomplete in release option + * fix config file disappear in bootstrap + * fix aarch64 bin_format + * fix pylint + +MIC Image Creator 0.24.2 Release Notes +==================================== +Released June 09 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * enable ssl_verify in config file + +Bug Fixes +--------- + * fix logfile incompleted issue + * fix md5sum to be compatible with utility md5sum + * fix locale issue in systemd service + +MIC Image Creator 0.24 Release Notes +==================================== +Released March 11 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * enhance to handle password with special characters + * change python-zypp require to python-zypp-tizen + * add --repo comand option for local build + * add --user and --password option for %repo directive of ks file + * clean up some codes relevant to EULA agreement + * add hostname showing in log + +Bug Fixes +--------- + * fix chroot failed by space in image not enough + * fix obsolete packages incorrect handling + * fix yum backend failed to cache packages + * fix bare ip in no_proxy not working + * fix repeated log showing with yum backend + * fix loop device latency timing + * fix zypp failed to download by changing 'cachedir' + * fix 'mkfs' not working caused by mic-bootstrap install failed + +MIC Image Creator 0.23.1 Release Notes +==================================== +Released January 23 2014 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * fix bmap creation failed if using tmpfs + +MIC Image Creator 0.23 Release Notes +==================================== +Released December 18 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * support arm64 architecture image creation in native mode + (to know more about arm64 support, check doc/KNOWN_ISSUES) + * new distribution support: Ubuntu 13.10 and OpenSUSE 13.1 + * split the "native" running mode support to a separated sub-package + * reduced the dependencies(packages) of mic main package dramatically + * add new options '--interactive' and '--non-interactive' to enable/disable interaction + * add new option '--uuid' for 'part' in ks file to set filesystem uuid + * export more environment variables related to installer framework for loop format + +Bug Fixes +--------- + * fix bootstrap handling if bootstrap package failed + * fix 'mapper_device' key error + * fix detailed error messages missing in mounting + * fix version comparing issue of urlgrabber in Fedora + + MIC Image Creator 0.22.3 Release Notes +=========================================================== +Released Nov 19 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * export mapper device for installer framework + * update BmapCreate to 2.0 + * enable internal setarch in bootstrap environment + +Bug Fixes +--------- + + MIC Image Creator 0.22.2 Release Notes +=========================================================== +Released Nov 18 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * export mapper device for installer framework + * update BmapCreate to 2.0 + +Bug Fixes +--------- + + MIC Image Creator 0.22 Release Notes +=========================================================== + +Released October 24 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * refactor msger module to ulitize logging module + * refine error class module + * improve installation in virtualenv + * add bash completion support + * add zsh completion support + * export mapper device related to installer framework + * update BmapCreate to the latest version + +Bug Fixes +--------- + - fix customized plugin_dir not work in bootstrap + - fix packing process exit on Ubuntu + - fix loop device alloaction failed on openSUSE + - fix incorrect number showing during installing + - set owner of cacheidr/outdir to SUDO_USER + - correct project url in setup.py + - fix mic not work when mic.conf disappear + + MIC Image Creator 0.21.2 Release Notes +=========================================================== + +Released September 28 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * use sync mode in kpartx command + + MIC Image Creator 0.21.1 Release Notes +=========================================================== + +Released September 18 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * prefer to use 'pbzip2'/'pigz' to compress image if possible + * won't fallback to native mode if bootstrap not found + +Bug Fixes +--------- + * fix exiting at packaging images on Ubuntu distro + + MIC Image Creator 0.21 Release Notes +=========================================================== + +Released August 28 2013 + +This release note documents the changes included in the new release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * new distribution support: Fedora 19 + * refactor part of chroot modules for better cleanup handling + * add an alias "installerfw_plugins" for installerfw + * remove unnecessary fuser dependency for "fuser" command + * enable proxy with user authentication setting + * correct no_proxy handling in openSUSE + * kill processes inside chroot after post script running + * ulitize 'dmsetup' to avoid possible dm device unaccessible issue + +Bug Fixes +--------- + * fix bootloader options omitted + * warn failed boot partition flags set + * fix wrong file descriptor issue + * fix some requires + + + MIC Image Creator 0.20 Release Notes +=========================================================== +Released Jule 08 2013 + +This release note documents the changes included in the MIC 0.20 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * new distribution support: CentOS 6 + * drop image creation if checked packages not present in image + * introduce 'installerfw' command in kickstart to customize configuration + * improve output message of post scripts + +Bug Fixes +--------- + * fix rpm not support 'VCS' tag traceback + + MIC Image Creator 0.19.3 Release Notes +=========================================================== +Released July 01 2013 + +This release note documents the changes included in the MIC 0.19.3 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * fix vfat UUID issue in liveusb + + MIC Image Creator 0.19.2 Release Notes +=========================================================== +Released July 01 2013 + +This release note documents the changes included in the MIC 0.19.2 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * introduce 'installerfw' command in kickstart + * export several environment to keep image information + +Bug Fixes +--------- + + MIC Image Creator 0.19.1 Release Notes +=========================================================== +Released June 21 2013 + +This release note documents the changes included in the MIC 0.19.1 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + +Bug Fixes +--------- + * fall back to old arch detecting + MIC Image Creator 0.19 Release Notes =========================================================== Released May 16 2013 @@ -18,12 +415,332 @@ Bug Fixes * fix symlink bind mount left issue * fix '/var/lock' non-existent throw traceback + MIC Image Creator 0.18 Release Notes +=========================================================== +Released Apr 03 2013 + +This release note documents the changes included in the MIC 0.18 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * put build_id before image name for release option + * mount build directory as tmpfs to speed up + * enable --priority in ks to set priority + * upgrade qemu (mic's depends) to 1.4.0 + +Bug Fixes +--------- + * fix debuginfo rpm swig attribute lost + * fix release option failure with slash + * fix man page lost in some distros + * fix bmap file packed to tarball + +Code Cleanup +------------ + * unify import statements to absolute import + * clean up many undefined in partitionfs.py/loop.py/livecd.py + * clean up some useless try and raise blocks + * clean up some bad indentations + * improve some error messages + + MIC Image Creator 0.17 Release Notes +=========================================================== +Released Feb 28 2013 + +This release note documents the changes included in the MIC 0.17 release. And +the release contains new features, enhancements and bug fixes. + +New Features & Ehancements +-------------------------- + * support new distribution Fedora 18 + * enable to handle more than 3 partitions + * support partition without mount point + * move vcs infomation from .vcs to .packages + +Bug Fixes +--------- + * fix failure when symbolic link vmlinuz in single disk + * fix incorrect alignment handling + * fix requires squashfs in opensuse + * fix unicode error in myurlgrab + +Code Cleanup +------------ + * clean up some mess in utils/misc.py + * clean up pylint issue in creator.py + +Release 0.16 - Wed Jan 30 2013 +=========================================================== + * add GPT support for UEFI format + - add --ptable=gpt option in kickstart to enable GPT + - add simple GPT parser to parse PARTUUID + - use PARTUUID in syslinux.cfg if gpt required + - clean up the mess code and improve readability + * improve alignment to reduce unused space + - add interface add_disks for PartitionedMount + - clean up the mess code in partitionfs.py + * append timestamp suffix to image name by default + * add a new option --nocache in kickstart to disable caching rpm + * add a new value 'vcs' for --record-pkgs to save VCS info + * add a new option --fstab-entry for raw image to set fstab entry + * introduce a new option '--install-pkgs' in mic + - valid values imagined: source, debuginfo, debugsource + - enable 'debuginfo' to install '-debuginfo' rpm in the meantime + * improve loop device generating and cleaning mechanism + * code clean: + - update documentation + - rename MANIFEST to MD5SUM + - clean up moblin stuff in kickstart + - clean mic tests and keep only unittest + - sepreate the download related as independent module + * bug fix: + - cleanup zypp credential file to fix conflicts during installing + - fix the issue that outdir/cachedir is not dir + - recalculate package content recording + - refactor try except statement in baseimager + - fix existing loop images overwritten + +Release 0.15.3 - Wed Jan 23 2013 +=========================================================== + * urgent bug fix: + - fix loop device not cleaned issue + - fix bootstrap dirs not unmounted issue + - fix mic failed in kvmic issue + - fix binfmt register in bootstrap incorrect + - fix rpm path incorrect when local repo and sslverify used + * other bug fix: + - fix x86_64 image conflicts installing x86_64 and x86 rpm + - verify if logfile is a file + - fix type error when calling mknod + - fix the failure if ks under / dir + - clean up the mess 'directory not empty' + - fix type error when calling mknod + +Release 0.15 - Tue Dec 13 2012 +=========================================================== + * adapt new mechanism for bootstrap mode + - create 'mic-bootstrap-x86-arm' by obs build + - publish 'mic-bootstrap-x86-arm' into server repo + - use 'mic-bootstrap-x86-arm' as bootstrap env for x86 and arm image + * filesystem parameter support for ext[234] fs + - sample in ks: part / --size 1000 --fstype=ext3 --extoptions="-I 256" + - other filesystem will ignore option '--extoptions' + * update the documentation and man page + * multiple bootstrap path to support multi-instance mic + * use 'pkgmgr=auto' to select available backend in conf file + * improve loop device creation algorithm and create loop device by 'mknod' + * bug fix: + - bear unexpected checksum type when getting metadata + - avoid traceback when loopback is NoneType + - lseek limit to 2G in 32bit env + - split out username and password in zypp repo file + - use rpm real path instead of 'cp' if it's local repo + - fix local repo unavailable in bootstrap + - fix traceback when failed to unmap kpartx device + - fix timestamp incorrect issue in logfile + +Release 0.14.2 - Wed Nov 14 2012 +=========================================================== + * support dracut for live image + * update bmap version to 1.1 + +Release 0.14.1 - Fri Oct 15 2012 +=========================================================== + * support bmap file for ivi flashing tool + * just warning in chroot when not Tizen/MeeGo chroot dir + * fix logfile lost in bootstrap mode + * clean mounts in bootstrap when exiting + * bug fix: + - fix https proxy issue in yum backend + - avoid traceback when loop instance is NoneType + +Release 0.14 - Thu Aug 02 2012 +=========================================================== + * use cached metadata when checksum is not changed + * skip non-fatal error in ks file and prompt user to handle + * prompt user to handle when failed to apply img configure + * replace hard name with device uuid in etc/fstab + * enhance extlinux cfg file for symbolic kernel like IVI + * support label assign for raw image + * bug fix: + - fix live image create failure when label assigned + - avoid traceback when converting unsupported type + - fix mic --version ugly output + +Release 0.13 - Wed Jul 12 2012 +=========================================================== + * create logfile as default when --release specifid + * use 'gzip' and 'bzip2' to pack image instead of python + * automatically detect path of 'env' for chroot + * record version and os info in build log and logfile + * bug fix: + - fix popup message in ubuntus + - fix unicode issue for logfile + - better fix for 'chroot raw' issue + +Release 0.12 - Wed Jun 20 2012 +=========================================================== + * use default value when @BUILD_ID@ and @ARCH@ not specified + * enhance proxy support in attachment retrieve + * add new --shrink opt for loop image to control img shrinking + * avoid invalid literal for loop device generation + * relocate and refactor selinux_check func + * remove prefix for make install + * bug fix: + - fix compres image in raw image + - fix src pkgs download failed issue + - fix convert failed issue + +Release 0.11 - Fri Jun 08 2012 +=========================================================== + * support new subcmd 'auto' to handle magic line in ks + * enhance the handle of authentication url and https proxy + * support packing images together and support compressed file format + * reset LD_PRELOAD for chroot env + * centralized interface to check existing images + * avoid live image creating when using multi-partitions + * resolve the depends of python-urlgrabber + * bug fix: + - fix logfile context lost issue + - fix attachment package url handling + - fix mic ch raw failed issue + +Release 0.10 - Tue May 15 2012 +=========================================================== + * container support using '%attachment' section in ks + * add --compress-to option to support zip format in loop image + * auto-detect config and plugindir to meet virtualenv and customized install + * remove all hardcoded info in setup.py and use sys.prefix for installing + * tolerate some OS errors in the image configurations stage + * extra patch: + - fix zypp missing password when using username passwd + - some fixes to enhance authentication url + - refine repostr structure to fix comma issue in baseurl + +Release 0.9 - Fri Apr 13 2012 +=========================================================== + * support pre-install package with zypp backend + * sync /etc/mic/mic.conf to bootstrap + * enhance sorting for version comparsion in zypp + * rewrite chroot tar image using xml format mount point file + * fix the incorrect number showing in fs src pkgs download + * remove tests directory for dist in Makefile + * fix liveusb parted mkpart failure, revert mbr size expand in raw + * cleanup /tmp/repolic* dir in the EULA checking + +Release 0.8 - Mon Mar 26 2012 +=========================================================== + * partition alignment support + * remove bootloader option 'quiet vga' for raw + * update dist files in git source + * update unittest, add cases for chroot, msger, runner + * add 40 system test case for help + * rewrite loop device allocation mechanism + +Release 0.7 - Fri Mar 02 2012 +=========================================================== + * zypp backend: fixed a fatal issue of unreleasable loop devs + * zypp backend: more friendly output message + * backend: share cached rpm files between yum and zypp + * enhancement for multiple partition loop format + * make msger to accept Unicode string + * fixed a regression of compress option for FS format + * fixed issues in openSUSE12.1 + * new written man page + +Release 0.6 - Thu Feb 16 2012 +=========================================================== + * give hint when converted image existed + * conf.py: proxy scheme check + * space check before copy image + * zypp: abort with error msg for repo resolver issues + * runner.py refinement + * ks file syntax check for '%post' without '%end' + * support more compression formats than only bzip2 + * fix msg NoneType issue, causing exit after install + * bootstrap: + - catch creator error when retrieving bootstrap metadata + - correct matching .metadata file in bootstrap + +Release 0.5 - Mon Feb 06 2012 +=========================================================== + * Rewrite the algorithm of checking free space for download and install + * Add --shell option for convert to recreate image modified by internal shell + * Add -s option for chroot to unpack image + * Introduce --copy-kernel option for creator + * Remove the hardcoded default args for bootloader + * Disable logstderr and flush message buffer in disable_logstderr + * Deal with yum.conf inside yum backend by itself + * Bug fix: + - Fix rpmdb error in yum and zypp to avoid bad file descriptor message + - Fix MANIFEST syntax to be compliant with md5sum + - Correct dependencies for mic in bootstrap + +Release 0.4 - Fri Jan 06 2012 +=========================================================== + * Support bootstrap mode, run with '--runtime=bootstrap' + * Full support for taring-to output, use 'mic ch x.tar' + * Break dependency between backend and baseimage + * Check valid repos in ks file + * Space check update and catch no space exception + * Fix no prompt when cv and ch no existed image + * Fix NoneType 'createopts' when convert + * Fix no existed local_pkgs_path + +Release 0.3 - Mon Dec 26 2011 +=========================================================== + * Unit test support, run 'make test' + * Enable proxy support in config file + * Refine configmgr and pluginmgr + * Support multi instance with different cache dir + * Add 47 system test case + * Improve md5sum generation + * Add repo option --ssl_verify + * Add option --name_prefix + * Reformatted code according to PEP08 + * Backport from mic2: + - Add priority and cost option for repos + - Reinstroduced compress-disk-image option + +Release 0.2 - Tue Nov 29 2011 +=========================================================== + * Support btrfs and ext4 fstype for creator, convertor, and chroot + * Append distfiles and Makefile + * Check arch type from repo data + * Set rpm dbpath to fix 'rpm -qa' issue + * Fix chroot issue caused by image size + * Improve setup.py and make it compatible with python 2.5 + * Disable ca check for https + * Change default output dir name to ./mic-output + * untrack mic/__version__.py + * Fix some minor issues + +Release 0.1 - Thu Oct 27 2011 +=========================================================== + * Support three subcommand: create, convert, chroot + * Support five image types: fs, loop, raw, livecd, liveusb + * Support two package manager backend: yum and zypp + * Support the following global command line options: + - --verbose + - --debug + * Creator subcommand support the following command line options: + - --logfile=LOGFILE + - -c CONFIG, --config=CONFIG + - -k CACHEDIR, --cachedir=CACHEDIR + - -o OUTDIR, --outdir=OUTDIR + - -A ARCH, --arch=ARCH + - --release=RID + - --record-pkgs=RECORD_PKGS + - --pkgmgr=PKGMGR + - --local-pkgs-path=LOCAL_PKGS_PATH + Resource -------- * SITE: https://www.tizen.org/ * REPO: https://download.tizen.org/tools/ * DOCS: https://source.tizen.org/documentation/reference/mic-image-creator - * CODE: https://github.com/01org/mic + * CODE: https://review.tizen.org/git/tools/mic * BUGS: https://bugs.tizen.org/jira * HELP: general@lists.tizen.org @@ -31,7 +748,4 @@ Report Bugs ----------- when you found a bug, you can file this bug in our official bug tracker: https://bugs.tizen.org/jira -also you are welcome to file the issue at our github repository: -https://github.com/01org/mic - Thank you for using MIC and for taking the time to send us your feedback! diff --git a/doc/faq.rst b/doc/faq.rst deleted file mode 100644 index aef97c3..0000000 --- a/doc/faq.rst +++ /dev/null @@ -1,24 +0,0 @@ -FAQ -=== - -Q: When creating an image, MIC shows "Error <creator>: URLGrabber error: http://www.example.com/latest/repos/oss/ia32/packages/repodata/repomd.xml" - -A: Perhaps your network has some issues, or your proxy doesn't work. Try another proxy or find out the network issue. - -Q: MIC complains "Error <repo>: found 1 resolver problem, abort!" - -A: This is not an issue with MIC, but with the repo you used. Make sure the packages in the repo you used have proper dependencies. - -Q: I used '-A i586' to create an i586 image, but it showed "nothing provided ....". What's wrong with it? - -A: Use '-A i686'. i586 is lower than i686, so many packages will be missing from the installation. - -Q: MIC shows in the log: "file /usr/share/whatever conflicts between attempted installs of somepackageA and somepackageB" - -A: There are conflicts between some packages in the repo you used, but this is not an issue with MIC. Please make sure you are using a proper repo. - -Q: Error shows: Command 'modprobe' is not available in Fedora 17. - -A: In Fedora 17, when you use sudo, the PATH variable will be changed and you will lose some important paths. Run 'export PATH=/sbin:$PATH' before running MIC. - - diff --git a/doc/install.rst b/doc/install.rst index fef0fb7..bc5d668 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -10,6 +10,7 @@ Many mainstream Linux distributions are supported in Tizen Tools repository, inc - Ubuntu (12.10, 12.04, 11.10) - openSUSE (12.2, 12.1) - Feodra (17, 16) +- Debian (7.0) Ubuntu Installation ------------------- diff --git a/doc/man.rst b/doc/man.rst index 00d208e..aeba63d 100644 --- a/doc/man.rst +++ b/doc/man.rst @@ -5,36 +5,32 @@ SYNOPSIS ======== -| mic create SUBCOMMAND <ksfile> [OPTION] -| mic chroot [OPTION] <imgfile> -| mic convert [OPTION] <imgfile> <format> +| mic [GLOBAL-OPTS] create SUBCOMMAND <ksfile> [OPTION] +| mic [GLOBAL-OPTS] chroot [OPTION] <imgfile> DESCRIPTION =========== The tools `mic` is used to create and manipulate images for Linux distributions. -It is composed of three subcommand: `create`, `convert`, `chroot`. +It is composed of two subcommand: `create`, `chroot`. USAGE ===== create ------ -This command is used to create various images, including live CD, live USB, -loop, raw. +This command is used to create various images, including loop. Usage: - - | mic create(cr) SUBCOMMAND <ksfile> [OPTION] + | mic [GLOBAL-OPTS] create(cr) SUBCOMMAND <ksfile> [OPTION] Subcommands: - | help(?) give detailed help on a specific sub-command + | auto auto detect image type from magic header | fs create fs image, which is also chroot directory - | livecd create live CD image, used for CD booting - | liveusb create live USB image, used for USB booting | loop create loop image, including multi-partitions | raw create raw image, containing multi-partitions + | qcow create qcow image Options: @@ -50,23 +46,29 @@ Options: --pack-to=PACK_TO pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used --release=RID generate a release of RID with all necessary files, when @BUILD_ID@ is contained in kickstart file, it will be replaced by RID. sample values: "latest", "tizen_20120101.1" --copy-kernel copy kernel files from image /boot directory to the image output directory + --runtime=RUNTIME Specify runtime mode, avaiable: bootstrap + --install-pkgs=INSTALL_PKGS Specify what type of packages to be installed, valid: source, debuginfo, debugsource + --check-pkgs=CHECK_PKGS Check if given packages would be installed, packages should be separated by comma + --tmpfs Setup tmpdir as tmpfs to accelerate, experimental feature, use it if you have more than 4G memory + --strict-mode Abort creation of image, if there are some errors during rpm installation Options for fs image: --include-src generate a image with source rpms included; to enable it, user should specify the source repo in the ks file Options for loop image: --shrink whether to shrink loop images to minimal size - --compress-image=COMPRESS_IMAGE compress all loop images with 'gz' or 'bz2' + --compress-image=COMPRESS_IMAGE compress all loop images with 'gz' or 'bz2' or 'lzo' --compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image Options for raw image: --compress-image=COMPRESS_IMAGE compress all raw images with 'gz' or 'bz2' --compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image + --generate-bmap=GENERATE_BMAP also generate the block map file + --fstab-entry=FSTAB_ENTRY Set fstab entry, 'name' means using device names, 'uuid' means using filesystem uuid Examples: | mic create loop tizen.ks - | mic create livecd tizen.ks --release=latest | mic cr fs tizen.ks --local-pkgs-path=localrpm chroot @@ -85,34 +87,13 @@ Options: Examples: | mic chroot loop.img - | mic chroot tizen.iso - | mic ch -s tizenfs tizen.usbimg - -convert -------- -This command is used for converting an image to another format. - -Usage: - - | mic convert(cv) <imagefile> <destformat> - -Options: - - -h, --help show the help message - -S, --shell launch interactive shell before packing the new image in the converting - -Examples: - - | mic convert tizen.iso liveusb - | mic convert tizen.usbimg livecd - | mic cv --shell tizen.iso liveusb Advanced Usage ============== The advanced usage is just for bootstrap, please skip it if you don't care about it. The major purpose to use bootstrap is that some important packages (like rpm) are customized -a lot in the repo which you want to create image, and mic must use the customized rpm to +a lot in the repo which you want to create image, and mic must use the customized rpm to create images, or the images can't be boot. So mic will create a bootstrap using the repo in the ks file at first, then create the image via chrooting, which can make mic using the chroot environment with the customized rpm. @@ -147,13 +128,17 @@ tools repos, until the official released one being fixed. Failed to create btrfs image in openSUSE ---------------------------------------- -When creating btrfs image in openSUSE, it would hang up with showing image kernel -panic. This issue impact all openSUSE distributions: 12.1, 11.4, 11.3, etc +When creating btrfs image in openSUSE, it would hang up with showing image kernel +panic. This issue impact all openSUSE distributions: 12.1, 11.4, 11.3, etc REPORTING BUGS ============== -The source code is tracked in github.com: +The source code is tracked in review.tizen.org: + + https://review.tizen.org/git/tools/mic + +The bug is registered in tizen.org: - https://github.com/jfding/mic + https://bugs.tizen.org/jira Please report issues for bugs or feature requests. diff --git a/doc/usage.rst b/doc/usage.rst index 803dbe2..1dda0a0 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -8,7 +8,6 @@ Overview MIC offers three major functions: - creating an image with different format -- converting an image to another format - chrooting into an image Getting help @@ -25,9 +24,7 @@ How to get help: * mic --help * mic create --help - * mic help create * mic create loop --help - * mic create help loop Image formulation support ------------------------- @@ -44,15 +41,13 @@ Image formulation support * Including partition table and all the partitions * The image is bootable directly -- Livecd/liveusb - - * Mainly used for an ia32 build, it can be burned to CD or usbstick, which can be booted into a live system or installation UI - - fs * “fs” means file-system * mic can install all the Tizen files to the specified directory, which can be used directly as chroot env +- qcow + Create ------ @@ -60,25 +55,24 @@ Create :: - mic create(cr) SUBCOMMAND <ksfile> [OPTION] + mic [GLOBAL-OPTS] create(cr) SUBCOMMAND <ksfile> [OPTION] - Sub-commands, to specify image format, include: :: - help(?) give detailed help on a specific sub-command + auto auto detect image type from magic header fs create fs image, which is also a chroot directory - livecd create live CD image, used for CD booting - liveusb create live USB image, used for USB booting loop create loop image, including multi-partitions raw create raw image, containing multi-partitions + qcow create qcow image - <ksfile>: The kickstart file is a simple text file, containing a list of items about image partition, setup, Bootloader, packages to be installed, etc, each identified by a keyword. -In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/daily/trunk/ivi/latest/images/ivi-min... +In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/weekly/tizen/mobile/latest/images/... - Options include: @@ -107,27 +101,40 @@ In Tizen, the released image will have a ks file along with image. For example, --pack-to=PACK_TO Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used + --runtime=RUNTIME_MODE + Sets runtime mode, the default is bootstrap mode, valid + values: "bootstrap". "bootstrap" means mic uses one + tizen chroot environment to create image. --copy-kernel Copy kernel files from image /boot directory to the image output directory. - + --install-pkgs INSTALL_PKGS Specify what type of packages to be + installed, valid: source, debuginfo, debugsource + --check-pkgs=CHECK_PKGS + Check if given packages would be installed, + packages should be separated by comma + --tmpfs Setup tmpdir as tmpfs to accelerate, experimental feature, + use it if you have more than 4G memory + --strict-mode Abort creation of image, if there are some errors + during rpm installation + - Other options: -:: +:: - --runtime=RUNTIME_MODE - Sets runtime mode, the default is bootstrap mode, valid - values: "native", "bootstrap". "native" means mic uses - localhost environment to create image, while "bootstrap" - means mic uses one tizen chroot environment to create image. - --compress-image=COMPRESS_IMAGE (for loop & raw) + --compress-image=COMPRESS_IMAGE (for loop & raw) Sets the disk image compression. Note: The available values might depend on the used filesystem type. --compress-disk-image=COMPRESS_IMAGE Same with --compress-image --shrink (for loop) Whether to shrink loop images to minimal size + --include-src (for fs) + Generate a image with source rpms included --generate-bmap (for raw) Generate the block map file + --fstab-entry (for raw) + Set fstab entry, 'name' means using device names, + 'uuid' means using filesystem uuid - Examples: @@ -158,34 +165,6 @@ This command is used to chroot inside the image. It's a great enhancement of the :: mic ch loop.img - mic ch tizen.iso - mic ch -s tizenfs tizen.usbimg - -Convert -------- -This command is used for converting an image to another format. - - -- Usage: - -:: - - mic convert(cv) <imagefile> <destformat> - -- Options: - -:: - - -h, --help Show this help message and exit - -S, --shell Launch shell before packaging the converted image - -- Examples: - -:: - - mic cv tizen.iso liveusb - mic cv tizen.usbimg livecd - mic cv --shell tizen.iso liveusb Getting Start ------------- diff --git a/etc/bash_completion.d/mic.sh b/etc/bash_completion.d/mic.sh new file mode 100644 index 0000000..1cd19c9 --- /dev/null +++ b/etc/bash_completion.d/mic.sh @@ -0,0 +1,306 @@ +# bash completion for mic +# +# Copyright (c) 2013 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +__miccomp_1 () +{ + local c IFS=$' \t\n' + for c in $1; do + c="$c$2" + case $c in + --*=*|*.) ;; + *) c="$c " ;; + esac + printf '%s\n' "$c" + done +} + +__miccomp () +{ + local cur_="${3-$cur}" + + case "$cur_" in + --*=) + COMPREPLY=() + ;; + *) + local IFS=$'\n' + COMPREPLY=($(compgen -P "${2-}" \ + -W "$(__miccomp_1 "${1-}" "${4-}")" \ + -- "$cur_")) + ;; + esac +} + +__mic_find_on_cmdline () +{ + local word subcommand c=1 + while [ $c -lt $cword ]; do + word="${words[c]}" + for subcommand in $1; do + if [ "$subcommand" = "$word" ]; then + echo "$subcommand" + return + fi + done + let c++ + done +} + +__mic_expand_alias () +{ + case ${subcommand} in + cr*) subcommand="create" ;; + ch*) subcommand="chroot" ;; + c*v*) subcommand="convert" ;; + esac +} + +__mic_complete_opt() +{ + fs_exts=" + --include-src + " + loop_exts=" + --shrink + --compress-image= + --compress-disk-image= + " + raw_exts=" + --fstab-entry= + --generate-bmap + --compress-image= + --compress-disk-image= + " + + keyword="$(__mic_find_on_cmdline "fs loop raw")" + eval extensions="$"{${keyword}_exts} + + __miccomp "${options} ${extensions}" +} + +__mic_complete_val() +{ + _values=" + " + arch_values=" + ia64 + i686 + i586 + x86_64 + armv5l + armv6l + armv7l + armv7hl + armv7thl + armv7nhl + armv5tel + armv5tejl + armv7tnhl + " + pkgmgr_values=" + yum + zypper + " + release_values=" + latest + " + runtime_values=" + bootstrap + " + record_pkgs_values=" + vcs + name + content + license + " + fstab_entry_values=" + name + uuid + " + install_pkgs_values=" + source + debuginfo + debugsoure + " + compress_image_values=" + gz + bz2 + " + compress_disk_image_values=" + gz + bz2 + " + + declare -F _split_longopt &>/dev/null && _split_longopt + + prev_=${prev##--} + eval values="$"{${prev_//-/_}_values} + __miccomp "${values}" +} + +__mic_complete_arg() +{ + local before c=$((cword-1)) + while [[ $c -gt 0 ]]; do + before=${words[c]} && [[ "$before" != -* ]] && break; + let c-- + done + + case ${subcommand},${before} in + cr*,cr*) + __miccomp "${arguments}" + return 0 + ;; + c*v*,fs|loop|raw|livecd|liveusb) + __miccomp "${arguments}" + return 0 + ;; + help,help) + __miccomp "${arguments}" + return 0 + ;; + esac + + return 1 +} + +__mic () +{ + if [ -z "$subcommand" ]; then + case "$cur" in + -*) + __miccomp "${options}" + ;; + *) + __miccomp "${subcommands}" + ;; + esac + else + case "$cur" in + --*=*) + __mic_complete_val + ;; + -*) + __mic_complete_opt + ;; + *) + __mic_complete_arg || COMPREPLY=() + ;; + esac + fi +} + +__mic_main () +{ + subcommands=" + help + create + chroot + convert + " + alias=" + cr + ch + cv + " + + _opts=" + --version + " + common_opts=" + --help + --debug + --verbose + " + create_opts=" + --arch= + --tmpfs + --config= + --pkgmgr= + --outdir= + --pack-to= + --logfile= + --runtime= + --release= + --cachedir= + --copy-kernel + --check-pkgs= + --record-pkgs= + --install-pkgs= + --local-pkgs-path= + " + convert_opts=" + --shell + " + chroot_opts=" + --saveto= + " + help_opts=" + " + + _args=" + " + create_args=" + auto + fs + livecd + liveusb + loop + raw + " + convert_args=" + fs + livecd + liveusb + loop + raw + " + chroot_args=" + " + help_args=" + create + convert + chroot + " + + local cur prev words cword + if declare -F _get_comp_words_by_ref &>/dev/null ; then + _get_comp_words_by_ref cur prev words cword + else + cur=$2 prev=$3 words=("${COMP_WORDS[@]}") cword=$COMP_CWORD + fi + + local subcommand + subcommand="$(__mic_find_on_cmdline "$subcommands $alias")" + __mic_expand_alias + + local options arguments + eval options="$"{${subcommand}_opts}"\ $"{common_opts} + eval arguments="$"{${subcommand}_args} + + __mic && return + +} && +complete -F __mic_main -o bashdefault -o default -o nospace mic + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/etc/zsh_completion.d/_mic b/etc/zsh_completion.d/_mic new file mode 100644 index 0000000..a8e80c0 --- /dev/null +++ b/etc/zsh_completion.d/_mic @@ -0,0 +1,187 @@ +#compdef mic +# +# Copyright (c) 2013 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License + +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +_mic() { + typeset -A opt_args + local context state line curcontext="$curcontext" + + local ret=1 + + _arguments -C \ + {-h,--help}'[show this help message and exit]' \ + "--version[show program\'s version number and exit]" \ + {-d,--debug}'[print debug message]' \ + {-v,--verbose}'[verbose information]' \ + '1: :_mic_cmds' \ + '*::arg:->args' \ + && ret=0 + + + case "$state" in + (args) + curcontext="${curcontext%:*:*}:mic-cmd-$words[1]:" + case $words[1] in + (chroot|ch) + _arguments \ + {-h,--help}'[show this help message and exit]' \ + {-s,--saveto=}'[Save the unpacked image to specified dir]: :_files -/' \ + '1: :_files -/' \ + && ret=0 + ;; + (convert|cv) + _arguments \ + {-h,--help}'[show this help message and exit]' \ + {-S,--shell}'[Launch shell before packaging the converted image]' \ + && ret=0 + ;; + (create|cr) + _arguments -C \ + {-h,--help}'[show this help message and exit]' \ + '--logfile=[Path of logfile]:path' \ + '-c[Specify config file for mic]:file' \ + '-k[Cache directory to store the downloaded]: :_files -/' \ + '-o[Output directory]: :_files -/' \ + '-A[Specify repo architecture]:parameter' \ + '--release=[Generate a release of RID with all necessary files, when @BUILD_ID@ is contained in kickstart file, it will be replaced by RID]:parameter' \ + '--record-pkgs=[Record the info of installed packages, multiple values can be specified which joined by \",\", valid values: \"name\", \"content\", \"license\", \"vcs\"]: :_mic_create_filters -s ,' \ + '--pkgmgr=[Specify backend package manager]:parameter' \ + '--local-pkgs-path=[Path for local pkgs(rpms) to be installed]:path' \ + '--runtime=[Specify runtime mode, avaiable: bootstrap]: :(bootstrap)' \ + '--pack-to=[Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used]:parameter' \ + '--copy-kernel[Copy kernel files from image /boot directory to the image output directory.]' \ + '--install-pkgs=[Specify what type of packages to be installed, valid: source, debuginfo, debugsource]: :(source debuginfo debugsource)' \ + '--check-pkgs=[Check if given packages would be installed, packages should be separated by comma]' \ + '--tmpfs[Setup tmpdir as tmpfs to accelerate, experimental feature, use it if you have more than 4G memory]' \ + '1: :_mic_create_entities' \ + '*::create-arg:->create-args' \ + && ret=0 + case "$state" in + (create-args) + local -a common_ops + common_ops=( + '1:: :(`ls`)' \ + {-h,--help}'[show this help message and exit]' \ + '--logfile=[Path of logfile]:path' \ + '-c[Specify config file for mic]:file' \ + '-k[Cache directory to store the downloaded]: :_files -/' \ + '-o[Output directory]: :_files -/' \ + '-A[Specify repo architecture]:parameter' \ + '--release=[Generate a release of RID with all necessary files, when @BUILD_ID@ is contained in kickstart file, it will be replaced by RID]:parameter' \ + '--record-pkgs=[Record the info of installed packages, multiple values can be specified which joined by \",\", valid values: \"name\", \"content\", \"license\", \"vcs\"]: :_mic_create_filters -s ,' \ + '--pkgmgr=[Specify backend package manager]:parameter' \ + '--local-pkgs-path=[Path for local pkgs(rpms) to be installed]:path' \ + '--runtime=[Specify runtime mode, avaiable: bootstrap]: :(bootstrap)' \ + '--pack-to=[Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used]:parameter' \ + '--copy-kernel[Copy kernel files from image /boot directory to the image output directory.]' \ + '--install-pkgs=[Specify what type of packages to be installed, valid: source, debuginfo, debugsource]: :(source debuginfo debugsource)' \ + '--check-pkgs=[Check if given packages would be installed, packages should be separated by comma]' \ + '--tmpfs[Setup tmpdir as tmpfs to accelerate, experimental feature, use it if you have more than 4G memory]' \ + ) + case $words[1] in + (auto) + _arguments \ + $common_ops \ + && ret=0 + ;; + (fs) + _arguments \ + $common_ops \ + '--include-src[Generate a image with source rpms included]' \ + && ret=0 + ;; + (livecd) + _arguments \ + $common_ops \ + && ret=0 + ;; + (liveusb) + _arguments \ + $common_ops \ + && ret=0 + ;; + (loop) + _arguments \ + $common_ops \ + '--shrink[Whether to shrink loop images to minimal size]' \ + "--compress-image=[Compress all loop images with \'gz\' or \'bz2\']: :(gz bz2)" \ + "--compress-disk-image=[Same with --compress-image]: :(gz bz2)" \ + && ret=0 + ;; + (raw) + _arguments \ + $common_ops \ + "--fstab-entry=[Set fstab entry, \'name\' means using device names, \'uuid\' means using filesystem uuid]: :(name uuid)" \ + '--generate-bmap[also generate the block map file]' \ + '--compress-image=[Compress all raw images before package]: :(gz bz2)' \ + '--compress-disk-image=[Same with --compress-image]: :(gz bz2)' \ + && ret=0 + ;; + (help) + _arguments -C \ + '1: :_mic_create_entities' \ + && ret=0 + ;; + esac + ;; + esac + ;; + (help) + _arguments -C \ + '1: :_mic_cmds' \ + && ret=0 + ;; + esac + ;; + esac + + return ret +} + +(( $+functions[_mic_cmds] )) || +_mic_cmds() { + local commands; commands=( + 'chroot:chroot into an image' + 'convert:convert image format' + 'create:create an image' + 'help:give detailed help on a specific sub-command' + ) + _describe -t commands 'command' commands "$@" +} + +(( $+functions[_mic_create_entities] )) || +_mic_create_entities() { + local entities; entities=( + 'auto:auto detect image type from magic header' + 'fs:create fs image' + 'help:give detailed help on a specific sub-command' + 'livecd:create livecd image' + 'liveusb:create liveusb image' + 'loop:create loop image' + 'raw:create raw image' + ) + _describe -t entities 'entity' entities "$@" +} + +(( $+functions[_mic_create_filters] )) || +_mic_create_filters() { + local filters; filters=(name content license vcs) + _values $@ 'filter' "${filters[@]}" +} + +_mic "$@" + +# vim: ft=zsh sw=2 ts=2 et diff --git a/mic/__init__.py b/mic/__init__.py index 63c1d9c..7f0bbb8 100644 --- a/mic/__init__.py +++ b/mic/__init__.py @@ -1,4 +1,22 @@ +# +# Copyright (c) 2013 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import os, sys +__version__ = "0.27.1" + cur_path = os.path.dirname(__file__) or '.' sys.path.insert(0, cur_path + '/3rdparty') diff --git a/mic/archive.py b/mic/archive.py new file mode 100644 index 0000000..72f506f --- /dev/null +++ b/mic/archive.py @@ -0,0 +1,450 @@ +# +# Copyright (c) 2013 Intel Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +# Following messages should be disabled in pylint +# * Used * or ** magic (W0142) +# * Unused variable (W0612) +# * Used builtin function (W0141) +# * Invalid name for type (C0103) +# * Popen has no '%s' member (E1101) +# pylint: disable=W0142, W0612, W0141, C0103, E1101 + +""" Compression and Archiving + +Utility functions for creating archive files (tarballs, zip files, etc) +and compressing files (gzip, bzip2, lzop, etc) +""" + +import os +import shutil +import tempfile +import subprocess +from mic import msger + +__all__ = [ + "get_compress_formats", + "compress", + "decompress", + "get_archive_formats", + "get_archive_suffixes", + "make_archive", + "extract_archive", + "compressing", + "packing", + ] + +# TODO: refine Error class for archive/extract + +def which(binary, path=None): + """ Find 'binary' in the directories listed in 'path' + + @binary: the executable file to find + @path: the suposed path to search for, use $PATH if None + @retval: the absolute path if found, otherwise None + """ + if path is None: + path = os.environ["PATH"] + for apath in path.split(os.pathsep): + fpath = os.path.join(apath, binary) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + return fpath + return None + +def _call_external(cmdln_or_args): + """ Wapper for subprocess calls. + + @cmdln_or_args: command line to be joined before execution. + @retval: a tuple (returncode, outdata, errdata). + """ + if isinstance(cmdln_or_args, list): + shell = False + else: + shell = True + msger.info("Running command: " + " ".join(cmdln_or_args)) + + proc = subprocess.Popen(cmdln_or_args, shell=shell, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (outdata, errdata) = proc.communicate() + + return (proc.returncode, outdata, errdata) + +def _do_gzip(input_name, compression=True): + """ Compress/decompress the file with 'gzip' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + if which("pigz") is not None: + compressor = "pigz" + else: + compressor = "gzip" + + if compression: + cmdln = [compressor, "-f", input_name] + else: + cmdln = [compressor, "-d", "-f", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".gz" + else: + # suppose that file name is suffixed with ".gz" + output_name = os.path.splitext(input_name)[0] + + return output_name + +def _do_bzip2(input_name, compression=True): + """ Compress/decompress the file with 'bzip2' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + if which("pbzip2") is not None: + compressor = "pbzip2" + else: + compressor = "bzip2" + + if compression: + cmdln = [compressor, "-f", input_name] + else: + cmdln = [compressor, "-d", "-f", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".bz2" + else: + # suppose that file name is suffixed with ".bz2" + output_name = os.path.splitext(input_name)[0] + + return output_name + +def _do_lzop(input_name, compression=True): + """ Compress/decompress the file with 'lzop' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + compressor = "lzop" + + if compression: + cmdln = [compressor, "-f", "-U", input_name] + else: + cmdln = [compressor, "-d", "-f", "-U", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".lzo" + else: + # suppose that file name is suffixed with ".lzo" + output_name = os.path.splitext(input_name)[0] + + return output_name + +_COMPRESS_SUFFIXES = { + ".lzo" : [".lzo"], + ".gz" : [".gz"], + ".bz2" : [".bz2", ".bz"], + ".tar.lzo" : [".tar.lzo", ".tzo"], + ".tar.gz" : [".tar.gz", ".tgz", ".taz"], + ".tar.bz2" : [".tar.bz2", ".tbz", ".tbz2", ".tar.bz"], +} + +_COMPRESS_FORMATS = { + "gz" : _do_gzip, + "bz2": _do_bzip2, + "lzo": _do_lzop, +} + +def get_compress_formats(): + """ Get the list of the supported compression formats + + @retval: a list contained supported compress formats + """ + return _COMPRESS_FORMATS.keys() + +def get_compress_suffixes(): + """ Get the list of the support suffixes + + @retval: a list contained all suffixes + """ + suffixes = [] + for key in _COMPRESS_SUFFIXES.keys(): + suffix = _COMPRESS_SUFFIXES[key] + if (suffix): + suffixes.extend(suffix) + + return suffixes + +def compress(file_path, compress_format): + """ Compress a given file + + @file_path: the path of the file to compress + @compress_format: the compression format + @retval: the path of the compressed file + """ + if not os.path.isfile(file_path): + raise OSError, "can't compress a file not existed: '%s'" % file_path + + try: + func = _COMPRESS_FORMATS[compress_format] + except KeyError: + raise ValueError, "unknown compress format '%s'" % compress_format + return func(file_path, True) + +def decompress(file_path, decompress_format=None): + """ Decompess a give file + + @file_path: the path of the file to decompress + @decompress_format: the format for decompression, None for auto detection + @retval: the path of the decompressed file + """ + if not os.path.isfile(file_path): + raise OSError, "can't decompress a file not existed: '%s'" % file_path + + (file_name, file_ext) = os.path.splitext(file_path) + for key, suffixes in _COMPRESS_SUFFIXES.iteritems(): + if file_ext in suffixes: + file_ext = key + break + + if file_path != (file_name + file_ext): + shutil.move(file_path, file_name + file_ext) + file_path = file_name + file_ext + + if not decompress_format: + decompress_format = os.path.splitext(file_path)[1].lstrip(".") + + try: + func = _COMPRESS_FORMATS[decompress_format] + except KeyError: + raise ValueError, "unknown decompress format '%s'" % decompress_format + return func(file_path, False) + + +def _do_tar(archive_name, target_name): + """ Archive the directory or the file with 'tar' utility + + @archive_name: the name of the tarball file + @target_name: the name of the target to tar + @retval: the path of the archived file + """ + file_list = [] + if os.path.isdir(target_name): + target_dir = target_name + target_name = "." + for target_file in os.listdir(target_dir): + file_list.append(target_file) + else: + target_dir = os.path.dirname(target_name) + target_name = os.path.basename(target_name) + file_list.append(target_name) + cmdln = ["tar", "-C", target_dir, "-cf", archive_name] + cmdln.extend(file_list) + _call_external(cmdln) + + return archive_name + +def _do_untar(archive_name, target_dir=None): + """ Unarchive the archived file with 'tar' utility + + @archive_name: the name of the archived file + @target_dir: the directory which the target locates + raise exception if fail to extract archive + """ + if not target_dir: + target_dir = os.getcwd() + cmdln = ["tar", "-C", target_dir, "-xf", archive_name] + (returncode, stdout, stderr) = _call_external(cmdln) + if returncode != 0: + raise OSError, os.linesep.join([stdout, stderr]) + +def _imp_tarfile(archive_name, target_name): + """ Archive the directory or the file with tarfile module + + @archive_name: the name of the tarball file + @target_name: the name of the target to tar + @retval: the path of the archived file + """ + import tarfile + + msger.info("Taring files to %s using tarfile module" % archive_name) + tar = tarfile.open(archive_name, 'w') + if os.path.isdir(target_name): + for child in os.listdir(target_name): + tar.add(os.path.join(target_name, child), child) + else: + tar.add(target_name, os.path.basename(target_name)) + + tar.close() + return archive_name + +def _make_tarball(archive_name, target_name, compressor=None): + """ Create a tarball from all the files under 'target_name' or itself. + + @archive_name: the name of the archived file to create + @target_name: the directory or the file name to archive + @compressor: callback function to compress the tarball + @retval: indicate the compressing result + """ + archive_dir = os.path.dirname(archive_name) + tarball_name = tempfile.mktemp(suffix=".tar", dir=archive_dir) + + if which("tar") is not None: + _do_tar(tarball_name, target_name) + else: + _imp_tarfile(tarball_name, target_name) + + if compressor: + tarball_name = compressor(tarball_name, True) + + shutil.move(tarball_name, archive_name) + + return os.path.exists(archive_name) + +def _extract_tarball(archive_name, target_dir, compressor=None): + """ Extract a tarball to a target directory + + @archive_name: the name of the archived file to extract + @target_dir: the directory of the extracted target + """ + + _do_untar(archive_name, target_dir) + +def _make_zipfile(archive_name, target_name): + """ Create a zip file from all the files under 'target_name' or itself. + + @archive_name: the name of the archived file + @target_name: the directory or the file name to archive + @retval: indicate the archiving result + """ + import zipfile + + msger.info("Zipping files to %s using zipfile module" % archive_name) + arv = zipfile.ZipFile(archive_name, 'w', compression=zipfile.ZIP_DEFLATED) + + if os.path.isdir(target_name): + for dirpath, dirname, filenames in os.walk(target_name): + for filename in filenames: + filepath = os.path.normpath(os.path.join(dirpath, filename)) + arcname = os.path.relpath(filepath, target_name) + if os.path.isfile(filepath): + arv.write(filepath, arcname) + else: + arv.write(target_name, os.path.basename(target_name)) + + arv.close() + + return os.path.exists(archive_name) + +_ARCHIVE_SUFFIXES = { + "zip" : [".zip"], + "tar" : [".tar"], + "lzotar": [".tzo", ".tar.lzo"], + "gztar" : [".tgz", ".taz", ".tar.gz"], + "bztar" : [".tbz", ".tbz2", ".tar.bz", ".tar.bz2"], +} + +_ARCHIVE_FORMATS = { + "zip" : ( _make_zipfile, {} ), + "tar" : ( _make_tarball, {"compressor" : None} ), + "lzotar": ( _make_tarball, {"compressor" : _do_lzop} ), + "gztar" : ( _make_tarball, {"compressor" : _do_gzip} ), + "bztar" : ( _make_tarball, {"compressor" : _do_bzip2} ), +} + +def get_archive_formats(): + """ Get the list of the supported formats for archiving + + @retval: a list contained archive formats + """ + return _ARCHIVE_FORMATS.keys() + +def get_archive_suffixes(): + """ Get the list of the support suffixes + + @retval: a list contained all suffixes + """ + suffixes = [] + for name in _ARCHIVE_FORMATS.keys(): + suffix = _ARCHIVE_SUFFIXES.get(name, None) + if (suffix): + suffixes.extend(suffix) + + return suffixes + +def make_archive(archive_name, target_name): + """ Create an archive file (eg. tar or zip). + + @archive_name: the name of the archived file + @target_name: the directory or the file to archive + @retval: the archiving result + """ + if not os.path.exists(target_name): + raise OSError, "archive object does not exist: '%s'" % target_name + + for aformat, suffixes in _ARCHIVE_SUFFIXES.iteritems(): + if filter(archive_name.endswith, suffixes): + archive_format = aformat + break + else: + raise ValueError, "unknown archive suffix '%s'" % archive_name + + try: + func, kwargs = _ARCHIVE_FORMATS[archive_format] + except KeyError: + raise ValueError, "unknown archive format '%s'" % archive_format + + archive_name = os.path.abspath(archive_name) + target_name = os.path.abspath(target_name) + + archive_dir = os.path.dirname(archive_name) + if not os.path.exists(archive_dir): + os.makedirs(archive_dir) + + return func(archive_name, target_name, **kwargs) + +def extract_archive(archive_name, target_name): + """ Extract the given file + + @archive_name: the name of the archived file to extract + @target_name: the directory name where the target locates + raise exception if fail to extract archive + """ + if not os.path.exists(archive_name): + raise OSError, "archived object does not exist: '%s'" % archive_name + + archive_name = os.path.abspath(archive_name) + target_name = os.path.abspath(target_name) + + if os.path.exists(target_name) and not os.path.isdir(target_name): + raise OSError, "%s should be directory where extracted files locate"\ + % target_name + + if not os.path.exists(target_name): + os.makedirs(target_name) + + _extract_tarball(archive_name, target_name) + +packing = make_archive +compressing = compress + diff --git a/mic/bootstrap.py b/mic/bootstrap.py index 66c291b..5cfa8c7 100644 --- a/mic/bootstrap.py +++ b/mic/bootstrap.py @@ -135,7 +135,13 @@ class MiniBackend(object): # run transaction self.ts.order() cb = RPMInstallCallback(self.ts) - self.ts.run(cb.callback, '') + errs = self.ts.run(cb.callback, '') + + # ts.run() exit codes are, hmm, "creative": None means all ok, empty + # list means some errors happened in the transaction and non-empty + # list that there were errors preventing the ts from starting... + if errs is not None: + raise errors.BootstrapError("Transaction couldn't start: %s" % '\n'.join(errs)) def run_pkg_script(self, pkg, prog, script, arg): mychroot = lambda: os.chroot(self.rootdir) @@ -253,7 +259,7 @@ class Bootstrap(object): gloablmounts = None try: proxy.set_proxy_environ() - gloablmounts = setup_chrootenv(rootdir, bindmounts, False) + gloablmounts = setup_chrootenv(rootdir, bindmounts) sync_timesetting(rootdir) sync_passwdfile(rootdir) retcode = subprocess.call(cmd, preexec_fn=mychroot, env=env, shell=shell) @@ -263,8 +269,8 @@ class Bootstrap(object): value = '%s: %s' % (value, ' '.join(cmd)) raise RuntimeError, value, tb finally: - if self.logfile and os.path.isfile(self.logfile): - msger.log(file(self.logfile).read()) + #if self.logfile and os.path.isfile(self.logfile): + # msger.log(file(self.logfile).read()) cleanup_chrootenv(rootdir, bindmounts, gloablmounts) proxy.unset_proxy_environ() return retcode diff --git a/mic/chroot.py b/mic/chroot.py index 99fb9a2..a2bafdf 100644 --- a/mic/chroot.py +++ b/mic/chroot.py @@ -17,253 +17,244 @@ from __future__ import with_statement import os +import re import shutil import subprocess from mic import msger from mic.conf import configmgr -from mic.utils import misc, errors, runner, fs_related +from mic.utils import misc, errors, runner, fs_related, lock -chroot_lockfd = -1 -chroot_lock = "" +##################################################################### +### GLOBAL CONSTANTS +##################################################################### + +chroot_bindmounts = None +chroot_lock = None BIND_MOUNTS = ( "/proc", "/proc/sys/fs/binfmt_misc", "/sys", "/dev", "/dev/pts", - "/dev/shm", "/var/lib/dbus", "/var/run/dbus", "/var/lock", + "/lib/modules", ) -def cleanup_after_chroot(targettype,imgmount,tmpdir,tmpmnt): - if imgmount and targettype == "img": - imgmount.cleanup() - - if tmpdir: - shutil.rmtree(tmpdir, ignore_errors = True) - - if tmpmnt: - shutil.rmtree(tmpmnt, ignore_errors = True) - -def check_bind_mounts(chrootdir, bindmounts): - chrootmounts = [] - for mount in bindmounts.split(";"): - if not mount: +##################################################################### +### GLOBAL ROUTINE +##################################################################### + +def ELF_arch(chrootdir): + """ detect the architecture of an ELF file """ + #FIXME: if chkfiles are symlink, it will be complex + chkfiles = ('/bin/bash', '/sbin/init') + # regular expression to arch mapping + mapping = { + r"Intel 80[0-9]86": "i686", + r"x86-64": "x86_64", + r"ARM": "arm", + } + + for path in chkfiles: + cpath = os.path.join(chrootdir, path.lstrip('/')) + if not os.path.exists(cpath): continue - srcdst = mount.split(":") - if len(srcdst) == 1: - srcdst.append("none") + outs = runner.outs(['file', cpath]) + for ptn in mapping.keys(): + if re.search(ptn, outs): + return mapping[ptn] - if not os.path.isdir(srcdst[0]): - return False + raise errors.CreatorError("Failed to detect architecture of chroot: %s" % + chrootdir) - if srcdst[1] == "" or srcdst[1] == "none": - srcdst[1] = None +def get_bindmounts(chrootdir, bindmounts = None): + """ calculate all bind mount entries for global usage """ + # bindmounts should be a string like '/dev:/dev' + # FIXME: refine the bindmounts from string to dict + global chroot_bindmounts - if srcdst[0] in BIND_MOUNTS or srcdst[0] == '/': - continue + def totuple(string): + """ convert string contained ':' to a tuple """ + if ':' in string: + src, dst = string.split(':', 1) + else: + src = string + dst = None - if chrootdir: - if not srcdst[1]: - srcdst[1] = os.path.abspath(os.path.expanduser(srcdst[0])) - else: - srcdst[1] = os.path.abspath(os.path.expanduser(srcdst[1])) + return (src or None, dst or None) - tmpdir = chrootdir + "/" + srcdst[1] - if os.path.isdir(tmpdir): - msger.warning("Warning: dir %s has existed." % tmpdir) + if chroot_bindmounts: + return chroot_bindmounts - return True + chroot_bindmounts = [] + bindmounts = bindmounts or "" + mountlist = [] -def cleanup_mounts(chrootdir): - umountcmd = misc.find_binary_path("umount") - abs_chrootdir = os.path.abspath(chrootdir) - mounts = open('/proc/mounts').readlines() - for line in reversed(mounts): - if abs_chrootdir not in line: + for mount in bindmounts.split(";"): + if not mount: continue - point = line.split()[1] - - # '/' to avoid common name prefix - if abs_chrootdir == point or point.startswith(abs_chrootdir + '/'): - args = [ umountcmd, "-l", point ] - ret = runner.quiet(args) - if ret != 0: - msger.warning("failed to unmount %s" % point) + (src, dst) = totuple(mount) - return 0 + if src in BIND_MOUNTS or src == '/': + continue -def setup_chrootenv(chrootdir, bindmounts = None, mountparent = True): - global chroot_lockfd, chroot_lock + if not os.path.exists(src): + os.makedirs(src) - def get_bind_mounts(chrootdir, bindmounts, mountparent = True): - chrootmounts = [] - if bindmounts in ("", None): - bindmounts = "" + if dst and os.path.isdir("%s/%s" % (chrootdir, dst)): + msger.warning("%s existed in %s , skip it." % (dst, chrootdir)) + continue - for mount in bindmounts.split(";"): - if not mount: - continue + mountlist.append(totuple(mount)) - srcdst = mount.split(":") - srcdst[0] = os.path.abspath(os.path.expanduser(srcdst[0])) - if len(srcdst) == 1: - srcdst.append("none") + for mntpoint in BIND_MOUNTS: + if os.path.isdir(mntpoint): + mountlist.append(tuple((mntpoint, None))) - # if some bindmount is not existed, but it's created inside - # chroot, this is not expected - if not os.path.exists(srcdst[0]): - os.makedirs(srcdst[0]) + for pair in mountlist: + if pair[0] == "/lib/modules": + opt = "ro" + else: + opt = None + bmount = fs_related.BindChrootMount(pair[0], chrootdir, pair[1], opt) + chroot_bindmounts.append(bmount) - if not os.path.isdir(srcdst[0]): - continue + return chroot_bindmounts - if srcdst[0] in BIND_MOUNTS or srcdst[0] == '/': - msger.verbose("%s will be mounted by default." % srcdst[0]) - continue +##################################################################### +### SETUP CHROOT ENVIRONMENT +##################################################################### - if srcdst[1] == "" or srcdst[1] == "none": - srcdst[1] = None - else: - srcdst[1] = os.path.abspath(os.path.expanduser(srcdst[1])) - if os.path.isdir(chrootdir + "/" + srcdst[1]): - msger.warning("%s has existed in %s , skip it."\ - % (srcdst[1], chrootdir)) - continue - - chrootmounts.append(fs_related.BindChrootMount(srcdst[0], - chrootdir, - srcdst[1])) - - """Default bind mounts""" - for pt in BIND_MOUNTS: - if not os.path.exists(pt): - continue - chrootmounts.append(fs_related.BindChrootMount(pt, - chrootdir, - None)) - - if mountparent: - chrootmounts.append(fs_related.BindChrootMount("/", - chrootdir, - "/parentroot", - "ro")) - - for kernel in os.listdir("/lib/modules"): - chrootmounts.append(fs_related.BindChrootMount( - "/lib/modules/"+kernel, - chrootdir, - None, - "ro")) - - return chrootmounts - - def bind_mount(chrootmounts): - for b in chrootmounts: - msger.verbose("bind_mount: %s -> %s" % (b.src, b.dest)) - b.mount() - - def setup_resolv(chrootdir): - try: - shutil.copyfile("/etc/resolv.conf", chrootdir + "/etc/resolv.conf") - except: - pass +def bind_mount(chrootmounts): + """ perform bind mounting """ + for mnt in chrootmounts: + msger.verbose("bind_mount: %s -> %s" % (mnt.src, mnt.dest)) + mnt.mount() - globalmounts = get_bind_mounts(chrootdir, bindmounts, mountparent) - bind_mount(globalmounts) +def setup_resolv(chrootdir): + """ resolve network """ + try: + shutil.copyfile("/etc/resolv.conf", chrootdir + "/etc/resolv.conf") + except (OSError, IOError): + pass +def setup_mtab(chrootdir): + """ adjust mount table """ + try: + mtab = "/etc/mtab" + dstmtab = chrootdir + mtab + if not os.path.islink(dstmtab): + shutil.copyfile(mtab, dstmtab) + except (OSError, IOError): + pass + +def setup_chrootenv(chrootdir, bindmounts = None): + """ setup chroot environment """ + global chroot_lock + + # acquire the lock + if not chroot_lock: + lockpath = os.path.join(chrootdir, '.chroot.lock') + chroot_lock = lock.SimpleLockfile(lockpath) + chroot_lock.acquire() + # bind mounting + bind_mount(get_bindmounts(chrootdir, bindmounts)) + # setup resolv.conf setup_resolv(chrootdir) + # update /etc/mtab + setup_mtab(chrootdir) - mtab = "/etc/mtab" - dstmtab = chrootdir + mtab - if not os.path.islink(dstmtab): - shutil.copyfile(mtab, dstmtab) + return None - chroot_lock = os.path.join(chrootdir, ".chroot.lock") - chroot_lockfd = open(chroot_lock, "w") +###################################################################### +### CLEANUP CHROOT ENVIRONMENT +###################################################################### - return globalmounts +def bind_unmount(chrootmounts): + """ perform bind unmounting """ + for mnt in reversed(chrootmounts): + msger.verbose("bind_unmount: %s -> %s" % (mnt.src, mnt.dest)) + mnt.unmount() -def cleanup_chrootenv(chrootdir, bindmounts=None, globalmounts=()): - global chroot_lockfd, chroot_lock - - def bind_unmount(chrootmounts): - for b in reversed(chrootmounts): - msger.verbose("bind_unmount: %s -> %s" % (b.src, b.dest)) - b.unmount() - - def cleanup_resolv(chrootdir): +def cleanup_resolv(chrootdir): + """ clear resolv.conf """ + try: + fdes = open(chrootdir + "/etc/resolv.conf", "w") + fdes.truncate(0) + fdes.close() + except (OSError, IOError): + pass + +def kill_proc_inchroot(chrootdir): + """ kill all processes running inside chrootdir """ + import glob + for fpath in glob.glob("/proc/*/root"): try: - fd = open(chrootdir + "/etc/resolv.conf", "w") - fd.truncate(0) - fd.close() - except: + if os.readlink(fpath) == chrootdir: + pid = int(fpath.split("/")[2]) + os.kill(pid, 9) + except (OSError, ValueError): pass - def kill_processes(chrootdir): - import glob - for fp in glob.glob("/proc/*/root"): - try: - if os.readlink(fp) == chrootdir: - pid = int(fp.split("/")[2]) - os.kill(pid, 9) - except: - pass - - def cleanup_mountdir(chrootdir, bindmounts): - if bindmounts == "" or bindmounts == None: - return - chrootmounts = [] - for mount in bindmounts.split(";"): - if not mount: - continue +def cleanup_mtab(chrootdir): + """ remove mtab file """ + if os.path.exists(chrootdir + "/etc/mtab"): + os.unlink(chrootdir + "/etc/mtab") - srcdst = mount.split(":") - - if len(srcdst) == 1: - srcdst.append("none") - - if srcdst[0] == "/": - continue - - if srcdst[1] == "" or srcdst[1] == "none": - srcdst[1] = srcdst[0] - - srcdst[1] = os.path.abspath(os.path.expanduser(srcdst[1])) - tmpdir = chrootdir + "/" + srcdst[1] - if os.path.isdir(tmpdir): - if len(os.listdir(tmpdir)) == 0: - shutil.rmtree(tmpdir, ignore_errors = True) - else: - msger.warning("Warning: dir %s isn't empty." % tmpdir) - - chroot_lockfd.close() - bind_unmount(globalmounts) - - if not fs_related.my_fuser(chroot_lock): - tmpdir = chrootdir + "/parentroot" - if os.path.exists(tmpdir) and len(os.listdir(tmpdir)) == 0: - shutil.rmtree(tmpdir, ignore_errors = True) - - cleanup_resolv(chrootdir) - - if os.path.exists(chrootdir + "/etc/mtab"): - os.unlink(chrootdir + "/etc/mtab") - - kill_processes(chrootdir) +def cleanup_mounts(chrootdir): + """ clean up all mount entries owned by chrootdir """ + umountcmd = misc.find_binary_path("umount") + mounts = open('/proc/mounts').readlines() + for line in reversed(mounts): + if chrootdir not in line: + continue - cleanup_mountdir(chrootdir, bindmounts) + point = line.split()[1] -def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"): - def mychroot(): - os.chroot(chrootdir) - os.chdir("/") + # '/' to avoid common name prefix + if chrootdir == point or point.startswith(chrootdir + '/'): + args = [ umountcmd, "-l", point ] + ret = runner.quiet(args) + if ret != 0: + msger.warning("failed to unmount %s" % point) + if os.path.isdir(point) and len(os.listdir(point)) == 0: + shutil.rmtree(point) + else: + msger.warning("%s is not directory or is not empty" % point) +def cleanup_chrootenv(chrootdir, bindmounts=None, globalmounts=()): + """ clean up chroot environment """ + global chroot_lock + + # kill processes + kill_proc_inchroot(chrootdir) + # clean mtab + cleanup_mtab(chrootdir) + # clean resolv.conf + cleanup_resolv(chrootdir) + # bind umounting + bind_unmount(get_bindmounts(chrootdir, bindmounts)) + # FIXME: need to clean up mounts? + #cleanup_mounts(chrootdir) + + # release the lock + if chroot_lock: + chroot_lock.release() + chroot_lock = None + + return None + +##################################################################### +### CHROOT STUFF +##################################################################### + +def savefs_before_chroot(chrootdir, saveto = None): + """ backup chrootdir to another directory before chrooting in """ if configmgr.chroot['saveto']: savefs = True saveto = configmgr.chroot['saveto'] @@ -293,44 +284,38 @@ def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"): else: msger.warning(wrnmsg) - dev_null = os.open("/dev/null", os.O_WRONLY) - files_to_check = ["/bin/bash", "/sbin/init"] - - architecture_found = False - - """ Register statically-linked qemu-arm if it is an ARM fs """ - qemu_emulator = None +def cleanup_after_chroot(targettype, imgmount, tmpdir, tmpmnt): + """ clean up all temporary directories after chrooting """ + if imgmount and targettype == "img": + imgmount.cleanup() - for ftc in files_to_check: - ftc = "%s/%s" % (chrootdir,ftc) + if tmpdir: + shutil.rmtree(tmpdir, ignore_errors = True) - # Return code of 'file' is "almost always" 0 based on some man pages - # so we need to check the file existance first. - if not os.path.exists(ftc): - continue + if tmpmnt: + shutil.rmtree(tmpmnt, ignore_errors = True) - for line in runner.outs(['file', ftc]).splitlines(): - if 'ARM' in line: - qemu_emulator = misc.setup_qemu_emulator(chrootdir, "arm") - architecture_found = True - break +def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"): + """ chroot the chrootdir and execute the command """ + def mychroot(): + """ pre-execute function """ + os.chroot(chrootdir) + os.chdir("/") - if 'Intel' in line: - architecture_found = True - break + arch = ELF_arch(chrootdir) + if arch == "arm": + qemu_emulator = misc.setup_qemu_emulator(chrootdir, "arm") + else: + qemu_emulator = None - if architecture_found: - break + savefs_before_chroot(chrootdir, None) - os.close(dev_null) - if not architecture_found: - raise errors.CreatorError("Failed to get architecture from any of the " - "following files %s from chroot." \ - % files_to_check) + globalmounts = None try: msger.info("Launching shell. Exit to continue.\n" "----------------------------------") + globalmounts = setup_chrootenv(chrootdir, bindmounts) subprocess.call(execute, preexec_fn = mychroot, shell=True) diff --git a/mic/cmd_chroot.py b/mic/cmd_chroot.py new file mode 100755 index 0000000..d7d76ef --- /dev/null +++ b/mic/cmd_chroot.py @@ -0,0 +1,71 @@ +#!/usr/bin/python -tt
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Implementation of subcmd: chroot
+"""
+
+import os
+import os, sys, re
+import pwd
+import argparse
+
+from mic import msger
+from mic.utils import misc, errors
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+def _root_confirm():
+ """Make sure command is called by root
+ There are a lot of commands needed to be run during creating images,
+ some of them must be run with root privilege like mount, kpartx"""
+ if os.geteuid() != 0:
+ msger.error('Root permission is required to continue, abort')
+
+def main(parser, args, argv):
+ """mic choot entry point."""
+
+ #args is argparser namespace, argv is the input cmd line
+ if args is None:
+ raise errors.Usage("Invalid arguments")
+
+ targetimage = args.imagefile
+ if not os.path.exists(targetimage):
+ raise errors.CreatorError("Cannot find the image: %s"
+ % targetimage)
+
+ _root_confirm()
+
+ configmgr.chroot['saveto'] = args.saveto
+
+ imagetype = misc.get_image_type(targetimage)
+ if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"):
+ imagetype = "loop"
+
+ chrootclass = None
+ for pname, pcls in pluginmgr.get_plugins('imager').iteritems():
+ if pname == imagetype and hasattr(pcls, "do_chroot"):
+ chrootclass = pcls
+ break
+
+ if not chrootclass:
+ raise errors.CreatorError("Cannot support image type: %s" \
+ % imagetype)
+
+ chrootclass.do_chroot(targetimage, args.cmd)
+
+
diff --git a/mic/cmd_create.py b/mic/cmd_create.py new file mode 100755 index 0000000..ce574a1 --- /dev/null +++ b/mic/cmd_create.py @@ -0,0 +1,234 @@ +#!/usr/bin/python -tt
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Implementation of subcmd: create
+"""
+
+import os
+import os, sys, re
+import pwd
+import argparse
+
+from mic import msger
+from mic.utils import errors, rpmmisc
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+def main(parser, args, argv):
+ """mic create entry point."""
+ #args is argparser namespace, argv is the input cmd line
+ if args is None:
+ raise errors.Usage("Invalid arguments")
+
+ if not os.path.exists(args.ksfile):
+ raise errors.CreatorError("Can't find the file: %s" % args.ksfile)
+
+ if os.geteuid() != 0:
+ msger.error("Root permission is required, abort")
+
+ try:
+ w = pwd.getpwuid(os.geteuid())
+ except KeyError:
+ msger.warning("Might fail in compressing stage for undetermined user")
+
+ abspath = lambda pth: os.path.abspath(os.path.expanduser(pth))
+ if args.logfile:
+ logfile_abs_path = abspath(args.logfile)
+ if os.path.isdir(logfile_abs_path):
+ raise errors.Usage("logfile's path %s should be file"
+ % args.logfile)
+ configmgr.create['logfile'] = logfile_abs_path
+ configmgr.set_logfile()
+
+ if args.subcommand == "auto":
+ do_auto(parser, args.ksfile, argv)
+ return
+
+ #check the imager type
+ createrClass = None
+ for subcmd, klass in pluginmgr.get_plugins('imager').iteritems():
+ if subcmd == args.subcommand and hasattr(klass, 'do_create'):
+ createrClass = klass
+
+ if createrClass is None:
+ raise errors.CreatorError("Can't support subcommand %s" % args.subcommand)
+
+ if args.config:
+ configmgr.reset()
+ configmgr._siteconf = args.config
+
+ if args.outdir is not None:
+ configmgr.create['outdir'] = abspath(args.outdir)
+ if args.cachedir is not None:
+ configmgr.create['cachedir'] = abspath(args.cachedir)
+ os.environ['ZYPP_LOCKFILE_ROOT'] = configmgr.create['cachedir']
+
+ for cdir in ('outdir', 'cachedir'):
+ if os.path.exists(configmgr.create[cdir]) \
+ and not os.path.isdir(configmgr.create[cdir]):
+ raise errors.Usage('Invalid directory specified: %s' \
+ % configmgr.create[cdir])
+ if not os.path.exists(configmgr.create[cdir]):
+ os.makedirs(configmgr.create[cdir])
+ if os.getenv('SUDO_UID', '') and os.getenv('SUDO_GID', ''):
+ os.chown(configmgr.create[cdir],
+ int(os.getenv('SUDO_UID')),
+ int(os.getenv('SUDO_GID')))
+
+ if args.local_pkgs_path is not None:
+ if not os.path.exists(args.local_pkgs_path):
+ raise errors.Usage('Local pkgs directory: \'%s\' not exist' \
+ % args.local_pkgs_path)
+ configmgr.create['local_pkgs_path'] = args.local_pkgs_path
+
+ if args.release:
+ configmgr.create['release'] = args.release.rstrip('/')
+
+ if args.record_pkgs:
+ configmgr.create['record_pkgs'] = []
+ for infotype in args.record_pkgs.split(','):
+ if infotype not in ('name', 'content', 'license', 'vcs'):
+ raise errors.Usage('Invalid pkg recording: %s, valid ones:'
+ ' "name", "content", "license", "vcs"' \
+ % infotype)
+
+ configmgr.create['record_pkgs'].append(infotype)
+
+ if args.strict_mode:
+ configmgr.create['strict_mode'] = args.strict_mode
+ if args.arch is not None:
+ supported_arch = sorted(rpmmisc.archPolicies.keys(), reverse=True)
+ if args.arch in supported_arch:
+ configmgr.create['arch'] = args.arch
+ else:
+ raise errors.Usage('Invalid architecture: "%s".\n'
+ ' Supported architectures are: \n'
+ ' %s' % (args.arch,
+ ', '.join(supported_arch)))
+
+ if args.pkgmgr is not None:
+ configmgr.create['pkgmgr'] = args.pkgmgr
+
+ if args.runtime:
+ configmgr.set_runtime(args.runtime)
+
+ if args.pack_to is not None:
+ configmgr.create['pack_to'] = args.pack_to
+
+ if args.copy_kernel:
+ configmgr.create['copy_kernel'] = args.copy_kernel
+
+ if args.install_pkgs:
+ configmgr.create['install_pkgs'] = []
+ for pkgtype in args.install_pkgs.split(','):
+ if pkgtype not in ('source', 'debuginfo', 'debugsource'):
+ raise errors.Usage('Invalid parameter specified: "%s", '
+ 'valid values: source, debuginfo, '
+ 'debusource' % pkgtype)
+
+ configmgr.create['install_pkgs'].append(pkgtype)
+
+ if args.check_pkgs:
+ for pkg in args.check_pkgs.split(','):
+ configmgr.create['check_pkgs'].append(pkg)
+
+ if args.enabletmpfs:
+ configmgr.create['enabletmpfs'] = args.enabletmpfs
+
+ if args.repourl:
+ for item in args.repourl:
+ try:
+ key, val = item.split('=')
+ except:
+ continue
+ configmgr.create['repourl'][key] = val
+
+ if args.repo:
+ for optvalue in args.repo:
+ repo = {}
+ for item in optvalue.split(';'):
+ try:
+ key, val = item.split('=')
+ except:
+ continue
+ repo[key.strip()] = val.strip()
+ if 'name' in repo:
+ configmgr.create['extrarepos'][repo['name']] = repo
+
+ if args.ignore_ksrepo:
+ configmgr.create['ignore_ksrepo'] = args.ignore_ksrepo
+
+ creater = createrClass()
+ creater.do_create(args)
+
+def do_auto(parser, ksfile, argv):
+ """${cmd_name}: auto detect image type from magic header
+
+ Usage:
+ ${name} ${cmd_name} <ksfile>
+
+ ${cmd_option_list}
+ """
+ def parse_magic_line(re_str, pstr, ptype='mic'):
+ ptn = re.compile(re_str)
+ m = ptn.match(pstr)
+ if not m or not m.groups():
+ return None
+
+ inline_argv = m.group(1).strip()
+ if ptype == 'mic':
+ m2 = re.search('(?P<format>\w+)', inline_argv)
+ elif ptype == 'mic2':
+ m2 = re.search('(-f|--format(=)?)\s*(?P<format>\w+)',
+ inline_argv)
+ else:
+ return None
+
+ if m2:
+ cmdname = m2.group('format')
+ inline_argv = inline_argv.replace(m2.group(0), '')
+ return (cmdname, inline_argv)
+
+ return None
+
+ if not os.path.exists(ksfile):
+ raise errors.CreatorError("Can't find the file: %s" % ksfile)
+
+ with open(ksfile, 'r') as rf:
+ first_line = rf.readline()
+
+ mic_re = '^#\s*-\*-mic-options-\*-\s+(.*)\s+-\*-mic-options-\*-'
+ mic2_re = '^#\s*-\*-mic2-options-\*-\s+(.*)\s+-\*-mic2-options-\*-'
+
+ result = parse_magic_line(mic_re, first_line, 'mic') \
+ or parse_magic_line(mic2_re, first_line, 'mic2')
+ if not result:
+ raise errors.KsError("Invalid magic line in file: %s" % ksfile)
+
+ ksargv = ' '.join(result).split()
+
+ argv.remove("auto")
+ index = argv.index("create")
+ #insert the subcommand
+ argv.insert(index+1, ksargv[0])
+ options = argv + ksargv[1:]
+
+ args = parser.parse_args(options)
+
+ main(parser, args, options)
+
diff --git a/mic/conf.py b/mic/conf.py index e37334c..d844c65 100644 --- a/mic/conf.py +++ b/mic/conf.py @@ -27,13 +27,15 @@ DEFAULT_GSITECONF = '/etc/mic/mic.conf' def get_siteconf(): - mic_path = os.path.dirname(__file__) + if hasattr(sys, 'real_prefix'): + return os.path.join(sys.prefix, "etc/mic/mic.conf") + else: + return DEFAULT_GSITECONF - m = re.match(r"(?P<prefix>.*)\/lib(64)?\/.*", mic_path) - if m and m.group('prefix') != "/usr": - return os.path.join(m.group('prefix'), "etc/mic/mic.conf") - - return DEFAULT_GSITECONF +def inbootstrap(): + if os.path.exists(os.path.join("/", ".chroot.lock")): + return True + return (os.stat("/").st_ino != 2) class ConfigMgr(object): prefer_backends = ["zypp", "yum"] @@ -46,7 +48,7 @@ class ConfigMgr(object): "tmpdir": '/var/tmp/mic', "cachedir": '/var/tmp/mic/cache', "outdir": './mic-output', - + "destdir": None, "arch": None, # None means auto-detect "pkgmgr": "auto", "name": "output", @@ -56,17 +58,23 @@ class ConfigMgr(object): "local_pkgs_path": None, "release": None, "logfile": None, + "releaselog": False, "record_pkgs": [], "pack_to": None, "name_prefix": None, "name_suffix": None, "proxy": None, "no_proxy": None, + "ssl_verify": "yes", "copy_kernel": False, "install_pkgs": None, + "check_pkgs": [], "repourl": {}, "localrepos": [], # save localrepos "runtime": "bootstrap", + "extrarepos": {}, + "ignore_ksrepo": False, + "strict_mode": False, }, 'chroot': { "saveto": None, @@ -77,6 +85,7 @@ class ConfigMgr(object): 'bootstrap': { "rootdir": '/var/tmp/mic-bootstrap', "packages": [], + "distro_name": "", }, } @@ -121,7 +130,7 @@ class ConfigMgr(object): def __set_ksconf(self, ksconf): if not os.path.isfile(ksconf): - msger.error('Cannot find ks file: %s' % ksconf) + raise errors.KsError('Cannot find ks file: %s' % ksconf) self.__ksconf = ksconf self._parse_kickstart(ksconf) @@ -130,11 +139,21 @@ class ConfigMgr(object): _ksconf = property(__get_ksconf, __set_ksconf) def _parse_siteconf(self, siteconf): + + if os.getenv("MIC_PLUGIN_DIR"): + self.common["plugin_dir"] = os.environ["MIC_PLUGIN_DIR"] + + if siteconf and not os.path.exists(siteconf): + msger.warning("cannot find config file: %s" % siteconf) + siteconf = None + if not siteconf: - return + self.common["distro_name"] = "Tizen" + # append common section items to other sections + for section in self.DEFAULTS.keys(): + if section != "common": + getattr(self, section).update(self.common) - if not os.path.exists(siteconf): - msger.warning("cannot read config file: %s" % siteconf) return parser = ConfigParser.SafeConfigParser() @@ -155,7 +174,7 @@ class ConfigMgr(object): if m: scheme = m.group(1) if scheme not in ('http', 'https', 'ftp', 'socks'): - msger.error("%s: proxy scheme is incorrect" % siteconf) + raise errors.ConfigError("%s: proxy scheme is incorrect" % siteconf) else: msger.warning("%s: proxy url w/o scheme, use http as default" % siteconf) @@ -191,14 +210,29 @@ class ConfigMgr(object): self.create['name_prefix'], self.create['name_suffix']) + self.create['destdir'] = self.create['outdir'] + if self.create['release'] is not None: + self.create['destdir'] = "%s/%s/images/%s/" % (self.create['outdir'], + self.create['release'], + self.create['name']) + self.create['name'] = self.create['release'] + '_' + self.create['name'] + + if not self.create['logfile']: + self.create['logfile'] = os.path.join(self.create['destdir'], + self.create['name'] + ".log") + self.create['releaselog'] = True + self.set_logfile() + msger.info("Retrieving repo metadata:") - ksrepos = misc.get_repostrs_from_ks(ks) + ksrepos = kickstart.get_repos(ks, + self.create['extrarepos'], + self.create['ignore_ksrepo']) if not ksrepos: raise errors.KsError('no valid repos found in ks file') for repo in ksrepos: - if 'baseurl' in repo and repo['baseurl'].startswith("file:"): - repourl = repo['baseurl'].replace('file:', '') + if hasattr(repo, 'baseurl') and repo.baseurl.startswith("file:"): + repourl = repo.baseurl.replace('file:', '') repourl = "/%s" % repourl.lstrip('/') self.create['localrepos'].append(repourl) @@ -216,7 +250,7 @@ class ConfigMgr(object): else: if len(target_archlist) == 1: self.create['arch'] = str(target_archlist[0]) - msger.info("\nUse detected arch %s." % target_archlist[0]) + msger.info("Use detected arch %s." % target_archlist[0]) else: raise errors.ConfigError("Please specify a valid arch, " "the choice can be: %s" \ @@ -228,9 +262,23 @@ class ConfigMgr(object): misc.selinux_check(self.create['arch'], [p.fstype for p in ks.handler.partition.partitions]) + def set_logfile(self, logfile = None): + if not logfile: + logfile = self.create['logfile'] + + logfile_dir = os.path.dirname(self.create['logfile']) + if not os.path.exists(logfile_dir): + os.makedirs(logfile_dir) + msger.set_interactive(False) + if inbootstrap(): + mode = 'a' + else: + mode = 'w' + msger.set_logfile(self.create['logfile'], mode) + def set_runtime(self, runtime): - if runtime not in ("bootstrap", "native"): - msger.error("Invalid runtime mode: %s" % runtime) + if runtime != "bootstrap": + raise errors.CreatorError("Invalid runtime mode: %s, only 'bootstrap' mode is allowed." % runtime) if misc.get_distro()[0] in ("tizen", "Tizen"): runtime = "native" diff --git a/mic/creator.py b/mic/creator.py deleted file mode 100644 index af5fb82..0000000 --- a/mic/creator.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys, re -from optparse import SUPPRESS_HELP - -from mic import msger, rt_util -from mic.utils import cmdln, errors, rpmmisc -from mic.conf import configmgr -from mic.plugin import pluginmgr - - -class Creator(cmdln.Cmdln): - """${name}: create an image - - Usage: - ${name} SUBCOMMAND <ksfile> [OPTS] - - ${command_list} - ${option_list} - """ - - name = 'mic create(cr)' - - def __init__(self, *args, **kwargs): - cmdln.Cmdln.__init__(self, *args, **kwargs) - self._subcmds = [] - - # get cmds from pluginmgr - # mix-in do_subcmd interface - for subcmd, klass in pluginmgr.get_plugins('imager').iteritems(): - if not hasattr(klass, 'do_create'): - msger.warning("Unsurpport subcmd: %s" % subcmd) - continue - - func = getattr(klass, 'do_create') - setattr(self.__class__, "do_"+subcmd, func) - self._subcmds.append(subcmd) - - def get_optparser(self): - optparser = cmdln.CmdlnOptionParser(self) - optparser.add_option('-d', '--debug', action='store_true', - dest='debug', - help=SUPPRESS_HELP) - optparser.add_option('-v', '--verbose', action='store_true', - dest='verbose', - help=SUPPRESS_HELP) - optparser.add_option('', '--logfile', type='string', dest='logfile', - default=None, - help='Path of logfile') - optparser.add_option('-c', '--config', type='string', dest='config', - default=None, - help='Specify config file for mic') - optparser.add_option('-k', '--cachedir', type='string', action='store', - dest='cachedir', default=None, - help='Cache directory to store the downloaded') - optparser.add_option('-o', '--outdir', type='string', action='store', - dest='outdir', default=None, - help='Output directory') - optparser.add_option('-A', '--arch', type='string', dest='arch', - default=None, - help='Specify repo architecture') - optparser.add_option('', '--release', type='string', dest='release', - default=None, metavar='RID', - help='Generate a release of RID with all necessary' - ' files, when @BUILD_ID@ is contained in ' - 'kickstart file, it will be replaced by RID') - optparser.add_option("", "--record-pkgs", type="string", - dest="record_pkgs", default=None, - help='Record the info of installed packages, ' - 'multiple values can be specified which ' - 'joined by ",", valid values: "name", ' - '"content", "license", "vcs"') - optparser.add_option('', '--pkgmgr', type='string', dest='pkgmgr', - default=None, - help='Specify backend package manager') - optparser.add_option('', '--local-pkgs-path', type='string', - dest='local_pkgs_path', default=None, - help='Path for local pkgs(rpms) to be installed') - optparser.add_option('', '--runtime', type='string', - dest='runtime', default=None, - help='Specify runtime mode, avaiable: bootstrap, native') - # --taring-to is alias to --pack-to - optparser.add_option('', '--taring-to', type='string', - dest='pack_to', default=None, - help=SUPPRESS_HELP) - optparser.add_option('', '--pack-to', type='string', - dest='pack_to', default=None, - help='Pack the images together into the specified' - ' achive, extension supported: .zip, .tar, ' - '.tar.gz, .tar.bz2, etc. by default, .tar ' - 'will be used') - optparser.add_option('', '--copy-kernel', action='store_true', - dest='copy_kernel', - help='Copy kernel files from image /boot directory' - ' to the image output directory.') - optparser.add_option('', '--install-pkgs', type='string', action='store', - dest='install_pkgs', default=None, - help='Specify what type of packages to be installed,' - ' valid: source, debuginfo, debugsource') - optparser.add_option('', '--tmpfs', action='store_true', dest='enabletmpfs', - help='Setup tmpdir as tmpfs to accelerate, experimental' - ' feature, use it if you have more than 4G memory') - optparser.add_option('', '--repourl', action='append', - dest='repourl', default=[], - help=SUPPRESS_HELP) - return optparser - - def preoptparse(self, argv): - optparser = self.get_optparser() - - largs = [] - rargs = [] - while argv: - arg = argv.pop(0) - - if arg in ('-h', '--help'): - rargs.append(arg) - - elif optparser.has_option(arg): - largs.append(arg) - - if optparser.get_option(arg).takes_value(): - try: - largs.append(argv.pop(0)) - except IndexError: - raise errors.Usage("option %s requires arguments" % arg) - - else: - if arg.startswith("--"): - if "=" in arg: - opt = arg.split("=")[0] - else: - opt = None - elif arg.startswith("-") and len(arg) > 2: - opt = arg[0:2] - else: - opt = None - - if opt and optparser.has_option(opt): - largs.append(arg) - else: - rargs.append(arg) - - return largs + rargs - - def postoptparse(self): - abspath = lambda pth: os.path.abspath(os.path.expanduser(pth)) - - if self.options.verbose: - msger.set_loglevel('verbose') - if self.options.debug: - msger.set_loglevel('debug') - - if self.options.logfile: - logfile_abs_path = abspath(self.options.logfile) - if os.path.isdir(logfile_abs_path): - raise errors.Usage("logfile's path %s should be file" - % self.options.logfile) - if not os.path.exists(os.path.dirname(logfile_abs_path)): - os.makedirs(os.path.dirname(logfile_abs_path)) - msger.set_interactive(False) - msger.set_logfile(logfile_abs_path) - configmgr.create['logfile'] = self.options.logfile - - if self.options.config: - configmgr.reset() - configmgr._siteconf = self.options.config - - if self.options.outdir is not None: - configmgr.create['outdir'] = abspath(self.options.outdir) - if self.options.cachedir is not None: - configmgr.create['cachedir'] = abspath(self.options.cachedir) - os.environ['ZYPP_LOCKFILE_ROOT'] = configmgr.create['cachedir'] - - for cdir in ('outdir', 'cachedir'): - if os.path.exists(configmgr.create[cdir]) \ - and not os.path.isdir(configmgr.create[cdir]): - msger.error('Invalid directory specified: %s' \ - % configmgr.create[cdir]) - - if self.options.local_pkgs_path is not None: - if not os.path.exists(self.options.local_pkgs_path): - msger.error('Local pkgs directory: \'%s\' not exist' \ - % self.options.local_pkgs_path) - configmgr.create['local_pkgs_path'] = self.options.local_pkgs_path - - if self.options.release: - configmgr.create['release'] = self.options.release.rstrip('/') - - if self.options.record_pkgs: - configmgr.create['record_pkgs'] = [] - for infotype in self.options.record_pkgs.split(','): - if infotype not in ('name', 'content', 'license', 'vcs'): - raise errors.Usage('Invalid pkg recording: %s, valid ones:' - ' "name", "content", "license", "vcs"' \ - % infotype) - - configmgr.create['record_pkgs'].append(infotype) - - if self.options.arch is not None: - supported_arch = sorted(rpmmisc.archPolicies.keys(), reverse=True) - if self.options.arch in supported_arch: - configmgr.create['arch'] = self.options.arch - else: - raise errors.Usage('Invalid architecture: "%s".\n' - ' Supported architectures are: \n' - ' %s' % (self.options.arch, - ', '.join(supported_arch))) - - if self.options.pkgmgr is not None: - configmgr.create['pkgmgr'] = self.options.pkgmgr - - if self.options.runtime: - configmgr.set_runtime(self.options.runtime) - - if self.options.pack_to is not None: - configmgr.create['pack_to'] = self.options.pack_to - - if self.options.copy_kernel: - configmgr.create['copy_kernel'] = self.options.copy_kernel - - if self.options.install_pkgs: - configmgr.create['install_pkgs'] = [] - for pkgtype in self.options.install_pkgs.split(','): - if pkgtype not in ('source', 'debuginfo', 'debugsource'): - raise errors.Usage('Invalid parameter specified: "%s", ' - 'valid values: source, debuginfo, ' - 'debusource' % pkgtype) - - configmgr.create['install_pkgs'].append(pkgtype) - - if self.options.enabletmpfs: - configmgr.create['enabletmpfs'] = self.options.enabletmpfs - - if self.options.repourl: - for item in self.options.repourl: - try: - key, val = item.split('=') - except: - continue - configmgr.create['repourl'][key] = val - - def main(self, argv=None): - if argv is None: - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - self.optparser = self.get_optparser() - if self.optparser: - try: - argv = self.preoptparse(argv) - self.options, args = self.optparser.parse_args(argv) - - except cmdln.CmdlnUserError, ex: - msg = "%s: %s\nTry '%s help' for info.\n"\ - % (self.name, ex, self.name) - msger.error(msg) - - except cmdln.StopOptionProcessing, ex: - return 0 - else: - # optparser=None means no process for opts - self.options, args = None, argv[1:] - - if not args: - return self.emptyline() - - self.postoptparse() - - return self.cmd(args) - - def precmd(self, argv): # check help before cmd - - if '-h' in argv or '?' in argv or '--help' in argv or 'help' in argv: - return argv - - if len(argv) == 1: - return ['help', argv[0]] - - if os.geteuid() != 0: - raise msger.error("Root permission is required, abort") - - return argv - - def do_auto(self, subcmd, opts, *args): - """${cmd_name}: auto detect image type from magic header - - Usage: - ${name} ${cmd_name} <ksfile> - - ${cmd_option_list} - """ - def parse_magic_line(re_str, pstr, ptype='mic'): - ptn = re.compile(re_str) - m = ptn.match(pstr) - if not m or not m.groups(): - return None - - inline_argv = m.group(1).strip() - if ptype == 'mic': - m2 = re.search('(?P<format>\w+)', inline_argv) - elif ptype == 'mic2': - m2 = re.search('(-f|--format(=)?)\s*(?P<format>\w+)', - inline_argv) - else: - return None - - if m2: - cmdname = m2.group('format') - inline_argv = inline_argv.replace(m2.group(0), '') - return (cmdname, inline_argv) - - return None - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - if not os.path.exists(args[0]): - raise errors.CreatorError("Can't find the file: %s" % args[0]) - - with open(args[0], 'r') as rf: - first_line = rf.readline() - - mic_re = '^#\s*-\*-mic-options-\*-\s+(.*)\s+-\*-mic-options-\*-' - mic2_re = '^#\s*-\*-mic2-options-\*-\s+(.*)\s+-\*-mic2-options-\*-' - - result = parse_magic_line(mic_re, first_line, 'mic') \ - or parse_magic_line(mic2_re, first_line, 'mic2') - if not result: - raise errors.KsError("Invalid magic line in file: %s" % args[0]) - - if result[0] not in self._subcmds: - raise errors.KsError("Unsupport format '%s' in %s" - % (result[0], args[0])) - - argv = ' '.join(result + args).split() - self.main(argv) - diff --git a/mic/helpformat.py b/mic/helpformat.py new file mode 100755 index 0000000..e17f716 --- /dev/null +++ b/mic/helpformat.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Local additions to commandline parsing."""
+
+import os
+import re
+import functools
+
+from argparse import RawDescriptionHelpFormatter, ArgumentTypeError
+
+class MICHelpFormatter(RawDescriptionHelpFormatter):
+ """Changed default argparse help output by request from cmdln lovers."""
+
+ def __init__(self, *args, **kwargs):
+ super(MICHelpFormatter, self).__init__(*args, **kwargs)
+ self._aliases = {}
+
+ def add_argument(self, action):
+ """Collect aliases."""
+
+ if action.choices:
+ for item, parser in action.choices.iteritems():
+ self._aliases[str(item)] = parser.get_default('alias')
+
+ return super(MICHelpFormatter, self).add_argument(action)
+
+ def format_help(self):
+ """
+ There is no safe and documented way in argparse to reformat
+ help output through APIs as almost all of them are private,
+ so this method just parses the output and changes it.
+ """
+ result = []
+ subcomm = False
+ for line in super(MICHelpFormatter, self).format_help().split('\n'):
+ if line.strip().startswith('{'):
+ continue
+ if line.startswith('optional arguments:'):
+ line = 'Global Options:'
+ if line.startswith('usage:'):
+ line = "Usage: mic [GLOBAL-OPTS] SUBCOMMAND [OPTS]"
+ if subcomm:
+ match = re.match("[ ]+([^ ]+)[ ]+(.+)", line)
+ if match:
+ name, help_text = match.group(1), match.group(2)
+ alias = self._aliases.get(name) or ''
+ if alias:
+ alias = "(%s)" % alias
+ line = " %-22s%s" % ("%s %s" % (name, alias), help_text)
+ if line.strip().startswith('subcommands:'):
+ line = 'Subcommands:'
+ subcomm = True
+ result.append(line)
+ return '\n'.join(result)
+
+def subparser(func):
+ """Convenient decorator for subparsers."""
+ @functools.wraps(func)
+ def wrapper(parser):
+ """
+ Create subparser
+ Set first line of function's docstring as a help
+ and the rest of the lines as a description.
+ Set attribute 'module' of subparser to 'cmd'+first part of function name
+ """
+ splitted = func.__doc__.split('\n')
+ name = func.__name__.split('_')[0]
+ subpar = parser.add_parser(name, help=splitted[0],
+ description='\n'.join(splitted[1:]),
+ formatter_class=RawDescriptionHelpFormatter)
+ subpar.set_defaults(module="cmd_%s" % name)
+ return func(subpar)
+ return wrapper
+
diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index 4a501dd..01936a1 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -26,13 +26,17 @@ import subprocess import re import tarfile import glob +import json +from datetime import datetime import rpm from mic import kickstart -from mic import msger +from mic import msger, __version__ as VERSION from mic.utils.errors import CreatorError, Abort from mic.utils import misc, grabber, runner, fs_related as fs +from mic.chroot import kill_proc_inchroot +from mic.archive import get_archive_suffixes class BaseImageCreator(object): """Installs a system to a chroot directory. @@ -47,6 +51,8 @@ class BaseImageCreator(object): imgcreate.ImageCreator(ks, "foo").create() """ + # Output image format + img_format = '' def __del__(self): self.cleanup() @@ -63,6 +69,7 @@ class BaseImageCreator(object): """ self.pkgmgr = pkgmgr + self.distro_name = "" self.__builddir = None self.__bindmounts = [] @@ -73,10 +80,13 @@ class BaseImageCreator(object): self.cachedir = "/var/tmp/mic/cache" self.workdir = "/var/tmp/mic/build" self.destdir = "." + self.installerfw_prefix = "INSTALLERFW_" self.target_arch = "noarch" + self.strict_mode = False self._local_pkgs_path = None self.pack_to = None self.repourl = {} + self.multiple_partitions = False # If the kernel is save to the destdir when copy_kernel cmd is called. self._need_copy_kernel = False @@ -86,10 +96,10 @@ class BaseImageCreator(object): if createopts: # Mapping table for variables that have different names. optmap = {"pkgmgr" : "pkgmgr_name", - "outdir" : "destdir", "arch" : "target_arch", "local_pkgs_path" : "_local_pkgs_path", "copy_kernel" : "_need_copy_kernel", + "strict_mode" : "strict_mode", } # update setting from createopts @@ -102,23 +112,21 @@ class BaseImageCreator(object): self.destdir = os.path.abspath(os.path.expanduser(self.destdir)) - if 'release' in createopts and createopts['release']: - self.name = createopts['release'] + '_' + self.name - if self.pack_to: if '@NAME@' in self.pack_to: self.pack_to = self.pack_to.replace('@NAME@', self.name) (tar, ext) = os.path.splitext(self.pack_to) - if ext in (".gz", ".bz2") and tar.endswith(".tar"): + if ext in (".gz", ".bz2", ".lzo", ".bz") and tar.endswith(".tar"): ext = ".tar" + ext - if ext not in misc.pack_formats: + if ext not in get_archive_suffixes(): self.pack_to += ".tar" self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"] # Output image file names self.outimage = [] - + # Output info related with manifest + self.image_files = {} # A flag to generate checksum self._genchecksum = False @@ -144,6 +152,8 @@ class BaseImageCreator(object): if part.fstype and part.fstype == "btrfs": self._dep_checks.append("mkfs.btrfs") break + if len(self.ks.handler.partition.partitions) > 1: + self.multiple_partitions = True if self.target_arch and self.target_arch.startswith("arm"): for dep in self._dep_checks: @@ -401,6 +411,25 @@ class BaseImageCreator(object): s += "sysfs /sys sysfs defaults 0 0\n" return s + def _set_part_env(self, pnum, prop, value): + """ This is a helper function which generates an environment variable + for a property "prop" with value "value" of a partition number "pnum". + + The naming convention is: + * Variables start with INSTALLERFW_PART + * Then goes the partition number, the order is the same as + specified in the KS file + * Then goes the property name + """ + + if value == None: + value = "" + else: + value = str(value) + + name = self.installerfw_prefix + ("PART%d_" % pnum) + prop + return { name : value } + def _get_post_scripts_env(self, in_chroot): """Return an environment dict for %post scripts. @@ -408,13 +437,56 @@ class BaseImageCreator(object): variables for %post scripts by return a dict containing the desired environment. - By default, this returns an empty dict. - in_chroot -- whether this %post script is to be executed chroot()ed into _instroot. - """ - return {} + + env = {} + pnum = 0 + + for p in kickstart.get_partitions(self.ks): + env.update(self._set_part_env(pnum, "SIZE", p.size)) + env.update(self._set_part_env(pnum, "MOUNTPOINT", p.mountpoint)) + env.update(self._set_part_env(pnum, "FSTYPE", p.fstype)) + env.update(self._set_part_env(pnum, "LABEL", p.label)) + env.update(self._set_part_env(pnum, "FSOPTS", p.fsopts)) + env.update(self._set_part_env(pnum, "BOOTFLAG", p.active)) + env.update(self._set_part_env(pnum, "ALIGN", p.align)) + env.update(self._set_part_env(pnum, "TYPE_ID", p.part_type)) + env.update(self._set_part_env(pnum, "UUID", p.uuid)) + env.update(self._set_part_env(pnum, "DEVNODE", + "/dev/%s%d" % (p.disk, pnum + 1))) + env.update(self._set_part_env(pnum, "DISK_DEVNODE", + "/dev/%s" % p.disk)) + pnum += 1 + + # Count of paritions + env[self.installerfw_prefix + "PART_COUNT"] = str(pnum) + + # Partition table format + ptable_format = self.ks.handler.bootloader.ptable + env[self.installerfw_prefix + "PTABLE_FORMAT"] = ptable_format + + # The kerned boot parameters + kernel_opts = self.ks.handler.bootloader.appendLine + env[self.installerfw_prefix + "KERNEL_OPTS"] = kernel_opts + + # Name of the image creation tool + env[self.installerfw_prefix + "INSTALLER_NAME"] = "mic" + + # The real current location of the mounted file-systems + if in_chroot: + mount_prefix = "/" + else: + mount_prefix = self._instroot + env[self.installerfw_prefix + "MOUNT_PREFIX"] = mount_prefix + + # These are historical variables which lack the common name prefix + if not in_chroot: + env["INSTALL_ROOT"] = self._instroot + env["IMG_NAME"] = self._name + + return env def __get_imgname(self): return self.name @@ -621,9 +693,15 @@ class BaseImageCreator(object): raise CreatorError("No repositories specified") def __write_fstab(self): - fstab = open(self._instroot + "/etc/fstab", "w") - fstab.write(self._get_fstab()) - fstab.close() + if kickstart.use_installerfw(self.ks, "fstab"): + # The fstab file will be generated by installer framework scripts + # instead. + return None + fstab_contents = self._get_fstab() + if fstab_contents: + fstab = open(self._instroot + "/etc/fstab", "w") + fstab.write(fstab_contents) + fstab.close() def __create_minimal_dev(self): """Create a minimal /dev so that we don't corrupt the host /dev""" @@ -703,11 +781,11 @@ class BaseImageCreator(object): "/usr/bin"): fs.makedirs(self._instroot + d) - if self.target_arch and self.target_arch.startswith("arm"): + if self.target_arch and self.target_arch.startswith("arm") or \ + self.target_arch == "aarch64": self.qemu_emulator = misc.setup_qemu_emulator(self._instroot, self.target_arch) - self.get_cachedir(cachedir) # bind mount system directories into _instroot @@ -789,6 +867,8 @@ class BaseImageCreator(object): if not self.__builddir: return + kill_proc_inchroot(self._instroot) + self.unmount() shutil.rmtree(self.__builddir, ignore_errors = True) @@ -857,6 +937,10 @@ class BaseImageCreator(object): for pkg in self._preinstall_pkgs: pkg_manager.preInstall(pkg) + def __check_packages(self, pkg_manager): + for pkg in self.check_pkgs: + pkg_manager.checkPackage(pkg) + def __attachment_packages(self, pkg_manager): if not self.ks: return @@ -886,7 +970,7 @@ class BaseImageCreator(object): if not os.path.exists(fpath): # download pkgs try: - fpath = grabber.myurlgrab(url, fpath, proxies, None) + fpath = grabber.myurlgrab(url.full, fpath, proxies, None) except CreatorError: raise @@ -904,11 +988,28 @@ class BaseImageCreator(object): into the install root. By default, the packages are installed from the repository URLs specified in the kickstart. - repo_urls -- a dict which maps a repository name to a repository URL; + repo_urls -- a dict which maps a repository name to a repository; if supplied, this causes any repository URLs specified in the kickstart to be overridden. """ + def checkScriptletError(dirname, suffix): + if os.path.exists(dirname): + list = os.listdir(dirname) + for line in list: + filepath = os.path.join(dirname, line) + if os.path.isfile(filepath) and 0 < line.find(suffix): + return True + else: + continue + + return False + + def get_ssl_verify(ssl_verify=None): + if ssl_verify is not None: + return not ssl_verify.lower().strip() == 'no' + else: + return not self.ssl_verify.lower().strip() == 'no' # initialize pkg list to install if self.ks: @@ -924,6 +1025,9 @@ class BaseImageCreator(object): self._excluded_pkgs = None self._required_groups = None + if not repo_urls: + repo_urls = self.extrarepos + pkg_manager = self.get_pkg_manager() pkg_manager.setup() @@ -931,12 +1035,13 @@ class BaseImageCreator(object): if 'debuginfo' in self.install_pkgs: pkg_manager.install_debuginfo = True - for repo in kickstart.get_repos(self.ks, repo_urls): + for repo in kickstart.get_repos(self.ks, repo_urls, self.ignore_ksrepo): (name, baseurl, mirrorlist, inc, exc, proxy, proxy_username, proxy_password, debuginfo, source, gpgkey, disable, ssl_verify, nocache, cost, priority) = repo + ssl_verify = get_ssl_verify(ssl_verify) yr = pkg_manager.addRepository(name, baseurl, mirrorlist, proxy, proxy_username, proxy_password, inc, exc, ssl_verify, nocache, cost, priority) @@ -954,6 +1059,7 @@ class BaseImageCreator(object): self.__select_groups(pkg_manager) self.__deselect_packages(pkg_manager) self.__localinst_packages(pkg_manager) + self.__check_packages(pkg_manager) BOOT_SAFEGUARD = 256L * 1024 * 1024 # 256M checksize = self._root_fs_avail @@ -961,6 +1067,11 @@ class BaseImageCreator(object): checksize -= BOOT_SAFEGUARD if self.target_arch: pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_IGNOREARCH) + + # If we have multiple partitions, don't check diskspace when rpm run transaction + # because rpm check '/' partition only. + if self.multiple_partitions: + pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_DISKSPACE) pkg_manager.runInstall(checksize) except CreatorError, e: raise @@ -974,6 +1085,9 @@ class BaseImageCreator(object): finally: pkg_manager.close() + if checkScriptletError(self._instroot + "/tmp/.postscript/error/", "_error"): + raise CreatorError('scriptlet errors occurred') + # hook post install self.postinstall() @@ -1004,10 +1118,10 @@ class BaseImageCreator(object): os.chmod(path, 0700) env = self._get_post_scripts_env(s.inChroot) + if 'PATH' not in env: + env['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin' if not s.inChroot: - env["INSTALL_ROOT"] = self._instroot - env["IMG_NAME"] = self._name preexec = None script = path else: @@ -1016,11 +1130,13 @@ class BaseImageCreator(object): try: try: - subprocess.call([s.interp, script], - preexec_fn = preexec, - env = env, - stdout = sys.stdout, - stderr = sys.stderr) + p = subprocess.Popen([s.interp, script], + preexec_fn = preexec, + env = env, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT) + for entry in p.communicate()[0].splitlines(): + msger.info(entry) except OSError, (err, msg): raise CreatorError("Failed to execute %%post script " "with '%s' : %s" % (s.interp, msg)) @@ -1090,7 +1206,7 @@ class BaseImageCreator(object): md5sum = misc.get_md5sum(image_name) with open(image_name + ".md5sum", "w") as f: - f.write("%s %s" % (md5sum, os.path.basename(image_name))) + f.write("%s %s" % (md5sum, os.path.basename(image_name))) self.outimage.append(image_name+".md5sum") def package(self, destdir = "."): @@ -1199,13 +1315,8 @@ class BaseImageCreator(object): outimages.append(new_kspath) # save log file, logfile is only available in creator attrs - if hasattr(self, 'logfile') and not self.logfile: - log_path = _rpath(self.name + ".log") - # touch the log file, else outimages will filter it out - with open(log_path, 'w') as wf: - wf.write('') - msger.set_logfile(log_path) - outimages.append(_rpath(self.name + ".log")) + if hasattr(self, 'releaselog') and self.releaselog: + outimages.append(self.logfile) # rename iso and usbimg for f in os.listdir(destdir): @@ -1218,22 +1329,32 @@ class BaseImageCreator(object): os.rename(_rpath(f), _rpath(newf)) outimages.append(_rpath(newf)) - # generate MD5SUMS - with open(_rpath("MD5SUMS"), "w") as wf: - for f in os.listdir(destdir): - if f == "MD5SUMS": - continue + # generate MD5SUMS SHA1SUMS SHA256SUMS + def generate_hashsum(hash_name, hash_method): + with open(_rpath(hash_name), "w") as wf: + for f in os.listdir(destdir): + if f.endswith('SUMS'): + continue - if os.path.isdir(os.path.join(destdir, f)): - continue + if os.path.isdir(os.path.join(destdir, f)): + continue + + hash_value = hash_method(_rpath(f)) + # There needs to be two spaces between the sum and + # filepath to match the syntax with md5sum,sha1sum, + # sha256sum. This way also *sum -c *SUMS can be used. + wf.write("%s %s\n" % (hash_value, f)) + + outimages.append("%s/%s" % (destdir, hash_name)) - md5sum = misc.get_md5sum(_rpath(f)) - # There needs to be two spaces between the sum and - # filepath to match the syntax with md5sum. - # This way also md5sum -c MD5SUMS can be used by users - wf.write("%s *%s\n" % (md5sum, f)) + hash_dict = { + 'MD5SUMS' : misc.get_md5sum, + 'SHA1SUMS' : misc.get_sha1sum, + 'SHA256SUMS' : misc.get_sha256sum + } - outimages.append("%s/MD5SUMS" % destdir) + for k, v in hash_dict.items(): + generate_hashsum(k, v) # Filter out the nonexist file for fp in outimages[:]: @@ -1269,4 +1390,32 @@ class BaseImageCreator(object): def get_pkg_manager(self): return self.pkgmgr(target_arch = self.target_arch, instroot = self._instroot, - cachedir = self.cachedir) + cachedir = self.cachedir, + strict_mode = self.strict_mode) + + def create_manifest(self): + def get_pack_suffix(): + return '.' + self.pack_to.split('.', 1)[1] + + if not os.path.exists(self.destdir): + os.makedirs(self.destdir) + + now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + manifest_dict = {'version': VERSION, + 'created': now} + if self.img_format: + manifest_dict.update({'format': self.img_format}) + + if hasattr(self, 'logfile') and self.logfile: + manifest_dict.update({'log_file': self.logfile}) + + if self.image_files: + if self.pack_to: + self.image_files.update({'pack': get_pack_suffix()}) + manifest_dict.update({self.img_format: self.image_files}) + + msger.info('Creating manifest file...') + manifest_file_path = os.path.join(self.destdir, 'manifest.json') + with open(manifest_file_path, 'w') as fest_file: + json.dump(manifest_dict, fest_file, indent=4) + self.outimage.append(manifest_file_path) diff --git a/mic/imager/fs.py b/mic/imager/fs.py index d53b29c..f2ccb78 100644 --- a/mic/imager/fs.py +++ b/mic/imager/fs.py @@ -24,6 +24,8 @@ from mic.utils.fs_related import find_binary_path from mic.imager.baseimager import BaseImageCreator class FsImageCreator(BaseImageCreator): + img_format = 'fs' + def __init__(self, cfgmgr = None, pkgmgr = None): self.zips = { "tar.bz2" : "" @@ -48,6 +50,7 @@ class FsImageCreator(BaseImageCreator): self._save_recording_pkgs(destdir) if not self.pack_to: + self.image_files = {'image_files': [self.name]} fsdir = os.path.join(destdir, self.name) misc.check_space_pre_cp(self._instroot, destdir) @@ -61,6 +64,7 @@ class FsImageCreator(BaseImageCreator): self.outimage.append(fsdir) else: + self.image_files = {'image_files': [self.pack_to]} (tar, comp) = os.path.splitext(self.pack_to) try: tarcreat = {'.tar': '-cf', @@ -78,7 +82,6 @@ class FsImageCreator(BaseImageCreator): tar = find_binary_path('tar') tar_cmdline = [tar, "--numeric-owner", "--preserve-permissions", - "--preserve-order", "--one-file-system", "--directory", self._instroot] @@ -96,4 +99,3 @@ class FsImageCreator(BaseImageCreator): "Cmdline: %s" % (" ".join(tar_cmdline))) self.outimage.append(dst) - diff --git a/mic/imager/livecd.py b/mic/imager/livecd.py deleted file mode 100644 index a992ee0..0000000 --- a/mic/imager/livecd.py +++ /dev/null @@ -1,750 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys -import glob -import shutil - -from mic import kickstart, msger -from mic.utils import fs_related, rpmmisc, runner, misc -from mic.utils.errors import CreatorError -from mic.imager.loop import LoopImageCreator - - -class LiveImageCreatorBase(LoopImageCreator): - """A base class for LiveCD image creators. - - This class serves as a base class for the architecture-specific LiveCD - image creator subclass, LiveImageCreator. - - LiveImageCreator creates a bootable ISO containing the system image, - bootloader, bootloader configuration, kernel and initramfs. - """ - - def __init__(self, creatoropts = None, pkgmgr = None): - """Initialise a LiveImageCreator instance. - - This method takes the same arguments as ImageCreator.__init__(). - """ - LoopImageCreator.__init__(self, creatoropts, pkgmgr) - - #Controls whether to use squashfs to compress the image. - self.skip_compression = False - - #Controls whether an image minimizing snapshot should be created. - # - #This snapshot can be used when copying the system image from the ISO in - #order to minimize the amount of data that needs to be copied; simply, - #it makes it possible to create a version of the image's filesystem with - #no spare space. - self.skip_minimize = False - - #A flag which indicates i act as a convertor default false - self.actasconvertor = False - - #The bootloader timeout from kickstart. - if self.ks: - self._timeout = kickstart.get_timeout(self.ks, 10) - else: - self._timeout = 10 - - #The default kernel type from kickstart. - if self.ks: - self._default_kernel = kickstart.get_default_kernel(self.ks, - "kernel") - else: - self._default_kernel = None - - if self.ks: - parts = kickstart.get_partitions(self.ks) - if len(parts) > 1: - raise CreatorError("Can't support multi partitions in ks file " - "for this image type") - # FIXME: rename rootfs img to self.name, - # else can't find files when create iso - self._instloops[0]['name'] = self.name + ".img" - - self.__isodir = None - - self.__modules = ["=ata", - "sym53c8xx", - "aic7xxx", - "=usb", - "=firewire", - "=mmc", - "=pcmcia", - "mptsas"] - if self.ks: - self.__modules.extend(kickstart.get_modules(self.ks)) - - self._dep_checks.extend(["isohybrid", - "unsquashfs", - "mksquashfs", - "dd", - "genisoimage"]) - - # - # Hooks for subclasses - # - def _configure_bootloader(self, isodir): - """Create the architecture specific booloader configuration. - - This is the hook where subclasses must create the booloader - configuration in order to allow a bootable ISO to be built. - - isodir -- the directory where the contents of the ISO are to - be staged - """ - raise CreatorError("Bootloader configuration is arch-specific, " - "but not implemented for this arch!") - def _get_menu_options(self): - """Return a menu options string for syslinux configuration. - """ - if self.ks is None: - return "liveinst autoinst" - r = kickstart.get_menu_args(self.ks) - return r - - def _get_kernel_options(self): - """Return a kernel options string for bootloader configuration. - - This is the hook where subclasses may specify a set of kernel - options which should be included in the images bootloader - configuration. - - A sensible default implementation is provided. - """ - - if self.ks is None: - r = "ro rd.live.image" - else: - r = kickstart.get_kernel_args(self.ks) - - return r - - def _get_mkisofs_options(self, isodir): - """Return the architecture specific mkisosfs options. - - This is the hook where subclasses may specify additional arguments - to mkisofs, e.g. to enable a bootable ISO to be built. - - By default, an empty list is returned. - """ - return [] - - # - # Helpers for subclasses - # - def _has_checkisomd5(self): - """Check whether checkisomd5 is available in the install root.""" - def _exists(path): - return os.path.exists(self._instroot + path) - - if _exists("/usr/bin/checkisomd5") and os.path.exists("/usr/bin/implantisomd5"): - return True - - return False - - def __restore_file(self,path): - try: - os.unlink(path) - except: - pass - if os.path.exists(path + '.rpmnew'): - os.rename(path + '.rpmnew', path) - - def _mount_instroot(self, base_on = None): - LoopImageCreator._mount_instroot(self, base_on) - self.__write_initrd_conf(self._instroot + "/etc/sysconfig/mkinitrd") - self.__write_dracut_conf(self._instroot + "/etc/dracut.conf.d/02livecd.conf") - - def _unmount_instroot(self): - self.__restore_file(self._instroot + "/etc/sysconfig/mkinitrd") - self.__restore_file(self._instroot + "/etc/dracut.conf.d/02livecd.conf") - LoopImageCreator._unmount_instroot(self) - - def __ensure_isodir(self): - if self.__isodir is None: - self.__isodir = self._mkdtemp("iso-") - return self.__isodir - - def _get_isodir(self): - return self.__ensure_isodir() - - def _set_isodir(self, isodir = None): - self.__isodir = isodir - - def _create_bootconfig(self): - """Configure the image so that it's bootable.""" - self._configure_bootloader(self.__ensure_isodir()) - - def _get_post_scripts_env(self, in_chroot): - env = LoopImageCreator._get_post_scripts_env(self, in_chroot) - - if not in_chroot: - env["LIVE_ROOT"] = self.__ensure_isodir() - - return env - def __write_dracut_conf(self, path): - if not os.path.exists(os.path.dirname(path)): - fs_related.makedirs(os.path.dirname(path)) - f = open(path, "a") - f.write('add_dracutmodules+=" dmsquash-live pollcdrom "') - f.close() - - def __write_initrd_conf(self, path): - content = "" - if not os.path.exists(os.path.dirname(path)): - fs_related.makedirs(os.path.dirname(path)) - f = open(path, "w") - - content += 'LIVEOS="yes"\n' - content += 'PROBE="no"\n' - content += 'MODULES+="squashfs ext3 ext2 vfat msdos "\n' - content += 'MODULES+="sr_mod sd_mod ide-cd cdrom "\n' - - for module in self.__modules: - if module == "=usb": - content += 'MODULES+="ehci_hcd uhci_hcd ohci_hcd "\n' - content += 'MODULES+="usb_storage usbhid "\n' - elif module == "=firewire": - content += 'MODULES+="firewire-sbp2 firewire-ohci "\n' - content += 'MODULES+="sbp2 ohci1394 ieee1394 "\n' - elif module == "=mmc": - content += 'MODULES+="mmc_block sdhci sdhci-pci "\n' - elif module == "=pcmcia": - content += 'MODULES+="pata_pcmcia "\n' - else: - content += 'MODULES+="' + module + ' "\n' - f.write(content) - f.close() - - def __create_iso(self, isodir): - iso = self._outdir + "/" + self.name + ".iso" - genisoimage = fs_related.find_binary_path("genisoimage") - args = [genisoimage, - "-J", "-r", - "-hide-rr-moved", "-hide-joliet-trans-tbl", - "-V", self.fslabel, - "-o", iso] - - args.extend(self._get_mkisofs_options(isodir)) - - args.append(isodir) - - if runner.show(args) != 0: - raise CreatorError("ISO creation failed!") - - """ It should be ok still even if you haven't isohybrid """ - isohybrid = None - try: - isohybrid = fs_related.find_binary_path("isohybrid") - except: - pass - - if isohybrid: - args = [isohybrid, "-partok", iso ] - if runner.show(args) != 0: - raise CreatorError("Hybrid ISO creation failed!") - - self.__implant_md5sum(iso) - - def __implant_md5sum(self, iso): - """Implant an isomd5sum.""" - if os.path.exists("/usr/bin/implantisomd5"): - implantisomd5 = "/usr/bin/implantisomd5" - else: - msger.warning("isomd5sum not installed; not setting up mediacheck") - implantisomd5 = "" - return - - runner.show([implantisomd5, iso]) - - def _stage_final_image(self): - try: - fs_related.makedirs(self.__ensure_isodir() + "/LiveOS") - - minimal_size = self._resparse() - - if not self.skip_minimize: - fs_related.create_image_minimizer(self.__isodir + \ - "/LiveOS/osmin.img", - self._image, - minimal_size) - - if self.skip_compression: - shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img") - else: - fs_related.makedirs(os.path.join( - os.path.dirname(self._image), - "LiveOS")) - shutil.move(self._image, - os.path.join(os.path.dirname(self._image), - "LiveOS", "ext3fs.img")) - fs_related.mksquashfs(os.path.dirname(self._image), - self.__isodir + "/LiveOS/squashfs.img") - - self.__create_iso(self.__isodir) - - if self.pack_to: - isoimg = os.path.join(self._outdir, self.name + ".iso") - packimg = os.path.join(self._outdir, self.pack_to) - misc.packing(packimg, isoimg) - os.unlink(isoimg) - - finally: - shutil.rmtree(self.__isodir, ignore_errors = True) - self.__isodir = None - -class x86LiveImageCreator(LiveImageCreatorBase): - """ImageCreator for x86 machines""" - def _get_mkisofs_options(self, isodir): - return [ "-b", "isolinux/isolinux.bin", - "-c", "isolinux/boot.cat", - "-no-emul-boot", "-boot-info-table", - "-boot-load-size", "4" ] - - def _get_required_packages(self): - return ["syslinux", "syslinux-extlinux"] + \ - LiveImageCreatorBase._get_required_packages(self) - - def _get_isolinux_stanzas(self, isodir): - return "" - - def __find_syslinux_menu(self): - for menu in ["vesamenu.c32", "menu.c32"]: - if os.path.isfile(self._instroot + "/usr/share/syslinux/" + menu): - return menu - - raise CreatorError("syslinux not installed : " - "no suitable /usr/share/syslinux/*menu.c32 found") - - def __find_syslinux_mboot(self): - # - # We only need the mboot module if we have any xen hypervisors - # - if not glob.glob(self._instroot + "/boot/xen.gz*"): - return None - - return "mboot.c32" - - def __copy_syslinux_files(self, isodir, menu, mboot = None): - files = ["isolinux.bin", menu] - if mboot: - files += [mboot] - - for f in files: - path = self._instroot + "/usr/share/syslinux/" + f - - if not os.path.isfile(path): - raise CreatorError("syslinux not installed : " - "%s not found" % path) - - shutil.copy(path, isodir + "/isolinux/") - - def __copy_syslinux_background(self, isodest): - background_path = self._instroot + \ - "/usr/share/branding/default/syslinux/syslinux-vesa-splash.jpg" - - if not os.path.exists(background_path): - return False - - shutil.copyfile(background_path, isodest) - - return True - - def __copy_kernel_and_initramfs(self, isodir, version, index): - bootdir = self._instroot + "/boot" - isDracut = False - - if self._alt_initrd_name: - src_initrd_path = os.path.join(bootdir, self._alt_initrd_name) - else: - if os.path.exists(bootdir + "/initramfs-" + version + ".img"): - src_initrd_path = os.path.join(bootdir, "initramfs-" +version+ ".img") - isDracut = True - else: - src_initrd_path = os.path.join(bootdir, "initrd-" +version+ ".img") - - try: - msger.debug("copy %s to %s" % (bootdir + "/vmlinuz-" + version, isodir + "/isolinux/vmlinuz" + index)) - shutil.copyfile(bootdir + "/vmlinuz-" + version, - isodir + "/isolinux/vmlinuz" + index) - - msger.debug("copy %s to %s" % (src_initrd_path, isodir + "/isolinux/initrd" + index + ".img")) - shutil.copyfile(src_initrd_path, - isodir + "/isolinux/initrd" + index + ".img") - except: - raise CreatorError("Unable to copy valid kernels or initrds, " - "please check the repo.") - - is_xen = False - if os.path.exists(bootdir + "/xen.gz-" + version[:-3]): - shutil.copyfile(bootdir + "/xen.gz-" + version[:-3], - isodir + "/isolinux/xen" + index + ".gz") - is_xen = True - - return (is_xen,isDracut) - - def __is_default_kernel(self, kernel, kernels): - if len(kernels) == 1: - return True - - if kernel == self._default_kernel: - return True - - if kernel.startswith("kernel-") and kernel[7:] == self._default_kernel: - return True - - return False - - def __get_basic_syslinux_config(self, **args): - return """ -default %(menu)s -timeout %(timeout)d - -%(background)s -menu title Welcome to %(distroname)s! -menu color border 0 #ffffffff #00000000 -menu color sel 7 #ff000000 #ffffffff -menu color title 0 #ffffffff #00000000 -menu color tabmsg 0 #ffffffff #00000000 -menu color unsel 0 #ffffffff #00000000 -menu color hotsel 0 #ff000000 #ffffffff -menu color hotkey 7 #ffffffff #ff000000 -menu color timeout_msg 0 #ffffffff #00000000 -menu color timeout 0 #ffffffff #00000000 -menu color cmdline 0 #ffffffff #00000000 -menu hidden -menu clear -""" % args - - def __get_image_stanza(self, is_xen, isDracut, **args): - if isDracut: - args["rootlabel"] = "live:CDLABEL=%(fslabel)s" % args - else: - args["rootlabel"] = "CDLABEL=%(fslabel)s" % args - if not is_xen: - template = """label %(short)s - menu label %(long)s - kernel vmlinuz%(index)s - append initrd=initrd%(index)s.img root=%(rootlabel)s rootfstype=iso9660 %(liveargs)s %(extra)s -""" - else: - template = """label %(short)s - menu label %(long)s - kernel mboot.c32 - append xen%(index)s.gz --- vmlinuz%(index)s root=%(rootlabel)s rootfstype=iso9660 %(liveargs)s %(extra)s --- initrd%(index)s.img -""" - return template % args - - def __get_image_stanzas(self, isodir): - versions = [] - kernels = self._get_kernel_versions() - for kernel in kernels: - for version in kernels[kernel]: - versions.append(version) - - if not versions: - raise CreatorError("Unable to find valid kernels, " - "please check the repo") - - kernel_options = self._get_kernel_options() - - """ menu can be customized highly, the format is: - - short_name1:long_name1:extra_opts1;short_name2:long_name2:extra_opts2 - - e.g.: autoinst:InstallationOnly:systemd.unit=installer-graphical.service - but in order to keep compatible with old format, these are still ok: - - liveinst autoinst - liveinst;autoinst - liveinst::;autoinst:: - """ - oldmenus = {"basic": { - "short": "basic", - "long": "Installation Only (Text based)", - "extra": "basic nosplash 4" - }, - "liveinst": { - "short": "liveinst", - "long": "Installation Only", - "extra": "liveinst nosplash 4" - }, - "autoinst": { - "short": "autoinst", - "long": "Autoinstall (Deletes all existing content)", - "extra": "autoinst nosplash 4" - }, - "netinst": { - "short": "netinst", - "long": "Network Installation", - "extra": "netinst 4" - }, - "verify": { - "short": "check", - "long": "Verify and", - "extra": "check" - } - } - menu_options = self._get_menu_options() - menus = menu_options.split(";") - for i in range(len(menus)): - menus[i] = menus[i].split(":") - if len(menus) == 1 and len(menus[0]) == 1: - """ Keep compatible with the old usage way """ - menus = menu_options.split() - for i in range(len(menus)): - menus[i] = [menus[i]] - - cfg = "" - - default_version = None - default_index = None - index = "0" - netinst = None - for version in versions: - (is_xen, isDracut) = self.__copy_kernel_and_initramfs(isodir, version, index) - if index == "0": - self._isDracut = isDracut - - default = self.__is_default_kernel(kernel, kernels) - - if default: - long = "Boot %s" % self.distro_name - elif kernel.startswith("kernel-"): - long = "Boot %s(%s)" % (self.name, kernel[7:]) - else: - long = "Boot %s(%s)" % (self.name, kernel) - - oldmenus["verify"]["long"] = "%s %s" % (oldmenus["verify"]["long"], - long) - # tell dracut not to ask for LUKS passwords or activate mdraid sets - if isDracut: - kern_opts = kernel_options + " rd.luks=0 rd.md=0 rd.dm=0" - else: - kern_opts = kernel_options - - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kern_opts, - long = long, - short = "linux" + index, - extra = "", - index = index) - - if default: - cfg += "menu default\n" - default_version = version - default_index = index - - for menu in menus: - if not menu[0]: - continue - short = menu[0] + index - - if len(menu) >= 2: - long = menu[1] - else: - if menu[0] in oldmenus.keys(): - if menu[0] == "verify" and not self._has_checkisomd5(): - continue - if menu[0] == "netinst": - netinst = oldmenus[menu[0]] - continue - long = oldmenus[menu[0]]["long"] - extra = oldmenus[menu[0]]["extra"] - else: - long = short.upper() + " X" + index - extra = "" - - if len(menu) >= 3: - extra = menu[2] - - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kernel_options, - long = long, - short = short, - extra = extra, - index = index) - - index = str(int(index) + 1) - - if not default_version: - default_version = versions[0] - if not default_index: - default_index = "0" - - if netinst: - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kernel_options, - long = netinst["long"], - short = netinst["short"], - extra = netinst["extra"], - index = default_index) - - return cfg - - def __get_memtest_stanza(self, isodir): - memtest = glob.glob(self._instroot + "/boot/memtest86*") - if not memtest: - return "" - - shutil.copyfile(memtest[0], isodir + "/isolinux/memtest") - - return """label memtest - menu label Memory Test - kernel memtest -""" - - def __get_local_stanza(self, isodir): - return """label local - menu label Boot from local drive - localboot 0xffff -""" - - def _configure_syslinux_bootloader(self, isodir): - """configure the boot loader""" - fs_related.makedirs(isodir + "/isolinux") - - menu = self.__find_syslinux_menu() - - self.__copy_syslinux_files(isodir, menu, - self.__find_syslinux_mboot()) - - background = "" - if self.__copy_syslinux_background(isodir + "/isolinux/splash.jpg"): - background = "menu background splash.jpg" - - cfg = self.__get_basic_syslinux_config(menu = menu, - background = background, - name = self.name, - timeout = self._timeout * 10, - distroname = self.distro_name) - - cfg += self.__get_image_stanzas(isodir) - cfg += self.__get_memtest_stanza(isodir) - cfg += self.__get_local_stanza(isodir) - cfg += self._get_isolinux_stanzas(isodir) - - cfgf = open(isodir + "/isolinux/isolinux.cfg", "w") - cfgf.write(cfg) - cfgf.close() - - def __copy_efi_files(self, isodir): - if not os.path.exists(self._instroot + "/boot/efi/EFI/redhat/grub.efi"): - return False - shutil.copy(self._instroot + "/boot/efi/EFI/redhat/grub.efi", - isodir + "/EFI/boot/grub.efi") - shutil.copy(self._instroot + "/boot/grub/splash.xpm.gz", - isodir + "/EFI/boot/splash.xpm.gz") - - return True - - def __get_basic_efi_config(self, **args): - return """ -default=0 -splashimage=/EFI/boot/splash.xpm.gz -timeout %(timeout)d -hiddenmenu - -""" %args - - def __get_efi_image_stanza(self, **args): - return """title %(long)s - kernel /EFI/boot/vmlinuz%(index)s root=CDLABEL=%(fslabel)s rootfstype=iso9660 %(liveargs)s %(extra)s - initrd /EFI/boot/initrd%(index)s.img -""" %args - - def __get_efi_image_stanzas(self, isodir, name): - # FIXME: this only supports one kernel right now... - - kernel_options = self._get_kernel_options() - checkisomd5 = self._has_checkisomd5() - - cfg = "" - - for index in range(0, 9): - # we don't support xen kernels - if os.path.exists("%s/EFI/boot/xen%d.gz" %(isodir, index)): - continue - cfg += self.__get_efi_image_stanza(fslabel = self.fslabel, - liveargs = kernel_options, - long = name, - extra = "", index = index) - if checkisomd5: - cfg += self.__get_efi_image_stanza( - fslabel = self.fslabel, - liveargs = kernel_options, - long = "Verify and Boot " + name, - extra = "check", - index = index) - break - - return cfg - - def _configure_efi_bootloader(self, isodir): - """Set up the configuration for an EFI bootloader""" - fs_related.makedirs(isodir + "/EFI/boot") - - if not self.__copy_efi_files(isodir): - shutil.rmtree(isodir + "/EFI") - return - - for f in os.listdir(isodir + "/isolinux"): - os.link("%s/isolinux/%s" %(isodir, f), - "%s/EFI/boot/%s" %(isodir, f)) - - - cfg = self.__get_basic_efi_config(name = self.name, - timeout = self._timeout) - cfg += self.__get_efi_image_stanzas(isodir, self.name) - - cfgf = open(isodir + "/EFI/boot/grub.conf", "w") - cfgf.write(cfg) - cfgf.close() - - # first gen mactel machines get the bootloader name wrong apparently - if rpmmisc.getBaseArch() == "i386": - os.link(isodir + "/EFI/boot/grub.efi", - isodir + "/EFI/boot/boot.efi") - os.link(isodir + "/EFI/boot/grub.conf", - isodir + "/EFI/boot/boot.conf") - - # for most things, we want them named boot$efiarch - efiarch = {"i386": "ia32", "x86_64": "x64"} - efiname = efiarch[rpmmisc.getBaseArch()] - os.rename(isodir + "/EFI/boot/grub.efi", - isodir + "/EFI/boot/boot%s.efi" %(efiname,)) - os.link(isodir + "/EFI/boot/grub.conf", - isodir + "/EFI/boot/boot%s.conf" %(efiname,)) - - - def _configure_bootloader(self, isodir): - self._configure_syslinux_bootloader(isodir) - self._configure_efi_bootloader(isodir) - -arch = rpmmisc.getBaseArch() -if arch in ("i386", "x86_64"): - LiveCDImageCreator = x86LiveImageCreator -elif arch.startswith("arm"): - LiveCDImageCreator = LiveImageCreatorBase -else: - raise CreatorError("Architecture not supported!") diff --git a/mic/imager/liveusb.py b/mic/imager/liveusb.py deleted file mode 100644 index a909928..0000000 --- a/mic/imager/liveusb.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import re - -from mic import msger -from mic.utils import misc, fs_related, runner -from mic.utils.errors import CreatorError, MountError -from mic.utils.partitionedfs import PartitionedMount -from mic.imager.livecd import LiveCDImageCreator - - -class LiveUSBImageCreator(LiveCDImageCreator): - def __init__(self, *args): - LiveCDImageCreator.__init__(self, *args) - - self._dep_checks.extend(["kpartx", "parted"]) - - # remove dependency of genisoimage in parent class - if "genisoimage" in self._dep_checks: - self._dep_checks.remove("genisoimage") - - def _create_usbimg(self, isodir): - overlaysizemb = 64 #default - #skipcompress = self.skip_compression? - fstype = "vfat" - homesizemb=0 - swapsizemb=0 - homefile="home.img" - plussize=128 - kernelargs=None - - if fstype == 'vfat': - if overlaysizemb > 2047: - raise CreatorError("Can't have an overlay of 2048MB or " - "greater on VFAT") - - if homesizemb > 2047: - raise CreatorError("Can't have an home overlay of 2048MB or " - "greater on VFAT") - - if swapsizemb > 2047: - raise CreatorError("Can't have an swap overlay of 2048MB or " - "greater on VFAT") - - livesize = misc.get_file_size(isodir + "/LiveOS") - - usbimgsize = (overlaysizemb + \ - homesizemb + \ - swapsizemb + \ - livesize + \ - plussize) * 1024L * 1024L - - disk = fs_related.SparseLoopbackDisk("%s/%s.usbimg" \ - % (self._outdir, self.name), - usbimgsize) - usbmnt = self._mkdtemp("usb-mnt") - usbloop = PartitionedMount(usbmnt) - usbloop.add_disk('/dev/sdb', disk) - - usbloop.add_partition(usbimgsize/1024/1024, - "/dev/sdb", - "/", - fstype, - boot=True) - - usbloop.mount() - - try: - fs_related.makedirs(usbmnt + "/LiveOS") - - if os.path.exists(isodir + "/LiveOS/squashfs.img"): - shutil.copyfile(isodir + "/LiveOS/squashfs.img", - usbmnt + "/LiveOS/squashfs.img") - else: - fs_related.mksquashfs(os.path.dirname(self._image), - usbmnt + "/LiveOS/squashfs.img") - - if os.path.exists(isodir + "/LiveOS/osmin.img"): - shutil.copyfile(isodir + "/LiveOS/osmin.img", - usbmnt + "/LiveOS/osmin.img") - - if fstype == "vfat" or fstype == "msdos": - uuid = usbloop.partitions[0]['mount'].uuid - label = usbloop.partitions[0]['mount'].fslabel - usblabel = "UUID=%s-%s" % (uuid[0:4], uuid[4:8]) - overlaysuffix = "-%s-%s-%s" % (label, uuid[0:4], uuid[4:8]) - else: - diskmount = usbloop.partitions[0]['mount'] - usblabel = "UUID=%s" % diskmount.uuid - overlaysuffix = "-%s-%s" % (diskmount.fslabel, diskmount.uuid) - - args = ['cp', "-Rf", isodir + "/isolinux", usbmnt + "/syslinux"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't copy isolinux directory %s" \ - % (isodir + "/isolinux/*")) - - if os.path.isfile("/usr/share/syslinux/isolinux.bin"): - syslinux_path = "/usr/share/syslinux" - elif os.path.isfile("/usr/lib/syslinux/isolinux.bin"): - syslinux_path = "/usr/lib/syslinux" - else: - raise CreatorError("syslinux not installed : " - "cannot find syslinux installation path") - - for f in ("isolinux.bin", "vesamenu.c32"): - path = os.path.join(syslinux_path, f) - if os.path.isfile(path): - args = ['cp', path, usbmnt + "/syslinux/"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't copy syslinux file " + path) - else: - raise CreatorError("syslinux not installed: " - "syslinux file %s not found" % path) - - fd = open(isodir + "/isolinux/isolinux.cfg", "r") - text = fd.read() - fd.close() - pattern = re.compile('CDLABEL=[^ ]*') - text = pattern.sub(usblabel, text) - pattern = re.compile('rootfstype=[^ ]*') - text = pattern.sub("rootfstype=" + fstype, text) - if kernelargs: - text = text.replace("rd.live.image", "rd.live.image " + kernelargs) - - if overlaysizemb > 0: - msger.info("Initializing persistent overlay file") - overfile = "overlay" + overlaysuffix - if fstype == "vfat": - args = ['dd', - "if=/dev/zero", - "of=" + usbmnt + "/LiveOS/" + overfile, - "count=%d" % overlaysizemb, - "bs=1M"] - else: - args = ['dd', - "if=/dev/null", - "of=" + usbmnt + "/LiveOS/" + overfile, - "count=1", - "bs=1M", - "seek=%d" % overlaysizemb] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create overlay file") - text = text.replace("rd.live.image", "rd.live.image rd.live.overlay=" + usblabel) - text = text.replace(" ro ", " rw ") - - if swapsizemb > 0: - msger.info("Initializing swap file") - swapfile = usbmnt + "/LiveOS/" + "swap.img" - args = ['dd', - "if=/dev/zero", - "of=" + swapfile, - "count=%d" % swapsizemb, - "bs=1M"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create swap file") - args = ["mkswap", "-f", swapfile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't mkswap on swap file") - - if homesizemb > 0: - msger.info("Initializing persistent /home") - homefile = usbmnt + "/LiveOS/" + homefile - if fstype == "vfat": - args = ['dd', - "if=/dev/zero", - "of=" + homefile, - "count=%d" % homesizemb, - "bs=1M"] - else: - args = ['dd', - "if=/dev/null", - "of=" + homefile, - "count=1", - "bs=1M", - "seek=%d" % homesizemb] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create home file") - - mkfscmd = fs_related.find_binary_path("/sbin/mkfs." + fstype) - if fstype == "ext2" or fstype == "ext3": - args = [mkfscmd, "-F", "-j", homefile] - else: - args = [mkfscmd, homefile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't mke2fs home file") - if fstype == "ext2" or fstype == "ext3": - tune2fs = fs_related.find_binary_path("tune2fs") - args = [tune2fs, - "-c0", - "-i0", - "-ouser_xattr,acl", - homefile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't tune2fs home file") - - if fstype == "vfat" or fstype == "msdos": - syslinuxcmd = fs_related.find_binary_path("syslinux") - syslinuxcfg = usbmnt + "/syslinux/syslinux.cfg" - args = [syslinuxcmd, - "-d", - "syslinux", - usbloop.partitions[0]["device"]] - - elif fstype == "ext2" or fstype == "ext3": - extlinuxcmd = fs_related.find_binary_path("extlinux") - syslinuxcfg = usbmnt + "/syslinux/extlinux.conf" - args = [extlinuxcmd, - "-i", - usbmnt + "/syslinux"] - - else: - raise CreatorError("Invalid file system type: %s" % (fstype)) - - os.unlink(usbmnt + "/syslinux/isolinux.cfg") - fd = open(syslinuxcfg, "w") - fd.write(text) - fd.close() - rc = runner.show(args) - if rc: - raise CreatorError("Can't install boot loader.") - - finally: - usbloop.unmount() - usbloop.cleanup() - - # Need to do this after image is unmounted and device mapper is closed - msger.info("set MBR") - mbrfile = "/usr/lib/syslinux/mbr.bin" - if not os.path.exists(mbrfile): - mbrfile = "/usr/share/syslinux/mbr.bin" - if not os.path.exists(mbrfile): - raise CreatorError("mbr.bin file didn't exist.") - mbrsize = os.path.getsize(mbrfile) - outimg = "%s/%s.usbimg" % (self._outdir, self.name) - - args = ['dd', - "if=" + mbrfile, - "of=" + outimg, - "seek=0", - "conv=notrunc", - "bs=1", - "count=%d" % (mbrsize)] - rc = runner.show(args) - if rc: - raise CreatorError("Can't set MBR.") - - def _stage_final_image(self): - try: - isodir = self._get_isodir() - fs_related.makedirs(isodir + "/LiveOS") - - minimal_size = self._resparse() - - if not self.skip_minimize: - fs_related.create_image_minimizer(isodir + "/LiveOS/osmin.img", - self._image, - minimal_size) - - if self.skip_compression: - shutil.move(self._image, - isodir + "/LiveOS/ext3fs.img") - else: - fs_related.makedirs(os.path.join( - os.path.dirname(self._image), - "LiveOS")) - shutil.move(self._image, - os.path.join(os.path.dirname(self._image), - "LiveOS", "ext3fs.img")) - fs_related.mksquashfs(os.path.dirname(self._image), - isodir + "/LiveOS/squashfs.img") - - self._create_usbimg(isodir) - - if self.pack_to: - usbimg = os.path.join(self._outdir, self.name + ".usbimg") - packimg = os.path.join(self._outdir, self.pack_to) - misc.packing(packimg, usbimg) - os.unlink(usbimg) - - finally: - shutil.rmtree(isodir, ignore_errors = True) - self._set_isodir(None) - diff --git a/mic/imager/loop.py b/mic/imager/loop.py index 4d05ef2..51d8181 100644 --- a/mic/imager/loop.py +++ b/mic/imager/loop.py @@ -23,6 +23,7 @@ from mic import kickstart, msger from mic.utils.errors import CreatorError, MountError from mic.utils import misc, runner, fs_related as fs from mic.imager.baseimager import BaseImageCreator +from mic.archive import packing, compressing # The maximum string length supported for LoopImageCreator.fslabel @@ -98,6 +99,7 @@ class LoopImageCreator(BaseImageCreator): When specifying multiple partitions in kickstart file, each partition will be created as a separated loop image. """ + img_format = 'loop' def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None, @@ -153,6 +155,8 @@ class LoopImageCreator(BaseImageCreator): 'fstype': part.fstype or 'ext3', 'extopts': part.extopts or None, 'loop': None, # to be created in _mount_instroot + 'uuid': part.uuid or None, + 'kspart' : part, }) self._instloops = allloops @@ -161,7 +165,7 @@ class LoopImageCreator(BaseImageCreator): self.__fsopts = None self._instloops = [] - self.__imgdir = None + self._imgdir = None if self.ks: self.__image_size = kickstart.get_image_size(self.ks, @@ -209,9 +213,9 @@ class LoopImageCreator(BaseImageCreator): fslabel = property(__get_fslabel, __set_fslabel) def __get_image(self): - if self.__imgdir is None: + if self._imgdir is None: raise CreatorError("_image is not valid before calling mount()") - return os.path.join(self.__imgdir, self._img_name) + return os.path.join(self._imgdir, self._img_name) #The location of the image file. # #This is the path to the filesystem image. Subclasses may use this path @@ -300,8 +304,8 @@ class LoopImageCreator(BaseImageCreator): shutil.copyfile(base_on, self._image) def _check_imgdir(self): - if self.__imgdir is None: - self.__imgdir = self._mkdtemp() + if self._imgdir is None: + self._imgdir = self._mkdtemp() # @@ -310,7 +314,7 @@ class LoopImageCreator(BaseImageCreator): def _mount_instroot(self, base_on=None): if base_on and os.path.isfile(base_on): - self.__imgdir = os.path.dirname(base_on) + self._imgdir = os.path.dirname(base_on) imgname = os.path.basename(base_on) self._base_on(base_on) self._set_image_size(misc.get_file_size(self._image)) @@ -323,7 +327,9 @@ class LoopImageCreator(BaseImageCreator): "size": self.__image_size or 4096L, "fstype": self.__fstype or "ext3", "extopts": None, - "loop": None + "loop": None, + "uuid": None, + "kspart": None }) self._check_imgdir() @@ -341,15 +347,16 @@ class LoopImageCreator(BaseImageCreator): elif fstype in ("vfat", "msdos"): MyDiskMount = fs.VfatDiskMount else: - msger.error('Cannot support fstype: %s' % fstype) + raise MountError('Cannot support fstype: %s' % fstype) loop['loop'] = MyDiskMount(fs.SparseLoopbackDisk( - os.path.join(self.__imgdir, imgname), + os.path.join(self._imgdir, imgname), size), mp, fstype, self._blocksize, - loop['label']) + loop['label'], + fsuuid = loop['uuid']) if fstype in ("ext2", "ext3", "ext4"): loop['loop'].extopts = loop['extopts'] @@ -358,6 +365,11 @@ class LoopImageCreator(BaseImageCreator): msger.verbose('Mounting image "%s" on "%s"' % (imgname, mp)) fs.makedirs(mp) loop['loop'].mount() + # Make an autogenerated uuid avaialble in _get_post_scripts_env() + if loop['kspart'] and loop['kspart'].uuid is None and \ + loop['loop'].uuid: + loop['kspart'].uuid = loop['loop'].uuid + except MountError, e: raise @@ -376,21 +388,29 @@ class LoopImageCreator(BaseImageCreator): self._resparse() for item in self._instloops: - imgfile = os.path.join(self.__imgdir, item['name']) + imgfile = os.path.join(self._imgdir, item['name']) if item['fstype'] == "ext4": runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s ' % imgfile) + self.image_files.setdefault('partitions', {}).update( + {item['mountpoint']: item['label']}) if self.compress_image: - misc.compressing(imgfile, self.compress_image) + compressing(imgfile, self.compress_image) + self.image_files.setdefault('image_files', []).append( + '.'.join([item['name'], self.compress_image])) + else: + self.image_files.setdefault('image_files', []).append(item['name']) if not self.pack_to: - for item in os.listdir(self.__imgdir): - shutil.move(os.path.join(self.__imgdir, item), + for item in os.listdir(self._imgdir): + shutil.move(os.path.join(self._imgdir, item), os.path.join(self._outdir, item)) else: msger.info("Pack all loop images together to %s" % self.pack_to) dstfile = os.path.join(self._outdir, self.pack_to) - misc.packing(dstfile, self.__imgdir) + packing(dstfile, self._imgdir) + self.image_files['image_files'] = [self.pack_to] + if self.pack_to: mountfp_xml = os.path.splitext(self.pack_to)[0] @@ -412,7 +432,11 @@ class LoopImageCreator(BaseImageCreator): for item in self._attachment: if not os.path.exists(item): continue - dpath = os.path.join(self.__imgdir, os.path.basename(item)) + dpath = os.path.join(self._imgdir, os.path.basename(item)) msger.verbose("Copy attachment %s to %s" % (item, dpath)) shutil.copy(item, dpath) + def create_manifest(self): + if self.compress_image: + self.image_files.update({'compress': self.compress_image}) + super(LoopImageCreator, self).create_manifest() diff --git a/mic/imager/raw.py b/mic/imager/raw.py index 8535d66..08f9683 100644..100755 --- a/mic/imager/raw.py +++ b/mic/imager/raw.py @@ -24,7 +24,7 @@ from mic.utils import fs_related, runner, misc from mic.utils.partitionedfs import PartitionedMount from mic.utils.errors import CreatorError, MountError from mic.imager.baseimager import BaseImageCreator - +from mic.archive import packing, compressing class RawImageCreator(BaseImageCreator): """Installs a system into a file containing a partitioned disk image. @@ -34,6 +34,7 @@ class RawImageCreator(BaseImageCreator): and the system installed into an virtual disk. The disk image can subsequently be booted in a virtual machine or accessed with kpartx """ + img_format = 'raw' def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None, generate_bmap=None, fstab_entry="uuid"): """Initialize a ApplianceImageCreator instance. @@ -56,10 +57,13 @@ class RawImageCreator(BaseImageCreator): self.appliance_release = None self.compress_image = compress_image self.bmap_needed = generate_bmap + self._need_extlinux = not kickstart.use_installerfw(self.ks, "bootloader") #self.getsource = False #self.listpkg = False - self._dep_checks.extend(["sync", "kpartx", "parted", "extlinux"]) + self._dep_checks.extend(["sync", "kpartx", "parted"]) + if self._need_extlinux: + self._dep_checks.extend(["extlinux"]) def configure(self, repodata = None): import subprocess @@ -75,7 +79,7 @@ class RawImageCreator(BaseImageCreator): def _get_fstab(self): s = "" - for mp in self.__instloop.mountOrder: + for mp in self.__instloop.mount_order: p = None for p1 in self.__instloop.partitions: if p1['mountpoint'] == mp: @@ -210,10 +214,84 @@ class RawImageCreator(BaseImageCreator): self.__instloop.mount() self._create_mkinitrd_config() + def mount(self, base_on = None, cachedir = None): + """ + This method calls the base class' 'mount()' method and then creates + block device nodes corresponding to the image's partitions in the image + itself. Namely, the image has /dev/loopX device corresponding to the + entire image, and per-partition /dev/mapper/* devices. + + We copy these files to image's "/dev" directory in order to enable + scripts which run in the image chroot environment to access own raw + partitions. For example, this can be used to install the bootloader to + the MBR (say, from an installer framework plugin). + """ + + def copy_devnode(src, dest): + """A helper function for copying device nodes.""" + + if not src: + return + + stat_obj = os.stat(src) + assert stat.S_ISBLK(stat_obj.st_mode) + + os.mknod(dest, stat_obj.st_mode, + os.makedev(os.major(stat_obj.st_rdev), + os.minor(stat_obj.st_rdev))) + # os.mknod uses process umask may create a nod with different + # permissions, so we use os.chmod to make sure permissions are + # correct. + os.chmod(dest, stat_obj.st_mode) + + BaseImageCreator.mount(self, base_on, cachedir) + + # Copy the disk loop devices + for name in self.__disks.keys(): + loopdev = self.__disks[name].device + copy_devnode(loopdev, self._instroot + loopdev) + + # Copy per-partition dm nodes + os.mkdir(self._instroot + "/dev/mapper", os.stat("/dev/mapper").st_mode) + for p in self.__instloop.partitions: + copy_devnode(p['mapper_device'], + self._instroot + p['mapper_device']) + copy_devnode(p['mpath_device'], + self._instroot + p['mpath_device']) + + def unmount(self): + """ + Remove loop/dm device nodes which we created in 'mount()' and call the + base class' 'unmount()' method. + """ + + for p in self.__instloop.partitions: + if p['mapper_device']: + path = self._instroot + p['mapper_device'] + if os.path.exists(path): + os.unlink(path) + if p['mpath_device']: + path = self._instroot + p['mpath_device'] + if os.path.exists(path): + os.unlink(path) + + path = self._instroot + "/dev/mapper" + if os.path.exists(path): + shutil.rmtree(path, ignore_errors=True) + + for name in self.__disks.keys(): + if self.__disks[name].device: + path = self._instroot + self.__disks[name].device + if os.path.exists(path): + os.unlink(path) + + BaseImageCreator.unmount(self) + def _get_required_packages(self): required_packages = BaseImageCreator._get_required_packages(self) - if not self.target_arch or not self.target_arch.startswith("arm"): - required_packages += ["syslinux", "syslinux-extlinux"] + if self._need_extlinux: + if not self.target_arch or not self.target_arch.startswith("arm"): + required_packages += ["syslinux", "syslinux-extlinux"] return required_packages def _get_excluded_packages(self): @@ -316,7 +394,6 @@ class RawImageCreator(BaseImageCreator): msger.debug("Installing syslinux bootloader '%s' to %s" % \ (mbrfile, loopdev)) - mbrsize = os.stat(mbrfile)[stat.ST_SIZE] rc = runner.show(['dd', 'if=%s' % mbrfile, 'of=' + loopdev]) if rc != 0: raise MountError("Unable to set MBR to %s" % loopdev) @@ -335,7 +412,8 @@ class RawImageCreator(BaseImageCreator): def _create_bootconfig(self): #If syslinux is available do the required configurations. - if os.path.exists("%s/usr/share/syslinux/" % (self._instroot)) \ + if self._need_extlinux \ + and os.path.exists("%s/usr/share/syslinux/" % (self._instroot)) \ and os.path.exists("%s/boot/extlinux/" % (self._instroot)): self._create_syslinux_config() self._install_syslinux() @@ -350,22 +428,57 @@ class RawImageCreator(BaseImageCreator): def _resparse(self, size = None): return self.__instloop.resparse(size) + def _get_post_scripts_env(self, in_chroot): + env = BaseImageCreator._get_post_scripts_env(self, in_chroot) + + # Export the file-system UUIDs and partition UUIDs (AKA PARTUUIDs) + for p in self.__instloop.partitions: + env.update(self._set_part_env(p['ks_pnum'], "UUID", p['uuid'])) + env.update(self._set_part_env(p['ks_pnum'], "PARTUUID", p['partuuid'])) + env.update(self._set_part_env(p['ks_pnum'], "DEVNODE_NOW", + p['mapper_device'])) + env.update(self._set_part_env(p['ks_pnum'], "DISK_DEVNODE_NOW", + self.__disks[p['disk_name']].device)) + + return env + def _stage_final_image(self): """Stage the final system image in _outdir. write meta data """ self._resparse() + self.image_files.update({'disks': self.__disks.keys()}) + + if not (self.compress_image or self.pack_to): + for imgfile in os.listdir(self.__imgdir): + if imgfile.endswith('.raw'): + for disk in self.__disks.keys(): + if imgfile.find(disk) != -1: + self.image_files.setdefault(disk, {}).update( + {'image': imgfile}) + self.image_files.setdefault('image_files', + []).append(imgfile) if self.compress_image: for imgfile in os.listdir(self.__imgdir): if imgfile.endswith('.raw') or imgfile.endswith('bin'): imgpath = os.path.join(self.__imgdir, imgfile) - misc.compressing(imgpath, self.compress_image) + msger.info("Compressing image %s" % imgfile) + compressing(imgpath, self.compress_image) + if imgfile.endswith('.raw') and not self.pack_to: + for disk in self.__disks.keys(): + if imgfile.find(disk) != -1: + imgname = '%s.%s' % (imgfile, self.compress_image) + self.image_files.setdefault(disk, {}).update( + {'image': imgname}) + self.image_files.setdefault('image_files', + []).append(imgname) if self.pack_to: dst = os.path.join(self._outdir, self.pack_to) msger.info("Pack all raw images to %s" % dst) - misc.packing(dst, self.__imgdir) + packing(dst, self.__imgdir) + self.image_files.update({'image_files': self.pack_to}) else: msger.debug("moving disks to stage location") for imgfile in os.listdir(self.__imgdir): @@ -373,6 +486,7 @@ class RawImageCreator(BaseImageCreator): dst = os.path.join(self._outdir, imgfile) msger.debug("moving %s to %s" % (src,dst)) shutil.move(src,dst) + self._write_image_xml() def _write_image_xml(self): @@ -464,18 +578,22 @@ class RawImageCreator(BaseImageCreator): if self.bmap_needed is None: return - from mic.utils import BmapCreate msger.info("Generating the map file(s)") for name in self.__disks.keys(): image = self._full_path(self.__imgdir, name, self.__disk_format) bmap_file = self._full_path(self._outdir, name, "bmap") + self.image_files.setdefault(name, {}).update({'bmap': \ + os.path.basename(bmap_file)}) msger.debug("Generating block map file '%s'" % bmap_file) + + bmaptoolcmd = misc.find_binary_path('bmaptool') + rc = runner.show([bmaptoolcmd, 'create', image, '-o', bmap_file]) + if rc != 0: + raise CreatorError("Failed to create bmap file: %s" % bmap_file) - try: - creator = BmapCreate.BmapCreate(image, bmap_file) - creator.generate() - del creator - except BmapCreate.Error as err: - raise CreatorError("Failed to create bmap file: %s" % str(err)) + def create_manifest(self): + if self.compress_image: + self.image_files.update({'compress': self.compress_image}) + super(RawImageCreator, self).create_manifest() diff --git a/mic/kickstart/__init__.py b/mic/kickstart/__init__.py index eb8bee2..a4b4f80 100644 --- a/mic/kickstart/__init__.py +++ b/mic/kickstart/__init__.py @@ -20,6 +20,7 @@ import os, sys, re import shutil import subprocess import string +import collections import pykickstart.sections as kssections import pykickstart.commands as kscommands @@ -32,7 +33,8 @@ from pykickstart.handlers.control import dataMap from mic import msger from mic.utils import errors, misc, runner, fs_related as fs -from custom_commands import desktop, micrepo, micboot, partition +from custom_commands import desktop, micrepo, micboot, partition, installerfw +from mic.utils.safeurl import SafeURL AUTH_URL_PTN = r"(?P<scheme>.*)://(?P<username>.*)(:?P<password>.*)?@(?P<url>.*)" @@ -101,6 +103,7 @@ def read_kickstart(path): commandMap[using_version]["bootloader"] = micboot.Mic_Bootloader commandMap[using_version]["part"] = partition.Mic_Partition commandMap[using_version]["partition"] = partition.Mic_Partition + commandMap[using_version]["installerfw_plugins"] = installerfw.Mic_installerfw dataMap[using_version]["RepoData"] = micrepo.Mic_RepoData dataMap[using_version]["PartData"] = partition.Mic_PartData superclass = ksversion.returnClassForVersion(version=using_version) @@ -160,6 +163,10 @@ class LanguageConfig(KickstartConfig): f.write("LANG=\"" + kslang.lang + "\"\n") f.close() + f = open(self.path("/etc/locale.conf"), "w+") + f.write("LANG=\"" + kslang.lang + "\"\n") + f.close() + class KeyboardConfig(KickstartConfig): """A class to apply a kickstart keyboard configuration to a system.""" @apply_wrapper @@ -636,6 +643,15 @@ class NetworkConfig(KickstartConfig): self.write_hosts(hostname) self.write_resolv(nodns, nameservers) +def use_installerfw(ks, feature): + """ Check if the installer framework has to be used for a feature + "feature". """ + + features = ks.handler.installerfw.features + if features: + if feature in features or "all" in features: + return True + return False def get_image_size(ks, default = None): __size = 0 @@ -702,61 +718,47 @@ def get_default_kernel(ks, default = None): return default return ks.handler.bootloader.default -def get_repos(ks, repo_urls=None): +RepoType = collections.namedtuple("Repo", + "name, baseurl, mirrorlist, includepkgs, excludepkgs, proxy, \ + proxy_username, proxy_password, debuginfo, \ + source, gpgkey, disable, ssl_verify, nocache, \ + cost, priority") + +def Repo(name, baseurl, mirrorlist=None, includepkgs=[], excludepkgs=[], proxy=None, + proxy_username=None, proxy_password=None, debuginfo=None, + source=None, gpgkey=None, disable=None, ssl_verify=None, + nocache=False, cost=None, priority=None): + return RepoType(name, baseurl, mirrorlist, includepkgs, excludepkgs, proxy, + proxy_username, proxy_password, debuginfo, + source, gpgkey, disable, ssl_verify, nocache, + cost, priority) + + +def get_repos(ks, repo_urls=None, ignore_ksrepo=False): repos = {} - for repo in ks.handler.repo.repoList: - inc = [] - if hasattr(repo, "includepkgs"): - inc.extend(repo.includepkgs) - - exc = [] - if hasattr(repo, "excludepkgs"): - exc.extend(repo.excludepkgs) - - baseurl = repo.baseurl - mirrorlist = repo.mirrorlist - - if repo_urls and repo.name in repo_urls: - baseurl = repo_urls[repo.name] - mirrorlist = None - - if repos.has_key(repo.name): - msger.warning("Overriding already specified repo %s" %(repo.name,)) - - proxy = None - if hasattr(repo, "proxy"): - proxy = repo.proxy - proxy_username = None - if hasattr(repo, "proxy_username"): - proxy_username = repo.proxy_username - proxy_password = None - if hasattr(repo, "proxy_password"): - proxy_password = repo.proxy_password - if hasattr(repo, "debuginfo"): - debuginfo = repo.debuginfo - if hasattr(repo, "source"): - source = repo.source - if hasattr(repo, "gpgkey"): - gpgkey = repo.gpgkey - if hasattr(repo, "disable"): - disable = repo.disable - ssl_verify = True - if hasattr(repo, "ssl_verify"): - ssl_verify = repo.ssl_verify == "yes" - nocache = False - if hasattr(repo, "nocache"): - nocache = repo.nocache - cost = None - if hasattr(repo, "cost"): - cost = repo.cost - priority = None - if hasattr(repo, "priority"): - priority = repo.priority - - repos[repo.name] = (repo.name, baseurl, mirrorlist, inc, exc, - proxy, proxy_username, proxy_password, debuginfo, - source, gpgkey, disable, ssl_verify, nocache, - cost, priority) + for repodata in ks.handler.repo.repoList: + repo = {} + for field in RepoType._fields: + if hasattr(repodata, field) and getattr(repodata, field): + repo[field] = getattr(repodata, field) + + if hasattr(repodata, 'baseurl') and getattr(repodata, 'baseurl'): + repo['baseurl'] = SafeURL(getattr(repodata, 'baseurl'), + getattr(repodata, 'user', None), + getattr(repodata, 'passwd', None)) + + if 'name' in repo: + repos[repo['name']] = Repo(**repo) + + if repo_urls: + if ignore_ksrepo: + repos = {} + for name, repo in repo_urls.items(): + if 'baseurl' in repo: + repo['baseurl'] = SafeURL(repo.get('baseurl'), + repo.get('user', None), + repo.get('passwd', None)) + repos[name] = Repo(**repo) return repos.values() diff --git a/mic/kickstart/custom_commands/__init__.py b/mic/kickstart/custom_commands/__init__.py index d9a1fe5..5f4c440 100644 --- a/mic/kickstart/custom_commands/__init__.py +++ b/mic/kickstart/custom_commands/__init__.py @@ -1,11 +1,12 @@ from desktop import Mic_Desktop from micrepo import Mic_Repo, Mic_RepoData from partition import Mic_Partition - +from installerfw import Mic_installerfw __all__ = ( "Mic_Desktop", "Mic_Repo", "Mic_RepoData", "Mic_Partition", -)
\ No newline at end of file + "Mic_installerfw", +) diff --git a/mic/kickstart/custom_commands/installerfw.py b/mic/kickstart/custom_commands/installerfw.py new file mode 100644 index 0000000..4fa39ed --- /dev/null +++ b/mic/kickstart/custom_commands/installerfw.py @@ -0,0 +1,65 @@ +#!/usr/bin/python -tt +# +# Copyright (c) 2013 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from pykickstart.base import * +from pykickstart.options import * +from mic import msger + +class Mic_installerfw(KickstartCommand): + """ This class implements the "installerfw_plugins" KS option. The argument + of the option is a comman-separated list of MIC features which have to be + disabled and instead, will be done in the installer. For example, + "installerfw_plugins=bootloader" disables all the MIC code which installs + the bootloader to the target images, and instead, the bootlodaer will be + installed by the installer framework plugin. + + The plugin is a program which is external to MIC, it comes from the + installation repositories and can be executed by MIC in order to perform + various configuration actions. The main point here is to make sure MIC has + no hard-wired knoledge about the target OS configuration. """ + + removedKeywords = KickstartCommand.removedKeywords + removedAttrs = KickstartCommand.removedAttrs + + def __init__(self, *args, **kwargs): + KickstartCommand.__init__(self, *args, **kwargs) + self.op = self._getParser() + self.features = None + + def __str__(self): + retval = KickstartCommand.__str__(self) + + if self.features: + retval += "# Enable installer framework plugins\ninstallerfw_plugins\n" + + return retval + + def _getParser(self): + op = KSOptionParser() + return op + + def parse(self, args): + (_, extra) = self.op.parse_args(args=args, lineno=self.lineno) + + if len(extra) != 1: + msg = "Kickstart command \"%s\" requires one " \ + "argumet - a list of legacy features to disable" % self.currentCmd + raise KickstartValueError, formatErrorMsg(self.lineno, msg = msg) + + self.features = extra[0].split(",") + + return self diff --git a/mic/kickstart/custom_commands/micrepo.py b/mic/kickstart/custom_commands/micrepo.py index b31576e..b38ae77 100644 --- a/mic/kickstart/custom_commands/micrepo.py +++ b/mic/kickstart/custom_commands/micrepo.py @@ -1,4 +1,3 @@ -#!/usr/bin/python -tt # # Copyright (c) 2008, 2009, 2010 Intel, Inc. # @@ -17,111 +16,58 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -from pykickstart.base import * -from pykickstart.errors import * -from pykickstart.options import * -from pykickstart.commands.repo import * +from pykickstart.commands.repo import F14_RepoData, F14_Repo -class Mic_RepoData(F8_RepoData): - def __init__(self, baseurl="", mirrorlist=None, name="", priority=None, - includepkgs=(), excludepkgs=(), save=False, proxy=None, - proxy_username=None, proxy_password=None, debuginfo=False, - source=False, gpgkey=None, disable=False, ssl_verify="yes", - nocache=False): - kw = {} - # F8_RepoData keywords - if includepkgs: - kw['includepkgs'] = includepkgs - if excludepkgs: - kw['excludepkgs'] = excludepkgs +class Mic_RepoData(F14_RepoData): + "Mic customized repo data" - #FC6_RepoData keywords - if baseurl: - kw['baseurl'] = baseurl - if mirrorlist: - kw['mirrorlist'] = mirrorlist - if name: - kw['name'] = name + def __init__(self, *args, **kw): + F14_RepoData.__init__(self, *args, **kw) + for field in ('save', 'proxyuser', 'proxypasswd', 'debuginfo', + 'disable', 'source', 'gpgkey', 'ssl_verify', 'priority', + 'nocache', 'user', 'passwd'): + setattr(self, field, kw.get(field)) - F8_RepoData.__init__(self, **kw) - self.save = save - self.proxy = proxy - self.proxy_username = proxy_username - self.proxy_password = proxy_password - self.debuginfo = debuginfo - self.disable = disable - self.source = source - self.gpgkey = gpgkey - self.ssl_verify = ssl_verify.lower() - self.priority = priority - self.nocache = nocache + if hasattr(self, 'proxy') and not self.proxy: + # TODO: remove this code, since it only for back-compatible. + # Some code behind only accept None but not empty string + # for default proxy + self.proxy = None def _getArgsAsStr(self): - retval = F8_RepoData._getArgsAsStr(self) + retval = F14_RepoData._getArgsAsStr(self) - if self.save: - retval += " --save" - if self.proxy: - retval += " --proxy=%s" % self.proxy - if self.proxy_username: - retval += " --proxyuser=%s" % self.proxy_username - if self.proxy_password: - retval += " --proxypasswd=%s" % self.proxy_password - if self.debuginfo: - retval += " --debuginfo" - if self.source: - retval += " --source" - if self.gpgkey: - retval += " --gpgkey=%s" % self.gpgkey - if self.disable: - retval += " --disable" - if self.ssl_verify: - retval += " --ssl_verify=%s" % self.ssl_verify - if self.priority: - retval += " --priority=%s" % self.priority - if self.nocache: - retval += " --nocache" + for field in ('proxyuser', 'proxypasswd', 'user', 'passwd', + 'gpgkey', 'ssl_verify', 'priority', + ): + if hasattr(self, field) and getattr(self, field): + retval += ' --%s="%s"' % (field, getattr(self, field)) - return retval + for field in ('save', 'diable', 'nocache', 'source', 'debuginfo'): + if hasattr(self, field) and getattr(self, field): + retval += ' --%s' % field -class Mic_Repo(F8_Repo): - def __init__(self, writePriority=0, repoList=None): - F8_Repo.__init__(self, writePriority, repoList) + return retval - def __str__(self): - retval = "" - for repo in self.repoList: - retval += repo.__str__() - return retval +class Mic_Repo(F14_Repo): + "Mic customized repo command" def _getParser(self): - def list_cb (option, opt_str, value, parser): - for d in value.split(','): - parser.values.ensure_value(option.dest, []).append(d) + op = F14_Repo._getParser(self) + op.add_option('--user') + op.add_option('--passwd') + op.add_option("--proxyuser") + op.add_option("--proxypasswd") + + op.add_option("--save", action="store_true", default=False) + op.add_option("--debuginfo", action="store_true", default=False) + op.add_option("--source", action="store_true", default=False) + op.add_option("--disable", action="store_true", default=False) + op.add_option("--nocache", action="store_true", default=False) - op = F8_Repo._getParser(self) - op.add_option("--save", action="store_true", dest="save", - default=False) - op.add_option("--proxy", type="string", action="store", dest="proxy", - default=None, nargs=1) - op.add_option("--proxyuser", type="string", action="store", - dest="proxy_username", default=None, nargs=1) - op.add_option("--proxypasswd", type="string", action="store", - dest="proxy_password", default=None, nargs=1) - op.add_option("--debuginfo", action="store_true", dest="debuginfo", - default=False) - op.add_option("--source", action="store_true", dest="source", - default=False) - op.add_option("--disable", action="store_true", dest="disable", - default=False) - op.add_option("--gpgkey", type="string", action="store", dest="gpgkey", - default=None, nargs=1) - op.add_option("--ssl_verify", type="string", action="store", - dest="ssl_verify", default="yes") - op.add_option("--priority", type="int", action="store", dest="priority", - default=None) - op.add_option("--nocache", action="store_true", dest="nocache", - default=False) + op.add_option("--gpgkey") + op.add_option("--priority", type="int") + op.add_option("--ssl_verify", default=None) return op diff --git a/mic/kickstart/custom_commands/partition.py b/mic/kickstart/custom_commands/partition.py index 59a87fb..d4c39bf 100644 --- a/mic/kickstart/custom_commands/partition.py +++ b/mic/kickstart/custom_commands/partition.py @@ -27,6 +27,7 @@ class Mic_PartData(FC4_PartData): self.align = kwargs.get("align", None) self.extopts = kwargs.get("extopts", None) self.part_type = kwargs.get("part_type", None) + self.uuid = kwargs.get("uuid", None) def _getArgsAsStr(self): retval = FC4_PartData._getArgsAsStr(self) @@ -37,6 +38,8 @@ class Mic_PartData(FC4_PartData): retval += " --extoptions=%s" % self.extopts if self.part_type: retval += " --part-type=%s" % self.part_type + if self.uuid: + retval += " --uuid=%s" % self.uuid return retval @@ -54,4 +57,5 @@ class Mic_Partition(FC4_Partition): default=None) op.add_option("--part-type", type="string", action="store", dest="part_type", default=None) + op.add_option("--uuid", dest="uuid", action="store", type="string") return op diff --git a/mic/msger.py b/mic/msger.py index 9afc85b..49bdc2f 100644 --- a/mic/msger.py +++ b/mic/msger.py @@ -1,7 +1,5 @@ -#!/usr/bin/python -tt -# vim: ai ts=4 sts=4 et sw=4 # -# Copyright (c) 2009, 2010, 2011 Intel, Inc. +# Copyright (c) 2013 Intel, Inc. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -16,294 +14,415 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os,sys -import re -import time - -__ALL__ = ['set_mode', - 'get_loglevel', - 'set_loglevel', - 'set_logfile', - 'raw', - 'debug', - 'verbose', - 'info', - 'warning', - 'error', - 'ask', - 'pause', - ] - -# COLORs in ANSI -INFO_COLOR = 32 # green -WARN_COLOR = 33 # yellow -ERR_COLOR = 31 # red -ASK_COLOR = 34 # blue -NO_COLOR = 0 - -PREFIX_RE = re.compile('^<(.*?)>\s*(.*)', re.S) - -INTERACTIVE = True - -LOG_LEVEL = 1 -LOG_LEVELS = { - 'quiet': 0, - 'normal': 1, - 'verbose': 2, - 'debug': 3, - 'never': 4, - } - -LOG_FILE_FP = None -LOG_CONTENT = '' -CATCHERR_BUFFILE_FD = -1 -CATCHERR_BUFFILE_PATH = None -CATCHERR_SAVED_2 = -1 - -def _general_print(head, color, msg = None, stream = None, level = 'normal'): - global LOG_CONTENT - if not stream: - stream = sys.stdout - - if LOG_LEVELS[level] > LOG_LEVEL: - # skip - return - - # encode raw 'unicode' str to utf8 encoded str - if msg and isinstance(msg, unicode): - msg = msg.encode('utf-8', 'ignore') - - errormsg = '' - if CATCHERR_BUFFILE_FD > 0: - size = os.lseek(CATCHERR_BUFFILE_FD , 0, os.SEEK_END) - os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET) - errormsg = os.read(CATCHERR_BUFFILE_FD, size) - os.ftruncate(CATCHERR_BUFFILE_FD, 0) - - # append error msg to LOG - if errormsg: - LOG_CONTENT += errormsg - - # append normal msg to LOG - save_msg = msg.strip() if msg else None - if save_msg: - timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime()) - LOG_CONTENT += timestr + save_msg + '\n' - - if errormsg: - _color_print('', NO_COLOR, errormsg, stream, level) - - _color_print(head, color, msg, stream, level) - -def _color_print(head, color, msg, stream, level): - colored = True - if color == NO_COLOR or \ - not stream.isatty() or \ - os.getenv('ANSI_COLORS_DISABLED') is not None: - colored = False - - if head.startswith('\r'): - # need not \n at last - newline = False - else: - newline = True - - if colored: - head = '\033[%dm%s:\033[0m ' %(color, head) - if not newline: - # ESC cmd to clear line - head = '\033[2K' + head - else: - if head: - head += ': ' - if head.startswith('\r'): - head = head.lstrip() - newline = True - - if msg is not None: - if isinstance(msg, unicode): - msg = msg.encode('utf8', 'ignore') - - stream.write('%s%s' % (head, msg)) - if newline: - stream.write('\n') - - stream.flush() - -def _color_perror(head, color, msg, level = 'normal'): - if CATCHERR_BUFFILE_FD > 0: - _general_print(head, color, msg, sys.stdout, level) - else: - _general_print(head, color, msg, sys.stderr, level) - -def _split_msg(head, msg): - if isinstance(msg, list): - msg = '\n'.join(map(str, msg)) - - if msg.startswith('\n'): - # means print \n at first - msg = msg.lstrip() - head = '\n' + head - - elif msg.startswith('\r'): - # means print \r at first - msg = msg.lstrip() - head = '\r' + head - - m = PREFIX_RE.match(msg) - if m: - head += ' <%s>' % m.group(1) - msg = m.group(2) - - return head, msg - -def get_loglevel(): - return (k for k,v in LOG_LEVELS.items() if v==LOG_LEVEL).next() - -def set_loglevel(level): - global LOG_LEVEL - if level not in LOG_LEVELS: - # no effect - return +# Following messages should be disabled in pylint: +# * Too many instance attributes (R0902) +# * Too few public methods (R0903) +# * Too many public methods (R0904) +# * Anomalous backslash in string (W1401) +# * __init__ method from base class %r is not called (W0231) +# * __init__ method from a non direct base class %r is called (W0233) +# * Invalid name for type (C0103) +# * RootLogger has no '%s' member (E1103) +# pylint: disable=R0902,R0903,R0904,W1401,W0231,W0233,C0103,E1103 + +""" This logging module is fully compatible with the old msger module, and + it supports interactive mode, logs the messages with specified levels + to specified stream, can also catch all error messages including the + involved 3rd party modules to the logger +""" +import os +import sys +import logging +import tempfile + +__ALL__ = [ + 'get_loglevel', + 'set_loglevel', + 'set_logfile', + 'enable_interactive', + 'disable_interactive', + 'enable_logstderr', + 'disable_logstderr', + 'raw', + 'debug', + 'verbose', + 'info', + 'warning', + 'error', + 'select', + 'choice', + 'ask', + 'pause', +] + + +# define the color constants +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) + +# color sequence for tty terminal +COLOR_SEQ = "\033[%dm" +# reset sequence for tty terminal +RESET_SEQ = "\033[0m" + +# new log level +RAWTEXT = 25 +VERBOSE = 15 + +# define colors for log levels +COLORS = { + 'DEBUG': COLOR_SEQ % BLUE, + 'VERBOSE': COLOR_SEQ % MAGENTA, + 'INFO': COLOR_SEQ % GREEN, + 'WARNING': COLOR_SEQ % YELLOW, + 'ERROR': COLOR_SEQ % RED, +} + + +class LevelFilter(logging.Filter): + """ A filter that selects logging message with specified level """ + def __init__(self, levels): + self._levels = levels + + def filter(self, record): + if self._levels: + return record.levelname in self._levels + return False + + +class MicStreamHandler(logging.StreamHandler): + """ A stream handler that print colorized levelname in tty terminal """ + def __init__(self, stream=None): + logging.StreamHandler.__init__(self, stream) + msg_fmt = "%(color)s%(levelname)s:%(reset)s %(message)s" + self.setFormatter(logging.Formatter(fmt=msg_fmt)) + + def _use_color(self): + """ Check if to print in color or not """ + in_emacs = (os.getenv("EMACS") and + os.getenv("INSIDE_EMACS", "").endswith(",comint")) + return self.stream.isatty() and not in_emacs + + def format(self, record): + """ Format the logging record if need color """ + record.color = record.reset = "" + if self._use_color(): + record.color = COLORS[record.levelname] + record.reset = RESET_SEQ + return logging.StreamHandler.format(self, record) + + +class RedirectedStderr(object): + """ A faked error stream that redirect stderr to a temp file """ + def __init__(self): + self.tmpfile = tempfile.NamedTemporaryFile() + self.fderr = None + self.value = None + + def __del__(self): + self.close() + + def close(self): + """ Close the temp file and clear the buffer """ + try: + self.value = None + self.tmpfile.close() + except OSError: + pass + + def truncate(self): + """ Truncate the tempfile to size zero """ + if self.tmpfile: + os.ftruncate(self.tmpfile.fileno(), 0) + + def redirect(self): + """ Redirect stderr to the temp file """ + self.fderr = os.dup(2) + os.dup2(self.tmpfile.fileno(), 2) + + def restore(self): + """ Restore the stderr and read the bufferred data """ + os.dup2(self.fderr, 2) + self.fderr = None + + if self.tmpfile: + self.tmpfile.seek(0, 0) + self.value = self.tmpfile.read() + + def getvalue(self): + """ Read the bufferred data """ + if self.tmpfile: + self.tmpfile.seek(0, 0) + self.value = self.tmpfile.read() + os.ftruncate(self.tmpfile.fileno(), 0) + return self.value + return None - LOG_LEVEL = LOG_LEVELS[level] +class MicFileHandler(logging.FileHandler): + """ This file handler is supposed to catch the stderr output from + all modules even 3rd party modules involed, as it redirects + the stderr stream to a temp file stream, if logfile assigned, + it will flush the record to file stream, else it's a buffer + handler; once logfile assigned, the buffer will be flushed + """ + def __init__(self, filename=None, mode='w', encoding=None, capacity=10): + # we don't use FileHandler to initialize, + # because filename might be expected to None + logging.Handler.__init__(self) + self.stream = None + if filename: + self.baseFilename = os.path.abspath(filename) + else: + self.baseFilename = None + self.mode = mode + self.encoding = None + self.capacity = capacity + # buffering the records + self.buffer = [] + + # set formater locally + msg_fmt = "[%(asctime)s] %(message)s" + date_fmt = "%m/%d %H:%M:%S %Z" + self.setFormatter(logging.Formatter(fmt=msg_fmt, datefmt=date_fmt)) + self.olderr = sys.stderr + self.stderr = RedirectedStderr() + self.errmsg = None + + def set_logfile(self, filename, mode='w'): + """ Set logfile path to make it possible flush records not-on-fly """ + self.baseFilename = os.path.abspath(filename) + self.mode = mode + + def redirect_stderr(self): + """ Start to redirect stderr for catching all error output """ + self.stderr.redirect() + + def restore_stderr(self): + """ Restore stderr stream and log the error messages to both stderr + and log file if error messages are not empty + """ + self.stderr.restore() + self.errmsg = self.stderr.value + if self.errmsg: + self.logstderr() + + def logstderr(self): + """ Log catched error message from stderr redirector """ + if not self.errmsg: + return + + sys.stdout.write(self.errmsg) + sys.stdout.flush() + + record = logging.makeLogRecord({'msg': self.errmsg}) + self.buffer.append(record) + + # truncate the redirector for the errors is logged + self.stderr.truncate() + self.errmsg = None + + def emit(self, record): + """ Emit the log record to Handler """ + # if there error message catched, log it first + self.errmsg = self.stderr.getvalue() + if self.errmsg: + self.logstderr() + + # if no logfile assigned, it's a buffer handler + if not self.baseFilename: + self.buffer.append(record) + if len(self.buffer) >= self.capacity: + self.buffer = [] + else: + self.flushing(record) + + def flushing(self, record=None): + """ Flush buffer and record to logfile """ + # NOTE: 'flushing' can't be named 'flush' because of 'emit' calling it + # set file stream position to SEEK_END(=2) + if self.stream: + self.stream.seek(0, 2) + # if bufferred, flush it + if self.buffer: + for arecord in self.buffer: + logging.FileHandler.emit(self, arecord) + self.buffer = [] + # if recorded, flush it + if record: + logging.FileHandler.emit(self, record) + + def close(self): + """ Close handler after flushing the buffer """ + # if any left in buffer, flush it + if self.stream: + self.flushing() + logging.FileHandler.close(self) + + +class MicLogger(logging.Logger): + """ The MIC logger class, it supports interactive mode, and logs the + messages with specified levels tospecified stream, also can catch + all error messages including the involved 3rd party modules + """ + def __init__(self, name, level=logging.INFO): + logging.Logger.__init__(self, name, level) + self.propagate = False + self.interactive = True + self.logfile = None + self._allhandlers = { + 'default': logging.StreamHandler(sys.stdout), + 'stdout': MicStreamHandler(sys.stdout), + 'stderr': MicStreamHandler(sys.stderr), + 'logfile': MicFileHandler(), + } + + self._allhandlers['default'].addFilter(LevelFilter(['RAWTEXT'])) + self._allhandlers['default'].setFormatter( + logging.Formatter(fmt="%(message)s")) + self.addHandler(self._allhandlers['default']) + + self._allhandlers['stdout'].addFilter(LevelFilter(['DEBUG', 'VERBOSE', + 'INFO'])) + self.addHandler(self._allhandlers['stdout']) + + self._allhandlers['stderr'].addFilter(LevelFilter(['WARNING', + 'ERROR'])) + self.addHandler(self._allhandlers['stderr']) + + self.addHandler(self._allhandlers['logfile']) + + def set_logfile(self, filename, mode='w'): + """ Set logfile path """ + self.logfile = filename + self._allhandlers['logfile'].set_logfile(self.logfile, mode) + + def enable_logstderr(self): + """ Start to log all error messages """ + if self.logfile: + self._allhandlers['logfile'].redirect_stderr() + + def disable_logstderr(self): + """ Stop to log all error messages """ + if self.logfile: + self._allhandlers['logfile'].restore_stderr() + + def verbose(self, msg, *args, **kwargs): + """ Log a message with level VERBOSE """ + if self.isEnabledFor(VERBOSE): + self._log(VERBOSE, msg, args, **kwargs) + + def raw(self, msg, *args, **kwargs): + """ Log a message in raw text format """ + if self.isEnabledFor(RAWTEXT): + self._log(RAWTEXT, msg, args, **kwargs) + + def select(self, msg, optdict, default=None): + """ Log a message in interactive mode """ + if not optdict.keys(): + return default + if default is None: + default = optdict.keys()[0] + msg += " [%s](%s): " % ('/'.join(optdict.keys()), default) + if not self.interactive or self.logfile: + reply = default + self.raw(msg + reply) + else: + while True: + reply = raw_input(msg).strip() + if not reply or reply in optdict: + break + if not reply: + reply = default + return optdict[reply] -def set_interactive(mode=True): - global INTERACTIVE - if mode: - INTERACTIVE = True - else: - INTERACTIVE = False -def log(msg=''): - # log msg to LOG_CONTENT then save to logfile - global LOG_CONTENT - if msg: - LOG_CONTENT += msg +def error(msg): + """ Log a message with level ERROR on the MIC logger """ + LOGGER.error(msg) + sys.exit(2) -def raw(msg=''): - _general_print('', NO_COLOR, msg) +def warning(msg): + """ Log a message with level WARNING on the MIC logger """ + LOGGER.warning(msg) def info(msg): - head, msg = _split_msg('Info', msg) - _general_print(head, INFO_COLOR, msg) + """ Log a message with level INFO on the MIC logger """ + LOGGER.info(msg) def verbose(msg): - head, msg = _split_msg('Verbose', msg) - _general_print(head, INFO_COLOR, msg, level = 'verbose') - -def warning(msg): - head, msg = _split_msg('Warning', msg) - _color_perror(head, WARN_COLOR, msg) + """ Log a message with level VERBOSE on the MIC logger """ + LOGGER.verbose(msg) def debug(msg): - head, msg = _split_msg('Debug', msg) - _color_perror(head, ERR_COLOR, msg, level = 'debug') + """ Log a message with level DEBUG on the MIC logger """ + LOGGER.debug(msg) -def error(msg): - head, msg = _split_msg('Error', msg) - _color_perror(head, ERR_COLOR, msg) - sys.exit(1) - -def ask(msg, default=True): - _general_print('\rQ', ASK_COLOR, '') - try: - if default: - msg += '(Y/n) ' - else: - msg += '(y/N) ' - if INTERACTIVE: - while True: - repl = raw_input(msg) - if repl.lower() == 'y': - return True - elif repl.lower() == 'n': - return False - elif not repl.strip(): - # <Enter> - return default - - # else loop - else: - if default: - msg += ' Y' - else: - msg += ' N' - _general_print('', NO_COLOR, msg) +def raw(msg): + """ Log a message on the MIC logger in raw text format""" + LOGGER.raw(msg) - return default - except KeyboardInterrupt: - sys.stdout.write('\n') - sys.exit(2) +def select(msg, optdict, default=None): + """ Show an interactive scene in tty terminal and + logs them on MIC logger + """ + return LOGGER.select(msg, optdict, default) -def choice(msg, choices, default=0): - if default >= len(choices): - return None - _general_print('\rQ', ASK_COLOR, '') - try: - msg += " [%s] " % '/'.join(choices) - if INTERACTIVE: - while True: - repl = raw_input(msg) - if repl in choices: - return repl - elif not repl.strip(): - return choices[default] - else: - msg += choices[default] - _general_print('', NO_COLOR, msg) +def choice(msg, optlist, default=0): + """ Give some alternatives to users for answering the question """ + return LOGGER.select(msg, dict(zip(optlist, optlist)), optlist[default]) - return choices[default] - except KeyboardInterrupt: - sys.stdout.write('\n') - sys.exit(2) +def ask(msg, ret=True): + """ Ask users to answer 'yes' or 'no' to the question """ + answers = {'y': True, 'n': False} + default = {True: 'y', False: 'n'}[ret] + return LOGGER.select(msg, answers, default) def pause(msg=None): - if INTERACTIVE: - _general_print('\rQ', ASK_COLOR, '') - if msg is None: - msg = 'press <ENTER> to continue ...' - raw_input(msg) + """ Pause for any key """ + if msg is None: + msg = "press ANY KEY to continue ..." + raw_input(msg) -def set_logfile(fpath): - global LOG_FILE_FP +def set_logfile(logfile, mode='w'): + """ Set logfile path to the MIC logger """ + LOGGER.set_logfile(logfile, mode) - def _savelogf(): - if LOG_FILE_FP: - fp = open(LOG_FILE_FP, 'w') - fp.write(LOG_CONTENT) - fp.close() +def set_loglevel(level): + """ Set loglevel to the MIC logger """ + if isinstance(level, basestring): + level = logging.getLevelName(level) + LOGGER.setLevel(level) - if LOG_FILE_FP is not None: - warning('duplicate log file configuration') +def get_loglevel(): + """ Get the loglevel of the MIC logger """ + return logging.getLevelName(LOGGER.level) - LOG_FILE_FP = fpath +def disable_interactive(): + """ Disable the interactive mode """ + LOGGER.interactive = False - import atexit - atexit.register(_savelogf) +def enable_interactive(): + """ Enable the interactive mode """ + LOGGER.interactive = True -def enable_logstderr(fpath): - global CATCHERR_BUFFILE_FD - global CATCHERR_BUFFILE_PATH - global CATCHERR_SAVED_2 +def set_interactive(value): + """ Set the interactive mode (for compatibility) """ + if value: + enable_interactive() + else: + disable_interactive() - if os.path.exists(fpath): - os.remove(fpath) - CATCHERR_BUFFILE_PATH = fpath - CATCHERR_BUFFILE_FD = os.open(CATCHERR_BUFFILE_PATH, os.O_RDWR|os.O_CREAT) - CATCHERR_SAVED_2 = os.dup(2) - os.dup2(CATCHERR_BUFFILE_FD, 2) +def enable_logstderr(fpath=None): + """ Start to log all error message on the MIC logger """ + LOGGER.enable_logstderr() def disable_logstderr(): - global CATCHERR_BUFFILE_FD - global CATCHERR_BUFFILE_PATH - global CATCHERR_SAVED_2 - - raw(msg = None) # flush message buffer and print it. - os.dup2(CATCHERR_SAVED_2, 2) - os.close(CATCHERR_SAVED_2) - os.close(CATCHERR_BUFFILE_FD) - os.unlink(CATCHERR_BUFFILE_PATH) - CATCHERR_BUFFILE_FD = -1 - CATCHERR_BUFFILE_PATH = None - CATCHERR_SAVED_2 = -1 + """ Stop to log all error message on the MIC logger """ + LOGGER.disable_logstderr() + + +# add two level to the MIC logger: 'VERBOSE', 'RAWTEXT' +logging.addLevelName(VERBOSE, 'VERBOSE') +logging.addLevelName(RAWTEXT, 'RAWTEXT') +# initial the MIC logger +logging.setLoggerClass(MicLogger) +LOGGER = logging.getLogger("MIC") diff --git a/mic/plugin.py b/mic/plugin.py index 18c93ad..d7150d1 100644 --- a/mic/plugin.py +++ b/mic/plugin.py @@ -80,7 +80,7 @@ class PluginMgr(object): except ImportError, err: msg = 'Failed to load plugin %s/%s: %s' \ % (os.path.basename(pdir), mod, err) - msger.warning(msg) + msger.verbose(msg) del(sys.path[0]) diff --git a/mic/pluginbase.py b/mic/pluginbase.py index 6ac195b..541ac8a 100644 --- a/mic/pluginbase.py +++ b/mic/pluginbase.py @@ -64,15 +64,18 @@ class ImagerPlugin(_Plugin): image = os.path.join(destdir, name) if not os.path.exists(image): continue - if msger.ask("Target image/dir: %s already exists, " - "clean up and continue?" % image): + "clean up the old files and continue?" % image): if os.path.isdir(image): - shutil.rmtree(image) + for path, dirs, files in os.walk(os.path.abspath(image)): + for fname in files: + fpath = os.path.join(path, fname) + if not fpath.endswith('.log'): + os.remove(fpath) else: os.unlink(image) else: - raise errors.Abort("Cancled") + raise errors.Abort("Canceled") def do_create(self): pass diff --git a/mic/rt_util.py b/mic/rt_util.py index 2a31f4a..3897efc 100644 --- a/mic/rt_util.py +++ b/mic/rt_util.py @@ -22,15 +22,47 @@ import glob import re import shutil import subprocess +import ctypes from mic import bootstrap, msger from mic.conf import configmgr from mic.utils import errors, proxy from mic.utils.fs_related import find_binary_path, makedirs -from mic.chroot import setup_chrootenv, cleanup_chrootenv +from mic.chroot import setup_chrootenv, cleanup_chrootenv, ELF_arch + +_libc = ctypes.cdll.LoadLibrary(None) +_errno = ctypes.c_int.in_dll(_libc, "errno") +_libc.personality.argtypes = [ctypes.c_ulong] +_libc.personality.restype = ctypes.c_int +_libc.unshare.argtypes = [ctypes.c_int,] +_libc.unshare.restype = ctypes.c_int expath = lambda p: os.path.abspath(os.path.expanduser(p)) +PER_LINUX32=0x0008 +PER_LINUX=0x0000 +personality_defs = { + 'x86_64': PER_LINUX, 'ppc64': PER_LINUX, 'sparc64': PER_LINUX, + 'i386': PER_LINUX32, 'i586': PER_LINUX32, 'i686': PER_LINUX32, + 'ppc': PER_LINUX32, 'sparc': PER_LINUX32, 'sparcv9': PER_LINUX32, + 'ia64' : PER_LINUX, 'alpha' : PER_LINUX, + 's390' : PER_LINUX32, 's390x' : PER_LINUX, +} + +def condPersonality(per=None): + if per is None or per in ('noarch',): + return + if personality_defs.get(per, None) is None: + return + res = _libc.personality(personality_defs[per]) + if res == -1: + raise OSError(_errno.value, os.strerror(_errno.value)) + +def inbootstrap(): + if os.path.exists(os.path.join("/", ".chroot.lock")): + return True + return (os.stat("/").st_ino != 2) + def bootstrap_mic(argv=None): @@ -72,25 +104,26 @@ def bootstrap_mic(argv=None): rootdir = os.path.join(rootdir, "bootstrap") bsenv.dirsetup(rootdir) - sync_mic(rootdir) + sync_mic(rootdir, plugin=cropts['plugin_dir']) #FIXME: sync the ks file to bootstrap if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)): safecopy(configmgr._ksconf, rootdir) msger.info("Start mic in bootstrap: %s\n" % rootdir) + bsarch = ELF_arch(rootdir) + if bsarch in personality_defs: + condPersonality(bsarch) bindmounts = get_bindmounts(cropts) ret = bsenv.run(argv, cwd, rootdir, bindmounts) except errors.BootstrapError, err: - msger.warning('\n%s' % err) - if msger.ask("Switch to native mode and continue?"): - return - raise + raise errors.CreatorError("Failed to download/install bootstrap package " \ + "or the package is in bad format: %s" % err) except RuntimeError, err: #change exception type but keep the trace back value, tb = sys.exc_info()[1:] - raise errors.BootstrapError, value, tb + raise errors.CreatorError, value, tb else: sys.exit(ret) finally: @@ -123,13 +156,15 @@ def get_mic_binpath(): fp = None try: import pkg_resources # depends on 'setuptools' - except ImportError: - pass - else: dist = pkg_resources.get_distribution('mic') # the real script is under EGG_INFO/scripts if dist.has_metadata('scripts/mic'): fp = os.path.join(dist.egg_info, "scripts/mic") + except ImportError: + pass + except pkg_resources.DistributionNotFound: + pass + if fp: return fp @@ -150,12 +185,12 @@ def get_mic_modpath(): return os.path.dirname(path) def get_mic_libpath(): - # TBD: so far mic lib path is hard coded - return "/usr/lib/mic" + return os.getenv("MIC_LIBRARY_PATH", "/usr/lib/mic") # the hard code path is prepared for bootstrap def sync_mic(bootstrap, binpth = '/usr/bin/mic', libpth='/usr/lib', + plugin='/usr/lib/mic/plugins', pylib = '/usr/lib/python2.7/site-packages', conf = '/etc/mic/mic.conf'): _path = lambda p: os.path.join(bootstrap, p.lstrip('/')) @@ -163,6 +198,7 @@ def sync_mic(bootstrap, binpth = '/usr/bin/mic', micpaths = { 'binpth': get_mic_binpath(), 'libpth': get_mic_libpath(), + 'plugin': plugin, 'pylib': get_mic_modpath(), 'conf': '/etc/mic/mic.conf', } diff --git a/mic/utils/BmapCreate.py b/mic/utils/BmapCreate.py deleted file mode 100644 index 65b19a5..0000000 --- a/mic/utils/BmapCreate.py +++ /dev/null @@ -1,298 +0,0 @@ -""" This module implements the block map (bmap) creation functionality and -provides the corresponding API in form of the 'BmapCreate' class. - -The idea is that while images files may generally be very large (e.g., 4GiB), -they may nevertheless contain only little real data, e.g., 512MiB. This data -are files, directories, file-system meta-data, partition table, etc. When -copying the image to the target device, you do not have to copy all the 4GiB of -data, you can copy only 512MiB of it, which is 4 times less, so copying should -presumably be 4 times faster. - -The block map file is an XML file which contains a list of blocks which have to -be copied to the target device. The other blocks are not used and there is no -need to copy them. The XML file also contains some additional information like -block size, image size, count of mapped blocks, etc. There are also many -commentaries, so it is human-readable. - -The image has to be a sparse file. Generally, this means that when you generate -this image file, you should start with a huge sparse file which contains a -single hole spanning the entire file. Then you should partition it, write all -the data (probably by means of loop-back mounting the image or parts of it), -etc. The end result should be a sparse file where mapped areas represent useful -parts of the image and holes represent useless parts of the image, which do not -have to be copied when copying the image to the target device. - -This module uses the FIBMAP ioctl to detect holes. """ - -# Disable the following pylint recommendations: -# * Too many instance attributes - R0902 -# * Too few public methods - R0903 -# pylint: disable=R0902,R0903 - -import hashlib -from mic.utils.misc import human_size -from mic.utils import Fiemap - -# The bmap format version we generate -SUPPORTED_BMAP_VERSION = "1.3" - -_BMAP_START_TEMPLATE = \ -"""<?xml version="1.0" ?> -<!-- This file contains the block map for an image file, which is basically - a list of useful (mapped) block numbers in the image file. In other words, - it lists only those blocks which contain data (boot sector, partition - table, file-system metadata, files, directories, extents, etc). These - blocks have to be copied to the target device. The other blocks do not - contain any useful data and do not have to be copied to the target - device. - - The block map an optimization which allows to copy or flash the image to - the image quicker than copying of flashing the entire image. This is - because with bmap less data is copied: <MappedBlocksCount> blocks instead - of <BlocksCount> blocks. - - Besides the machine-readable data, this file contains useful commentaries - which contain human-readable information like image size, percentage of - mapped data, etc. - - The 'version' attribute is the block map file format version in the - 'major.minor' format. The version major number is increased whenever an - incompatible block map format change is made. The minor number changes - in case of minor backward-compatible changes. --> - -<bmap version="%s"> - <!-- Image size in bytes: %s --> - <ImageSize> %u </ImageSize> - - <!-- Size of a block in bytes --> - <BlockSize> %u </BlockSize> - - <!-- Count of blocks in the image file --> - <BlocksCount> %u </BlocksCount> - -""" - -class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ - pass - -class BmapCreate: - """ This class implements the bmap creation functionality. To generate a - bmap for an image (which is supposedly a sparse file), you should first - create an instance of 'BmapCreate' and provide: - - * full path or a file-like object of the image to create bmap for - * full path or a file object to use for writing the results to - - Then you should invoke the 'generate()' method of this class. It will use - the FIEMAP ioctl to generate the bmap. """ - - def _open_image_file(self): - """ Open the image file. """ - - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" \ - % (self._image_path, err)) - - self._f_image_needs_close = True - - def _open_bmap_file(self): - """ Open the bmap file. """ - - try: - self._f_bmap = open(self._bmap_path, 'w+') - except IOError as err: - raise Error("cannot open bmap file '%s': %s" \ - % (self._bmap_path, err)) - - self._f_bmap_needs_close = True - - def __init__(self, image, bmap): - """ Initialize a class instance: - * image - full path or a file-like object of the image to create bmap - for - * bmap - full path or a file object to use for writing the resulting - bmap to """ - - self.image_size = None - self.image_size_human = None - self.block_size = None - self.blocks_cnt = None - self.mapped_cnt = None - self.mapped_size = None - self.mapped_size_human = None - self.mapped_percent = None - - self._mapped_count_pos1 = None - self._mapped_count_pos2 = None - self._sha1_pos = None - - self._f_image_needs_close = False - self._f_bmap_needs_close = False - - if hasattr(image, "read"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - if hasattr(bmap, "read"): - self._f_bmap = bmap - self._bmap_path = bmap.name - else: - self._bmap_path = bmap - self._open_bmap_file() - - self.fiemap = Fiemap.Fiemap(self._f_image) - - self.image_size = self.fiemap.image_size - self.image_size_human = human_size(self.image_size) - if self.image_size == 0: - raise Error("cannot generate bmap for zero-sized image file '%s'" \ - % self._image_path) - - self.block_size = self.fiemap.block_size - self.blocks_cnt = self.fiemap.blocks_cnt - - def _bmap_file_start(self): - """ A helper function which generates the starting contents of the - block map file: the header comment, image size, block size, etc. """ - - # We do not know the amount of mapped blocks at the moment, so just put - # whitespaces instead of real numbers. Assume the longest possible - # numbers. - mapped_count = ' ' * len(str(self.image_size)) - mapped_size_human = ' ' * len(self.image_size_human) - - xml = _BMAP_START_TEMPLATE \ - % (SUPPORTED_BMAP_VERSION, self.image_size_human, - self.image_size, self.block_size, self.blocks_cnt) - xml += " <!-- Count of mapped blocks: " - - self._f_bmap.write(xml) - self._mapped_count_pos1 = self._f_bmap.tell() - - # Just put white-spaces instead of real information about mapped blocks - xml = "%s or %.1f -->\n" % (mapped_size_human, 100.0) - xml += " <MappedBlocksCount> " - - self._f_bmap.write(xml) - self._mapped_count_pos2 = self._f_bmap.tell() - - xml = "%s </MappedBlocksCount>\n\n" % mapped_count - - # pylint: disable=C0301 - xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n" - xml += " the SHA1 checksum has be zeoro (40 ASCII \"0\" symbols). -->\n" - xml += " <BmapFileSHA1> " - - self._f_bmap.write(xml) - self._sha1_pos = self._f_bmap.tell() - - xml = "0" * 40 + " </BmapFileSHA1>\n\n" - xml += " <!-- The block map which consists of elements which may either be a\n" - xml += " range of blocks or a single block. The 'sha1' attribute (if present)\n" - xml += " is the SHA1 checksum of this blocks range. -->\n" - xml += " <BlockMap>\n" - # pylint: enable=C0301 - - self._f_bmap.write(xml) - - def _bmap_file_end(self): - """ A helper function which generates the final parts of the block map - file: the ending tags and the information about the amount of mapped - blocks. """ - - xml = " </BlockMap>\n" - xml += "</bmap>\n" - - self._f_bmap.write(xml) - - self._f_bmap.seek(self._mapped_count_pos1) - self._f_bmap.write("%s or %.1f%%" % \ - (self.mapped_size_human, self.mapped_percent)) - - self._f_bmap.seek(self._mapped_count_pos2) - self._f_bmap.write("%u" % self.mapped_cnt) - - self._f_bmap.seek(0) - sha1 = hashlib.sha1(self._f_bmap.read()).hexdigest() - self._f_bmap.seek(self._sha1_pos) - self._f_bmap.write("%s" % sha1) - - def _calculate_sha1(self, first, last): - """ A helper function which calculates SHA1 checksum for the range of - blocks of the image file: from block 'first' to block 'last'. """ - - start = first * self.block_size - end = (last + 1) * self.block_size - - self._f_image.seek(start) - hash_obj = hashlib.new("sha1") - - chunk_size = 1024*1024 - to_read = end - start - read = 0 - - while read < to_read: - if read + chunk_size > to_read: - chunk_size = to_read - read - chunk = self._f_image.read(chunk_size) - hash_obj.update(chunk) - read += chunk_size - - return hash_obj.hexdigest() - - def generate(self, include_checksums = True): - """ Generate bmap for the image file. If 'include_checksums' is 'True', - also generate SHA1 checksums for block ranges. """ - - # Save image file position in order to restore it at the end - image_pos = self._f_image.tell() - - self._bmap_file_start() - - # Generate the block map and write it to the XML block map - # file as we go. - self.mapped_cnt = 0 - for first, last in self.fiemap.get_mapped_ranges(0, self.blocks_cnt): - self.mapped_cnt += last - first + 1 - if include_checksums: - sha1 = self._calculate_sha1(first, last) - sha1 = " sha1=\"%s\"" % sha1 - else: - sha1 = "" - - if first != last: - self._f_bmap.write(" <Range%s> %s-%s </Range>\n" \ - % (sha1, first, last)) - else: - self._f_bmap.write(" <Range%s> %s </Range>\n" \ - % (sha1, first)) - - self.mapped_size = self.mapped_cnt * self.block_size - self.mapped_size_human = human_size(self.mapped_size) - self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt - - self._bmap_file_end() - - try: - self._f_bmap.flush() - except IOError as err: - raise Error("cannot flush the bmap file '%s': %s" \ - % (self._bmap_path, err)) - - self._f_image.seek(image_pos) - - def __del__(self): - """ The class destructor which closes the opened files. """ - - if self._f_image_needs_close: - self._f_image.close() - if self._f_bmap_needs_close: - self._f_bmap.close() diff --git a/mic/utils/Fiemap.py b/mic/utils/Fiemap.py deleted file mode 100644 index f2db6ff..0000000 --- a/mic/utils/Fiemap.py +++ /dev/null @@ -1,252 +0,0 @@ -""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl -allows to find holes and mapped areas in a file. """ - -# Note, a lot of code in this module is not very readable, because it deals -# with the rather complex FIEMAP ioctl. To understand the code, you need to -# know the FIEMAP interface, which is documented in the -# Documentation/filesystems/fiemap.txt file in the Linux kernel sources. - -# Disable the following pylint recommendations: -# * Too many instance attributes (R0902) -# pylint: disable=R0902 - -import os -import struct -import array -import fcntl -from mic.utils.misc import get_block_size - -# Format string for 'struct fiemap' -_FIEMAP_FORMAT = "=QQLLLL" -# sizeof(struct fiemap) -_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) -# Format string for 'struct fiemap_extent' -_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" -# sizeof(struct fiemap_extent) -_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) -# The FIEMAP ioctl number -_FIEMAP_IOCTL = 0xC020660B - -# Minimum buffer which is required for 'class Fiemap' to operate -MIN_BUFFER_SIZE = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE -# The default buffer size for 'class Fiemap' -DEFAULT_BUFFER_SIZE = 256 * 1024 - -class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ - pass - -class Fiemap: - """ This class provides API to the FIEMAP ioctl. Namely, it allows to - iterate over all mapped blocks and over all holes. """ - - def _open_image_file(self): - """ Open the image file. """ - - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" \ - % (self._image_path, err)) - - self._f_image_needs_close = True - - def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE): - """ Initialize a class instance. The 'image' argument is full path to - the file to operate on, or a file object to operate on. - - The 'buf_size' argument is the size of the buffer for 'struct - fiemap_extent' elements which will be used when invoking the FIEMAP - ioctl. The larger is the buffer, the less times the FIEMAP ioctl will - be invoked. """ - - self._f_image_needs_close = False - - if hasattr(image, "fileno"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - # Validate 'buf_size' - if buf_size < MIN_BUFFER_SIZE: - raise Error("too small buffer (%d bytes), minimum is %d bytes" \ - % (buf_size, MIN_BUFFER_SIZE)) - - # How many 'struct fiemap_extent' elements fit the buffer - buf_size -= _FIEMAP_SIZE - self._fiemap_extent_cnt = buf_size / _FIEMAP_EXTENT_SIZE - self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE - self._buf_size += _FIEMAP_SIZE - - # Allocate a mutable buffer for the FIEMAP ioctl - self._buf = array.array('B', [0] * self._buf_size) - - self.image_size = os.fstat(self._f_image.fileno()).st_size - - try: - self.block_size = get_block_size(self._f_image) - except IOError as err: - raise Error("cannot get block size for '%s': %s" \ - % (self._image_path, err)) - - self.blocks_cnt = self.image_size + self.block_size - 1 - self.blocks_cnt /= self.block_size - - # Synchronize the image file to make sure FIEMAP returns correct values - try: - self._f_image.flush() - except IOError as err: - raise Error("cannot flush image file '%s': %s" \ - % (self._image_path, err)) - try: - os.fsync(self._f_image.fileno()), - except OSError as err: - raise Error("cannot synchronize image file '%s': %s " \ - % (self._image_path, err.strerror)) - - # Check if the FIEMAP ioctl is supported - self.block_is_mapped(0) - - def __del__(self): - """ The class destructor which closes the opened files. """ - - if self._f_image_needs_close: - self._f_image.close() - - def _invoke_fiemap(self, block, count): - """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from - block number 'block'. - - The full result of the operation is stored in 'self._buf' on exit. - Returns the unpacked 'struct fiemap' data structure in form of a python - list (just like 'struct.upack()'). """ - - if block < 0 or block >= self.blocks_cnt: - raise Error("bad block number %d, should be within [0, %d]" \ - % (block, self.blocks_cnt)) - - # Initialize the 'struct fiemap' part of the buffer - struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size, - count * self.block_size, 0, 0, - self._fiemap_extent_cnt, 0) - - try: - fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1) - except IOError as err: - error_msg = "the FIEMAP ioctl failed for '%s': %s" \ - % (self._image_path, err) - if err.errno == os.errno.EPERM or err.errno == os.errno.EACCES: - # The FIEMAP ioctl was added in kernel version 2.6.28 in 2008 - error_msg += " (looks like your kernel does not support FIEMAP)" - - raise Error(error_msg) - - return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) - - def block_is_mapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is mapped and 'False' otherwise. """ - - struct_fiemap = self._invoke_fiemap(block, 1) - - # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field. - # If it contains zero, the block is not mapped, otherwise it is - # mapped. - return bool(struct_fiemap[3]) - - def block_is_unmapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is not mapped (hole) and 'False' otherwise. """ - - return not self.block_is_mapped(block) - - def _unpack_fiemap_extent(self, index): - """ Unpack a 'struct fiemap_extent' structure object number 'index' - from the internal 'self._buf' buffer. """ - - offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index - return struct.unpack(_FIEMAP_EXTENT_FORMAT, - self._buf[offset : offset + _FIEMAP_EXTENT_SIZE]) - - def _do_get_mapped_ranges(self, start, count): - """ Implements most the functionality for the 'get_mapped_ranges()' - generator: invokes the FIEMAP ioctl, walks through the mapped - extents and yields mapped block ranges. However, the ranges may be - consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' - simply merges them. """ - - block = start - while block < start + count: - struct_fiemap = self._invoke_fiemap(block, count) - - mapped_extents = struct_fiemap[3] - if mapped_extents == 0: - # No more mapped blocks - return - - extent = 0 - while extent < mapped_extents: - fiemap_extent = self._unpack_fiemap_extent(extent) - - # Start of the extent - extent_start = fiemap_extent[0] - # Starting block number of the extent - extent_block = extent_start / self.block_size - # Length of the extent - extent_len = fiemap_extent[2] - # Count of blocks in the extent - extent_count = extent_len / self.block_size - - # Extent length and offset have to be block-aligned - assert extent_start % self.block_size == 0 - assert extent_len % self.block_size == 0 - - if extent_block > start + count - 1: - return - - first = max(extent_block, block) - last = min(extent_block + extent_count, start + count) - 1 - yield (first, last) - - extent += 1 - - block = extent_block + extent_count - - def get_mapped_ranges(self, start, count): - """ A generator which yields ranges of mapped blocks in the file. The - ranges are tuples of 2 elements: [first, last], where 'first' is the - first mapped block and 'last' is the last mapped block. - - The ranges are yielded for the area of the file of size 'count' blocks, - starting from block 'start'. """ - - iterator = self._do_get_mapped_ranges(start, count) - - first_prev, last_prev = iterator.next() - - for first, last in iterator: - if last_prev == first - 1: - last_prev = last - else: - yield (first_prev, last_prev) - first_prev, last_prev = first, last - - yield (first_prev, last_prev) - - def get_unmapped_ranges(self, start, count): - """ Just like 'get_mapped_ranges()', but yields unmapped block ranges - instead (holes). """ - - hole_first = start - for first, last in self._do_get_mapped_ranges(start, count): - if first > hole_first: - yield (hole_first, first - 1) - - hole_first = last + 1 - - if hole_first < start + count: - yield (hole_first, start + count - 1) diff --git a/mic/utils/cmdln.py b/mic/utils/cmdln.py deleted file mode 100644 index b099473..0000000 --- a/mic/utils/cmdln.py +++ /dev/null @@ -1,1586 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2002-2007 ActiveState Software Inc. -# License: MIT (see LICENSE.txt for license details) -# Author: Trent Mick -# Home: http://trentm.com/projects/cmdln/ - -"""An improvement on Python's standard cmd.py module. - -As with cmd.py, this module provides "a simple framework for writing -line-oriented command intepreters." This module provides a 'RawCmdln' -class that fixes some design flaws in cmd.Cmd, making it more scalable -and nicer to use for good 'cvs'- or 'svn'-style command line interfaces -or simple shells. And it provides a 'Cmdln' class that add -optparse-based option processing. Basically you use it like this: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.alias('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - -See the README.txt or <http://trentm.com/projects/cmdln/> for more -details. -""" - -__version_info__ = (1, 1, 2) -__version__ = '.'.join(map(str, __version_info__)) - -import os -import sys -import re -import cmd -import optparse -from pprint import pprint -import sys - - - - -#---- globals - -LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3) - -# An unspecified optional argument when None is a meaningful value. -_NOT_SPECIFIED = ("Not", "Specified") - -# Pattern to match a TypeError message from a call that -# failed because of incorrect number of arguments (see -# Python/getargs.c). -_INCORRECT_NUM_ARGS_RE = re.compile( - r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))") - - - -#---- exceptions - -class CmdlnError(Exception): - """A cmdln.py usage error.""" - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - -class CmdlnUserError(Exception): - """An error by a user of a cmdln-based tool/shell.""" - pass - - - -#---- public methods and classes - -def alias(*aliases): - """Decorator to add aliases for Cmdln.do_* command handlers. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.alias("!", "sh") - def do_shell(self, argv): - #...implement 'shell' command - """ - def decorate(f): - if not hasattr(f, "aliases"): - f.aliases = [] - f.aliases += aliases - return f - return decorate - - -class RawCmdln(cmd.Cmd): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.RawCmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - def do_status(self, argv): - print "handle 'svn status' command" - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - See <http://trentm.com/projects/cmdln> for more information. - """ - name = None # if unset, defaults basename(sys.argv[0]) - prompt = None # if unset, defaults to self.name+"> " - version = None # if set, default top-level options include --version - - # Default messages for some 'help' command error cases. - # They are interpolated with one arg: the command. - nohelp = "no help on '%s'" - unknowncmd = "unknown command: '%s'" - - helpindent = '' # string with which to indent help output - - def __init__(self, completekey='tab', - stdin=None, stdout=None, stderr=None): - """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None) - - The optional argument 'completekey' is the readline name of a - completion key; it defaults to the Tab key. If completekey is - not None and the readline module is available, command completion - is done automatically. - - The optional arguments 'stdin', 'stdout' and 'stderr' specify - alternate input, output and error output file objects; if not - specified, sys.* are used. - - If 'stdout' but not 'stderr' is specified, stdout is used for - error output. This is to provide least surprise for users used - to only the 'stdin' and 'stdout' options with cmd.Cmd. - """ - import sys - if self.name is None: - self.name = os.path.basename(sys.argv[0]) - if self.prompt is None: - self.prompt = self.name+"> " - self._name_str = self._str(self.name) - self._prompt_str = self._str(self.prompt) - if stdin is not None: - self.stdin = stdin - else: - self.stdin = sys.stdin - if stdout is not None: - self.stdout = stdout - else: - self.stdout = sys.stdout - if stderr is not None: - self.stderr = stderr - elif stdout is not None: - self.stderr = stdout - else: - self.stderr = sys.stderr - self.cmdqueue = [] - self.completekey = completekey - self.cmdlooping = False - - def get_optparser(self): - """Hook for subclasses to set the option parser for the - top-level command/shell. - - This option parser is used retrieved and used by `.main()' to - handle top-level options. - - The default implements a single '-h|--help' option. Sub-classes - can return None to have no options at the top-level. Typically - an instance of CmdlnOptionParser should be returned. - """ - version = (self.version is not None - and "%s %s" % (self._name_str, self.version) - or None) - return CmdlnOptionParser(self, version=version) - - def postoptparse(self): - """Hook method executed just after `.main()' parses top-level - options. - - When called `self.options' holds the results of the option parse. - """ - pass - - def main(self, argv=None, loop=LOOP_NEVER): - """A possible mainline handler for a script, like so: - - import cmdln - class MyCmd(cmdln.Cmdln): - name = "mycmd" - ... - - if __name__ == "__main__": - MyCmd().main() - - By default this will use sys.argv to issue a single command to - 'MyCmd', then exit. The 'loop' argument can be use to control - interactive shell behaviour. - - Arguments: - "argv" (optional, default sys.argv) is the command to run. - It must be a sequence, where the first element is the - command name and subsequent elements the args for that - command. - "loop" (optional, default LOOP_NEVER) is a constant - indicating if a command loop should be started (i.e. an - interactive shell). Valid values (constants on this module): - LOOP_ALWAYS start loop and run "argv", if any - LOOP_NEVER run "argv" (or .emptyline()) and exit - LOOP_IF_EMPTY run "argv", if given, and exit; - otherwise, start loop - """ - if argv is None: - import sys - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - self.optparser = self.get_optparser() - if self.optparser: # i.e. optparser=None means don't process for opts - try: - self.options, args = self.optparser.parse_args(argv[1:]) - except CmdlnUserError, ex: - msg = "%s: %s\nTry '%s help' for info.\n"\ - % (self.name, ex, self.name) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return 1 - except StopOptionProcessing, ex: - return 0 - else: - self.options, args = None, argv[1:] - self.postoptparse() - - if loop == LOOP_ALWAYS: - if args: - self.cmdqueue.append(args) - return self.cmdloop() - elif loop == LOOP_NEVER: - if args: - return self.cmd(args) - else: - return self.emptyline() - elif loop == LOOP_IF_EMPTY: - if args: - return self.cmd(args) - else: - return self.cmdloop() - - def cmd(self, argv): - """Run one command and exit. - - "argv" is the arglist for the command to run. argv[0] is the - command to run. If argv is an empty list then the - 'emptyline' handler is run. - - Returns the return value from the command handler. - """ - assert isinstance(argv, (list, tuple)), \ - "'argv' is not a sequence: %r" % argv - retval = None - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - return retval - - def _str(self, s): - """Safely convert the given str/unicode to a string for printing.""" - try: - return str(s) - except UnicodeError: - #XXX What is the proper encoding to use here? 'utf-8' seems - # to work better than "getdefaultencoding" (usually - # 'ascii'), on OS X at least. - #import sys - #return s.encode(sys.getdefaultencoding(), "replace") - return s.encode("utf-8", "replace") - - def cmdloop(self, intro=None): - """Repeatedly issue a prompt, accept input, parse into an argv, and - dispatch (via .precmd(), .onecmd() and .postcmd()), passing them - the argv. In other words, start a shell. - - "intro" (optional) is a introductory message to print when - starting the command loop. This overrides the class - "intro" attribute, if any. - """ - self.cmdlooping = True - self.preloop() - if self.use_rawinput and self.completekey: - try: - import readline - self.old_completer = readline.get_completer() - readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") - except ImportError: - pass - try: - if intro is None: - intro = self.intro - if intro: - intro_str = self._str(intro) - self.stdout.write(intro_str+'\n') - self.stop = False - retval = None - while not self.stop: - if self.cmdqueue: - argv = self.cmdqueue.pop(0) - assert isinstance(argv, (list, tuple)), \ - "item on 'cmdqueue' is not a sequence: %r" % argv - else: - if self.use_rawinput: - try: - line = raw_input(self._prompt_str) - except EOFError: - line = 'EOF' - else: - self.stdout.write(self._prompt_str) - self.stdout.flush() - line = self.stdin.readline() - if not len(line): - line = 'EOF' - else: - line = line[:-1] # chop '\n' - argv = line2argv(line) - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - self.lastretval = retval - self.postloop() - finally: - if self.use_rawinput and self.completekey: - try: - import readline - readline.set_completer(self.old_completer) - except ImportError: - pass - self.cmdlooping = False - return retval - - def precmd(self, argv): - """Hook method executed just before the command argv is - interpreted, but after the input prompt is generated and issued. - - "argv" is the cmd to run. - - Returns an argv to run (i.e. this method can modify the command - to run). - """ - return argv - - def postcmd(self, argv): - """Hook method executed just after a command dispatch is finished. - - "argv" is the command that was run. - """ - pass - - def cmdexc(self, argv): - """Called if an exception is raised in any of precmd(), onecmd(), - or postcmd(). If True is returned, the exception is deemed to have - been dealt with. Otherwise, the exception is re-raised. - - The default implementation handles CmdlnUserError's, which - typically correspond to user error in calling commands (as - opposed to programmer error in the design of the script using - cmdln.py). - """ - import sys - type, exc, traceback = sys.exc_info() - if isinstance(exc, CmdlnUserError): - msg = "%s %s: %s\nTry '%s help %s' for info.\n"\ - % (self.name, argv[0], exc, self.name, argv[0]) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return True - - def onecmd(self, argv): - if not argv: - return self.emptyline() - self.lastcmd = argv - cmdname = self._get_canonical_cmd_name(argv[0]) - if cmdname: - handler = self._get_cmd_handler(cmdname) - if handler: - return self._dispatch_cmd(handler, argv) - return self.default(argv) - - def _dispatch_cmd(self, handler, argv): - return handler(argv) - - def default(self, argv): - """Hook called to handle a command for which there is no handler. - - "argv" is the command and arguments to run. - - The default implementation writes and error message to stderr - and returns an error exit status. - - Returns a numeric command exit status. - """ - errmsg = self._str(self.unknowncmd % (argv[0],)) - if self.cmdlooping: - self.stderr.write(errmsg+"\n") - else: - self.stderr.write("%s: %s\nTry '%s help' for info.\n" - % (self._name_str, errmsg, self._name_str)) - self.stderr.flush() - return 1 - - def parseline(self, line): - # This is used by Cmd.complete (readline completer function) to - # massage the current line buffer before completion processing. - # We override to drop special '!' handling. - line = line.strip() - if not line: - return None, None, line - elif line[0] == '?': - line = 'help ' + line[1:] - i, n = 0, len(line) - while i < n and line[i] in self.identchars: i = i+1 - cmd, arg = line[:i], line[i:].strip() - return cmd, arg, line - - def helpdefault(self, cmd, known): - """Hook called to handle help on a command for which there is no - help handler. - - "cmd" is the command name on which help was requested. - "known" is a boolean indicating if this command is known - (i.e. if there is a handler for it). - - Returns a return code. - """ - if known: - msg = self._str(self.nohelp % (cmd,)) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" % (self.name, msg)) - else: - msg = self.unknowncmd % (cmd,) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" - "Try '%s help' for info.\n" - % (self.name, msg, self.name)) - self.stderr.flush() - return 1 - - def do_help(self, argv): - """${cmd_name}: give detailed help on a specific sub-command - - Usage: - ${name} help [COMMAND] - """ - if len(argv) > 1: # asking for help on a particular command - doc = None - cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1] - if not cmdname: - return self.helpdefault(argv[1], False) - else: - helpfunc = getattr(self, "help_"+cmdname, None) - if helpfunc: - doc = helpfunc() - else: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - if doc is None: - return self.helpdefault(argv[1], handler != None) - else: # bare "help" command - doc = self.__class__.__doc__ # try class docstring - if doc is None: - # Try to provide some reasonable useful default help. - if self.cmdlooping: prefix = "" - else: prefix = self.name+' ' - doc = """Usage: - %sCOMMAND [ARGS...] - %shelp [COMMAND] - - ${option_list} - ${command_list} - ${help_list} - """ % (prefix, prefix) - cmdname = None - - if doc: # *do* have help content, massage and print that - doc = self._help_reindent(doc) - doc = self._help_preprocess(doc, cmdname) - doc = doc.rstrip() + '\n' # trim down trailing space - self.stdout.write(self._str(doc)) - self.stdout.flush() - do_help.aliases = ["?"] - - def _help_reindent(self, help, indent=None): - """Hook to re-indent help strings before writing to stdout. - - "help" is the help content to re-indent - "indent" is a string with which to indent each line of the - help content after normalizing. If unspecified or None - then the default is use: the 'self.helpindent' class - attribute. By default this is the empty string, i.e. - no indentation. - - By default, all common leading whitespace is removed and then - the lot is indented by 'self.helpindent'. When calculating the - common leading whitespace the first line is ignored -- hence - help content for Conan can be written as follows and have the - expected indentation: - - def do_crush(self, ...): - '''${cmd_name}: crush your enemies, see them driven before you... - - c.f. Conan the Barbarian''' - """ - if indent is None: - indent = self.helpindent - lines = help.splitlines(0) - _dedentlines(lines, skip_first_line=True) - lines = [(indent+line).rstrip() for line in lines] - return '\n'.join(lines) - - def _help_preprocess(self, help, cmdname): - """Hook to preprocess a help string before writing to stdout. - - "help" is the help string to process. - "cmdname" is the canonical sub-command name for which help - is being given, or None if the help is not specific to a - command. - - By default the following template variables are interpolated in - help content. (Note: these are similar to Python 2.4's - string.Template interpolation but not quite.) - - ${name} - The tool's/shell's name, i.e. 'self.name'. - ${option_list} - A formatted table of options for this shell/tool. - ${command_list} - A formatted table of available sub-commands. - ${help_list} - A formatted table of additional help topics (i.e. 'help_*' - methods with no matching 'do_*' method). - ${cmd_name} - The name (and aliases) for this sub-command formatted as: - "NAME (ALIAS1, ALIAS2, ...)". - ${cmd_usage} - A formatted usage block inferred from the command function - signature. - ${cmd_option_list} - A formatted table of options for this sub-command. (This is - only available for commands using the optparse integration, - i.e. using @cmdln.option decorators or manually setting the - 'optparser' attribute on the 'do_*' method.) - - Returns the processed help. - """ - preprocessors = { - "${name}": self._help_preprocess_name, - "${option_list}": self._help_preprocess_option_list, - "${command_list}": self._help_preprocess_command_list, - "${help_list}": self._help_preprocess_help_list, - "${cmd_name}": self._help_preprocess_cmd_name, - "${cmd_usage}": self._help_preprocess_cmd_usage, - "${cmd_option_list}": self._help_preprocess_cmd_option_list, - } - - for marker, preprocessor in preprocessors.items(): - if marker in help: - help = preprocessor(help, cmdname) - return help - - def _help_preprocess_name(self, help, cmdname=None): - return help.replace("${name}", self.name) - - def _help_preprocess_option_list(self, help, cmdname=None): - marker = "${option_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - if self.optparser: - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - self.optparser.formatter.indent_increment = 4 - self.optparser.formatter.current_indent = indent_width - block = self.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - - def _help_preprocess_command_list(self, help, cmdname=None): - marker = "${command_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Find any aliases for commands. - token2canonical = self._get_canonical_map() - aliases = {} - for token, cmdname in token2canonical.items(): - if token == cmdname: continue - aliases.setdefault(cmdname, []).append(token) - - # Get the list of (non-hidden) commands and their - # documentation, if any. - cmdnames = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): - cmdnames[attr[3:]] = True - cmdnames = cmdnames.keys() - cmdnames.sort() - linedata = [] - for cmdname in cmdnames: - if aliases.get(cmdname): - a = aliases[cmdname] - a.sort() - cmdstr = "%s (%s)" % (cmdname, ", ".join(a)) - else: - cmdstr = cmdname - doc = None - try: - helpfunc = getattr(self, 'help_'+cmdname) - except AttributeError: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - else: - doc = helpfunc() - - # Strip "${cmd_name}: " from the start of a command's doc. Best - # practice dictates that command help strings begin with this, but - # it isn't at all wanted for the command list. - to_strip = "${cmd_name}:" - if doc and doc.startswith(to_strip): - #log.debug("stripping %r from start of %s's help string", - # to_strip, cmdname) - doc = doc[len(to_strip):].lstrip() - linedata.append( (cmdstr, doc) ) - - if linedata: - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = indent + "Commands:\n" \ - + '\n'.join(lines) + "\n\n" - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _gen_names_and_attrs(self): - # Inheritance says we have to look in class and - # base classes; order is not important. - names = [] - classes = [self.__class__] - while classes: - aclass = classes.pop(0) - if aclass.__bases__: - classes = classes + list(aclass.__bases__) - for name in dir(aclass): - yield (name, getattr(aclass, name)) - - def _help_preprocess_help_list(self, help, cmdname=None): - marker = "${help_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Determine the additional help topics, if any. - helpnames = {} - token2cmdname = self._get_canonical_map() - for attrname, attr in self._gen_names_and_attrs(): - if not attrname.startswith("help_"): continue - helpname = attrname[5:] - if helpname not in token2cmdname: - helpnames[helpname] = attr - - if helpnames: - linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()] - linedata.sort() - - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = (indent - + "Additional help topics (run `%s help TOPIC'):\n" % self.name - + '\n'.join(lines) - + "\n\n") - else: - block = '' - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _help_preprocess_cmd_name(self, help, cmdname=None): - marker = "${cmd_name}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - s = cmdname - if hasattr(handler, "aliases"): - s += " (%s)" % (", ".join(handler.aliases)) - help = help.replace(marker, s) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_usage(self, help, cmdname=None): - marker = "${cmd_usage}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Extract the introspection bits we need. - func = handler.im_func - if func.func_defaults: - func_defaults = list(func.func_defaults) - else: - func_defaults = [] - co_argcount = func.func_code.co_argcount - co_varnames = func.func_code.co_varnames - co_flags = func.func_code.co_flags - CO_FLAGS_ARGS = 4 - CO_FLAGS_KWARGS = 8 - - # Adjust argcount for possible *args and **kwargs arguments. - argcount = co_argcount - if co_flags & CO_FLAGS_ARGS: argcount += 1 - if co_flags & CO_FLAGS_KWARGS: argcount += 1 - - # Determine the usage string. - usage = "%s %s" % (self.name, cmdname) - if argcount <= 2: # handler ::= do_FOO(self, argv) - usage += " [ARGS...]" - elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...) - argnames = list(co_varnames[3:argcount]) - tail = "" - if co_flags & CO_FLAGS_KWARGS: - name = argnames.pop(-1) - import warnings - # There is no generally accepted mechanism for passing - # keyword arguments from the command line. Could - # *perhaps* consider: arg=value arg2=value2 ... - warnings.warn("argument '**%s' on '%s.%s' command " - "handler will never get values" - % (name, self.__class__.__name__, - func.func_name)) - if co_flags & CO_FLAGS_ARGS: - name = argnames.pop(-1) - tail = "[%s...]" % name.upper() - while func_defaults: - func_defaults.pop(-1) - name = argnames.pop(-1) - tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail) - while argnames: - name = argnames.pop(-1) - tail = "%s %s" % (name.upper(), tail) - usage += ' ' + tail - - block_lines = [ - self.helpindent + "Usage:", - self.helpindent + ' '*4 + usage - ] - block = '\n'.join(block_lines) + '\n\n' - - help = help.replace(indent+marker+suffix, block, 1) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_option_list(self, help, cmdname=None): - marker = "${cmd_option_list}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - if hasattr(handler, "optparser"): - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - handler.optparser.formatter.indent_increment = 4 - handler.optparser.formatter.current_indent = indent_width - block = handler.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _get_canonical_cmd_name(self, token): - map = self._get_canonical_map() - return map.get(token, None) - - def _get_canonical_map(self): - """Return a mapping of available command names and aliases to - their canonical command name. - """ - cacheattr = "_token2canonical" - if not hasattr(self, cacheattr): - # Get the list of commands and their aliases, if any. - token2canonical = {} - cmd2funcname = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): cmdname = attr[3:] - elif attr.startswith("_do_"): cmdname = attr[4:] - else: - continue - cmd2funcname[cmdname] = attr - token2canonical[cmdname] = cmdname - for cmdname, funcname in cmd2funcname.items(): # add aliases - func = getattr(self, funcname) - aliases = getattr(func, "aliases", []) - for alias in aliases: - if alias in cmd2funcname: - import warnings - warnings.warn("'%s' alias for '%s' command conflicts " - "with '%s' handler" - % (alias, cmdname, cmd2funcname[alias])) - continue - token2canonical[alias] = cmdname - setattr(self, cacheattr, token2canonical) - return getattr(self, cacheattr) - - def _get_cmd_handler(self, cmdname): - handler = None - try: - handler = getattr(self, 'do_' + cmdname) - except AttributeError: - try: - # Private command handlers begin with "_do_". - handler = getattr(self, '_do_' + cmdname) - except AttributeError: - pass - return handler - - def _do_EOF(self, argv): - # Default EOF handler - # Note: an actual EOF is redirected to this command. - #TODO: separate name for this. Currently it is available from - # command-line. Is that okay? - self.stdout.write('\n') - self.stdout.flush() - self.stop = True - - def emptyline(self): - # Different from cmd.Cmd: don't repeat the last command for an - # emptyline. - if self.cmdlooping: - pass - else: - return self.do_help(["help"]) - - -#---- optparse.py extension to fix (IMO) some deficiencies -# -# See the class _OptionParserEx docstring for details. -# - -class StopOptionProcessing(Exception): - """Indicate that option *and argument* processing should stop - cleanly. This is not an error condition. It is similar in spirit to - StopIteration. This is raised by _OptionParserEx's default "help" - and "version" option actions and can be raised by custom option - callbacks too. - - Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx) - usage is: - - parser = CmdlnOptionParser(mycmd) - parser.add_option("-f", "--force", dest="force") - ... - try: - opts, args = parser.parse_args() - except StopOptionProcessing: - # normal termination, "--help" was probably given - sys.exit(0) - """ - -class _OptionParserEx(optparse.OptionParser): - """An optparse.OptionParser that uses exceptions instead of sys.exit. - - This class is an extension of optparse.OptionParser that differs - as follows: - - Correct (IMO) the default OptionParser error handling to never - sys.exit(). Instead OptParseError exceptions are passed through. - - Add the StopOptionProcessing exception (a la StopIteration) to - indicate normal termination of option processing. - See StopOptionProcessing's docstring for details. - - I'd also like to see the following in the core optparse.py, perhaps - as a RawOptionParser which would serve as a base class for the more - generally used OptionParser (that works as current): - - Remove the implicit addition of the -h|--help and --version - options. They can get in the way (e.g. if want '-?' and '-V' for - these as well) and it is not hard to do: - optparser.add_option("-h", "--help", action="help") - optparser.add_option("--version", action="version") - These are good practices, just not valid defaults if they can - get in the way. - """ - def error(self, msg): - raise optparse.OptParseError(msg) - - def exit(self, status=0, msg=None): - if status == 0: - raise StopOptionProcessing(msg) - else: - #TODO: don't lose status info here - raise optparse.OptParseError(msg) - - - -#---- optparse.py-based option processing support - -class CmdlnOptionParser(_OptionParserEx): - """An optparse.OptionParser class more appropriate for top-level - Cmdln options. For parsing of sub-command options, see - SubCmdOptionParser. - - Changes: - - disable_interspersed_args() by default, because a Cmdln instance - has sub-commands which may themselves have options. - - Redirect print_help() to the Cmdln.do_help() which is better - equiped to handle the "help" action. - - error() will raise a CmdlnUserError: OptionParse.error() is meant - to be called for user errors. Raising a well-known error here can - make error handling clearer. - - Also see the changes in _OptionParserEx. - """ - def __init__(self, cmdln, **kwargs): - self.cmdln = cmdln - kwargs["prog"] = self.cmdln.name - _OptionParserEx.__init__(self, **kwargs) - self.disable_interspersed_args() - - def print_help(self, file=None): - self.cmdln.onecmd(["help"]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -class SubCmdOptionParser(_OptionParserEx): - def set_cmdln_info(self, cmdln, subcmd): - """Called by Cmdln to pass relevant info about itself needed - for print_help(). - """ - self.cmdln = cmdln - self.subcmd = subcmd - - def print_help(self, file=None): - self.cmdln.onecmd(["help", self.subcmd]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -def option(*args, **kwargs): - """Decorator to add an option to the optparser argument of a Cmdln - subcommand. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.option("-f", "--force", help="force removal") - def do_remove(self, subcmd, opts, *args): - #... - """ - #XXX Is there a possible optimization for many options to not have a - # large stack depth here? - def decorate(f): - if not hasattr(f, "optparser"): - f.optparser = SubCmdOptionParser() - f.optparser.add_option(*args, **kwargs) - return f - return decorate - - -class Cmdln(RawCmdln): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - 'Cmdln' extends 'RawCmdln' by providing optparse option processing - integration. See this class' _dispatch_cmd() docstring and - <http://trentm.com/projects/cmdln> for more information. - """ - def _dispatch_cmd(self, handler, argv): - """Introspect sub-command handler signature to determine how to - dispatch the command. The raw handler provided by the base - 'RawCmdln' class is still supported: - - def do_foo(self, argv): - # 'argv' is the vector of command line args, argv[0] is - # the command name itself (i.e. "foo" or an alias) - pass - - In addition, if the handler has more than 2 arguments option - processing is automatically done (using optparse): - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar(self, subcmd, opts, *args): - # subcmd = <"bar" or an alias> - # opts = <an optparse.Values instance> - if opts.verbose: - print "lots of debugging output..." - # args = <tuple of arguments> - for arg in args: - bar(arg) - - TODO: explain that "*args" can be other signatures as well. - - The `cmdln.option` decorator corresponds to an `add_option()` - method call on an `optparse.OptionParser` instance. - - You can declare a specific number of arguments: - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar2(self, subcmd, opts, bar_one, bar_two): - #... - - and an appropriate error message will be raised/printed if the - command is called with a different number of args. - """ - co_argcount = handler.im_func.func_code.co_argcount - if co_argcount == 2: # handler ::= do_foo(self, argv) - return handler(argv) - elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...) - try: - optparser = handler.optparser - except AttributeError: - optparser = handler.im_func.optparser = SubCmdOptionParser() - assert isinstance(optparser, SubCmdOptionParser) - optparser.set_cmdln_info(self, argv[0]) - try: - opts, args = optparser.parse_args(argv[1:]) - except StopOptionProcessing: - #TODO: this doesn't really fly for a replacement of - # optparse.py behaviour, does it? - return 0 # Normal command termination - - try: - return handler(argv[0], opts, *args) - except TypeError, ex: - # Some TypeError's are user errors: - # do_foo() takes at least 4 arguments (3 given) - # do_foo() takes at most 5 arguments (6 given) - # do_foo() takes exactly 5 arguments (6 given) - # Raise CmdlnUserError for these with a suitably - # massaged error message. - import sys - tb = sys.exc_info()[2] # the traceback object - if tb.tb_next is not None: - # If the traceback is more than one level deep, then the - # TypeError do *not* happen on the "handler(...)" call - # above. In that we don't want to handle it specially - # here: it would falsely mask deeper code errors. - raise - msg = ex.args[0] - match = _INCORRECT_NUM_ARGS_RE.search(msg) - if match: - msg = list(match.groups()) - msg[1] = int(msg[1]) - 3 - if msg[1] == 1: - msg[2] = msg[2].replace("arguments", "argument") - msg[3] = int(msg[3]) - 3 - msg = ''.join(map(str, msg)) - raise CmdlnUserError(msg) - else: - raise - else: - raise CmdlnError("incorrect argcount for %s(): takes %d, must " - "take 2 for 'argv' signature or 3+ for 'opts' " - "signature" % (handler.__name__, co_argcount)) - - - -#---- internal support functions - -def _format_linedata(linedata, indent, indent_width): - """Format specific linedata into a pleasant layout. - - "linedata" is a list of 2-tuples of the form: - (<item-display-string>, <item-docstring>) - "indent" is a string to use for one level of indentation - "indent_width" is a number of columns by which the - formatted data will be indented when printed. - - The <item-display-string> column is held to 15 columns. - """ - lines = [] - WIDTH = 78 - indent_width - SPACING = 2 - NAME_WIDTH_LOWER_BOUND = 13 - NAME_WIDTH_UPPER_BOUND = 16 - NAME_WIDTH = max([len(s) for s,d in linedata]) - if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND: - NAME_WIDTH = NAME_WIDTH_LOWER_BOUND - else: - NAME_WIDTH = NAME_WIDTH_UPPER_BOUND - - DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING - for namestr, doc in linedata: - line = indent + namestr - if len(namestr) <= NAME_WIDTH: - line += ' ' * (NAME_WIDTH + SPACING - len(namestr)) - else: - lines.append(line) - line = indent + ' ' * (NAME_WIDTH + SPACING) - line += _summarize_doc(doc, DOC_WIDTH) - lines.append(line.rstrip()) - return lines - -def _summarize_doc(doc, length=60): - r"""Parse out a short one line summary from the given doclines. - - "doc" is the doc string to summarize. - "length" is the max length for the summary - - >>> _summarize_doc("this function does this") - 'this function does this' - >>> _summarize_doc("this function does this", 10) - 'this fu...' - >>> _summarize_doc("this function does this\nand that") - 'this function does this and that' - >>> _summarize_doc("this function does this\n\nand that") - 'this function does this' - """ - import re - if doc is None: - return "" - assert length > 3, "length <= 3 is absurdly short for a doc summary" - doclines = doc.strip().splitlines(0) - if not doclines: - return "" - - summlines = [] - for i, line in enumerate(doclines): - stripped = line.strip() - if not stripped: - break - summlines.append(stripped) - if len(''.join(summlines)) >= length: - break - - summary = ' '.join(summlines) - if len(summary) > length: - summary = summary[:length-3] + "..." - return summary - - -def line2argv(line): - r"""Parse the given line into an argument vector. - - "line" is the line of input to parse. - - This may get niggly when dealing with quoting and escaping. The - current state of this parsing may not be completely thorough/correct - in this respect. - - >>> from cmdln import line2argv - >>> line2argv("foo") - ['foo'] - >>> line2argv("foo bar") - ['foo', 'bar'] - >>> line2argv("foo bar ") - ['foo', 'bar'] - >>> line2argv(" foo bar") - ['foo', 'bar'] - - Quote handling: - - >>> line2argv("'foo bar'") - ['foo bar'] - >>> line2argv('"foo bar"') - ['foo bar'] - >>> line2argv(r'"foo\"bar"') - ['foo"bar'] - >>> line2argv("'foo bar' spam") - ['foo bar', 'spam'] - >>> line2argv("'foo 'bar spam") - ['foo bar', 'spam'] - - >>> line2argv('some\tsimple\ttests') - ['some', 'simple', 'tests'] - >>> line2argv('a "more complex" test') - ['a', 'more complex', 'test'] - >>> line2argv('a more="complex test of " quotes') - ['a', 'more=complex test of ', 'quotes'] - >>> line2argv('a more" complex test of " quotes') - ['a', 'more complex test of ', 'quotes'] - >>> line2argv('an "embedded \\"quote\\""') - ['an', 'embedded "quote"'] - - # Komodo bug 48027 - >>> line2argv('foo bar C:\\') - ['foo', 'bar', 'C:\\'] - - # Komodo change 127581 - >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"') - ['\\test\\slash', 'foo bar', 'foo"bar'] - - # Komodo change 127629 - >>> if sys.platform == "win32": - ... line2argv(r'\foo\bar') == ['\\foo\\bar'] - ... line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar'] - ... line2argv('"foo') == ['foo'] - ... else: - ... line2argv(r'\foo\bar') == ['foobar'] - ... line2argv(r'\\foo\\bar') == ['\\foo\\bar'] - ... try: - ... line2argv('"foo') - ... except ValueError, ex: - ... "not terminated" in str(ex) - True - True - True - """ - import string - line = line.strip() - argv = [] - state = "default" - arg = None # the current argument being parsed - i = -1 - while 1: - i += 1 - if i >= len(line): break - ch = line[i] - - if ch == "\\" and i+1 < len(line): - # escaped char always added to arg, regardless of state - if arg is None: arg = "" - if (sys.platform == "win32" - or state in ("double-quoted", "single-quoted") - ) and line[i+1] not in tuple('"\''): - arg += ch - i += 1 - arg += line[i] - continue - - if state == "single-quoted": - if ch == "'": - state = "default" - else: - arg += ch - elif state == "double-quoted": - if ch == '"': - state = "default" - else: - arg += ch - elif state == "default": - if ch == '"': - if arg is None: arg = "" - state = "double-quoted" - elif ch == "'": - if arg is None: arg = "" - state = "single-quoted" - elif ch in string.whitespace: - if arg is not None: - argv.append(arg) - arg = None - else: - if arg is None: arg = "" - arg += ch - if arg is not None: - argv.append(arg) - if not sys.platform == "win32" and state != "default": - raise ValueError("command line is not terminated: unfinished %s " - "segment" % state) - return argv - - -def argv2line(argv): - r"""Put together the given argument vector into a command line. - - "argv" is the argument vector to process. - - >>> from cmdln import argv2line - >>> argv2line(['foo']) - 'foo' - >>> argv2line(['foo', 'bar']) - 'foo bar' - >>> argv2line(['foo', 'bar baz']) - 'foo "bar baz"' - >>> argv2line(['foo"bar']) - 'foo"bar' - >>> print argv2line(['foo" bar']) - 'foo" bar' - >>> print argv2line(["foo' bar"]) - "foo' bar" - >>> argv2line(["foo'bar"]) - "foo'bar" - """ - escapedArgs = [] - for arg in argv: - if ' ' in arg and '"' not in arg: - arg = '"'+arg+'"' - elif ' ' in arg and "'" not in arg: - arg = "'"+arg+"'" - elif ' ' in arg: - arg = arg.replace('"', r'\"') - arg = '"'+arg+'"' - escapedArgs.append(arg) - return ' '.join(escapedArgs) - - -# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook -def _dedentlines(lines, tabsize=8, skip_first_line=False): - """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines - - "lines" is a list of lines to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - Same as dedent() except operates on a sequence of lines. Note: the - lines list is modified **in-place**. - """ - DEBUG = False - if DEBUG: - print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ - % (tabsize, skip_first_line) - indents = [] - margin = None - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - indent = 0 - for ch in line: - if ch == ' ': - indent += 1 - elif ch == '\t': - indent += tabsize - (indent % tabsize) - elif ch in '\r\n': - continue # skip all-whitespace lines - else: - break - else: - continue # skip all-whitespace lines - if DEBUG: print "dedent: indent=%d: %r" % (indent, line) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if DEBUG: print "dedent: margin=%r" % margin - - if margin is not None and margin > 0: - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - removed = 0 - for j, ch in enumerate(line): - if ch == ' ': - removed += 1 - elif ch == '\t': - removed += tabsize - (removed % tabsize) - elif ch in '\r\n': - if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line - lines[i] = lines[i][j:] - break - else: - raise ValueError("unexpected non-whitespace char %r in " - "line %r while removing %d-space margin" - % (ch, line, margin)) - if DEBUG: - print "dedent: %r: %r -> removed %d/%d"\ - % (line, ch, removed, margin) - if removed == margin: - lines[i] = lines[i][j+1:] - break - elif removed > margin: - lines[i] = ' '*(removed-margin) + lines[i][j+1:] - break - return lines - -def _dedent(text, tabsize=8, skip_first_line=False): - """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text - - "text" is the text to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - textwrap.dedent(s), but don't expand tabs to spaces - """ - lines = text.splitlines(1) - _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) - return ''.join(lines) - - -def _get_indent(marker, s, tab_width=8): - """_get_indent(marker, s, tab_width=8) -> - (<indentation-of-'marker'>, <indentation-width>)""" - # Figure out how much the marker is indented. - INDENT_CHARS = tuple(' \t') - start = s.index(marker) - i = start - while i > 0: - if s[i-1] not in INDENT_CHARS: - break - i -= 1 - indent = s[i:start] - indent_width = 0 - for ch in indent: - if ch == ' ': - indent_width += 1 - elif ch == '\t': - indent_width += tab_width - (indent_width % tab_width) - return indent, indent_width - -def _get_trailing_whitespace(marker, s): - """Return the whitespace content trailing the given 'marker' in string 's', - up to and including a newline. - """ - suffix = '' - start = s.index(marker) + len(marker) - i = start - while i < len(s): - if s[i] in ' \t': - suffix += s[i] - elif s[i] in '\r\n': - suffix += s[i] - if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n': - suffix += s[i+1] - break - else: - break - i += 1 - return suffix - - - -#---- bash completion support -# Note: This is still experimental. I expect to change this -# significantly. -# -# To get Bash completion for a cmdln.Cmdln class, run the following -# bash command: -# $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname -# For example: -# $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn -# -#TODO: Simplify the above so don't have to given path to script (try to -# find it on PATH, if possible). Could also make class name -# optional if there is only one in the module (common case). - -if __name__ == "__main__" and len(sys.argv) == 6: - def _log(s): - return # no-op, comment out for debugging - from os.path import expanduser - fout = open(expanduser("~/tmp/bashcpln.log"), 'a') - fout.write(str(s) + '\n') - fout.close() - - # Recipe: module_from_path (1.0.1+) - def _module_from_path(path): - import imp, os, sys - path = os.path.expanduser(path) - dir = os.path.dirname(path) or os.curdir - name = os.path.splitext(os.path.basename(path))[0] - sys.path.insert(0, dir) - try: - iinfo = imp.find_module(name, [dir]) - return imp.load_module(name, *iinfo) - finally: - sys.path.remove(dir) - - def _get_bash_cplns(script_path, class_name, cmd_name, - token, preceding_token): - _log('--') - _log('get_cplns(%r, %r, %r, %r, %r)' - % (script_path, class_name, cmd_name, token, preceding_token)) - comp_line = os.environ["COMP_LINE"] - comp_point = int(os.environ["COMP_POINT"]) - _log("COMP_LINE: %r" % comp_line) - _log("COMP_POINT: %r" % comp_point) - - try: - script = _module_from_path(script_path) - except ImportError, ex: - _log("error importing `%s': %s" % (script_path, ex)) - return [] - shell = getattr(script, class_name)() - cmd_map = shell._get_canonical_map() - del cmd_map["EOF"] - - # Determine if completing the sub-command name. - parts = comp_line[:comp_point].split(None, 1) - _log(parts) - if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]): - #TODO: if parts[1].startswith('-'): handle top-level opts - _log("complete sub-command names") - matches = {} - for name, canon_name in cmd_map.items(): - if name.startswith(token): - matches[name] = canon_name - if not matches: - return [] - elif len(matches) == 1: - return matches.keys() - elif len(set(matches.values())) == 1: - return [matches.values()[0]] - else: - return matches.keys() - - # Otherwise, complete options for the given sub-command. - #TODO: refine this so it does the right thing with option args - if token.startswith('-'): - cmd_name = comp_line.split(None, 2)[1] - try: - cmd_canon_name = cmd_map[cmd_name] - except KeyError: - return [] - handler = shell._get_cmd_handler(cmd_canon_name) - optparser = getattr(handler, "optparser", None) - if optparser is None: - optparser = SubCmdOptionParser() - opt_strs = [] - for option in optparser.option_list: - for opt_str in option._short_opts + option._long_opts: - if opt_str.startswith(token): - opt_strs.append(opt_str) - return opt_strs - - return [] - - for cpln in _get_bash_cplns(*sys.argv[1:]): - print cpln - diff --git a/mic/utils/errors.py b/mic/utils/errors.py index 8d720f9..f64b452 100644 --- a/mic/utils/errors.py +++ b/mic/utils/errors.py @@ -1,7 +1,5 @@ -#!/usr/bin/python -tt # -# Copyright (c) 2007 Red Hat, Inc. -# Copyright (c) 2011 Intel, Inc. +# Copyright (c) 2011~2013 Intel, Inc. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -16,11 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. +""" Collection of all error class """ + class CreatorError(Exception): - """An exception base class for all imgcreate errors.""" - keyword = '<creator>' + """ Based class for all mic creator errors """ + keyword = None def __init__(self, msg): + Exception.__init__(self) self.msg = msg def __str__(self): @@ -28,44 +29,50 @@ class CreatorError(Exception): self.msg = self.msg.encode('utf-8', 'ignore') else: self.msg = str(self.msg) - return self.keyword + self.msg + return self.msg -class Usage(CreatorError): - keyword = '<usage>' + def __repr__(self): + if not hasattr(self, 'keyword') or not self.keyword: + self.keyword = self.__class__.__name__ + return "<%s> %s" % (self.keyword, str(self)) - def __str__(self): - if isinstance(self.msg, unicode): - self.msg = self.msg.encode('utf-8', 'ignore') - else: - self.msg = str(self.msg) - return self.keyword + self.msg + ', please use "--help" for more info' +class Usage(CreatorError): + """ Error class for Usage """ + pass class Abort(CreatorError): - keyword = '' + """ Error class for Abort """ + pass class ConfigError(CreatorError): - keyword = '<config>' + """ Error class for Config file """ + keyword = 'Config' class KsError(CreatorError): - keyword = '<kickstart>' + """ Error class for Kickstart module """ + keyword = 'Kickstart' class RepoError(CreatorError): - keyword = '<repo>' + """ Error class for Repository related """ + keyword = 'Repository' class RpmError(CreatorError): - keyword = '<rpm>' + """ Error class for RPM related """ + keyword = 'RPM' class MountError(CreatorError): - keyword = '<mount>' + """ Error class for Mount related """ + keyword = 'Mount' class SnapshotError(CreatorError): - keyword = '<snapshot>' + """ Error class for Snapshot related """ + keyword = 'Snapshot' class SquashfsError(CreatorError): - keyword = '<squashfs>' + """ Error class for Squashfs related """ + keyword = 'Squashfs' class BootstrapError(CreatorError): - keyword = '<bootstrap>' + """ Error class for Bootstrap related """ + keyword = 'Bootstrap' -class RuntimeError(CreatorError): - keyword = '<runtime>' diff --git a/mic/utils/fs_related.py b/mic/utils/fs_related.py index 56b9a4f..a6c2c0b 100644 --- a/mic/utils/fs_related.py +++ b/mic/utils/fs_related.py @@ -24,6 +24,7 @@ import stat import random import string import time +import uuid from mic import msger from mic.utils import runner @@ -87,28 +88,11 @@ def resize2fs(fs, size): else: return runner.show([resize2fs, fs, "%sK" % (size / 1024,)]) -def my_fuser(fp): - fuser = find_binary_path("fuser") - if not os.path.exists(fp): - return False - - rc = runner.quiet([fuser, "-s", fp]) - if rc == 0: - for pid in runner.outs([fuser, fp]).split(): - fd = open("/proc/%s/cmdline" % pid, "r") - cmdline = fd.read() - fd.close() - if cmdline[:-1] == "/bin/bash": - return True - - # not found - return False - class BindChrootMount: """Represents a bind mount of a directory into a chroot.""" def __init__(self, src, chroot, dest = None, option = None): self.root = os.path.abspath(os.path.expanduser(chroot)) - self.option = option + self.mount_option = option self.orig_src = self.src = src if os.path.islink(src): @@ -133,23 +117,26 @@ class BindChrootMount: return False - def has_chroot_instance(self): - lock = os.path.join(self.root, ".chroot.lock") - return my_fuser(lock) - def mount(self): if self.mounted or self.ismounted(): return - makedirs(self.dest) - rc = runner.show([self.mountcmd, "--bind", self.src, self.dest]) + try: + makedirs(self.dest) + except OSError, err: + if err.errno == errno.ENOSPC: + msger.warning("No space left on device '%s'" % err.filename) + return + + if self.mount_option: + cmdline = [self.mountcmd, "-o" ,"bind", "-o", "%s" % \ + self.mount_option, self.src, self.dest] + else: + cmdline = [self.mountcmd, "-o" ,"bind", self.src, self.dest] + rc, errout = runner.runtool(cmdline, catch=2) if rc != 0: - raise MountError("Bind-mounting '%s' to '%s' failed" % - (self.src, self.dest)) - if self.option: - rc = runner.show([self.mountcmd, "--bind", "-o", "remount,%s" % self.option, self.dest]) - if rc != 0: - raise MountError("Bind-remounting '%s' failed" % self.dest) + raise MountError("Bind-mounting '%s' to '%s' failed: %s" % + (self.src, self.dest, errout)) self.mounted = True if os.path.islink(self.orig_src): @@ -158,10 +145,7 @@ class BindChrootMount: os.symlink(self.src, dest) def unmount(self): - if self.has_chroot_instance(): - return - - if self.ismounted(): + if self.mounted or self.ismounted(): runner.show([self.umountcmd, "-l", self.dest]) self.mounted = False @@ -434,11 +418,11 @@ class DiskMount(Mount): class ExtDiskMount(DiskMount): """A DiskMount object that is able to format/resize ext[23] filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): + def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid=None): DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) self.blocksize = blocksize self.fslabel = fslabel.replace("/", "") - self.uuid = None + self.uuid = fsuuid or str(uuid.uuid4()) self.skipformat = skipformat self.fsopts = fsopts self.extopts = None @@ -459,7 +443,7 @@ class ExtDiskMount(DiskMount): msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) cmdlist = [self.mkfscmd, "-F", "-L", self.fslabel, "-m", "1", "-b", - str(self.blocksize)] + str(self.blocksize), "-U", self.uuid] if self.extopts: cmdlist.extend(self.extopts.split()) cmdlist.extend([self.disk.device]) @@ -473,18 +457,8 @@ class ExtDiskMount(DiskMount): msger.debug("Tuning filesystem on %s" % self.disk.device) runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) - rc, out = runner.runtool([self.dumpe2fs, '-h', self.disk.device], - catch=2) - if rc != 0: - raise MountError("Error dumpe2fs %s filesystem on disk %s:\n%s" % - (self.fstype, self.disk.device, out)) - # FIXME: specify uuid in mkfs parameter - try: - self.uuid = self.__parse_field(out, "Filesystem UUID") - except: - self.uuid = None - def __resize_filesystem(self, size = None): + msger.info("Resizing filesystem ...") current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] if size is None: @@ -526,6 +500,7 @@ class ExtDiskMount(DiskMount): "Block count")) * self.blocksize def __resize_to_minimal(self): + msger.info("Resizing filesystem to minimal ...") self.__fsck() # @@ -556,11 +531,13 @@ class ExtDiskMount(DiskMount): class VfatDiskMount(DiskMount): """A DiskMount object that is able to format vfat/msdos filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): + def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid = None): DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) self.blocksize = blocksize self.fslabel = fslabel.replace("/", "") - self.uuid = "%08X" % int(time.time()) + rand1 = random.randint(0, 2**16 - 1) + rand2 = random.randint(0, 2**16 - 1) + self.uuid = fsuuid or "%04X-%04X" % (rand1, rand2) self.skipformat = skipformat self.fsopts = fsopts self.fsckcmd = find_binary_path("fsck." + self.fstype) @@ -571,7 +548,8 @@ class VfatDiskMount(DiskMount): return msger.verbose("Formating %s filesystem on %s" % (self.fstype, self.disk.device)) - rc = runner.show([self.mkfscmd, "-n", self.fslabel, "-i", self.uuid, self.disk.device]) + rc = runner.show([self.mkfscmd, "-n", self.fslabel, + "-i", self.uuid.replace("-", ""), self.disk.device]) if rc != 0: raise MountError("Error creating %s filesystem on disk %s" % (self.fstype,self.disk.device)) @@ -637,12 +615,12 @@ class VfatDiskMount(DiskMount): class BtrfsDiskMount(DiskMount): """A DiskMount object that is able to format/resize btrfs filesystems.""" - def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None): + def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True, skipformat = False, fsopts = None, fsuuid = None): self.__check_btrfs() DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir) self.blocksize = blocksize self.fslabel = fslabel.replace("/", "") - self.uuid = None + self.uuid = fsuuid or None self.skipformat = skipformat self.fsopts = fsopts self.blkidcmd = find_binary_path("blkid") @@ -866,7 +844,7 @@ class LoopDevice(object): def register(self, device): self.device = device self.loopid = None - self.created = True + #self.created = True def reg_atexit(self): import atexit @@ -929,7 +907,7 @@ class LoopDevice(object): self.cleanup() self.device = None except MountError, e: - msger.error("%s" % e) + raise CreatorError("%s" % e) def cleanup(self): @@ -938,12 +916,6 @@ class LoopDevice(object): if self._kpseek(self.device): - if self.created: - for i in range(3, os.sysconf("SC_OPEN_MAX")): - try: - os.close(i) - except: - pass runner.quiet([self.kpartxcmd, "-d", self.device]) if self._loseek(self.device): runner.quiet([self.losetupcmd, "-d", self.device]) @@ -973,7 +945,7 @@ def get_loop_device(losetupcmd, lofile): # provide an avaible loop device rc, out = runner.runtool([losetupcmd, "--find"]) - if rc == 0: + if rc == 0 and out: loopdev = out.split()[0] devinst.register(loopdev) if not loopdev or not os.path.exists(loopdev): diff --git a/mic/utils/gpt_parser.py b/mic/utils/gpt_parser.py index 5d43b70..895600d 100644 --- a/mic/utils/gpt_parser.py +++ b/mic/utils/gpt_parser.py @@ -296,7 +296,7 @@ class GptParser: entry['first_lba'], entry['last_lba'], entry['flags'], - entry['name'].encode('UTF-16')) + entry['name'].encode('UTF-16LE')) # Write the updated entry to the disk entry_offs = header['ptable_offs'] + \ diff --git a/mic/utils/grabber.py b/mic/utils/grabber.py index 45e30b4..c42dc02 100644 --- a/mic/utils/grabber.py +++ b/mic/utils/grabber.py @@ -1,8 +1,22 @@ #!/usr/bin/python +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import os import sys -import rpm import fcntl import struct import termios @@ -10,14 +24,11 @@ import termios from mic import msger from mic.utils import runner from mic.utils.errors import CreatorError +from mic.utils.safeurl import SafeURL from urlgrabber import grabber from urlgrabber import __version__ as grabber_version -if rpm.labelCompare(grabber_version.split('.'), '3.9.0'.split('.')) == -1: - msger.warning("Version of python-urlgrabber is %s, lower than '3.9.0', " - "you may encounter some network issues" % grabber_version) - def myurlgrab(url, filename, proxies, progress_obj = None): g = grabber.URLGrabber() if progress_obj is None: @@ -35,6 +46,8 @@ def myurlgrab(url, filename, proxies, progress_obj = None): else: try: + # cast url to str here, sometimes it can be unicode, + # but pycurl only accept str filename = g.urlgrab(url=str(url), filename=filename, ssl_verify_host=False, @@ -44,9 +57,14 @@ def myurlgrab(url, filename, proxies, progress_obj = None): quote=0, progress_obj=progress_obj) except grabber.URLGrabError, err: + tmp = SafeURL(url) msg = str(err) + if msg.find(url) < 0: - msg += ' on %s' % url + msg += ' on %s' % tmp + else: + msg = msg.replace(url, tmp) + raise CreatorError(msg) return filename @@ -79,11 +97,10 @@ class TextProgress(object): def start(self, filename, url, *args, **kwargs): self.url = url self.termwidth = terminal_width() - msger.info("\r%-*s" % (self.termwidth, " ")) if self.total is None: - msger.info("\rRetrieving %s ..." % truncate_url(self.url, self.termwidth - 15)) + msger.info("Retrieving %s ..." % truncate_url(self.url, self.termwidth - 15)) else: - msger.info("\rRetrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)) + msger.info("Retrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)) def update(self, *args): pass diff --git a/mic/utils/lock.py b/mic/utils/lock.py new file mode 100644 index 0000000..ca8d5a0 --- /dev/null +++ b/mic/utils/lock.py @@ -0,0 +1,49 @@ +# Copyright (c) 2011 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or any later version. + +import os +import errno + +class LockfileError(Exception): + """ Lockfile Exception""" + pass + +class SimpleLockfile(object): + """ Simple implementation of lockfile """ + def __init__(self, fpath): + self.fpath = fpath + self.lockf = None + + def acquire(self): + """ acquire the lock """ + try: + self.lockf = os.open(self.fpath, + os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except OSError as err: + if err.errno == errno.EEXIST: + raise LockfileError("File %s is locked already" % self.fpath) + raise + finally: + if self.lockf: + os.close(self.lockf) + + def release(self): + """ release the lock """ + try: + os.remove(self.fpath) + except OSError as err: + if err.errno == errno.ENOENT: + pass + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, *args): + self.release() + + def __del__(self): + self.release() diff --git a/mic/utils/misc.py b/mic/utils/misc.py index e7dbf2f..1e337d9 100644..100755 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -15,6 +15,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. +from __future__ import with_statement import os import sys import time @@ -46,6 +47,7 @@ from mic.utils.grabber import myurlgrab from mic.utils.proxy import get_proxy_for from mic.utils import runner from mic.utils import rpmmisc +from mic.utils.safeurl import SafeURL RPM_RE = re.compile("(.*)\.(.*) (.*)-(.*)") @@ -112,15 +114,21 @@ def get_distro(): return (dist, ver, id) -def get_distro_str(): +def get_hostname(): + """Get hostname + """ + return platform.node() + +def get_hostname_distro_str(): """Get composited string for current linux distribution """ (dist, ver, id) = get_distro() + hostname = get_hostname() if not dist: - return 'Unknown Linux Distro' + return "%s(Unknown Linux Distribution)" % hostname else: - distro_str = ' '.join(map(str.strip, (dist, ver, id))) + distro_str = ' '.join(map(str.strip, (hostname, dist, ver, id))) return distro_str.strip() _LOOP_RULE_PTH = None @@ -173,88 +181,12 @@ def extract_rpm(rpmfile, targetdir): p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE) p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p1.stdout.close() (sout, serr) = p2.communicate() msger.verbose(sout or serr) os.chdir(olddir) -def compressing(fpath, method): - comp_map = { - "gz": "gzip", - "bz2": "bzip2" - } - if method not in comp_map: - raise CreatorError("Unsupport compress format: %s, valid values: %s" - % (method, ','.join(comp_map.keys()))) - cmd = find_binary_path(comp_map[method]) - rc = runner.show([cmd, "-f", fpath]) - if rc: - raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath)) - -def taring(dstfile, target): - import tarfile - basen, ext = os.path.splitext(dstfile) - comp = {".tar": None, - ".gz": "gz", # for .tar.gz - ".bz2": "bz2", # for .tar.bz2 - ".tgz": "gz", - ".tbz": "bz2"}[ext] - - # specify tarball file path - if not comp: - tarpath = dstfile - elif basen.endswith(".tar"): - tarpath = basen - else: - tarpath = basen + ".tar" - wf = tarfile.open(tarpath, 'w') - - if os.path.isdir(target): - for item in os.listdir(target): - wf.add(os.path.join(target, item), item) - else: - wf.add(target, os.path.basename(target)) - wf.close() - - if comp: - compressing(tarpath, comp) - # when dstfile ext is ".tgz" and ".tbz", should rename - if not basen.endswith(".tar"): - shutil.move("%s.%s" % (tarpath, comp), dstfile) - -def ziping(dstfile, target): - import zipfile - wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED) - if os.path.isdir(target): - for item in os.listdir(target): - fpath = os.path.join(target, item) - if not os.path.isfile(fpath): - continue - wf.write(fpath, item, zipfile.ZIP_DEFLATED) - else: - wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED) - wf.close() - -pack_formats = { - ".tar": taring, - ".tar.gz": taring, - ".tar.bz2": taring, - ".tgz": taring, - ".tbz": taring, - ".zip": ziping, -} - -def packing(dstfile, target): - (base, ext) = os.path.splitext(dstfile) - if ext in (".gz", ".bz2") and base.endswith(".tar"): - ext = ".tar" + ext - if ext not in pack_formats: - raise CreatorError("Unsupport pack format: %s, valid values: %s" - % (ext, ','.join(pack_formats.keys()))) - func = pack_formats[ext] - # func should be callable - func(dstfile, target) - def human_size(size): """Return human readable string for Bytes size """ @@ -329,6 +261,11 @@ def calc_hashes(file_path, hash_names, start = 0, end = None): def get_md5sum(fpath): return calc_hashes(fpath, ('md5', ))[0] +def get_sha1sum(fpath): + return calc_hashes(fpath, ('sha1', ))[0] + +def get_sha256sum(fpath): + return calc_hashes(fpath, ('sha256', ))[0] def normalize_ksfile(ksconf, release, arch): ''' @@ -549,13 +486,17 @@ def get_repostrs_from_ks(ks): if 'name' not in repo: repo['name'] = _get_temp_reponame(repodata.baseurl) + if hasattr(repodata, 'baseurl') and getattr(repodata, 'baseurl'): + repo['baseurl'] = SafeURL(getattr(repodata, 'baseurl'), + getattr(repodata, 'user', None), + getattr(repodata, 'passwd', None)) kickstart_repos.append(repo) return kickstart_repos def _get_uncompressed_data_from_url(url, filename, proxies): - filename = myurlgrab(url, filename, proxies) + filename = myurlgrab(url.full, filename, proxies) suffix = None if filename.endswith(".gz"): suffix = ".gz" @@ -569,7 +510,7 @@ def _get_uncompressed_data_from_url(url, filename, proxies): def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, sumtype=None, checksum=None): - url = os.path.join(baseurl, filename) + url = baseurl.join(filename) filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename))) if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"): filename = os.path.splitext(filename_tmp)[0] @@ -591,23 +532,22 @@ def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, def get_metadata_from_repos(repos, cachedir): my_repo_metadata = [] for repo in repos: - reponame = repo['name'] - baseurl = repo['baseurl'] - + reponame = repo.name + baseurl = repo.baseurl - if 'proxy' in repo: - proxy = repo['proxy'] + if hasattr(repo, 'proxy'): + proxy = repo.proxy else: proxy = get_proxy_for(baseurl) proxies = None if proxy: - proxies = {str(baseurl.split(":")[0]):str(proxy)} + proxies = {str(baseurl.split(":")[0]): str(proxy)} makedirs(os.path.join(cachedir, reponame)) - url = os.path.join(baseurl, "repodata/repomd.xml") + url = baseurl.join("repodata/repomd.xml") filename = os.path.join(cachedir, reponame, 'repomd.xml') - repomd = myurlgrab(url, filename, proxies) + repomd = myurlgrab(url.full, filename, proxies) try: root = xmlparse(repomd) except SyntaxError: @@ -713,35 +653,8 @@ def get_rpmver_in_repo(repometadata): return None def get_arch(repometadata): - def uniqarch(archlist=[]): - uniq_arch = [] - for i in range(len(archlist)): - if archlist[i] not in rpmmisc.archPolicies.keys(): - continue - need_append = True - j = 0 - while j < len(uniq_arch): - if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'): - need_append = False - break - if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'): - if need_append: - uniq_arch[j] = archlist[i] - need_append = False - else: - uniq_arch.remove(uniq_arch[j]) - continue - j += 1 - if need_append: - uniq_arch.append(archlist[i]) - - return uniq_arch - - - ret_uniq_arch = [] - ret_arch_list = [] + archlist = [] for repo in repometadata: - archlist = [] if repo["primary"].endswith(".xml"): root = xmlparse(repo["primary"]) ns = root.getroot().tag @@ -759,13 +672,28 @@ def get_arch(repometadata): con.close() - uniq_arch = uniqarch(archlist) - if not ret_uniq_arch and len(uniq_arch) == 1: - ret_uniq_arch = uniq_arch - ret_arch_list += uniq_arch + uniq_arch = [] + for i in range(len(archlist)): + if archlist[i] not in rpmmisc.archPolicies.keys(): + continue + need_append = True + j = 0 + while j < len(uniq_arch): + if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'): + need_append = False + break + if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'): + if need_append: + uniq_arch[j] = archlist[i] + need_append = False + else: + uniq_arch.remove(uniq_arch[j]) + continue + j += 1 + if need_append: + uniq_arch.append(archlist[i]) - ret_arch_list = uniqarch(ret_arch_list) - return ret_uniq_arch, ret_arch_list + return uniq_arch, archlist def get_package(pkg, repometadata, arch = None): ver = "" @@ -820,7 +748,7 @@ def get_package(pkg, repometadata, arch = None): con.close() if target_repo: makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"])) - url = os.path.join(target_repo["baseurl"], pkgpath) + url = target_repo["baseurl"].join(pkgpath) filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath))) if os.path.exists(filename): ret = rpmmisc.checkRpmIntegrity('rpm', filename) @@ -831,7 +759,7 @@ def get_package(pkg, repometadata, arch = None): (os.path.basename(filename), filename)) os.unlink(filename) - pkg = myurlgrab(str(url), filename, target_repo["proxies"]) + pkg = myurlgrab(url.full, filename, target_repo["proxies"]) return pkg else: return None @@ -968,28 +896,31 @@ def setup_qemu_emulator(rootdir, arch): # qemu_emulator is a special case, we can't use find_binary_path # qemu emulator should be a statically-linked executable file - qemu_emulator = "/usr/bin/qemu-arm" - if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator): - qemu_emulator = "/usr/bin/qemu-arm-static" - if not os.path.exists(qemu_emulator): - raise CreatorError("Please install a statically-linked qemu-arm") - - # qemu emulator version check - armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')] - if arch in armv7_list: # need qemu (>=0.13.0) - qemuout = runner.outs([qemu_emulator, "-h"]) - m = re.search("version\s*([.\d]+)", qemuout) - if m: - qemu_version = m.group(1) - if qemu_version < "0.13": - raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch)) + if arch == "aarch64": + node = "/proc/sys/fs/binfmt_misc/aarch64" + if os.path.exists("/usr/bin/qemu-arm64") and is_statically_linked("/usr/bin/qemu-arm64"): + arm_binary = "qemu-arm64" + elif os.path.exists("/usr/bin/qemu-aarch64") and is_statically_linked("/usr/bin/qemu-aarch64"): + arm_binary = "qemu-aarch64" + elif os.path.exists("/usr/bin/qemu-arm64-static"): + arm_binary = "qemu-arm64-static" + elif os.path.exists("/usr/bin/qemu-aarch64-static"): + arm_binary = "qemu-aarch64-static" else: - msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator) + raise CreatorError("Please install a statically-linked %s" % arm_binary) + else: + node = "/proc/sys/fs/binfmt_misc/arm" + arm_binary = "qemu-arm" + if not os.path.exists("/usr/bin/qemu-arm") or not is_statically_linked("/usr/bin/qemu-arm"): + arm_binary = "qemu-arm-static" + if not os.path.exists("/usr/bin/%s" % arm_binary): + raise CreatorError("Please install a statically-linked %s" % arm_binary) + + qemu_emulator = "/usr/bin/%s" % arm_binary if not os.path.exists(rootdir + "/usr/bin"): makedirs(rootdir + "/usr/bin") - shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static") - qemu_emulator = "/usr/bin/qemu-arm-static" + shutil.copy(qemu_emulator, rootdir + qemu_emulator) # disable selinux, selinux will block qemu emulator to run if os.path.exists("/usr/sbin/setenforce"): @@ -997,19 +928,19 @@ def setup_qemu_emulator(rootdir, arch): runner.show(["/usr/sbin/setenforce", "0"]) # unregister it if it has been registered and is a dynamically-linked executable - node = "/proc/sys/fs/binfmt_misc/arm" if os.path.exists(node): qemu_unregister_string = "-1\n" - fd = open("/proc/sys/fs/binfmt_misc/arm", "w") - fd.write(qemu_unregister_string) - fd.close() + with open(node, "w") as fd: + fd.write(qemu_unregister_string) # register qemu emulator for interpreting other arch executable file if not os.path.exists(node): - qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator - fd = open("/proc/sys/fs/binfmt_misc/register", "w") - fd.write(qemu_arm_string) - fd.close() + if arch == "aarch64": + qemu_arm_string = ":aarch64:M::\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\xb7:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff:%s:\n" % qemu_emulator + else: + qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator + with open("/proc/sys/fs/binfmt_misc/register", "w") as fd: + fd.write(qemu_arm_string) return qemu_emulator diff --git a/mic/utils/partitionedfs.py b/mic/utils/partitionedfs.py index f1102d4..cac360b 100644 --- a/mic/utils/partitionedfs.py +++ b/mic/utils/partitionedfs.py @@ -22,7 +22,7 @@ import os from mic import msger from mic.utils import runner -from mic.utils.errors import MountError +from mic.utils.errors import MountError, CreatorError from mic.utils.fs_related import * from mic.utils.gpt_parser import GptParser @@ -34,6 +34,13 @@ GPT_OVERHEAD = 34 # Size of a sector in bytes SECTOR_SIZE = 512 +def resolve_ref(ref): + real = os.readlink(ref) + if not real.startswith('/'): + return os.path.realpath(os.path.join(ref, real)) + else: + return real + class PartitionedMount(Mount): def __init__(self, mountdir, skipformat = False): Mount.__init__(self, mountdir) @@ -41,12 +48,13 @@ class PartitionedMount(Mount): self.partitions = [] self.subvolumes = [] self.mapped = False - self.mountOrder = [] - self.unmountOrder = [] + self.mount_order = [] + self.unmount_order = [] self.parted = find_binary_path("parted") self.kpartx = find_binary_path("kpartx") self.mkswap = find_binary_path("mkswap") - self.btrfscmd=None + self.dmsetup = find_binary_path("dmsetup") + self.btrfscmd = None self.mountcmd = find_binary_path("mount") self.umountcmd = find_binary_path("umount") self.skipformat = skipformat @@ -96,15 +104,17 @@ class PartitionedMount(Mount): def add_partition(self, size, disk_name, mountpoint, fstype = None, label=None, fsopts = None, boot = False, align = None, part_type = None): - """ Add the next partition. Prtitions have to be added in the + """ Add the next partition. Partitions have to be added in the first-to-last order. """ + ks_pnum = len(self.partitions) + # Converting MB to sectors for parted size = size * 1024 * 1024 / self.sector_size # We need to handle subvolumes for btrfs if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1: - self.btrfscmd=find_binary_path("btrfs") + self.btrfscmd = find_binary_path("btrfs") subvol = None opts = fsopts.split(",") for opt in opts: @@ -119,6 +129,8 @@ class PartitionedMount(Mount): 'fsopts': fsopts, # Filesystem mount options 'disk_name': disk_name, # physical disk name holding partition 'device': None, # kpartx device node for partition + 'mapper_device': None, # mapper device node + 'mpath_device': None, # multipath device of device mapper 'mount': None, # Mount object 'subvol': subvol, # Subvolume name 'boot': boot, # Bootable flag @@ -136,18 +148,22 @@ class PartitionedMount(Mount): break fsopts = ",".join(opts) - part = { 'size': size, # In sectors + part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file + 'size': size, # In sectors 'mountpoint': mountpoint, # Mount relative to chroot 'fstype': fstype, # Filesystem type 'fsopts': fsopts, # Filesystem mount options 'label': label, # Partition label 'disk_name': disk_name, # physical disk name holding partition 'device': None, # kpartx device node for partition + 'mapper_device': None, # mapper device node + 'mpath_device': None, # multipath device of device mapper 'mount': None, # Mount object 'num': None, # Partition number 'boot': boot, # Bootable flag 'align': align, # Partition alignment 'part_type' : part_type, # Partition type + 'uuid': None, # Partition UUID (no-GPT use) 'partuuid': None } # Partition UUID (GPT-only) self.__add_partition(part) @@ -181,7 +197,7 @@ class PartitionedMount(Mount): # in which case it would map to the 1-byte "partition type" # filed at offset 3 of the partition entry. raise MountError("setting custom partition type is only " \ - "imlemented for GPT partitions") + "implemented for GPT partitions") # Get the disk where the partition is located d = self.disks[p['disk_name']] @@ -204,7 +220,7 @@ class PartitionedMount(Mount): # If not first partition and we do have alignment set we need # to align the partition. # FIXME: This leaves a empty spaces to the disk. To fill the - # gaps we could enlargea the previous partition? + # gaps we could enlarge the previous partition? # Calc how much the alignment is off. align_sectors = d['offset'] % (p['align'] * 1024 / self.sector_size) @@ -245,8 +261,8 @@ class PartitionedMount(Mount): p['start'], p['start'] + p['size'] - 1, p['size'], p['size'] * self.sector_size)) - # Once all the partitions have been layed out, we can calculate the - # minumim disk sizes. + # Once all the partitions have been laid out, we can calculate the + # minimum disk sizes. for disk_name, d in self.disks.items(): d['min_size'] = d['offset'] if d['ptable_format'] == 'gpt': @@ -263,20 +279,17 @@ class PartitionedMount(Mount): rc, out = runner.runtool(args, catch = 3) out = out.strip() - if out: - msger.debug('"parted" output: %s' % out) - - if rc != 0: - # We don't throw exception when return code is not 0, because - # parted always fails to reload part table with loop devices. This - # prevents us from distinguishing real errors based on return - # code. - msger.debug("WARNING: parted returned '%s' instead of 0" % rc) + msger.debug("'parted': exitcode: %d, output: %s" % (rc, out)) + # We don't throw exception when return code is not 0, because + # parted always fails to reload part table with loop devices. This + # prevents us from distinguishing real errors based on return + # code. + return rc, out def __create_partition(self, device, parttype, fstype, start, size): """ Create a partition on an image described by the 'device' object. """ - # Start is included to the size so we need to substract one from the end. + # Start is included to the size so we need to subtract one from the end. end = start + size - 1 msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" % (parttype, start, end, size)) @@ -344,8 +357,16 @@ class PartitionedMount(Mount): flag_name = "boot" msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \ (flag_name, p['num'], d['disk'].device)) - self.__run_parted(["-s", d['disk'].device, "set", - "%d" % p['num'], flag_name, "on"]) + cmd = ["-s", d['disk'].device, + "set", "%d" % p['num'], + flag_name, "on"] + exitcode, output = self.__run_parted(cmd) + if exitcode != 0: + msger.warning( + "partition '%s' is marked with --active, " + "but flag '%s' can't be set: " + "exitcode: %s, output: %s" + % (p['mountpoint'], flag_name, exitcode, output)) # If the partition table format is "gpt", find out PARTUUIDs for all # the partitions. And if users specified custom parition type UUIDs, @@ -355,12 +376,12 @@ class PartitionedMount(Mount): continue pnum = 0 - gpt_parser = GptParser(d['disk'].device, SECTOR_SIZE) + gpt_parser = GptParser(disk['disk'].device, SECTOR_SIZE) # Iterate over all GPT partitions on this disk for entry in gpt_parser.get_partitions(): pnum += 1 # Find the matching partition in the 'self.partitions' list - for n in d['partitions']: + for n in disk['partitions']: p = self.partitions[n] if p['num'] == pnum: # Found, fetch PARTUUID (partition's unique ID) @@ -388,8 +409,8 @@ class PartitionedMount(Mount): continue msger.debug("Running kpartx on %s" % d['disk'].device ) - rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) - kpartxOutput = kpartxOutput.splitlines() + rc, kpartx_output = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) + kpartx_output = kpartx_output.splitlines() if rc != 0: raise MountError("Failed to query partition mapping for '%s'" % @@ -397,12 +418,12 @@ class PartitionedMount(Mount): # Strip trailing blank and mask verbose output i = 0 - while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop": + while i < len(kpartx_output) and kpartx_output[i][0:4] != "loop": i = i + 1 - kpartxOutput = kpartxOutput[i:] + kpartx_output = kpartx_output[i:] # Make sure kpartx reported the right count of partitions - if len(kpartxOutput) != d['numpart']: + if len(kpartx_output) != d['numpart']: # If this disk has more than 3 partitions, then in case of MBR # paritions there is an extended parition. Different versions # of kpartx behave differently WRT the extended partition - @@ -411,16 +432,16 @@ class PartitionedMount(Mount): # table type is "msdos" and the amount of partitions is more # than 3, we just assume kpartx mapped the extended parition # and we remove it. - if len(kpartxOutput) == d['numpart'] + 1 \ - and d['ptable_format'] == 'msdos' and len(kpartxOutput) > 3: - kpartxOutput.pop(3) + if len(kpartx_output) == d['numpart'] + 1 \ + and d['ptable_format'] == 'msdos' and len(kpartx_output) > 3: + kpartx_output.pop(3) else: raise MountError("Unexpected number of partitions from " \ "kpartx: %d != %d" % \ - (len(kpartxOutput), d['numpart'])) + (len(kpartx_output), d['numpart'])) - for i in range(len(kpartxOutput)): - line = kpartxOutput[i] + for i in range(len(kpartx_output)): + line = kpartx_output[i] newdev = line.split()[0] mapperdev = "/dev/mapper/" + newdev loopdev = d['disk'].device + newdev[-1] @@ -428,6 +449,7 @@ class PartitionedMount(Mount): msger.debug("Dev %s: %s -> %s" % (newdev, loopdev, mapperdev)) pnum = d['partitions'][i] self.partitions[pnum]['device'] = loopdev + self.partitions[pnum]['mapper_device'] = mapperdev # grub's install wants partitions to be named # to match their parent device + partition num @@ -438,20 +460,35 @@ class PartitionedMount(Mount): os.symlink(mapperdev, loopdev) msger.debug("Adding partx mapping for %s" % d['disk'].device) - rc = runner.show([self.kpartx, "-v", "-a", d['disk'].device]) + rc = runner.show([self.kpartx, "-v", "-sa", d['disk'].device]) if rc != 0: # Make sure that the device maps are also removed on error case. # The d['mapped'] isn't set to True if the kpartx fails so # failed mapping will not be cleaned on cleanup either. - runner.quiet([self.kpartx, "-d", d['disk'].device]) + runner.quiet([self.kpartx, "-sd", d['disk'].device]) raise MountError("Failed to map partitions for '%s'" % d['disk'].device) - # FIXME: there is a bit delay for multipath device setup, - # wait 10ms for the setup + for p in self.partitions: + if p['mapper_device'] and os.path.islink(p['mapper_device']): + p['mpath_device'] = resolve_ref(p['mapper_device']) + else: + p['mpath_device'] = '' + + # FIXME: need a better way to fix the latency import time - time.sleep(10) + time.sleep(1) + + if not os.path.exists(mapperdev): + # load mapper device if not updated + runner.quiet([self.dmsetup, "mknodes"]) + # still not updated, roll back + if not os.path.exists(mapperdev): + runner.quiet([self.kpartx, "-sd", d['disk'].device]) + raise MountError("Failed to load mapper devices for '%s'" % + d['disk'].device) + d['mapped'] = True def __unmap_partitions(self): @@ -467,7 +504,7 @@ class PartitionedMount(Mount): self.partitions[pnum]['device'] = None msger.debug("Unmapping %s" % d['disk'].device) - rc = runner.quiet([self.kpartx, "-d", d['disk'].device]) + rc = runner.quiet([self.kpartx, "-sd", d['disk'].device]) if rc != 0: raise MountError("Failed to unmap partitions for '%s'" % d['disk'].device) @@ -478,12 +515,12 @@ class PartitionedMount(Mount): msger.debug("Calculating mount order") for p in self.partitions: if p['mountpoint']: - self.mountOrder.append(p['mountpoint']) - self.unmountOrder.append(p['mountpoint']) + self.mount_order.append(p['mountpoint']) + self.unmount_order.append(p['mountpoint']) - self.mountOrder.sort() - self.unmountOrder.sort() - self.unmountOrder.reverse() + self.mount_order.sort() + self.unmount_order.sort() + self.unmount_order.reverse() def cleanup(self): Mount.cleanup(self) @@ -498,7 +535,7 @@ class PartitionedMount(Mount): def unmount(self): self.__unmount_subvolumes() - for mp in self.unmountOrder: + for mp in self.unmount_order: if mp == 'swap': continue p = None @@ -510,7 +547,8 @@ class PartitionedMount(Mount): if p['mount'] != None: try: # Create subvolume snapshot here - if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created: + if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and \ + not self.snapshot_created: self.__create_subvolume_snapshots(p, p["mount"]) p['mount'].cleanup() except: @@ -520,14 +558,15 @@ class PartitionedMount(Mount): # Only for btrfs def __get_subvolume_id(self, rootpath, subvol): if not self.btrfscmd: - self.btrfscmd=find_binary_path("btrfs") + self.btrfscmd = find_binary_path("btrfs") argv = [ self.btrfscmd, "subvolume", "list", rootpath ] rc, out = runner.runtool(argv) msger.debug(out) if rc != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc)) + raise MountError("Failed to get subvolume id from %s'," + "return code: %d." % (rootpath, rc)) subvolid = -1 for line in out.splitlines(): @@ -566,7 +605,8 @@ class PartitionedMount(Mount): opts.remove(opt) break fsopts = ",".join(opts) - subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts) + subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], + subvol['mountpoint'], fsopts) if subvolume_metadata: fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w") @@ -619,9 +659,11 @@ class PartitionedMount(Mount): subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) # Set default subvolume if subvolid != -1: - rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]) + rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % + subvolid, pdisk.mountdir]) if rc != 0: - raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc)) + raise MountError("Failed to set default subvolume id: %d', return code: %d." + % (subvolid, rc)) self.__create_subvolume_metadata(p, pdisk) @@ -716,7 +758,8 @@ class PartitionedMount(Mount): snapshotpath = subvolpath + "_%s-1" % snapshotts rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]) if rc != 0: - raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc)) + raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code:" + "%d." % (snapshotpath, subvolpath, rc)) self.snapshot_created = True @@ -729,7 +772,7 @@ class PartitionedMount(Mount): self.__map_partitions() self.__calculate_mountorder() - for mp in self.mountOrder: + for mp in self.mount_order: p = None for p1 in self.partitions: if p1['mountpoint'] == mp: @@ -755,18 +798,18 @@ class PartitionedMount(Mount): if p['mountpoint'] == "/": rmmountdir = True if p['fstype'] == "vfat" or p['fstype'] == "msdos": - myDiskMount = VfatDiskMount + my_disk_mount = VfatDiskMount elif p['fstype'] in ("ext2", "ext3", "ext4"): - myDiskMount = ExtDiskMount + my_disk_mount = ExtDiskMount elif p['fstype'] == "btrfs": - myDiskMount = BtrfsDiskMount + my_disk_mount = BtrfsDiskMount else: raise MountError("Fail to support file system " + p['fstype']) if p['fstype'] == "btrfs" and not p['fsopts']: p['fsopts'] = "subvolid=0" - pdisk = myDiskMount(RawDisk(p['size'] * self.sector_size, p['device']), + pdisk = my_disk_mount(RawDisk(p['size'] * self.sector_size, p['device']), self.mountdir + p['mountpoint'], p['fstype'], 4096, diff --git a/mic/utils/proxy.py b/mic/utils/proxy.py index 91451a2..acb05e4 100644 --- a/mic/utils/proxy.py +++ b/mic/utils/proxy.py @@ -16,6 +16,7 @@ # Temple Place - Suite 330, Boston, MA 02111-1307, USA. import os +import re import urlparse _my_proxies = {} @@ -40,9 +41,9 @@ def unset_proxy_environ(): if env in os.environ: del os.environ[env] - ENV=env.upper() - if ENV in os.environ: - del os.environ[ENV] + env_upper = env.upper() + if env_upper in os.environ: + del os.environ[env_upper] def _set_proxies(proxy = None, no_proxy = None): """Return a dictionary of scheme -> proxy server URL mappings. @@ -73,16 +74,16 @@ def _set_proxies(proxy = None, no_proxy = None): _my_noproxy = value def _ip_to_int(ip): - ipint=0 - shift=24 + ipint = 0 + shift = 24 for dec in ip.split("."): ipint |= int(dec) << shift shift -= 8 return ipint def _int_to_ip(val): - ipaddr="" - shift=0 + ipaddr = "" + shift = 0 for i in range(4): dec = val >> shift dec &= 0xff @@ -107,11 +108,11 @@ def _set_noproxy_list(): if item[0] != '.' and item.find("/") == -1: # Need to match it - _my_noproxy_list.append({"match":0,"needle":item}) + _my_noproxy_list.append({"match":0, "needle":item}) elif item[0] == '.': # Need to match at tail - _my_noproxy_list.append({"match":1,"needle":item}) + _my_noproxy_list.append({"match":1, "needle":item}) elif item.find("/") > 3: # IP/MASK, need to match at head @@ -125,17 +126,18 @@ def _set_noproxy_list(): netmask = ~((1<<(32-netmask)) - 1) ip &= netmask else: - shift=24 - netmask=0 + shift = 24 + netmask = 0 for dec in mask.split("."): netmask |= int(dec) << shift shift -= 8 ip &= netmask - _my_noproxy_list.append({"match":2,"needle":ip,"netmask":netmask}) + _my_noproxy_list.append({"match":2, "needle":ip, "netmask":netmask}) def _isnoproxy(url): - (scheme, host, path, parm, query, frag) = urlparse.urlparse(url) + host = urlparse.urlparse(url)[1] + # urlparse.urlparse(url) returns (scheme, host, path, parm, query, frag) if '@' in host: user_pass, host = host.split('@', 1) @@ -145,7 +147,7 @@ def _isnoproxy(url): hostisip = _isip(host) for item in _my_noproxy_list: - if hostisip and item["match"] <= 1: + if hostisip and item["match"] == 1: continue if item["match"] == 2 and hostisip: @@ -157,7 +159,7 @@ def _isnoproxy(url): return True if item["match"] == 1: - if host.rfind(item["needle"]) > 0: + if re.match(r".*%s$" % item["needle"], host): return True return False diff --git a/mic/utils/rpmmisc.py b/mic/utils/rpmmisc.py index af15763..351f2b8 100644 --- a/mic/utils/rpmmisc.py +++ b/mic/utils/rpmmisc.py @@ -35,6 +35,7 @@ class RPMInstallCallback: self.callbackfilehandles = {} self.total_actions = 0 self.total_installed = 0 + self.total_installing = 0 self.installed_pkg_names = [] self.total_removed = 0 self.mark = "+" @@ -67,17 +68,17 @@ class RPMInstallCallback: l = len(str(self.total_actions)) size = "%s.%s" % (l, l) fmt_done = "[%" + size + "s/%" + size + "s]" - done = fmt_done % (self.total_installed + self.total_removed, + done = fmt_done % (self.total_installing, self.total_actions) marks = self.marks - (2 * l) width = "%s.%s" % (marks, marks) fmt_bar = "%-" + width + "s" if progress: bar = fmt_bar % (self.mark * int(marks * (percent / 100.0)), ) - fmt = "\r %-10.10s: %-20.20s " + bar + " " + done + fmt = "%-10.10s: %-20.20s " + bar + " " + done else: bar = fmt_bar % (self.mark * marks, ) - fmt = " %-10.10s: %-20.20s " + bar + " " + done + fmt = "%-10.10s: %-20.20s " + bar + " " + done return fmt def _logPkgString(self, hdr): @@ -138,6 +139,12 @@ class RPMInstallCallback: #pkgtup = self._dopkgtup(hdr) self.logString.append(self._logPkgString(hdr)) + elif what == rpm.RPMCALLBACK_INST_START: + self.total_installing += 1 + + elif what == rpm.RPMCALLBACK_UNINST_STOP: + pass + elif what == rpm.RPMCALLBACK_INST_PROGRESS: if h is not None: percent = (self.total_installed*100L)/self.total_actions @@ -284,6 +291,7 @@ archPolicies = { "i686": "i686:i586:i486:i386", "i586": "i586:i486:i386", "ia64": "ia64:i686:i586:i486:i386", + "aarch64": "aarch64", "armv7tnhl": "armv7tnhl:armv7thl:armv7nhl:armv7hl", "armv7thl": "armv7thl:armv7hl", "armv7nhl": "armv7nhl:armv7hl", @@ -430,171 +438,3 @@ def getSigInfo(hdr): infotuple = (sigtype, sigdate, sigid) return error, infotuple -def checkRepositoryEULA(name, repo): - """ This function is to check the EULA file if provided. - return True: no EULA or accepted - return False: user declined the EULA - """ - - import tempfile - import shutil - import urlparse - import urllib2 as u2 - import httplib - from mic.utils.errors import CreatorError - - def _check_and_download_url(u2opener, url, savepath): - try: - if u2opener: - f = u2opener.open(url) - else: - f = u2.urlopen(url) - except u2.HTTPError, httperror: - if httperror.code in (404, 503): - return None - else: - raise CreatorError(httperror) - except OSError, oserr: - if oserr.errno == 2: - return None - else: - raise CreatorError(oserr) - except IOError, oserr: - if hasattr(oserr, "reason") and oserr.reason.errno == 2: - return None - else: - raise CreatorError(oserr) - except u2.URLError, err: - raise CreatorError(err) - except httplib.HTTPException, e: - raise CreatorError(e) - - # save to file - licf = open(savepath, "w") - licf.write(f.read()) - licf.close() - f.close() - - return savepath - - def _pager_file(savepath): - - if os.path.splitext(savepath)[1].upper() in ('.HTM', '.HTML'): - pagers = ('w3m', 'links', 'lynx', 'less', 'more') - else: - pagers = ('less', 'more') - - file_showed = False - for pager in pagers: - cmd = "%s %s" % (pager, savepath) - try: - os.system(cmd) - except OSError: - continue - else: - file_showed = True - break - - if not file_showed: - f = open(savepath) - msger.raw(f.read()) - f.close() - msger.pause() - - # when proxy needed, make urllib2 follow it - proxy = repo.proxy - proxy_username = repo.proxy_username - proxy_password = repo.proxy_password - - if not proxy: - proxy = get_proxy_for(repo.baseurl[0]) - - handlers = [] - auth_handler = u2.HTTPBasicAuthHandler(u2.HTTPPasswordMgrWithDefaultRealm()) - u2opener = None - if proxy: - if proxy_username: - proxy_netloc = urlparse.urlsplit(proxy).netloc - if proxy_password: - proxy_url = 'http://%s:%s@%s' % (proxy_username, proxy_password, proxy_netloc) - else: - proxy_url = 'http://%s@%s' % (proxy_username, proxy_netloc) - else: - proxy_url = proxy - - proxy_support = u2.ProxyHandler({'http': proxy_url, - 'https': proxy_url, - 'ftp': proxy_url}) - handlers.append(proxy_support) - - # download all remote files to one temp dir - baseurl = None - repo_lic_dir = tempfile.mkdtemp(prefix = 'repolic') - - for url in repo.baseurl: - tmphandlers = handlers[:] - - (scheme, host, path, parm, query, frag) = urlparse.urlparse(url.rstrip('/') + '/') - if scheme not in ("http", "https", "ftp", "ftps", "file"): - raise CreatorError("Error: invalid url %s" % url) - - if '@' in host: - try: - user_pass, host = host.split('@', 1) - if ':' in user_pass: - user, password = user_pass.split(':', 1) - except ValueError, e: - raise CreatorError('Bad URL: %s' % url) - - msger.verbose("adding HTTP auth: %s, XXXXXXXX" %(user)) - auth_handler.add_password(None, host, user, password) - tmphandlers.append(auth_handler) - url = scheme + "://" + host + path + parm + query + frag - - if tmphandlers: - u2opener = u2.build_opener(*tmphandlers) - - # try to download - repo_eula_url = urlparse.urljoin(url, "LICENSE.txt") - repo_eula_path = _check_and_download_url( - u2opener, - repo_eula_url, - os.path.join(repo_lic_dir, repo.id + '_LICENSE.txt')) - if repo_eula_path: - # found - baseurl = url - break - - if not baseurl: - shutil.rmtree(repo_lic_dir) #cleanup - return True - - # show the license file - msger.info('For the software packages in this yum repo:') - msger.info(' %s: %s' % (name, baseurl)) - msger.info('There is an "End User License Agreement" file that need to be checked.') - msger.info('Please read the terms and conditions outlined in it and answer the followed qustions.') - msger.pause() - - _pager_file(repo_eula_path) - - # Asking for the "Accept/Decline" - if not msger.ask('Would you agree to the terms and conditions outlined in the above End User License Agreement?'): - msger.warning('Will not install pkgs from this repo.') - shutil.rmtree(repo_lic_dir) #cleanup - return False - - # try to find support_info.html for extra infomation - repo_info_url = urlparse.urljoin(baseurl, "support_info.html") - repo_info_path = _check_and_download_url( - u2opener, - repo_info_url, - os.path.join(repo_lic_dir, repo.id + '_support_info.html')) - if repo_info_path: - msger.info('There is one more file in the repo for additional support information, please read it') - msger.pause() - _pager_file(repo_info_path) - - #cleanup - shutil.rmtree(repo_lic_dir) - return True diff --git a/mic/utils/runner.py b/mic/utils/runner.py index fded3c9..6d68e26 100644 --- a/mic/utils/runner.py +++ b/mic/utils/runner.py @@ -19,6 +19,7 @@ import os import subprocess from mic import msger +from mic.utils import errors def runtool(cmdln_or_args, catch=1): """ wrapper for most of the subprocess calls @@ -70,7 +71,7 @@ def runtool(cmdln_or_args, catch=1): except OSError, e: if e.errno == 2: # [Errno 2] No such file or directory - msger.error('Cannot run command: %s, lost dependency?' % cmd) + raise errors.CreatorError('Cannot run command: %s, lost dependency?' % cmd) else: raise # relay finally: diff --git a/mic/utils/safeurl.py b/mic/utils/safeurl.py new file mode 100644 index 0000000..0a82f8e --- /dev/null +++ b/mic/utils/safeurl.py @@ -0,0 +1,92 @@ +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" +This module provides a class SafeURL which can contain url/user/password read +from config file, and hide plain user and password when it print to screen +""" +import os.path +import urllib +from urlparse import urlsplit, urlunsplit + + +def join_userpass(href, user, passwd): + """Return authenticated URL with user and passwd embeded""" + if not user and not passwd: + return href + + if passwd: + userpass = '%s:%s' % (urllib.quote(user, safe=''), + urllib.quote(passwd, safe='')) + else: + userpass = urllib.quote(user, safe='') + + parts = urlsplit(href) + netloc = '%s@%s' % (userpass, parts[1]) + comps = list(parts) + comps[1] = netloc + return urlunsplit(comps) + + +def split_userpass(href): + """Returns (href, user, passwd) of an authenticated URL""" + parts = urlsplit(href) + + netloc = parts[1] + if '@' not in netloc: + return href, None, None + + userpass, netloc = netloc.split('@', 1) + if ':' in userpass: + user, passwd = [ urllib.unquote(i) + for i in userpass.split(':', 1) ] + else: + user, passwd = userpass, None + + comps = list(parts) + comps[1] = netloc + return urlunsplit(comps), user, passwd + + +class SafeURL(str): + '''SafeURL can hide user info when it's printed to console. + Use property full to get url with user info + ''' + def __new__(cls, urlstring, user=None, passwd=None): + """Imuutable object""" + href, user1, passwd1 = split_userpass(urlstring) + user = user if user else user1 + passwd = passwd if passwd else passwd1 + + obj = super(SafeURL, cls).__new__(cls, href) + obj.user = user + obj.passwd = passwd + obj.full = join_userpass(href, user, passwd) + + parts = urlsplit(href) + obj.scheme = parts[0] + obj.netloc = parts[1] + obj.path = parts[2] + obj.host = parts.hostname + obj.port = parts.port + return obj + + def join(self, *path): + """Returns a new SafeURL with new path. Search part is removed since + after join path is changed, keep the same search part is useless. + """ + idx = self.full.find('?') + url = self.full if idx < 0 else self.full[:idx] + return SafeURL(os.path.join(url.rstrip('/'), *path)) diff --git a/packaging/Makefile b/packaging/Makefile deleted file mode 100644 index ae431d6..0000000 --- a/packaging/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -PKG_NAME := mic -SPECFILE = $(addsuffix .spec, $(PKG_NAME)) -PKG_VERSION := $(shell grep '^Version: ' $(SPECFILE)|awk '{print $$2}') - -TARBALL := $(PKG_NAME)_$(PKG_VERSION).tar.gz - -dsc: $(TARBALL) - $(eval MD5=$(shell md5sum $(TARBALL) | sed "s/ / $(shell stat -c '%s' $(TARBALL)) /")) - @sed -i 's/^Version:.*/Version: $(PKG_VERSION)/' $(PKG_NAME).dsc - @sed -i 's/ [a-f0-9]\+ [0-9]\+ $(PKG_NAME).*tar.*/ $(MD5)/' $(PKG_NAME).dsc - -$(TARBALL): - cd "$$(git rev-parse --show-toplevel)" \ - && git archive --prefix $(PKG_NAME)-$(PKG_VERSION)/ HEAD \ - | gzip > "$(CURDIR)/$(TARBALL)" - -clean: - rm -f $(PKG_NAME)*.tar.gz - -all: clean dsc - diff --git a/packaging/mic.changes b/packaging/mic.changes index 6bae117..150074e 100644 --- a/packaging/mic.changes +++ b/packaging/mic.changes @@ -1,5 +1,114 @@ -* Thu Jun 06 2013 Anas Nashif <anas.nashif@intel.com> accepted/tizen/20130603.185841@c80dd03 -- Fixed run time requirements +* Wed May 25 2016 Jianzhong Fang <jz.fang@samsung.com> - 0.27.1 + * new distribution support: Ubuntu 16.04, Fedora 23 + * add raw image format support + + * bug fix: + - Remove BmapCreate and Filemap source code from MIC (#DEVT-151) + +* Mon Mar 28 2016 Jianzhong Fang <jz.fang@samsung.com> - 0.27 + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, openSUSE 13.2 + * generate manifest file to describe image information + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum (#DEVT-224) + * drop mic-native support (#DEVT-248) + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support (#DEVT-243, #DEVT-263) + * use argparse module to parse the cmd line (#DEVT-52) + + * bug fix: + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure + - fix qemu arm and arm64 issues + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply (#DEVT-254) + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive + +* Tue Mar 11 2014 Gui Chen <gui.chen@intel.com> - 0.24 + * enhance to handle password with special characters + * change python-zypp require to python-zypp-tizen + * add --repo comand option for local build + * add --user and --password option for %repo directive of ks file + * clean up some codes relevant to EULA agreement + * add hostname showing in log + * bug fix: + - fix chroot failed by space in image not enough + - fix obsolete packages incorrect handling + - fix yum backend failed to cache packages + - fix bare ip in no_proxy not working + - fix repeated log showing with yum backend + - fix loop device latency timing + - fix zypp failed to download by changing 'cachedir' + - fix 'mkfs' not working caused by mic-bootstrap install failed + +* Fri Dec 12 2013 Gui Chen <gui.chen@intel.com> - 0.23 + * new distribution support: Ubuntu 13.10 and OpenSUSE 13.1 + * split requirements to subpackage 'mic-native' to reduce mic's dependencies + * support arm64 architecture image creation in native mode + * new option '--interactive'/'--non-interactive' to enable/disable interaction + * new option '--uuid' for 'part' in ks file to set filesystem uuid + * export more variables related to installer framework for loop format + * bug fix: + - fix bootstrap handling if bootstrap package failed + - fix 'mapper_device' key error + - fix detailed error messages missing in mounting + - fix version comparing issue of urlgrabber in Fedora + +* Thu Oct 24 2013 Gui Chen <gui.chen@intel.com> - 0.22 + * use __version__ variable instead of VERSION file + * refactor msger module to ulitize logging module + * refine error class module + * improve installation in virtualenv + * add bash completion support + * add zsh completion support + * export mapper device related to installer framework + * update BmapCreate to the latest version + * bug fix: + - fix customized plugin_dir not work in bootstrap + - fix packing process exit on Ubuntu + - fix loop device alloaction failed on openSUSE + - fix incorrect number showing during installing + - set owner of cacheidr/outdir to SUDO_USER + - correct project url in setup.py + - fix mic not work when mic.conf disappear + +* Mon Aug 26 2013 Gui Chen <gui.chen@intel.com> - 0.21 + * new distribution support: Fedora 19 + * refactor chroot module to correct the logic + * add an alias for installerfw - installerfw_plugins + * remove fuser dependency to avoid some unmount issue + * enable proxy setting with authentication + * don't get proxy info from /etc/sysconfig/proxy ever + * kill processes inside chroot after post script running + * bug fix: + - fix bootloader options omitted + - raise when incorrectly set partition flags 'legacy_boot' + - fix wrong file descriptor issue + - fix some requires + +* Mon Jul 01 2013 Gui Chen <gui.chen@intel.com> - 0.20 + * new distribution support: CentOS 6 + * drop image creation if checked packages not present in image + * introduce 'installerfw' command in kickstart to customize configuration + * improve output message of post scripts + * bug fix: + - fix rpm not support 'VCS' tag traceback * Thu May 16 2013 Gui Chen <gui.chen@intel.com> - 0.19 - new distribution support: Ubuntu 13.04 and openSUSE 12.3 diff --git a/packaging/mic.dsc b/packaging/mic.dsc deleted file mode 100644 index bf39ca0..0000000 --- a/packaging/mic.dsc +++ /dev/null @@ -1,11 +0,0 @@ -Format: 1.0 -Source: mic -Binary: mic -Architecture: all -Version: 0.19 -Maintainer: Jian-feng Ding <jian-feng.ding@intel.com> -Homepage: http://www.tizen.org -Standards-Version: 3.8.0 -Build-Depends: debhelper (>= 7.0.15), dpatch, cdbs, python-dev, python-support, python-docutils -Files: - 1f266944838a142e657aa0244e7f15e5 1684957 mic_0.15.tar.gz diff --git a/packaging/mic.spec b/packaging/mic.spec index 11733ac..3f70bd3 100644 --- a/packaging/mic.spec +++ b/packaging/mic.spec @@ -1,51 +1,45 @@ +%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%{!?python_version: %define python_version %(%{__python} -c "import sys; sys.stdout.write(sys.version[:3])")} + +%define rc_version 0 + +%if 0%{?rc_version} +%define release_prefix 0.rc%{rc_version}. +%endif + Name: mic Summary: Image Creator for Linux Distributions -Version: 0.19 -Release: 0 -Group: System/Utilities -License: GPL-2.0 +Version: 0.27.1 +Release: %{?release_prefix}%{?opensuse_bs:<CI_CNT>.<B_CNT>}%{!?opensuse_bs:0} +Group: Development/Tools +License: GPLv2 BuildArch: noarch URL: http://www.tizen.org -Source0: %{name}-%{version}.tar.gz +Source0: %{name}_%{version}.tar.gz %if 0%{?tizen_version:1} Source1001: mic.manifest %endif -Requires: python-rpm -Requires: util-linux -Requires: coreutils -Requires: python >= 2.5 -Requires: e2fsprogs -Requires: dosfstools >= 2.11-8 -Requires: syslinux >= 3.82 -Requires: kpartx -Requires: parted -Requires: device-mapper -Requires: /usr/bin/genisoimage -Requires: cpio -#Requires: isomd5sum -Requires: gzip -Requires: bzip2 -Requires: python-urlgrabber -Requires: yum >= 3.2.24 -%if ! 0%{?centos_version} -%if 0%{?suse_version} -Requires: btrfsprogs -%else -Requires: btrfs-progs + +Requires: python >= 2.6 +Requires: python-urlgrabber >= 3.9.0 +%if 0%{?suse_version} || 0%{?tizen_version:1} +Requires: python-xml %endif + +%if "%{?python_version}" < "2.7" +Requires: python-argparse %endif -%if 0%{?suse_version} -Requires: squashfs >= 4.0 -Requires: python-m2crypto +%if 0%{?tizen_version:1} +Requires: python-rpm %else -Requires: squashfs >= 4.0 -Requires: python-M2Crypto +Requires: rpm-python %endif -%if 0%{?fedora_version} || 0%{?centos_version} -Requires: syslinux-extlinux -%endif +Requires: cpio +# not neccessary +Requires: gzip +Requires: bzip2 %if 0%{?tizen_version:1} Requires: qemu-linux-user @@ -53,8 +47,6 @@ Requires: qemu-linux-user Requires: qemu-arm-static %endif -Requires: python-zypp - BuildRequires: python-devel %if ! 0%{?tizen_version:1} BuildRequires: python-docutils @@ -80,7 +72,7 @@ cp %{SOURCE1001} . %build CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build %if ! 0%{?tizen_version:1} -%__make man +make man %endif %install @@ -92,11 +84,19 @@ rm -rf %{buildroot} %endif # install man page -mkdir -p %{buildroot}%{_mandir}/man1 +mkdir -p %{buildroot}/%{_prefix}/share/man/man1 %if ! 0%{?tizen_version:1} -install -m644 doc/mic.1 %{buildroot}%{_mandir}/man1 +install -m644 doc/mic.1 %{buildroot}/%{_prefix}/share/man/man1 %endif +# install bash completion +install -d -m0755 %{buildroot}/%{_sysconfdir}/bash_completion.d/ +install -Dp -m0755 etc/bash_completion.d/%{name}.sh %{buildroot}/%{_sysconfdir}/bash_completion.d/ + +# install zsh completion +install -d -m0755 %{buildroot}/%{_sysconfdir}/zsh_completion.d/ +install -Dp -m0755 etc/zsh_completion.d/_%{name} %{buildroot}/%{_sysconfdir}/zsh_completion.d/ + %if ! 0%{?centos_version} %fdupes %{buildroot} %endif @@ -120,4 +120,7 @@ install -m644 doc/mic.1 %{buildroot}%{_mandir}/man1 %{python_sitelib}/* %dir %{_prefix}/lib/%{name} %{_prefix}/lib/%{name}/* -%{_bindir}/* +%{_bindir}/mic +%{_sysconfdir}/bash_completion.d +%{_sysconfdir}/zsh_completion.d + diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index 955f813..8f3112d 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -16,8 +16,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os, sys -import re +import os import tempfile import glob from string import Template @@ -31,7 +30,8 @@ from mic.utils import misc, rpmmisc from mic.utils.grabber import TextProgress from mic.utils.proxy import get_proxy_for from mic.utils.errors import CreatorError -from mic.imager.baseimager import BaseImageCreator +from mic.utils.safeurl import SafeURL + YUMCONF_TEMP = """[main] installroot=$installroot @@ -45,6 +45,10 @@ sslverify=1 """ class MyYumRepository(yum.yumRepo.YumRepository): + def __init__(self, repoid, nocache): + super(MyYumRepository, self).__init__(repoid) + self.nocache = nocache + def __del__(self): pass @@ -108,12 +112,13 @@ from mic.pluginbase import BackendPlugin class Yum(BackendPlugin, yum.YumBase): name = 'yum' - def __init__(self, target_arch, instroot, cachedir): + def __init__(self, target_arch, instroot, cachedir, strict_mode = False): yum.YumBase.__init__(self) self.cachedir = cachedir self.instroot = instroot self.target_arch = target_arch + self.strict_mode = strict_mode if self.target_arch: if not rpmUtils.arch.arches.has_key(self.target_arch): @@ -127,6 +132,7 @@ class Yum(BackendPlugin, yum.YumBase): self.__pkgs_license = {} self.__pkgs_content = {} self.__pkgs_vcsinfo = {} + self.check_pkgs = [] self.install_debuginfo = False @@ -149,14 +155,6 @@ class Yum(BackendPlugin, yum.YumBase): yum.YumBase.close(self) self.closeRpmDB() - if not os.path.exists("/etc/fedora-release") and \ - not os.path.exists("/etc/meego-release"): - for i in range(3, os.sysconf("SC_OPEN_MAX")): - try: - os.close(i) - except: - pass - def __del__(self): pass @@ -194,6 +192,9 @@ class Yum(BackendPlugin, yum.YumBase): # FIXME: handle pre-install package return None + def checkPackage(self, pkg): + self.check_pkgs.append(pkg) + def selectPackage(self, pkg): """Select a given package. Can be specified with name.arch or name* @@ -268,8 +269,7 @@ class Yum(BackendPlugin, yum.YumBase): option = option.replace("$basearch", rpmUtils.arch.getBaseArch()) option = option.replace("$arch", rpmUtils.arch.getCanonArch()) return option - - repo = MyYumRepository(name) + repo = MyYumRepository(name, nocache) # Set proxy repo.proxy = proxy @@ -277,12 +277,7 @@ class Yum(BackendPlugin, yum.YumBase): repo.proxy_password = proxy_password if url: - repo.baseurl.append(_varSubstitute(url)) - - # check LICENSE files - if not rpmmisc.checkRepositoryEULA(name, repo): - msger.warning('skip repo:%s for failed EULA confirmation' % name) - return None + repo.baseurl.append(_varSubstitute(url.full)) if mirrorlist: repo.mirrorlist = _varSubstitute(mirrorlist) @@ -293,7 +288,6 @@ class Yum(BackendPlugin, yum.YumBase): repo.setAttribute(k, v) repo.sslverify = ssl_verify - repo.cache = not nocache repo.basecachedir = self.cachedir repo.base_persistdir = self.conf.persistdir @@ -369,6 +363,12 @@ class Yum(BackendPlugin, yum.YumBase): else: self.__pkgs_license[license] = [pkg_long_name] + if pkg.name in self.check_pkgs: + self.check_pkgs.remove(pkg.name) + + if self.check_pkgs: + raise CreatorError('Packages absent in image: %s' % ','.join(self.check_pkgs)) + total_count = len(dlpkgs) cached_count = 0 download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs)) @@ -377,7 +377,7 @@ class Yum(BackendPlugin, yum.YumBase): for po in dlpkgs: local = po.localPkg() repo = filter(lambda r: r.id == po.repoid, self.repos.listEnabled())[0] - if not repo.cache and os.path.exists(local): + if repo.nocache and os.path.exists(local): os.unlink(local) if not os.path.exists(local): continue @@ -386,7 +386,7 @@ class Yum(BackendPlugin, yum.YumBase): % (os.path.basename(local), local)) else: download_total_size -= int(po.packagesize) - cached_count +=1 + cached_count += 1 cache_avail_size = misc.get_filesystem_avail(self.cachedir) if cache_avail_size < download_total_size: @@ -439,7 +439,9 @@ class Yum(BackendPlugin, yum.YumBase): installlogfile = "%s/__catched_stderr.buf" % (self.instroot) msger.enable_logstderr(installlogfile) - self.runTransaction(cb) + transactionResult = self.runTransaction(cb) + if transactionResult.return_code != 0 and self.strict_mode: + raise CreatorError("mic failes to install some packages") self._cleanupRpmdbLocks(self.conf.installroot) except rpmUtils.RpmUtilsError, e: @@ -452,6 +454,23 @@ class Yum(BackendPlugin, yum.YumBase): msger.disable_logstderr() def getVcsInfo(self): + if self.__pkgs_vcsinfo: + return self.__pkgs_vcsinfo + if not self.ts: + self.__initialize_transaction() + mi = self.ts.dbMatch() + for hdr in mi: + lname = misc.RPM_FMT % { + 'name': hdr['name'], + 'arch': hdr['arch'], + 'version': hdr['version'], + 'release': hdr['release'] + } + try: + self.__pkgs_vcsinfo[lname] = hdr['VCS'] + except ValueError, KeyError: + self.__pkgs_vcsinfo[lname] = None + return self.__pkgs_vcsinfo def getAllContent(self): @@ -472,18 +491,18 @@ class Yum(BackendPlugin, yum.YumBase): def package_url(self, pkgname): pkgs = self.pkgSack.searchNevra(name=pkgname) if pkgs: - proxy = None - proxies = None - url = pkgs[0].remote_url - repoid = pkgs[0].repoid - repos = filter(lambda r: r.id == repoid, self.repos.listEnabled()) - - if repos: - proxy = repos[0].proxy + pkg = pkgs[0] + + repo = pkg.repo + url = SafeURL(repo.baseurl[0]).join(pkg.remote_path) + + proxy = repo.proxy if not proxy: proxy = get_proxy_for(url) if proxy: proxies = {str(url.split(':')[0]): str(proxy)} + else: + proxies = None return (url, proxies) diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index c760859..9358cbe 100755..100644 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -54,10 +54,11 @@ from mic.pluginbase import BackendPlugin class Zypp(BackendPlugin): name = 'zypp' - def __init__(self, target_arch, instroot, cachedir): + def __init__(self, target_arch, instroot, cachedir, strict_mode = False): self.cachedir = cachedir self.instroot = instroot self.target_arch = target_arch + self.strict_mode = strict_mode self.__pkgs_license = {} self.__pkgs_content = {} @@ -73,11 +74,14 @@ class Zypp(BackendPlugin): self.incpkgs = {} self.excpkgs = {} self.pre_pkgs = [] + self.check_pkgs = [] self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE, rpm.RPMPROB_FILTER_REPLACEPKG ] self.has_prov_query = True self.install_debuginfo = False + # this can't be changed, it is used by zypp + self.tmp_file_path = '/var/tmp' def doFileLogSetup(self, uid, logfile): # don't do the file log for the livecd as it can lead to open fds @@ -98,14 +102,6 @@ class Zypp(BackendPlugin): self.closeRpmDB() - if not os.path.exists("/etc/fedora-release") and \ - not os.path.exists("/etc/meego-release"): - for i in range(3, os.sysconf("SC_OPEN_MAX")): - try: - os.close(i) - except: - pass - def __del__(self): self.close() @@ -124,11 +120,15 @@ class Zypp(BackendPlugin): def setup(self): self._cleanupRpmdbLocks(self.instroot) + # '/var/tmp' is used by zypp to build cache, so make sure + # if it exists + if not os.path.exists(self.tmp_file_path ): + os.makedirs(self.tmp_file_path) def whatObsolete(self, pkg): query = zypp.PoolQuery() query.addKind(zypp.ResKind.package) - query.addAttribute(zypp.SolvAttr.obsoletes, pkg) + query.addDependency(zypp.SolvAttr.obsoletes, pkg.name(), pkg.edition()) query.setMatchExact() for pi in query.queryResults(self.Z.pool()): return pi @@ -137,14 +137,14 @@ class Zypp(BackendPlugin): def _zyppQueryPackage(self, pkg): query = zypp.PoolQuery() query.addKind(zypp.ResKind.package) - query.addAttribute(zypp.SolvAttr.name,pkg) + query.addAttribute(zypp.SolvAttr.name, pkg) query.setMatchExact() for pi in query.queryResults(self.Z.pool()): return pi return None def _splitPkgString(self, pkg): - sp = pkg.rsplit(".",1) + sp = pkg.rsplit(".", 1) name = sp[0] arch = None if len(sp) == 2: @@ -211,15 +211,15 @@ class Zypp(BackendPlugin): if endx and startx: pattern = '%s' % (pkg[1:-1]) q.setMatchRegex() - q.addAttribute(zypp.SolvAttr.name,pattern) + q.addAttribute(zypp.SolvAttr.name, pattern) elif arch: q.setMatchExact() - q.addAttribute(zypp.SolvAttr.name,name) + q.addAttribute(zypp.SolvAttr.name, name) else: q.setMatchExact() - q.addAttribute(zypp.SolvAttr.name,pkg) + q.addAttribute(zypp.SolvAttr.name, pkg) for pitem in sorted( q.queryResults(self.Z.pool()), @@ -234,10 +234,10 @@ class Zypp(BackendPlugin): continue found = True - obspkg = self.whatObsolete(item.name()) + obspkg = self.whatObsolete(item) if arch: if arch == str(item.arch()): - item.status().setToBeInstalled (zypp.ResStatus.USER) + pitem.status().setToBeInstalled (zypp.ResStatus.USER) else: markPoolItem(obspkg, pitem) if not ispattern: @@ -262,7 +262,7 @@ class Zypp(BackendPlugin): continue found = True - obspkg = self.whatObsolete(item.name()) + obspkg = self.whatObsolete(item) markPoolItem(obspkg, pitem) break @@ -284,29 +284,32 @@ class Zypp(BackendPlugin): if not ispattern: if pkgarch: if name == pkgname and str(item.arch()) == pkgarch: - return True; + return True else: if name == pkgname: - return True; + return True else: if startx and name.endswith(pkg[1:]): - return True; + return True if endx and name.startswith(pkg[:-1]): - return True; + return True - return False; + return False def deselectPackage(self, pkg): """collect packages should not be installed""" self.to_deselect.append(pkg) def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT): + def compareGroup(pitem): + item = zypp.asKindPattern(pitem) + return item.repoInfo().priority() if not self.Z: self.__initialize_zypp() found = False - q=zypp.PoolQuery() + q = zypp.PoolQuery() q.addKind(zypp.ResKind.pattern) - for pitem in q.queryResults(self.Z.pool()): + for pitem in sorted(q.queryResults(self.Z.pool()), key=compareGroup): item = zypp.asKindPattern(pitem) summary = "%s" % item.summary() name = "%s" % item.name() @@ -361,11 +364,6 @@ class Zypp(BackendPlugin): for pkg in exc: self.excpkgs[pkg] = name - # check LICENSE files - if not rpmmisc.checkRepositoryEULA(name, repo): - msger.warning('skip repo:%s for failed EULA confirmation' % name) - return None - if mirrorlist: repo.mirrorlist = mirrorlist @@ -382,13 +380,14 @@ class Zypp(BackendPlugin): repo_info.setEnabled(repo.enabled) repo_info.setAutorefresh(repo.autorefresh) repo_info.setKeepPackages(repo.keeppackages) - baseurl = zypp.Url(repo.baseurl[0]) + baseurl = zypp.Url(repo.baseurl[0].full) if not ssl_verify: baseurl.setQueryParam("ssl_verify", "no") if proxy: - scheme, host, path, parm, query, frag = urlparse.urlparse(proxy) + host = urlparse.urlparse(proxy)[1] + # scheme, host, path, parm, query, frag = urlparse.urlparse(proxy) - proxyinfo = host.split(":") + proxyinfo = host.rsplit(":", 1) host = proxyinfo[0] port = "80" @@ -399,10 +398,28 @@ class Zypp(BackendPlugin): host = proxy.rsplit(':', 1)[0] port = proxy.rsplit(':', 1)[1] + # parse user/pass from proxy host + proxyinfo = host.rsplit("@", 1) + if len(proxyinfo) == 2: + host = proxyinfo[1] + # Known Issue: If password contains ":", which should be + # quoted, for example, use '123%3Aabc' instead of 123:abc + userpassinfo = proxyinfo[0].rsplit(":", 1) + if len(userpassinfo) == 2: + proxy_username = userpassinfo[0] + proxy_password = userpassinfo[1] + elif len(userpassinfo) == 1: + proxy_username = userpassinfo[0] + baseurl.setQueryParam ("proxy", host) baseurl.setQueryParam ("proxyport", port) + if proxy_username: + baseurl.setQueryParam ("proxyuser", proxy_username) + if proxy_password: + baseurl.setQueryParam ("proxypass", proxy_password) + else: + baseurl.setQueryParam ("proxy", "_none_") - repo.baseurl[0] = baseurl.asCompleteString() self.repos.append(repo) repo_info.addBaseUrl(baseurl) @@ -443,6 +460,9 @@ class Zypp(BackendPlugin): def preInstall(self, pkg): self.pre_pkgs.append(pkg) + def checkPackage(self, pkg): + self.check_pkgs.append(pkg) + def runInstall(self, checksize = 0): os.environ["HOME"] = "/" os.environ["LD_PRELOAD"] = "" @@ -451,12 +471,16 @@ class Zypp(BackendPlugin): todo = zypp.GetResolvablesToInsDel(self.Z.pool()) installed_pkgs = todo._toInstall dlpkgs = [] + for pitem in installed_pkgs: if not zypp.isKindPattern(pitem) and \ not self.inDeselectPackages(pitem): item = zypp.asKindPackage(pitem) dlpkgs.append(item) + if item.name() in self.check_pkgs: + self.check_pkgs.remove(item.name()) + if not self.install_debuginfo or str(item.arch()) == "noarch": continue @@ -468,6 +492,9 @@ class Zypp(BackendPlugin): msger.warning("No debuginfo rpm found for: %s" \ % item.name()) + if self.check_pkgs: + raise CreatorError('Packages absent in image: %s' % ','.join(self.check_pkgs)) + # record all pkg and the content localpkgs = self.localpkgs.keys() for pkg in dlpkgs: @@ -544,9 +571,11 @@ class Zypp(BackendPlugin): if download_count > 0: msger.info("Downloading packages ...") self.downloadPkgs(dlpkgs, download_count) + except CreatorError, e: + raise CreatorError("Package download failed: %s" %(e,)) + try: self.installPkgs(dlpkgs) - except (RepoError, RpmError): raise except Exception, e: @@ -567,7 +596,11 @@ class Zypp(BackendPlugin): 'version': hdr['version'], 'release': hdr['release'] } - self.__pkgs_vcsinfo[lname] = hdr['VCS'] + try: + self.__pkgs_vcsinfo[lname] = hdr['VCS'] + except ValueError: + # if rpm not support VCS, set to None + self.__pkgs_vcsinfo[lname] = None return self.__pkgs_vcsinfo @@ -686,7 +719,7 @@ class Zypp(BackendPlugin): def getLocalPkgPath(self, po): repoinfo = po.repoInfo() cacheroot = repoinfo.packagesPath() - location= po.location() + location = po.location() rpmpath = str(location.filename()) pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath)) return pkgpath @@ -749,7 +782,7 @@ class Zypp(BackendPlugin): proxies = self.get_proxies(po) try: - filename = myurlgrab(url, filename, proxies, progress_obj) + filename = myurlgrab(url.full, filename, proxies, progress_obj) except CreatorError: self.close() raise @@ -844,7 +877,8 @@ class Zypp(BackendPlugin): if len(errors) == 0: msger.warning('scriptlet or other non-fatal errors occurred ' 'during transaction.') - + if self.strict_mode: + raise CreatorError("mic failes to install some packages") else: for e in errors: msger.warning(e[0]) @@ -868,7 +902,7 @@ class Zypp(BackendPlugin): % (package, deppkg)) elif sense == rpm.RPMDEP_SENSE_CONFLICTS: - msger.warning("[%s] Conflicts with [%s]" %(package,deppkg)) + msger.warning("[%s] Conflicts with [%s]" % (package, deppkg)) raise RepoError("Unresolved dependencies, transaction failed.") @@ -931,18 +965,12 @@ class Zypp(BackendPlugin): except IndexError: return None - baseurl = repo.baseurl[0] - - index = baseurl.find("?") - if index > -1: - baseurl = baseurl[:index] - location = pobj.location() location = str(location.filename()) if location.startswith("./"): location = location[2:] - return os.path.join(baseurl, location) + return repo.baseurl[0].join(location) def package_url(self, pkgname): diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index 8e758db..d74530f 100644 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -15,11 +15,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os -import sys - from mic import chroot, msger, rt_util -from mic.utils import cmdln, misc, errors, fs_related +from mic.utils import misc, errors, fs_related from mic.imager import fs from mic.conf import configmgr from mic.plugin import pluginmgr @@ -29,12 +26,7 @@ class FsPlugin(ImagerPlugin): name = 'fs' @classmethod - @cmdln.option("--include-src", - dest="include_src", - action="store_true", - default=False, - help="Generate a image with source rpms included") - def do_create(self, subcmd, opts, *args): + def do_create(self, args): """${cmd_name}: create fs image Usage: @@ -43,11 +35,11 @@ class FsPlugin(ImagerPlugin): ${cmd_option_list} """ - if len(args) != 1: - raise errors.Usage("Extra arguments given") + if args is None: + raise errors.Usage("Invalid arguments.") creatoropts = configmgr.create - ksconf = args[0] + ksconf = args.ksfile if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf @@ -65,10 +57,6 @@ class FsPlugin(ImagerPlugin): configmgr._ksconf = ksconf - # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. - if creatoropts['release'] is not None: - creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) - # try to find the pkgmgr pkgmgr = None backends = pluginmgr.get_plugins('backend') @@ -90,7 +78,7 @@ class FsPlugin(ImagerPlugin): ','.join(backends.keys()))) creator = fs.FsImageCreator(creatoropts, pkgmgr) - creator._include_src = opts.include_src + creator._include_src = args.include_src if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs @@ -105,19 +93,22 @@ class FsPlugin(ImagerPlugin): creator.mount(None, creatoropts["cachedir"]) creator.install() #Download the source packages ###private options - if opts.include_src: + if args.include_src: installed_pkgs = creator.get_installed_packages() msger.info('--------------------------------------------------') msger.info('Generating the image with source rpms included ...') - if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], creator._instroot, creatoropts["cachedir"]): + if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], + creator._instroot, creatoropts["cachedir"]): msger.warning("Source packages can't be downloaded") creator.configure(creatoropts["repomd"]) creator.copy_kernel() creator.unmount() - creator.package(creatoropts["outdir"]) + creator.package(creatoropts["destdir"]) + creator.create_manifest() if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.release_output(ksconf, creatoropts['destdir'], + creatoropts['release']) creator.print_outimage_info() except errors.CreatorError: raise diff --git a/plugins/imager/livecd_plugin.py b/plugins/imager/livecd_plugin.py deleted file mode 100644 index d24ef59..0000000 --- a/plugins/imager/livecd_plugin.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import tempfile - -from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors -from mic.conf import configmgr -import mic.imager.livecd as livecd -from mic.plugin import pluginmgr - -from mic.pluginbase import ImagerPlugin -class LiveCDPlugin(ImagerPlugin): - name = 'livecd' - - @classmethod - def do_create(self, subcmd, opts, *args): - """${cmd_name}: create livecd image - - Usage: - ${name} ${cmd_name} <ksfile> [OPTS] - - ${cmd_option_list} - """ - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - creatoropts = configmgr.create - ksconf = args[0] - - if creatoropts['runtime'] == 'bootstrap': - configmgr._ksconf = ksconf - rt_util.bootstrap_mic() - - if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): - msger.warning('livecd cannot support arm images, Quit') - return - - recording_pkgs = [] - if len(creatoropts['record_pkgs']) > 0: - recording_pkgs = creatoropts['record_pkgs'] - - if creatoropts['release'] is not None: - if 'name' not in recording_pkgs: - recording_pkgs.append('name') - if 'vcs' not in recording_pkgs: - recording_pkgs.append('vcs') - - configmgr._ksconf = ksconf - - # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. - if creatoropts['release'] is not None: - creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) - - # try to find the pkgmgr - pkgmgr = None - backends = pluginmgr.get_plugins('backend') - if 'auto' == creatoropts['pkgmgr']: - for key in configmgr.prefer_backends: - if key in backends: - pkgmgr = backends[key] - break - else: - for key in backends.keys(): - if key == creatoropts['pkgmgr']: - pkgmgr = backends[key] - break - - if not pkgmgr: - raise errors.CreatorError("Can't find backend: %s, " - "available choices: %s" % - (creatoropts['pkgmgr'], - ','.join(backends.keys()))) - - creator = livecd.LiveCDImageCreator(creatoropts, pkgmgr) - - if len(recording_pkgs) > 0: - creator._recording_pkgs = recording_pkgs - - self.check_image_exists(creator.destdir, - creator.pack_to, - [creator.name + ".iso"], - creatoropts['release']) - - try: - creator.check_depend_tools() - creator.mount(None, creatoropts["cachedir"]) - creator.install() - creator.configure(creatoropts["repomd"]) - creator.copy_kernel() - creator.unmount() - creator.package(creatoropts["outdir"]) - if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - msger.info("Finished.") - return 0 - - @classmethod - def do_chroot(cls, target, cmd=[]): - os_image = cls.do_unpack(target) - os_image_dir = os.path.dirname(os_image) - - # unpack image to target dir - imgsize = misc.get_file_size(os_image) * 1024L * 1024L - imgtype = misc.get_image_type(os_image) - if imgtype == "btrfsimg": - fstype = "btrfs" - myDiskMount = fs_related.BtrfsDiskMount - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - myDiskMount = fs_related.ExtDiskMount - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - - extmnt = misc.mkdtemp() - extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), - extmnt, - fstype, - 4096, - "%s label" % fstype) - try: - extloop.mount() - - except errors.MountError: - extloop.cleanup() - shutil.rmtree(extmnt, ignore_errors = True) - shutil.rmtree(os_image_dir, ignore_errors = True) - raise - - try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) - else: - cmdline = "/bin/bash" - envcmd = fs_related.find_binary_inchroot("env", extmnt) - if envcmd: - cmdline = "%s HOME=/root %s" % (envcmd, cmdline) - chroot.chroot(extmnt, None, cmdline) - except: - raise errors.CreatorError("Failed to chroot to %s." %target) - finally: - chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) - - @classmethod - def do_pack(cls, base_on): - import subprocess - - def __mkinitrd(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) - - def __run_post_cleanups(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] - - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to run post cleanups: %s" % msg) - - convertoropts = configmgr.convert - convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] - convertor = livecd.LiveCDImageCreator(convertoropts) - imgtype = misc.get_image_type(base_on) - if imgtype == "btrfsimg": - fstype = "btrfs" - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - convertor._set_fstype(fstype) - try: - convertor.mount(base_on) - __mkinitrd(convertor) - convertor._create_bootconfig() - __run_post_cleanups(convertor) - convertor.launch_shell(convertoropts['shell']) - convertor.unmount() - convertor.package() - convertor.print_outimage_info() - finally: - shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) - - @classmethod - def do_unpack(cls, srcimg): - img = srcimg - imgmnt = misc.mkdtemp() - imgloop = fs_related.DiskMount(fs_related.LoopbackDisk(img, 0), imgmnt) - try: - imgloop.mount() - except errors.MountError: - imgloop.cleanup() - raise - - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(imgmnt + "/squashfs.img"): - squashimg = imgmnt + "/squashfs.img" - else: - squashimg = imgmnt + "/LiveOS/squashfs.img" - - tmpoutdir = misc.mkdtemp() - # unsquashfs requires outdir mustn't exist - shutil.rmtree(tmpoutdir, ignore_errors = True) - misc.uncompress_squashfs(squashimg, tmpoutdir) - - try: - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(tmpoutdir + "/os.img"): - os_image = tmpoutdir + "/os.img" - else: - os_image = tmpoutdir + "/LiveOS/ext3fs.img" - - if not os.path.exists(os_image): - raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " - "LiveOS/ext3fs.img nor os.img exist" %img) - - imgname = os.path.basename(srcimg) - imgname = os.path.splitext(imgname)[0] + ".img" - rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) - shutil.copyfile(os_image, rtimage) - - finally: - imgloop.cleanup() - shutil.rmtree(tmpoutdir, ignore_errors = True) - shutil.rmtree(imgmnt, ignore_errors = True) - - return rtimage diff --git a/plugins/imager/liveusb_plugin.py b/plugins/imager/liveusb_plugin.py deleted file mode 100644 index 7aa8927..0000000 --- a/plugins/imager/liveusb_plugin.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import tempfile - -from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors -from mic.utils.partitionedfs import PartitionedMount -from mic.conf import configmgr -from mic.plugin import pluginmgr - -import mic.imager.liveusb as liveusb - -from mic.pluginbase import ImagerPlugin -class LiveUSBPlugin(ImagerPlugin): - name = 'liveusb' - - @classmethod - def do_create(self, subcmd, opts, *args): - """${cmd_name}: create liveusb image - - Usage: - ${name} ${cmd_name} <ksfile> [OPTS] - - ${cmd_option_list} - """ - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - creatoropts = configmgr.create - ksconf = args[0] - - if creatoropts['runtime'] == "bootstrap": - configmgr._ksconf = ksconf - rt_util.bootstrap_mic() - - if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): - msger.warning('liveusb cannot support arm images, Quit') - return - - recording_pkgs = [] - if len(creatoropts['record_pkgs']) > 0: - recording_pkgs = creatoropts['record_pkgs'] - - if creatoropts['release'] is not None: - if 'name' not in recording_pkgs: - recording_pkgs.append('name') - if 'vcs' not in recording_pkgs: - recording_pkgs.append('vcs') - - configmgr._ksconf = ksconf - - # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. - if creatoropts['release'] is not None: - creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) - - # try to find the pkgmgr - pkgmgr = None - backends = pluginmgr.get_plugins('backend') - if 'auto' == creatoropts['pkgmgr']: - for key in configmgr.prefer_backends: - if key in backends: - pkgmgr = backends[key] - break - else: - for key in backends.keys(): - if key == creatoropts['pkgmgr']: - pkgmgr = backends[key] - break - - if not pkgmgr: - raise errors.CreatorError("Can't find backend: %s, " - "available choices: %s" % - (creatoropts['pkgmgr'], - ','.join(backends.keys()))) - - creator = liveusb.LiveUSBImageCreator(creatoropts, pkgmgr) - - if len(recording_pkgs) > 0: - creator._recording_pkgs = recording_pkgs - - self.check_image_exists(creator.destdir, - creator.pack_to, - [creator.name + ".usbimg"], - creatoropts['release']) - try: - creator.check_depend_tools() - creator.mount(None, creatoropts["cachedir"]) - creator.install() - creator.configure(creatoropts["repomd"]) - creator.copy_kernel() - creator.unmount() - creator.package(creatoropts["outdir"]) - if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - msger.info("Finished.") - return 0 - - @classmethod - def do_chroot(cls, target, cmd=[]): - os_image = cls.do_unpack(target) - os_image_dir = os.path.dirname(os_image) - - # unpack image to target dir - imgsize = misc.get_file_size(os_image) * 1024L * 1024L - imgtype = misc.get_image_type(os_image) - if imgtype == "btrfsimg": - fstype = "btrfs" - myDiskMount = fs_related.BtrfsDiskMount - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - myDiskMount = fs_related.ExtDiskMount - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - - extmnt = misc.mkdtemp() - extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), - extmnt, - fstype, - 4096, - "%s label" % fstype) - - try: - extloop.mount() - - except errors.MountError: - extloop.cleanup() - shutil.rmtree(extmnt, ignore_errors = True) - raise - - try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) - else: - cmdline = "/bin/bash" - envcmd = fs_related.find_binary_inchroot("env", extmnt) - if envcmd: - cmdline = "%s HOME=/root %s" % (envcmd, cmdline) - chroot.chroot(extmnt, None, cmdline) - except: - raise errors.CreatorError("Failed to chroot to %s." %target) - finally: - chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) - - @classmethod - def do_pack(cls, base_on): - import subprocess - - def __mkinitrd(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] - try: - subprocess.call(args, preexec_fn = instance._chroot) - - except OSError, (err, msg): - raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) - - def __run_post_cleanups(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] - - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to run post cleanups: %s" % msg) - - convertoropts = configmgr.convert - convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] - convertor = liveusb.LiveUSBImageCreator(convertoropts) - imgtype = misc.get_image_type(base_on) - if imgtype == "btrfsimg": - fstype = "btrfs" - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstyp) - convertor._set_fstype(fstype) - try: - convertor.mount(base_on) - __mkinitrd(convertor) - convertor._create_bootconfig() - __run_post_cleanups(convertor) - convertor.launch_shell(convertoropts['shell']) - convertor.unmount() - convertor.package() - convertor.print_outimage_info() - finally: - shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) - - @classmethod - def do_unpack(cls, srcimg): - img = srcimg - imgsize = misc.get_file_size(img) * 1024L * 1024L - imgmnt = misc.mkdtemp() - disk = fs_related.SparseLoopbackDisk(img, imgsize) - imgloop = PartitionedMount(imgmnt, skipformat = True) - imgloop.add_disk('/dev/sdb', disk) - imgloop.add_partition(imgsize/1024/1024, "/dev/sdb", "/", "vfat", boot=False) - try: - imgloop.mount() - except errors.MountError: - imgloop.cleanup() - raise - - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(imgmnt + "/squashfs.img"): - squashimg = imgmnt + "/squashfs.img" - else: - squashimg = imgmnt + "/LiveOS/squashfs.img" - - tmpoutdir = misc.mkdtemp() - # unsquashfs requires outdir mustn't exist - shutil.rmtree(tmpoutdir, ignore_errors = True) - misc.uncompress_squashfs(squashimg, tmpoutdir) - - try: - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(tmpoutdir + "/os.img"): - os_image = tmpoutdir + "/os.img" - else: - os_image = tmpoutdir + "/LiveOS/ext3fs.img" - - if not os.path.exists(os_image): - raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " - "LiveOS/ext3fs.img nor os.img exist" %img) - imgname = os.path.basename(srcimg) - imgname = os.path.splitext(imgname)[0] + ".img" - rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) - shutil.copyfile(os_image, rtimage) - - finally: - imgloop.cleanup() - shutil.rmtree(tmpoutdir, ignore_errors = True) - shutil.rmtree(imgmnt, ignore_errors = True) - - return rtimage diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index 8f4b030..1830230 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -20,7 +20,7 @@ import shutil import tempfile from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors, cmdln +from mic.utils import misc, fs_related, errors from mic.conf import configmgr from mic.plugin import pluginmgr from mic.imager.loop import LoopImageCreator, load_mountpoints @@ -30,16 +30,7 @@ class LoopPlugin(ImagerPlugin): name = 'loop' @classmethod - @cmdln.option("--compress-disk-image", dest="compress_image", - type='choice', choices=("gz", "bz2"), default=None, - help="Same with --compress-image") - # alias to compress-image for compatibility - @cmdln.option("--compress-image", dest="compress_image", - type='choice', choices=("gz", "bz2"), default=None, - help="Compress all loop images with 'gz' or 'bz2'") - @cmdln.option("--shrink", action='store_true', default=False, - help="Whether to shrink loop images to minimal size") - def do_create(self, subcmd, opts, *args): + def do_create(self, args): """${cmd_name}: create loop image Usage: @@ -48,11 +39,11 @@ class LoopPlugin(ImagerPlugin): ${cmd_option_list} """ - if len(args) != 1: - raise errors.Usage("Extra arguments given") + if args is None: + raise errors.Usage("Invalid arguments") creatoropts = configmgr.create - ksconf = args[0] + ksconf = args.ksfile if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf @@ -70,12 +61,6 @@ class LoopPlugin(ImagerPlugin): configmgr._ksconf = ksconf - # Called After setting the configmgr._ksconf - # as the creatoropts['name'] is reset there. - if creatoropts['release'] is not None: - creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], - creatoropts['release'], - creatoropts['name']) # try to find the pkgmgr pkgmgr = None backends = pluginmgr.get_plugins('backend') @@ -98,8 +83,8 @@ class LoopPlugin(ImagerPlugin): creator = LoopImageCreator(creatoropts, pkgmgr, - opts.compress_image, - opts.shrink) + args.compress_image, + args.shrink) if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs @@ -118,11 +103,12 @@ class LoopPlugin(ImagerPlugin): creator.configure(creatoropts["repomd"]) creator.copy_kernel() creator.unmount() - creator.package(creatoropts["outdir"]) + creator.package(creatoropts["destdir"]) + creator.create_manifest() if creatoropts['release'] is not None: creator.release_output(ksconf, - creatoropts['outdir'], + creatoropts['destdir'], creatoropts['release']) creator.print_outimage_info() @@ -158,7 +144,7 @@ class LoopPlugin(ImagerPlugin): elif fstype in ("vfat", "msdos"): myDiskMount = fs_related.VfatDiskMount else: - msger.error("Cannot support fstype: %s" % fstype) + raise errors.CreatorError("Cannot support fstype: %s" % fstype) name = os.path.join(tmpdir, name) size = size * 1024L * 1024L @@ -233,8 +219,8 @@ class LoopPlugin(ImagerPlugin): raise try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) + if cmd is not None: + cmdline = cmd else: cmdline = "/bin/bash" envcmd = fs_related.find_binary_inchroot("env", extmnt) diff --git a/plugins/imager/qcow_plugin.py b/plugins/imager/qcow_plugin.py new file mode 100644 index 0000000..8acf572 --- /dev/null +++ b/plugins/imager/qcow_plugin.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil + +from mic import msger, rt_util +from mic.conf import configmgr +from mic.plugin import pluginmgr +from mic.pluginbase import ImagerPlugin +from mic.imager.loop import LoopImageCreator +from mic.utils import errors, fs_related, runner + +class QcowImageCreator(LoopImageCreator): + img_format = 'qcow' + + def __init__(self, creatoropts=None, pkgmgr=None): + LoopImageCreator.__init__(self, creatoropts, pkgmgr) + self.cmd_qemuimg = 'qemu-img' + + def _stage_final_image(self): + try: + self.cmd_qemuimg = fs_related.find_binary_path('qemu-img') + except errors.CreatorError: + return LoopImageCreator._stage_final_image(self) + + self._resparse() + + imgfile = None + for item in self._instloops: + if item['mountpoint'] == '/': + if item['fstype'] == "ext4": + runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s' + % imgfile) + self.image_files.setdefault('partitions', {}).update( + {item['mountpoint']: item['label']}) + imgfile = os.path.join(self._imgdir, item['name']) + + if imgfile: + qemuimage = imgfile + ".x86" + runner.show("%s convert -O qcow2 %s %s" + % (self.cmd_qemuimg, imgfile, qemuimage)) + os.unlink(imgfile) + os.rename(qemuimage, imgfile) + + for item in os.listdir(self._imgdir): + shutil.move(os.path.join(self._imgdir, item), + os.path.join(self._outdir, item)) + +class QcowPlugin(ImagerPlugin): + name = 'qcow' + + @classmethod + def do_create(cls, args): + """${cmd_name}: create qcow image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + if args is None: + raise errors.Usage("Invalid arguments") + + creatoropts = configmgr.create + ksconf = args.ksfile + + if creatoropts['runtime'] == "bootstrap": + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = QcowImageCreator(creatoropts, + pkgmgr) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + image_names = [creator.name + ".img"] + image_names.extend(creator.get_image_names()) + cls.check_image_exists(creator.destdir, + creator.pack_to, + image_names, + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["destdir"]) + creator.create_manifest() + + if creatoropts['release'] is not None: + creator.release_output(ksconf, + creatoropts['destdir'], + creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(cls, target, cmd=[]): + pass + + @classmethod + def do_unpack(cls, srcimg): + pass diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py index 1b9631d..09a9714 100644..100755 --- a/plugins/imager/raw_plugin.py +++ b/plugins/imager/raw_plugin.py @@ -21,7 +21,7 @@ import re import tempfile from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors, runner, cmdln +from mic.utils import misc, fs_related, errors, runner from mic.conf import configmgr from mic.plugin import pluginmgr from mic.utils.partitionedfs import PartitionedMount @@ -33,19 +33,7 @@ class RawPlugin(ImagerPlugin): name = 'raw' @classmethod - @cmdln.option("--compress-disk-image", dest="compress_image", type='choice', - choices=("gz", "bz2"), default=None, - help="Same with --compress-image") - @cmdln.option("--compress-image", dest="compress_image", type='choice', - choices=("gz", "bz2"), default = None, - help="Compress all raw images before package") - @cmdln.option("--generate-bmap", action="store_true", default = None, - help="also generate the block map file") - @cmdln.option("--fstab-entry", dest="fstab_entry", type='choice', - choices=("name", "uuid"), default="uuid", - help="Set fstab entry, 'name' means using device names, " - "'uuid' means using filesystem uuid") - def do_create(self, subcmd, opts, *args): + def do_create(self, args): """${cmd_name}: create raw image Usage: @@ -54,11 +42,8 @@ class RawPlugin(ImagerPlugin): ${cmd_option_list} """ - if len(args) != 1: - raise errors.Usage("Extra arguments given") - creatoropts = configmgr.create - ksconf = args[0] + ksconf = args.ksfile if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf @@ -76,10 +61,6 @@ class RawPlugin(ImagerPlugin): configmgr._ksconf = ksconf - # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there. - if creatoropts['release'] is not None: - creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name']) - # try to find the pkgmgr pkgmgr = None backends = pluginmgr.get_plugins('backend') @@ -100,8 +81,8 @@ class RawPlugin(ImagerPlugin): (creatoropts['pkgmgr'], ','.join(backends.keys()))) - creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image, - opts.generate_bmap, opts.fstab_entry) + creator = raw.RawImageCreator(creatoropts, pkgmgr, args.compress_image, + args.generate_bmap, args.fstab_entry) if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs @@ -121,9 +102,10 @@ class RawPlugin(ImagerPlugin): creator.copy_kernel() creator.unmount() creator.generate_bmap() - creator.package(creatoropts["outdir"]) + creator.package(creatoropts["destdir"]) + creator.create_manifest() if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release']) + creator.release_output(ksconf, creatoropts['destdir'], creatoropts['release']) creator.print_outimage_info() except errors.CreatorError: @@ -166,7 +148,7 @@ class RawPlugin(ImagerPlugin): else: root_mounted = False partition_mounts = 0 - for line in runner.outs([partedcmd,"-s",img,"unit","B","print"]).splitlines(): + for line in runner.outs([ partedcmd, "-s", img, "unit", "B", "print" ]).splitlines(): line = line.strip() # Lines that start with number are the partitions, @@ -178,12 +160,12 @@ class RawPlugin(ImagerPlugin): line = line.replace(",","") # Example of parted output lines that are handled: - # Number Start End Size Type File system Flags + # Number Start End Size Type File system Flags # 1 512B 3400000511B 3400000000B primary # 2 3400531968B 3656384511B 255852544B primary linux-swap(v1) - # 3 3656384512B 3720347647B 63963136B primary fat16 boot, lba + # 3 3656384512B 3720347647B 63963136B primary fat16 boot, lba - partition_info = re.split("\s+",line) + partition_info = re.split("\s+", line) size = partition_info[3].split("B")[0] @@ -193,18 +175,19 @@ class RawPlugin(ImagerPlugin): # not recognize properly. # TODO: Can we make better assumption? fstype = "btrfs" - elif partition_info[5] in ["ext2","ext3","ext4","btrfs"]: + elif partition_info[5] in [ "ext2", "ext3", "ext4", "btrfs" ]: fstype = partition_info[5] - elif partition_info[5] in ["fat16","fat32"]: + elif partition_info[5] in [ "fat16", "fat32" ]: fstype = "vfat" elif "swap" in partition_info[5]: fstype = "swap" else: - raise errors.CreatorError("Could not recognize partition fs type '%s'." % partition_info[5]) + raise errors.CreatorError("Could not recognize partition fs type '%s'." % + partition_info[5]) if rootpart and rootpart == line[0]: mountpoint = '/' - elif not root_mounted and fstype in ["ext2","ext3","ext4","btrfs"]: + elif not root_mounted and fstype in [ "ext2", "ext3", "ext4", "btrfs" ]: # TODO: Check that this is actually the valid root partition from /etc/fstab mountpoint = "/" root_mounted = True @@ -220,9 +203,11 @@ class RawPlugin(ImagerPlugin): else: boot = False - msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" % (size, fstype, mountpoint, boot)) + msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" % + (size, fstype, mountpoint, boot)) # TODO: add_partition should take bytes as size parameter. - imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint, fstype = fstype, boot = boot) + imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint, + fstype = fstype, boot = boot) try: imgloop.mount() diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index fe98c34..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[install] -prefix=$PREFIX @@ -1,47 +1,64 @@ #!/usr/bin/env python +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" + Main for installing mic +""" + + import os, sys import glob from distutils.core import setup -try: - import setuptools - # enable "setup.py develop", optional -except ImportError: - pass + MOD_NAME = 'mic' -version_path = 'VERSION' -if not os.path.isfile(version_path): - print 'No VERSION file in topdir, abort' - sys.exit(1) + +def check_debian(): + """--install-layout is recognized after 2.5""" + if sys.version_info[:2] > (2, 5): + if len(sys.argv) > 1 and 'install' in sys.argv: + try: + import platform + (dist, _, _) = platform.linux_distribution() + # for debian-like distros, mods will be installed to + # ${PYTHONLIB}/dist-packages + if dist in ('debian', 'Ubuntu'): + sys.argv.append('--install-layout=deb') + except AttributeError: + pass + + +def create_conf_file(): + """Apply prefix to mic.conf.in to generate actual mic.conf""" + with open('etc/mic.conf.in') as source_file: + conf_str = source_file.read() + conf_str = conf_str.replace('@PREFIX@', PREFIX) + with open(CONF_FILE, 'w') as conf_file: + conf_file.write(conf_str) + try: - # first line should be the version number - version = open(version_path).readline().strip() - if not version: - print 'VERSION file is invalid, abort' - sys.exit(1) - - ver_file = open('%s/__version__.py' % MOD_NAME, 'w') - ver_file.write("VERSION = \"%s\"\n" % version) - ver_file.close() -except IOError: - print 'WARNING: Cannot write version number file' - -# --install-layout is recognized after 2.5 -if sys.version_info[:2] > (2, 5): - if len(sys.argv) > 1 and 'install' in sys.argv: - try: - import platform - (dist, ver, id) = platform.linux_distribution() - - # for debian-like distros, mods will be installed to - # ${PYTHONLIB}/dist-packages - if dist in ('debian', 'Ubuntu'): - sys.argv.append('--install-layout=deb') - except: - pass + import mic + VERSION = mic.__version__ +except (ImportError, AttributeError): + VERSION = "dev" + +check_debian() PACKAGES = [MOD_NAME, MOD_NAME + '/utils', @@ -57,53 +74,25 @@ PACKAGES = [MOD_NAME, IMAGER_PLUGINS = glob.glob(os.path.join("plugins", "imager", "*.py")) BACKEND_PLUGINS = glob.glob(os.path.join("plugins", "backend", "*.py")) -# the following code to do a simple parse for '--prefix' opts -prefix = sys.prefix -is_next = False -for arg in sys.argv: - if is_next: - prefix = arg - break - if '--prefix=' in arg: - prefix = arg[9:] - break - elif '--prefix' == arg: - is_next = True - -# get the installation path of mic.conf -prefix = os.path.abspath(os.path.expanduser(prefix)).rstrip('/') -if prefix.lstrip('/') == 'usr': - etc_prefix = '/etc' -else: - etc_prefix = os.path.join(prefix, 'etc') - -conffile = 'etc/mic.conf' -if os.path.isfile('%s/mic/mic.conf' % etc_prefix): - conffile += '.new' - -# apply prefix to mic.conf.in to generate actual mic.conf -conf_str = file('etc/mic.conf.in').read() -conf_str = conf_str.replace('@PREFIX@', prefix) -with file(conffile, 'w') as wf: - wf.write(conf_str) +PREFIX = sys.prefix +# if real_prefix, it must be in virtualenv, use prefix as root +ROOT = sys.prefix if hasattr(sys, 'real_prefix') else '' -try: - os.environ['PREFIX'] = prefix - setup(name=MOD_NAME, - version = version, - description = 'Image Creator for Linux Distributions', - author='Jian-feng Ding, Qiang Zhang, Gui Chen', - author_email='jian-feng.ding@intel.com, qiang.z.zhang@intel.com, gui.chen@intel.com', - url='https://github.com/jfding/mic', - scripts=[ - 'tools/mic', - ], - packages = PACKAGES, - data_files = [("%s/lib/mic/plugins/imager" % prefix, IMAGER_PLUGINS), - ("%s/lib/mic/plugins/backend" % prefix, BACKEND_PLUGINS), - ("%s/mic" % etc_prefix, [conffile])] - ) -finally: - # remove dynamic file distfiles/mic.conf - os.unlink(conffile) +CONF_FILE = 'etc/mic.conf' +create_conf_file() +setup(name=MOD_NAME, + version = VERSION, + description = 'Image Creator for Linux Distributions', + author='Jian-feng Ding, Qiang Zhang, Gui Chen', + author_email='jian-feng.ding@intel.com, qiang.z.zhang@intel.com,\ + gui.chen@intel.com', + url='https://github.com/01org/mic', + scripts=[ + 'tools/mic', + ], + packages = PACKAGES, + data_files = [("%s/lib/mic/plugins/imager" % PREFIX, IMAGER_PLUGINS), + ("%s/lib/mic/plugins/backend" % PREFIX, BACKEND_PLUGINS), + ("%s/etc/mic" % ROOT, [CONF_FILE])] +) diff --git a/tests/suite.py b/tests/suite.py index 7a7475d..dea87ab 100644 --- a/tests/suite.py +++ b/tests/suite.py @@ -4,9 +4,10 @@ import unittest import test_configmgr import test_pluginmgr import test_baseimager -import test_msger +#import test_msger import test_runner import test_chroot +import test_proxy if os.getuid() != 0: raise SystemExit("Root permission is needed") @@ -15,8 +16,9 @@ suite = unittest.TestSuite() suite.addTests(test_pluginmgr.suite()) suite.addTests(test_configmgr.suite()) suite.addTests(test_baseimager.suite()) -suite.addTests(test_msger.suite()) +#suite.addTests(test_msger.suite()) suite.addTests(test_runner.suite()) suite.addTests(test_chroot.suite()) +suite.addTests(test_proxy.suite()) result = unittest.TextTestRunner(verbosity=2).run(suite) sys.exit(not result.wasSuccessful()) diff --git a/tests/test_archive.py b/tests/test_archive.py new file mode 100644 index 0000000..609e7cc --- /dev/null +++ b/tests/test_archive.py @@ -0,0 +1,692 @@ +""" +It is used to test mic/archive.py +""" + + +import os +import shutil +import unittest + +from mic import archive + + +class ArchiveTest(unittest.TestCase): + """ + test pulic methods in archive.py + """ + def setUp(self): + """Create files and directories for later use""" + self.relative_file = './sdfb.gxdf.bzws.zzz' + abs_file = '/tmp/adsdfb.gxdf.bzws.zzz' + bare_file = 'abc.def.bz.zzz' + self.relative_dir = './sdf.zzz' + abs_dir = '/tmp/asdf.zzz' + bare_dir = 'abd.zzz' + self.wrong_format_file = './sdbs.werxdf.bz.zzz' + + self.files = [self.relative_file, abs_file, bare_file] + self.dirs = [self.relative_dir, abs_dir, bare_dir] + for file_item in self.files: + os.system('touch %s' % file_item) + + for dir_item in self.dirs: + self.create_dir(dir_item) + shutil.copy(self.relative_file, '%s/1.txt' % dir_item) + shutil.copy(self.relative_file, '%s/2.txt' % dir_item) + self.create_dir('%s/dir1' % dir_item) + self.create_dir('%s/dir2' % dir_item) + + def tearDown(self): + """Clean up unuseful file and directory """ + try: + for file_item in self.files: + os.remove(file_item) + for dir_item in self.dirs: + shutil.rmtree(dir_item, ignore_errors=True) + except OSError: + pass + + def create_dir(self, dir_name): + """Create directories and ignore any erros """ + try: + os.makedirs(dir_name) + except OSError: + pass + + def test_get_compress_formats(self): + """Test get compress format """ + compress_list = archive.get_compress_formats() + compress_list.sort() + self.assertEqual(compress_list, ['bz2', 'gz', 'lzo']) + + def test_compress_negtive_file_path_is_required(self): + """Test if the first parameter: file path is empty""" + self.assertRaises(OSError, archive.compress, '', 'bz2') + #with self.assertRaises(OSError): + # archive.compress('', 'bz2') + + def test_compress_negtive_compress_format_is_required(self): + """Test if the second parameter: compress format is empty""" + self.assertRaises(ValueError, archive.compress, + self.relative_file, '') + #with self.assertRaises(ValueError): + # archive.compress(self.relative_file, '') + + def test_compress_negtive_parameters_are_all_required(self): + """Test if two parameters are both empty""" + self.assertRaises(OSError, archive.compress, '', '') + #with self.assertRaises(OSError): + # archive.compress('', '') + + def test_compress_negtive_file_not_exist(self): + """Test target file does not exist""" + self.assertRaises(OSError, archive.compress, 'a.py', 'bz2') + #with self.assertRaises(OSError): + # archive.compress('a.py', 'bz2') + + def test_compress_negtive_file_is_dir(self): + """Test target is one direcoty, which is not supported""" + self.assertRaises(OSError, archive.compress, + self.relative_dir, 'bz2') + # with self.assertRaises(OSError): + # archive.compress(self.relative_dir, 'bz2') + + def test_compress_negtive_wrong_compress_format(self): + """Test wrong compress format""" + self.assertRaises(ValueError, archive.compress, + self.relative_file, 'bzip2') + #with self.assertRaises(ValueError): + # archive.compress(self.relative_file, 'bzip2') + + def _compress_negtive_gz_command_not_exists(self): + #TODO: test if command like 'pigz', 'gzip' does not exist + pass + + def _compress_negtive_lzo_command_not_exists(self): + #TODO: test if command 'lzop' does not exist + pass + + def _compress_negtive_bz2_command_not_exists(self): + #TODO: test if command like 'pbzip2', 'bzip2' does not exist + pass + + def test_compress_gz(self): + """Test compress format: gz""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def test_compress_bz2(self): + """Test compress format: bz2""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def _test_compress_lzo(self): + """Test compress format: lzo""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def test_decompress_negtive_file_path_is_required(self): + """Test if the first parameter: file to be uncompressed is empty""" + self.assertRaises(OSError, archive.compress, '', 'bz') + #with self.assertRaises(OSError): + # archive.decompress('', 'bz') + + def test_decompress_compress_format_is_empty(self): + """Test if the second parameter: compress format is empty string""" + output_name = archive.compress(self.relative_file, 'gz') + self.assertEqual('%s.gz' % self.relative_file, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(self.relative_file)) + archive.decompress(output_name, '') + self.assertTrue(os.path.exists(self.relative_file)) + + def test_decompress_negtive_parameters_are_empty(self): + """Test if two parameters are both empty string""" + self.assertRaises(OSError, archive.decompress, '', '') + #with self.assertRaises(OSError): + # archive.decompress('', '') + + def test_decompress_negtive_file_not_exist(self): + """Test decompress target does not exist""" + self.assertRaises(OSError, archive.decompress, + 'tresa.py', 'bz2') + #with self.assertRaises(OSError): + # archive.decompress('tresa.py', 'bz2') + + def test_decompress_negtive_path_is_dir(self): + """Test decompress target is a directory""" + self.assertRaises(OSError, archive.decompress, + self.relative_dir, 'bz2') + #with self.assertRaises(OSError): + # archive.decompress(self.relative_dir, 'bz2') + + def _decompress_negtive_not_corresponding(self): + # TODO: test if path is .lzo, but given format is bz2 + pass + + def test_decompress_negtive_wrong_compress_format(self): + """Test wrong decompress format""" + self.assertRaises(ValueError, archive.decompress, + self.relative_file, 'bzip2') + #with self.assertRaises(ValueError): + # archive.decompress(self.relative_file, 'bzip2') + + def test_decompress_negtive_wrong_file_format(self): + """Test wrong target format""" + self.assertRaises(Exception, archive.decompress, + self.wrong_format_file, 'bz2') + #with self.assertRaises(Exception): + # archive.decompress(self.wrong_format_file, 'bz2') + + def test_decompress_gz(self): + """Test decompress + Format: gz + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'gz') + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_gz_no_compress_format(self): + """Test decompress + Format: gz + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_bz2(self): + """Test decompress + Format: bz2 + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'bz2') + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_bz2_no_compress_format(self): + """Test decompress + Format: bz2 + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def _test_decompress_lzo(self): + """Test decompress + Format: lzo + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'lzo') + self.assertTrue(os.path.exists(file_item)) + + def _test_decompress_lzo_no_compress_format(self): + """Test decompress + Format: lzo + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def test_get_archive_formats(self): + """Test get archive format""" + archive_formats = archive.get_archive_formats() + archive_formats.sort() + self.assertEqual(archive_formats, + ["bztar", "gztar", "lzotar", "tar", 'zip']) + + def test_get_archive_suffixes(self): + """Test get archive suffixes""" + archive_suffixes = archive.get_archive_suffixes() + archive_suffixes.sort() + + self.assertEqual(archive_suffixes, + ['.tar', '.tar.bz', '.tar.bz2', '.tar.gz', '.tar.lzo', + '.taz', '.tbz', '.tbz2', '.tgz', '.tzo', '.zip']) + + def test_make_archive_negtive_archive_name_is_required(self): + """Test if first parameter: file path is empty""" + self.assertRaises(Exception, archive.make_archive, + '', self.relative_dir) + #with self.assertRaises(Exception): + # archive.make_archive('', self.relative_dir) + + def test_extract_archive_negtive_archive_name_is_required(self): + """Test if first parameter: file path is empty""" + self.assertRaises(Exception, archive.extract_archive, + '', self.relative_dir) + #with self.assertRaises(Exception): + # archive.extract_archive('', self.relative_dir) + + def test_make_archive_negtive_target_name_is_required(self): + """Test if second parameter: target name is empty""" + self.assertRaises(Exception, archive.make_archive, 'a.zip', '') + #with self.assertRaises(Exception): + # archive.make_archive('a.zip', '') + + def _extract_archive_negtive_target_name_is_required(self): + # Not sure if the current dir will be used ? + # TODO: + pass + + def test_make_archive_negtive_parameters_are_empty(self): + """Test if both parameters are empty""" + self.assertRaises(Exception, archive.make_archive, '', '') + #with self.assertRaises(Exception): + # archive.make_archive('', '') + + def test_extract_archive_negtive_parameters_are_empty(self): + """Test if both parameters are empty""" + self.assertRaises(Exception, archive.extract_archive, '', '') + #with self.assertRaises(Exception): + # archive.extract_archive('', '') + + def test_make_archive_negtive_target_path_not_exists(self): + """Test if file path does not exist""" + fake_file = 'abcdfsdf' + self.assertRaises(Exception, archive.make_archive, + 'a.tar', fake_file) + #with self.assertRaises(Exception): + # archive.make_archive('a.tar', fake_file) + + self.assertRaises(Exception, archive.make_archive, + 'a.zip', fake_file) + #with self.assertRaises(Exception): + # archive.make_archive('a.zip', fake_file) + + def test_extract_archive_negtive_path_not_exists(self): + """Test if file path does not exist""" + fake_file = 'abcdfsdf' + self.assertRaises(Exception, archive.extract_archive, + fake_file, self.relative_dir) + #with self.assertRaises(Exception): + # archive.extract_archive(fake_file, self.relative_dir) + + def test_extract_archive_negtive_target_is_file(self): + """Test if the extract target is file""" + out_file = '%s.tar' % self.relative_dir + self.assertTrue(archive.make_archive(out_file, self.relative_dir)) + self.assertTrue(os.path.exists(out_file)) + self.assertRaises(Exception, archive.extract_archive, + out_file, self.relative_file) + #with self.assertRaises(Exception): + # archive.extract_archive(out_file, self.relative_file) + os.remove(out_file) + + def test_make_archive_wrong_format(self): + """Test wrong make_archive format""" + self.assertRaises(Exception, archive.make_archive, + 'a.sfsfrwe', self.relative_dir) + #with self.assertRaises(Exception): + # archive.make_archive('a.sfsfrwe', self.relative_dir) + + def test_make_archive_tar_with_different_name(self): + """ Test make_archive format: tar + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abcd.tar' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar(self): + """ Test make_archive format: tar""" + for item in self.files + self.dirs: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_extract_archive_tar(self): + """ Test extract format: tar""" + for item in self.files: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join( + out_dir, + os.path.basename(item)))) + shutil.rmtree(out_dir) + + for item in self.dirs: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, '1.txt'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, '2.txt'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, 'dir1'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, 'dir2'))) + shutil.rmtree(out_dir) + + def test_make_archive_zip_with_different_name(self): + """ Test make_archive format: zip + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'a.zip' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_zip(self): + """ Test make_archive format: zip""" + for item in self.files + self.dirs: + out_file = '%s.zip' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_zip(self): + """ Test extract archive format: zip""" + for item in self.files + self.dirs: + out_file = '%s.zip' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def _test_make_archive_tzo_with_different_name(self): + """ Test make_archive format: tzo + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abc.tzo' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _test_make_archive_tzo(self): + """ Test make_archive format: tzo""" + for item in self.files + self.dirs: + out_file = '%s.tzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tzo(self): + """ Test extract format: tzo""" + for item in self.files + self.dirs: + out_file = '%s.tzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def _test_make_archive_tar_lzo_with_different_name(self): + """ Test make_archive format: lzo + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abc.tar.lzo' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _test_make_archive_tar_lzo(self): + """ Test make_archive format: lzo""" + for item in self.files + self.dirs: + out_file = '%s.tar.lzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_lzo(self): + """ Test extract_archive format: lzo""" + for item in self.files + self.dirs: + out_file = '%s.tar.lzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_taz_with_different_name(self): + """ Test make_archive format: taz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abcd.taz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_taz(self): + """ Test make_archive format: taz""" + for item in self.files + self.dirs: + out_file = '%s.taz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_taz(self): + """ Test extract archive format: taz""" + for item in self.files + self.dirs: + out_file = '%s.taz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tgz_with_different_name(self): + """ Test make_archive format: tgz + It packs the source with anotehr name""" + for item in self.files + self.dirs: + out_file = 'abc.tgz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tgz(self): + """ Test make_archive format: tgz""" + for item in self.files + self.dirs: + out_file = '%s.tgz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tgz(self): + """ Test extract archive format: tgz""" + for item in self.files + self.dirs: + out_file = '%s.tgz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_gz_with_different_name(self): + """ Test make_archive format: tar.gz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'erwe.tar.gz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_gz(self): + """ Test make_archive format: tar.gz""" + for item in self.files + self.dirs: + out_file = '%s.tar.gz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_gz(self): + """ Test extract archive format: tar.gz""" + for item in self.files + self.dirs: + out_file = '%s.tar.gz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tbz_with_different_name(self): + """ Test make_archive format: tbz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'sdfsd.tbz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tbz(self): + """ Test make_archive format: tbz""" + for item in self.files + self.dirs: + out_file = '%s.tbz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tbz(self): + """ Test extract format: tbz""" + for item in self.files + self.dirs: + out_file = '%s.tbz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tbz2_with_different_name(self): + """ Test make_archive format: tbz2 + It packs source with another name""" + for item in self.files + self.dirs: + out_file = 'sfsfd.tbz2' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tbz2(self): + """ Test make_archive format: tbz2""" + for item in self.files + self.dirs: + out_file = '%s.tbz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tbz2(self): + """ Test extract format: tbz2""" + for item in self.files + self.dirs: + out_file = '%s.tbz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_bz_with_different_name(self): + """ Test make_archive format: tar.bz + It packs source with antoher name""" + for item in self.files + self.dirs: + out_file = 'sdf.tar.bz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_bz(self): + """ Test make_archive format: tar.bz""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_bz(self): + """ Test extract format: tar.bz""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_bz2_with_different_name(self): + """ Test make_archive format: tar.bz2 + it packs the source with another name """ + for item in self.files + self.dirs: + out_file = 'df.tar.bz2' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_bz2(self): + """ Test make_archive format: tar.bz2""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_bz2(self): + """ Test extract format: tar.bz2""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_configmgr.py b/tests/test_configmgr.py index bc2e4a4..2f15f10 100644 --- a/tests/test_configmgr.py +++ b/tests/test_configmgr.py @@ -33,7 +33,7 @@ class ConfigMgrTest(unittest.TestCase): os.makedirs(CACHEDIR) self.configmgr.create['cachedir'] = CACHEDIR self.level = msger.get_loglevel() - msger.set_loglevel('quiet') + msger.set_loglevel('RAWTEXT') def tearDown(self): msger.set_loglevel(self.level) @@ -68,7 +68,8 @@ class ConfigMgrTest(unittest.TestCase): self.configmgr._ksconf = KSCONF self.assertTrue(isinstance(self.configmgr.create['ks'], KickstartParser)) #self.assertEqual(self.configmgr.create['name'], 'test') - self.assertDictEqual(repomd[0], self.configmgr.create['repomd'][0]) + #self.assertDictEqual(repomd[0], self.configmgr.create['repomd'][0]) + self.assertEqual(repomd[0], self.configmgr.create['repomd'][0]) self.assertEqual(self.configmgr.create['arch'], 'i686') if __name__ == "__main__": diff --git a/tests/test_msger.py b/tests/test_msger.py deleted file mode 100644 index be57b2f..0000000 --- a/tests/test_msger.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import StringIO -import unittest -from mic import msger - -def suite(): - return unittest.makeSuite(MsgerTest) - -class MsgerTest(unittest.TestCase): - - def setUp(self): - self.stdout = sys.stdout - self.stderr = sys.stderr - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - msger.set_loglevel('normal') - self.loglevel = msger.LOG_LEVEL - - def tearDown(self): - msger.LOG_LEVEL = self.loglevel - sys.stdout = self.stdout - sys.stderr = self.stderr - - def testRaw(self): - excepted = "hello\n" - msger.raw("hello") - self.assertEqual(excepted, sys.stdout.getvalue()) - - def testInfo(self): - excepted = "Info: hello\n" - msger.info("hello") - self.assertEqual(excepted, sys.stdout.getvalue()) - - def testWarning(self): - excepted = "Warning: hello\n" - msger.warning("hello") - self.assertEqual(excepted, sys.stderr.getvalue()) - - def testVerbose(self): - excepted = "Verbose: hello\n" - msger.verbose("hello") - self.assertEqual("", sys.stdout.getvalue()) - msger.set_loglevel("verbose") - msger.verbose("hello") - self.assertEqual(excepted, sys.stdout.getvalue()) - - def testDebug(self): - excepted = "Debug: hello\n" - msger.debug("hello") - self.assertEqual("", sys.stdout.getvalue()) - msger.set_loglevel("debug") - msger.debug("hello") - self.assertEqual(excepted, sys.stderr.getvalue()) - - def testLogstderr(self): - excepted = "hello\n" - cwd = os.getcwd() - msger.enable_logstderr(cwd + "/__tmp_err.log") - print >>sys.stderr, "hello" - msger.disable_logstderr() - self.assertEqual(excepted, sys.stderr.getvalue()) - - def testLoglevel(self): - # test default value - self.assertEqual("normal", msger.get_loglevel()) - # test no effect value - msger.set_loglevel("zzzzzz") - self.assertEqual("normal", msger.get_loglevel()) - # test effect value - msger.set_loglevel("verbose") - self.assertEqual("verbose", msger.get_loglevel()) - msger.set_loglevel("debug") - self.assertEqual("debug", msger.get_loglevel()) - msger.set_loglevel("quiet") - self.assertEqual("quiet", msger.get_loglevel()) - -if __name__ == "__main__": - unittest.main() - diff --git a/tests/test_pluginmgr.py b/tests/test_pluginmgr.py index 6263ee9..aad4b5e 100644 --- a/tests/test_pluginmgr.py +++ b/tests/test_pluginmgr.py @@ -43,7 +43,7 @@ class PluginMgrTest(unittest.TestCase): self.plugin._add_plugindir(noexistdir) warn = "Warning: Plugin dir is not a directory or does not exist: " \ "%s\n" % noexistdir - self.assertEqual(sys.stderr.getvalue(), warn) + #self.assertEqual(sys.stderr.getvalue(), warn) def testBackendPlugins(self): expect = ['zypptest', 'yumtest'] diff --git a/tests/test_proxy.py b/tests/test_proxy.py new file mode 100644 index 0000000..9655b58 --- /dev/null +++ b/tests/test_proxy.py @@ -0,0 +1,34 @@ +#!/usr/bin/python + +import unittest +from mic.utils import proxy + +def suite(): + return unittest.makeSuite(ProxyTest) + +class ProxyTest(unittest.TestCase): + + def test_proxy(self): + proxy.set_proxies('http://proxy.some.com:11', '1.2.3.4') + self.assertEqual(proxy.get_proxy_for('http://1.2.3.4'), None) + self.assertEqual(proxy.get_proxy_for('http://download.tizen.org'), 'http://proxy.some.com:11') + + proxy.set_proxies('http://proxy.some.com:11', 'download.am.org') + self.assertEqual(proxy.get_proxy_for('http://download.am.org'), None) + self.assertEqual(proxy.get_proxy_for('https://download.am.org'), None) + self.assertEqual(proxy.get_proxy_for('http://download.tizen.org'), 'http://proxy.some.com:11') + + proxy.set_proxies('http://proxy.some.com:11', '1.2.3.0/24') + self.assertEqual(proxy.get_proxy_for('http://1.2.3.4'), None) + self.assertEqual(proxy.get_proxy_for('http://1.2.3.0'), None) + self.assertEqual(proxy.get_proxy_for('http://1.2.3.255'), None) + self.assertEqual(proxy.get_proxy_for('http://download.tizen.org'), 'http://proxy.some.com:11') + + proxy.set_proxies('http://proxy.some.com:11', '.hello.com') + self.assertEqual(proxy.get_proxy_for('http://linux.hello.com'), None) + self.assertEqual(proxy.get_proxy_for('http://linux.hello.com.org'), 'http://proxy.some.com:11') + + + +if __name__ == "__main__": + unittest.main() @@ -1,243 +1,288 @@ #!/usr/bin/env python + +#Copyright (c) 2011 Intel, Inc. # -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License +#This program is free software; you can redistribute it and/or modify it +#under the terms of the GNU General Public License as published by the Free +#Software Foundation; version 2 of the License # -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +#This program is distributed in the hope that it will be useful, but +#WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys, errno -from mic import msger, creator -from mic.utils import cmdln, misc, errors +# +# pylint: disable-msg=E0611, E1101, R0201 +# E0611: no name in module, some attributes are set during running, so ignore it +# E1101: %s %r has no %r member, some attributes are set during running, +# so ignore it +# R0201: Method could be a function + +""" + This mudule is entry for mic. + It defines a class named MicCmd inheriting Cmdln, and supplies interfaces like + 'create, chroot, convert' and also some parameters for command 'mic'. +""" +import os +import sys +import errno + +from argparse import ArgumentParser, SUPPRESS + +from mic import msger, __version__ as VERSION +from mic.utils import misc, errors from mic.conf import configmgr from mic.plugin import pluginmgr -from mic.__version__ import VERSION +from mic.helpformat import MICHelpFormatter, subparser + +@subparser +def chroot_parser(parser): + """chroot into an image -def optparser_setup(func): - """Setup optparser for a function""" - if not hasattr(func, "optparser"): - func.optparser = cmdln.SubCmdOptionParser() - func.optparser.disable_interspersed_args() - return func - - -class MicCmd(cmdln.Cmdln): + Examples: + mic chroot platform.img + mic chroot platform.img ls """ - Usage: mic SUBCOMMAND [OPTS] [ARGS...] - - mic Means the Image Creation tool - Try 'mic help SUBCOMMAND' for help on a specific subcommand. - - ${command_list} - global ${option_list} - ${help_list} + parser.add_argument('imagefile', help='Path of image file') + parser.add_argument('-s', '--saveto', action = 'store', dest = 'saveto', default = None, + help = "Save the unpacked image to specified dir") + parser.add_argument('-c', '--cmd', dest = 'cmd', default = None, + help = "command which will be executed in chroot environment") + parser.set_defaults(alias="ch") + return parser + +@subparser +def create_parser(parser): + """create an image + Examples: + $ mic -d -v create auto handset_blackbay.ks + $ mic -d -v cr loop handset_blackbay.ks --logfile=mic.log """ - name = 'mic' - version = VERSION - - def print_version(self): - msger.raw("%s %s (%s)" % (self.name, - self.version, - misc.get_distro_str())) - - def get_optparser(self): - optparser = cmdln.CmdlnOptionParser(self, version=self.version) - # hook optparse print_version here - optparser.print_version = self.print_version - optparser.add_option('-d', '--debug', action='store_true', - dest='debug', - help='print debug message') - optparser.add_option('-v', '--verbose', action='store_true', - dest='verbose', - help='verbose information') - return optparser - - def postoptparse(self): - if self.options.verbose: - msger.set_loglevel('verbose') - - if self.options.debug: - try: - import rpm - rpm.setVerbosity(rpm.RPMLOG_NOTICE) - except ImportError: - pass - - msger.set_loglevel('debug') - - self.print_version() - - def help_create(self): - cr = creator.Creator() - cr.optparser = cr.get_optparser() - doc = cr.__doc__ - doc = cr._help_reindent(doc) - doc = cr._help_preprocess(doc, None) - doc = doc.replace(cr.name, "${cmd_name}", 1) - doc = doc.rstrip() + '\n' - return doc - - @cmdln.alias("cr") - def do_create(self, argv): - cr = creator.Creator() - cr.main(argv[1:]) - - def _root_confirm(self): - if os.geteuid() != 0: - msger.error('Root permission is required to continue, abort') - - @cmdln.alias("cv") - @cmdln.option("-S", "--shell", - action="store_true", dest="shell", default=False, - help="Launch shell before packaging the converted image") - def do_convert(self, subcmd, opts, *args): - """${cmd_name}: convert image format - - Usage: - mic convert <imagefile> <destformat> - - ${cmd_option_list} + parent_parser = ArgumentParser(add_help=False) + parent_parser.add_argument('ksfile', help='Path of ksfile'); + parent_parser.add_argument('--logfile', dest='logfile', default=None, + help='Path of logfile') + parent_parser.add_argument('-c', '--config', dest='config', default=None, + help='Specify config file for mic') + parent_parser.add_argument('-k', '--cachedir', action='store', + dest='cachedir', default=None, + help='Cache directory to store the downloaded') + parent_parser.add_argument('-o', '--outdir', action='store', dest='outdir', + default=None, help='Output directory') + parent_parser.add_argument('-A', '--arch', dest='arch', default=None, + help='Specify repo architecture') + parent_parser.add_argument('--release', dest='release', default=None, metavar='RID', + help='Generate a release of RID with all necessary' + ' files, when @BUILD_ID@ is contained in ' + 'kickstart file, it will be replaced by RID') + parent_parser.add_argument("--record-pkgs", dest="record_pkgs", default=None, + help='Record the info of installed packages, ' + 'multiple values can be specified which ' + 'joined by ",", valid values: "name", ' + '"content", "license", "vcs"') + parent_parser.add_argument('--pkgmgr', dest='pkgmgr', default=None, + help='Specify backend package manager') + parent_parser.add_argument('--local-pkgs-path', dest='local_pkgs_path', default=None, + help='Path for local pkgs(rpms) to be installed') + parent_parser.add_argument('--runtime', dest='runtime', default=None, + help='Specify runtime mode, avaiable: bootstrap') + # --taring-to is alias to --pack-to + parent_parser.add_argument('--taring-to', dest='pack_to', default=None, + help=SUPPRESS) + parent_parser.add_argument('--pack-to', dest='pack_to', default=None, + help='Pack the images together into the specified' + ' achive, extension supported: .zip, .tar, ' + '.tar.gz, .tar.bz2, etc. by default, .tar ' + 'will be used') + parent_parser.add_argument('--copy-kernel', action='store_true', dest='copy_kernel', + help='Copy kernel files from image /boot directory' + ' to the image output directory.') + parent_parser.add_argument('--install-pkgs', action='store', dest='install_pkgs', default=None, + help='Specify what type of packages to be installed,' + ' valid: source, debuginfo, debugsource') + parent_parser.add_argument('--check-pkgs', action='store', dest='check_pkgs', default=[], + help='Check if given packages would be installed, ' + 'packages should be separated by comma') + parent_parser.add_argument('--tmpfs', action='store_true', dest='enabletmpfs', + help='Setup tmpdir as tmpfs to accelerate, experimental' + ' feature, use it if you have more than 4G memory') + parent_parser.add_argument('--repourl', action='append', dest='repourl', default=[], + help=SUPPRESS) + parent_parser.add_argument('-R', '--repo', action='append', + dest='repo', default=[], + help=SUPPRESS) + parent_parser.add_argument('--ignore-ksrepo', action='store_true', + dest='ignore_ksrepo', default=False, + help=SUPPRESS) + parent_parser.add_argument('--strict-mode', action='store_true', + dest='strict_mode', default=False, + help='Abort creation of image, if there are some errors' + ' during rpm installation. ') + parser.set_defaults(alias="cr") + + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + auto_parser = subparsers.add_parser('auto', parents=[parent_parser], help='auto detect image type from magic header') + + fs_parser = subparsers.add_parser('fs', parents=[parent_parser], + help='create fs image') + fs_parser.add_argument("--include-src", dest = "include_src",action = "store_true", + default = False, help = "Generate a image with source rpms included") + + loop_parser = subparsers.add_parser('loop', parents=[parent_parser], help='create loop image') + + loop_parser.add_argument("--compress-disk-image", dest="compress_image", + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + # alias to compress-image for compatibility + loop_parser.add_argument("--compress-image", dest="compress_image", + choices=("gz", "bz2"), default=None, + help="Compress all loop images with 'gz' or 'bz2'") + loop_parser.add_argument("--shrink", action='store_true', default=False, + help="Whether to shrink loop images to minimal size") + + qcow_parser = subparsers.add_parser('qcow', parents=[parent_parser], help='create qcow image') + + raw_parser = subparsers.add_parser('raw', parents=[parent_parser], help='create raw image') + + raw_parser.add_argument("--compress-disk-image", dest="compress_image", + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + raw_parser.add_argument("--compress-image", dest="compress_image", + choices=("gz", "bz2"), default = None, + help="Compress all raw images before package") + raw_parser.add_argument("--generate-bmap", action="store_true", default = None, + help="also generate the block map file") + raw_parser.add_argument("--fstab-entry", dest="fstab_entry", choices=("name", "uuid"), default="uuid", + help="Set fstab entry, 'name' means using device names, " + "'uuid' means using filesystem uuid") + return parser + +def main(argv): + """Script entry point.""" + + def print_version(): + """log name, verion, hostname""" + + name = 'mic' + msger.raw("%s %s (%s)" % (name, + VERSION, + misc.get_hostname_distro_str())) + + def has_parameter(arg, arglist): """ - - if not args: - # print help - handler = self._get_cmd_handler('convert') - if hasattr(handler, "optparser"): - handler.optparser.print_help() - return 1 - - if len(args) == 1: - raise errors.Usage("It need 2 arguments (1 given)") - elif len(args) == 2: - (srcimg, destformat) = args - else: - raise errors.Usage("Extra argument given") - - if not os.path.exists(srcimg): - raise errors.CreatorError("Cannot find the image: %s" % srcimg) - - self._root_confirm() - - configmgr.convert['shell'] = opts.shell - - srcformat = misc.get_image_type(srcimg) - if srcformat == "ext3fsimg": - srcformat = "loop" - - srcimager = None - destimager = None - for iname, icls in pluginmgr.get_plugins('imager').iteritems(): - if iname == srcformat and hasattr(icls, "do_unpack"): - srcimager = icls - if iname == destformat and hasattr(icls, "do_pack"): - destimager = icls - - if (srcimager and destimager) is None: - raise errors.CreatorError("Can't convert from %s to %s" \ - % (srcformat, destformat)) - - else: - maptab = { - "livecd": "iso", - "liveusb": "usbimg", - "loop": "img", - } - - if destformat in maptab: - imgname = os.path.splitext(os.path.basename(srcimg))[0] - dstname = "{0}.{1}".format(imgname, maptab[destformat]) - - if os.path.exists(dstname): - if msger.ask("Converted image %s seems existed, " - "remove and continue?" % dstname): - os.unlink(dstname) - else: - raise errors.Abort("Canceled") - - base_on = srcimager.do_unpack(srcimg) - destimager.do_pack(base_on) - - @cmdln.alias("ch") - @cmdln.option('-s', '--saveto', - action='store', dest='saveto', default=None, - help="Save the unpacked image to specified dir") - @optparser_setup - def do_chroot(self, subcmd, opts, *args): - """${cmd_name}: chroot into an image - - Usage: - mic chroot [options] <imagefile> [command [arg]...] - - ${cmd_option_list} + Helper function. + Check if argument requires parameter by analyzing + its action. Parameter is required only for 'store' and 'append' actions """ + if arg.startswith('-'): + for args in arglist: + if arg in (args['short'], args['long']): + if args.get('action') in (None, 'store', 'append'): + return True + return False + + # Create top level parser + epilog = "Try 'mic SUBCOMMAND --help' for help on a specific subcommand." + description = "mic - the Image Creation tool" + parser = ArgumentParser(description=description, epilog=epilog, + formatter_class=MICHelpFormatter) + + # List of global arguments + # The main purpose of this structure is to contain arguments + # of add_argument. This is used to do aliasing properly + # (see code under the comment 'replace aliases with real commands') + global_args = [{'short': '-V', 'long': '--version', 'action': 'version', + 'version': '%(prog)s ' + VERSION}, + {'short': '-d', 'long': '--debug', 'action': 'store_true', + 'help': 'debug output'}, + {'short': '-v', 'long': '--verbose', 'action': 'store_true', + 'help': 'verbose output'}, + {'short': '-i', 'long': '--interactive', 'action': 'store_true', + 'dest': 'interactive', 'default': 'True', 'help': 'interactive output'}, + {'short': '', 'long': '--non-interactive', 'action': 'store_false', + 'dest': 'interactive', 'default': 'True', 'help': 'non-interactive output'}, + ] + + for args in global_args: + parser_kwargs = {} + for key in ('action', 'help', 'version', 'default', 'dest'): + if key in args: + parser_kwargs[key] = args[key] + + if args['short'] is '': + parser.add_argument(args['long'], **parser_kwargs) + else: + parser.add_argument(args['short'], args['long'], **parser_kwargs) + + # hacked by the request of cmdln lovers + parser.format_usage = parser.format_help + + # Create parsers for subcommands + subparsers = parser.add_subparsers(title='subcommands') + + # collect aliases + aliases = {} + for name, obj in globals().iteritems(): + if name.endswith('_parser') and callable(obj): + aliases[obj(subparsers).get_default('alias')] = name.split('_')[0] + + # replace aliases with real commands + for i, arg in enumerate(argv[1:]): + if not arg.startswith('-'): + # argv[i] is previous argument to arg + if not has_parameter(argv[i], global_args) and arg in aliases: + argv[i+1] = aliases[arg] + break - if not args: - # print help - handler = self._get_cmd_handler('chroot') - if hasattr(handler, "optparser"): - handler.optparser.print_help() - return 1 - - targetimage = args[0] - - if not os.path.exists(targetimage): - raise errors.CreatorError("Cannot find the image: %s" - % targetimage) + # Parse arguments + args = parser.parse_args(argv[1:]) - self._root_confirm() + if args.interactive: + msger.enable_interactive() + else: + msger.disable_interactive() - configmgr.chroot['saveto'] = opts.saveto + if args.verbose: + msger.set_loglevel('VERBOSE') - imagetype = misc.get_image_type(targetimage) - if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"): - imagetype = "loop" + if args.debug: + try: + import rpm + rpm.setVerbosity(rpm.RPMLOG_NOTICE) + except ImportError: + pass - chrootclass = None - for pname, pcls in pluginmgr.get_plugins('imager').iteritems(): - if pname == imagetype and hasattr(pcls, "do_chroot"): - chrootclass = pcls - break + msger.set_loglevel('DEBUG') - if not chrootclass: - raise errors.CreatorError("Cannot support image type: %s" \ - % imagetype) + print_version() - chrootclass.do_chroot(targetimage, args[1:]) + # Import target module and call 'main' from it + module = __import__("mic.%s" % args.module, fromlist=[args.module]) + return module.main(parser, args, argv[1:]) + if __name__ == "__main__": try: - mic = MicCmd() - sys.exit(mic.main()) - + sys.exit(main(sys.argv)) except KeyboardInterrupt: msger.error('\n^C catched, program aborted.') - - # catch 'no space left' exception, etc - except IOError, e: - if e.errno == errno.ENOSPC: + except IOError as ioerr: + # catch 'no space left' exception, etc + if ioerr.errno == errno.ENOSPC: msger.error('\nNo space left on device') raise - - except errors.Usage, usage: + except errors.Usage as usage: msger.error(str(usage)) - - except errors.Abort, msg: + except errors.Abort as msg: msger.info(str(msg)) - - except errors.CreatorError, err: - if msger.get_loglevel() == 'debug': + except errors.CreatorError as err: + if msger.get_loglevel() == 'DEBUG': import traceback msger.error(traceback.format_exc()) else: - msger.error('\n'+str(err)) + msger.error(str(err)) |