diff options
author | SoonKyu Park <sk7.park@samsung.com> | 2016-04-11 15:24:24 +0900 |
---|---|---|
committer | SoonKyu Park <sk7.park@samsung.com> | 2016-04-11 15:24:24 +0900 |
commit | 89453d58de729caad7613a007b5dda9c19e7d003 (patch) | |
tree | 40741dc528b6a5f546a4287124f8c523324e5910 | |
parent | 9312c04f58c63883aa556124e618c16e531519a3 (diff) | |
parent | 97a7db20d2297c9f66716057c20f19265751bfa7 (diff) | |
download | mic-89453d58de729caad7613a007b5dda9c19e7d003.tar.gz mic-89453d58de729caad7613a007b5dda9c19e7d003.tar.bz2 mic-89453d58de729caad7613a007b5dda9c19e7d003.zip |
Handle conflicts from git merge release-20160315
47 files changed, 2428 insertions, 6089 deletions
@@ -1,17 +1,38 @@ -Release 0.24.4 - Thu Jan 22 2015 - Lihong Sun <lihongx.sun@intel.com> +Release 0.27 - Mon Mar 28 2016 - Jianzhong Fang <jz.fang@samsung.com> ===================================================================== + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, Fedora 23, openSUSE 13.2 * generate manifest file to describe image information - * update mount option to make it more general - * bug fix: - - fix incorrect logfile location of '--release' if creation failed - - update dependency package for mic - - add catching 'distribution not found' exception + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum + * drop mic-native support + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support + * use argparse module to parse the cmd line -Release 0.24.3 - Mon Nov 24 2014 - Yongfeng Du <dolpher.du@intel.com> -===================================================================== - * bug fix: - - fix qemu arm and arm64 issues - - remove --preserve-order option in taring fs image + * bug fix: + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure + - fix qemu arm and arm64 issues + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive Release 0.24 - Tue Mar 11 2014 - Gui Chen <gui.chen@intel.com> ===================================================================== @@ -8,7 +8,7 @@ The tool offers three major functions: - image creation - image conversion bwtween two different formats -- chrooting into an image +- chrooting into an image With the MIC tool, users can create different types of images for different verticals, including live CD images, live USB images, raw images for KVM, @@ -20,7 +20,6 @@ image format, a very useful function for those sensitive to image format. It can also provide plugin mechanism for developers to expand image type or image options, and even to hook. - Resource -------- * SITE: https://www.tizen.org/ @@ -35,7 +34,7 @@ Resource License ------- MIC is Open Source and is distributed under the GPLv2 License. -Please see the COPYING file included with this software +Please see the COPYING file included with this software Contacts diff --git a/debian/changelog b/debian/changelog index f4e487d..38f28d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,26 +1,38 @@ -mic (0.24.4) unstable; urgency=low +mic (0.27) unstable; urgency=low + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, Fedora 23, openSUSE 13.2 * generate manifest file to describe image information - * update mount option to make it more general + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum + * drop mic-native support + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support + * use argparse module to parse the cmd line * bug fix: - - fix incorrect logfile location of '--release' if creation failed - - update dependency package for mic - - add catching 'distribution not found' exception - -- Lihong Sun <lihongx.sun@intel.com> Thu, 22 Jan 2015 17:25:35 +0800 - -mic (0.24.3) unstable; urgency=low - * bug fix: - - fix qemu arm and arm64 issues - - remove --preserve-order option in taring fs image - - -- Yongfeng Du <dolpher.du@intel.com> Mon, 24 Nov 2014 17:25:35 +0800 - -mic (0.24.2) unstable; urgency=low - * enable ssl_verify in config file - * fix logfile incompleted issue - * fix md5sum to be compatible with utility md5sum - * fix locale issue in systemd service - - -- Gui Chen <gui.chen@intel.com> Wed, 09 Jul 2014 17:25:35 +0800 + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure + - fix qemu arm and arm64 issues + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive + + -- Jianzhong Fang <jz.fang@samsung.com> Mon, 28 Mar 2016 19:50:35 +0800 mic (0.24) unstable; urgency=low * enhance to handle password with special characters @@ -39,7 +51,7 @@ mic (0.24) unstable; urgency=low - fix zypp failed to download by changing 'cachedir' - fix 'mkfs' not working caused by mic-bootstrap install failed - -- Gui Chen <gui.chen@intel.com> Thu, 24 Oct 2013 17:25:35 +0800 + -- Gui Chen <gui.chen@intel.com> Tue, 11 Mar 2014 17:25:35 +0800 mic (0.23-1) unstable; urgency=low * new distribution support: Ubuntu 13.10 and OpenSUSE 13.1 @@ -54,7 +66,7 @@ mic (0.23-1) unstable; urgency=low - fix detailed error messages missing in mounting - fix version comparing issue of urlgrabber in Fedora - -- Gui Chen <gui.chen@intel.com> Thu, 24 Oct 2013 17:25:35 +0800 + -- Gui Chen <gui.chen@intel.com> Fri, 12 Dec 2013 17:25:35 +0800 mic (0.22-1) unstable; urgency=low * use __version__ variable instead of VERSION file diff --git a/debian/control b/debian/control index b48673c..8d921f5 100644 --- a/debian/control +++ b/debian/control @@ -24,31 +24,3 @@ Description: image creator for Linux distributions convert an image to a specified type; subcommand chroot is used to chroot into an image. -Package: mic-native -Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, - util-linux, - coreutils, - psmisc, - e2fsprogs (>= 1.41), - dosfstools, - isomd5sum, - genisoimage, - dmsetup, - kpartx, - parted, - squashfs-tools (>= 4.0), - yum (>= 3.2), - syslinux (>= 2:4.05), - extlinux (>= 2:4.05), - python-zypp-tizen, - python-m2crypto, - mic, -Recommends: - qemu-arm-static | qemu-user-static, - binfmt-support, - btrfs-tools, - udisks | hal -Description: Native support for mic - The native support package for mic, it includes all requirements - for mic native running. diff --git a/debian/mic-native.install b/debian/mic-native.install deleted file mode 100644 index 29caf35..0000000 --- a/debian/mic-native.install +++ /dev/null @@ -1 +0,0 @@ -usr/bin/mic-native diff --git a/debian/rules b/debian/rules index c9f50ec..41a13a9 100755 --- a/debian/rules +++ b/debian/rules @@ -32,7 +32,6 @@ install: build install -m644 doc/mic.1 $(CURDIR)/debian/tmp/usr/share/man/man1 install -m755 etc/bash_completion.d/mic.sh $(CURDIR)/debian/tmp/etc/bash_completion.d/ install -m755 etc/zsh_completion.d/_mic $(CURDIR)/debian/tmp/etc/zsh_completion.d/_mic - install -m755 tools/mic $(CURDIR)/debian/tmp/usr/bin/mic-native python setup.py install --root=$(CURDIR)/debian/tmp --prefix=/usr binary-indep: build install diff --git a/doc/RELEASE_NOTES b/doc/RELEASE_NOTES index 0cae44d..74a509a 100644 --- a/doc/RELEASE_NOTES +++ b/doc/RELEASE_NOTES @@ -1,43 +1,48 @@ -MIC Image Creator 0.24.4 Release Notes +MIC Image Creator 0.27 Release Notes ====================================== -Released Jan 22 2015 +Released Mar 28 2016 This release note documents the changes included in the new release. And the release contains new features, enhancements and bug fixes. -New Features & Enhancements ---------------------------- +New Features & Ehancements +-------------------------- + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, Fedora 23, openSUSE 13.2 * generate manifest file to describe image information - * update mount option to make it more general + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum + * drop mic-native support + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support + * use argparse module to parse the cmd line Bug Fixes --------- - * fix incorrect logfile location of '--release' if creation failed - * update dependency package for mic - * add catching 'distribution not found' exception + * fix logfile incomplete in release option + * fix config file disappear in bootstrap + * fix aarch64 bin_format + * fix pylint + * fix real path of device mapper causing initrd failure + * fix qemu arm and arm64 issues + * fix AttributeError in zypp backend + * fix 'python-xml' depends used by cElements + * fix xml requirements + * fix logfile not in release of '--release' when creation failed + * fix copyright missing issue + * fix syslinux installation path issue in Arch Linux + * fix priority option of ks file not apply + * fix need to check loop device after excute 'losetup --find' + * fix check scriptlet error file on /tmp/.postscript/error/ + * fix broken tar archive -MIC Image Creator 0.24.3 Release Notes -====================================== -Released Nov 24 2014 - -This release note documents the changes included in the new release. And -the release contains new features, enhancements and bug fixes. - -Bug Fixes ---------- - * fix qemu arm and arm64 issues - * remove --preserve-order option in taring fs image - -Resource --------- - * SITE: https://www.tizen.org/ - * REPO: https://download.tizen.org/tools/ - * DOCS: https://source.tizen.org/documentation/reference/mic-image-creator - * CODE: https://github.com/01org/mic - * BUGS: https://bugs.tizen.org/jira - * HELP: general@lists.tizen.org - Report Bugs ----------- when you found a bug, you can file this bug in our official bug tracker: diff --git a/doc/install.rst b/doc/install.rst index fef0fb7..bc5d668 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -10,6 +10,7 @@ Many mainstream Linux distributions are supported in Tizen Tools repository, inc - Ubuntu (12.10, 12.04, 11.10) - openSUSE (12.2, 12.1) - Feodra (17, 16) +- Debian (7.0) Ubuntu Installation ------------------- diff --git a/doc/man.rst b/doc/man.rst index 00d208e..c58e2f6 100644 --- a/doc/man.rst +++ b/doc/man.rst @@ -5,36 +5,31 @@ SYNOPSIS ======== -| mic create SUBCOMMAND <ksfile> [OPTION] -| mic chroot [OPTION] <imgfile> -| mic convert [OPTION] <imgfile> <format> +| mic [GLOBAL-OPTS] create SUBCOMMAND <ksfile> [OPTION] +| mic [GLOBAL-OPTS] chroot [OPTION] <imgfile> DESCRIPTION =========== The tools `mic` is used to create and manipulate images for Linux distributions. -It is composed of three subcommand: `create`, `convert`, `chroot`. +It is composed of two subcommand: `create`, `chroot`. USAGE ===== create ------ -This command is used to create various images, including live CD, live USB, -loop, raw. +This command is used to create various images, including loop. Usage: - - | mic create(cr) SUBCOMMAND <ksfile> [OPTION] + | mic [GLOBAL-OPTS] create(cr) SUBCOMMAND <ksfile> [OPTION] Subcommands: - | help(?) give detailed help on a specific sub-command + | auto auto detect image type from magic header | fs create fs image, which is also chroot directory - | livecd create live CD image, used for CD booting - | liveusb create live USB image, used for USB booting | loop create loop image, including multi-partitions - | raw create raw image, containing multi-partitions + | qcow create qcow image Options: @@ -50,23 +45,23 @@ Options: --pack-to=PACK_TO pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used --release=RID generate a release of RID with all necessary files, when @BUILD_ID@ is contained in kickstart file, it will be replaced by RID. sample values: "latest", "tizen_20120101.1" --copy-kernel copy kernel files from image /boot directory to the image output directory + --runtime=RUNTIME Specify runtime mode, avaiable: bootstrap + --install-pkgs=INSTALL_PKGS Specify what type of packages to be installed, valid: source, debuginfo, debugsource + --check-pkgs=CHECK_PKGS Check if given packages would be installed, packages should be separated by comma + --tmpfs Setup tmpdir as tmpfs to accelerate, experimental feature, use it if you have more than 4G memory + --strict-mode Abort creation of image, if there are some errors during rpm installation Options for fs image: --include-src generate a image with source rpms included; to enable it, user should specify the source repo in the ks file Options for loop image: --shrink whether to shrink loop images to minimal size - --compress-image=COMPRESS_IMAGE compress all loop images with 'gz' or 'bz2' - --compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image - -Options for raw image: - --compress-image=COMPRESS_IMAGE compress all raw images with 'gz' or 'bz2' + --compress-image=COMPRESS_IMAGE compress all loop images with 'gz' or 'bz2' or 'lzo' --compress-disk-image=COMPRESS_DISK_IMAGE same with --compress-image Examples: | mic create loop tizen.ks - | mic create livecd tizen.ks --release=latest | mic cr fs tizen.ks --local-pkgs-path=localrpm chroot @@ -85,34 +80,13 @@ Options: Examples: | mic chroot loop.img - | mic chroot tizen.iso - | mic ch -s tizenfs tizen.usbimg - -convert -------- -This command is used for converting an image to another format. - -Usage: - - | mic convert(cv) <imagefile> <destformat> - -Options: - - -h, --help show the help message - -S, --shell launch interactive shell before packing the new image in the converting - -Examples: - - | mic convert tizen.iso liveusb - | mic convert tizen.usbimg livecd - | mic cv --shell tizen.iso liveusb Advanced Usage ============== The advanced usage is just for bootstrap, please skip it if you don't care about it. The major purpose to use bootstrap is that some important packages (like rpm) are customized -a lot in the repo which you want to create image, and mic must use the customized rpm to +a lot in the repo which you want to create image, and mic must use the customized rpm to create images, or the images can't be boot. So mic will create a bootstrap using the repo in the ks file at first, then create the image via chrooting, which can make mic using the chroot environment with the customized rpm. @@ -147,13 +121,17 @@ tools repos, until the official released one being fixed. Failed to create btrfs image in openSUSE ---------------------------------------- -When creating btrfs image in openSUSE, it would hang up with showing image kernel -panic. This issue impact all openSUSE distributions: 12.1, 11.4, 11.3, etc +When creating btrfs image in openSUSE, it would hang up with showing image kernel +panic. This issue impact all openSUSE distributions: 12.1, 11.4, 11.3, etc REPORTING BUGS ============== The source code is tracked in github.com: - https://github.com/jfding/mic + https://github.com/01org/mic + +The bug is registered in tizen.org: + + https://bugs.tizen.org/jira Please report issues for bugs or feature requests. diff --git a/doc/usage.rst b/doc/usage.rst index 803dbe2..aed950b 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -8,7 +8,6 @@ Overview MIC offers three major functions: - creating an image with different format -- converting an image to another format - chrooting into an image Getting help @@ -25,9 +24,7 @@ How to get help: * mic --help * mic create --help - * mic help create * mic create loop --help - * mic create help loop Image formulation support ------------------------- @@ -38,21 +35,13 @@ Image formulation support * For a configuration with multiple partitions, which is specified in the kickstartfile, mic will generate multiple loop images * And multiple loop images can be packed into a single archive file -- Raw - - * “raw” format means something like hard disk dumping - * Including partition table and all the partitions - * The image is bootable directly - -- Livecd/liveusb - - * Mainly used for an ia32 build, it can be burned to CD or usbstick, which can be booted into a live system or installation UI - - fs * “fs” means file-system * mic can install all the Tizen files to the specified directory, which can be used directly as chroot env +- qcow + Create ------ @@ -60,25 +49,23 @@ Create :: - mic create(cr) SUBCOMMAND <ksfile> [OPTION] + mic [GLOBAL-OPTS] create(cr) SUBCOMMAND <ksfile> [OPTION] - Sub-commands, to specify image format, include: :: - help(?) give detailed help on a specific sub-command + auto auto detect image type from magic header fs create fs image, which is also a chroot directory - livecd create live CD image, used for CD booting - liveusb create live USB image, used for USB booting loop create loop image, including multi-partitions - raw create raw image, containing multi-partitions + qcow create qcow image - <ksfile>: The kickstart file is a simple text file, containing a list of items about image partition, setup, Bootloader, packages to be installed, etc, each identified by a keyword. -In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/daily/trunk/ivi/latest/images/ivi-min... +In Tizen, the released image will have a ks file along with image. For example, you can download the ks file from: http://download.tizen.org/releases/weekly/tizen/mobile/latest/images/... - Options include: @@ -107,28 +94,35 @@ In Tizen, the released image will have a ks file along with image. For example, --pack-to=PACK_TO Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used + --runtime=RUNTIME_MODE + Sets runtime mode, the default is bootstrap mode, valid + values: "bootstrap". "bootstrap" means mic uses one + tizen chroot environment to create image. --copy-kernel Copy kernel files from image /boot directory to the image output directory. - + --install-pkgs INSTALL_PKGS Specify what type of packages to be + installed, valid: source, debuginfo, debugsource + --check-pkgs=CHECK_PKGS + Check if given packages would be installed, + packages should be separated by comma + --tmpfs Setup tmpdir as tmpfs to accelerate, experimental feature, + use it if you have more than 4G memory + --strict-mode Abort creation of image, if there are some errors + during rpm installation + - Other options: -:: +:: - --runtime=RUNTIME_MODE - Sets runtime mode, the default is bootstrap mode, valid - values: "native", "bootstrap". "native" means mic uses - localhost environment to create image, while "bootstrap" - means mic uses one tizen chroot environment to create image. - --compress-image=COMPRESS_IMAGE (for loop & raw) + --compress-image=COMPRESS_IMAGE (for loop) Sets the disk image compression. Note: The available values might depend on the used filesystem type. --compress-disk-image=COMPRESS_IMAGE Same with --compress-image --shrink (for loop) Whether to shrink loop images to minimal size - --generate-bmap (for raw) - Generate the block map file - + --include-src (for fs) + Generate a image with source rpms included - Examples: :: @@ -158,34 +152,6 @@ This command is used to chroot inside the image. It's a great enhancement of the :: mic ch loop.img - mic ch tizen.iso - mic ch -s tizenfs tizen.usbimg - -Convert -------- -This command is used for converting an image to another format. - - -- Usage: - -:: - - mic convert(cv) <imagefile> <destformat> - -- Options: - -:: - - -h, --help Show this help message and exit - -S, --shell Launch shell before packaging the converted image - -- Examples: - -:: - - mic cv tizen.iso liveusb - mic cv tizen.usbimg livecd - mic cv --shell tizen.iso liveusb Getting Start ------------- diff --git a/etc/bash_completion.d/mic.sh b/etc/bash_completion.d/mic.sh index 91ea3c0..1cd19c9 100644 --- a/etc/bash_completion.d/mic.sh +++ b/etc/bash_completion.d/mic.sh @@ -119,7 +119,6 @@ __mic_complete_val() latest " runtime_values=" - native bootstrap " record_pkgs_values=" diff --git a/etc/zsh_completion.d/_mic b/etc/zsh_completion.d/_mic index 65e3433..a8e80c0 100644 --- a/etc/zsh_completion.d/_mic +++ b/etc/zsh_completion.d/_mic @@ -60,7 +60,7 @@ _mic() { '--record-pkgs=[Record the info of installed packages, multiple values can be specified which joined by \",\", valid values: \"name\", \"content\", \"license\", \"vcs\"]: :_mic_create_filters -s ,' \ '--pkgmgr=[Specify backend package manager]:parameter' \ '--local-pkgs-path=[Path for local pkgs(rpms) to be installed]:path' \ - '--runtime=[Specify runtime mode, avaiable: bootstrap, native]: :(bootstrap native)' \ + '--runtime=[Specify runtime mode, avaiable: bootstrap]: :(bootstrap)' \ '--pack-to=[Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used]:parameter' \ '--copy-kernel[Copy kernel files from image /boot directory to the image output directory.]' \ '--install-pkgs=[Specify what type of packages to be installed, valid: source, debuginfo, debugsource]: :(source debuginfo debugsource)' \ @@ -84,7 +84,7 @@ _mic() { '--record-pkgs=[Record the info of installed packages, multiple values can be specified which joined by \",\", valid values: \"name\", \"content\", \"license\", \"vcs\"]: :_mic_create_filters -s ,' \ '--pkgmgr=[Specify backend package manager]:parameter' \ '--local-pkgs-path=[Path for local pkgs(rpms) to be installed]:path' \ - '--runtime=[Specify runtime mode, avaiable: bootstrap, native]: :(bootstrap native)' \ + '--runtime=[Specify runtime mode, avaiable: bootstrap]: :(bootstrap)' \ '--pack-to=[Pack the images together into the specified achive, extension supported: .zip, .tar, .tar.gz, .tar.bz2, etc. by default, .tar will be used]:parameter' \ '--copy-kernel[Copy kernel files from image /boot directory to the image output directory.]' \ '--install-pkgs=[Specify what type of packages to be installed, valid: source, debuginfo, debugsource]: :(source debuginfo debugsource)' \ diff --git a/mic/__init__.py b/mic/__init__.py index bb35a9f..3aef805 100644 --- a/mic/__init__.py +++ b/mic/__init__.py @@ -16,7 +16,7 @@ import os, sys -__version__ = "0.24.4" +__version__ = "0.27" cur_path = os.path.dirname(__file__) or '.' sys.path.insert(0, cur_path + '/3rdparty') diff --git a/mic/archive.py b/mic/archive.py new file mode 100644 index 0000000..72f506f --- /dev/null +++ b/mic/archive.py @@ -0,0 +1,450 @@ +# +# Copyright (c) 2013 Intel Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +# Following messages should be disabled in pylint +# * Used * or ** magic (W0142) +# * Unused variable (W0612) +# * Used builtin function (W0141) +# * Invalid name for type (C0103) +# * Popen has no '%s' member (E1101) +# pylint: disable=W0142, W0612, W0141, C0103, E1101 + +""" Compression and Archiving + +Utility functions for creating archive files (tarballs, zip files, etc) +and compressing files (gzip, bzip2, lzop, etc) +""" + +import os +import shutil +import tempfile +import subprocess +from mic import msger + +__all__ = [ + "get_compress_formats", + "compress", + "decompress", + "get_archive_formats", + "get_archive_suffixes", + "make_archive", + "extract_archive", + "compressing", + "packing", + ] + +# TODO: refine Error class for archive/extract + +def which(binary, path=None): + """ Find 'binary' in the directories listed in 'path' + + @binary: the executable file to find + @path: the suposed path to search for, use $PATH if None + @retval: the absolute path if found, otherwise None + """ + if path is None: + path = os.environ["PATH"] + for apath in path.split(os.pathsep): + fpath = os.path.join(apath, binary) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + return fpath + return None + +def _call_external(cmdln_or_args): + """ Wapper for subprocess calls. + + @cmdln_or_args: command line to be joined before execution. + @retval: a tuple (returncode, outdata, errdata). + """ + if isinstance(cmdln_or_args, list): + shell = False + else: + shell = True + msger.info("Running command: " + " ".join(cmdln_or_args)) + + proc = subprocess.Popen(cmdln_or_args, shell=shell, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (outdata, errdata) = proc.communicate() + + return (proc.returncode, outdata, errdata) + +def _do_gzip(input_name, compression=True): + """ Compress/decompress the file with 'gzip' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + if which("pigz") is not None: + compressor = "pigz" + else: + compressor = "gzip" + + if compression: + cmdln = [compressor, "-f", input_name] + else: + cmdln = [compressor, "-d", "-f", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".gz" + else: + # suppose that file name is suffixed with ".gz" + output_name = os.path.splitext(input_name)[0] + + return output_name + +def _do_bzip2(input_name, compression=True): + """ Compress/decompress the file with 'bzip2' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + if which("pbzip2") is not None: + compressor = "pbzip2" + else: + compressor = "bzip2" + + if compression: + cmdln = [compressor, "-f", input_name] + else: + cmdln = [compressor, "-d", "-f", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".bz2" + else: + # suppose that file name is suffixed with ".bz2" + output_name = os.path.splitext(input_name)[0] + + return output_name + +def _do_lzop(input_name, compression=True): + """ Compress/decompress the file with 'lzop' utility. + + @input_name: the file name to compress/decompress + @compress: True for compressing, False for decompressing + @retval: the path of the compressed/decompressed file + """ + compressor = "lzop" + + if compression: + cmdln = [compressor, "-f", "-U", input_name] + else: + cmdln = [compressor, "-d", "-f", "-U", input_name] + + _call_external(cmdln) + + if compression: + output_name = input_name + ".lzo" + else: + # suppose that file name is suffixed with ".lzo" + output_name = os.path.splitext(input_name)[0] + + return output_name + +_COMPRESS_SUFFIXES = { + ".lzo" : [".lzo"], + ".gz" : [".gz"], + ".bz2" : [".bz2", ".bz"], + ".tar.lzo" : [".tar.lzo", ".tzo"], + ".tar.gz" : [".tar.gz", ".tgz", ".taz"], + ".tar.bz2" : [".tar.bz2", ".tbz", ".tbz2", ".tar.bz"], +} + +_COMPRESS_FORMATS = { + "gz" : _do_gzip, + "bz2": _do_bzip2, + "lzo": _do_lzop, +} + +def get_compress_formats(): + """ Get the list of the supported compression formats + + @retval: a list contained supported compress formats + """ + return _COMPRESS_FORMATS.keys() + +def get_compress_suffixes(): + """ Get the list of the support suffixes + + @retval: a list contained all suffixes + """ + suffixes = [] + for key in _COMPRESS_SUFFIXES.keys(): + suffix = _COMPRESS_SUFFIXES[key] + if (suffix): + suffixes.extend(suffix) + + return suffixes + +def compress(file_path, compress_format): + """ Compress a given file + + @file_path: the path of the file to compress + @compress_format: the compression format + @retval: the path of the compressed file + """ + if not os.path.isfile(file_path): + raise OSError, "can't compress a file not existed: '%s'" % file_path + + try: + func = _COMPRESS_FORMATS[compress_format] + except KeyError: + raise ValueError, "unknown compress format '%s'" % compress_format + return func(file_path, True) + +def decompress(file_path, decompress_format=None): + """ Decompess a give file + + @file_path: the path of the file to decompress + @decompress_format: the format for decompression, None for auto detection + @retval: the path of the decompressed file + """ + if not os.path.isfile(file_path): + raise OSError, "can't decompress a file not existed: '%s'" % file_path + + (file_name, file_ext) = os.path.splitext(file_path) + for key, suffixes in _COMPRESS_SUFFIXES.iteritems(): + if file_ext in suffixes: + file_ext = key + break + + if file_path != (file_name + file_ext): + shutil.move(file_path, file_name + file_ext) + file_path = file_name + file_ext + + if not decompress_format: + decompress_format = os.path.splitext(file_path)[1].lstrip(".") + + try: + func = _COMPRESS_FORMATS[decompress_format] + except KeyError: + raise ValueError, "unknown decompress format '%s'" % decompress_format + return func(file_path, False) + + +def _do_tar(archive_name, target_name): + """ Archive the directory or the file with 'tar' utility + + @archive_name: the name of the tarball file + @target_name: the name of the target to tar + @retval: the path of the archived file + """ + file_list = [] + if os.path.isdir(target_name): + target_dir = target_name + target_name = "." + for target_file in os.listdir(target_dir): + file_list.append(target_file) + else: + target_dir = os.path.dirname(target_name) + target_name = os.path.basename(target_name) + file_list.append(target_name) + cmdln = ["tar", "-C", target_dir, "-cf", archive_name] + cmdln.extend(file_list) + _call_external(cmdln) + + return archive_name + +def _do_untar(archive_name, target_dir=None): + """ Unarchive the archived file with 'tar' utility + + @archive_name: the name of the archived file + @target_dir: the directory which the target locates + raise exception if fail to extract archive + """ + if not target_dir: + target_dir = os.getcwd() + cmdln = ["tar", "-C", target_dir, "-xf", archive_name] + (returncode, stdout, stderr) = _call_external(cmdln) + if returncode != 0: + raise OSError, os.linesep.join([stdout, stderr]) + +def _imp_tarfile(archive_name, target_name): + """ Archive the directory or the file with tarfile module + + @archive_name: the name of the tarball file + @target_name: the name of the target to tar + @retval: the path of the archived file + """ + import tarfile + + msger.info("Taring files to %s using tarfile module" % archive_name) + tar = tarfile.open(archive_name, 'w') + if os.path.isdir(target_name): + for child in os.listdir(target_name): + tar.add(os.path.join(target_name, child), child) + else: + tar.add(target_name, os.path.basename(target_name)) + + tar.close() + return archive_name + +def _make_tarball(archive_name, target_name, compressor=None): + """ Create a tarball from all the files under 'target_name' or itself. + + @archive_name: the name of the archived file to create + @target_name: the directory or the file name to archive + @compressor: callback function to compress the tarball + @retval: indicate the compressing result + """ + archive_dir = os.path.dirname(archive_name) + tarball_name = tempfile.mktemp(suffix=".tar", dir=archive_dir) + + if which("tar") is not None: + _do_tar(tarball_name, target_name) + else: + _imp_tarfile(tarball_name, target_name) + + if compressor: + tarball_name = compressor(tarball_name, True) + + shutil.move(tarball_name, archive_name) + + return os.path.exists(archive_name) + +def _extract_tarball(archive_name, target_dir, compressor=None): + """ Extract a tarball to a target directory + + @archive_name: the name of the archived file to extract + @target_dir: the directory of the extracted target + """ + + _do_untar(archive_name, target_dir) + +def _make_zipfile(archive_name, target_name): + """ Create a zip file from all the files under 'target_name' or itself. + + @archive_name: the name of the archived file + @target_name: the directory or the file name to archive + @retval: indicate the archiving result + """ + import zipfile + + msger.info("Zipping files to %s using zipfile module" % archive_name) + arv = zipfile.ZipFile(archive_name, 'w', compression=zipfile.ZIP_DEFLATED) + + if os.path.isdir(target_name): + for dirpath, dirname, filenames in os.walk(target_name): + for filename in filenames: + filepath = os.path.normpath(os.path.join(dirpath, filename)) + arcname = os.path.relpath(filepath, target_name) + if os.path.isfile(filepath): + arv.write(filepath, arcname) + else: + arv.write(target_name, os.path.basename(target_name)) + + arv.close() + + return os.path.exists(archive_name) + +_ARCHIVE_SUFFIXES = { + "zip" : [".zip"], + "tar" : [".tar"], + "lzotar": [".tzo", ".tar.lzo"], + "gztar" : [".tgz", ".taz", ".tar.gz"], + "bztar" : [".tbz", ".tbz2", ".tar.bz", ".tar.bz2"], +} + +_ARCHIVE_FORMATS = { + "zip" : ( _make_zipfile, {} ), + "tar" : ( _make_tarball, {"compressor" : None} ), + "lzotar": ( _make_tarball, {"compressor" : _do_lzop} ), + "gztar" : ( _make_tarball, {"compressor" : _do_gzip} ), + "bztar" : ( _make_tarball, {"compressor" : _do_bzip2} ), +} + +def get_archive_formats(): + """ Get the list of the supported formats for archiving + + @retval: a list contained archive formats + """ + return _ARCHIVE_FORMATS.keys() + +def get_archive_suffixes(): + """ Get the list of the support suffixes + + @retval: a list contained all suffixes + """ + suffixes = [] + for name in _ARCHIVE_FORMATS.keys(): + suffix = _ARCHIVE_SUFFIXES.get(name, None) + if (suffix): + suffixes.extend(suffix) + + return suffixes + +def make_archive(archive_name, target_name): + """ Create an archive file (eg. tar or zip). + + @archive_name: the name of the archived file + @target_name: the directory or the file to archive + @retval: the archiving result + """ + if not os.path.exists(target_name): + raise OSError, "archive object does not exist: '%s'" % target_name + + for aformat, suffixes in _ARCHIVE_SUFFIXES.iteritems(): + if filter(archive_name.endswith, suffixes): + archive_format = aformat + break + else: + raise ValueError, "unknown archive suffix '%s'" % archive_name + + try: + func, kwargs = _ARCHIVE_FORMATS[archive_format] + except KeyError: + raise ValueError, "unknown archive format '%s'" % archive_format + + archive_name = os.path.abspath(archive_name) + target_name = os.path.abspath(target_name) + + archive_dir = os.path.dirname(archive_name) + if not os.path.exists(archive_dir): + os.makedirs(archive_dir) + + return func(archive_name, target_name, **kwargs) + +def extract_archive(archive_name, target_name): + """ Extract the given file + + @archive_name: the name of the archived file to extract + @target_name: the directory name where the target locates + raise exception if fail to extract archive + """ + if not os.path.exists(archive_name): + raise OSError, "archived object does not exist: '%s'" % archive_name + + archive_name = os.path.abspath(archive_name) + target_name = os.path.abspath(target_name) + + if os.path.exists(target_name) and not os.path.isdir(target_name): + raise OSError, "%s should be directory where extracted files locate"\ + % target_name + + if not os.path.exists(target_name): + os.makedirs(target_name) + + _extract_tarball(archive_name, target_name) + +packing = make_archive +compressing = compress + diff --git a/mic/cmd_chroot.py b/mic/cmd_chroot.py new file mode 100755 index 0000000..d7d76ef --- /dev/null +++ b/mic/cmd_chroot.py @@ -0,0 +1,71 @@ +#!/usr/bin/python -tt
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Implementation of subcmd: chroot
+"""
+
+import os
+import os, sys, re
+import pwd
+import argparse
+
+from mic import msger
+from mic.utils import misc, errors
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+def _root_confirm():
+ """Make sure command is called by root
+ There are a lot of commands needed to be run during creating images,
+ some of them must be run with root privilege like mount, kpartx"""
+ if os.geteuid() != 0:
+ msger.error('Root permission is required to continue, abort')
+
+def main(parser, args, argv):
+ """mic choot entry point."""
+
+ #args is argparser namespace, argv is the input cmd line
+ if args is None:
+ raise errors.Usage("Invalid arguments")
+
+ targetimage = args.imagefile
+ if not os.path.exists(targetimage):
+ raise errors.CreatorError("Cannot find the image: %s"
+ % targetimage)
+
+ _root_confirm()
+
+ configmgr.chroot['saveto'] = args.saveto
+
+ imagetype = misc.get_image_type(targetimage)
+ if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"):
+ imagetype = "loop"
+
+ chrootclass = None
+ for pname, pcls in pluginmgr.get_plugins('imager').iteritems():
+ if pname == imagetype and hasattr(pcls, "do_chroot"):
+ chrootclass = pcls
+ break
+
+ if not chrootclass:
+ raise errors.CreatorError("Cannot support image type: %s" \
+ % imagetype)
+
+ chrootclass.do_chroot(targetimage, args.cmd)
+
+
diff --git a/mic/cmd_create.py b/mic/cmd_create.py new file mode 100755 index 0000000..ce574a1 --- /dev/null +++ b/mic/cmd_create.py @@ -0,0 +1,234 @@ +#!/usr/bin/python -tt
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Implementation of subcmd: create
+"""
+
+import os
+import os, sys, re
+import pwd
+import argparse
+
+from mic import msger
+from mic.utils import errors, rpmmisc
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+def main(parser, args, argv):
+ """mic create entry point."""
+ #args is argparser namespace, argv is the input cmd line
+ if args is None:
+ raise errors.Usage("Invalid arguments")
+
+ if not os.path.exists(args.ksfile):
+ raise errors.CreatorError("Can't find the file: %s" % args.ksfile)
+
+ if os.geteuid() != 0:
+ msger.error("Root permission is required, abort")
+
+ try:
+ w = pwd.getpwuid(os.geteuid())
+ except KeyError:
+ msger.warning("Might fail in compressing stage for undetermined user")
+
+ abspath = lambda pth: os.path.abspath(os.path.expanduser(pth))
+ if args.logfile:
+ logfile_abs_path = abspath(args.logfile)
+ if os.path.isdir(logfile_abs_path):
+ raise errors.Usage("logfile's path %s should be file"
+ % args.logfile)
+ configmgr.create['logfile'] = logfile_abs_path
+ configmgr.set_logfile()
+
+ if args.subcommand == "auto":
+ do_auto(parser, args.ksfile, argv)
+ return
+
+ #check the imager type
+ createrClass = None
+ for subcmd, klass in pluginmgr.get_plugins('imager').iteritems():
+ if subcmd == args.subcommand and hasattr(klass, 'do_create'):
+ createrClass = klass
+
+ if createrClass is None:
+ raise errors.CreatorError("Can't support subcommand %s" % args.subcommand)
+
+ if args.config:
+ configmgr.reset()
+ configmgr._siteconf = args.config
+
+ if args.outdir is not None:
+ configmgr.create['outdir'] = abspath(args.outdir)
+ if args.cachedir is not None:
+ configmgr.create['cachedir'] = abspath(args.cachedir)
+ os.environ['ZYPP_LOCKFILE_ROOT'] = configmgr.create['cachedir']
+
+ for cdir in ('outdir', 'cachedir'):
+ if os.path.exists(configmgr.create[cdir]) \
+ and not os.path.isdir(configmgr.create[cdir]):
+ raise errors.Usage('Invalid directory specified: %s' \
+ % configmgr.create[cdir])
+ if not os.path.exists(configmgr.create[cdir]):
+ os.makedirs(configmgr.create[cdir])
+ if os.getenv('SUDO_UID', '') and os.getenv('SUDO_GID', ''):
+ os.chown(configmgr.create[cdir],
+ int(os.getenv('SUDO_UID')),
+ int(os.getenv('SUDO_GID')))
+
+ if args.local_pkgs_path is not None:
+ if not os.path.exists(args.local_pkgs_path):
+ raise errors.Usage('Local pkgs directory: \'%s\' not exist' \
+ % args.local_pkgs_path)
+ configmgr.create['local_pkgs_path'] = args.local_pkgs_path
+
+ if args.release:
+ configmgr.create['release'] = args.release.rstrip('/')
+
+ if args.record_pkgs:
+ configmgr.create['record_pkgs'] = []
+ for infotype in args.record_pkgs.split(','):
+ if infotype not in ('name', 'content', 'license', 'vcs'):
+ raise errors.Usage('Invalid pkg recording: %s, valid ones:'
+ ' "name", "content", "license", "vcs"' \
+ % infotype)
+
+ configmgr.create['record_pkgs'].append(infotype)
+
+ if args.strict_mode:
+ configmgr.create['strict_mode'] = args.strict_mode
+ if args.arch is not None:
+ supported_arch = sorted(rpmmisc.archPolicies.keys(), reverse=True)
+ if args.arch in supported_arch:
+ configmgr.create['arch'] = args.arch
+ else:
+ raise errors.Usage('Invalid architecture: "%s".\n'
+ ' Supported architectures are: \n'
+ ' %s' % (args.arch,
+ ', '.join(supported_arch)))
+
+ if args.pkgmgr is not None:
+ configmgr.create['pkgmgr'] = args.pkgmgr
+
+ if args.runtime:
+ configmgr.set_runtime(args.runtime)
+
+ if args.pack_to is not None:
+ configmgr.create['pack_to'] = args.pack_to
+
+ if args.copy_kernel:
+ configmgr.create['copy_kernel'] = args.copy_kernel
+
+ if args.install_pkgs:
+ configmgr.create['install_pkgs'] = []
+ for pkgtype in args.install_pkgs.split(','):
+ if pkgtype not in ('source', 'debuginfo', 'debugsource'):
+ raise errors.Usage('Invalid parameter specified: "%s", '
+ 'valid values: source, debuginfo, '
+ 'debusource' % pkgtype)
+
+ configmgr.create['install_pkgs'].append(pkgtype)
+
+ if args.check_pkgs:
+ for pkg in args.check_pkgs.split(','):
+ configmgr.create['check_pkgs'].append(pkg)
+
+ if args.enabletmpfs:
+ configmgr.create['enabletmpfs'] = args.enabletmpfs
+
+ if args.repourl:
+ for item in args.repourl:
+ try:
+ key, val = item.split('=')
+ except:
+ continue
+ configmgr.create['repourl'][key] = val
+
+ if args.repo:
+ for optvalue in args.repo:
+ repo = {}
+ for item in optvalue.split(';'):
+ try:
+ key, val = item.split('=')
+ except:
+ continue
+ repo[key.strip()] = val.strip()
+ if 'name' in repo:
+ configmgr.create['extrarepos'][repo['name']] = repo
+
+ if args.ignore_ksrepo:
+ configmgr.create['ignore_ksrepo'] = args.ignore_ksrepo
+
+ creater = createrClass()
+ creater.do_create(args)
+
+def do_auto(parser, ksfile, argv):
+ """${cmd_name}: auto detect image type from magic header
+
+ Usage:
+ ${name} ${cmd_name} <ksfile>
+
+ ${cmd_option_list}
+ """
+ def parse_magic_line(re_str, pstr, ptype='mic'):
+ ptn = re.compile(re_str)
+ m = ptn.match(pstr)
+ if not m or not m.groups():
+ return None
+
+ inline_argv = m.group(1).strip()
+ if ptype == 'mic':
+ m2 = re.search('(?P<format>\w+)', inline_argv)
+ elif ptype == 'mic2':
+ m2 = re.search('(-f|--format(=)?)\s*(?P<format>\w+)',
+ inline_argv)
+ else:
+ return None
+
+ if m2:
+ cmdname = m2.group('format')
+ inline_argv = inline_argv.replace(m2.group(0), '')
+ return (cmdname, inline_argv)
+
+ return None
+
+ if not os.path.exists(ksfile):
+ raise errors.CreatorError("Can't find the file: %s" % ksfile)
+
+ with open(ksfile, 'r') as rf:
+ first_line = rf.readline()
+
+ mic_re = '^#\s*-\*-mic-options-\*-\s+(.*)\s+-\*-mic-options-\*-'
+ mic2_re = '^#\s*-\*-mic2-options-\*-\s+(.*)\s+-\*-mic2-options-\*-'
+
+ result = parse_magic_line(mic_re, first_line, 'mic') \
+ or parse_magic_line(mic2_re, first_line, 'mic2')
+ if not result:
+ raise errors.KsError("Invalid magic line in file: %s" % ksfile)
+
+ ksargv = ' '.join(result).split()
+
+ argv.remove("auto")
+ index = argv.index("create")
+ #insert the subcommand
+ argv.insert(index+1, ksargv[0])
+ options = argv + ksargv[1:]
+
+ args = parser.parse_args(options)
+
+ main(parser, args, options)
+
diff --git a/mic/conf.py b/mic/conf.py index 742fcd5..d844c65 100644 --- a/mic/conf.py +++ b/mic/conf.py @@ -74,6 +74,7 @@ class ConfigMgr(object): "runtime": "bootstrap", "extrarepos": {}, "ignore_ksrepo": False, + "strict_mode": False, }, 'chroot': { "saveto": None, @@ -276,8 +277,8 @@ class ConfigMgr(object): msger.set_logfile(self.create['logfile'], mode) def set_runtime(self, runtime): - if runtime not in ("bootstrap", "native"): - raise errors.CreatorError("Invalid runtime mode: %s" % runtime) + if runtime != "bootstrap": + raise errors.CreatorError("Invalid runtime mode: %s, only 'bootstrap' mode is allowed." % runtime) if misc.get_distro()[0] in ("tizen", "Tizen"): runtime = "native" diff --git a/mic/creator.py b/mic/creator.py deleted file mode 100644 index dde1013..0000000 --- a/mic/creator.py +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys, re -import pwd -from optparse import SUPPRESS_HELP - -from mic import msger -from mic.utils import cmdln, errors, rpmmisc -from mic.conf import configmgr -from mic.plugin import pluginmgr - - -class Creator(cmdln.Cmdln): - """${name}: create an image - - Usage: - ${name} SUBCOMMAND <ksfile> [OPTS] - - ${command_list} - ${option_list} - """ - - name = 'mic create(cr)' - - def __init__(self, *args, **kwargs): - cmdln.Cmdln.__init__(self, *args, **kwargs) - self._subcmds = [] - - # get cmds from pluginmgr - # mix-in do_subcmd interface - for subcmd, klass in pluginmgr.get_plugins('imager').iteritems(): - if not hasattr(klass, 'do_create'): - msger.warning("Unsurpport subcmd: %s" % subcmd) - continue - - func = getattr(klass, 'do_create') - setattr(self.__class__, "do_"+subcmd, func) - self._subcmds.append(subcmd) - - def get_optparser(self): - optparser = cmdln.CmdlnOptionParser(self) - optparser.add_option('-d', '--debug', action='store_true', - dest='debug', - help=SUPPRESS_HELP) - optparser.add_option('-v', '--verbose', action='store_true', - dest='verbose', - help=SUPPRESS_HELP) - optparser.add_option('', '--logfile', type='string', dest='logfile', - default=None, - help='Path of logfile') - optparser.add_option('-c', '--config', type='string', dest='config', - default=None, - help='Specify config file for mic') - optparser.add_option('-k', '--cachedir', type='string', action='store', - dest='cachedir', default=None, - help='Cache directory to store the downloaded') - optparser.add_option('-o', '--outdir', type='string', action='store', - dest='outdir', default=None, - help='Output directory') - optparser.add_option('-A', '--arch', type='string', dest='arch', - default=None, - help='Specify repo architecture') - optparser.add_option('', '--release', type='string', dest='release', - default=None, metavar='RID', - help='Generate a release of RID with all necessary' - ' files, when @BUILD_ID@ is contained in ' - 'kickstart file, it will be replaced by RID') - optparser.add_option("", "--record-pkgs", type="string", - dest="record_pkgs", default=None, - help='Record the info of installed packages, ' - 'multiple values can be specified which ' - 'joined by ",", valid values: "name", ' - '"content", "license", "vcs"') - optparser.add_option('', '--pkgmgr', type='string', dest='pkgmgr', - default=None, - help='Specify backend package manager') - optparser.add_option('', '--local-pkgs-path', type='string', - dest='local_pkgs_path', default=None, - help='Path for local pkgs(rpms) to be installed') - optparser.add_option('', '--runtime', type='string', - dest='runtime', default=None, - help='Specify runtime mode, avaiable: bootstrap, native') - # --taring-to is alias to --pack-to - optparser.add_option('', '--taring-to', type='string', - dest='pack_to', default=None, - help=SUPPRESS_HELP) - optparser.add_option('', '--pack-to', type='string', - dest='pack_to', default=None, - help='Pack the images together into the specified' - ' achive, extension supported: .zip, .tar, ' - '.tar.gz, .tar.bz2, etc. by default, .tar ' - 'will be used') - optparser.add_option('', '--copy-kernel', action='store_true', - dest='copy_kernel', - help='Copy kernel files from image /boot directory' - ' to the image output directory.') - optparser.add_option('', '--install-pkgs', type='string', action='store', - dest='install_pkgs', default=None, - help='Specify what type of packages to be installed,' - ' valid: source, debuginfo, debugsource') - optparser.add_option('', '--check-pkgs', type='string', action='store', - dest='check_pkgs', default=[], - help='Check if given packages would be installed, ' - 'packages should be separated by comma') - optparser.add_option('', '--tmpfs', action='store_true', dest='enabletmpfs', - help='Setup tmpdir as tmpfs to accelerate, experimental' - ' feature, use it if you have more than 4G memory') - optparser.add_option('', '--repourl', action='append', - dest='repourl', default=[], - help=SUPPRESS_HELP) - optparser.add_option('-R', '--repo', action='append', - dest='repo', default=[], - help=SUPPRESS_HELP) - optparser.add_option('', '--ignore-ksrepo', action='store_true', - dest='ignore_ksrepo', default=False, - help=SUPPRESS_HELP) - return optparser - - def preoptparse(self, argv): - optparser = self.get_optparser() - - largs = [] - rargs = [] - while argv: - arg = argv.pop(0) - - if arg in ('-h', '--help'): - rargs.append(arg) - - elif optparser.has_option(arg): - largs.append(arg) - - if optparser.get_option(arg).takes_value(): - try: - largs.append(argv.pop(0)) - except IndexError: - raise errors.Usage("option %s requires arguments" % arg) - - else: - if arg.startswith("--"): - if "=" in arg: - opt = arg.split("=")[0] - else: - opt = None - elif arg.startswith("-") and len(arg) > 2: - opt = arg[0:2] - else: - opt = None - - if opt and optparser.has_option(opt): - largs.append(arg) - else: - rargs.append(arg) - - return largs + rargs - - def postoptparse(self): - abspath = lambda pth: os.path.abspath(os.path.expanduser(pth)) - - if self.options.verbose: - msger.set_loglevel('VERBOSE') - if self.options.debug: - msger.set_loglevel('DEBUG') - - if self.options.logfile: - logfile_abs_path = abspath(self.options.logfile) - if os.path.isdir(logfile_abs_path): - raise errors.Usage("logfile's path %s should be file" - % self.options.logfile) - configmgr.create['logfile'] = logfile_abs_path - configmgr.set_logfile() - - if self.options.config: - configmgr.reset() - configmgr._siteconf = self.options.config - - if self.options.outdir is not None: - configmgr.create['outdir'] = abspath(self.options.outdir) - if self.options.cachedir is not None: - configmgr.create['cachedir'] = abspath(self.options.cachedir) - os.environ['ZYPP_LOCKFILE_ROOT'] = configmgr.create['cachedir'] - - for cdir in ('outdir', 'cachedir'): - if os.path.exists(configmgr.create[cdir]) \ - and not os.path.isdir(configmgr.create[cdir]): - raise errors.Usage('Invalid directory specified: %s' \ - % configmgr.create[cdir]) - if not os.path.exists(configmgr.create[cdir]): - os.makedirs(configmgr.create[cdir]) - if os.getenv('SUDO_UID', '') and os.getenv('SUDO_GID', ''): - os.chown(configmgr.create[cdir], - int(os.getenv('SUDO_UID')), - int(os.getenv('SUDO_GID'))) - - if self.options.local_pkgs_path is not None: - if not os.path.exists(self.options.local_pkgs_path): - raise errors.Usage('Local pkgs directory: \'%s\' not exist' \ - % self.options.local_pkgs_path) - configmgr.create['local_pkgs_path'] = self.options.local_pkgs_path - - if self.options.release: - configmgr.create['release'] = self.options.release.rstrip('/') - - if self.options.record_pkgs: - configmgr.create['record_pkgs'] = [] - for infotype in self.options.record_pkgs.split(','): - if infotype not in ('name', 'content', 'license', 'vcs'): - raise errors.Usage('Invalid pkg recording: %s, valid ones:' - ' "name", "content", "license", "vcs"' \ - % infotype) - - configmgr.create['record_pkgs'].append(infotype) - - if self.options.arch is not None: - supported_arch = sorted(rpmmisc.archPolicies.keys(), reverse=True) - if self.options.arch in supported_arch: - configmgr.create['arch'] = self.options.arch - else: - raise errors.Usage('Invalid architecture: "%s".\n' - ' Supported architectures are: \n' - ' %s' % (self.options.arch, - ', '.join(supported_arch))) - - if self.options.pkgmgr is not None: - configmgr.create['pkgmgr'] = self.options.pkgmgr - - if self.options.runtime: - configmgr.set_runtime(self.options.runtime) - - if self.options.pack_to is not None: - configmgr.create['pack_to'] = self.options.pack_to - - if self.options.copy_kernel: - configmgr.create['copy_kernel'] = self.options.copy_kernel - - if self.options.install_pkgs: - configmgr.create['install_pkgs'] = [] - for pkgtype in self.options.install_pkgs.split(','): - if pkgtype not in ('source', 'debuginfo', 'debugsource'): - raise errors.Usage('Invalid parameter specified: "%s", ' - 'valid values: source, debuginfo, ' - 'debusource' % pkgtype) - - configmgr.create['install_pkgs'].append(pkgtype) - - if self.options.check_pkgs: - for pkg in self.options.check_pkgs.split(','): - configmgr.create['check_pkgs'].append(pkg) - - if self.options.enabletmpfs: - configmgr.create['enabletmpfs'] = self.options.enabletmpfs - - if self.options.repourl: - for item in self.options.repourl: - try: - key, val = item.split('=') - except: - continue - configmgr.create['repourl'][key] = val - - if self.options.repo: - for optvalue in self.options.repo: - repo = {} - for item in optvalue.split(';'): - try: - key, val = item.split('=') - except: - continue - repo[key.strip()] = val.strip() - if 'name' in repo: - configmgr.create['extrarepos'][repo['name']] = repo - - if self.options.ignore_ksrepo: - configmgr.create['ignore_ksrepo'] = self.options.ignore_ksrepo - - def main(self, argv=None): - if argv is None: - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - self.optparser = self.get_optparser() - if self.optparser: - try: - argv = self.preoptparse(argv) - self.options, args = self.optparser.parse_args(argv) - - except cmdln.CmdlnUserError, ex: - msg = "%s: %s\nTry '%s help' for info.\n"\ - % (self.name, ex, self.name) - raise errors.Usage(msg) - - except cmdln.StopOptionProcessing, ex: - return 0 - else: - # optparser=None means no process for opts - self.options, args = None, argv[1:] - - if not args: - return self.emptyline() - - self.postoptparse() - - return self.cmd(args) - - def precmd(self, argv): # check help before cmd - - if '-h' in argv or '?' in argv or '--help' in argv or 'help' in argv: - return argv - - if len(argv) == 1: - return ['help', argv[0]] - - if os.geteuid() != 0: - msger.error("Root permission is required, abort") - - try: - w = pwd.getpwuid(os.geteuid()) - except KeyError: - msger.warning("Might fail in compressing stage for undetermined user") - - return argv - - def do_auto(self, subcmd, opts, *args): - """${cmd_name}: auto detect image type from magic header - - Usage: - ${name} ${cmd_name} <ksfile> - - ${cmd_option_list} - """ - def parse_magic_line(re_str, pstr, ptype='mic'): - ptn = re.compile(re_str) - m = ptn.match(pstr) - if not m or not m.groups(): - return None - - inline_argv = m.group(1).strip() - if ptype == 'mic': - m2 = re.search('(?P<format>\w+)', inline_argv) - elif ptype == 'mic2': - m2 = re.search('(-f|--format(=)?)\s*(?P<format>\w+)', - inline_argv) - else: - return None - - if m2: - cmdname = m2.group('format') - inline_argv = inline_argv.replace(m2.group(0), '') - return (cmdname, inline_argv) - - return None - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - if not os.path.exists(args[0]): - raise errors.CreatorError("Can't find the file: %s" % args[0]) - - with open(args[0], 'r') as rf: - first_line = rf.readline() - - mic_re = '^#\s*-\*-mic-options-\*-\s+(.*)\s+-\*-mic-options-\*-' - mic2_re = '^#\s*-\*-mic2-options-\*-\s+(.*)\s+-\*-mic2-options-\*-' - - result = parse_magic_line(mic_re, first_line, 'mic') \ - or parse_magic_line(mic2_re, first_line, 'mic2') - if not result: - raise errors.KsError("Invalid magic line in file: %s" % args[0]) - - if result[0] not in self._subcmds: - raise errors.KsError("Unsupport format '%s' in %s" - % (result[0], args[0])) - - argv = ' '.join(result + args).split() - self.main(argv) - diff --git a/mic/helpformat.py b/mic/helpformat.py new file mode 100755 index 0000000..e17f716 --- /dev/null +++ b/mic/helpformat.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Local additions to commandline parsing."""
+
+import os
+import re
+import functools
+
+from argparse import RawDescriptionHelpFormatter, ArgumentTypeError
+
+class MICHelpFormatter(RawDescriptionHelpFormatter):
+ """Changed default argparse help output by request from cmdln lovers."""
+
+ def __init__(self, *args, **kwargs):
+ super(MICHelpFormatter, self).__init__(*args, **kwargs)
+ self._aliases = {}
+
+ def add_argument(self, action):
+ """Collect aliases."""
+
+ if action.choices:
+ for item, parser in action.choices.iteritems():
+ self._aliases[str(item)] = parser.get_default('alias')
+
+ return super(MICHelpFormatter, self).add_argument(action)
+
+ def format_help(self):
+ """
+ There is no safe and documented way in argparse to reformat
+ help output through APIs as almost all of them are private,
+ so this method just parses the output and changes it.
+ """
+ result = []
+ subcomm = False
+ for line in super(MICHelpFormatter, self).format_help().split('\n'):
+ if line.strip().startswith('{'):
+ continue
+ if line.startswith('optional arguments:'):
+ line = 'Global Options:'
+ if line.startswith('usage:'):
+ line = "Usage: mic [GLOBAL-OPTS] SUBCOMMAND [OPTS]"
+ if subcomm:
+ match = re.match("[ ]+([^ ]+)[ ]+(.+)", line)
+ if match:
+ name, help_text = match.group(1), match.group(2)
+ alias = self._aliases.get(name) or ''
+ if alias:
+ alias = "(%s)" % alias
+ line = " %-22s%s" % ("%s %s" % (name, alias), help_text)
+ if line.strip().startswith('subcommands:'):
+ line = 'Subcommands:'
+ subcomm = True
+ result.append(line)
+ return '\n'.join(result)
+
+def subparser(func):
+ """Convenient decorator for subparsers."""
+ @functools.wraps(func)
+ def wrapper(parser):
+ """
+ Create subparser
+ Set first line of function's docstring as a help
+ and the rest of the lines as a description.
+ Set attribute 'module' of subparser to 'cmd'+first part of function name
+ """
+ splitted = func.__doc__.split('\n')
+ name = func.__name__.split('_')[0]
+ subpar = parser.add_parser(name, help=splitted[0],
+ description='\n'.join(splitted[1:]),
+ formatter_class=RawDescriptionHelpFormatter)
+ subpar.set_defaults(module="cmd_%s" % name)
+ return func(subpar)
+ return wrapper
+
diff --git a/mic/imager/baseimager.py b/mic/imager/baseimager.py index 5f7b014..01936a1 100644 --- a/mic/imager/baseimager.py +++ b/mic/imager/baseimager.py @@ -36,6 +36,7 @@ from mic import msger, __version__ as VERSION from mic.utils.errors import CreatorError, Abort from mic.utils import misc, grabber, runner, fs_related as fs from mic.chroot import kill_proc_inchroot +from mic.archive import get_archive_suffixes class BaseImageCreator(object): """Installs a system to a chroot directory. @@ -81,9 +82,11 @@ class BaseImageCreator(object): self.destdir = "." self.installerfw_prefix = "INSTALLERFW_" self.target_arch = "noarch" + self.strict_mode = False self._local_pkgs_path = None self.pack_to = None self.repourl = {} + self.multiple_partitions = False # If the kernel is save to the destdir when copy_kernel cmd is called. self._need_copy_kernel = False @@ -96,6 +99,7 @@ class BaseImageCreator(object): "arch" : "target_arch", "local_pkgs_path" : "_local_pkgs_path", "copy_kernel" : "_need_copy_kernel", + "strict_mode" : "strict_mode", } # update setting from createopts @@ -112,9 +116,9 @@ class BaseImageCreator(object): if '@NAME@' in self.pack_to: self.pack_to = self.pack_to.replace('@NAME@', self.name) (tar, ext) = os.path.splitext(self.pack_to) - if ext in (".gz", ".bz2") and tar.endswith(".tar"): + if ext in (".gz", ".bz2", ".lzo", ".bz") and tar.endswith(".tar"): ext = ".tar" + ext - if ext not in misc.pack_formats: + if ext not in get_archive_suffixes(): self.pack_to += ".tar" self._dep_checks = ["ls", "bash", "cp", "echo", "modprobe"] @@ -148,6 +152,8 @@ class BaseImageCreator(object): if part.fstype and part.fstype == "btrfs": self._dep_checks.append("mkfs.btrfs") break + if len(self.ks.handler.partition.partitions) > 1: + self.multiple_partitions = True if self.target_arch and self.target_arch.startswith("arm"): for dep in self._dep_checks: @@ -987,6 +993,18 @@ class BaseImageCreator(object): the kickstart to be overridden. """ + def checkScriptletError(dirname, suffix): + if os.path.exists(dirname): + list = os.listdir(dirname) + for line in list: + filepath = os.path.join(dirname, line) + if os.path.isfile(filepath) and 0 < line.find(suffix): + return True + else: + continue + + return False + def get_ssl_verify(ssl_verify=None): if ssl_verify is not None: return not ssl_verify.lower().strip() == 'no' @@ -1049,6 +1067,11 @@ class BaseImageCreator(object): checksize -= BOOT_SAFEGUARD if self.target_arch: pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_IGNOREARCH) + + # If we have multiple partitions, don't check diskspace when rpm run transaction + # because rpm check '/' partition only. + if self.multiple_partitions: + pkg_manager._add_prob_flags(rpm.RPMPROB_FILTER_DISKSPACE) pkg_manager.runInstall(checksize) except CreatorError, e: raise @@ -1062,6 +1085,9 @@ class BaseImageCreator(object): finally: pkg_manager.close() + if checkScriptletError(self._instroot + "/tmp/.postscript/error/", "_error"): + raise CreatorError('scriptlet errors occurred') + # hook post install self.postinstall() @@ -1303,22 +1329,32 @@ class BaseImageCreator(object): os.rename(_rpath(f), _rpath(newf)) outimages.append(_rpath(newf)) - # generate MD5SUMS - with open(_rpath("MD5SUMS"), "w") as wf: - for f in os.listdir(destdir): - if f == "MD5SUMS": - continue + # generate MD5SUMS SHA1SUMS SHA256SUMS + def generate_hashsum(hash_name, hash_method): + with open(_rpath(hash_name), "w") as wf: + for f in os.listdir(destdir): + if f.endswith('SUMS'): + continue - if os.path.isdir(os.path.join(destdir, f)): - continue + if os.path.isdir(os.path.join(destdir, f)): + continue + + hash_value = hash_method(_rpath(f)) + # There needs to be two spaces between the sum and + # filepath to match the syntax with md5sum,sha1sum, + # sha256sum. This way also *sum -c *SUMS can be used. + wf.write("%s %s\n" % (hash_value, f)) + + outimages.append("%s/%s" % (destdir, hash_name)) - md5sum = misc.get_md5sum(_rpath(f)) - # There needs to be two spaces between the sum and - # filepath to match the syntax with md5sum. - # This way also md5sum -c MD5SUMS can be used by users - wf.write("%s %s\n" % (md5sum, f)) + hash_dict = { + 'MD5SUMS' : misc.get_md5sum, + 'SHA1SUMS' : misc.get_sha1sum, + 'SHA256SUMS' : misc.get_sha256sum + } - outimages.append("%s/MD5SUMS" % destdir) + for k, v in hash_dict.items(): + generate_hashsum(k, v) # Filter out the nonexist file for fp in outimages[:]: @@ -1354,7 +1390,8 @@ class BaseImageCreator(object): def get_pkg_manager(self): return self.pkgmgr(target_arch = self.target_arch, instroot = self._instroot, - cachedir = self.cachedir) + cachedir = self.cachedir, + strict_mode = self.strict_mode) def create_manifest(self): def get_pack_suffix(): diff --git a/mic/imager/livecd.py b/mic/imager/livecd.py deleted file mode 100644 index df613a4..0000000 --- a/mic/imager/livecd.py +++ /dev/null @@ -1,757 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys -import glob -import shutil - -from mic import kickstart, msger -from mic.utils import fs_related, rpmmisc, runner, misc -from mic.utils.errors import CreatorError -from mic.imager.loop import LoopImageCreator -from mic.imager.baseimager import BaseImageCreator - - -class LiveImageCreatorBase(LoopImageCreator): - """A base class for LiveCD image creators. - - This class serves as a base class for the architecture-specific LiveCD - image creator subclass, LiveImageCreator. - - LiveImageCreator creates a bootable ISO containing the system image, - bootloader, bootloader configuration, kernel and initramfs. - """ - - img_format = 'livecd' - - def __init__(self, creatoropts = None, pkgmgr = None): - """Initialise a LiveImageCreator instance. - - This method takes the same arguments as ImageCreator.__init__(). - """ - LoopImageCreator.__init__(self, creatoropts, pkgmgr) - - #Controls whether to use squashfs to compress the image. - self.skip_compression = False - - #Controls whether an image minimizing snapshot should be created. - # - #This snapshot can be used when copying the system image from the ISO in - #order to minimize the amount of data that needs to be copied; simply, - #it makes it possible to create a version of the image's filesystem with - #no spare space. - self.skip_minimize = False - - #A flag which indicates i act as a convertor default false - self.actasconvertor = False - - #The bootloader timeout from kickstart. - if self.ks: - self._timeout = kickstart.get_timeout(self.ks, 10) - else: - self._timeout = 10 - - #The default kernel type from kickstart. - if self.ks: - self._default_kernel = kickstart.get_default_kernel(self.ks, - "kernel") - else: - self._default_kernel = None - - if self.ks: - parts = kickstart.get_partitions(self.ks) - if len(parts) > 1: - raise CreatorError("Can't support multi partitions in ks file " - "for this image type") - # FIXME: rename rootfs img to self.name, - # else can't find files when create iso - self._instloops[0]['name'] = self.name + ".img" - - self.__isodir = None - - self.__modules = ["=ata", - "sym53c8xx", - "aic7xxx", - "=usb", - "=firewire", - "=mmc", - "=pcmcia", - "mptsas"] - if self.ks: - self.__modules.extend(kickstart.get_modules(self.ks)) - - self._dep_checks.extend(["isohybrid", - "unsquashfs", - "mksquashfs", - "dd", - "genisoimage"]) - - # - # Hooks for subclasses - # - def _configure_bootloader(self, isodir): - """Create the architecture specific booloader configuration. - - This is the hook where subclasses must create the booloader - configuration in order to allow a bootable ISO to be built. - - isodir -- the directory where the contents of the ISO are to - be staged - """ - raise CreatorError("Bootloader configuration is arch-specific, " - "but not implemented for this arch!") - def _get_menu_options(self): - """Return a menu options string for syslinux configuration. - """ - if self.ks is None: - return "liveinst autoinst" - r = kickstart.get_menu_args(self.ks) - return r - - def _get_kernel_options(self): - """Return a kernel options string for bootloader configuration. - - This is the hook where subclasses may specify a set of kernel - options which should be included in the images bootloader - configuration. - - A sensible default implementation is provided. - """ - - if self.ks is None: - r = "ro rd.live.image" - else: - r = kickstart.get_kernel_args(self.ks) - - return r - - def _get_mkisofs_options(self, isodir): - """Return the architecture specific mkisosfs options. - - This is the hook where subclasses may specify additional arguments - to mkisofs, e.g. to enable a bootable ISO to be built. - - By default, an empty list is returned. - """ - return [] - - # - # Helpers for subclasses - # - def _has_checkisomd5(self): - """Check whether checkisomd5 is available in the install root.""" - def _exists(path): - return os.path.exists(self._instroot + path) - - if _exists("/usr/bin/checkisomd5") and os.path.exists("/usr/bin/implantisomd5"): - return True - - return False - - def __restore_file(self,path): - try: - os.unlink(path) - except: - pass - if os.path.exists(path + '.rpmnew'): - os.rename(path + '.rpmnew', path) - - def _mount_instroot(self, base_on = None): - LoopImageCreator._mount_instroot(self, base_on) - self.__write_initrd_conf(self._instroot + "/etc/sysconfig/mkinitrd") - self.__write_dracut_conf(self._instroot + "/etc/dracut.conf.d/02livecd.conf") - - def _unmount_instroot(self): - self.__restore_file(self._instroot + "/etc/sysconfig/mkinitrd") - self.__restore_file(self._instroot + "/etc/dracut.conf.d/02livecd.conf") - LoopImageCreator._unmount_instroot(self) - - def __ensure_isodir(self): - if self.__isodir is None: - self.__isodir = self._mkdtemp("iso-") - return self.__isodir - - def _get_isodir(self): - return self.__ensure_isodir() - - def _set_isodir(self, isodir = None): - self.__isodir = isodir - - def _create_bootconfig(self): - """Configure the image so that it's bootable.""" - self._configure_bootloader(self.__ensure_isodir()) - - def _get_post_scripts_env(self, in_chroot): - env = LoopImageCreator._get_post_scripts_env(self, in_chroot) - - if not in_chroot: - env["LIVE_ROOT"] = self.__ensure_isodir() - - return env - def __write_dracut_conf(self, path): - if not os.path.exists(os.path.dirname(path)): - fs_related.makedirs(os.path.dirname(path)) - f = open(path, "a") - f.write('add_dracutmodules+=" dmsquash-live pollcdrom "') - f.close() - - def __write_initrd_conf(self, path): - content = "" - if not os.path.exists(os.path.dirname(path)): - fs_related.makedirs(os.path.dirname(path)) - f = open(path, "w") - - content += 'LIVEOS="yes"\n' - content += 'PROBE="no"\n' - content += 'MODULES+="squashfs ext3 ext2 vfat msdos "\n' - content += 'MODULES+="sr_mod sd_mod ide-cd cdrom "\n' - - for module in self.__modules: - if module == "=usb": - content += 'MODULES+="ehci_hcd uhci_hcd ohci_hcd "\n' - content += 'MODULES+="usb_storage usbhid "\n' - elif module == "=firewire": - content += 'MODULES+="firewire-sbp2 firewire-ohci "\n' - content += 'MODULES+="sbp2 ohci1394 ieee1394 "\n' - elif module == "=mmc": - content += 'MODULES+="mmc_block sdhci sdhci-pci "\n' - elif module == "=pcmcia": - content += 'MODULES+="pata_pcmcia "\n' - else: - content += 'MODULES+="' + module + ' "\n' - f.write(content) - f.close() - - def __create_iso(self, isodir): - iso = self._outdir + "/" + self.name + ".iso" - genisoimage = fs_related.find_binary_path("genisoimage") - args = [genisoimage, - "-J", "-r", - "-hide-rr-moved", "-hide-joliet-trans-tbl", - "-V", self.fslabel, - "-o", iso] - - args.extend(self._get_mkisofs_options(isodir)) - - args.append(isodir) - - if runner.show(args) != 0: - raise CreatorError("ISO creation failed!") - - """ It should be ok still even if you haven't isohybrid """ - isohybrid = None - try: - isohybrid = fs_related.find_binary_path("isohybrid") - except: - pass - - if isohybrid: - args = [isohybrid, "-partok", iso ] - if runner.show(args) != 0: - raise CreatorError("Hybrid ISO creation failed!") - - self.__implant_md5sum(iso) - - def __implant_md5sum(self, iso): - """Implant an isomd5sum.""" - if os.path.exists("/usr/bin/implantisomd5"): - implantisomd5 = "/usr/bin/implantisomd5" - else: - msger.warning("isomd5sum not installed; not setting up mediacheck") - implantisomd5 = "" - return - - runner.show([implantisomd5, iso]) - - def _stage_final_image(self): - try: - fs_related.makedirs(self.__ensure_isodir() + "/LiveOS") - - minimal_size = self._resparse() - - if not self.skip_minimize: - fs_related.create_image_minimizer(self.__isodir + \ - "/LiveOS/osmin.img", - self._image, - minimal_size) - - if self.skip_compression: - shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img") - else: - fs_related.makedirs(os.path.join( - os.path.dirname(self._image), - "LiveOS")) - shutil.move(self._image, - os.path.join(os.path.dirname(self._image), - "LiveOS", "ext3fs.img")) - fs_related.mksquashfs(os.path.dirname(self._image), - self.__isodir + "/LiveOS/squashfs.img") - - self.__create_iso(self.__isodir) - - if self.pack_to: - isoimg = os.path.join(self._outdir, self.name + ".iso") - packimg = os.path.join(self._outdir, self.pack_to) - misc.packing(packimg, isoimg) - os.unlink(isoimg) - self.image_files.update({'image_files': [self.pack_to]}) - else: - self.image_files.update({'image_files': [self.name + ".iso"]}) - - finally: - shutil.rmtree(self.__isodir, ignore_errors = True) - self.__isodir = None - - -class x86LiveImageCreator(LiveImageCreatorBase): - """ImageCreator for x86 machines""" - def _get_mkisofs_options(self, isodir): - return [ "-b", "isolinux/isolinux.bin", - "-c", "isolinux/boot.cat", - "-no-emul-boot", "-boot-info-table", - "-boot-load-size", "4" ] - - def _get_required_packages(self): - return ["syslinux", "syslinux-extlinux"] + \ - LiveImageCreatorBase._get_required_packages(self) - - def _get_isolinux_stanzas(self, isodir): - return "" - - def __find_syslinux_menu(self): - for menu in ["vesamenu.c32", "menu.c32"]: - if os.path.isfile(self._instroot + "/usr/share/syslinux/" + menu): - return menu - - raise CreatorError("syslinux not installed : " - "no suitable /usr/share/syslinux/*menu.c32 found") - - def __find_syslinux_mboot(self): - # - # We only need the mboot module if we have any xen hypervisors - # - if not glob.glob(self._instroot + "/boot/xen.gz*"): - return None - - return "mboot.c32" - - def __copy_syslinux_files(self, isodir, menu, mboot = None): - files = ["isolinux.bin", menu] - if mboot: - files += [mboot] - - for f in files: - path = self._instroot + "/usr/share/syslinux/" + f - - if not os.path.isfile(path): - raise CreatorError("syslinux not installed : " - "%s not found" % path) - - shutil.copy(path, isodir + "/isolinux/") - - def __copy_syslinux_background(self, isodest): - background_path = self._instroot + \ - "/usr/share/branding/default/syslinux/syslinux-vesa-splash.jpg" - - if not os.path.exists(background_path): - return False - - shutil.copyfile(background_path, isodest) - - return True - - def __copy_kernel_and_initramfs(self, isodir, version, index): - bootdir = self._instroot + "/boot" - isDracut = False - - if self._alt_initrd_name: - src_initrd_path = os.path.join(bootdir, self._alt_initrd_name) - else: - if os.path.exists(bootdir + "/initramfs-" + version + ".img"): - src_initrd_path = os.path.join(bootdir, "initramfs-" +version+ ".img") - isDracut = True - else: - src_initrd_path = os.path.join(bootdir, "initrd-" +version+ ".img") - - try: - msger.debug("copy %s to %s" % (bootdir + "/vmlinuz-" + version, isodir + "/isolinux/vmlinuz" + index)) - shutil.copyfile(bootdir + "/vmlinuz-" + version, - isodir + "/isolinux/vmlinuz" + index) - - msger.debug("copy %s to %s" % (src_initrd_path, isodir + "/isolinux/initrd" + index + ".img")) - shutil.copyfile(src_initrd_path, - isodir + "/isolinux/initrd" + index + ".img") - except: - raise CreatorError("Unable to copy valid kernels or initrds, " - "please check the repo.") - - is_xen = False - if os.path.exists(bootdir + "/xen.gz-" + version[:-3]): - shutil.copyfile(bootdir + "/xen.gz-" + version[:-3], - isodir + "/isolinux/xen" + index + ".gz") - is_xen = True - - return (is_xen,isDracut) - - def __is_default_kernel(self, kernel, kernels): - if len(kernels) == 1: - return True - - if kernel == self._default_kernel: - return True - - if kernel.startswith("kernel-") and kernel[7:] == self._default_kernel: - return True - - return False - - def __get_basic_syslinux_config(self, **args): - return """ -default %(menu)s -timeout %(timeout)d - -%(background)s -menu title Welcome to %(distroname)s! -menu color border 0 #ffffffff #00000000 -menu color sel 7 #ff000000 #ffffffff -menu color title 0 #ffffffff #00000000 -menu color tabmsg 0 #ffffffff #00000000 -menu color unsel 0 #ffffffff #00000000 -menu color hotsel 0 #ff000000 #ffffffff -menu color hotkey 7 #ffffffff #ff000000 -menu color timeout_msg 0 #ffffffff #00000000 -menu color timeout 0 #ffffffff #00000000 -menu color cmdline 0 #ffffffff #00000000 -menu hidden -menu clear -""" % args - - def __get_image_stanza(self, is_xen, isDracut, **args): - if isDracut: - args["rootlabel"] = "live:CDLABEL=%(fslabel)s" % args - else: - args["rootlabel"] = "CDLABEL=%(fslabel)s" % args - if not is_xen: - template = """label %(short)s - menu label %(long)s - kernel vmlinuz%(index)s - append initrd=initrd%(index)s.img root=%(rootlabel)s rootfstype=iso9660 %(liveargs)s %(extra)s -""" - else: - template = """label %(short)s - menu label %(long)s - kernel mboot.c32 - append xen%(index)s.gz --- vmlinuz%(index)s root=%(rootlabel)s rootfstype=iso9660 %(liveargs)s %(extra)s --- initrd%(index)s.img -""" - return template % args - - def __get_image_stanzas(self, isodir): - versions = [] - kernels = self._get_kernel_versions() - for kernel in kernels: - for version in kernels[kernel]: - versions.append(version) - - if not versions: - raise CreatorError("Unable to find valid kernels, " - "please check the repo") - - kernel_options = self._get_kernel_options() - - """ menu can be customized highly, the format is: - - short_name1:long_name1:extra_opts1;short_name2:long_name2:extra_opts2 - - e.g.: autoinst:InstallationOnly:systemd.unit=installer-graphical.service - but in order to keep compatible with old format, these are still ok: - - liveinst autoinst - liveinst;autoinst - liveinst::;autoinst:: - """ - oldmenus = {"basic": { - "short": "basic", - "long": "Installation Only (Text based)", - "extra": "basic nosplash 4" - }, - "liveinst": { - "short": "liveinst", - "long": "Installation Only", - "extra": "liveinst nosplash 4" - }, - "autoinst": { - "short": "autoinst", - "long": "Autoinstall (Deletes all existing content)", - "extra": "autoinst nosplash 4" - }, - "netinst": { - "short": "netinst", - "long": "Network Installation", - "extra": "netinst 4" - }, - "verify": { - "short": "check", - "long": "Verify and", - "extra": "check" - } - } - menu_options = self._get_menu_options() - menus = menu_options.split(";") - for i in range(len(menus)): - menus[i] = menus[i].split(":") - if len(menus) == 1 and len(menus[0]) == 1: - """ Keep compatible with the old usage way """ - menus = menu_options.split() - for i in range(len(menus)): - menus[i] = [menus[i]] - - cfg = "" - - default_version = None - default_index = None - index = "0" - netinst = None - for version in versions: - (is_xen, isDracut) = self.__copy_kernel_and_initramfs(isodir, version, index) - if index == "0": - self._isDracut = isDracut - - default = self.__is_default_kernel(kernel, kernels) - - if default: - long = "Boot %s" % self.distro_name - elif kernel.startswith("kernel-"): - long = "Boot %s(%s)" % (self.name, kernel[7:]) - else: - long = "Boot %s(%s)" % (self.name, kernel) - - oldmenus["verify"]["long"] = "%s %s" % (oldmenus["verify"]["long"], - long) - # tell dracut not to ask for LUKS passwords or activate mdraid sets - if isDracut: - kern_opts = kernel_options + " rd.luks=0 rd.md=0 rd.dm=0" - else: - kern_opts = kernel_options - - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kern_opts, - long = long, - short = "linux" + index, - extra = "", - index = index) - - if default: - cfg += "menu default\n" - default_version = version - default_index = index - - for menu in menus: - if not menu[0]: - continue - short = menu[0] + index - - if len(menu) >= 2: - long = menu[1] - else: - if menu[0] in oldmenus.keys(): - if menu[0] == "verify" and not self._has_checkisomd5(): - continue - if menu[0] == "netinst": - netinst = oldmenus[menu[0]] - continue - long = oldmenus[menu[0]]["long"] - extra = oldmenus[menu[0]]["extra"] - else: - long = short.upper() + " X" + index - extra = "" - - if len(menu) >= 3: - extra = ' '.join(menu[2:]) - - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kernel_options, - long = long, - short = short, - extra = extra, - index = index) - - index = str(int(index) + 1) - - if not default_version: - default_version = versions[0] - if not default_index: - default_index = "0" - - if netinst: - cfg += self.__get_image_stanza(is_xen, isDracut, - fslabel = self.fslabel, - liveargs = kernel_options, - long = netinst["long"], - short = netinst["short"], - extra = netinst["extra"], - index = default_index) - - return cfg - - def __get_memtest_stanza(self, isodir): - memtest = glob.glob(self._instroot + "/boot/memtest86*") - if not memtest: - return "" - - shutil.copyfile(memtest[0], isodir + "/isolinux/memtest") - - return """label memtest - menu label Memory Test - kernel memtest -""" - - def __get_local_stanza(self, isodir): - return """label local - menu label Boot from local drive - localboot 0xffff -""" - - def _configure_syslinux_bootloader(self, isodir): - """configure the boot loader""" - fs_related.makedirs(isodir + "/isolinux") - - menu = self.__find_syslinux_menu() - - self.__copy_syslinux_files(isodir, menu, - self.__find_syslinux_mboot()) - - background = "" - if self.__copy_syslinux_background(isodir + "/isolinux/splash.jpg"): - background = "menu background splash.jpg" - - cfg = self.__get_basic_syslinux_config(menu = menu, - background = background, - name = self.name, - timeout = self._timeout * 10, - distroname = self.distro_name) - - cfg += self.__get_image_stanzas(isodir) - cfg += self.__get_memtest_stanza(isodir) - cfg += self.__get_local_stanza(isodir) - cfg += self._get_isolinux_stanzas(isodir) - - cfgf = open(isodir + "/isolinux/isolinux.cfg", "w") - cfgf.write(cfg) - cfgf.close() - - def __copy_efi_files(self, isodir): - if not os.path.exists(self._instroot + "/boot/efi/EFI/redhat/grub.efi"): - return False - shutil.copy(self._instroot + "/boot/efi/EFI/redhat/grub.efi", - isodir + "/EFI/boot/grub.efi") - shutil.copy(self._instroot + "/boot/grub/splash.xpm.gz", - isodir + "/EFI/boot/splash.xpm.gz") - - return True - - def __get_basic_efi_config(self, **args): - return """ -default=0 -splashimage=/EFI/boot/splash.xpm.gz -timeout %(timeout)d -hiddenmenu - -""" %args - - def __get_efi_image_stanza(self, **args): - return """title %(long)s - kernel /EFI/boot/vmlinuz%(index)s root=CDLABEL=%(fslabel)s rootfstype=iso9660 %(liveargs)s %(extra)s - initrd /EFI/boot/initrd%(index)s.img -""" %args - - def __get_efi_image_stanzas(self, isodir, name): - # FIXME: this only supports one kernel right now... - - kernel_options = self._get_kernel_options() - checkisomd5 = self._has_checkisomd5() - - cfg = "" - - for index in range(0, 9): - # we don't support xen kernels - if os.path.exists("%s/EFI/boot/xen%d.gz" %(isodir, index)): - continue - cfg += self.__get_efi_image_stanza(fslabel = self.fslabel, - liveargs = kernel_options, - long = name, - extra = "", index = index) - if checkisomd5: - cfg += self.__get_efi_image_stanza( - fslabel = self.fslabel, - liveargs = kernel_options, - long = "Verify and Boot " + name, - extra = "check", - index = index) - break - - return cfg - - def _configure_efi_bootloader(self, isodir): - """Set up the configuration for an EFI bootloader""" - fs_related.makedirs(isodir + "/EFI/boot") - - if not self.__copy_efi_files(isodir): - shutil.rmtree(isodir + "/EFI") - return - - for f in os.listdir(isodir + "/isolinux"): - os.link("%s/isolinux/%s" %(isodir, f), - "%s/EFI/boot/%s" %(isodir, f)) - - - cfg = self.__get_basic_efi_config(name = self.name, - timeout = self._timeout) - cfg += self.__get_efi_image_stanzas(isodir, self.name) - - cfgf = open(isodir + "/EFI/boot/grub.conf", "w") - cfgf.write(cfg) - cfgf.close() - - # first gen mactel machines get the bootloader name wrong apparently - if rpmmisc.getBaseArch() == "i386": - os.link(isodir + "/EFI/boot/grub.efi", - isodir + "/EFI/boot/boot.efi") - os.link(isodir + "/EFI/boot/grub.conf", - isodir + "/EFI/boot/boot.conf") - - # for most things, we want them named boot$efiarch - efiarch = {"i386": "ia32", "x86_64": "x64"} - efiname = efiarch[rpmmisc.getBaseArch()] - os.rename(isodir + "/EFI/boot/grub.efi", - isodir + "/EFI/boot/boot%s.efi" %(efiname,)) - os.link(isodir + "/EFI/boot/grub.conf", - isodir + "/EFI/boot/boot%s.conf" %(efiname,)) - - - def _configure_bootloader(self, isodir): - self._configure_syslinux_bootloader(isodir) - self._configure_efi_bootloader(isodir) - -arch = rpmmisc.getBaseArch() -if arch in ("i386", "x86_64"): - LiveCDImageCreator = x86LiveImageCreator -elif arch.startswith("arm"): - LiveCDImageCreator = LiveImageCreatorBase -else: - raise CreatorError("Architecture not supported!") diff --git a/mic/imager/liveusb.py b/mic/imager/liveusb.py deleted file mode 100644 index 3c01f39..0000000 --- a/mic/imager/liveusb.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import re - -from mic import msger -from mic.utils import misc, fs_related, runner -from mic.utils.errors import CreatorError, MountError -from mic.utils.partitionedfs import PartitionedMount -from mic.imager.livecd import LiveCDImageCreator - - -class LiveUSBImageCreator(LiveCDImageCreator): - img_format = 'liveusb' - - def __init__(self, *args): - LiveCDImageCreator.__init__(self, *args) - - self._dep_checks.extend(["kpartx", "parted"]) - - # remove dependency of genisoimage in parent class - if "genisoimage" in self._dep_checks: - self._dep_checks.remove("genisoimage") - - def _create_usbimg(self, isodir): - overlaysizemb = 64 #default - #skipcompress = self.skip_compression? - fstype = "vfat" - homesizemb=0 - swapsizemb=0 - homefile="home.img" - plussize=128 - kernelargs=None - - if fstype == 'vfat': - if overlaysizemb > 2047: - raise CreatorError("Can't have an overlay of 2048MB or " - "greater on VFAT") - - if homesizemb > 2047: - raise CreatorError("Can't have an home overlay of 2048MB or " - "greater on VFAT") - - if swapsizemb > 2047: - raise CreatorError("Can't have an swap overlay of 2048MB or " - "greater on VFAT") - - livesize = misc.get_file_size(isodir + "/LiveOS") - - usbimgsize = (overlaysizemb + \ - homesizemb + \ - swapsizemb + \ - livesize + \ - plussize) * 1024L * 1024L - - disk = fs_related.SparseLoopbackDisk("%s/%s.usbimg" \ - % (self._outdir, self.name), - usbimgsize) - usbmnt = self._mkdtemp("usb-mnt") - usbloop = PartitionedMount(usbmnt) - usbloop.add_disk('/dev/sdb', disk) - - usbloop.add_partition(usbimgsize/1024/1024, - "/dev/sdb", - "/", - fstype, - boot=True) - - usbloop.mount() - - try: - fs_related.makedirs(usbmnt + "/LiveOS") - - if os.path.exists(isodir + "/LiveOS/squashfs.img"): - shutil.copyfile(isodir + "/LiveOS/squashfs.img", - usbmnt + "/LiveOS/squashfs.img") - else: - fs_related.mksquashfs(os.path.dirname(self._image), - usbmnt + "/LiveOS/squashfs.img") - - if os.path.exists(isodir + "/LiveOS/osmin.img"): - shutil.copyfile(isodir + "/LiveOS/osmin.img", - usbmnt + "/LiveOS/osmin.img") - - if fstype == "vfat" or fstype == "msdos": - uuid = usbloop.partitions[0]['mount'].uuid - label = usbloop.partitions[0]['mount'].fslabel - usblabel = "UUID=%s" % (uuid) - overlaysuffix = "-%s-%s" % (label, uuid) - else: - diskmount = usbloop.partitions[0]['mount'] - usblabel = "UUID=%s" % diskmount.uuid - overlaysuffix = "-%s-%s" % (diskmount.fslabel, diskmount.uuid) - - args = ['cp', "-Rf", isodir + "/isolinux", usbmnt + "/syslinux"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't copy isolinux directory %s" \ - % (isodir + "/isolinux/*")) - - if os.path.isfile("/usr/share/syslinux/isolinux.bin"): - syslinux_path = "/usr/share/syslinux" - elif os.path.isfile("/usr/lib/syslinux/isolinux.bin"): - syslinux_path = "/usr/lib/syslinux" - else: - raise CreatorError("syslinux not installed : " - "cannot find syslinux installation path") - - for f in ("isolinux.bin", "vesamenu.c32"): - path = os.path.join(syslinux_path, f) - if os.path.isfile(path): - args = ['cp', path, usbmnt + "/syslinux/"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't copy syslinux file " + path) - else: - raise CreatorError("syslinux not installed: " - "syslinux file %s not found" % path) - - fd = open(isodir + "/isolinux/isolinux.cfg", "r") - text = fd.read() - fd.close() - pattern = re.compile('CDLABEL=[^ ]*') - text = pattern.sub(usblabel, text) - pattern = re.compile('rootfstype=[^ ]*') - text = pattern.sub("rootfstype=" + fstype, text) - if kernelargs: - text = text.replace("rd.live.image", "rd.live.image " + kernelargs) - - if overlaysizemb > 0: - msger.info("Initializing persistent overlay file") - overfile = "overlay" + overlaysuffix - if fstype == "vfat": - args = ['dd', - "if=/dev/zero", - "of=" + usbmnt + "/LiveOS/" + overfile, - "count=%d" % overlaysizemb, - "bs=1M"] - else: - args = ['dd', - "if=/dev/null", - "of=" + usbmnt + "/LiveOS/" + overfile, - "count=1", - "bs=1M", - "seek=%d" % overlaysizemb] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create overlay file") - text = text.replace("rd.live.image", "rd.live.image rd.live.overlay=" + usblabel) - text = text.replace(" ro ", " rw ") - - if swapsizemb > 0: - msger.info("Initializing swap file") - swapfile = usbmnt + "/LiveOS/" + "swap.img" - args = ['dd', - "if=/dev/zero", - "of=" + swapfile, - "count=%d" % swapsizemb, - "bs=1M"] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create swap file") - args = ["mkswap", "-f", swapfile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't mkswap on swap file") - - if homesizemb > 0: - msger.info("Initializing persistent /home") - homefile = usbmnt + "/LiveOS/" + homefile - if fstype == "vfat": - args = ['dd', - "if=/dev/zero", - "of=" + homefile, - "count=%d" % homesizemb, - "bs=1M"] - else: - args = ['dd', - "if=/dev/null", - "of=" + homefile, - "count=1", - "bs=1M", - "seek=%d" % homesizemb] - rc = runner.show(args) - if rc: - raise CreatorError("Can't create home file") - - mkfscmd = fs_related.find_binary_path("/sbin/mkfs." + fstype) - if fstype == "ext2" or fstype == "ext3": - args = [mkfscmd, "-F", "-j", homefile] - else: - args = [mkfscmd, homefile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't mke2fs home file") - if fstype == "ext2" or fstype == "ext3": - tune2fs = fs_related.find_binary_path("tune2fs") - args = [tune2fs, - "-c0", - "-i0", - "-ouser_xattr,acl", - homefile] - rc = runner.show(args) - if rc: - raise CreatorError("Can't tune2fs home file") - - if fstype == "vfat" or fstype == "msdos": - syslinuxcmd = fs_related.find_binary_path("syslinux") - syslinuxcfg = usbmnt + "/syslinux/syslinux.cfg" - args = [syslinuxcmd, - "-d", - "syslinux", - usbloop.partitions[0]["device"]] - - elif fstype == "ext2" or fstype == "ext3": - extlinuxcmd = fs_related.find_binary_path("extlinux") - syslinuxcfg = usbmnt + "/syslinux/extlinux.conf" - args = [extlinuxcmd, - "-i", - usbmnt + "/syslinux"] - - else: - raise CreatorError("Invalid file system type: %s" % (fstype)) - - os.unlink(usbmnt + "/syslinux/isolinux.cfg") - fd = open(syslinuxcfg, "w") - fd.write(text) - fd.close() - rc = runner.show(args) - if rc: - raise CreatorError("Can't install boot loader.") - - finally: - usbloop.unmount() - usbloop.cleanup() - - # Need to do this after image is unmounted and device mapper is closed - msger.info("set MBR") - mbrfile = "/usr/lib/syslinux/mbr.bin" - if not os.path.exists(mbrfile): - mbrfile = "/usr/share/syslinux/mbr.bin" - if not os.path.exists(mbrfile): - raise CreatorError("mbr.bin file didn't exist.") - mbrsize = os.path.getsize(mbrfile) - outimg = "%s/%s.usbimg" % (self._outdir, self.name) - - args = ['dd', - "if=" + mbrfile, - "of=" + outimg, - "seek=0", - "conv=notrunc", - "bs=1", - "count=%d" % (mbrsize)] - rc = runner.show(args) - if rc: - raise CreatorError("Can't set MBR.") - - def _stage_final_image(self): - try: - isodir = self._get_isodir() - fs_related.makedirs(isodir + "/LiveOS") - - minimal_size = self._resparse() - - if not self.skip_minimize: - fs_related.create_image_minimizer(isodir + "/LiveOS/osmin.img", - self._image, - minimal_size) - - if self.skip_compression: - shutil.move(self._image, - isodir + "/LiveOS/ext3fs.img") - else: - fs_related.makedirs(os.path.join( - os.path.dirname(self._image), - "LiveOS")) - shutil.move(self._image, - os.path.join(os.path.dirname(self._image), - "LiveOS", "ext3fs.img")) - fs_related.mksquashfs(os.path.dirname(self._image), - isodir + "/LiveOS/squashfs.img") - - self._create_usbimg(isodir) - - if self.pack_to: - self.image_files.update({'image_files': self.pack_to}) - usbimg = os.path.join(self._outdir, self.name + ".usbimg") - packimg = os.path.join(self._outdir, self.pack_to) - misc.packing(packimg, usbimg) - os.unlink(usbimg) - else: - self.image_files.update({'image_files': self.name + ".usbimg"}) - - finally: - shutil.rmtree(isodir, ignore_errors = True) - self._set_isodir(None) - diff --git a/mic/imager/loop.py b/mic/imager/loop.py index 049406c..51d8181 100644 --- a/mic/imager/loop.py +++ b/mic/imager/loop.py @@ -23,6 +23,7 @@ from mic import kickstart, msger from mic.utils.errors import CreatorError, MountError from mic.utils import misc, runner, fs_related as fs from mic.imager.baseimager import BaseImageCreator +from mic.archive import packing, compressing # The maximum string length supported for LoopImageCreator.fslabel @@ -164,7 +165,7 @@ class LoopImageCreator(BaseImageCreator): self.__fsopts = None self._instloops = [] - self.__imgdir = None + self._imgdir = None if self.ks: self.__image_size = kickstart.get_image_size(self.ks, @@ -212,9 +213,9 @@ class LoopImageCreator(BaseImageCreator): fslabel = property(__get_fslabel, __set_fslabel) def __get_image(self): - if self.__imgdir is None: + if self._imgdir is None: raise CreatorError("_image is not valid before calling mount()") - return os.path.join(self.__imgdir, self._img_name) + return os.path.join(self._imgdir, self._img_name) #The location of the image file. # #This is the path to the filesystem image. Subclasses may use this path @@ -303,8 +304,8 @@ class LoopImageCreator(BaseImageCreator): shutil.copyfile(base_on, self._image) def _check_imgdir(self): - if self.__imgdir is None: - self.__imgdir = self._mkdtemp() + if self._imgdir is None: + self._imgdir = self._mkdtemp() # @@ -313,7 +314,7 @@ class LoopImageCreator(BaseImageCreator): def _mount_instroot(self, base_on=None): if base_on and os.path.isfile(base_on): - self.__imgdir = os.path.dirname(base_on) + self._imgdir = os.path.dirname(base_on) imgname = os.path.basename(base_on) self._base_on(base_on) self._set_image_size(misc.get_file_size(self._image)) @@ -349,7 +350,7 @@ class LoopImageCreator(BaseImageCreator): raise MountError('Cannot support fstype: %s' % fstype) loop['loop'] = MyDiskMount(fs.SparseLoopbackDisk( - os.path.join(self.__imgdir, imgname), + os.path.join(self._imgdir, imgname), size), mp, fstype, @@ -387,27 +388,27 @@ class LoopImageCreator(BaseImageCreator): self._resparse() for item in self._instloops: - imgfile = os.path.join(self.__imgdir, item['name']) + imgfile = os.path.join(self._imgdir, item['name']) if item['fstype'] == "ext4": runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s ' % imgfile) self.image_files.setdefault('partitions', {}).update( {item['mountpoint']: item['label']}) if self.compress_image: - misc.compressing(imgfile, self.compress_image) + compressing(imgfile, self.compress_image) self.image_files.setdefault('image_files', []).append( '.'.join([item['name'], self.compress_image])) else: self.image_files.setdefault('image_files', []).append(item['name']) if not self.pack_to: - for item in os.listdir(self.__imgdir): - shutil.move(os.path.join(self.__imgdir, item), + for item in os.listdir(self._imgdir): + shutil.move(os.path.join(self._imgdir, item), os.path.join(self._outdir, item)) else: msger.info("Pack all loop images together to %s" % self.pack_to) dstfile = os.path.join(self._outdir, self.pack_to) - misc.packing(dstfile, self.__imgdir) + packing(dstfile, self._imgdir) self.image_files['image_files'] = [self.pack_to] @@ -431,7 +432,7 @@ class LoopImageCreator(BaseImageCreator): for item in self._attachment: if not os.path.exists(item): continue - dpath = os.path.join(self.__imgdir, os.path.basename(item)) + dpath = os.path.join(self._imgdir, os.path.basename(item)) msger.verbose("Copy attachment %s to %s" % (item, dpath)) shutil.copy(item, dpath) diff --git a/mic/imager/raw.py b/mic/imager/raw.py deleted file mode 100644 index 6b53ac6..0000000 --- a/mic/imager/raw.py +++ /dev/null @@ -1,596 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import stat -import shutil - -from mic import kickstart, msger -from mic.utils import fs_related, runner, misc -from mic.utils.partitionedfs import PartitionedMount -from mic.utils.errors import CreatorError, MountError -from mic.imager.baseimager import BaseImageCreator - - -class RawImageCreator(BaseImageCreator): - """Installs a system into a file containing a partitioned disk image. - - ApplianceImageCreator is an advanced ImageCreator subclass; a sparse file - is formatted with a partition table, each partition loopback mounted - and the system installed into an virtual disk. The disk image can - subsequently be booted in a virtual machine or accessed with kpartx - """ - img_format = 'raw' - - def __init__(self, creatoropts=None, pkgmgr=None, compress_image=None, generate_bmap=None, fstab_entry="uuid"): - """Initialize a ApplianceImageCreator instance. - - This method takes the same arguments as ImageCreator.__init__() - """ - BaseImageCreator.__init__(self, creatoropts, pkgmgr) - - self.__instloop = None - self.__imgdir = None - self.__disks = {} - self.__disk_format = "raw" - self._disk_names = [] - self._ptable_format = self.ks.handler.bootloader.ptable - self.vmem = 512 - self.vcpu = 1 - self.checksum = False - self.use_uuid = fstab_entry == "uuid" - self.appliance_version = None - self.appliance_release = None - self.compress_image = compress_image - self.bmap_needed = generate_bmap - self._need_extlinux = not kickstart.use_installerfw(self.ks, "bootloader") - #self.getsource = False - #self.listpkg = False - - self._dep_checks.extend(["sync", "kpartx", "parted"]) - if self._need_extlinux: - self._dep_checks.extend(["extlinux"]) - - def configure(self, repodata = None): - import subprocess - def chroot(): - os.chroot(self._instroot) - os.chdir("/") - - if os.path.exists(self._instroot + "/usr/bin/Xorg"): - subprocess.call(["/bin/chmod", "u+s", "/usr/bin/Xorg"], - preexec_fn = chroot) - - BaseImageCreator.configure(self, repodata) - - def _get_fstab(self): - s = "" - for mp in self.__instloop.mountOrder: - p = None - for p1 in self.__instloop.partitions: - if p1['mountpoint'] == mp: - p = p1 - break - - if self.use_uuid and p['uuid']: - device = "UUID=%s" % p['uuid'] - else: - device = "/dev/%s%-d" % (p['disk_name'], p['num']) - - s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % { - 'device': device, - 'mountpoint': p['mountpoint'], - 'fstype': p['fstype'], - 'fsopts': "defaults,noatime" if not p['fsopts'] else p['fsopts']} - - if p['mountpoint'] == "/": - for subvol in self.__instloop.subvolumes: - if subvol['mountpoint'] == "/": - continue - s += "%(device)s %(mountpoint)s %(fstype)s %(fsopts)s 0 0\n" % { - 'device': "/dev/%s%-d" % (p['disk_name'], p['num']), - 'mountpoint': subvol['mountpoint'], - 'fstype': p['fstype'], - 'fsopts': "defaults,noatime" if not subvol['fsopts'] else subvol['fsopts']} - - s += "devpts /dev/pts devpts gid=5,mode=620 0 0\n" - s += "tmpfs /dev/shm tmpfs defaults 0 0\n" - s += "proc /proc proc defaults 0 0\n" - s += "sysfs /sys sysfs defaults 0 0\n" - return s - - def _create_mkinitrd_config(self): - """write to tell which modules to be included in initrd""" - - mkinitrd = "" - mkinitrd += "PROBE=\"no\"\n" - mkinitrd += "MODULES+=\"ext3 ata_piix sd_mod libata scsi_mod\"\n" - mkinitrd += "rootfs=\"ext3\"\n" - mkinitrd += "rootopts=\"defaults\"\n" - - msger.debug("Writing mkinitrd config %s/etc/sysconfig/mkinitrd" \ - % self._instroot) - os.makedirs(self._instroot + "/etc/sysconfig/",mode=644) - cfg = open(self._instroot + "/etc/sysconfig/mkinitrd", "w") - cfg.write(mkinitrd) - cfg.close() - - def _get_parts(self): - if not self.ks: - raise CreatorError("Failed to get partition info, " - "please check your kickstart setting.") - - # Set a default partition if no partition is given out - if not self.ks.handler.partition.partitions: - partstr = "part / --size 1900 --ondisk sda --fstype=ext3" - args = partstr.split() - pd = self.ks.handler.partition.parse(args[1:]) - if pd not in self.ks.handler.partition.partitions: - self.ks.handler.partition.partitions.append(pd) - - # partitions list from kickstart file - return kickstart.get_partitions(self.ks) - - def get_disk_names(self): - """ Returns a list of physical target disk names (e.g., 'sdb') which - will be created. """ - - if self._disk_names: - return self._disk_names - - #get partition info from ks handler - parts = self._get_parts() - - for i in range(len(parts)): - if parts[i].disk: - disk_name = parts[i].disk - else: - raise CreatorError("Failed to create disks, no --ondisk " - "specified in partition line of ks file") - - if parts[i].mountpoint and not parts[i].fstype: - raise CreatorError("Failed to create disks, no --fstype " - "specified for partition with mountpoint " - "'%s' in the ks file") - - self._disk_names.append(disk_name) - - return self._disk_names - - def _full_name(self, name, extention): - """ Construct full file name for a file we generate. """ - return "%s-%s.%s" % (self.name, name, extention) - - def _full_path(self, path, name, extention): - """ Construct full file path to a file we generate. """ - return os.path.join(path, self._full_name(name, extention)) - - # - # Actual implemention - # - def _mount_instroot(self, base_on = None): - parts = self._get_parts() - self.__instloop = PartitionedMount(self._instroot) - - for p in parts: - self.__instloop.add_partition(int(p.size), - p.disk, - p.mountpoint, - p.fstype, - p.label, - fsopts = p.fsopts, - boot = p.active, - align = p.align, - part_type = p.part_type) - - self.__instloop.layout_partitions(self._ptable_format) - - # Create the disks - self.__imgdir = self._mkdtemp() - for disk_name, disk in self.__instloop.disks.items(): - full_path = self._full_path(self.__imgdir, disk_name, "raw") - msger.debug("Adding disk %s as %s with size %s bytes" \ - % (disk_name, full_path, disk['min_size'])) - - disk_obj = fs_related.SparseLoopbackDisk(full_path, - disk['min_size']) - self.__disks[disk_name] = disk_obj - self.__instloop.add_disk(disk_name, disk_obj) - - self.__instloop.mount() - self._create_mkinitrd_config() - - def mount(self, base_on = None, cachedir = None): - """ - This method calls the base class' 'mount()' method and then creates - block device nodes corresponding to the image's partitions in the image - itself. Namely, the image has /dev/loopX device corresponding to the - entire image, and per-partition /dev/mapper/* devices. - - We copy these files to image's "/dev" directory in order to enable - scripts which run in the image chroot environment to access own raw - partitions. For example, this can be used to install the bootloader to - the MBR (say, from an installer framework plugin). - """ - - def copy_devnode(src, dest): - """A helper function for copying device nodes.""" - - if not src: - return - - stat_obj = os.stat(src) - assert stat.S_ISBLK(stat_obj.st_mode) - - os.mknod(dest, stat_obj.st_mode, - os.makedev(os.major(stat_obj.st_rdev), - os.minor(stat_obj.st_rdev))) - # os.mknod uses process umask may create a nod with different - # permissions, so we use os.chmod to make sure permissions are - # correct. - os.chmod(dest, stat_obj.st_mode) - - BaseImageCreator.mount(self, base_on, cachedir) - - # Copy the disk loop devices - for name in self.__disks.keys(): - loopdev = self.__disks[name].device - copy_devnode(loopdev, self._instroot + loopdev) - - # Copy per-partition dm nodes - os.mkdir(self._instroot + "/dev/mapper", os.stat("/dev/mapper").st_mode) - for p in self.__instloop.partitions: - copy_devnode(p['mapper_device'], - self._instroot + p['mapper_device']) - - def unmount(self): - """ - Remove loop/dm device nodes which we created in 'mount()' and call the - base class' 'unmount()' method. - """ - - for p in self.__instloop.partitions: - if p['mapper_device']: - path = self._instroot + p['mapper_device'] - if os.path.exists(path): - os.unlink(path) - - path = self._instroot + "/dev/mapper" - if os.path.exists(path): - os.rmdir(path) - - for name in self.__disks.keys(): - if self.__disks[name].device: - path = self._instroot + self.__disks[name].device - if os.path.exists(path): - os.unlink(path) - - BaseImageCreator.unmount(self) - - def _get_required_packages(self): - required_packages = BaseImageCreator._get_required_packages(self) - if self._need_extlinux: - if not self.target_arch or not self.target_arch.startswith("arm"): - required_packages += ["syslinux", "syslinux-extlinux"] - return required_packages - - def _get_excluded_packages(self): - return BaseImageCreator._get_excluded_packages(self) - - def _get_syslinux_boot_config(self): - rootdev = None - root_part_uuid = None - for p in self.__instloop.partitions: - if p['mountpoint'] == "/": - rootdev = "/dev/%s%-d" % (p['disk_name'], p['num']) - root_part_uuid = p['partuuid'] - - return (rootdev, root_part_uuid) - - def _create_syslinux_config(self): - - splash = os.path.join(self._instroot, "boot/extlinux") - if os.path.exists(splash): - splashline = "menu background splash.jpg" - else: - splashline = "" - - (rootdev, root_part_uuid) = self._get_syslinux_boot_config() - options = self.ks.handler.bootloader.appendLine - - #XXX don't hardcode default kernel - see livecd code - syslinux_conf = "" - syslinux_conf += "prompt 0\n" - syslinux_conf += "timeout 1\n" - syslinux_conf += "\n" - syslinux_conf += "default vesamenu.c32\n" - syslinux_conf += "menu autoboot Starting %s...\n" % self.distro_name - syslinux_conf += "menu hidden\n" - syslinux_conf += "\n" - syslinux_conf += "%s\n" % splashline - syslinux_conf += "menu title Welcome to %s!\n" % self.distro_name - syslinux_conf += "menu color border 0 #ffffffff #00000000\n" - syslinux_conf += "menu color sel 7 #ffffffff #ff000000\n" - syslinux_conf += "menu color title 0 #ffffffff #00000000\n" - syslinux_conf += "menu color tabmsg 0 #ffffffff #00000000\n" - syslinux_conf += "menu color unsel 0 #ffffffff #00000000\n" - syslinux_conf += "menu color hotsel 0 #ff000000 #ffffffff\n" - syslinux_conf += "menu color hotkey 7 #ffffffff #ff000000\n" - syslinux_conf += "menu color timeout_msg 0 #ffffffff #00000000\n" - syslinux_conf += "menu color timeout 0 #ffffffff #00000000\n" - syslinux_conf += "menu color cmdline 0 #ffffffff #00000000\n" - - versions = [] - kernels = self._get_kernel_versions() - symkern = "%s/boot/vmlinuz" % self._instroot - - if os.path.lexists(symkern): - v = os.path.realpath(symkern).replace('%s-' % symkern, "") - syslinux_conf += "label %s\n" % self.distro_name.lower() - syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v) - syslinux_conf += "\tlinux ../vmlinuz\n" - if self._ptable_format == 'msdos': - rootstr = rootdev - else: - if not root_part_uuid: - raise MountError("Cannot find the root GPT partition UUID") - rootstr = "PARTUUID=%s" % root_part_uuid - syslinux_conf += "\tappend ro root=%s %s\n" % (rootstr, options) - syslinux_conf += "\tmenu default\n" - else: - for kernel in kernels: - for version in kernels[kernel]: - versions.append(version) - - footlabel = 0 - for v in versions: - syslinux_conf += "label %s%d\n" \ - % (self.distro_name.lower(), footlabel) - syslinux_conf += "\tmenu label %s (%s)\n" % (self.distro_name, v) - syslinux_conf += "\tlinux ../vmlinuz-%s\n" % v - syslinux_conf += "\tappend ro root=%s %s\n" \ - % (rootdev, options) - if footlabel == 0: - syslinux_conf += "\tmenu default\n" - footlabel += 1; - - msger.debug("Writing syslinux config %s/boot/extlinux/extlinux.conf" \ - % self._instroot) - cfg = open(self._instroot + "/boot/extlinux/extlinux.conf", "w") - cfg.write(syslinux_conf) - cfg.close() - - def _install_syslinux(self): - for name in self.__disks.keys(): - loopdev = self.__disks[name].device - - # Set MBR - mbrfile = "%s/usr/share/syslinux/" % self._instroot - if self._ptable_format == 'gpt': - mbrfile += "gptmbr.bin" - else: - mbrfile += "mbr.bin" - - msger.debug("Installing syslinux bootloader '%s' to %s" % \ - (mbrfile, loopdev)) - - rc = runner.show(['dd', 'if=%s' % mbrfile, 'of=' + loopdev]) - if rc != 0: - raise MountError("Unable to set MBR to %s" % loopdev) - - - # Ensure all data is flushed to disk before doing syslinux install - runner.quiet('sync') - - fullpathsyslinux = fs_related.find_binary_path("extlinux") - rc = runner.show([fullpathsyslinux, - "-i", - "%s/boot/extlinux" % self._instroot]) - if rc != 0: - raise MountError("Unable to install syslinux bootloader to %s" \ - % loopdev) - - def _create_bootconfig(self): - #If syslinux is available do the required configurations. - if self._need_extlinux \ - and os.path.exists("%s/usr/share/syslinux/" % (self._instroot)) \ - and os.path.exists("%s/boot/extlinux/" % (self._instroot)): - self._create_syslinux_config() - self._install_syslinux() - - def _unmount_instroot(self): - if not self.__instloop is None: - try: - self.__instloop.cleanup() - except MountError, err: - msger.warning("%s" % err) - - def _resparse(self, size = None): - return self.__instloop.resparse(size) - - def _get_post_scripts_env(self, in_chroot): - env = BaseImageCreator._get_post_scripts_env(self, in_chroot) - - # Export the file-system UUIDs and partition UUIDs (AKA PARTUUIDs) - for p in self.__instloop.partitions: - env.update(self._set_part_env(p['ks_pnum'], "UUID", p['uuid'])) - env.update(self._set_part_env(p['ks_pnum'], "PARTUUID", p['partuuid'])) - env.update(self._set_part_env(p['ks_pnum'], "DEVNODE_NOW", - p['mapper_device'])) - env.update(self._set_part_env(p['ks_pnum'], "DISK_DEVNODE_NOW", - self.__disks[p['disk_name']].device)) - - return env - - def _stage_final_image(self): - """Stage the final system image in _outdir. - write meta data - """ - self._resparse() - self.image_files.update({'disks': self.__disks.keys()}) - - if not (self.compress_image or self.pack_to): - for imgfile in os.listdir(self.__imgdir): - if imgfile.endswith('.raw'): - for disk in self.__disks.keys(): - if imgfile.find(disk) != -1: - self.image_files.setdefault(disk, {}).update( - {'image': imgfile}) - self.image_files.setdefault('image_files', - []).append(imgfile) - - if self.compress_image: - for imgfile in os.listdir(self.__imgdir): - if imgfile.endswith('.raw') or imgfile.endswith('bin'): - imgpath = os.path.join(self.__imgdir, imgfile) - msger.info("Compressing image %s" % imgfile) - misc.compressing(imgpath, self.compress_image) - if imgfile.endswith('.raw') and not self.pack_to: - for disk in self.__disks.keys(): - if imgfile.find(disk) != -1: - imgname = '%s.%s' % (imgfile, self.compress_image) - self.image_files.setdefault(disk, {}).update( - {'image': imgname}) - self.image_files.setdefault('image_files', - []).append(imgname) - - if self.pack_to: - dst = os.path.join(self._outdir, self.pack_to) - msger.info("Pack all raw images to %s" % dst) - misc.packing(dst, self.__imgdir) - self.image_files.update({'image_files': self.pack_to}) - else: - msger.debug("moving disks to stage location") - for imgfile in os.listdir(self.__imgdir): - src = os.path.join(self.__imgdir, imgfile) - dst = os.path.join(self._outdir, imgfile) - msger.debug("moving %s to %s" % (src,dst)) - shutil.move(src,dst) - - self._write_image_xml() - - def _write_image_xml(self): - imgarch = "i686" - if self.target_arch and self.target_arch.startswith("arm"): - imgarch = "arm" - xml = "<image>\n" - - name_attributes = "" - if self.appliance_version: - name_attributes += " version='%s'" % self.appliance_version - if self.appliance_release: - name_attributes += " release='%s'" % self.appliance_release - xml += " <name%s>%s</name>\n" % (name_attributes, self.name) - xml += " <domain>\n" - # XXX don't hardcode - determine based on the kernel we installed for - # grub baremetal vs xen - xml += " <boot type='hvm'>\n" - xml += " <guest>\n" - xml += " <arch>%s</arch>\n" % imgarch - xml += " </guest>\n" - xml += " <os>\n" - xml += " <loader dev='hd'/>\n" - xml += " </os>\n" - - i = 0 - for name in self.__disks.keys(): - full_name = self._full_name(name, self.__disk_format) - xml += " <drive disk='%s' target='hd%s'/>\n" \ - % (full_name, chr(ord('a') + i)) - i = i + 1 - - xml += " </boot>\n" - xml += " <devices>\n" - xml += " <vcpu>%s</vcpu>\n" % self.vcpu - xml += " <memory>%d</memory>\n" %(self.vmem * 1024) - for network in self.ks.handler.network.network: - xml += " <interface/>\n" - xml += " <graphics/>\n" - xml += " </devices>\n" - xml += " </domain>\n" - xml += " <storage>\n" - - if self.checksum is True: - for name in self.__disks.keys(): - diskpath = self._full_path(self._outdir, name, \ - self.__disk_format) - full_name = self._full_name(name, self.__disk_format) - - msger.debug("Generating disk signature for %s" % full_name) - - xml += " <disk file='%s' use='system' format='%s'>\n" \ - % (full_name, self.__disk_format) - - hashes = misc.calc_hashes(diskpath, ('sha1', 'sha256')) - - xml += " <checksum type='sha1'>%s</checksum>\n" \ - % hashes[0] - xml += " <checksum type='sha256'>%s</checksum>\n" \ - % hashes[1] - xml += " </disk>\n" - else: - for name in self.__disks.keys(): - full_name = self._full_name(name, self.__disk_format) - xml += " <disk file='%s' use='system' format='%s'/>\n" \ - % (full_name, self.__disk_format) - - xml += " </storage>\n" - xml += "</image>\n" - - msger.debug("writing image XML to %s/%s.xml" %(self._outdir, self.name)) - cfg = open("%s/%s.xml" % (self._outdir, self.name), "w") - cfg.write(xml) - cfg.close() - - def generate_bmap(self): - """ Generate block map file for the image. The idea is that while disk - images we generate may be large (e.g., 4GiB), they may actually contain - only little real data, e.g., 512MiB. This data are files, directories, - file-system meta-data, partition table, etc. In other words, when - flashing the image to the target device, you do not have to copy all the - 4GiB of data, you can copy only 512MiB of it, which is 4 times faster. - - This function generates the block map file for an arbitrary image that - mic has generated. The block map file is basically an XML file which - contains a list of blocks which have to be copied to the target device. - The other blocks are not used and there is no need to copy them. """ - - if self.bmap_needed is None: - return - - from mic.utils import BmapCreate - msger.info("Generating the map file(s)") - - for name in self.__disks.keys(): - image = self._full_path(self.__imgdir, name, self.__disk_format) - bmap_file = self._full_path(self._outdir, name, "bmap") - self.image_files.setdefault(name, {}).update({'bmap': \ - os.path.basename(bmap_file)}) - - msger.debug("Generating block map file '%s'" % bmap_file) - - try: - creator = BmapCreate.BmapCreate(image, bmap_file) - creator.generate() - del creator - except BmapCreate.Error as err: - raise CreatorError("Failed to create bmap file: %s" % str(err)) - - def create_manifest(self): - if self.compress_image: - self.image_files.update({'compress': self.compress_image}) - super(RawImageCreator, self).create_manifest() diff --git a/mic/utils/BmapCreate.py b/mic/utils/BmapCreate.py deleted file mode 100644 index 6934f1a..0000000 --- a/mic/utils/BmapCreate.py +++ /dev/null @@ -1,354 +0,0 @@ -# Copyright (c) 2012-2013 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License, version 2, -# as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -""" -This module implements the block map (bmap) creation functionality and provides -the corresponding API in form of the 'BmapCreate' class. - -The idea is that while images files may generally be very large (e.g., 4GiB), -they may nevertheless contain only little real data, e.g., 512MiB. This data -are files, directories, file-system meta-data, partition table, etc. When -copying the image to the target device, you do not have to copy all the 4GiB of -data, you can copy only 512MiB of it, which is 4 times less, so copying should -presumably be 4 times faster. - -The block map file is an XML file which contains a list of blocks which have to -be copied to the target device. The other blocks are not used and there is no -need to copy them. The XML file also contains some additional information like -block size, image size, count of mapped blocks, etc. There are also many -commentaries, so it is human-readable. - -The image has to be a sparse file. Generally, this means that when you generate -this image file, you should start with a huge sparse file which contains a -single hole spanning the entire file. Then you should partition it, write all -the data (probably by means of loop-back mounting the image or parts of it), -etc. The end result should be a sparse file where mapped areas represent useful -parts of the image and holes represent useless parts of the image, which do not -have to be copied when copying the image to the target device. - -This module uses the FIBMAP ioctl to detect holes. -""" - -# Disable the following pylint recommendations: -# * Too many instance attributes - R0902 -# * Too few public methods - R0903 -# pylint: disable=R0902,R0903 - -import hashlib -import logging -from mic.utils.misc import human_size -from mic.utils import Filemap - -# The bmap format version we generate. -# -# Changelog: -# o 1.3 -> 2.0: -# Support SHA256 and SHA512 checksums, in 1.3 only SHA1 was supported. -# "BmapFileChecksum" is used instead of "BmapFileSHA1", and "chksum=" -# attribute is used instead "sha1=". Introduced "ChecksumType" tag. This is -# an incompatible change. -# Note, bmap format 1.4 is identical to 2.0. Version 1.4 was a mistake, -# instead of incrementing the major version number, we incremented minor -# version number. Unfortunately, the mistake slipped into bmap-tools version -# 3.0, and was only fixed in bmap-tools v3.1. -SUPPORTED_BMAP_VERSION = "2.0" - -_BMAP_START_TEMPLATE = \ -"""<?xml version="1.0" ?> -<!-- This file contains the block map for an image file, which is basically - a list of useful (mapped) block numbers in the image file. In other words, - it lists only those blocks which contain data (boot sector, partition - table, file-system metadata, files, directories, extents, etc). These - blocks have to be copied to the target device. The other blocks do not - contain any useful data and do not have to be copied to the target - device. - - The block map an optimization which allows to copy or flash the image to - the image quicker than copying of flashing the entire image. This is - because with bmap less data is copied: <MappedBlocksCount> blocks instead - of <BlocksCount> blocks. - - Besides the machine-readable data, this file contains useful commentaries - which contain human-readable information like image size, percentage of - mapped data, etc. - - The 'version' attribute is the block map file format version in the - 'major.minor' format. The version major number is increased whenever an - incompatible block map format change is made. The minor number changes - in case of minor backward-compatible changes. --> - -<bmap version="%s"> - <!-- Image size in bytes: %s --> - <ImageSize> %u </ImageSize> - - <!-- Size of a block in bytes --> - <BlockSize> %u </BlockSize> - - <!-- Count of blocks in the image file --> - <BlocksCount> %u </BlocksCount> - -""" - -class Error(Exception): - """ - A class for exceptions generated by this module. We currently support only - one type of exceptions, and we basically throw human-readable problem - description in case of errors. - """ - pass - -class BmapCreate(object): - """ - This class implements the bmap creation functionality. To generate a bmap - for an image (which is supposedly a sparse file), you should first create - an instance of 'BmapCreate' and provide: - - * full path or a file-like object of the image to create bmap for - * full path or a file object to use for writing the results to - - Then you should invoke the 'generate()' method of this class. It will use - the FIEMAP ioctl to generate the bmap. - """ - - def __init__(self, image, bmap, chksum_type="sha256", log=None): - """ - Initialize a class instance: - * image - full path or a file-like object of the image to create bmap - for - * bmap - full path or a file object to use for writing the resulting - bmap to - * chksum - type of the check sum to use in the bmap file (all checksum - types which python's "hashlib" module supports are allowed). - * log - the logger object to use for printing messages. - """ - - self._log = log - if self._log is None: - self._log = logging.getLogger(__name__) - - self.image_size = None - self.image_size_human = None - self.block_size = None - self.blocks_cnt = None - self.mapped_cnt = None - self.mapped_size = None - self.mapped_size_human = None - self.mapped_percent = None - - self._mapped_count_pos1 = None - self._mapped_count_pos2 = None - self._chksum_pos = None - - self._f_image_needs_close = False - self._f_bmap_needs_close = False - - self._cs_type = chksum_type.lower() - try: - self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) - except ValueError as err: - raise Error("cannot initialize hash function \"%s\": %s" % - (self._cs_type, err)) - - if hasattr(image, "read"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - if hasattr(bmap, "read"): - self._f_bmap = bmap - self._bmap_path = bmap.name - else: - self._bmap_path = bmap - self._open_bmap_file() - - try: - self.filemap = Filemap.filemap(self._f_image, self._log) - except (Filemap.Error, Filemap.ErrorNotSupp) as err: - raise Error("cannot generate bmap: %s" % err) - - self.image_size = self.filemap.image_size - self.image_size_human = human_size(self.image_size) - if self.image_size == 0: - raise Error("cannot generate bmap for zero-sized image file '%s'" - % self._image_path) - - self.block_size = self.filemap.block_size - self.blocks_cnt = self.filemap.blocks_cnt - - def __del__(self): - """The class destructor which closes the opened files.""" - if self._f_image_needs_close: - self._f_image.close() - if self._f_bmap_needs_close: - self._f_bmap.close() - - def _open_image_file(self): - """Open the image file.""" - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" - % (self._image_path, err)) - - self._f_image_needs_close = True - - def _open_bmap_file(self): - """Open the bmap file.""" - try: - self._f_bmap = open(self._bmap_path, 'w+') - except IOError as err: - raise Error("cannot open bmap file '%s': %s" - % (self._bmap_path, err)) - - self._f_bmap_needs_close = True - - def _bmap_file_start(self): - """ - A helper function which generates the starting contents of the block - map file: the header comment, image size, block size, etc. - """ - - # We do not know the amount of mapped blocks at the moment, so just put - # whitespaces instead of real numbers. Assume the longest possible - # numbers. - - xml = _BMAP_START_TEMPLATE \ - % (SUPPORTED_BMAP_VERSION, self.image_size_human, - self.image_size, self.block_size, self.blocks_cnt) - xml += " <!-- Count of mapped blocks: " - - self._f_bmap.write(xml) - self._mapped_count_pos1 = self._f_bmap.tell() - - xml = "%s or %s -->\n" % (' ' * len(self.image_size_human), - ' ' * len("100.0%")) - xml += " <MappedBlocksCount> " - - self._f_bmap.write(xml) - self._mapped_count_pos2 = self._f_bmap.tell() - - xml = "%s </MappedBlocksCount>\n\n" % (' ' * len(str(self.blocks_cnt))) - - # pylint: disable=C0301 - xml += " <!-- Type of checksum used in this file -->\n" - xml += " <ChecksumType> %s </ChecksumType>\n\n" % self._cs_type - - xml += " <!-- The checksum of this bmap file. When it is calculated, the value of\n" - xml += " the checksum has be zero (all ASCII \"0\" symbols). -->\n" - xml += " <BmapFileChecksum> " - - self._f_bmap.write(xml) - self._chksum_pos = self._f_bmap.tell() - - xml = "0" * self._cs_len + " </BmapFileChecksum>\n\n" - xml += " <!-- The block map which consists of elements which may either be a\n" - xml += " range of blocks or a single block. The 'chksum' attribute\n" - xml += " (if present) is the checksum of this blocks range. -->\n" - xml += " <BlockMap>\n" - # pylint: enable=C0301 - - self._f_bmap.write(xml) - - def _bmap_file_end(self): - """ - A helper function which generates the final parts of the block map - file: the ending tags and the information about the amount of mapped - blocks. - """ - - xml = " </BlockMap>\n" - xml += "</bmap>\n" - - self._f_bmap.write(xml) - - self._f_bmap.seek(self._mapped_count_pos1) - self._f_bmap.write("%s or %.1f%%" - % (self.mapped_size_human, self.mapped_percent)) - - self._f_bmap.seek(self._mapped_count_pos2) - self._f_bmap.write("%u" % self.mapped_cnt) - - self._f_bmap.seek(0) - hash_obj = hashlib.new(self._cs_type) - hash_obj.update(self._f_bmap.read()) - chksum = hash_obj.hexdigest() - self._f_bmap.seek(self._chksum_pos) - self._f_bmap.write("%s" % chksum) - - def _calculate_chksum(self, first, last): - """ - A helper function which calculates checksum for the range of blocks of - the image file: from block 'first' to block 'last'. - """ - - start = first * self.block_size - end = (last + 1) * self.block_size - - self._f_image.seek(start) - hash_obj = hashlib.new(self._cs_type) - - chunk_size = 1024*1024 - to_read = end - start - read = 0 - - while read < to_read: - if read + chunk_size > to_read: - chunk_size = to_read - read - chunk = self._f_image.read(chunk_size) - hash_obj.update(chunk) - read += chunk_size - - return hash_obj.hexdigest() - - def generate(self, include_checksums=True): - """ - Generate bmap for the image file. If 'include_checksums' is 'True', - also generate checksums for block ranges. - """ - - # Save image file position in order to restore it at the end - image_pos = self._f_image.tell() - - self._bmap_file_start() - - # Generate the block map and write it to the XML block map - # file as we go. - self.mapped_cnt = 0 - for first, last in self.filemap.get_mapped_ranges(0, self.blocks_cnt): - self.mapped_cnt += last - first + 1 - if include_checksums: - chksum = self._calculate_chksum(first, last) - chksum = " chksum=\"%s\"" % chksum - else: - chksum = "" - - if first != last: - self._f_bmap.write(" <Range%s> %s-%s </Range>\n" - % (chksum, first, last)) - else: - self._f_bmap.write(" <Range%s> %s </Range>\n" - % (chksum, first)) - - self.mapped_size = self.mapped_cnt * self.block_size - self.mapped_size_human = human_size(self.mapped_size) - self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt - - self._bmap_file_end() - - try: - self._f_bmap.flush() - except IOError as err: - raise Error("cannot flush the bmap file '%s': %s" - % (self._bmap_path, err)) - - self._f_image.seek(image_pos) diff --git a/mic/utils/Filemap.py b/mic/utils/Filemap.py deleted file mode 100644 index 81d16c1..0000000 --- a/mic/utils/Filemap.py +++ /dev/null @@ -1,520 +0,0 @@ -# Copyright (c) 2012 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License, version 2, -# as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -""" -This module implements python implements a way to get file block. Two methods -are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of -the file seek syscall. The former is implemented by the 'FilemapFiemap' class, -the latter is implemented by the 'FilemapSeek' class. Both classes provide the -same API. The 'filemap' function automatically selects which class can be used -and returns an instance of the class. -""" - -# Disable the following pylint recommendations: -# * Too many instance attributes (R0902) -# pylint: disable=R0902 - -import os -import struct -import array -import fcntl -import tempfile -import logging -from mic.utils.misc import get_block_size - - -class ErrorNotSupp(Exception): - """ - An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature - is not supported either by the kernel or the file-system. - """ - pass - -class Error(Exception): - """A class for all the other exceptions raised by this module.""" - pass - - -class _FilemapBase(object): - """ - This is a base class for a couple of other classes in this module. This - class simply performs the common parts of the initialization process: opens - the image file, gets its size, etc. The 'log' parameter is the logger object - to use for printing messages. - """ - - def __init__(self, image, log=None): - """ - Initialize a class instance. The 'image' argument is full path to the - file or file object to operate on. - """ - - self._log = log - if self._log is None: - self._log = logging.getLogger(__name__) - - self._f_image_needs_close = False - - if hasattr(image, "fileno"): - self._f_image = image - self._image_path = image.name - else: - self._image_path = image - self._open_image_file() - - try: - self.image_size = os.fstat(self._f_image.fileno()).st_size - except IOError as err: - raise Error("cannot get information about file '%s': %s" - % (self._f_image.name, err)) - - try: - self.block_size = get_block_size(self._f_image) - except IOError as err: - raise Error("cannot get block size for '%s': %s" - % (self._image_path, err)) - - self.blocks_cnt = self.image_size + self.block_size - 1 - self.blocks_cnt /= self.block_size - - try: - self._f_image.flush() - except IOError as err: - raise Error("cannot flush image file '%s': %s" - % (self._image_path, err)) - - try: - os.fsync(self._f_image.fileno()), - except OSError as err: - raise Error("cannot synchronize image file '%s': %s " - % (self._image_path, err.strerror)) - - self._log.debug("opened image \"%s\"" % self._image_path) - self._log.debug("block size %d, blocks count %d, image size %d" - % (self.block_size, self.blocks_cnt, self.image_size)) - - def __del__(self): - """The class destructor which just closes the image file.""" - if self._f_image_needs_close: - self._f_image.close() - - def _open_image_file(self): - """Open the image file.""" - try: - self._f_image = open(self._image_path, 'rb') - except IOError as err: - raise Error("cannot open image file '%s': %s" - % (self._image_path, err)) - - self._f_image_needs_close = True - - def block_is_mapped(self, block): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. It returns - 'True' if block number 'block' of the image file is mapped and 'False' - otherwise. - """ - - raise Error("the method is not implemented") - - def block_is_unmapped(self, block): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. It returns - 'True' if block number 'block' of the image file is not mapped (hole) - and 'False' otherwise. - """ - - raise Error("the method is not implemented") - - def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. This is a - generator which yields ranges of mapped blocks in the file. The ranges - are tuples of 2 elements: [first, last], where 'first' is the first - mapped block and 'last' is the last mapped block. - - The ranges are yielded for the area of the file of size 'count' blocks, - starting from block 'start'. - """ - - raise Error("the method is not implemented") - - def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201 - """ - This method has has to be implemented by child classes. Just like - 'get_mapped_ranges()', but yields unmapped block ranges instead - (holes). - """ - - raise Error("the method is not implemented") - - -# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call -_SEEK_DATA = 3 -_SEEK_HOLE = 4 - -def _lseek(file_obj, offset, whence): - """This is a helper function which invokes 'os.lseek' for file object - 'file_obj' and with specified 'offset' and 'whence'. The 'whence' - argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When - there is no more data or hole starting from 'offset', this function - returns '-1'. Otherwise the data or hole position is returned.""" - - try: - return os.lseek(file_obj.fileno(), offset, whence) - except OSError as err: - # The 'lseek' system call returns the ENXIO if there is no data or - # hole starting from the specified offset. - if err.errno == os.errno.ENXIO: - return -1 - elif err.errno == os.errno.EINVAL: - raise ErrorNotSupp("the kernel or file-system does not support " - "\"SEEK_HOLE\" and \"SEEK_DATA\"") - else: - raise - -class FilemapSeek(_FilemapBase): - """ - This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping. - Unfortunately, the current implementation requires the caller to have write - access to the image file. - """ - - def __init__(self, image, log=None): - """Refer the '_FilemapBase' class for the documentation.""" - - # Call the base class constructor first - _FilemapBase.__init__(self, image, log) - self._log.debug("FilemapSeek: initializing") - - self._probe_seek_hole() - - def _probe_seek_hole(self): - """ - Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'. - Unfortunately, there seems to be no clean way for detecting this, - because often the system just fakes them by just assuming that all - files are fully mapped, so 'SEEK_HOLE' always returns EOF and - 'SEEK_DATA' always returns the requested offset. - - I could not invent a better way of detecting the fake 'SEEK_HOLE' - implementation than just to create a temporary file in the same - directory where the image file resides. It would be nice to change this - to something better. - """ - - directory = os.path.dirname(self._image_path) - - try: - tmp_obj = tempfile.TemporaryFile("w+", dir=directory) - except IOError as err: - raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" - % (directory, err)) - - try: - os.ftruncate(tmp_obj.fileno(), self.block_size) - except OSError as err: - raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s" - % (directory, err)) - - offs = _lseek(tmp_obj, 0, _SEEK_HOLE) - if offs != 0: - # We are dealing with the stub 'SEEK_HOLE' implementation which - # always returns EOF. - self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs) - raise ErrorNotSupp("the file-system does not support " - "\"SEEK_HOLE\" and \"SEEK_DATA\" but only " - "provides a stub implementation") - - tmp_obj.close() - - def block_is_mapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA) - if offs == -1: - result = False - else: - result = (offs / self.block_size == block) - - self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s" - % (block, result)) - return result - - def block_is_unmapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - return not self.block_is_mapped(block) - - def _get_ranges(self, start, count, whence1, whence2): - """ - This function implements 'get_mapped_ranges()' and - 'get_unmapped_ranges()' depending on what is passed in the 'whence1' - and 'whence2' arguments. - """ - - assert whence1 != whence2 - end = start * self.block_size - limit = end + count * self.block_size - - while True: - start = _lseek(self._f_image, end, whence1) - if start == -1 or start >= limit or start == self.image_size: - break - - end = _lseek(self._f_image, start, whence2) - if end == -1 or end == self.image_size: - end = self.blocks_cnt * self.block_size - if end > limit: - end = limit - - start_blk = start / self.block_size - end_blk = end / self.block_size - 1 - self._log.debug("FilemapSeek: yielding range (%d, %d)" - % (start_blk, end_blk)) - yield (start_blk, end_blk) - - def get_mapped_ranges(self, start, count): - """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE) - - def get_unmapped_ranges(self, start, count): - """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA) - - -# Below goes the FIEMAP ioctl implementation, which is not very readable -# because it deals with the rather complex FIEMAP ioctl. To understand the -# code, you need to know the FIEMAP interface, which is documented in the -# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources. - -# Format string for 'struct fiemap' -_FIEMAP_FORMAT = "=QQLLLL" -# sizeof(struct fiemap) -_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT) -# Format string for 'struct fiemap_extent' -_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL" -# sizeof(struct fiemap_extent) -_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT) -# The FIEMAP ioctl number -_FIEMAP_IOCTL = 0xC020660B -# This FIEMAP ioctl flag which instructs the kernel to sync the file before -# reading the block map -_FIEMAP_FLAG_SYNC = 0x00000001 -# Size of the buffer for 'struct fiemap_extent' elements which will be used -# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the -# FIEMAP ioctl will be invoked. -_FIEMAP_BUFFER_SIZE = 256 * 1024 - -class FilemapFiemap(_FilemapBase): - """ - This class provides API to the FIEMAP ioctl. Namely, it allows to iterate - over all mapped blocks and over all holes. - - This class synchronizes the image file every time it invokes the FIEMAP - ioctl in order to work-around early FIEMAP implementation kernel bugs. - """ - - def __init__(self, image, log=None): - """ - Initialize a class instance. The 'image' argument is full the file - object to operate on. - """ - - # Call the base class constructor first - _FilemapBase.__init__(self, image, log) - self._log.debug("FilemapFiemap: initializing") - - self._buf_size = _FIEMAP_BUFFER_SIZE - - # Calculate how many 'struct fiemap_extent' elements fit the buffer - self._buf_size -= _FIEMAP_SIZE - self._fiemap_extent_cnt = self._buf_size / _FIEMAP_EXTENT_SIZE - assert self._fiemap_extent_cnt > 0 - self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE - self._buf_size += _FIEMAP_SIZE - - # Allocate a mutable buffer for the FIEMAP ioctl - self._buf = array.array('B', [0] * self._buf_size) - - # Check if the FIEMAP ioctl is supported - self.block_is_mapped(0) - - def _invoke_fiemap(self, block, count): - """ - Invoke the FIEMAP ioctl for 'count' blocks of the file starting from - block number 'block'. - - The full result of the operation is stored in 'self._buf' on exit. - Returns the unpacked 'struct fiemap' data structure in form of a python - list (just like 'struct.upack()'). - """ - - if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt): - raise Error("bad block number %d, should be within [0, %d]" - % (block, self.blocks_cnt)) - - # Initialize the 'struct fiemap' part of the buffer. We use the - # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is - # synchronized. The reason for this is that early FIEMAP - # implementations had many bugs related to cached dirty data, and - # synchronizing the file is a necessary work-around. - struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size, - count * self.block_size, _FIEMAP_FLAG_SYNC, 0, - self._fiemap_extent_cnt, 0) - - try: - fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1) - except IOError as err: - # Note, the FIEMAP ioctl is supported by the Linux kernel starting - # from version 2.6.28 (year 2008). - if err.errno == os.errno.EOPNOTSUPP: - errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ - "by the file-system" - self._log.debug(errstr) - raise ErrorNotSupp(errstr) - if err.errno == os.errno.ENOTTY: - errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \ - "by the kernel" - self._log.debug(errstr) - raise ErrorNotSupp(errstr) - raise Error("the FIEMAP ioctl failed for '%s': %s" - % (self._image_path, err)) - - return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) - - def block_is_mapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - struct_fiemap = self._invoke_fiemap(block, 1) - - # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field. - # If it contains zero, the block is not mapped, otherwise it is - # mapped. - result = bool(struct_fiemap[3]) - self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s" - % (block, result)) - return result - - def block_is_unmapped(self, block): - """Refer the '_FilemapBase' class for the documentation.""" - return not self.block_is_mapped(block) - - def _unpack_fiemap_extent(self, index): - """ - Unpack a 'struct fiemap_extent' structure object number 'index' from - the internal 'self._buf' buffer. - """ - - offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index - return struct.unpack(_FIEMAP_EXTENT_FORMAT, - self._buf[offset : offset + _FIEMAP_EXTENT_SIZE]) - - def _do_get_mapped_ranges(self, start, count): - """ - Implements most the functionality for the 'get_mapped_ranges()' - generator: invokes the FIEMAP ioctl, walks through the mapped extents - and yields mapped block ranges. However, the ranges may be consecutive - (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges - them. - """ - - block = start - while block < start + count: - struct_fiemap = self._invoke_fiemap(block, count) - - mapped_extents = struct_fiemap[3] - if mapped_extents == 0: - # No more mapped blocks - return - - extent = 0 - while extent < mapped_extents: - fiemap_extent = self._unpack_fiemap_extent(extent) - - # Start of the extent - extent_start = fiemap_extent[0] - # Starting block number of the extent - extent_block = extent_start / self.block_size - # Length of the extent - extent_len = fiemap_extent[2] - # Count of blocks in the extent - extent_count = extent_len / self.block_size - - # Extent length and offset have to be block-aligned - assert extent_start % self.block_size == 0 - assert extent_len % self.block_size == 0 - - if extent_block > start + count - 1: - return - - first = max(extent_block, block) - last = min(extent_block + extent_count, start + count) - 1 - yield (first, last) - - extent += 1 - - block = extent_block + extent_count - - def get_mapped_ranges(self, start, count): - """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapFiemap: get_mapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - iterator = self._do_get_mapped_ranges(start, count) - first_prev, last_prev = iterator.next() - - for first, last in iterator: - if last_prev == first - 1: - last_prev = last - else: - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (first_prev, last_prev)) - yield (first_prev, last_prev) - first_prev, last_prev = first, last - - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (first_prev, last_prev)) - yield (first_prev, last_prev) - - def get_unmapped_ranges(self, start, count): - """Refer the '_FilemapBase' class for the documentation.""" - self._log.debug("FilemapFiemap: get_unmapped_ranges(%d, %d(%d))" - % (start, count, start + count - 1)) - hole_first = start - for first, last in self._do_get_mapped_ranges(start, count): - if first > hole_first: - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, first - 1)) - yield (hole_first, first - 1) - - hole_first = last + 1 - - if hole_first < start + count: - self._log.debug("FilemapFiemap: yielding range (%d, %d)" - % (hole_first, start + count - 1)) - yield (hole_first, start + count - 1) - - -def filemap(image, log=None): - """ - Create and return an instance of a Filemap class - 'FilemapFiemap' or - 'FilemapSeek', depending on what the system we run on supports. If the - FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is - returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the - 'FilemapSeek' class is returned. If none of these are supported, the - function generates an 'Error' type exception. - """ - - try: - return FilemapFiemap(image, log) - except ErrorNotSupp: - return FilemapSeek(image, log) diff --git a/mic/utils/cmdln.py b/mic/utils/cmdln.py deleted file mode 100644 index b099473..0000000 --- a/mic/utils/cmdln.py +++ /dev/null @@ -1,1586 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2002-2007 ActiveState Software Inc. -# License: MIT (see LICENSE.txt for license details) -# Author: Trent Mick -# Home: http://trentm.com/projects/cmdln/ - -"""An improvement on Python's standard cmd.py module. - -As with cmd.py, this module provides "a simple framework for writing -line-oriented command intepreters." This module provides a 'RawCmdln' -class that fixes some design flaws in cmd.Cmd, making it more scalable -and nicer to use for good 'cvs'- or 'svn'-style command line interfaces -or simple shells. And it provides a 'Cmdln' class that add -optparse-based option processing. Basically you use it like this: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.alias('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - -See the README.txt or <http://trentm.com/projects/cmdln/> for more -details. -""" - -__version_info__ = (1, 1, 2) -__version__ = '.'.join(map(str, __version_info__)) - -import os -import sys -import re -import cmd -import optparse -from pprint import pprint -import sys - - - - -#---- globals - -LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3) - -# An unspecified optional argument when None is a meaningful value. -_NOT_SPECIFIED = ("Not", "Specified") - -# Pattern to match a TypeError message from a call that -# failed because of incorrect number of arguments (see -# Python/getargs.c). -_INCORRECT_NUM_ARGS_RE = re.compile( - r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))") - - - -#---- exceptions - -class CmdlnError(Exception): - """A cmdln.py usage error.""" - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - -class CmdlnUserError(Exception): - """An error by a user of a cmdln-based tool/shell.""" - pass - - - -#---- public methods and classes - -def alias(*aliases): - """Decorator to add aliases for Cmdln.do_* command handlers. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.alias("!", "sh") - def do_shell(self, argv): - #...implement 'shell' command - """ - def decorate(f): - if not hasattr(f, "aliases"): - f.aliases = [] - f.aliases += aliases - return f - return decorate - - -class RawCmdln(cmd.Cmd): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.RawCmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - def do_status(self, argv): - print "handle 'svn status' command" - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - See <http://trentm.com/projects/cmdln> for more information. - """ - name = None # if unset, defaults basename(sys.argv[0]) - prompt = None # if unset, defaults to self.name+"> " - version = None # if set, default top-level options include --version - - # Default messages for some 'help' command error cases. - # They are interpolated with one arg: the command. - nohelp = "no help on '%s'" - unknowncmd = "unknown command: '%s'" - - helpindent = '' # string with which to indent help output - - def __init__(self, completekey='tab', - stdin=None, stdout=None, stderr=None): - """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None) - - The optional argument 'completekey' is the readline name of a - completion key; it defaults to the Tab key. If completekey is - not None and the readline module is available, command completion - is done automatically. - - The optional arguments 'stdin', 'stdout' and 'stderr' specify - alternate input, output and error output file objects; if not - specified, sys.* are used. - - If 'stdout' but not 'stderr' is specified, stdout is used for - error output. This is to provide least surprise for users used - to only the 'stdin' and 'stdout' options with cmd.Cmd. - """ - import sys - if self.name is None: - self.name = os.path.basename(sys.argv[0]) - if self.prompt is None: - self.prompt = self.name+"> " - self._name_str = self._str(self.name) - self._prompt_str = self._str(self.prompt) - if stdin is not None: - self.stdin = stdin - else: - self.stdin = sys.stdin - if stdout is not None: - self.stdout = stdout - else: - self.stdout = sys.stdout - if stderr is not None: - self.stderr = stderr - elif stdout is not None: - self.stderr = stdout - else: - self.stderr = sys.stderr - self.cmdqueue = [] - self.completekey = completekey - self.cmdlooping = False - - def get_optparser(self): - """Hook for subclasses to set the option parser for the - top-level command/shell. - - This option parser is used retrieved and used by `.main()' to - handle top-level options. - - The default implements a single '-h|--help' option. Sub-classes - can return None to have no options at the top-level. Typically - an instance of CmdlnOptionParser should be returned. - """ - version = (self.version is not None - and "%s %s" % (self._name_str, self.version) - or None) - return CmdlnOptionParser(self, version=version) - - def postoptparse(self): - """Hook method executed just after `.main()' parses top-level - options. - - When called `self.options' holds the results of the option parse. - """ - pass - - def main(self, argv=None, loop=LOOP_NEVER): - """A possible mainline handler for a script, like so: - - import cmdln - class MyCmd(cmdln.Cmdln): - name = "mycmd" - ... - - if __name__ == "__main__": - MyCmd().main() - - By default this will use sys.argv to issue a single command to - 'MyCmd', then exit. The 'loop' argument can be use to control - interactive shell behaviour. - - Arguments: - "argv" (optional, default sys.argv) is the command to run. - It must be a sequence, where the first element is the - command name and subsequent elements the args for that - command. - "loop" (optional, default LOOP_NEVER) is a constant - indicating if a command loop should be started (i.e. an - interactive shell). Valid values (constants on this module): - LOOP_ALWAYS start loop and run "argv", if any - LOOP_NEVER run "argv" (or .emptyline()) and exit - LOOP_IF_EMPTY run "argv", if given, and exit; - otherwise, start loop - """ - if argv is None: - import sys - argv = sys.argv - else: - argv = argv[:] # don't modify caller's list - - self.optparser = self.get_optparser() - if self.optparser: # i.e. optparser=None means don't process for opts - try: - self.options, args = self.optparser.parse_args(argv[1:]) - except CmdlnUserError, ex: - msg = "%s: %s\nTry '%s help' for info.\n"\ - % (self.name, ex, self.name) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return 1 - except StopOptionProcessing, ex: - return 0 - else: - self.options, args = None, argv[1:] - self.postoptparse() - - if loop == LOOP_ALWAYS: - if args: - self.cmdqueue.append(args) - return self.cmdloop() - elif loop == LOOP_NEVER: - if args: - return self.cmd(args) - else: - return self.emptyline() - elif loop == LOOP_IF_EMPTY: - if args: - return self.cmd(args) - else: - return self.cmdloop() - - def cmd(self, argv): - """Run one command and exit. - - "argv" is the arglist for the command to run. argv[0] is the - command to run. If argv is an empty list then the - 'emptyline' handler is run. - - Returns the return value from the command handler. - """ - assert isinstance(argv, (list, tuple)), \ - "'argv' is not a sequence: %r" % argv - retval = None - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - return retval - - def _str(self, s): - """Safely convert the given str/unicode to a string for printing.""" - try: - return str(s) - except UnicodeError: - #XXX What is the proper encoding to use here? 'utf-8' seems - # to work better than "getdefaultencoding" (usually - # 'ascii'), on OS X at least. - #import sys - #return s.encode(sys.getdefaultencoding(), "replace") - return s.encode("utf-8", "replace") - - def cmdloop(self, intro=None): - """Repeatedly issue a prompt, accept input, parse into an argv, and - dispatch (via .precmd(), .onecmd() and .postcmd()), passing them - the argv. In other words, start a shell. - - "intro" (optional) is a introductory message to print when - starting the command loop. This overrides the class - "intro" attribute, if any. - """ - self.cmdlooping = True - self.preloop() - if self.use_rawinput and self.completekey: - try: - import readline - self.old_completer = readline.get_completer() - readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") - except ImportError: - pass - try: - if intro is None: - intro = self.intro - if intro: - intro_str = self._str(intro) - self.stdout.write(intro_str+'\n') - self.stop = False - retval = None - while not self.stop: - if self.cmdqueue: - argv = self.cmdqueue.pop(0) - assert isinstance(argv, (list, tuple)), \ - "item on 'cmdqueue' is not a sequence: %r" % argv - else: - if self.use_rawinput: - try: - line = raw_input(self._prompt_str) - except EOFError: - line = 'EOF' - else: - self.stdout.write(self._prompt_str) - self.stdout.flush() - line = self.stdin.readline() - if not len(line): - line = 'EOF' - else: - line = line[:-1] # chop '\n' - argv = line2argv(line) - try: - argv = self.precmd(argv) - retval = self.onecmd(argv) - self.postcmd(argv) - except: - if not self.cmdexc(argv): - raise - retval = 1 - self.lastretval = retval - self.postloop() - finally: - if self.use_rawinput and self.completekey: - try: - import readline - readline.set_completer(self.old_completer) - except ImportError: - pass - self.cmdlooping = False - return retval - - def precmd(self, argv): - """Hook method executed just before the command argv is - interpreted, but after the input prompt is generated and issued. - - "argv" is the cmd to run. - - Returns an argv to run (i.e. this method can modify the command - to run). - """ - return argv - - def postcmd(self, argv): - """Hook method executed just after a command dispatch is finished. - - "argv" is the command that was run. - """ - pass - - def cmdexc(self, argv): - """Called if an exception is raised in any of precmd(), onecmd(), - or postcmd(). If True is returned, the exception is deemed to have - been dealt with. Otherwise, the exception is re-raised. - - The default implementation handles CmdlnUserError's, which - typically correspond to user error in calling commands (as - opposed to programmer error in the design of the script using - cmdln.py). - """ - import sys - type, exc, traceback = sys.exc_info() - if isinstance(exc, CmdlnUserError): - msg = "%s %s: %s\nTry '%s help %s' for info.\n"\ - % (self.name, argv[0], exc, self.name, argv[0]) - self.stderr.write(self._str(msg)) - self.stderr.flush() - return True - - def onecmd(self, argv): - if not argv: - return self.emptyline() - self.lastcmd = argv - cmdname = self._get_canonical_cmd_name(argv[0]) - if cmdname: - handler = self._get_cmd_handler(cmdname) - if handler: - return self._dispatch_cmd(handler, argv) - return self.default(argv) - - def _dispatch_cmd(self, handler, argv): - return handler(argv) - - def default(self, argv): - """Hook called to handle a command for which there is no handler. - - "argv" is the command and arguments to run. - - The default implementation writes and error message to stderr - and returns an error exit status. - - Returns a numeric command exit status. - """ - errmsg = self._str(self.unknowncmd % (argv[0],)) - if self.cmdlooping: - self.stderr.write(errmsg+"\n") - else: - self.stderr.write("%s: %s\nTry '%s help' for info.\n" - % (self._name_str, errmsg, self._name_str)) - self.stderr.flush() - return 1 - - def parseline(self, line): - # This is used by Cmd.complete (readline completer function) to - # massage the current line buffer before completion processing. - # We override to drop special '!' handling. - line = line.strip() - if not line: - return None, None, line - elif line[0] == '?': - line = 'help ' + line[1:] - i, n = 0, len(line) - while i < n and line[i] in self.identchars: i = i+1 - cmd, arg = line[:i], line[i:].strip() - return cmd, arg, line - - def helpdefault(self, cmd, known): - """Hook called to handle help on a command for which there is no - help handler. - - "cmd" is the command name on which help was requested. - "known" is a boolean indicating if this command is known - (i.e. if there is a handler for it). - - Returns a return code. - """ - if known: - msg = self._str(self.nohelp % (cmd,)) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" % (self.name, msg)) - else: - msg = self.unknowncmd % (cmd,) - if self.cmdlooping: - self.stderr.write(msg + '\n') - else: - self.stderr.write("%s: %s\n" - "Try '%s help' for info.\n" - % (self.name, msg, self.name)) - self.stderr.flush() - return 1 - - def do_help(self, argv): - """${cmd_name}: give detailed help on a specific sub-command - - Usage: - ${name} help [COMMAND] - """ - if len(argv) > 1: # asking for help on a particular command - doc = None - cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1] - if not cmdname: - return self.helpdefault(argv[1], False) - else: - helpfunc = getattr(self, "help_"+cmdname, None) - if helpfunc: - doc = helpfunc() - else: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - if doc is None: - return self.helpdefault(argv[1], handler != None) - else: # bare "help" command - doc = self.__class__.__doc__ # try class docstring - if doc is None: - # Try to provide some reasonable useful default help. - if self.cmdlooping: prefix = "" - else: prefix = self.name+' ' - doc = """Usage: - %sCOMMAND [ARGS...] - %shelp [COMMAND] - - ${option_list} - ${command_list} - ${help_list} - """ % (prefix, prefix) - cmdname = None - - if doc: # *do* have help content, massage and print that - doc = self._help_reindent(doc) - doc = self._help_preprocess(doc, cmdname) - doc = doc.rstrip() + '\n' # trim down trailing space - self.stdout.write(self._str(doc)) - self.stdout.flush() - do_help.aliases = ["?"] - - def _help_reindent(self, help, indent=None): - """Hook to re-indent help strings before writing to stdout. - - "help" is the help content to re-indent - "indent" is a string with which to indent each line of the - help content after normalizing. If unspecified or None - then the default is use: the 'self.helpindent' class - attribute. By default this is the empty string, i.e. - no indentation. - - By default, all common leading whitespace is removed and then - the lot is indented by 'self.helpindent'. When calculating the - common leading whitespace the first line is ignored -- hence - help content for Conan can be written as follows and have the - expected indentation: - - def do_crush(self, ...): - '''${cmd_name}: crush your enemies, see them driven before you... - - c.f. Conan the Barbarian''' - """ - if indent is None: - indent = self.helpindent - lines = help.splitlines(0) - _dedentlines(lines, skip_first_line=True) - lines = [(indent+line).rstrip() for line in lines] - return '\n'.join(lines) - - def _help_preprocess(self, help, cmdname): - """Hook to preprocess a help string before writing to stdout. - - "help" is the help string to process. - "cmdname" is the canonical sub-command name for which help - is being given, or None if the help is not specific to a - command. - - By default the following template variables are interpolated in - help content. (Note: these are similar to Python 2.4's - string.Template interpolation but not quite.) - - ${name} - The tool's/shell's name, i.e. 'self.name'. - ${option_list} - A formatted table of options for this shell/tool. - ${command_list} - A formatted table of available sub-commands. - ${help_list} - A formatted table of additional help topics (i.e. 'help_*' - methods with no matching 'do_*' method). - ${cmd_name} - The name (and aliases) for this sub-command formatted as: - "NAME (ALIAS1, ALIAS2, ...)". - ${cmd_usage} - A formatted usage block inferred from the command function - signature. - ${cmd_option_list} - A formatted table of options for this sub-command. (This is - only available for commands using the optparse integration, - i.e. using @cmdln.option decorators or manually setting the - 'optparser' attribute on the 'do_*' method.) - - Returns the processed help. - """ - preprocessors = { - "${name}": self._help_preprocess_name, - "${option_list}": self._help_preprocess_option_list, - "${command_list}": self._help_preprocess_command_list, - "${help_list}": self._help_preprocess_help_list, - "${cmd_name}": self._help_preprocess_cmd_name, - "${cmd_usage}": self._help_preprocess_cmd_usage, - "${cmd_option_list}": self._help_preprocess_cmd_option_list, - } - - for marker, preprocessor in preprocessors.items(): - if marker in help: - help = preprocessor(help, cmdname) - return help - - def _help_preprocess_name(self, help, cmdname=None): - return help.replace("${name}", self.name) - - def _help_preprocess_option_list(self, help, cmdname=None): - marker = "${option_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - if self.optparser: - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - self.optparser.formatter.indent_increment = 4 - self.optparser.formatter.current_indent = indent_width - block = self.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - - def _help_preprocess_command_list(self, help, cmdname=None): - marker = "${command_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Find any aliases for commands. - token2canonical = self._get_canonical_map() - aliases = {} - for token, cmdname in token2canonical.items(): - if token == cmdname: continue - aliases.setdefault(cmdname, []).append(token) - - # Get the list of (non-hidden) commands and their - # documentation, if any. - cmdnames = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): - cmdnames[attr[3:]] = True - cmdnames = cmdnames.keys() - cmdnames.sort() - linedata = [] - for cmdname in cmdnames: - if aliases.get(cmdname): - a = aliases[cmdname] - a.sort() - cmdstr = "%s (%s)" % (cmdname, ", ".join(a)) - else: - cmdstr = cmdname - doc = None - try: - helpfunc = getattr(self, 'help_'+cmdname) - except AttributeError: - handler = self._get_cmd_handler(cmdname) - if handler: - doc = handler.__doc__ - else: - doc = helpfunc() - - # Strip "${cmd_name}: " from the start of a command's doc. Best - # practice dictates that command help strings begin with this, but - # it isn't at all wanted for the command list. - to_strip = "${cmd_name}:" - if doc and doc.startswith(to_strip): - #log.debug("stripping %r from start of %s's help string", - # to_strip, cmdname) - doc = doc[len(to_strip):].lstrip() - linedata.append( (cmdstr, doc) ) - - if linedata: - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = indent + "Commands:\n" \ - + '\n'.join(lines) + "\n\n" - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _gen_names_and_attrs(self): - # Inheritance says we have to look in class and - # base classes; order is not important. - names = [] - classes = [self.__class__] - while classes: - aclass = classes.pop(0) - if aclass.__bases__: - classes = classes + list(aclass.__bases__) - for name in dir(aclass): - yield (name, getattr(aclass, name)) - - def _help_preprocess_help_list(self, help, cmdname=None): - marker = "${help_list}" - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Determine the additional help topics, if any. - helpnames = {} - token2cmdname = self._get_canonical_map() - for attrname, attr in self._gen_names_and_attrs(): - if not attrname.startswith("help_"): continue - helpname = attrname[5:] - if helpname not in token2cmdname: - helpnames[helpname] = attr - - if helpnames: - linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()] - linedata.sort() - - subindent = indent + ' '*4 - lines = _format_linedata(linedata, subindent, indent_width+4) - block = (indent - + "Additional help topics (run `%s help TOPIC'):\n" % self.name - + '\n'.join(lines) - + "\n\n") - else: - block = '' - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _help_preprocess_cmd_name(self, help, cmdname=None): - marker = "${cmd_name}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - s = cmdname - if hasattr(handler, "aliases"): - s += " (%s)" % (", ".join(handler.aliases)) - help = help.replace(marker, s) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_usage(self, help, cmdname=None): - marker = "${cmd_usage}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - - # Extract the introspection bits we need. - func = handler.im_func - if func.func_defaults: - func_defaults = list(func.func_defaults) - else: - func_defaults = [] - co_argcount = func.func_code.co_argcount - co_varnames = func.func_code.co_varnames - co_flags = func.func_code.co_flags - CO_FLAGS_ARGS = 4 - CO_FLAGS_KWARGS = 8 - - # Adjust argcount for possible *args and **kwargs arguments. - argcount = co_argcount - if co_flags & CO_FLAGS_ARGS: argcount += 1 - if co_flags & CO_FLAGS_KWARGS: argcount += 1 - - # Determine the usage string. - usage = "%s %s" % (self.name, cmdname) - if argcount <= 2: # handler ::= do_FOO(self, argv) - usage += " [ARGS...]" - elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...) - argnames = list(co_varnames[3:argcount]) - tail = "" - if co_flags & CO_FLAGS_KWARGS: - name = argnames.pop(-1) - import warnings - # There is no generally accepted mechanism for passing - # keyword arguments from the command line. Could - # *perhaps* consider: arg=value arg2=value2 ... - warnings.warn("argument '**%s' on '%s.%s' command " - "handler will never get values" - % (name, self.__class__.__name__, - func.func_name)) - if co_flags & CO_FLAGS_ARGS: - name = argnames.pop(-1) - tail = "[%s...]" % name.upper() - while func_defaults: - func_defaults.pop(-1) - name = argnames.pop(-1) - tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail) - while argnames: - name = argnames.pop(-1) - tail = "%s %s" % (name.upper(), tail) - usage += ' ' + tail - - block_lines = [ - self.helpindent + "Usage:", - self.helpindent + ' '*4 + usage - ] - block = '\n'.join(block_lines) + '\n\n' - - help = help.replace(indent+marker+suffix, block, 1) - return help - - #TODO: this only makes sense as part of the Cmdln class. - # Add hooks to add help preprocessing template vars and put - # this one on that class. - def _help_preprocess_cmd_option_list(self, help, cmdname=None): - marker = "${cmd_option_list}" - handler = self._get_cmd_handler(cmdname) - if not handler: - raise CmdlnError("cannot preprocess '%s' into help string: " - "could not find command handler for %r" - % (marker, cmdname)) - indent, indent_width = _get_indent(marker, help) - suffix = _get_trailing_whitespace(marker, help) - if hasattr(handler, "optparser"): - # Setup formatting options and format. - # - Indentation of 4 is better than optparse default of 2. - # C.f. Damian Conway's discussion of this in Perl Best - # Practices. - handler.optparser.formatter.indent_increment = 4 - handler.optparser.formatter.current_indent = indent_width - block = handler.optparser.format_option_help() + '\n' - else: - block = "" - - help = help.replace(indent+marker+suffix, block, 1) - return help - - def _get_canonical_cmd_name(self, token): - map = self._get_canonical_map() - return map.get(token, None) - - def _get_canonical_map(self): - """Return a mapping of available command names and aliases to - their canonical command name. - """ - cacheattr = "_token2canonical" - if not hasattr(self, cacheattr): - # Get the list of commands and their aliases, if any. - token2canonical = {} - cmd2funcname = {} # use a dict to strip duplicates - for attr in self.get_names(): - if attr.startswith("do_"): cmdname = attr[3:] - elif attr.startswith("_do_"): cmdname = attr[4:] - else: - continue - cmd2funcname[cmdname] = attr - token2canonical[cmdname] = cmdname - for cmdname, funcname in cmd2funcname.items(): # add aliases - func = getattr(self, funcname) - aliases = getattr(func, "aliases", []) - for alias in aliases: - if alias in cmd2funcname: - import warnings - warnings.warn("'%s' alias for '%s' command conflicts " - "with '%s' handler" - % (alias, cmdname, cmd2funcname[alias])) - continue - token2canonical[alias] = cmdname - setattr(self, cacheattr, token2canonical) - return getattr(self, cacheattr) - - def _get_cmd_handler(self, cmdname): - handler = None - try: - handler = getattr(self, 'do_' + cmdname) - except AttributeError: - try: - # Private command handlers begin with "_do_". - handler = getattr(self, '_do_' + cmdname) - except AttributeError: - pass - return handler - - def _do_EOF(self, argv): - # Default EOF handler - # Note: an actual EOF is redirected to this command. - #TODO: separate name for this. Currently it is available from - # command-line. Is that okay? - self.stdout.write('\n') - self.stdout.flush() - self.stop = True - - def emptyline(self): - # Different from cmd.Cmd: don't repeat the last command for an - # emptyline. - if self.cmdlooping: - pass - else: - return self.do_help(["help"]) - - -#---- optparse.py extension to fix (IMO) some deficiencies -# -# See the class _OptionParserEx docstring for details. -# - -class StopOptionProcessing(Exception): - """Indicate that option *and argument* processing should stop - cleanly. This is not an error condition. It is similar in spirit to - StopIteration. This is raised by _OptionParserEx's default "help" - and "version" option actions and can be raised by custom option - callbacks too. - - Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx) - usage is: - - parser = CmdlnOptionParser(mycmd) - parser.add_option("-f", "--force", dest="force") - ... - try: - opts, args = parser.parse_args() - except StopOptionProcessing: - # normal termination, "--help" was probably given - sys.exit(0) - """ - -class _OptionParserEx(optparse.OptionParser): - """An optparse.OptionParser that uses exceptions instead of sys.exit. - - This class is an extension of optparse.OptionParser that differs - as follows: - - Correct (IMO) the default OptionParser error handling to never - sys.exit(). Instead OptParseError exceptions are passed through. - - Add the StopOptionProcessing exception (a la StopIteration) to - indicate normal termination of option processing. - See StopOptionProcessing's docstring for details. - - I'd also like to see the following in the core optparse.py, perhaps - as a RawOptionParser which would serve as a base class for the more - generally used OptionParser (that works as current): - - Remove the implicit addition of the -h|--help and --version - options. They can get in the way (e.g. if want '-?' and '-V' for - these as well) and it is not hard to do: - optparser.add_option("-h", "--help", action="help") - optparser.add_option("--version", action="version") - These are good practices, just not valid defaults if they can - get in the way. - """ - def error(self, msg): - raise optparse.OptParseError(msg) - - def exit(self, status=0, msg=None): - if status == 0: - raise StopOptionProcessing(msg) - else: - #TODO: don't lose status info here - raise optparse.OptParseError(msg) - - - -#---- optparse.py-based option processing support - -class CmdlnOptionParser(_OptionParserEx): - """An optparse.OptionParser class more appropriate for top-level - Cmdln options. For parsing of sub-command options, see - SubCmdOptionParser. - - Changes: - - disable_interspersed_args() by default, because a Cmdln instance - has sub-commands which may themselves have options. - - Redirect print_help() to the Cmdln.do_help() which is better - equiped to handle the "help" action. - - error() will raise a CmdlnUserError: OptionParse.error() is meant - to be called for user errors. Raising a well-known error here can - make error handling clearer. - - Also see the changes in _OptionParserEx. - """ - def __init__(self, cmdln, **kwargs): - self.cmdln = cmdln - kwargs["prog"] = self.cmdln.name - _OptionParserEx.__init__(self, **kwargs) - self.disable_interspersed_args() - - def print_help(self, file=None): - self.cmdln.onecmd(["help"]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -class SubCmdOptionParser(_OptionParserEx): - def set_cmdln_info(self, cmdln, subcmd): - """Called by Cmdln to pass relevant info about itself needed - for print_help(). - """ - self.cmdln = cmdln - self.subcmd = subcmd - - def print_help(self, file=None): - self.cmdln.onecmd(["help", self.subcmd]) - - def error(self, msg): - raise CmdlnUserError(msg) - - -def option(*args, **kwargs): - """Decorator to add an option to the optparser argument of a Cmdln - subcommand. - - Example: - class MyShell(cmdln.Cmdln): - @cmdln.option("-f", "--force", help="force removal") - def do_remove(self, subcmd, opts, *args): - #... - """ - #XXX Is there a possible optimization for many options to not have a - # large stack depth here? - def decorate(f): - if not hasattr(f, "optparser"): - f.optparser = SubCmdOptionParser() - f.optparser.add_option(*args, **kwargs) - return f - return decorate - - -class Cmdln(RawCmdln): - """An improved (on cmd.Cmd) framework for building multi-subcommand - scripts (think "svn" & "cvs") and simple shells (think "pdb" and - "gdb"). - - A simple example: - - import cmdln - - class MySVN(cmdln.Cmdln): - name = "svn" - - @cmdln.aliases('stat', 'st') - @cmdln.option('-v', '--verbose', action='store_true' - help='print verbose information') - def do_status(self, subcmd, opts, *paths): - print "handle 'svn status' command" - - #... - - if __name__ == "__main__": - shell = MySVN() - retval = shell.main() - sys.exit(retval) - - 'Cmdln' extends 'RawCmdln' by providing optparse option processing - integration. See this class' _dispatch_cmd() docstring and - <http://trentm.com/projects/cmdln> for more information. - """ - def _dispatch_cmd(self, handler, argv): - """Introspect sub-command handler signature to determine how to - dispatch the command. The raw handler provided by the base - 'RawCmdln' class is still supported: - - def do_foo(self, argv): - # 'argv' is the vector of command line args, argv[0] is - # the command name itself (i.e. "foo" or an alias) - pass - - In addition, if the handler has more than 2 arguments option - processing is automatically done (using optparse): - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar(self, subcmd, opts, *args): - # subcmd = <"bar" or an alias> - # opts = <an optparse.Values instance> - if opts.verbose: - print "lots of debugging output..." - # args = <tuple of arguments> - for arg in args: - bar(arg) - - TODO: explain that "*args" can be other signatures as well. - - The `cmdln.option` decorator corresponds to an `add_option()` - method call on an `optparse.OptionParser` instance. - - You can declare a specific number of arguments: - - @cmdln.option('-v', '--verbose', action='store_true') - def do_bar2(self, subcmd, opts, bar_one, bar_two): - #... - - and an appropriate error message will be raised/printed if the - command is called with a different number of args. - """ - co_argcount = handler.im_func.func_code.co_argcount - if co_argcount == 2: # handler ::= do_foo(self, argv) - return handler(argv) - elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...) - try: - optparser = handler.optparser - except AttributeError: - optparser = handler.im_func.optparser = SubCmdOptionParser() - assert isinstance(optparser, SubCmdOptionParser) - optparser.set_cmdln_info(self, argv[0]) - try: - opts, args = optparser.parse_args(argv[1:]) - except StopOptionProcessing: - #TODO: this doesn't really fly for a replacement of - # optparse.py behaviour, does it? - return 0 # Normal command termination - - try: - return handler(argv[0], opts, *args) - except TypeError, ex: - # Some TypeError's are user errors: - # do_foo() takes at least 4 arguments (3 given) - # do_foo() takes at most 5 arguments (6 given) - # do_foo() takes exactly 5 arguments (6 given) - # Raise CmdlnUserError for these with a suitably - # massaged error message. - import sys - tb = sys.exc_info()[2] # the traceback object - if tb.tb_next is not None: - # If the traceback is more than one level deep, then the - # TypeError do *not* happen on the "handler(...)" call - # above. In that we don't want to handle it specially - # here: it would falsely mask deeper code errors. - raise - msg = ex.args[0] - match = _INCORRECT_NUM_ARGS_RE.search(msg) - if match: - msg = list(match.groups()) - msg[1] = int(msg[1]) - 3 - if msg[1] == 1: - msg[2] = msg[2].replace("arguments", "argument") - msg[3] = int(msg[3]) - 3 - msg = ''.join(map(str, msg)) - raise CmdlnUserError(msg) - else: - raise - else: - raise CmdlnError("incorrect argcount for %s(): takes %d, must " - "take 2 for 'argv' signature or 3+ for 'opts' " - "signature" % (handler.__name__, co_argcount)) - - - -#---- internal support functions - -def _format_linedata(linedata, indent, indent_width): - """Format specific linedata into a pleasant layout. - - "linedata" is a list of 2-tuples of the form: - (<item-display-string>, <item-docstring>) - "indent" is a string to use for one level of indentation - "indent_width" is a number of columns by which the - formatted data will be indented when printed. - - The <item-display-string> column is held to 15 columns. - """ - lines = [] - WIDTH = 78 - indent_width - SPACING = 2 - NAME_WIDTH_LOWER_BOUND = 13 - NAME_WIDTH_UPPER_BOUND = 16 - NAME_WIDTH = max([len(s) for s,d in linedata]) - if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND: - NAME_WIDTH = NAME_WIDTH_LOWER_BOUND - else: - NAME_WIDTH = NAME_WIDTH_UPPER_BOUND - - DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING - for namestr, doc in linedata: - line = indent + namestr - if len(namestr) <= NAME_WIDTH: - line += ' ' * (NAME_WIDTH + SPACING - len(namestr)) - else: - lines.append(line) - line = indent + ' ' * (NAME_WIDTH + SPACING) - line += _summarize_doc(doc, DOC_WIDTH) - lines.append(line.rstrip()) - return lines - -def _summarize_doc(doc, length=60): - r"""Parse out a short one line summary from the given doclines. - - "doc" is the doc string to summarize. - "length" is the max length for the summary - - >>> _summarize_doc("this function does this") - 'this function does this' - >>> _summarize_doc("this function does this", 10) - 'this fu...' - >>> _summarize_doc("this function does this\nand that") - 'this function does this and that' - >>> _summarize_doc("this function does this\n\nand that") - 'this function does this' - """ - import re - if doc is None: - return "" - assert length > 3, "length <= 3 is absurdly short for a doc summary" - doclines = doc.strip().splitlines(0) - if not doclines: - return "" - - summlines = [] - for i, line in enumerate(doclines): - stripped = line.strip() - if not stripped: - break - summlines.append(stripped) - if len(''.join(summlines)) >= length: - break - - summary = ' '.join(summlines) - if len(summary) > length: - summary = summary[:length-3] + "..." - return summary - - -def line2argv(line): - r"""Parse the given line into an argument vector. - - "line" is the line of input to parse. - - This may get niggly when dealing with quoting and escaping. The - current state of this parsing may not be completely thorough/correct - in this respect. - - >>> from cmdln import line2argv - >>> line2argv("foo") - ['foo'] - >>> line2argv("foo bar") - ['foo', 'bar'] - >>> line2argv("foo bar ") - ['foo', 'bar'] - >>> line2argv(" foo bar") - ['foo', 'bar'] - - Quote handling: - - >>> line2argv("'foo bar'") - ['foo bar'] - >>> line2argv('"foo bar"') - ['foo bar'] - >>> line2argv(r'"foo\"bar"') - ['foo"bar'] - >>> line2argv("'foo bar' spam") - ['foo bar', 'spam'] - >>> line2argv("'foo 'bar spam") - ['foo bar', 'spam'] - - >>> line2argv('some\tsimple\ttests') - ['some', 'simple', 'tests'] - >>> line2argv('a "more complex" test') - ['a', 'more complex', 'test'] - >>> line2argv('a more="complex test of " quotes') - ['a', 'more=complex test of ', 'quotes'] - >>> line2argv('a more" complex test of " quotes') - ['a', 'more complex test of ', 'quotes'] - >>> line2argv('an "embedded \\"quote\\""') - ['an', 'embedded "quote"'] - - # Komodo bug 48027 - >>> line2argv('foo bar C:\\') - ['foo', 'bar', 'C:\\'] - - # Komodo change 127581 - >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"') - ['\\test\\slash', 'foo bar', 'foo"bar'] - - # Komodo change 127629 - >>> if sys.platform == "win32": - ... line2argv(r'\foo\bar') == ['\\foo\\bar'] - ... line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar'] - ... line2argv('"foo') == ['foo'] - ... else: - ... line2argv(r'\foo\bar') == ['foobar'] - ... line2argv(r'\\foo\\bar') == ['\\foo\\bar'] - ... try: - ... line2argv('"foo') - ... except ValueError, ex: - ... "not terminated" in str(ex) - True - True - True - """ - import string - line = line.strip() - argv = [] - state = "default" - arg = None # the current argument being parsed - i = -1 - while 1: - i += 1 - if i >= len(line): break - ch = line[i] - - if ch == "\\" and i+1 < len(line): - # escaped char always added to arg, regardless of state - if arg is None: arg = "" - if (sys.platform == "win32" - or state in ("double-quoted", "single-quoted") - ) and line[i+1] not in tuple('"\''): - arg += ch - i += 1 - arg += line[i] - continue - - if state == "single-quoted": - if ch == "'": - state = "default" - else: - arg += ch - elif state == "double-quoted": - if ch == '"': - state = "default" - else: - arg += ch - elif state == "default": - if ch == '"': - if arg is None: arg = "" - state = "double-quoted" - elif ch == "'": - if arg is None: arg = "" - state = "single-quoted" - elif ch in string.whitespace: - if arg is not None: - argv.append(arg) - arg = None - else: - if arg is None: arg = "" - arg += ch - if arg is not None: - argv.append(arg) - if not sys.platform == "win32" and state != "default": - raise ValueError("command line is not terminated: unfinished %s " - "segment" % state) - return argv - - -def argv2line(argv): - r"""Put together the given argument vector into a command line. - - "argv" is the argument vector to process. - - >>> from cmdln import argv2line - >>> argv2line(['foo']) - 'foo' - >>> argv2line(['foo', 'bar']) - 'foo bar' - >>> argv2line(['foo', 'bar baz']) - 'foo "bar baz"' - >>> argv2line(['foo"bar']) - 'foo"bar' - >>> print argv2line(['foo" bar']) - 'foo" bar' - >>> print argv2line(["foo' bar"]) - "foo' bar" - >>> argv2line(["foo'bar"]) - "foo'bar" - """ - escapedArgs = [] - for arg in argv: - if ' ' in arg and '"' not in arg: - arg = '"'+arg+'"' - elif ' ' in arg and "'" not in arg: - arg = "'"+arg+"'" - elif ' ' in arg: - arg = arg.replace('"', r'\"') - arg = '"'+arg+'"' - escapedArgs.append(arg) - return ' '.join(escapedArgs) - - -# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook -def _dedentlines(lines, tabsize=8, skip_first_line=False): - """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines - - "lines" is a list of lines to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - Same as dedent() except operates on a sequence of lines. Note: the - lines list is modified **in-place**. - """ - DEBUG = False - if DEBUG: - print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ - % (tabsize, skip_first_line) - indents = [] - margin = None - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - indent = 0 - for ch in line: - if ch == ' ': - indent += 1 - elif ch == '\t': - indent += tabsize - (indent % tabsize) - elif ch in '\r\n': - continue # skip all-whitespace lines - else: - break - else: - continue # skip all-whitespace lines - if DEBUG: print "dedent: indent=%d: %r" % (indent, line) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if DEBUG: print "dedent: margin=%r" % margin - - if margin is not None and margin > 0: - for i, line in enumerate(lines): - if i == 0 and skip_first_line: continue - removed = 0 - for j, ch in enumerate(line): - if ch == ' ': - removed += 1 - elif ch == '\t': - removed += tabsize - (removed % tabsize) - elif ch in '\r\n': - if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line - lines[i] = lines[i][j:] - break - else: - raise ValueError("unexpected non-whitespace char %r in " - "line %r while removing %d-space margin" - % (ch, line, margin)) - if DEBUG: - print "dedent: %r: %r -> removed %d/%d"\ - % (line, ch, removed, margin) - if removed == margin: - lines[i] = lines[i][j+1:] - break - elif removed > margin: - lines[i] = ' '*(removed-margin) + lines[i][j+1:] - break - return lines - -def _dedent(text, tabsize=8, skip_first_line=False): - """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text - - "text" is the text to dedent. - "tabsize" is the tab width to use for indent width calculations. - "skip_first_line" is a boolean indicating if the first line should - be skipped for calculating the indent width and for dedenting. - This is sometimes useful for docstrings and similar. - - textwrap.dedent(s), but don't expand tabs to spaces - """ - lines = text.splitlines(1) - _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) - return ''.join(lines) - - -def _get_indent(marker, s, tab_width=8): - """_get_indent(marker, s, tab_width=8) -> - (<indentation-of-'marker'>, <indentation-width>)""" - # Figure out how much the marker is indented. - INDENT_CHARS = tuple(' \t') - start = s.index(marker) - i = start - while i > 0: - if s[i-1] not in INDENT_CHARS: - break - i -= 1 - indent = s[i:start] - indent_width = 0 - for ch in indent: - if ch == ' ': - indent_width += 1 - elif ch == '\t': - indent_width += tab_width - (indent_width % tab_width) - return indent, indent_width - -def _get_trailing_whitespace(marker, s): - """Return the whitespace content trailing the given 'marker' in string 's', - up to and including a newline. - """ - suffix = '' - start = s.index(marker) + len(marker) - i = start - while i < len(s): - if s[i] in ' \t': - suffix += s[i] - elif s[i] in '\r\n': - suffix += s[i] - if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n': - suffix += s[i+1] - break - else: - break - i += 1 - return suffix - - - -#---- bash completion support -# Note: This is still experimental. I expect to change this -# significantly. -# -# To get Bash completion for a cmdln.Cmdln class, run the following -# bash command: -# $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname -# For example: -# $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn -# -#TODO: Simplify the above so don't have to given path to script (try to -# find it on PATH, if possible). Could also make class name -# optional if there is only one in the module (common case). - -if __name__ == "__main__" and len(sys.argv) == 6: - def _log(s): - return # no-op, comment out for debugging - from os.path import expanduser - fout = open(expanduser("~/tmp/bashcpln.log"), 'a') - fout.write(str(s) + '\n') - fout.close() - - # Recipe: module_from_path (1.0.1+) - def _module_from_path(path): - import imp, os, sys - path = os.path.expanduser(path) - dir = os.path.dirname(path) or os.curdir - name = os.path.splitext(os.path.basename(path))[0] - sys.path.insert(0, dir) - try: - iinfo = imp.find_module(name, [dir]) - return imp.load_module(name, *iinfo) - finally: - sys.path.remove(dir) - - def _get_bash_cplns(script_path, class_name, cmd_name, - token, preceding_token): - _log('--') - _log('get_cplns(%r, %r, %r, %r, %r)' - % (script_path, class_name, cmd_name, token, preceding_token)) - comp_line = os.environ["COMP_LINE"] - comp_point = int(os.environ["COMP_POINT"]) - _log("COMP_LINE: %r" % comp_line) - _log("COMP_POINT: %r" % comp_point) - - try: - script = _module_from_path(script_path) - except ImportError, ex: - _log("error importing `%s': %s" % (script_path, ex)) - return [] - shell = getattr(script, class_name)() - cmd_map = shell._get_canonical_map() - del cmd_map["EOF"] - - # Determine if completing the sub-command name. - parts = comp_line[:comp_point].split(None, 1) - _log(parts) - if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]): - #TODO: if parts[1].startswith('-'): handle top-level opts - _log("complete sub-command names") - matches = {} - for name, canon_name in cmd_map.items(): - if name.startswith(token): - matches[name] = canon_name - if not matches: - return [] - elif len(matches) == 1: - return matches.keys() - elif len(set(matches.values())) == 1: - return [matches.values()[0]] - else: - return matches.keys() - - # Otherwise, complete options for the given sub-command. - #TODO: refine this so it does the right thing with option args - if token.startswith('-'): - cmd_name = comp_line.split(None, 2)[1] - try: - cmd_canon_name = cmd_map[cmd_name] - except KeyError: - return [] - handler = shell._get_cmd_handler(cmd_canon_name) - optparser = getattr(handler, "optparser", None) - if optparser is None: - optparser = SubCmdOptionParser() - opt_strs = [] - for option in optparser.option_list: - for opt_str in option._short_opts + option._long_opts: - if opt_str.startswith(token): - opt_strs.append(opt_str) - return opt_strs - - return [] - - for cpln in _get_bash_cplns(*sys.argv[1:]): - print cpln - diff --git a/mic/utils/fs_related.py b/mic/utils/fs_related.py index 9356b3c..a6c2c0b 100644 --- a/mic/utils/fs_related.py +++ b/mic/utils/fs_related.py @@ -458,6 +458,7 @@ class ExtDiskMount(DiskMount): runner.show([self.tune2fs, "-c0", "-i0", "-Odir_index", "-ouser_xattr,acl", self.disk.device]) def __resize_filesystem(self, size = None): + msger.info("Resizing filesystem ...") current_size = os.stat(self.disk.lofile)[stat.ST_SIZE] if size is None: @@ -499,6 +500,7 @@ class ExtDiskMount(DiskMount): "Block count")) * self.blocksize def __resize_to_minimal(self): + msger.info("Resizing filesystem to minimal ...") self.__fsck() # @@ -842,7 +844,7 @@ class LoopDevice(object): def register(self, device): self.device = device self.loopid = None - self.created = True + #self.created = True def reg_atexit(self): import atexit diff --git a/mic/utils/grabber.py b/mic/utils/grabber.py index a0a0ae1..c42dc02 100644 --- a/mic/utils/grabber.py +++ b/mic/utils/grabber.py @@ -1,5 +1,20 @@ #!/usr/bin/python +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import os import sys import fcntl diff --git a/mic/utils/misc.py b/mic/utils/misc.py index 7813921..1e337d9 100755 --- a/mic/utils/misc.py +++ b/mic/utils/misc.py @@ -187,91 +187,6 @@ def extract_rpm(rpmfile, targetdir): os.chdir(olddir) -def compressing(fpath, method): - comp_map = { - "gz": ["pgzip", "pigz", "gzip"], - "bz2": ["pbzip2", "bzip2"], - } - if method not in comp_map: - raise CreatorError("Unsupport compress format: %s, valid values: %s" - % (method, ','.join(comp_map.keys()))) - cmd = None - for cmdname in comp_map[method]: - try: - cmd = find_binary_path(cmdname) - break - except CreatorError as err: - pass - if not cmd: - raise CreatorError("Command %s not available" % cmdname) - rc = runner.show([cmd, "-f", fpath]) - if rc: - raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath)) - -def taring(dstfile, target): - import tarfile - basen, ext = os.path.splitext(dstfile) - comp = {".tar": None, - ".gz": "gz", # for .tar.gz - ".bz2": "bz2", # for .tar.bz2 - ".tgz": "gz", - ".tbz": "bz2"}[ext] - - # specify tarball file path - if not comp: - tarpath = dstfile - elif basen.endswith(".tar"): - tarpath = basen - else: - tarpath = basen + ".tar" - wf = tarfile.open(tarpath, 'w') - - if os.path.isdir(target): - for item in os.listdir(target): - wf.add(os.path.join(target, item), item) - else: - wf.add(target, os.path.basename(target)) - wf.close() - - if comp: - compressing(tarpath, comp) - # when dstfile ext is ".tgz" and ".tbz", should rename - if not basen.endswith(".tar"): - shutil.move("%s.%s" % (tarpath, comp), dstfile) - -def ziping(dstfile, target): - import zipfile - wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED) - if os.path.isdir(target): - for item in os.listdir(target): - fpath = os.path.join(target, item) - if not os.path.isfile(fpath): - continue - wf.write(fpath, item, zipfile.ZIP_DEFLATED) - else: - wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED) - wf.close() - -pack_formats = { - ".tar": taring, - ".tar.gz": taring, - ".tar.bz2": taring, - ".tgz": taring, - ".tbz": taring, - ".zip": ziping, -} - -def packing(dstfile, target): - (base, ext) = os.path.splitext(dstfile) - if ext in (".gz", ".bz2") and base.endswith(".tar"): - ext = ".tar" + ext - if ext not in pack_formats: - raise CreatorError("Unsupport pack format: %s, valid values: %s" - % (ext, ','.join(pack_formats.keys()))) - func = pack_formats[ext] - # func should be callable - func(dstfile, target) - def human_size(size): """Return human readable string for Bytes size """ @@ -346,6 +261,11 @@ def calc_hashes(file_path, hash_names, start = 0, end = None): def get_md5sum(fpath): return calc_hashes(fpath, ('md5', ))[0] +def get_sha1sum(fpath): + return calc_hashes(fpath, ('sha1', ))[0] + +def get_sha256sum(fpath): + return calc_hashes(fpath, ('sha256', ))[0] def normalize_ksfile(ksconf, release, arch): ''' diff --git a/mic/utils/partitionedfs.py b/mic/utils/partitionedfs.py index d0aa652..cac360b 100644 --- a/mic/utils/partitionedfs.py +++ b/mic/utils/partitionedfs.py @@ -34,6 +34,13 @@ GPT_OVERHEAD = 34 # Size of a sector in bytes SECTOR_SIZE = 512 +def resolve_ref(ref): + real = os.readlink(ref) + if not real.startswith('/'): + return os.path.realpath(os.path.join(ref, real)) + else: + return real + class PartitionedMount(Mount): def __init__(self, mountdir, skipformat = False): Mount.__init__(self, mountdir) @@ -41,13 +48,13 @@ class PartitionedMount(Mount): self.partitions = [] self.subvolumes = [] self.mapped = False - self.mountOrder = [] - self.unmountOrder = [] + self.mount_order = [] + self.unmount_order = [] self.parted = find_binary_path("parted") self.kpartx = find_binary_path("kpartx") self.mkswap = find_binary_path("mkswap") self.dmsetup = find_binary_path("dmsetup") - self.btrfscmd=None + self.btrfscmd = None self.mountcmd = find_binary_path("mount") self.umountcmd = find_binary_path("umount") self.skipformat = skipformat @@ -107,7 +114,7 @@ class PartitionedMount(Mount): # We need to handle subvolumes for btrfs if fstype == "btrfs" and fsopts and fsopts.find("subvol=") != -1: - self.btrfscmd=find_binary_path("btrfs") + self.btrfscmd = find_binary_path("btrfs") subvol = None opts = fsopts.split(",") for opt in opts: @@ -123,6 +130,7 @@ class PartitionedMount(Mount): 'disk_name': disk_name, # physical disk name holding partition 'device': None, # kpartx device node for partition 'mapper_device': None, # mapper device node + 'mpath_device': None, # multipath device of device mapper 'mount': None, # Mount object 'subvol': subvol, # Subvolume name 'boot': boot, # Bootable flag @@ -149,6 +157,7 @@ class PartitionedMount(Mount): 'disk_name': disk_name, # physical disk name holding partition 'device': None, # kpartx device node for partition 'mapper_device': None, # mapper device node + 'mpath_device': None, # multipath device of device mapper 'mount': None, # Mount object 'num': None, # Partition number 'boot': boot, # Bootable flag @@ -367,12 +376,12 @@ class PartitionedMount(Mount): continue pnum = 0 - gpt_parser = GptParser(d['disk'].device, SECTOR_SIZE) + gpt_parser = GptParser(disk['disk'].device, SECTOR_SIZE) # Iterate over all GPT partitions on this disk for entry in gpt_parser.get_partitions(): pnum += 1 # Find the matching partition in the 'self.partitions' list - for n in d['partitions']: + for n in disk['partitions']: p = self.partitions[n] if p['num'] == pnum: # Found, fetch PARTUUID (partition's unique ID) @@ -400,8 +409,8 @@ class PartitionedMount(Mount): continue msger.debug("Running kpartx on %s" % d['disk'].device ) - rc, kpartxOutput = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) - kpartxOutput = kpartxOutput.splitlines() + rc, kpartx_output = runner.runtool([self.kpartx, "-l", "-v", d['disk'].device]) + kpartx_output = kpartx_output.splitlines() if rc != 0: raise MountError("Failed to query partition mapping for '%s'" % @@ -409,12 +418,12 @@ class PartitionedMount(Mount): # Strip trailing blank and mask verbose output i = 0 - while i < len(kpartxOutput) and kpartxOutput[i][0:4] != "loop": + while i < len(kpartx_output) and kpartx_output[i][0:4] != "loop": i = i + 1 - kpartxOutput = kpartxOutput[i:] + kpartx_output = kpartx_output[i:] # Make sure kpartx reported the right count of partitions - if len(kpartxOutput) != d['numpart']: + if len(kpartx_output) != d['numpart']: # If this disk has more than 3 partitions, then in case of MBR # paritions there is an extended parition. Different versions # of kpartx behave differently WRT the extended partition - @@ -423,16 +432,16 @@ class PartitionedMount(Mount): # table type is "msdos" and the amount of partitions is more # than 3, we just assume kpartx mapped the extended parition # and we remove it. - if len(kpartxOutput) == d['numpart'] + 1 \ - and d['ptable_format'] == 'msdos' and len(kpartxOutput) > 3: - kpartxOutput.pop(3) + if len(kpartx_output) == d['numpart'] + 1 \ + and d['ptable_format'] == 'msdos' and len(kpartx_output) > 3: + kpartx_output.pop(3) else: raise MountError("Unexpected number of partitions from " \ "kpartx: %d != %d" % \ - (len(kpartxOutput), d['numpart'])) + (len(kpartx_output), d['numpart'])) - for i in range(len(kpartxOutput)): - line = kpartxOutput[i] + for i in range(len(kpartx_output)): + line = kpartx_output[i] newdev = line.split()[0] mapperdev = "/dev/mapper/" + newdev loopdev = d['disk'].device + newdev[-1] @@ -461,6 +470,12 @@ class PartitionedMount(Mount): raise MountError("Failed to map partitions for '%s'" % d['disk'].device) + for p in self.partitions: + if p['mapper_device'] and os.path.islink(p['mapper_device']): + p['mpath_device'] = resolve_ref(p['mapper_device']) + else: + p['mpath_device'] = '' + # FIXME: need a better way to fix the latency import time time.sleep(1) @@ -500,12 +515,12 @@ class PartitionedMount(Mount): msger.debug("Calculating mount order") for p in self.partitions: if p['mountpoint']: - self.mountOrder.append(p['mountpoint']) - self.unmountOrder.append(p['mountpoint']) + self.mount_order.append(p['mountpoint']) + self.unmount_order.append(p['mountpoint']) - self.mountOrder.sort() - self.unmountOrder.sort() - self.unmountOrder.reverse() + self.mount_order.sort() + self.unmount_order.sort() + self.unmount_order.reverse() def cleanup(self): Mount.cleanup(self) @@ -520,7 +535,7 @@ class PartitionedMount(Mount): def unmount(self): self.__unmount_subvolumes() - for mp in self.unmountOrder: + for mp in self.unmount_order: if mp == 'swap': continue p = None @@ -532,7 +547,8 @@ class PartitionedMount(Mount): if p['mount'] != None: try: # Create subvolume snapshot here - if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and not self.snapshot_created: + if p['fstype'] == "btrfs" and p['mountpoint'] == "/" and \ + not self.snapshot_created: self.__create_subvolume_snapshots(p, p["mount"]) p['mount'].cleanup() except: @@ -542,14 +558,15 @@ class PartitionedMount(Mount): # Only for btrfs def __get_subvolume_id(self, rootpath, subvol): if not self.btrfscmd: - self.btrfscmd=find_binary_path("btrfs") + self.btrfscmd = find_binary_path("btrfs") argv = [ self.btrfscmd, "subvolume", "list", rootpath ] rc, out = runner.runtool(argv) msger.debug(out) if rc != 0: - raise MountError("Failed to get subvolume id from %s', return code: %d." % (rootpath, rc)) + raise MountError("Failed to get subvolume id from %s'," + "return code: %d." % (rootpath, rc)) subvolid = -1 for line in out.splitlines(): @@ -588,7 +605,8 @@ class PartitionedMount(Mount): opts.remove(opt) break fsopts = ",".join(opts) - subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], subvol['mountpoint'], fsopts) + subvolume_metadata += "%d\t%s\t%s\t%s\n" % (subvolid, subvol["subvol"], + subvol['mountpoint'], fsopts) if subvolume_metadata: fd = open("%s/.subvolume_metadata" % pdisk.mountdir, "w") @@ -641,9 +659,11 @@ class PartitionedMount(Mount): subvolid = self. __get_subvolume_id(pdisk.mountdir, subvol["subvol"]) # Set default subvolume if subvolid != -1: - rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % subvolid, pdisk.mountdir]) + rc = runner.show([ self.btrfscmd, "subvolume", "set-default", "%d" % + subvolid, pdisk.mountdir]) if rc != 0: - raise MountError("Failed to set default subvolume id: %d', return code: %d." % (subvolid, rc)) + raise MountError("Failed to set default subvolume id: %d', return code: %d." + % (subvolid, rc)) self.__create_subvolume_metadata(p, pdisk) @@ -738,7 +758,8 @@ class PartitionedMount(Mount): snapshotpath = subvolpath + "_%s-1" % snapshotts rc = runner.show([ self.btrfscmd, "subvolume", "snapshot", subvolpath, snapshotpath ]) if rc != 0: - raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code: %d." % (snapshotpath, subvolpath, rc)) + raise MountError("Failed to create subvolume snapshot '%s' for '%s', return code:" + "%d." % (snapshotpath, subvolpath, rc)) self.snapshot_created = True @@ -751,7 +772,7 @@ class PartitionedMount(Mount): self.__map_partitions() self.__calculate_mountorder() - for mp in self.mountOrder: + for mp in self.mount_order: p = None for p1 in self.partitions: if p1['mountpoint'] == mp: @@ -777,18 +798,18 @@ class PartitionedMount(Mount): if p['mountpoint'] == "/": rmmountdir = True if p['fstype'] == "vfat" or p['fstype'] == "msdos": - myDiskMount = VfatDiskMount + my_disk_mount = VfatDiskMount elif p['fstype'] in ("ext2", "ext3", "ext4"): - myDiskMount = ExtDiskMount + my_disk_mount = ExtDiskMount elif p['fstype'] == "btrfs": - myDiskMount = BtrfsDiskMount + my_disk_mount = BtrfsDiskMount else: raise MountError("Fail to support file system " + p['fstype']) if p['fstype'] == "btrfs" and not p['fsopts']: p['fsopts'] = "subvolid=0" - pdisk = myDiskMount(RawDisk(p['size'] * self.sector_size, p['device']), + pdisk = my_disk_mount(RawDisk(p['size'] * self.sector_size, p['device']), self.mountdir + p['mountpoint'], p['fstype'], 4096, diff --git a/mic/utils/proxy.py b/mic/utils/proxy.py index c1fb94f..acb05e4 100644 --- a/mic/utils/proxy.py +++ b/mic/utils/proxy.py @@ -41,9 +41,9 @@ def unset_proxy_environ(): if env in os.environ: del os.environ[env] - ENV=env.upper() - if ENV in os.environ: - del os.environ[ENV] + env_upper = env.upper() + if env_upper in os.environ: + del os.environ[env_upper] def _set_proxies(proxy = None, no_proxy = None): """Return a dictionary of scheme -> proxy server URL mappings. @@ -74,16 +74,16 @@ def _set_proxies(proxy = None, no_proxy = None): _my_noproxy = value def _ip_to_int(ip): - ipint=0 - shift=24 + ipint = 0 + shift = 24 for dec in ip.split("."): ipint |= int(dec) << shift shift -= 8 return ipint def _int_to_ip(val): - ipaddr="" - shift=0 + ipaddr = "" + shift = 0 for i in range(4): dec = val >> shift dec &= 0xff @@ -108,11 +108,11 @@ def _set_noproxy_list(): if item[0] != '.' and item.find("/") == -1: # Need to match it - _my_noproxy_list.append({"match":0,"needle":item}) + _my_noproxy_list.append({"match":0, "needle":item}) elif item[0] == '.': # Need to match at tail - _my_noproxy_list.append({"match":1,"needle":item}) + _my_noproxy_list.append({"match":1, "needle":item}) elif item.find("/") > 3: # IP/MASK, need to match at head @@ -126,17 +126,18 @@ def _set_noproxy_list(): netmask = ~((1<<(32-netmask)) - 1) ip &= netmask else: - shift=24 - netmask=0 + shift = 24 + netmask = 0 for dec in mask.split("."): netmask |= int(dec) << shift shift -= 8 ip &= netmask - _my_noproxy_list.append({"match":2,"needle":ip,"netmask":netmask}) + _my_noproxy_list.append({"match":2, "needle":ip, "netmask":netmask}) def _isnoproxy(url): - (scheme, host, path, parm, query, frag) = urlparse.urlparse(url) + host = urlparse.urlparse(url)[1] + # urlparse.urlparse(url) returns (scheme, host, path, parm, query, frag) if '@' in host: user_pass, host = host.split('@', 1) diff --git a/mic/utils/safeurl.py b/mic/utils/safeurl.py index d91509b..0a82f8e 100644 --- a/mic/utils/safeurl.py +++ b/mic/utils/safeurl.py @@ -1,3 +1,18 @@ +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + """ This module provides a class SafeURL which can contain url/user/password read from config file, and hide plain user and password when it print to screen diff --git a/packaging/mic.changes b/packaging/mic.changes index 5cb942d..41b3718 100644 --- a/packaging/mic.changes +++ b/packaging/mic.changes @@ -1,21 +1,37 @@ -* Thu Jan 22 2015 Lihong Sun <lihongx.sun@intel.com> - 0.24.4 +* Mon Mar 28 2016 Jianzhong Fang <jz.fang@samsung.com> - 0.27 + * new distribution support: CentOS 7, Debian 8, Fedora 21, + Fedora 22, Fedora 23, openSUSE 13.2 * generate manifest file to describe image information - * update mount option to make it more general - * bug fix: - - fix incorrect logfile location of '--release' if creation failed - - update dependency package for mic - - add catching 'distribution not found' exception + * refactor archive and compress module + * support sparse handle for tar command + * replace system V with systemd on locale setting + * add qcow2 image format support + * add strict mode for package installing + * enable ssl_verify option in config file + * enhance checksums of outputs: md5sum, sha1sum, sha256sum + * drop mic-native support + * update mount option + * revert bind mount config file to instroot + * drop liveusb, livecd and raw image formats support + * use argparse module to parse the cmd line -* Mon Nov 24 2014 Yongfeng Du <dolpher.du@intel.com> - 0.24.3 * bug fix: + - fix logfile incomplete in release option + - fix config file disappear in bootstrap + - fix aarch64 bin_format + - fix pylint + - fix real path of device mapper causing initrd failure - fix qemu arm and arm64 issues - - remove --preserve-order option in taring fs image - -* Wed Jul 09 2014 Gui Chen <gui.chen@intel.com> - 0.24.2 - * enable ssl_verify in config file - * fix logfile incompleted issue - * fix md5sum to be compatible with utility md5sum - * fix locale issue in systemd service + - fix AttributeError in zypp backend + - fix 'python-xml' depends used by cElements + - fix xml requirements + - fix logfile not in release of '--release' when creation failed + - fix copyright missing issue + - fix syslinux installation path issue in Arch Linux + - fix priority option of ks file not apply + - fix need to check loop device after excute 'losetup --find' + - fix check scriptlet error file on /tmp/.postscript/error/ + - fix broken tar archive * Tue Mar 11 2014 Gui Chen <gui.chen@intel.com> - 0.24 * enhance to handle password with special characters diff --git a/packaging/mic.spec b/packaging/mic.spec index 52a271f..b0607f2 100644 --- a/packaging/mic.spec +++ b/packaging/mic.spec @@ -1,4 +1,5 @@ %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%{!?python_version: %define python_version %(%{__python} -c "import sys; sys.stdout.write(sys.version[:3])")} %define rc_version 0 @@ -8,7 +9,7 @@ Name: mic Summary: Image Creator for Linux Distributions -Version: 0.24.4 +Version: 0.27 Release: %{?release_prefix}%{?opensuse_bs:<CI_CNT>.<B_CNT>}%{!?opensuse_bs:0} Group: Development/Tools License: GPLv2 @@ -22,6 +23,10 @@ Requires: python-urlgrabber >= 3.9.0 Requires: python-xml %endif +%if "%{?python_version}" < "2.7" +Requires: python-argparse +%endif + %if 0%{?tizen_version:1} Requires: python-rpm %else @@ -49,60 +54,6 @@ is used to create images with different types; subcommand convert is used to convert an image to a specified type; subcommand chroot is used to chroot into an image. -%package native -Summary: Native support for mic -Requires: util-linux -Requires: coreutils -Requires: psmisc -Requires: e2fsprogs -Requires: dosfstools >= 2.11 -Requires: kpartx -Requires: parted -Requires: device-mapper -Requires: syslinux >= 3.82 -%if ! 0%{?suse_version} -Requires: syslinux-extlinux >= 3.82 -%endif - -%if 0%{?suse_version} || 0%{?tizen_version:1} -Requires: squashfs >= 4.0 -Requires: python-m2crypto -%else -Requires: python-libs -Requires: squashfs-tools >= 4.0 -Requires: m2crypto -%endif - -%if 0%{?suse_version} || 0%{?tizen_version:1} -Requires: /usr/bin/qemu-arm -%else -Requires: qemu-arm-static -%endif - -%if ! 0%{?tizen_version:1} -Requires: isomd5sum -Requires: /usr/bin/genisoimage -%endif - -Requires: yum >= 3.2.24 -%if 0%{?tizen_version:1} -Requires: python-zypp -%else -Requires: python-zypp-tizen -%endif - -Requires: mic - -#%if 0%{?suse_version} -#Requires: btrfsprogs -#%else -#Requires: btrfs-progs -#%endif - -%description native -The native support package for mic, it includes all requirements -for mic native running. - %prep %setup -q -n %{name}-%{version} @@ -130,8 +81,6 @@ install -Dp -m0755 etc/bash_completion.d/%{name}.sh %{buildroot}/%{_sysconfdir}/ install -d -m0755 %{buildroot}/%{_sysconfdir}/zsh_completion.d/ install -Dp -m0755 etc/zsh_completion.d/_%{name} %{buildroot}/%{_sysconfdir}/zsh_completion.d/ -install -Dp -m0755 tools/mic %{buildroot}/%{_bindir}/mic-native - %files %defattr(-,root,root,-) %doc doc/* @@ -148,5 +97,3 @@ install -Dp -m0755 tools/mic %{buildroot}/%{_bindir}/mic-native %{_sysconfdir}/bash_completion.d %{_sysconfdir}/zsh_completion.d -%files native -%{_bindir}/mic-native diff --git a/plugins/backend/yumpkgmgr.py b/plugins/backend/yumpkgmgr.py index 6884551..8f3112d 100644 --- a/plugins/backend/yumpkgmgr.py +++ b/plugins/backend/yumpkgmgr.py @@ -16,8 +16,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os, sys -import re +import os import tempfile import glob from string import Template @@ -32,7 +31,6 @@ from mic.utils.grabber import TextProgress from mic.utils.proxy import get_proxy_for from mic.utils.errors import CreatorError from mic.utils.safeurl import SafeURL -from mic.imager.baseimager import BaseImageCreator YUMCONF_TEMP = """[main] @@ -114,12 +112,13 @@ from mic.pluginbase import BackendPlugin class Yum(BackendPlugin, yum.YumBase): name = 'yum' - def __init__(self, target_arch, instroot, cachedir): + def __init__(self, target_arch, instroot, cachedir, strict_mode = False): yum.YumBase.__init__(self) self.cachedir = cachedir self.instroot = instroot self.target_arch = target_arch + self.strict_mode = strict_mode if self.target_arch: if not rpmUtils.arch.arches.has_key(self.target_arch): @@ -387,7 +386,7 @@ class Yum(BackendPlugin, yum.YumBase): % (os.path.basename(local), local)) else: download_total_size -= int(po.packagesize) - cached_count +=1 + cached_count += 1 cache_avail_size = misc.get_filesystem_avail(self.cachedir) if cache_avail_size < download_total_size: @@ -440,7 +439,9 @@ class Yum(BackendPlugin, yum.YumBase): installlogfile = "%s/__catched_stderr.buf" % (self.instroot) msger.enable_logstderr(installlogfile) - self.runTransaction(cb) + transactionResult = self.runTransaction(cb) + if transactionResult.return_code != 0 and self.strict_mode: + raise CreatorError("mic failes to install some packages") self._cleanupRpmdbLocks(self.conf.installroot) except rpmUtils.RpmUtilsError, e: @@ -453,6 +454,23 @@ class Yum(BackendPlugin, yum.YumBase): msger.disable_logstderr() def getVcsInfo(self): + if self.__pkgs_vcsinfo: + return self.__pkgs_vcsinfo + if not self.ts: + self.__initialize_transaction() + mi = self.ts.dbMatch() + for hdr in mi: + lname = misc.RPM_FMT % { + 'name': hdr['name'], + 'arch': hdr['arch'], + 'version': hdr['version'], + 'release': hdr['release'] + } + try: + self.__pkgs_vcsinfo[lname] = hdr['VCS'] + except ValueError, KeyError: + self.__pkgs_vcsinfo[lname] = None + return self.__pkgs_vcsinfo def getAllContent(self): diff --git a/plugins/backend/zypppkgmgr.py b/plugins/backend/zypppkgmgr.py index b323da9..9358cbe 100644 --- a/plugins/backend/zypppkgmgr.py +++ b/plugins/backend/zypppkgmgr.py @@ -54,10 +54,11 @@ from mic.pluginbase import BackendPlugin class Zypp(BackendPlugin): name = 'zypp' - def __init__(self, target_arch, instroot, cachedir): + def __init__(self, target_arch, instroot, cachedir, strict_mode = False): self.cachedir = cachedir self.instroot = instroot self.target_arch = target_arch + self.strict_mode = strict_mode self.__pkgs_license = {} self.__pkgs_content = {} @@ -136,14 +137,14 @@ class Zypp(BackendPlugin): def _zyppQueryPackage(self, pkg): query = zypp.PoolQuery() query.addKind(zypp.ResKind.package) - query.addAttribute(zypp.SolvAttr.name,pkg) + query.addAttribute(zypp.SolvAttr.name, pkg) query.setMatchExact() for pi in query.queryResults(self.Z.pool()): return pi return None def _splitPkgString(self, pkg): - sp = pkg.rsplit(".",1) + sp = pkg.rsplit(".", 1) name = sp[0] arch = None if len(sp) == 2: @@ -210,15 +211,15 @@ class Zypp(BackendPlugin): if endx and startx: pattern = '%s' % (pkg[1:-1]) q.setMatchRegex() - q.addAttribute(zypp.SolvAttr.name,pattern) + q.addAttribute(zypp.SolvAttr.name, pattern) elif arch: q.setMatchExact() - q.addAttribute(zypp.SolvAttr.name,name) + q.addAttribute(zypp.SolvAttr.name, name) else: q.setMatchExact() - q.addAttribute(zypp.SolvAttr.name,pkg) + q.addAttribute(zypp.SolvAttr.name, pkg) for pitem in sorted( q.queryResults(self.Z.pool()), @@ -236,7 +237,7 @@ class Zypp(BackendPlugin): obspkg = self.whatObsolete(item) if arch: if arch == str(item.arch()): - item.status().setToBeInstalled (zypp.ResStatus.USER) + pitem.status().setToBeInstalled (zypp.ResStatus.USER) else: markPoolItem(obspkg, pitem) if not ispattern: @@ -283,29 +284,32 @@ class Zypp(BackendPlugin): if not ispattern: if pkgarch: if name == pkgname and str(item.arch()) == pkgarch: - return True; + return True else: if name == pkgname: - return True; + return True else: if startx and name.endswith(pkg[1:]): - return True; + return True if endx and name.startswith(pkg[:-1]): - return True; + return True - return False; + return False def deselectPackage(self, pkg): """collect packages should not be installed""" self.to_deselect.append(pkg) def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT): + def compareGroup(pitem): + item = zypp.asKindPattern(pitem) + return item.repoInfo().priority() if not self.Z: self.__initialize_zypp() found = False - q=zypp.PoolQuery() + q = zypp.PoolQuery() q.addKind(zypp.ResKind.pattern) - for pitem in q.queryResults(self.Z.pool()): + for pitem in sorted(q.queryResults(self.Z.pool()), key=compareGroup): item = zypp.asKindPattern(pitem) summary = "%s" % item.summary() name = "%s" % item.name() @@ -380,7 +384,8 @@ class Zypp(BackendPlugin): if not ssl_verify: baseurl.setQueryParam("ssl_verify", "no") if proxy: - scheme, host, path, parm, query, frag = urlparse.urlparse(proxy) + host = urlparse.urlparse(proxy)[1] + # scheme, host, path, parm, query, frag = urlparse.urlparse(proxy) proxyinfo = host.rsplit(":", 1) host = proxyinfo[0] @@ -566,9 +571,11 @@ class Zypp(BackendPlugin): if download_count > 0: msger.info("Downloading packages ...") self.downloadPkgs(dlpkgs, download_count) + except CreatorError, e: + raise CreatorError("Package download failed: %s" %(e,)) + try: self.installPkgs(dlpkgs) - except (RepoError, RpmError): raise except Exception, e: @@ -712,7 +719,7 @@ class Zypp(BackendPlugin): def getLocalPkgPath(self, po): repoinfo = po.repoInfo() cacheroot = repoinfo.packagesPath() - location= po.location() + location = po.location() rpmpath = str(location.filename()) pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath)) return pkgpath @@ -870,7 +877,8 @@ class Zypp(BackendPlugin): if len(errors) == 0: msger.warning('scriptlet or other non-fatal errors occurred ' 'during transaction.') - + if self.strict_mode: + raise CreatorError("mic failes to install some packages") else: for e in errors: msger.warning(e[0]) @@ -894,7 +902,7 @@ class Zypp(BackendPlugin): % (package, deppkg)) elif sense == rpm.RPMDEP_SENSE_CONFLICTS: - msger.warning("[%s] Conflicts with [%s]" %(package,deppkg)) + msger.warning("[%s] Conflicts with [%s]" % (package, deppkg)) raise RepoError("Unresolved dependencies, transaction failed.") diff --git a/plugins/imager/fs_plugin.py b/plugins/imager/fs_plugin.py index 15a7644..d74530f 100644 --- a/plugins/imager/fs_plugin.py +++ b/plugins/imager/fs_plugin.py @@ -15,11 +15,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import os -import sys - from mic import chroot, msger, rt_util -from mic.utils import cmdln, misc, errors, fs_related +from mic.utils import misc, errors, fs_related from mic.imager import fs from mic.conf import configmgr from mic.plugin import pluginmgr @@ -29,12 +26,7 @@ class FsPlugin(ImagerPlugin): name = 'fs' @classmethod - @cmdln.option("--include-src", - dest="include_src", - action="store_true", - default=False, - help="Generate a image with source rpms included") - def do_create(self, subcmd, opts, *args): + def do_create(self, args): """${cmd_name}: create fs image Usage: @@ -43,25 +35,15 @@ class FsPlugin(ImagerPlugin): ${cmd_option_list} """ - if len(args) != 1: - raise errors.Usage("Extra arguments given") + if args is None: + raise errors.Usage("Invalid arguments.") creatoropts = configmgr.create - ksconf = args[0] + ksconf = args.ksfile if creatoropts['runtime'] == 'bootstrap': configmgr._ksconf = ksconf rt_util.bootstrap_mic() - elif not rt_util.inbootstrap(): - try: - fs_related.find_binary_path('mic-native') - except errors.CreatorError: - if not msger.ask("Subpackage \"mic-native\" has not been " - "installed in your host system, still " - "continue with \"native\" running mode?", - False): - raise errors.Abort("Abort because subpackage 'mic-native' " - "has not been installed") recording_pkgs = [] if len(creatoropts['record_pkgs']) > 0: @@ -96,7 +78,7 @@ class FsPlugin(ImagerPlugin): ','.join(backends.keys()))) creator = fs.FsImageCreator(creatoropts, pkgmgr) - creator._include_src = opts.include_src + creator._include_src = args.include_src if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs @@ -111,11 +93,12 @@ class FsPlugin(ImagerPlugin): creator.mount(None, creatoropts["cachedir"]) creator.install() #Download the source packages ###private options - if opts.include_src: + if args.include_src: installed_pkgs = creator.get_installed_packages() msger.info('--------------------------------------------------') msger.info('Generating the image with source rpms included ...') - if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], creator._instroot, creatoropts["cachedir"]): + if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], + creator._instroot, creatoropts["cachedir"]): msger.warning("Source packages can't be downloaded") creator.configure(creatoropts["repomd"]) diff --git a/plugins/imager/livecd_plugin.py b/plugins/imager/livecd_plugin.py deleted file mode 100644 index 3b45b5a..0000000 --- a/plugins/imager/livecd_plugin.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import tempfile - -from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors -from mic.conf import configmgr -import mic.imager.livecd as livecd -from mic.plugin import pluginmgr - -from mic.pluginbase import ImagerPlugin -class LiveCDPlugin(ImagerPlugin): - name = 'livecd' - - @classmethod - def do_create(self, subcmd, opts, *args): - """${cmd_name}: create livecd image - - Usage: - ${name} ${cmd_name} <ksfile> [OPTS] - - ${cmd_option_list} - """ - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - creatoropts = configmgr.create - ksconf = args[0] - - if creatoropts['runtime'] == 'bootstrap': - configmgr._ksconf = ksconf - rt_util.bootstrap_mic() - elif not rt_util.inbootstrap(): - try: - fs_related.find_binary_path('mic-native') - except errors.CreatorError: - if not msger.ask("Subpackage \"mic-native\" has not been " - "installed in your host system, still " - "continue with \"native\" running mode?", - False): - raise errors.Abort("Abort because subpackage 'mic-native' " - "has not been installed") - - if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): - msger.warning('livecd cannot support arm images, Quit') - return - - recording_pkgs = [] - if len(creatoropts['record_pkgs']) > 0: - recording_pkgs = creatoropts['record_pkgs'] - - if creatoropts['release'] is not None: - if 'name' not in recording_pkgs: - recording_pkgs.append('name') - if 'vcs' not in recording_pkgs: - recording_pkgs.append('vcs') - - configmgr._ksconf = ksconf - - # try to find the pkgmgr - pkgmgr = None - backends = pluginmgr.get_plugins('backend') - if 'auto' == creatoropts['pkgmgr']: - for key in configmgr.prefer_backends: - if key in backends: - pkgmgr = backends[key] - break - else: - for key in backends.keys(): - if key == creatoropts['pkgmgr']: - pkgmgr = backends[key] - break - - if not pkgmgr: - raise errors.CreatorError("Can't find backend: %s, " - "available choices: %s" % - (creatoropts['pkgmgr'], - ','.join(backends.keys()))) - - creator = livecd.LiveCDImageCreator(creatoropts, pkgmgr) - - if len(recording_pkgs) > 0: - creator._recording_pkgs = recording_pkgs - - self.check_image_exists(creator.destdir, - creator.pack_to, - [creator.name + ".iso"], - creatoropts['release']) - - try: - creator.check_depend_tools() - creator.mount(None, creatoropts["cachedir"]) - creator.install() - creator.configure(creatoropts["repomd"]) - creator.copy_kernel() - creator.unmount() - creator.package(creatoropts["destdir"]) - creator.create_manifest() - if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['destdir'], creatoropts['release']) - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - msger.info("Finished.") - return 0 - - @classmethod - def do_chroot(cls, target, cmd=[]): - os_image = cls.do_unpack(target) - os_image_dir = os.path.dirname(os_image) - - # unpack image to target dir - imgsize = misc.get_file_size(os_image) * 1024L * 1024L - imgtype = misc.get_image_type(os_image) - if imgtype == "btrfsimg": - fstype = "btrfs" - myDiskMount = fs_related.BtrfsDiskMount - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - myDiskMount = fs_related.ExtDiskMount - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - - extmnt = misc.mkdtemp() - extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), - extmnt, - fstype, - 4096, - "%s label" % fstype) - try: - extloop.mount() - - except errors.MountError: - extloop.cleanup() - shutil.rmtree(extmnt, ignore_errors = True) - shutil.rmtree(os_image_dir, ignore_errors = True) - raise - - try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) - else: - cmdline = "/bin/bash" - envcmd = fs_related.find_binary_inchroot("env", extmnt) - if envcmd: - cmdline = "%s HOME=/root %s" % (envcmd, cmdline) - chroot.chroot(extmnt, None, cmdline) - except: - raise errors.CreatorError("Failed to chroot to %s." %target) - finally: - chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) - - @classmethod - def do_pack(cls, base_on): - import subprocess - - def __mkinitrd(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) - - def __run_post_cleanups(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] - - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to run post cleanups: %s" % msg) - - convertoropts = configmgr.convert - convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] - convertor = livecd.LiveCDImageCreator(convertoropts) - imgtype = misc.get_image_type(base_on) - if imgtype == "btrfsimg": - fstype = "btrfs" - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - convertor._set_fstype(fstype) - try: - convertor.mount(base_on) - __mkinitrd(convertor) - convertor._create_bootconfig() - __run_post_cleanups(convertor) - convertor.launch_shell(convertoropts['shell']) - convertor.unmount() - convertor.package() - convertor.print_outimage_info() - finally: - shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) - - @classmethod - def do_unpack(cls, srcimg): - img = srcimg - imgmnt = misc.mkdtemp() - imgloop = fs_related.DiskMount(fs_related.LoopbackDisk(img, 0), imgmnt) - try: - imgloop.mount() - except errors.MountError: - imgloop.cleanup() - raise - - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(imgmnt + "/squashfs.img"): - squashimg = imgmnt + "/squashfs.img" - else: - squashimg = imgmnt + "/LiveOS/squashfs.img" - - tmpoutdir = misc.mkdtemp() - # unsquashfs requires outdir mustn't exist - shutil.rmtree(tmpoutdir, ignore_errors = True) - misc.uncompress_squashfs(squashimg, tmpoutdir) - - try: - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(tmpoutdir + "/os.img"): - os_image = tmpoutdir + "/os.img" - else: - os_image = tmpoutdir + "/LiveOS/ext3fs.img" - - if not os.path.exists(os_image): - raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " - "LiveOS/ext3fs.img nor os.img exist" %img) - - imgname = os.path.basename(srcimg) - imgname = os.path.splitext(imgname)[0] + ".img" - rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) - shutil.copyfile(os_image, rtimage) - - finally: - imgloop.cleanup() - shutil.rmtree(tmpoutdir, ignore_errors = True) - shutil.rmtree(imgmnt, ignore_errors = True) - - return rtimage diff --git a/plugins/imager/liveusb_plugin.py b/plugins/imager/liveusb_plugin.py deleted file mode 100644 index a2f300f..0000000 --- a/plugins/imager/liveusb_plugin.py +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import tempfile - -from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors -from mic.utils.partitionedfs import PartitionedMount -from mic.conf import configmgr -from mic.plugin import pluginmgr - -import mic.imager.liveusb as liveusb - -from mic.pluginbase import ImagerPlugin -class LiveUSBPlugin(ImagerPlugin): - name = 'liveusb' - - @classmethod - def do_create(self, subcmd, opts, *args): - """${cmd_name}: create liveusb image - - Usage: - ${name} ${cmd_name} <ksfile> [OPTS] - - ${cmd_option_list} - """ - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - creatoropts = configmgr.create - ksconf = args[0] - - if creatoropts['runtime'] == "bootstrap": - configmgr._ksconf = ksconf - rt_util.bootstrap_mic() - elif not rt_util.inbootstrap(): - try: - fs_related.find_binary_path('mic-native') - except errors.CreatorError: - if not msger.ask("Subpackage \"mic-native\" has not been " - "installed in your host system, still " - "continue with \"native\" running mode?", - False): - raise errors.Abort("Abort because subpackage 'mic-native' " - "has not been installed") - - if creatoropts['arch'] and creatoropts['arch'].startswith('arm'): - msger.warning('liveusb cannot support arm images, Quit') - return - - recording_pkgs = [] - if len(creatoropts['record_pkgs']) > 0: - recording_pkgs = creatoropts['record_pkgs'] - - if creatoropts['release'] is not None: - if 'name' not in recording_pkgs: - recording_pkgs.append('name') - if 'vcs' not in recording_pkgs: - recording_pkgs.append('vcs') - - configmgr._ksconf = ksconf - - # try to find the pkgmgr - pkgmgr = None - backends = pluginmgr.get_plugins('backend') - if 'auto' == creatoropts['pkgmgr']: - for key in configmgr.prefer_backends: - if key in backends: - pkgmgr = backends[key] - break - else: - for key in backends.keys(): - if key == creatoropts['pkgmgr']: - pkgmgr = backends[key] - break - - if not pkgmgr: - raise errors.CreatorError("Can't find backend: %s, " - "available choices: %s" % - (creatoropts['pkgmgr'], - ','.join(backends.keys()))) - - creator = liveusb.LiveUSBImageCreator(creatoropts, pkgmgr) - - if len(recording_pkgs) > 0: - creator._recording_pkgs = recording_pkgs - - self.check_image_exists(creator.destdir, - creator.pack_to, - [creator.name + ".usbimg"], - creatoropts['release']) - try: - creator.check_depend_tools() - creator.mount(None, creatoropts["cachedir"]) - creator.install() - creator.configure(creatoropts["repomd"]) - creator.copy_kernel() - creator.unmount() - creator.package(creatoropts["destdir"]) - creator.create_manifest() - if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['destdir'], creatoropts['release']) - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - msger.info("Finished.") - return 0 - - @classmethod - def do_chroot(cls, target, cmd=[]): - os_image = cls.do_unpack(target) - os_image_dir = os.path.dirname(os_image) - - # unpack image to target dir - imgsize = misc.get_file_size(os_image) * 1024L * 1024L - imgtype = misc.get_image_type(os_image) - if imgtype == "btrfsimg": - fstype = "btrfs" - myDiskMount = fs_related.BtrfsDiskMount - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - myDiskMount = fs_related.ExtDiskMount - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstype) - - extmnt = misc.mkdtemp() - extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize), - extmnt, - fstype, - 4096, - "%s label" % fstype) - - try: - extloop.mount() - - except errors.MountError: - extloop.cleanup() - shutil.rmtree(extmnt, ignore_errors = True) - raise - - try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) - else: - cmdline = "/bin/bash" - envcmd = fs_related.find_binary_inchroot("env", extmnt) - if envcmd: - cmdline = "%s HOME=/root %s" % (envcmd, cmdline) - chroot.chroot(extmnt, None, cmdline) - except: - raise errors.CreatorError("Failed to chroot to %s." %target) - finally: - chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt) - - @classmethod - def do_pack(cls, base_on): - import subprocess - - def __mkinitrd(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ] - try: - subprocess.call(args, preexec_fn = instance._chroot) - - except OSError, (err, msg): - raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg) - - def __run_post_cleanups(instance): - kernelver = instance._get_kernel_versions().values()[0][0] - args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver] - - try: - subprocess.call(args, preexec_fn = instance._chroot) - except OSError, (err, msg): - raise errors.CreatorError("Failed to run post cleanups: %s" % msg) - - convertoropts = configmgr.convert - convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0] - convertor = liveusb.LiveUSBImageCreator(convertoropts) - imgtype = misc.get_image_type(base_on) - if imgtype == "btrfsimg": - fstype = "btrfs" - elif imgtype in ("ext3fsimg", "ext4fsimg"): - fstype = imgtype[:4] - else: - raise errors.CreatorError("Unsupported filesystem type: %s" % fstyp) - convertor._set_fstype(fstype) - try: - convertor.mount(base_on) - __mkinitrd(convertor) - convertor._create_bootconfig() - __run_post_cleanups(convertor) - convertor.launch_shell(convertoropts['shell']) - convertor.unmount() - convertor.package() - convertor.print_outimage_info() - finally: - shutil.rmtree(os.path.dirname(base_on), ignore_errors = True) - - @classmethod - def do_unpack(cls, srcimg): - img = srcimg - imgsize = misc.get_file_size(img) * 1024L * 1024L - imgmnt = misc.mkdtemp() - disk = fs_related.SparseLoopbackDisk(img, imgsize) - imgloop = PartitionedMount(imgmnt, skipformat = True) - imgloop.add_disk('/dev/sdb', disk) - imgloop.add_partition(imgsize/1024/1024, "/dev/sdb", "/", "vfat", boot=False) - try: - imgloop.mount() - except errors.MountError: - imgloop.cleanup() - raise - - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(imgmnt + "/squashfs.img"): - squashimg = imgmnt + "/squashfs.img" - else: - squashimg = imgmnt + "/LiveOS/squashfs.img" - - tmpoutdir = misc.mkdtemp() - # unsquashfs requires outdir mustn't exist - shutil.rmtree(tmpoutdir, ignore_errors = True) - misc.uncompress_squashfs(squashimg, tmpoutdir) - - try: - # legacy LiveOS filesystem layout support, remove for F9 or F10 - if os.path.exists(tmpoutdir + "/os.img"): - os_image = tmpoutdir + "/os.img" - else: - os_image = tmpoutdir + "/LiveOS/ext3fs.img" - - if not os.path.exists(os_image): - raise errors.CreatorError("'%s' is not a valid live CD ISO : neither " - "LiveOS/ext3fs.img nor os.img exist" %img) - imgname = os.path.basename(srcimg) - imgname = os.path.splitext(imgname)[0] + ".img" - rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname) - shutil.copyfile(os_image, rtimage) - - finally: - imgloop.cleanup() - shutil.rmtree(tmpoutdir, ignore_errors = True) - shutil.rmtree(imgmnt, ignore_errors = True) - - return rtimage diff --git a/plugins/imager/loop_plugin.py b/plugins/imager/loop_plugin.py index e306ad1..1830230 100644 --- a/plugins/imager/loop_plugin.py +++ b/plugins/imager/loop_plugin.py @@ -20,7 +20,7 @@ import shutil import tempfile from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors, cmdln +from mic.utils import misc, fs_related, errors from mic.conf import configmgr from mic.plugin import pluginmgr from mic.imager.loop import LoopImageCreator, load_mountpoints @@ -30,16 +30,7 @@ class LoopPlugin(ImagerPlugin): name = 'loop' @classmethod - @cmdln.option("--compress-disk-image", dest="compress_image", - type='choice', choices=("gz", "bz2"), default=None, - help="Same with --compress-image") - # alias to compress-image for compatibility - @cmdln.option("--compress-image", dest="compress_image", - type='choice', choices=("gz", "bz2"), default=None, - help="Compress all loop images with 'gz' or 'bz2'") - @cmdln.option("--shrink", action='store_true', default=False, - help="Whether to shrink loop images to minimal size") - def do_create(self, subcmd, opts, *args): + def do_create(self, args): """${cmd_name}: create loop image Usage: @@ -48,25 +39,15 @@ class LoopPlugin(ImagerPlugin): ${cmd_option_list} """ - if len(args) != 1: - raise errors.Usage("Extra arguments given") + if args is None: + raise errors.Usage("Invalid arguments") creatoropts = configmgr.create - ksconf = args[0] + ksconf = args.ksfile if creatoropts['runtime'] == "bootstrap": configmgr._ksconf = ksconf rt_util.bootstrap_mic() - elif not rt_util.inbootstrap(): - try: - fs_related.find_binary_path('mic-native') - except errors.CreatorError: - if not msger.ask("Subpackage \"mic-native\" has not been " - "installed in your host system, still " - "continue with \"native\" running mode?", - False): - raise errors.Abort("Abort because subpackage 'mic-native' " - "has not been installed") recording_pkgs = [] if len(creatoropts['record_pkgs']) > 0: @@ -102,8 +83,8 @@ class LoopPlugin(ImagerPlugin): creator = LoopImageCreator(creatoropts, pkgmgr, - opts.compress_image, - opts.shrink) + args.compress_image, + args.shrink) if len(recording_pkgs) > 0: creator._recording_pkgs = recording_pkgs @@ -238,8 +219,8 @@ class LoopPlugin(ImagerPlugin): raise try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) + if cmd is not None: + cmdline = cmd else: cmdline = "/bin/bash" envcmd = fs_related.find_binary_inchroot("env", extmnt) diff --git a/plugins/imager/qcow_plugin.py b/plugins/imager/qcow_plugin.py new file mode 100644 index 0000000..8acf572 --- /dev/null +++ b/plugins/imager/qcow_plugin.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import os +import shutil + +from mic import msger, rt_util +from mic.conf import configmgr +from mic.plugin import pluginmgr +from mic.pluginbase import ImagerPlugin +from mic.imager.loop import LoopImageCreator +from mic.utils import errors, fs_related, runner + +class QcowImageCreator(LoopImageCreator): + img_format = 'qcow' + + def __init__(self, creatoropts=None, pkgmgr=None): + LoopImageCreator.__init__(self, creatoropts, pkgmgr) + self.cmd_qemuimg = 'qemu-img' + + def _stage_final_image(self): + try: + self.cmd_qemuimg = fs_related.find_binary_path('qemu-img') + except errors.CreatorError: + return LoopImageCreator._stage_final_image(self) + + self._resparse() + + imgfile = None + for item in self._instloops: + if item['mountpoint'] == '/': + if item['fstype'] == "ext4": + runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s' + % imgfile) + self.image_files.setdefault('partitions', {}).update( + {item['mountpoint']: item['label']}) + imgfile = os.path.join(self._imgdir, item['name']) + + if imgfile: + qemuimage = imgfile + ".x86" + runner.show("%s convert -O qcow2 %s %s" + % (self.cmd_qemuimg, imgfile, qemuimage)) + os.unlink(imgfile) + os.rename(qemuimage, imgfile) + + for item in os.listdir(self._imgdir): + shutil.move(os.path.join(self._imgdir, item), + os.path.join(self._outdir, item)) + +class QcowPlugin(ImagerPlugin): + name = 'qcow' + + @classmethod + def do_create(cls, args): + """${cmd_name}: create qcow image + + Usage: + ${name} ${cmd_name} <ksfile> [OPTS] + + ${cmd_option_list} + """ + if args is None: + raise errors.Usage("Invalid arguments") + + creatoropts = configmgr.create + ksconf = args.ksfile + + if creatoropts['runtime'] == "bootstrap": + configmgr._ksconf = ksconf + rt_util.bootstrap_mic() + + recording_pkgs = [] + if len(creatoropts['record_pkgs']) > 0: + recording_pkgs = creatoropts['record_pkgs'] + + if creatoropts['release'] is not None: + if 'name' not in recording_pkgs: + recording_pkgs.append('name') + if 'vcs' not in recording_pkgs: + recording_pkgs.append('vcs') + + configmgr._ksconf = ksconf + + # try to find the pkgmgr + pkgmgr = None + backends = pluginmgr.get_plugins('backend') + if 'auto' == creatoropts['pkgmgr']: + for key in configmgr.prefer_backends: + if key in backends: + pkgmgr = backends[key] + break + else: + for key in backends.keys(): + if key == creatoropts['pkgmgr']: + pkgmgr = backends[key] + break + + if not pkgmgr: + raise errors.CreatorError("Can't find backend: %s, " + "available choices: %s" % + (creatoropts['pkgmgr'], + ','.join(backends.keys()))) + + creator = QcowImageCreator(creatoropts, + pkgmgr) + + if len(recording_pkgs) > 0: + creator._recording_pkgs = recording_pkgs + + image_names = [creator.name + ".img"] + image_names.extend(creator.get_image_names()) + cls.check_image_exists(creator.destdir, + creator.pack_to, + image_names, + creatoropts['release']) + + try: + creator.check_depend_tools() + creator.mount(None, creatoropts["cachedir"]) + creator.install() + creator.configure(creatoropts["repomd"]) + creator.copy_kernel() + creator.unmount() + creator.package(creatoropts["destdir"]) + creator.create_manifest() + + if creatoropts['release'] is not None: + creator.release_output(ksconf, + creatoropts['destdir'], + creatoropts['release']) + creator.print_outimage_info() + + except errors.CreatorError: + raise + finally: + creator.cleanup() + + msger.info("Finished.") + return 0 + + @classmethod + def do_chroot(cls, target, cmd=[]): + pass + + @classmethod + def do_unpack(cls, srcimg): + pass diff --git a/plugins/imager/raw_plugin.py b/plugins/imager/raw_plugin.py deleted file mode 100644 index e994036..0000000 --- a/plugins/imager/raw_plugin.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/python -tt -# -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import shutil -import re -import tempfile - -from mic import chroot, msger, rt_util -from mic.utils import misc, fs_related, errors, runner, cmdln -from mic.conf import configmgr -from mic.plugin import pluginmgr -from mic.utils.partitionedfs import PartitionedMount - -import mic.imager.raw as raw - -from mic.pluginbase import ImagerPlugin -class RawPlugin(ImagerPlugin): - name = 'raw' - - @classmethod - @cmdln.option("--compress-disk-image", dest="compress_image", type='choice', - choices=("gz", "bz2"), default=None, - help="Same with --compress-image") - @cmdln.option("--compress-image", dest="compress_image", type='choice', - choices=("gz", "bz2"), default = None, - help="Compress all raw images before package") - @cmdln.option("--generate-bmap", action="store_true", default = None, - help="also generate the block map file") - @cmdln.option("--fstab-entry", dest="fstab_entry", type='choice', - choices=("name", "uuid"), default="uuid", - help="Set fstab entry, 'name' means using device names, " - "'uuid' means using filesystem uuid") - def do_create(self, subcmd, opts, *args): - """${cmd_name}: create raw image - - Usage: - ${name} ${cmd_name} <ksfile> [OPTS] - - ${cmd_option_list} - """ - - if len(args) != 1: - raise errors.Usage("Extra arguments given") - - creatoropts = configmgr.create - ksconf = args[0] - - if creatoropts['runtime'] == "bootstrap": - configmgr._ksconf = ksconf - rt_util.bootstrap_mic() - elif not rt_util.inbootstrap(): - try: - fs_related.find_binary_path('mic-native') - except errors.CreatorError: - if not msger.ask("Subpackage \"mic-native\" has not been " - "installed in your host system, still " - "continue with \"native\" running mode?", - False): - raise errors.Abort("Abort because subpackage 'mic-native' " - "has not been installed") - - recording_pkgs = [] - if len(creatoropts['record_pkgs']) > 0: - recording_pkgs = creatoropts['record_pkgs'] - - if creatoropts['release'] is not None: - if 'name' not in recording_pkgs: - recording_pkgs.append('name') - if 'vcs' not in recording_pkgs: - recording_pkgs.append('vcs') - - configmgr._ksconf = ksconf - - # try to find the pkgmgr - pkgmgr = None - backends = pluginmgr.get_plugins('backend') - if 'auto' == creatoropts['pkgmgr']: - for key in configmgr.prefer_backends: - if key in backends: - pkgmgr = backends[key] - break - else: - for key in backends.keys(): - if key == creatoropts['pkgmgr']: - pkgmgr = backends[key] - break - - if not pkgmgr: - raise errors.CreatorError("Can't find backend: %s, " - "available choices: %s" % - (creatoropts['pkgmgr'], - ','.join(backends.keys()))) - - creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image, - opts.generate_bmap, opts.fstab_entry) - - if len(recording_pkgs) > 0: - creator._recording_pkgs = recording_pkgs - - images = ["%s-%s.raw" % (creator.name, disk_name) - for disk_name in creator.get_disk_names()] - self.check_image_exists(creator.destdir, - creator.pack_to, - images, - creatoropts['release']) - - try: - creator.check_depend_tools() - creator.mount(None, creatoropts["cachedir"]) - creator.install() - creator.configure(creatoropts["repomd"]) - creator.copy_kernel() - creator.unmount() - creator.generate_bmap() - creator.package(creatoropts["destdir"]) - creator.create_manifest() - if creatoropts['release'] is not None: - creator.release_output(ksconf, creatoropts['destdir'], creatoropts['release']) - creator.print_outimage_info() - - except errors.CreatorError: - raise - finally: - creator.cleanup() - - msger.info("Finished.") - return 0 - - @classmethod - def do_chroot(cls, target, cmd=[]): - img = target - imgsize = misc.get_file_size(img) * 1024L * 1024L - partedcmd = fs_related.find_binary_path("parted") - disk = fs_related.SparseLoopbackDisk(img, imgsize) - imgmnt = misc.mkdtemp() - imgloop = PartitionedMount(imgmnt, skipformat = True) - imgloop.add_disk('/dev/sdb', disk) - img_fstype = "ext3" - - msger.info("Partition Table:") - partnum = [] - for line in runner.outs([partedcmd, "-s", img, "print"]).splitlines(): - # no use strip to keep line output here - if "Number" in line: - msger.raw(line) - if line.strip() and line.strip()[0].isdigit(): - partnum.append(line.strip()[0]) - msger.raw(line) - - rootpart = None - if len(partnum) > 1: - rootpart = msger.choice("please choose root partition", partnum) - - # Check the partitions from raw disk. - # if choose root part, the mark it as mounted - if rootpart: - root_mounted = True - else: - root_mounted = False - partition_mounts = 0 - for line in runner.outs([partedcmd,"-s",img,"unit","B","print"]).splitlines(): - line = line.strip() - - # Lines that start with number are the partitions, - # because parted can be translated we can't refer to any text lines. - if not line or not line[0].isdigit(): - continue - - # Some vars have extra , as list seperator. - line = line.replace(",","") - - # Example of parted output lines that are handled: - # Number Start End Size Type File system Flags - # 1 512B 3400000511B 3400000000B primary - # 2 3400531968B 3656384511B 255852544B primary linux-swap(v1) - # 3 3656384512B 3720347647B 63963136B primary fat16 boot, lba - - partition_info = re.split("\s+",line) - - size = partition_info[3].split("B")[0] - - if len(partition_info) < 6 or partition_info[5] in ["boot"]: - # No filesystem can be found from partition line. Assuming - # btrfs, because that is the only MeeGo fs that parted does - # not recognize properly. - # TODO: Can we make better assumption? - fstype = "btrfs" - elif partition_info[5] in ["ext2","ext3","ext4","btrfs"]: - fstype = partition_info[5] - elif partition_info[5] in ["fat16","fat32"]: - fstype = "vfat" - elif "swap" in partition_info[5]: - fstype = "swap" - else: - raise errors.CreatorError("Could not recognize partition fs type '%s'." % partition_info[5]) - - if rootpart and rootpart == line[0]: - mountpoint = '/' - elif not root_mounted and fstype in ["ext2","ext3","ext4","btrfs"]: - # TODO: Check that this is actually the valid root partition from /etc/fstab - mountpoint = "/" - root_mounted = True - elif fstype == "swap": - mountpoint = "swap" - else: - # TODO: Assing better mount points for the rest of the partitions. - partition_mounts += 1 - mountpoint = "/media/partition_%d" % partition_mounts - - if "boot" in partition_info: - boot = True - else: - boot = False - - msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" % (size, fstype, mountpoint, boot)) - # TODO: add_partition should take bytes as size parameter. - imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint, fstype = fstype, boot = boot) - - try: - imgloop.mount() - - except errors.MountError: - imgloop.cleanup() - raise - - try: - if len(cmd) != 0: - cmdline = ' '.join(cmd) - else: - cmdline = "/bin/bash" - envcmd = fs_related.find_binary_inchroot("env", imgmnt) - if envcmd: - cmdline = "%s HOME=/root %s" % (envcmd, cmdline) - chroot.chroot(imgmnt, None, cmdline) - except: - raise errors.CreatorError("Failed to chroot to %s." %img) - finally: - chroot.cleanup_after_chroot("img", imgloop, None, imgmnt) - - @classmethod - def do_unpack(cls, srcimg): - srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L - srcmnt = misc.mkdtemp("srcmnt") - disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize) - srcloop = PartitionedMount(srcmnt, skipformat = True) - - srcloop.add_disk('/dev/sdb', disk) - srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False) - try: - srcloop.mount() - - except errors.MountError: - srcloop.cleanup() - raise - - image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img") - args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image] - - msger.info("`dd` image ...") - rc = runner.show(args) - srcloop.cleanup() - shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True) - - if rc != 0: - raise errors.CreatorError("Failed to dd") - else: - return image @@ -1,30 +1,64 @@ #!/usr/bin/env python +# Copyright (c) 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" + Main for installing mic +""" + + import os, sys import glob from distutils.core import setup + MOD_NAME = 'mic' + +def check_debian(): + """--install-layout is recognized after 2.5""" + if sys.version_info[:2] > (2, 5): + if len(sys.argv) > 1 and 'install' in sys.argv: + try: + import platform + (dist, _, _) = platform.linux_distribution() + # for debian-like distros, mods will be installed to + # ${PYTHONLIB}/dist-packages + if dist in ('debian', 'Ubuntu'): + sys.argv.append('--install-layout=deb') + except AttributeError: + pass + + +def create_conf_file(): + """Apply prefix to mic.conf.in to generate actual mic.conf""" + with open('etc/mic.conf.in') as source_file: + conf_str = source_file.read() + conf_str = conf_str.replace('@PREFIX@', PREFIX) + with open(CONF_FILE, 'w') as conf_file: + conf_file.write(conf_str) + + try: import mic - version = mic.__version__ + VERSION = mic.__version__ except (ImportError, AttributeError): - version = "dev" - -# --install-layout is recognized after 2.5 -if sys.version_info[:2] > (2, 5): - if len(sys.argv) > 1 and 'install' in sys.argv: - try: - import platform - (dist, ver, id) = platform.linux_distribution() - - # for debian-like distros, mods will be installed to - # ${PYTHONLIB}/dist-packages - if dist in ('debian', 'Ubuntu'): - sys.argv.append('--install-layout=deb') - except: - pass + VERSION = "dev" + +check_debian() PACKAGES = [MOD_NAME, MOD_NAME + '/utils', @@ -40,28 +74,25 @@ PACKAGES = [MOD_NAME, IMAGER_PLUGINS = glob.glob(os.path.join("plugins", "imager", "*.py")) BACKEND_PLUGINS = glob.glob(os.path.join("plugins", "backend", "*.py")) -prefix = sys.prefix +PREFIX = sys.prefix # if real_prefix, it must be in virtualenv, use prefix as root -root = sys.prefix if hasattr(sys, 'real_prefix') else '' +ROOT = sys.prefix if hasattr(sys, 'real_prefix') else '' -conffile = 'etc/mic.conf' -# apply prefix to mic.conf.in to generate actual mic.conf -conf_str = file('etc/mic.conf.in').read() -conf_str = conf_str.replace('@PREFIX@', prefix) -with file(conffile, 'w') as wf: - wf.write(conf_str) +CONF_FILE = 'etc/mic.conf' +create_conf_file() setup(name=MOD_NAME, - version = version, + version = VERSION, description = 'Image Creator for Linux Distributions', author='Jian-feng Ding, Qiang Zhang, Gui Chen', - author_email='jian-feng.ding@intel.com, qiang.z.zhang@intel.com, gui.chen@intel.com', + author_email='jian-feng.ding@intel.com, qiang.z.zhang@intel.com,\ + gui.chen@intel.com', url='https://github.com/01org/mic', scripts=[ 'tools/mic', ], packages = PACKAGES, - data_files = [("%s/lib/mic/plugins/imager" % prefix, IMAGER_PLUGINS), - ("%s/lib/mic/plugins/backend" % prefix, BACKEND_PLUGINS), - ("%s/etc/mic" % root, [conffile])] + data_files = [("%s/lib/mic/plugins/imager" % PREFIX, IMAGER_PLUGINS), + ("%s/lib/mic/plugins/backend" % PREFIX, BACKEND_PLUGINS), + ("%s/etc/mic" % ROOT, [CONF_FILE])] ) diff --git a/tests/test_archive.py b/tests/test_archive.py new file mode 100644 index 0000000..609e7cc --- /dev/null +++ b/tests/test_archive.py @@ -0,0 +1,692 @@ +""" +It is used to test mic/archive.py +""" + + +import os +import shutil +import unittest + +from mic import archive + + +class ArchiveTest(unittest.TestCase): + """ + test pulic methods in archive.py + """ + def setUp(self): + """Create files and directories for later use""" + self.relative_file = './sdfb.gxdf.bzws.zzz' + abs_file = '/tmp/adsdfb.gxdf.bzws.zzz' + bare_file = 'abc.def.bz.zzz' + self.relative_dir = './sdf.zzz' + abs_dir = '/tmp/asdf.zzz' + bare_dir = 'abd.zzz' + self.wrong_format_file = './sdbs.werxdf.bz.zzz' + + self.files = [self.relative_file, abs_file, bare_file] + self.dirs = [self.relative_dir, abs_dir, bare_dir] + for file_item in self.files: + os.system('touch %s' % file_item) + + for dir_item in self.dirs: + self.create_dir(dir_item) + shutil.copy(self.relative_file, '%s/1.txt' % dir_item) + shutil.copy(self.relative_file, '%s/2.txt' % dir_item) + self.create_dir('%s/dir1' % dir_item) + self.create_dir('%s/dir2' % dir_item) + + def tearDown(self): + """Clean up unuseful file and directory """ + try: + for file_item in self.files: + os.remove(file_item) + for dir_item in self.dirs: + shutil.rmtree(dir_item, ignore_errors=True) + except OSError: + pass + + def create_dir(self, dir_name): + """Create directories and ignore any erros """ + try: + os.makedirs(dir_name) + except OSError: + pass + + def test_get_compress_formats(self): + """Test get compress format """ + compress_list = archive.get_compress_formats() + compress_list.sort() + self.assertEqual(compress_list, ['bz2', 'gz', 'lzo']) + + def test_compress_negtive_file_path_is_required(self): + """Test if the first parameter: file path is empty""" + self.assertRaises(OSError, archive.compress, '', 'bz2') + #with self.assertRaises(OSError): + # archive.compress('', 'bz2') + + def test_compress_negtive_compress_format_is_required(self): + """Test if the second parameter: compress format is empty""" + self.assertRaises(ValueError, archive.compress, + self.relative_file, '') + #with self.assertRaises(ValueError): + # archive.compress(self.relative_file, '') + + def test_compress_negtive_parameters_are_all_required(self): + """Test if two parameters are both empty""" + self.assertRaises(OSError, archive.compress, '', '') + #with self.assertRaises(OSError): + # archive.compress('', '') + + def test_compress_negtive_file_not_exist(self): + """Test target file does not exist""" + self.assertRaises(OSError, archive.compress, 'a.py', 'bz2') + #with self.assertRaises(OSError): + # archive.compress('a.py', 'bz2') + + def test_compress_negtive_file_is_dir(self): + """Test target is one direcoty, which is not supported""" + self.assertRaises(OSError, archive.compress, + self.relative_dir, 'bz2') + # with self.assertRaises(OSError): + # archive.compress(self.relative_dir, 'bz2') + + def test_compress_negtive_wrong_compress_format(self): + """Test wrong compress format""" + self.assertRaises(ValueError, archive.compress, + self.relative_file, 'bzip2') + #with self.assertRaises(ValueError): + # archive.compress(self.relative_file, 'bzip2') + + def _compress_negtive_gz_command_not_exists(self): + #TODO: test if command like 'pigz', 'gzip' does not exist + pass + + def _compress_negtive_lzo_command_not_exists(self): + #TODO: test if command 'lzop' does not exist + pass + + def _compress_negtive_bz2_command_not_exists(self): + #TODO: test if command like 'pbzip2', 'bzip2' does not exist + pass + + def test_compress_gz(self): + """Test compress format: gz""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def test_compress_bz2(self): + """Test compress format: bz2""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def _test_compress_lzo(self): + """Test compress format: lzo""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + os.remove(output_name) + + def test_decompress_negtive_file_path_is_required(self): + """Test if the first parameter: file to be uncompressed is empty""" + self.assertRaises(OSError, archive.compress, '', 'bz') + #with self.assertRaises(OSError): + # archive.decompress('', 'bz') + + def test_decompress_compress_format_is_empty(self): + """Test if the second parameter: compress format is empty string""" + output_name = archive.compress(self.relative_file, 'gz') + self.assertEqual('%s.gz' % self.relative_file, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(self.relative_file)) + archive.decompress(output_name, '') + self.assertTrue(os.path.exists(self.relative_file)) + + def test_decompress_negtive_parameters_are_empty(self): + """Test if two parameters are both empty string""" + self.assertRaises(OSError, archive.decompress, '', '') + #with self.assertRaises(OSError): + # archive.decompress('', '') + + def test_decompress_negtive_file_not_exist(self): + """Test decompress target does not exist""" + self.assertRaises(OSError, archive.decompress, + 'tresa.py', 'bz2') + #with self.assertRaises(OSError): + # archive.decompress('tresa.py', 'bz2') + + def test_decompress_negtive_path_is_dir(self): + """Test decompress target is a directory""" + self.assertRaises(OSError, archive.decompress, + self.relative_dir, 'bz2') + #with self.assertRaises(OSError): + # archive.decompress(self.relative_dir, 'bz2') + + def _decompress_negtive_not_corresponding(self): + # TODO: test if path is .lzo, but given format is bz2 + pass + + def test_decompress_negtive_wrong_compress_format(self): + """Test wrong decompress format""" + self.assertRaises(ValueError, archive.decompress, + self.relative_file, 'bzip2') + #with self.assertRaises(ValueError): + # archive.decompress(self.relative_file, 'bzip2') + + def test_decompress_negtive_wrong_file_format(self): + """Test wrong target format""" + self.assertRaises(Exception, archive.decompress, + self.wrong_format_file, 'bz2') + #with self.assertRaises(Exception): + # archive.decompress(self.wrong_format_file, 'bz2') + + def test_decompress_gz(self): + """Test decompress + Format: gz + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'gz') + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_gz_no_compress_format(self): + """Test decompress + Format: gz + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'gz') + self.assertEqual('%s.gz' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_bz2(self): + """Test decompress + Format: bz2 + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'bz2') + self.assertTrue(os.path.exists(file_item)) + + def test_decompress_bz2_no_compress_format(self): + """Test decompress + Format: bz2 + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'bz2') + self.assertEqual('%s.bz2' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def _test_decompress_lzo(self): + """Test decompress + Format: lzo + both two parameters are given, one is target file, + the other is corresponding compress format""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name, 'lzo') + self.assertTrue(os.path.exists(file_item)) + + def _test_decompress_lzo_no_compress_format(self): + """Test decompress + Format: lzo + one parameters is given, only target file""" + for file_item in self.files: + output_name = archive.compress(file_item, 'lzo') + self.assertEqual('%s.lzo' % file_item, output_name) + self.assertTrue(os.path.exists(output_name)) + self.assertFalse(os.path.exists(file_item)) + archive.decompress(output_name) + self.assertTrue(os.path.exists(file_item)) + + def test_get_archive_formats(self): + """Test get archive format""" + archive_formats = archive.get_archive_formats() + archive_formats.sort() + self.assertEqual(archive_formats, + ["bztar", "gztar", "lzotar", "tar", 'zip']) + + def test_get_archive_suffixes(self): + """Test get archive suffixes""" + archive_suffixes = archive.get_archive_suffixes() + archive_suffixes.sort() + + self.assertEqual(archive_suffixes, + ['.tar', '.tar.bz', '.tar.bz2', '.tar.gz', '.tar.lzo', + '.taz', '.tbz', '.tbz2', '.tgz', '.tzo', '.zip']) + + def test_make_archive_negtive_archive_name_is_required(self): + """Test if first parameter: file path is empty""" + self.assertRaises(Exception, archive.make_archive, + '', self.relative_dir) + #with self.assertRaises(Exception): + # archive.make_archive('', self.relative_dir) + + def test_extract_archive_negtive_archive_name_is_required(self): + """Test if first parameter: file path is empty""" + self.assertRaises(Exception, archive.extract_archive, + '', self.relative_dir) + #with self.assertRaises(Exception): + # archive.extract_archive('', self.relative_dir) + + def test_make_archive_negtive_target_name_is_required(self): + """Test if second parameter: target name is empty""" + self.assertRaises(Exception, archive.make_archive, 'a.zip', '') + #with self.assertRaises(Exception): + # archive.make_archive('a.zip', '') + + def _extract_archive_negtive_target_name_is_required(self): + # Not sure if the current dir will be used ? + # TODO: + pass + + def test_make_archive_negtive_parameters_are_empty(self): + """Test if both parameters are empty""" + self.assertRaises(Exception, archive.make_archive, '', '') + #with self.assertRaises(Exception): + # archive.make_archive('', '') + + def test_extract_archive_negtive_parameters_are_empty(self): + """Test if both parameters are empty""" + self.assertRaises(Exception, archive.extract_archive, '', '') + #with self.assertRaises(Exception): + # archive.extract_archive('', '') + + def test_make_archive_negtive_target_path_not_exists(self): + """Test if file path does not exist""" + fake_file = 'abcdfsdf' + self.assertRaises(Exception, archive.make_archive, + 'a.tar', fake_file) + #with self.assertRaises(Exception): + # archive.make_archive('a.tar', fake_file) + + self.assertRaises(Exception, archive.make_archive, + 'a.zip', fake_file) + #with self.assertRaises(Exception): + # archive.make_archive('a.zip', fake_file) + + def test_extract_archive_negtive_path_not_exists(self): + """Test if file path does not exist""" + fake_file = 'abcdfsdf' + self.assertRaises(Exception, archive.extract_archive, + fake_file, self.relative_dir) + #with self.assertRaises(Exception): + # archive.extract_archive(fake_file, self.relative_dir) + + def test_extract_archive_negtive_target_is_file(self): + """Test if the extract target is file""" + out_file = '%s.tar' % self.relative_dir + self.assertTrue(archive.make_archive(out_file, self.relative_dir)) + self.assertTrue(os.path.exists(out_file)) + self.assertRaises(Exception, archive.extract_archive, + out_file, self.relative_file) + #with self.assertRaises(Exception): + # archive.extract_archive(out_file, self.relative_file) + os.remove(out_file) + + def test_make_archive_wrong_format(self): + """Test wrong make_archive format""" + self.assertRaises(Exception, archive.make_archive, + 'a.sfsfrwe', self.relative_dir) + #with self.assertRaises(Exception): + # archive.make_archive('a.sfsfrwe', self.relative_dir) + + def test_make_archive_tar_with_different_name(self): + """ Test make_archive format: tar + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abcd.tar' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar(self): + """ Test make_archive format: tar""" + for item in self.files + self.dirs: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_extract_archive_tar(self): + """ Test extract format: tar""" + for item in self.files: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join( + out_dir, + os.path.basename(item)))) + shutil.rmtree(out_dir) + + for item in self.dirs: + out_file = '%s.tar' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, '1.txt'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, '2.txt'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, 'dir1'))) + self.assertTrue(os.path.exists(os.path.join(out_dir, 'dir2'))) + shutil.rmtree(out_dir) + + def test_make_archive_zip_with_different_name(self): + """ Test make_archive format: zip + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'a.zip' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_zip(self): + """ Test make_archive format: zip""" + for item in self.files + self.dirs: + out_file = '%s.zip' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_zip(self): + """ Test extract archive format: zip""" + for item in self.files + self.dirs: + out_file = '%s.zip' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def _test_make_archive_tzo_with_different_name(self): + """ Test make_archive format: tzo + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abc.tzo' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _test_make_archive_tzo(self): + """ Test make_archive format: tzo""" + for item in self.files + self.dirs: + out_file = '%s.tzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tzo(self): + """ Test extract format: tzo""" + for item in self.files + self.dirs: + out_file = '%s.tzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def _test_make_archive_tar_lzo_with_different_name(self): + """ Test make_archive format: lzo + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abc.tar.lzo' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _test_make_archive_tar_lzo(self): + """ Test make_archive format: lzo""" + for item in self.files + self.dirs: + out_file = '%s.tar.lzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_lzo(self): + """ Test extract_archive format: lzo""" + for item in self.files + self.dirs: + out_file = '%s.tar.lzo' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_taz_with_different_name(self): + """ Test make_archive format: taz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'abcd.taz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_taz(self): + """ Test make_archive format: taz""" + for item in self.files + self.dirs: + out_file = '%s.taz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_taz(self): + """ Test extract archive format: taz""" + for item in self.files + self.dirs: + out_file = '%s.taz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tgz_with_different_name(self): + """ Test make_archive format: tgz + It packs the source with anotehr name""" + for item in self.files + self.dirs: + out_file = 'abc.tgz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tgz(self): + """ Test make_archive format: tgz""" + for item in self.files + self.dirs: + out_file = '%s.tgz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tgz(self): + """ Test extract archive format: tgz""" + for item in self.files + self.dirs: + out_file = '%s.tgz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_gz_with_different_name(self): + """ Test make_archive format: tar.gz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'erwe.tar.gz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_gz(self): + """ Test make_archive format: tar.gz""" + for item in self.files + self.dirs: + out_file = '%s.tar.gz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_gz(self): + """ Test extract archive format: tar.gz""" + for item in self.files + self.dirs: + out_file = '%s.tar.gz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tbz_with_different_name(self): + """ Test make_archive format: tbz + It packs the source with another name""" + for item in self.files + self.dirs: + out_file = 'sdfsd.tbz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tbz(self): + """ Test make_archive format: tbz""" + for item in self.files + self.dirs: + out_file = '%s.tbz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tbz(self): + """ Test extract format: tbz""" + for item in self.files + self.dirs: + out_file = '%s.tbz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tbz2_with_different_name(self): + """ Test make_archive format: tbz2 + It packs source with another name""" + for item in self.files + self.dirs: + out_file = 'sfsfd.tbz2' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tbz2(self): + """ Test make_archive format: tbz2""" + for item in self.files + self.dirs: + out_file = '%s.tbz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tbz2(self): + """ Test extract format: tbz2""" + for item in self.files + self.dirs: + out_file = '%s.tbz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_bz_with_different_name(self): + """ Test make_archive format: tar.bz + It packs source with antoher name""" + for item in self.files + self.dirs: + out_file = 'sdf.tar.bz' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_bz(self): + """ Test make_archive format: tar.bz""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_bz(self): + """ Test extract format: tar.bz""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + + def test_make_archive_tar_bz2_with_different_name(self): + """ Test make_archive format: tar.bz2 + it packs the source with another name """ + for item in self.files + self.dirs: + out_file = 'df.tar.bz2' + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def test_make_archive_tar_bz2(self): + """ Test make_archive format: tar.bz2""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + os.remove(out_file) + + def _extract_archive_tar_bz2(self): + """ Test extract format: tar.bz2""" + for item in self.files + self.dirs: + out_file = '%s.tar.bz2' % item + self.assertTrue(archive.make_archive(out_file, item)) + self.assertTrue(os.path.exists(out_file)) + + out_dir = 'un_tar_dir' + archive.extract_archive(out_file, out_dir) + self.assertTrue(os.path.exists(os.path.join(out_dir, item))) + shutil.rmtree(out_dir) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_configmgr.py b/tests/test_configmgr.py index a795046..2f15f10 100644 --- a/tests/test_configmgr.py +++ b/tests/test_configmgr.py @@ -68,7 +68,8 @@ class ConfigMgrTest(unittest.TestCase): self.configmgr._ksconf = KSCONF self.assertTrue(isinstance(self.configmgr.create['ks'], KickstartParser)) #self.assertEqual(self.configmgr.create['name'], 'test') - self.assertDictEqual(repomd[0], self.configmgr.create['repomd'][0]) + #self.assertDictEqual(repomd[0], self.configmgr.create['repomd'][0]) + self.assertEqual(repomd[0], self.configmgr.create['repomd'][0]) self.assertEqual(self.configmgr.create['arch'], 'i686') if __name__ == "__main__": @@ -1,251 +1,273 @@ #!/usr/bin/env python + +#Copyright (c) 2011 Intel, Inc. # -# Copyright (c) 2011 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License +#This program is free software; you can redistribute it and/or modify it +#under the terms of the GNU General Public License as published by the Free +#Software Foundation; version 2 of the License # -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +#This program is distributed in the hope that it will be useful, but +#WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., 59 # Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os, sys, errno -from mic import msger, creator, __version__ as VERSION -from mic.utils import cmdln, misc, errors +# +# pylint: disable-msg=E0611, E1101, R0201 +# E0611: no name in module, some attributes are set during running, so ignore it +# E1101: %s %r has no %r member, some attributes are set during running, +# so ignore it +# R0201: Method could be a function + +""" + This mudule is entry for mic. + It defines a class named MicCmd inheriting Cmdln, and supplies interfaces like + 'create, chroot, convert' and also some parameters for command 'mic'. +""" +import os +import sys +import errno + +from argparse import ArgumentParser, SUPPRESS + +from mic import msger, __version__ as VERSION +from mic.utils import misc, errors from mic.conf import configmgr from mic.plugin import pluginmgr +from mic.helpformat import MICHelpFormatter, subparser + +@subparser +def chroot_parser(parser): + """chroot into an image -def optparser_setup(func): - """Setup optparser for a function""" - if not hasattr(func, "optparser"): - func.optparser = cmdln.SubCmdOptionParser() - func.optparser.disable_interspersed_args() - return func - - -class MicCmd(cmdln.Cmdln): + Examples: + mic chroot platform.img + mic chroot platform.img ls """ - Usage: mic SUBCOMMAND [OPTS] [ARGS...] - - mic Means the Image Creation tool - Try 'mic help SUBCOMMAND' for help on a specific subcommand. - - ${command_list} - global ${option_list} - ${help_list} + parser.add_argument('imagefile', help='Path of image file') + parser.add_argument('-s', '--saveto', action = 'store', dest = 'saveto', default = None, + help = "Save the unpacked image to specified dir") + parser.add_argument('-c', '--cmd', dest = 'cmd', default = None, + help = "command which will be executed in chroot environment") + parser.set_defaults(alias="ch") + return parser + +@subparser +def create_parser(parser): + """create an image + Examples: + $ mic -d -v create auto handset_blackbay.ks + $ mic -d -v cr loop handset_blackbay.ks --logfile=mic.log """ - name = 'mic' - version = VERSION - - def print_version(self): - msger.raw("%s %s (%s)" % (self.name, - self.version, + parent_parser = ArgumentParser(add_help=False) + parent_parser.add_argument('ksfile', help='Path of ksfile'); + parent_parser.add_argument('--logfile', dest='logfile', default=None, + help='Path of logfile') + parent_parser.add_argument('-c', '--config', dest='config', default=None, + help='Specify config file for mic') + parent_parser.add_argument('-k', '--cachedir', action='store', + dest='cachedir', default=None, + help='Cache directory to store the downloaded') + parent_parser.add_argument('-o', '--outdir', action='store', dest='outdir', + default=None, help='Output directory') + parent_parser.add_argument('-A', '--arch', dest='arch', default=None, + help='Specify repo architecture') + parent_parser.add_argument('--release', dest='release', default=None, metavar='RID', + help='Generate a release of RID with all necessary' + ' files, when @BUILD_ID@ is contained in ' + 'kickstart file, it will be replaced by RID') + parent_parser.add_argument("--record-pkgs", dest="record_pkgs", default=None, + help='Record the info of installed packages, ' + 'multiple values can be specified which ' + 'joined by ",", valid values: "name", ' + '"content", "license", "vcs"') + parent_parser.add_argument('--pkgmgr', dest='pkgmgr', default=None, + help='Specify backend package manager') + parent_parser.add_argument('--local-pkgs-path', dest='local_pkgs_path', default=None, + help='Path for local pkgs(rpms) to be installed') + parent_parser.add_argument('--runtime', dest='runtime', default=None, + help='Specify runtime mode, avaiable: bootstrap') + # --taring-to is alias to --pack-to + parent_parser.add_argument('--taring-to', dest='pack_to', default=None, + help=SUPPRESS) + parent_parser.add_argument('--pack-to', dest='pack_to', default=None, + help='Pack the images together into the specified' + ' achive, extension supported: .zip, .tar, ' + '.tar.gz, .tar.bz2, etc. by default, .tar ' + 'will be used') + parent_parser.add_argument('--copy-kernel', action='store_true', dest='copy_kernel', + help='Copy kernel files from image /boot directory' + ' to the image output directory.') + parent_parser.add_argument('--install-pkgs', action='store', dest='install_pkgs', default=None, + help='Specify what type of packages to be installed,' + ' valid: source, debuginfo, debugsource') + parent_parser.add_argument('--check-pkgs', action='store', dest='check_pkgs', default=[], + help='Check if given packages would be installed, ' + 'packages should be separated by comma') + parent_parser.add_argument('--tmpfs', action='store_true', dest='enabletmpfs', + help='Setup tmpdir as tmpfs to accelerate, experimental' + ' feature, use it if you have more than 4G memory') + parent_parser.add_argument('--repourl', action='append', dest='repourl', default=[], + help=SUPPRESS) + parent_parser.add_argument('-R', '--repo', action='append', + dest='repo', default=[], + help=SUPPRESS) + parent_parser.add_argument('--ignore-ksrepo', action='store_true', + dest='ignore_ksrepo', default=False, + help=SUPPRESS) + parent_parser.add_argument('--strict-mode', action='store_true', + dest='strict_mode', default=False, + help='Abort creation of image, if there are some errors' + ' during rpm installation. ') + parser.set_defaults(alias="cr") + + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + auto_parser = subparsers.add_parser('auto', parents=[parent_parser], help='auto detect image type from magic header') + + fs_parser = subparsers.add_parser('fs', parents=[parent_parser], + help='create fs image') + fs_parser.add_argument("--include-src", dest = "include_src",action = "store_true", + default = False, help = "Generate a image with source rpms included") + + loop_parser = subparsers.add_parser('loop', parents=[parent_parser], help='create loop image') + + loop_parser.add_argument("--compress-disk-image", dest="compress_image", + choices=("gz", "bz2"), default=None, + help="Same with --compress-image") + # alias to compress-image for compatibility + loop_parser.add_argument("--compress-image", dest="compress_image", + choices=("gz", "bz2"), default=None, + help="Compress all loop images with 'gz' or 'bz2'") + loop_parser.add_argument("--shrink", action='store_true', default=False, + help="Whether to shrink loop images to minimal size") + + qcow_parser = subparsers.add_parser('qcow', parents=[parent_parser], help='create qcow image') + + return parser + +def main(argv): + """Script entry point.""" + + def print_version(): + """log name, verion, hostname""" + + name = 'mic' + msger.raw("%s %s (%s)" % (name, + VERSION, misc.get_hostname_distro_str())) - - def get_optparser(self): - optparser = cmdln.CmdlnOptionParser(self, version=self.version) - # hook optparse print_version here - optparser.print_version = self.print_version - optparser.add_option('-d', '--debug', action='store_true', - dest='debug', - help='print debug message') - optparser.add_option('-v', '--verbose', action='store_true', - dest='verbose', - help='verbose information') - optparser.add_option('-i', '--interactive', action='store_true', - dest='interactive', default='True', - help='interactive output') - optparser.add_option('--non-interactive', action='store_false', - dest='interactive', default='True', - help='non-interactive output') - return optparser - - def postoptparse(self): - if self.options.interactive: - msger.enable_interactive() - else: - msger.disable_interactive() - - if self.options.verbose: - msger.set_loglevel('VERBOSE') - - if self.options.debug: - try: - import rpm - rpm.setVerbosity(rpm.RPMLOG_NOTICE) - except ImportError: - pass - - msger.set_loglevel('DEBUG') - - self.print_version() - - def help_create(self): - crobj = creator.Creator() - crobj.optparser = crobj.get_optparser() - doc = crobj.__doc__ - doc = crobj._help_reindent(doc) - doc = crobj._help_preprocess(doc, None) - doc = doc.replace(crobj.name, "${cmd_name}", 1) - doc = doc.rstrip() + '\n' - return doc - - @cmdln.alias("cr") - def do_create(self, argv): - crobj = creator.Creator() - crobj.main(argv[1:]) - - def _root_confirm(self): - if os.geteuid() != 0: - msger.error('Root permission is required to continue, abort') - - @cmdln.alias("cv") - @cmdln.option("-S", "--shell", - action="store_true", dest="shell", default=False, - help="Launch shell before packaging the converted image") - def do_convert(self, subcmd, opts, *args): - """${cmd_name}: convert image format - - Usage: - mic convert <imagefile> <destformat> - - ${cmd_option_list} + + def has_parameter(arg, arglist): """ - - if not args: - # print help - handler = self._get_cmd_handler('convert') - if hasattr(handler, "optparser"): - handler.optparser.print_help() - return 1 - - if len(args) == 1: - raise errors.Usage("It need 2 arguments (1 given)") - elif len(args) == 2: - (srcimg, destformat) = args - else: - raise errors.Usage("Extra argument given") - - if not os.path.exists(srcimg): - raise errors.CreatorError("Cannot find the image: %s" % srcimg) - - self._root_confirm() - - configmgr.convert['shell'] = opts.shell - - srcformat = misc.get_image_type(srcimg) - if srcformat == "ext3fsimg": - srcformat = "loop" - - srcimager = None - destimager = None - for iname, icls in pluginmgr.get_plugins('imager').iteritems(): - if iname == srcformat and hasattr(icls, "do_unpack"): - srcimager = icls - if iname == destformat and hasattr(icls, "do_pack"): - destimager = icls - - if (srcimager and destimager) is None: - raise errors.CreatorError("Can't convert from %s to %s" \ - % (srcformat, destformat)) - - else: - maptab = { - "livecd": "iso", - "liveusb": "usbimg", - "loop": "img", - } - - if destformat in maptab: - imgname = os.path.splitext(os.path.basename(srcimg))[0] - dstname = "{0}.{1}".format(imgname, maptab[destformat]) - - if os.path.exists(dstname): - if msger.ask("Converted image %s seems existed, " - "remove and continue?" % dstname): - os.unlink(dstname) - else: - raise errors.Abort("Canceled") - - base_on = srcimager.do_unpack(srcimg) - destimager.do_pack(base_on) - - @cmdln.alias("ch") - @cmdln.option('-s', '--saveto', - action='store', dest='saveto', default=None, - help="Save the unpacked image to specified dir") - @optparser_setup - def do_chroot(self, subcmd, opts, *args): - """${cmd_name}: chroot into an image - - Usage: - mic chroot [options] <imagefile> [command [arg]...] - - ${cmd_option_list} + Helper function. + Check if argument requires parameter by analyzing + its action. Parameter is required only for 'store' and 'append' actions """ + if arg.startswith('-'): + for args in arglist: + if arg in (args['short'], args['long']): + if args.get('action') in (None, 'store', 'append'): + return True + return False + + # Create top level parser + epilog = "Try 'mic SUBCOMMAND --help' for help on a specific subcommand." + description = "mic - the Image Creation tool" + parser = ArgumentParser(description=description, epilog=epilog, + formatter_class=MICHelpFormatter) + + # List of global arguments + # The main purpose of this structure is to contain arguments + # of add_argument. This is used to do aliasing properly + # (see code under the comment 'replace aliases with real commands') + global_args = [{'short': '-V', 'long': '--version', 'action': 'version', + 'version': '%(prog)s ' + VERSION}, + {'short': '-d', 'long': '--debug', 'action': 'store_true', + 'help': 'debug output'}, + {'short': '-v', 'long': '--verbose', 'action': 'store_true', + 'help': 'verbose output'}, + {'short': '-i', 'long': '--interactive', 'action': 'store_true', + 'dest': 'interactive', 'default': 'True', 'help': 'interactive output'}, + {'short': '', 'long': '--non-interactive', 'action': 'store_false', + 'dest': 'interactive', 'default': 'True', 'help': 'non-interactive output'}, + ] + + for args in global_args: + parser_kwargs = {} + for key in ('action', 'help', 'version', 'default', 'dest'): + if key in args: + parser_kwargs[key] = args[key] + + if args['short'] is '': + parser.add_argument(args['long'], **parser_kwargs) + else: + parser.add_argument(args['short'], args['long'], **parser_kwargs) + + # hacked by the request of cmdln lovers + parser.format_usage = parser.format_help + + # Create parsers for subcommands + subparsers = parser.add_subparsers(title='subcommands') + + # collect aliases + aliases = {} + for name, obj in globals().iteritems(): + if name.endswith('_parser') and callable(obj): + aliases[obj(subparsers).get_default('alias')] = name.split('_')[0] + + # replace aliases with real commands + for i, arg in enumerate(argv[1:]): + if not arg.startswith('-'): + # argv[i] is previous argument to arg + if not has_parameter(argv[i], global_args) and arg in aliases: + argv[i+1] = aliases[arg] + break - if not args: - # print help - handler = self._get_cmd_handler('chroot') - if hasattr(handler, "optparser"): - handler.optparser.print_help() - return 1 - - targetimage = args[0] - - if not os.path.exists(targetimage): - raise errors.CreatorError("Cannot find the image: %s" - % targetimage) + # Parse arguments + args = parser.parse_args(argv[1:]) - self._root_confirm() + if args.interactive: + msger.enable_interactive() + else: + msger.disable_interactive() - configmgr.chroot['saveto'] = opts.saveto + if args.verbose: + msger.set_loglevel('VERBOSE') - imagetype = misc.get_image_type(targetimage) - if imagetype in ("ext3fsimg", "ext4fsimg", "btrfsimg"): - imagetype = "loop" + if args.debug: + try: + import rpm + rpm.setVerbosity(rpm.RPMLOG_NOTICE) + except ImportError: + pass - chrootclass = None - for pname, pcls in pluginmgr.get_plugins('imager').iteritems(): - if pname == imagetype and hasattr(pcls, "do_chroot"): - chrootclass = pcls - break + msger.set_loglevel('DEBUG') - if not chrootclass: - raise errors.CreatorError("Cannot support image type: %s" \ - % imagetype) + print_version() - chrootclass.do_chroot(targetimage, args[1:]) + # Import target module and call 'main' from it + module = __import__("mic.%s" % args.module, fromlist=[args.module]) + return module.main(parser, args, argv[1:]) + if __name__ == "__main__": try: - MIC = MicCmd() - sys.exit(MIC.main()) - + sys.exit(main(sys.argv)) except KeyboardInterrupt: msger.error('\n^C catched, program aborted.') - - # catch 'no space left' exception, etc - except IOError, ioerr: + except IOError as ioerr: + # catch 'no space left' exception, etc if ioerr.errno == errno.ENOSPC: msger.error('\nNo space left on device') raise - - except errors.Usage, usage: + except errors.Usage as usage: msger.error(str(usage)) - - except errors.Abort, msg: + except errors.Abort as msg: msger.info(str(msg)) - - except errors.CreatorError, err: + except errors.CreatorError as err: if msger.get_loglevel() == 'DEBUG': import traceback msger.error(traceback.format_exc()) |