Remove BmapCreate and Filemap source code from MIC

Change-Id: I1bcd94008d785fb82cb807b8678ce960cb6f1145

3 files changed, 5 insertions, 882 deletions

diff --git a/mic/imager/raw.py b/mic/imager/raw.py
index 649292b..08f9683 100755
--- a/mic/imager/raw.py
+++ b/mic/imager/raw.py
@@ -578,7 +578,6 @@ class RawImageCreator(BaseImageCreator):
         if self.bmap_needed is None:
             return
 
-        from mic.utils import BmapCreate
         msger.info("Generating the map file(s)")
 
         for name in self.__disks.keys():
@@ -588,13 +587,11 @@ class RawImageCreator(BaseImageCreator):
                                             os.path.basename(bmap_file)})
 
             msger.debug("Generating block map file '%s'" % bmap_file)
-
-            try:
-                creator = BmapCreate.BmapCreate(image, bmap_file)
-                creator.generate()
-                del creator
-            except BmapCreate.Error as err:
-                raise CreatorError("Failed to create bmap file: %s" % str(err))
+            
+            bmaptoolcmd = misc.find_binary_path('bmaptool')
+            rc = runner.show([bmaptoolcmd, 'create', image, '-o', bmap_file])
+            if rc != 0:
+                raise CreatorError("Failed to create bmap file: %s" % bmap_file)
 
     def create_manifest(self):
         if self.compress_image:
diff --git a/mic/utils/BmapCreate.py b/mic/utils/BmapCreate.py
deleted file mode 100755
index 6934f1a..0000000
--- a/mic/utils/BmapCreate.py
+++ /dev/null
@@ -1,354 +0,0 @@
-# Copyright (c) 2012-2013 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License, version 2,
-# as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-
-"""
-This module implements the block map (bmap) creation functionality and provides
-the corresponding API in form of the 'BmapCreate' class.
-
-The idea is that while images files may generally be very large (e.g., 4GiB),
-they may nevertheless contain only little real data, e.g., 512MiB. This data
-are files, directories, file-system meta-data, partition table, etc. When
-copying the image to the target device, you do not have to copy all the 4GiB of
-data, you can copy only 512MiB of it, which is 4 times less, so copying should
-presumably be 4 times faster.
-
-The block map file is an XML file which contains a list of blocks which have to
-be copied to the target device. The other blocks are not used and there is no
-need to copy them. The XML file also contains some additional information like
-block size, image size, count of mapped blocks, etc. There are also many
-commentaries, so it is human-readable.
-
-The image has to be a sparse file. Generally, this means that when you generate
-this image file, you should start with a huge sparse file which contains a
-single hole spanning the entire file. Then you should partition it, write all
-the data (probably by means of loop-back mounting the image or parts of it),
-etc. The end result should be a sparse file where mapped areas represent useful
-parts of the image and holes represent useless parts of the image, which do not
-have to be copied when copying the image to the target device.
-
-This module uses the FIBMAP ioctl to detect holes.
-"""
-
-# Disable the following pylint recommendations:
-#   *  Too many instance attributes - R0902
-#   *  Too few public methods - R0903
-# pylint: disable=R0902,R0903
-
-import hashlib
-import logging
-from mic.utils.misc import human_size
-from mic.utils import Filemap
-
-# The bmap format version we generate.
-#
-# Changelog:
-# o 1.3 -> 2.0:
-#   Support SHA256 and SHA512 checksums, in 1.3 only SHA1 was supported.
-#   "BmapFileChecksum" is used instead of "BmapFileSHA1", and "chksum="
-#   attribute is used instead "sha1=". Introduced "ChecksumType" tag. This is
-#   an incompatible change.
-#   Note, bmap format 1.4 is identical to 2.0. Version 1.4 was a mistake,
-#   instead of incrementing the major version number, we incremented minor
-#   version number. Unfortunately, the mistake slipped into bmap-tools version
-#   3.0, and was only fixed in bmap-tools v3.1.
-SUPPORTED_BMAP_VERSION = "2.0"
-
-_BMAP_START_TEMPLATE = \
-"""<?xml version="1.0" ?>
-<!-- This file contains the block map for an image file, which is basically
-     a list of useful (mapped) block numbers in the image file. In other words,
-     it lists only those blocks which contain data (boot sector, partition
-     table, file-system metadata, files, directories, extents, etc). These
-     blocks have to be copied to the target device. The other blocks do not
-     contain any useful data and do not have to be copied to the target
-     device.
-
-     The block map an optimization which allows to copy or flash the image to
-     the image quicker than copying of flashing the entire image. This is
-     because with bmap less data is copied: <MappedBlocksCount> blocks instead
-     of <BlocksCount> blocks.
-
-     Besides the machine-readable data, this file contains useful commentaries
-     which contain human-readable information like image size, percentage of
-     mapped data, etc.
-
-     The 'version' attribute is the block map file format version in the
-     'major.minor' format. The version major number is increased whenever an
-     incompatible block map format change is made. The minor number changes
-     in case of minor backward-compatible changes. -->
-
-<bmap version="%s">
-    <!-- Image size in bytes: %s -->
-    <ImageSize> %u </ImageSize>
-
-    <!-- Size of a block in bytes -->
-    <BlockSize> %u </BlockSize>
-
-    <!-- Count of blocks in the image file -->
-    <BlocksCount> %u </BlocksCount>
-
-"""
-
-class Error(Exception):
-    """
-    A class for exceptions generated by this module. We currently support only
-    one type of exceptions, and we basically throw human-readable problem
-    description in case of errors.
-    """
-    pass
-
-class BmapCreate(object):
-    """
-    This class implements the bmap creation functionality. To generate a bmap
-    for an image (which is supposedly a sparse file), you should first create
-    an instance of 'BmapCreate' and provide:
-
-    * full path or a file-like object of the image to create bmap for
-    * full path or a file object to use for writing the results to
-
-    Then you should invoke the 'generate()' method of this class. It will use
-    the FIEMAP ioctl to generate the bmap.
-    """
-
-    def __init__(self, image, bmap, chksum_type="sha256", log=None):
-        """
-        Initialize a class instance:
-        * image  - full path or a file-like object of the image to create bmap
-                   for
-        * bmap   - full path or a file object to use for writing the resulting
-                   bmap to
-        * chksum - type of the check sum to use in the bmap file (all checksum
-                   types which python's "hashlib" module supports are allowed).
-        * log     - the logger object to use for printing messages.
-        """
-
-        self._log = log
-        if self._log is None:
-            self._log = logging.getLogger(__name__)
-
-        self.image_size = None
-        self.image_size_human = None
-        self.block_size = None
-        self.blocks_cnt = None
-        self.mapped_cnt = None
-        self.mapped_size = None
-        self.mapped_size_human = None
-        self.mapped_percent = None
-
-        self._mapped_count_pos1 = None
-        self._mapped_count_pos2 = None
-        self._chksum_pos = None
-
-        self._f_image_needs_close = False
-        self._f_bmap_needs_close = False
-
-        self._cs_type = chksum_type.lower()
-        try:
-            self._cs_len = len(hashlib.new(self._cs_type).hexdigest())
-        except ValueError as err:
-            raise Error("cannot initialize hash function \"%s\": %s" %
-                        (self._cs_type, err))
-
-        if hasattr(image, "read"):
-            self._f_image = image
-            self._image_path = image.name
-        else:
-            self._image_path = image
-            self._open_image_file()
-
-        if hasattr(bmap, "read"):
-            self._f_bmap = bmap
-            self._bmap_path = bmap.name
-        else:
-            self._bmap_path = bmap
-            self._open_bmap_file()
-
-        try:
-            self.filemap = Filemap.filemap(self._f_image, self._log)
-        except (Filemap.Error, Filemap.ErrorNotSupp) as err:
-            raise Error("cannot generate bmap: %s" % err)
-
-        self.image_size = self.filemap.image_size
-        self.image_size_human = human_size(self.image_size)
-        if self.image_size == 0:
-            raise Error("cannot generate bmap for zero-sized image file '%s'"
-                        % self._image_path)
-
-        self.block_size = self.filemap.block_size
-        self.blocks_cnt = self.filemap.blocks_cnt
-
-    def __del__(self):
-        """The class destructor which closes the opened files."""
-        if self._f_image_needs_close:
-            self._f_image.close()
-        if self._f_bmap_needs_close:
-            self._f_bmap.close()
-
-    def _open_image_file(self):
-        """Open the image file."""
-        try:
-            self._f_image = open(self._image_path, 'rb')
-        except IOError as err:
-            raise Error("cannot open image file '%s': %s"
-                        % (self._image_path, err))
-
-        self._f_image_needs_close = True
-
-    def _open_bmap_file(self):
-        """Open the bmap file."""
-        try:
-            self._f_bmap = open(self._bmap_path, 'w+')
-        except IOError as err:
-            raise Error("cannot open bmap file '%s': %s"
-                        % (self._bmap_path, err))
-
-        self._f_bmap_needs_close = True
-
-    def _bmap_file_start(self):
-        """
-        A helper function which generates the starting contents of the block
-        map file: the header comment, image size, block size, etc.
-        """
-
-        # We do not know the amount of mapped blocks at the moment, so just put
-        # whitespaces instead of real numbers. Assume the longest possible
-        # numbers.
-
-        xml = _BMAP_START_TEMPLATE \
-               % (SUPPORTED_BMAP_VERSION, self.image_size_human,
-                  self.image_size, self.block_size, self.blocks_cnt)
-        xml += "    <!-- Count of mapped blocks: "
-
-        self._f_bmap.write(xml)
-        self._mapped_count_pos1 = self._f_bmap.tell()
-
-        xml  = "%s or %s   -->\n" % (' ' * len(self.image_size_human),
-                                   ' ' * len("100.0%"))
-        xml += "    <MappedBlocksCount> "
-
-        self._f_bmap.write(xml)
-        self._mapped_count_pos2 = self._f_bmap.tell()
-
-        xml  = "%s </MappedBlocksCount>\n\n" % (' ' * len(str(self.blocks_cnt)))
-
-        # pylint: disable=C0301
-        xml += "    <!-- Type of checksum used in this file -->\n"
-        xml += "    <ChecksumType> %s </ChecksumType>\n\n" % self._cs_type
-
-        xml += "    <!-- The checksum of this bmap file. When it is calculated, the value of\n"
-        xml += "         the checksum has be zero (all ASCII \"0\" symbols).  -->\n"
-        xml += "    <BmapFileChecksum> "
-
-        self._f_bmap.write(xml)
-        self._chksum_pos = self._f_bmap.tell()
-
-        xml = "0" * self._cs_len + " </BmapFileChecksum>\n\n"
-        xml += "    <!-- The block map which consists of elements which may either be a\n"
-        xml += "         range of blocks or a single block. The 'chksum' attribute\n"
-        xml += "         (if present) is the checksum of this blocks range. -->\n"
-        xml += "    <BlockMap>\n"
-        # pylint: enable=C0301
-
-        self._f_bmap.write(xml)
-
-    def _bmap_file_end(self):
-        """
-        A helper function which generates the final parts of the block map
-        file: the ending tags and the information about the amount of mapped
-        blocks.
-        """
-
-        xml =  "    </BlockMap>\n"
-        xml += "</bmap>\n"
-
-        self._f_bmap.write(xml)
-
-        self._f_bmap.seek(self._mapped_count_pos1)
-        self._f_bmap.write("%s or %.1f%%"
-                           % (self.mapped_size_human, self.mapped_percent))
-
-        self._f_bmap.seek(self._mapped_count_pos2)
-        self._f_bmap.write("%u" % self.mapped_cnt)
-
-        self._f_bmap.seek(0)
-        hash_obj = hashlib.new(self._cs_type)
-        hash_obj.update(self._f_bmap.read())
-        chksum = hash_obj.hexdigest()
-        self._f_bmap.seek(self._chksum_pos)
-        self._f_bmap.write("%s" % chksum)
-
-    def _calculate_chksum(self, first, last):
-        """
-        A helper function which calculates checksum for the range of blocks of
-        the image file: from block 'first' to block 'last'.
-        """
-
-        start = first * self.block_size
-        end = (last + 1) * self.block_size
-
-        self._f_image.seek(start)
-        hash_obj = hashlib.new(self._cs_type)
-
-        chunk_size = 1024*1024
-        to_read = end - start
-        read = 0
-
-        while read < to_read:
-            if read + chunk_size > to_read:
-                chunk_size = to_read - read
-            chunk = self._f_image.read(chunk_size)
-            hash_obj.update(chunk)
-            read += chunk_size
-
-        return hash_obj.hexdigest()
-
-    def generate(self, include_checksums=True):
-        """
-        Generate bmap for the image file. If 'include_checksums' is 'True',
-        also generate checksums for block ranges.
-        """
-
-        # Save image file position in order to restore it at the end
-        image_pos = self._f_image.tell()
-
-        self._bmap_file_start()
-
-        # Generate the block map and write it to the XML block map
-        # file as we go.
-        self.mapped_cnt = 0
-        for first, last in self.filemap.get_mapped_ranges(0, self.blocks_cnt):
-            self.mapped_cnt += last - first + 1
-            if include_checksums:
-                chksum = self._calculate_chksum(first, last)
-                chksum = " chksum=\"%s\"" % chksum
-            else:
-                chksum = ""
-
-            if first != last:
-                self._f_bmap.write("        <Range%s> %s-%s </Range>\n"
-                                   % (chksum, first, last))
-            else:
-                self._f_bmap.write("        <Range%s> %s </Range>\n"
-                                   % (chksum, first))
-
-        self.mapped_size = self.mapped_cnt * self.block_size
-        self.mapped_size_human = human_size(self.mapped_size)
-        self.mapped_percent = (self.mapped_cnt * 100.0) /  self.blocks_cnt
-
-        self._bmap_file_end()
-
-        try:
-            self._f_bmap.flush()
-        except IOError as err:
-            raise Error("cannot flush the bmap file '%s': %s"
-                        % (self._bmap_path, err))
-
-        self._f_image.seek(image_pos)
diff --git a/mic/utils/Filemap.py b/mic/utils/Filemap.py
deleted file mode 100755
index 81d16c1..0000000
--- a/mic/utils/Filemap.py
+++ /dev/null
@@ -1,520 +0,0 @@
-# Copyright (c) 2012 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License, version 2,
-# as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-
-"""
-This module implements python implements a way to get file block. Two methods
-are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of
-the file seek syscall. The former is implemented by the 'FilemapFiemap' class,
-the latter is implemented by the 'FilemapSeek' class. Both classes provide the
-same API. The 'filemap' function automatically selects which class can be used
-and returns an instance of the class.
-"""
-
-# Disable the following pylint recommendations:
-#   * Too many instance attributes (R0902)
-# pylint: disable=R0902
-
-import os
-import struct
-import array
-import fcntl
-import tempfile
-import logging
-from mic.utils.misc import get_block_size
-
-
-class ErrorNotSupp(Exception):
-    """
-    An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature
-    is not supported either by the kernel or the file-system.
-    """
-    pass
-
-class Error(Exception):
-    """A class for all the other exceptions raised by this module."""
-    pass
-
-
-class _FilemapBase(object):
-    """
-    This is a base class for a couple of other classes in this module. This
-    class simply performs the common parts of the initialization process: opens
-    the image file, gets its size, etc. The 'log' parameter is the logger object
-    to use for printing messages.
-    """
-
-    def __init__(self, image, log=None):
-        """
-        Initialize a class instance. The 'image' argument is full path to the
-        file or file object to operate on.
-        """
-
-        self._log = log
-        if self._log is None:
-            self._log = logging.getLogger(__name__)
-
-        self._f_image_needs_close = False
-
-        if hasattr(image, "fileno"):
-            self._f_image = image
-            self._image_path = image.name
-        else:
-            self._image_path = image
-            self._open_image_file()
-
-        try:
-            self.image_size = os.fstat(self._f_image.fileno()).st_size
-        except IOError as err:
-            raise Error("cannot get information about file '%s': %s"
-                        % (self._f_image.name, err))
-
-        try:
-            self.block_size = get_block_size(self._f_image)
-        except IOError as err:
-            raise Error("cannot get block size for '%s': %s"
-                        % (self._image_path, err))
-
-        self.blocks_cnt = self.image_size + self.block_size - 1
-        self.blocks_cnt /= self.block_size
-
-        try:
-            self._f_image.flush()
-        except IOError as err:
-            raise Error("cannot flush image file '%s': %s"
-                        % (self._image_path, err))
-
-        try:
-            os.fsync(self._f_image.fileno()),
-        except OSError as err:
-            raise Error("cannot synchronize image file '%s': %s "
-                        % (self._image_path, err.strerror))
-
-        self._log.debug("opened image \"%s\"" % self._image_path)
-        self._log.debug("block size %d, blocks count %d, image size %d"
-                        % (self.block_size, self.blocks_cnt, self.image_size))
-
-    def __del__(self):
-        """The class destructor which just closes the image file."""
-        if self._f_image_needs_close:
-            self._f_image.close()
-
-    def _open_image_file(self):
-        """Open the image file."""
-        try:
-            self._f_image = open(self._image_path, 'rb')
-        except IOError as err:
-            raise Error("cannot open image file '%s': %s"
-                        % (self._image_path, err))
-
-        self._f_image_needs_close = True
-
-    def block_is_mapped(self, block): # pylint: disable=W0613,R0201
-        """
-        This method has has to be implemented by child classes. It returns
-        'True' if block number 'block' of the image file is mapped and 'False'
-        otherwise.
-        """
-
-        raise Error("the method is not implemented")
-
-    def block_is_unmapped(self, block): # pylint: disable=W0613,R0201
-        """
-        This method has has to be implemented by child classes. It returns
-        'True' if block number 'block' of the image file is not mapped (hole)
-        and 'False' otherwise.
-        """
-
-        raise Error("the method is not implemented")
-
-    def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
-        """
-        This method has has to be implemented by child classes. This is a
-        generator which yields ranges of mapped blocks in the file. The ranges
-        are tuples of 2 elements: [first, last], where 'first' is the first
-        mapped block and 'last' is the last mapped block.
-
-        The ranges are yielded for the area of the file of size 'count' blocks,
-        starting from block 'start'.
-        """
-
-        raise Error("the method is not implemented")
-
-    def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201
-        """
-        This method has has to be implemented by child classes. Just like
-        'get_mapped_ranges()', but yields unmapped block ranges instead
-        (holes).
-        """
-
-        raise Error("the method is not implemented")
-
-
-# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
-_SEEK_DATA = 3
-_SEEK_HOLE = 4
-
-def _lseek(file_obj, offset, whence):
-    """This is a helper function which invokes 'os.lseek' for file object
-    'file_obj' and with specified 'offset' and 'whence'. The 'whence'
-    argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
-    there is no more data or hole starting from 'offset', this function
-    returns '-1'.  Otherwise the data or hole position is returned."""
-
-    try:
-        return os.lseek(file_obj.fileno(), offset, whence)
-    except OSError as err:
-        # The 'lseek' system call returns the ENXIO if there is no data or
-        # hole starting from the specified offset.
-        if err.errno == os.errno.ENXIO:
-            return -1
-        elif err.errno == os.errno.EINVAL:
-            raise ErrorNotSupp("the kernel or file-system does not support "
-                               "\"SEEK_HOLE\" and \"SEEK_DATA\"")
-        else:
-            raise
-
-class FilemapSeek(_FilemapBase):
-    """
-    This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
-    Unfortunately, the current implementation requires the caller to have write
-    access to the image file.
-    """
-
-    def __init__(self, image, log=None):
-        """Refer the '_FilemapBase' class for the documentation."""
-
-        # Call the base class constructor first
-        _FilemapBase.__init__(self, image, log)
-        self._log.debug("FilemapSeek: initializing")
-
-        self._probe_seek_hole()
-
-    def _probe_seek_hole(self):
-        """
-        Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
-        Unfortunately, there seems to be no clean way for detecting this,
-        because often the system just fakes them by just assuming that all
-        files are fully mapped, so 'SEEK_HOLE' always returns EOF and
-        'SEEK_DATA' always returns the requested offset.
-
-        I could not invent a better way of detecting the fake 'SEEK_HOLE'
-        implementation than just to create a temporary file in the same
-        directory where the image file resides. It would be nice to change this
-        to something better.
-        """
-
-        directory = os.path.dirname(self._image_path)
-
-        try:
-            tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
-        except IOError as err:
-            raise ErrorNotSupp("cannot create a temporary in \"%s\": %s"
-                              % (directory, err))
-
-        try:
-            os.ftruncate(tmp_obj.fileno(), self.block_size)
-        except OSError as err:
-            raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
-                               % (directory, err))
-
-        offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
-        if offs != 0:
-            # We are dealing with the stub 'SEEK_HOLE' implementation which
-            # always returns EOF.
-            self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
-            raise ErrorNotSupp("the file-system does not support "
-                               "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
-                               "provides a stub implementation")
-
-        tmp_obj.close()
-
-    def block_is_mapped(self, block):
-        """Refer the '_FilemapBase' class for the documentation."""
-        offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
-        if offs == -1:
-            result = False
-        else:
-            result = (offs / self.block_size == block)
-
-        self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
-                        % (block, result))
-        return result
-
-    def block_is_unmapped(self, block):
-        """Refer the '_FilemapBase' class for the documentation."""
-        return not self.block_is_mapped(block)
-
-    def _get_ranges(self, start, count, whence1, whence2):
-        """
-        This function implements 'get_mapped_ranges()' and
-        'get_unmapped_ranges()' depending on what is passed in the 'whence1'
-        and 'whence2' arguments.
-        """
-
-        assert whence1 != whence2
-        end = start * self.block_size
-        limit = end + count * self.block_size
-
-        while True:
-            start = _lseek(self._f_image, end, whence1)
-            if start == -1 or start >= limit or start == self.image_size:
-                break
-
-            end = _lseek(self._f_image, start, whence2)
-            if end == -1 or end == self.image_size:
-                end = self.blocks_cnt * self.block_size
-            if end > limit:
-                end = limit
-
-            start_blk = start / self.block_size
-            end_blk = end / self.block_size - 1
-            self._log.debug("FilemapSeek: yielding range (%d, %d)"
-                            % (start_blk, end_blk))
-            yield (start_blk, end_blk)
-
-    def get_mapped_ranges(self, start, count):
-        """Refer the '_FilemapBase' class for the documentation."""
-        self._log.debug("FilemapSeek: get_mapped_ranges(%d,  %d(%d))"
-                        % (start, count, start + count - 1))
-        return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
-
-    def get_unmapped_ranges(self, start, count):
-        """Refer the '_FilemapBase' class for the documentation."""
-        self._log.debug("FilemapSeek: get_unmapped_ranges(%d,  %d(%d))"
-                        % (start, count, start + count - 1))
-        return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA)
-
-
-# Below goes the FIEMAP ioctl implementation, which is not very readable
-# because it deals with the rather complex FIEMAP ioctl. To understand the
-# code, you need to know the FIEMAP interface, which is documented in the
-# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
-
-# Format string for 'struct fiemap'
-_FIEMAP_FORMAT = "=QQLLLL"
-# sizeof(struct fiemap)
-_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
-# Format string for 'struct fiemap_extent'
-_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
-# sizeof(struct fiemap_extent)
-_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
-# The FIEMAP ioctl number
-_FIEMAP_IOCTL = 0xC020660B
-# This FIEMAP ioctl flag which instructs the kernel to sync the file before
-# reading the block map
-_FIEMAP_FLAG_SYNC = 0x00000001
-# Size of the buffer for 'struct fiemap_extent' elements which will be used
-# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
-# FIEMAP ioctl will be invoked.
-_FIEMAP_BUFFER_SIZE = 256 * 1024
-
-class FilemapFiemap(_FilemapBase):
-    """
-    This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
-    over all mapped blocks and over all holes.
-
-    This class synchronizes the image file every time it invokes the FIEMAP
-    ioctl in order to work-around early FIEMAP implementation kernel bugs.
-    """
-
-    def __init__(self, image, log=None):
-        """
-        Initialize a class instance. The 'image' argument is full the file
-        object to operate on.
-        """
-
-        # Call the base class constructor first
-        _FilemapBase.__init__(self, image, log)
-        self._log.debug("FilemapFiemap: initializing")
-
-        self._buf_size = _FIEMAP_BUFFER_SIZE
-
-        # Calculate how many 'struct fiemap_extent' elements fit the buffer
-        self._buf_size -= _FIEMAP_SIZE
-        self._fiemap_extent_cnt = self._buf_size / _FIEMAP_EXTENT_SIZE
-        assert self._fiemap_extent_cnt > 0
-        self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
-        self._buf_size += _FIEMAP_SIZE
-
-        # Allocate a mutable buffer for the FIEMAP ioctl
-        self._buf = array.array('B', [0] * self._buf_size)
-
-        # Check if the FIEMAP ioctl is supported
-        self.block_is_mapped(0)
-
-    def _invoke_fiemap(self, block, count):
-        """
-        Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
-        block number 'block'.
-
-        The full result of the operation is stored in 'self._buf' on exit.
-        Returns the unpacked 'struct fiemap' data structure in form of a python
-        list (just like 'struct.upack()').
-        """
-
-        if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt):
-            raise Error("bad block number %d, should be within [0, %d]"
-                        % (block, self.blocks_cnt))
-
-        # Initialize the 'struct fiemap' part of the buffer. We use the
-        # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is
-        # synchronized. The reason for this is that early FIEMAP
-        # implementations had many bugs related to cached dirty data, and
-        # synchronizing the file is a necessary work-around.
-        struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
-                         count * self.block_size, _FIEMAP_FLAG_SYNC, 0,
-                         self._fiemap_extent_cnt, 0)
-
-        try:
-            fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
-        except IOError as err:
-            # Note, the FIEMAP ioctl is supported by the Linux kernel starting
-            # from version 2.6.28 (year 2008).
-            if err.errno == os.errno.EOPNOTSUPP:
-                errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
-                         "by the file-system"
-                self._log.debug(errstr)
-                raise ErrorNotSupp(errstr)
-            if err.errno == os.errno.ENOTTY:
-                errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
-                         "by the kernel"
-                self._log.debug(errstr)
-                raise ErrorNotSupp(errstr)
-            raise Error("the FIEMAP ioctl failed for '%s': %s"
-                        % (self._image_path, err))
-
-        return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
-
-    def block_is_mapped(self, block):
-        """Refer the '_FilemapBase' class for the documentation."""
-        struct_fiemap = self._invoke_fiemap(block, 1)
-
-        # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
-        # If it contains zero, the block is not mapped, otherwise it is
-        # mapped.
-        result = bool(struct_fiemap[3])
-        self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s"
-                        % (block, result))
-        return result
-
-    def block_is_unmapped(self, block):
-        """Refer the '_FilemapBase' class for the documentation."""
-        return not self.block_is_mapped(block)
-
-    def _unpack_fiemap_extent(self, index):
-        """
-        Unpack a 'struct fiemap_extent' structure object number 'index' from
-        the internal 'self._buf' buffer.
-        """
-
-        offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
-        return struct.unpack(_FIEMAP_EXTENT_FORMAT,
-                             self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
-
-    def _do_get_mapped_ranges(self, start, count):
-        """
-        Implements most the functionality for the  'get_mapped_ranges()'
-        generator: invokes the FIEMAP ioctl, walks through the mapped extents
-        and yields mapped block ranges. However, the ranges may be consecutive
-        (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
-        them.
-        """
-
-        block = start
-        while block < start + count:
-            struct_fiemap = self._invoke_fiemap(block, count)
-
-            mapped_extents = struct_fiemap[3]
-            if mapped_extents == 0:
-                # No more mapped blocks
-                return
-
-            extent = 0
-            while extent < mapped_extents:
-                fiemap_extent = self._unpack_fiemap_extent(extent)
-
-                # Start of the extent
-                extent_start = fiemap_extent[0]
-                # Starting block number of the extent
-                extent_block = extent_start / self.block_size
-                # Length of the extent
-                extent_len = fiemap_extent[2]
-                # Count of blocks in the extent
-                extent_count = extent_len / self.block_size
-
-                # Extent length and offset have to be block-aligned
-                assert extent_start % self.block_size == 0
-                assert extent_len % self.block_size == 0
-
-                if extent_block > start + count - 1:
-                    return
-
-                first = max(extent_block, block)
-                last = min(extent_block + extent_count, start + count) - 1
-                yield (first, last)
-
-                extent += 1
-
-            block = extent_block + extent_count
-
-    def get_mapped_ranges(self, start, count):
-        """Refer the '_FilemapBase' class for the documentation."""
-        self._log.debug("FilemapFiemap: get_mapped_ranges(%d,  %d(%d))"
-                        % (start, count, start + count - 1))
-        iterator = self._do_get_mapped_ranges(start, count)
-        first_prev, last_prev = iterator.next()
-
-        for first, last in iterator:
-            if last_prev == first - 1:
-                last_prev = last
-            else:
-                self._log.debug("FilemapFiemap: yielding range (%d, %d)"
-                                % (first_prev, last_prev))
-                yield (first_prev, last_prev)
-                first_prev, last_prev = first, last
-
-        self._log.debug("FilemapFiemap: yielding range (%d, %d)"
-                        % (first_prev, last_prev))
-        yield (first_prev, last_prev)
-
-    def get_unmapped_ranges(self, start, count):
-        """Refer the '_FilemapBase' class for the documentation."""
-        self._log.debug("FilemapFiemap: get_unmapped_ranges(%d,  %d(%d))"
-                        % (start, count, start + count - 1))
-        hole_first = start
-        for first, last in self._do_get_mapped_ranges(start, count):
-            if first > hole_first:
-                self._log.debug("FilemapFiemap: yielding range (%d, %d)"
-                                % (hole_first, first - 1))
-                yield (hole_first, first - 1)
-
-            hole_first = last + 1
-
-        if hole_first < start + count:
-            self._log.debug("FilemapFiemap: yielding range (%d, %d)"
-                            % (hole_first, start + count - 1))
-            yield (hole_first, start + count - 1)
-
-
-def filemap(image, log=None):
-    """
-    Create and return an instance of a Filemap class - 'FilemapFiemap' or
-    'FilemapSeek', depending on what the system we run on supports. If the
-    FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
-    returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
-    'FilemapSeek' class is returned. If none of these are supported, the
-    function generates an 'Error' type exception.
-    """
-
-    try:
-        return FilemapFiemap(image, log)
-    except ErrorNotSupp:
-        return FilemapSeek(image, log)