disk: implement check_valid_image() and checksum() methods of CephDisk, small…

disk: implement check_valid_image() and checksum() methods of CephDisk, small rework in download() method
storagedriver: fix make_free_space()

disk.py

@@ -453,7 +453,18 @@ class CephDisk(Disk):
 
 
     @property
     @property
     def checksum(self, blocksize=65536):
     def checksum(self, blocksize=65536):
-        raise NotImplementedError()
+        hash = md5()
+        with CephConnection(str(self.dir)) as conn:
+            with rbd.Image(conn.ioctx, self.name) as image:
+                size = image.size()
+                offset = 0
+                while offset + blocksize <= size:
+                    block = image.read(offset, blocksize)
+                    hash.update(block)
+                    offset += blocksize
+                block = image.read(offset, size - offset)
+                hash.update(block)
+        return hash.hexdigest()
 
 
     @classmethod
     @classmethod
     def deserialize(cls, desc):
     def deserialize(cls, desc):
@@ -492,8 +503,23 @@ class CephDisk(Disk):
     def check_valid_image(self):
     def check_valid_image(self):
         """Check wether the downloaded image is valid.
         """Check wether the downloaded image is valid.
         Set the proper type for valid images."""
         Set the proper type for valid images."""
-        # TODO
-        return True
+        format_map = [
+            ("iso", "iso"),
+            ("x86 boot sector", "iso")
+        ]
+        buff = None
+        with CephConnection(str(self.dir)) as conn:
+            with rbd.Image(conn.ioctx, self.name) as image:
+                # 2k may enough determine the file type
+                buff = image.read(0, 2048)
+        with magic.Magic() as m:
+            ftype = m.id_buffer(buff)
+            logger.debug("Downloaded file type is: %s", ftype)
+            for file_type, disk_format in format_map:
+                if file_type in ftype.lower():
+                    self.format = disk_format
+                    return True
+        return False
 
 
     def download(self, task, url, parent_id=None):
     def download(self, task, url, parent_id=None):
         """Download image from url."""
         """Download image from url."""
@@ -516,6 +542,9 @@ class CephDisk(Disk):
             # undocumented zlib feature http://stackoverflow.com/a/2424549
             # undocumented zlib feature http://stackoverflow.com/a/2424549
         elif ext == 'bz2':
         elif ext == 'bz2':
             decompressor = BZ2Decompressor()
             decompressor = BZ2Decompressor()
+        if ext == 'zip':
+            raise Exception("The zip format not supported "
+                            "with Ceph Block Device")
         clen = int(r.headers.get('content-length', MAXIMUM_SIZE))
         clen = int(r.headers.get('content-length', MAXIMUM_SIZE))
         if clen > MAXIMUM_SIZE:
         if clen > MAXIMUM_SIZE:
             raise FileTooBig()
             raise FileTooBig()
@@ -527,6 +556,7 @@ class CephDisk(Disk):
                 rbd_inst.create(conn.ioctx, self.name, int(MAXIMUM_SIZE))
                 rbd_inst.create(conn.ioctx, self.name, int(MAXIMUM_SIZE))
                 with rbd.Image(conn.ioctx, self.name) as image:
                 with rbd.Image(conn.ioctx, self.name) as image:
                     offset = 0
                     offset = 0
+                    actsize = 0
                     for chunk in r.iter_content(chunk_size=chunk_size):
                     for chunk in r.iter_content(chunk_size=chunk_size):
                         if ext in ('gz', 'bz'):
                         if ext in ('gz', 'bz'):
                             chunk = decompressor.decompress(chunk)
                             chunk = decompressor.decompress(chunk)
@@ -547,6 +577,7 @@ class CephDisk(Disk):
                     if ext == 'gz':
                     if ext == 'gz':
                         image.write(decompressor.flush(), offset)
                         image.write(decompressor.flush(), offset)
                     image.flush()
                     image.flush()
+                    image.resize(actsize)
                 self.size = CephDisk.get(conn.ioctx, self.dir, self.name).size
                 self.size = CephDisk.get(conn.ioctx, self.dir, self.name).size
                 logger.debug("Download finished %s (%s bytes)",
                 logger.debug("Download finished %s (%s bytes)",
                              self.name, self.size)
                              self.name, self.size)
@@ -563,6 +594,11 @@ class CephDisk(Disk):
             logger.error("Error occured %s. Download %s failed, %s removed.",
             logger.error("Error occured %s. Download %s failed, %s removed.",
                          e, url, disk_path)
                          e, url, disk_path)
             raise
             raise
+        else:
+            if not self.check_valid_image():
+                self.__remove_disk()
+                raise Exception("Invalid file format. Only iso files "
+                                "are allowed. Image from: %s" % url)
 
 
     def __remove_disk(self):
     def __remove_disk(self):
         with CephConnection(self.dir) as conn:
         with CephConnection(self.dir) as conn:
@@ -575,7 +611,7 @@ class CephDisk(Disk):
     def __snapshot(self, ioctx):
     def __snapshot(self, ioctx):
         ''' Creating snapshot with base image.
         ''' Creating snapshot with base image.
         '''
         '''
-        # Check if snapshot type and rbd format matchmatch
+        # Check if snapshot type and rbd format match
         if self.type != 'snapshot':
         if self.type != 'snapshot':
             raise Exception('Invalid type: %s' % self.type)
             raise Exception('Invalid type: %s' % self.type)
         if self.format != "rbd":
         if self.format != "rbd":

storagedriver.py

@@ -61,7 +61,7 @@ class download(AbortableTask):
         disk.download(self, url, parent_id)
         disk.download(self, url, parent_id)
         return {'size': disk.size,
         return {'size': disk.size,
                 'type': disk.format,
                 'type': disk.format,
-                'checksum': 0, }  # TODO: disk.checksum
+                'checksum': disk.checksum, }
 
 
 
 
 @celery.task()
 @celery.task()
@@ -173,13 +173,14 @@ def make_free_space(data_store_type, path, deletable_disks, percent=10):
         If free space is less than the given percent
         If free space is less than the given percent
         removes oldest files to satisfy the given requirement.
         removes oldest files to satisfy the given requirement.
     '''
     '''
+    ds_type = data_store_type
     logger.info("Free space on datastore: %s" %
     logger.info("Free space on datastore: %s" %
-                get_storage_stat(path).get('free_percent'))
-    while get_storage_stat(path).get('free_percent') < percent:
-        logger.debug(get_storage_stat(path))
+                get_storage_stat(ds_type, path).get('free_percent'))
+    while get_storage_stat(ds_type, path).get('free_percent') < percent:
+        logger.debug(get_storage_stat(ds_type, path))
         try:
         try:
             f = deletable_disks.pop(0)
             f = deletable_disks.pop(0)
-            if data_store_type == "ceph_block":
+            if ds_type == "ceph_block":
                 with CephConnection(str(path)) as conn:
                 with CephConnection(str(path)) as conn:
                     rbd_inst = rbd.RBD()
                     rbd_inst = rbd.RBD()
                     rbd_inst.remove(conn.ioctx, f)
                     rbd_inst.remove(conn.ioctx, f)