Commit 4812fcce by Czémán Arnold

disk: implement check_valid_image() and checksum() methods of CephDisk, small…

disk: implement check_valid_image() and checksum() methods of CephDisk, small rework in download() method
storagedriver: fix make_free_space()
parent d2bf1727
...@@ -453,7 +453,18 @@ class CephDisk(Disk): ...@@ -453,7 +453,18 @@ class CephDisk(Disk):
@property @property
def checksum(self, blocksize=65536): def checksum(self, blocksize=65536):
raise NotImplementedError() hash = md5()
with CephConnection(str(self.dir)) as conn:
with rbd.Image(conn.ioctx, self.name) as image:
size = image.size()
offset = 0
while offset + blocksize <= size:
block = image.read(offset, blocksize)
hash.update(block)
offset += blocksize
block = image.read(offset, size - offset)
hash.update(block)
return hash.hexdigest()
@classmethod @classmethod
def deserialize(cls, desc): def deserialize(cls, desc):
...@@ -492,8 +503,23 @@ class CephDisk(Disk): ...@@ -492,8 +503,23 @@ class CephDisk(Disk):
def check_valid_image(self): def check_valid_image(self):
"""Check wether the downloaded image is valid. """Check wether the downloaded image is valid.
Set the proper type for valid images.""" Set the proper type for valid images."""
# TODO format_map = [
return True ("iso", "iso"),
("x86 boot sector", "iso")
]
buff = None
with CephConnection(str(self.dir)) as conn:
with rbd.Image(conn.ioctx, self.name) as image:
# 2k may enough determine the file type
buff = image.read(0, 2048)
with magic.Magic() as m:
ftype = m.id_buffer(buff)
logger.debug("Downloaded file type is: %s", ftype)
for file_type, disk_format in format_map:
if file_type in ftype.lower():
self.format = disk_format
return True
return False
def download(self, task, url, parent_id=None): def download(self, task, url, parent_id=None):
"""Download image from url.""" """Download image from url."""
...@@ -516,6 +542,9 @@ class CephDisk(Disk): ...@@ -516,6 +542,9 @@ class CephDisk(Disk):
# undocumented zlib feature http://stackoverflow.com/a/2424549 # undocumented zlib feature http://stackoverflow.com/a/2424549
elif ext == 'bz2': elif ext == 'bz2':
decompressor = BZ2Decompressor() decompressor = BZ2Decompressor()
if ext == 'zip':
raise Exception("The zip format not supported "
"with Ceph Block Device")
clen = int(r.headers.get('content-length', MAXIMUM_SIZE)) clen = int(r.headers.get('content-length', MAXIMUM_SIZE))
if clen > MAXIMUM_SIZE: if clen > MAXIMUM_SIZE:
raise FileTooBig() raise FileTooBig()
...@@ -527,6 +556,7 @@ class CephDisk(Disk): ...@@ -527,6 +556,7 @@ class CephDisk(Disk):
rbd_inst.create(conn.ioctx, self.name, int(MAXIMUM_SIZE)) rbd_inst.create(conn.ioctx, self.name, int(MAXIMUM_SIZE))
with rbd.Image(conn.ioctx, self.name) as image: with rbd.Image(conn.ioctx, self.name) as image:
offset = 0 offset = 0
actsize = 0
for chunk in r.iter_content(chunk_size=chunk_size): for chunk in r.iter_content(chunk_size=chunk_size):
if ext in ('gz', 'bz'): if ext in ('gz', 'bz'):
chunk = decompressor.decompress(chunk) chunk = decompressor.decompress(chunk)
...@@ -547,6 +577,7 @@ class CephDisk(Disk): ...@@ -547,6 +577,7 @@ class CephDisk(Disk):
if ext == 'gz': if ext == 'gz':
image.write(decompressor.flush(), offset) image.write(decompressor.flush(), offset)
image.flush() image.flush()
image.resize(actsize)
self.size = CephDisk.get(conn.ioctx, self.dir, self.name).size self.size = CephDisk.get(conn.ioctx, self.dir, self.name).size
logger.debug("Download finished %s (%s bytes)", logger.debug("Download finished %s (%s bytes)",
self.name, self.size) self.name, self.size)
...@@ -563,6 +594,11 @@ class CephDisk(Disk): ...@@ -563,6 +594,11 @@ class CephDisk(Disk):
logger.error("Error occured %s. Download %s failed, %s removed.", logger.error("Error occured %s. Download %s failed, %s removed.",
e, url, disk_path) e, url, disk_path)
raise raise
else:
if not self.check_valid_image():
self.__remove_disk()
raise Exception("Invalid file format. Only iso files "
"are allowed. Image from: %s" % url)
def __remove_disk(self): def __remove_disk(self):
with CephConnection(self.dir) as conn: with CephConnection(self.dir) as conn:
...@@ -575,7 +611,7 @@ class CephDisk(Disk): ...@@ -575,7 +611,7 @@ class CephDisk(Disk):
def __snapshot(self, ioctx): def __snapshot(self, ioctx):
''' Creating snapshot with base image. ''' Creating snapshot with base image.
''' '''
# Check if snapshot type and rbd format matchmatch # Check if snapshot type and rbd format match
if self.type != 'snapshot': if self.type != 'snapshot':
raise Exception('Invalid type: %s' % self.type) raise Exception('Invalid type: %s' % self.type)
if self.format != "rbd": if self.format != "rbd":
......
...@@ -61,7 +61,7 @@ class download(AbortableTask): ...@@ -61,7 +61,7 @@ class download(AbortableTask):
disk.download(self, url, parent_id) disk.download(self, url, parent_id)
return {'size': disk.size, return {'size': disk.size,
'type': disk.format, 'type': disk.format,
'checksum': 0, } # TODO: disk.checksum 'checksum': disk.checksum, }
@celery.task() @celery.task()
...@@ -173,13 +173,14 @@ def make_free_space(data_store_type, path, deletable_disks, percent=10): ...@@ -173,13 +173,14 @@ def make_free_space(data_store_type, path, deletable_disks, percent=10):
If free space is less than the given percent If free space is less than the given percent
removes oldest files to satisfy the given requirement. removes oldest files to satisfy the given requirement.
''' '''
ds_type = data_store_type
logger.info("Free space on datastore: %s" % logger.info("Free space on datastore: %s" %
get_storage_stat(path).get('free_percent')) get_storage_stat(ds_type, path).get('free_percent'))
while get_storage_stat(path).get('free_percent') < percent: while get_storage_stat(ds_type, path).get('free_percent') < percent:
logger.debug(get_storage_stat(path)) logger.debug(get_storage_stat(ds_type, path))
try: try:
f = deletable_disks.pop(0) f = deletable_disks.pop(0)
if data_store_type == "ceph_block": if ds_type == "ceph_block":
with CephConnection(str(path)) as conn: with CephConnection(str(path)) as conn:
rbd_inst = rbd.RBD() rbd_inst = rbd.RBD()
rbd_inst.remove(conn.ioctx, f) rbd_inst.remove(conn.ioctx, f)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment