Commit c2d36f81 by Őry Máté

Merge branch 'storage-fixes'

parents 9dcd479e 3f0b123b
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
from contextlib import contextmanager from contextlib import contextmanager
import logging import logging
from os.path import join
import uuid import uuid
from django.db.models import (Model, BooleanField, CharField, DateTimeField, from django.db.models import (Model, BooleanField, CharField, DateTimeField,
...@@ -10,6 +11,7 @@ from django.utils import timezone ...@@ -10,6 +11,7 @@ from django.utils import timezone
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
from sizefield.models import FileSizeField from sizefield.models import FileSizeField
from datetime import timedelta
from acl.models import AclBase from acl.models import AclBase
from .tasks import local_tasks, remote_tasks from .tasks import local_tasks, remote_tasks
...@@ -36,14 +38,20 @@ class DataStore(Model): ...@@ -36,14 +38,20 @@ class DataStore(Model):
def __unicode__(self): def __unicode__(self):
return u'%s (%s)' % (self.name, self.path) return u'%s (%s)' % (self.name, self.path)
def get_remote_queue_name(self, queue_id): def get_remote_queue_name(self, queue_id, check_worker=True):
logger.debug("Checking for storage queue %s.%s", logger.debug("Checking for storage queue %s.%s",
self.hostname, queue_id) self.hostname, queue_id)
if local_tasks.check_queue(self.hostname, queue_id): if not check_worker or local_tasks.check_queue(self.hostname,
queue_id):
return self.hostname + '.' + queue_id return self.hostname + '.' + queue_id
else: else:
raise WorkerNotFound() raise WorkerNotFound()
def get_deletable_disks(self):
return [disk.filename for disk in
self.disk_set.filter(
destroyed__isnull=False) if disk.is_deletable()]
class Disk(AclBase, TimeStampedModel): class Disk(AclBase, TimeStampedModel):
...@@ -100,10 +108,12 @@ class Disk(AclBase, TimeStampedModel): ...@@ -100,10 +108,12 @@ class Disk(AclBase, TimeStampedModel):
@property @property
def path(self): def path(self):
return self.datastore.path + '/' + self.filename """Get the path where the files are stored."""
return join(self.datastore.path, self.filename)
@property @property
def format(self): def format(self):
"""Returns the proper file format for different type of images."""
return { return {
'qcow2-norm': 'qcow2', 'qcow2-norm': 'qcow2',
'qcow2-snap': 'qcow2', 'qcow2-snap': 'qcow2',
...@@ -114,6 +124,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -114,6 +124,7 @@ class Disk(AclBase, TimeStampedModel):
@property @property
def device_type(self): def device_type(self):
"""Returns the proper device prefix for different file format."""
return { return {
'qcow2-norm': 'vd', 'qcow2-norm': 'vd',
'qcow2-snap': 'vd', 'qcow2-snap': 'vd',
...@@ -122,7 +133,28 @@ class Disk(AclBase, TimeStampedModel): ...@@ -122,7 +133,28 @@ class Disk(AclBase, TimeStampedModel):
'raw-rw': 'vd', 'raw-rw': 'vd',
}[self.type] }[self.type]
def is_deletable(self):
"""Returns whether the file can be deleted.
Checks if all children and the disk itself is destroyed.
"""
yesterday = timezone.now() - timedelta(days=1)
return (self.destroyed is not None
and self.destroyed < yesterday) and not self.has_active_child()
def has_active_child(self):
"""Returns if disk has children that are not destroyed.
"""
return any((not i.is_deletable() for i in self.derivatives.all()))
def is_in_use(self): def is_in_use(self):
"""Returns if disk is attached to an active VM.
'In use' means the disk is attached to a VM which is not STOPPED, as
any other VMs leave the disk in an inconsistent state.
"""
return any([i.state != 'STOPPED' for i in self.instance_set.all()]) return any([i.state != 'STOPPED' for i in self.instance_set.all()])
def get_exclusive(self): def get_exclusive(self):
...@@ -139,7 +171,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -139,7 +171,7 @@ class Disk(AclBase, TimeStampedModel):
if self.type not in type_mapping.keys(): if self.type not in type_mapping.keys():
raise self.WrongDiskTypeError(self.type) raise self.WrongDiskTypeError(self.type)
filename = self.filename if self.type == 'iso' else str(uuid.uuid4()) filename = self.filename if self.type == 'iso' else None
new_type = type_mapping[self.type] new_type = type_mapping[self.type]
return Disk.objects.create(base=self, datastore=self.datastore, return Disk.objects.create(base=self, datastore=self.datastore,
...@@ -147,6 +179,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -147,6 +179,7 @@ class Disk(AclBase, TimeStampedModel):
size=self.size, type=new_type) size=self.size, type=new_type)
def get_vmdisk_desc(self): def get_vmdisk_desc(self):
"""Serialize disk object to the vmdriver."""
return { return {
'source': self.path, 'source': self.path,
'driver_type': self.format, 'driver_type': self.format,
...@@ -156,6 +189,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -156,6 +189,7 @@ class Disk(AclBase, TimeStampedModel):
} }
def get_disk_desc(self): def get_disk_desc(self):
"""Serialize disk object to the storage driver."""
return { return {
'name': self.filename, 'name': self.filename,
'dir': self.datastore.path, 'dir': self.datastore.path,
...@@ -165,20 +199,26 @@ class Disk(AclBase, TimeStampedModel): ...@@ -165,20 +199,26 @@ class Disk(AclBase, TimeStampedModel):
'type': 'snapshot' if self.type == 'qcow2-snap' else 'normal' 'type': 'snapshot' if self.type == 'qcow2-snap' else 'normal'
} }
def get_remote_queue_name(self, queue_id): def get_remote_queue_name(self, queue_id='storage', check_worker=True):
"""Returns the proper queue name based on the datastore."""
if self.datastore: if self.datastore:
return self.datastore.get_remote_queue_name(queue_id) return self.datastore.get_remote_queue_name(queue_id, check_worker)
else: else:
return None return None
def __unicode__(self): def __unicode__(self):
return u"%s (#%d)" % (self.name, self.id) return u"%s (#%d)" % (self.name, self.id or 0)
def clean(self, *args, **kwargs): def clean(self, *args, **kwargs):
if self.size == "" and self.base: if self.size == "" and self.base:
self.size = self.base.size self.size = self.base.size
super(Disk, self).clean(*args, **kwargs) super(Disk, self).clean(*args, **kwargs)
def save(self, *args, **kwargs):
if self.filename is None:
self.generate_filename()
return super(Disk, self).save(*args, **kwargs)
def deploy(self, user=None, task_uuid=None, timeout=15): def deploy(self, user=None, task_uuid=None, timeout=15):
"""Reify the disk model on the associated data store. """Reify the disk model on the associated data store.
...@@ -231,29 +271,84 @@ class Disk(AclBase, TimeStampedModel): ...@@ -231,29 +271,84 @@ class Disk(AclBase, TimeStampedModel):
return local_tasks.deploy.apply_async(args=[self, user], return local_tasks.deploy.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def generate_filename(self):
"""Generate a unique filename and set it on the object.
"""
self.filename = str(uuid.uuid4())
@classmethod @classmethod
def create_empty(cls, params={}, user=None): def create_empty(cls, instance=None, user=None, **kwargs):
disk = cls() """Create empty Disk object.
disk.__dict__.update(params)
:param instance: Instance attach the Disk to.
:type instane: vm.models.Instance or NoneType
:param user: Creator of the disk.
:type user: django.contrib.auth.User
:return: Disk object without a real image, to be .deploy()ed later.
"""
with disk_activity(code_suffix="create", user=user) as act:
disk = cls(**kwargs)
if disk.filename is None:
disk.generate_filename()
disk.save() disk.save()
act.disk = disk
act.save()
if instance:
instance.disks.add(disk)
return disk return disk
@classmethod @classmethod
def create_from_url_async(cls, url, params=None, user=None): def create_from_url_async(cls, url, instance=None, params=None, user=None):
"""Create disk object and download data from url asynchrnously.
:param url: URL of image to download.
:type url: string
:param instance: instance object to connect disk
:type instane: vm.models.Instance
:param params: disk custom parameters
:type params: dict
:param user: owner of the disk
:type user: django.contrib.auth.User
:return: Task
:rtype: AsyncResult
"""
return local_tasks.create_from_url.apply_async(kwargs={ return local_tasks.create_from_url.apply_async(kwargs={
'cls': cls, 'url': url, 'params': params, 'user': user}, 'cls': cls, 'url': url, 'instance': instance,
'params': params, 'user': user},
queue='localhost.man') queue='localhost.man')
def create_from_url(cls, url, params={}, user=None, task_uuid=None, @classmethod
abortable_task=None): def create_from_url(cls, url, instance=None, params=None, user=None,
task_uuid=None, abortable_task=None):
"""Create disk object and download data from url synchronusly.
:param url: image url to download.
:type url: url
:param instance: instnace object to connect disk
:type instane: vm.models.Instance
:param params: disk custom parameters
:type params: dict
:param user: owner of the disk
:type user: django.contrib.auth.User
:param task_uuid: TODO
:param abortable_task: TODO
:return: The created Disk object
:rtype: Disk
"""
disk = cls() disk = cls()
disk.filename = str(uuid.uuid4()) disk.generate_filename()
disk.type = "iso" disk.type = "iso"
disk.size = 1 disk.size = 1
disk.datastore = DataStore.objects.all()[0] # TODO get proper datastore
disk.datastore = DataStore.objects.get()
if params: if params:
disk.__dict__.update(params) disk.__dict__.update(params)
disk.save() disk.save()
if instance:
instance.disks.add(disk)
queue_name = disk.get_remote_queue_name('storage') queue_name = disk.get_remote_queue_name('storage')
def __on_abort(activity, error): def __on_abort(activity, error):
...@@ -284,6 +379,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -284,6 +379,7 @@ class Disk(AclBase, TimeStampedModel):
disk.size = size disk.size = size
disk.ready = True disk.ready = True
disk.save() disk.save()
return disk
def destroy(self, user=None, task_uuid=None): def destroy(self, user=None, task_uuid=None):
if self.destroyed: if self.destroyed:
...@@ -303,7 +399,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -303,7 +399,7 @@ class Disk(AclBase, TimeStampedModel):
queue='localhost.man') queue='localhost.man')
def restore(self, user=None, task_uuid=None): def restore(self, user=None, task_uuid=None):
"""Restore destroyed disk. """Recover destroyed disk from trash if possible.
""" """
# TODO # TODO
pass pass
...@@ -328,12 +424,11 @@ class Disk(AclBase, TimeStampedModel): ...@@ -328,12 +424,11 @@ class Disk(AclBase, TimeStampedModel):
with disk_activity(code_suffix='save_as', disk=self, with disk_activity(code_suffix='save_as', disk=self,
task_uuid=task_uuid, user=user, timeout=300): task_uuid=task_uuid, user=user, timeout=300):
filename = str(uuid.uuid4())
new_type, new_base = mapping[self.type] new_type, new_base = mapping[self.type]
disk = Disk.objects.create(base=new_base, datastore=self.datastore, disk = Disk.objects.create(base=new_base, datastore=self.datastore,
filename=filename, name=self.name, name=self.name, size=self.size,
size=self.size, type=new_type) type=new_type)
queue_name = self.get_remote_queue_name('storage') queue_name = self.get_remote_queue_name('storage')
remote_tasks.merge.apply_async(args=[self.get_disk_desc(), remote_tasks.merge.apply_async(args=[self.get_disk_desc(),
......
...@@ -48,3 +48,9 @@ class create_from_url(AbortableTask): ...@@ -48,3 +48,9 @@ class create_from_url(AbortableTask):
task_uuid=create_from_url.request.id, task_uuid=create_from_url.request.id,
abortable_task=self, abortable_task=self,
user=user) user=user)
@celery.task
def create_empty(Disk, instance, params, user):
Disk.create_empty(instance, params, user,
task_uuid=create_empty.request.id)
from storage.models import DataStore from storage.models import DataStore
import os import os
from django.utils import timezone
from datetime import timedelta
from manager.mancelery import celery from manager.mancelery import celery
import logging import logging
from storage.tasks import remote_tasks from storage.tasks import remote_tasks
...@@ -21,10 +19,8 @@ def garbage_collector(timeout=15): ...@@ -21,10 +19,8 @@ def garbage_collector(timeout=15):
:type timeoit: int :type timeoit: int
""" """
for ds in DataStore.objects.all(): for ds in DataStore.objects.all():
time_before = timezone.now() - timedelta(days=1)
file_list = os.listdir(ds.path) file_list = os.listdir(ds.path)
disk_list = [disk.filename for disk in disk_list = ds.get_deletable_disks()
ds.disk_set.filter(destroyed__lt=time_before)]
queue_name = ds.get_remote_queue_name('storage') queue_name = ds.get_remote_queue_name('storage')
for i in set(file_list).intersection(disk_list): for i in set(file_list).intersection(disk_list):
logger.info("Image: %s at Datastore: %s moved to trash folder." % logger.info("Image: %s at Datastore: %s moved to trash folder." %
......
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
from datetime import timedelta
from django.test import TestCase
from django.utils import timezone
from ..models import Disk, DataStore
old = timezone.now() - timedelta(days=2)
new = timezone.now() - timedelta(hours=2)
class DiskTestCase(TestCase):
n = 0
def setUp(self):
self.ds = DataStore.objects.create(path="/datastore",
hostname="devenv", name="default")
def _disk(self, destroyed=None, base=None):
self.n += 1
n = "d%d" % self.n
return Disk.objects.create(name=n, filename=n, base=base, size=1,
destroyed=destroyed, datastore=self.ds)
def test_deletable_not_destroyed(self):
d = self._disk()
assert not d.is_deletable()
def test_deletable_newly_destroyed(self):
d = self._disk(destroyed=new)
assert not d.is_deletable()
def test_deletable_no_child(self):
d = self._disk(destroyed=old)
assert d.is_deletable()
def test_deletable_child_not_destroyed(self):
d = self._disk()
self._disk(base=d, destroyed=old)
self._disk(base=d)
assert not d.is_deletable()
def test_deletable_child_newly_destroyed(self):
d = self._disk(destroyed=old)
self._disk(base=d, destroyed=new)
self._disk(base=d)
assert not d.is_deletable()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment