models.py 17.1 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _, ugettext_noop
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

Guba Sándor committed
34
from .tasks import local_tasks, storage_tasks
35
from celery.exceptions import TimeoutError
36
from common.models import WorkerNotFound, HumanReadableException
37 38 39 40

logger = logging.getLogger(__name__)


41
class DataStore(Model):
Guba Sándor committed
42

43 44
    """Collection of virtual disks.
    """
45 46 47 48
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
49

50 51 52 53 54 55 56 57
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

58 59
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
60 61
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
62
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
63 64 65 66 67 68
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
69 70
        else:
            raise WorkerNotFound()
71

72 73 74
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
75
                    destroyed__isnull=False) if disk.is_deletable]
76

77

Bach Dániel committed
78
class Disk(TimeStampedModel):
Guba Sándor committed
79

80 81 82 83
    """A virtual disk.
    """
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
84
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
85 86
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
87 88 89
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
90
    size = FileSizeField(null=True, default=None)
91 92 93
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
94
                        verbose_name=_("device number"))
95
    destroyed = DateTimeField(blank=True, default=None, null=True)
96

Guba Sándor committed
97 98
    is_ready = BooleanField(default=False)

99 100 101 102
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')
103 104 105
        permissions = (
            ('create_empty_disk', _('Can create an empty disk.')),
            ('download_disk', _('Can download a disk.')))
106

107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
    class DiskError(HumanReadableException):
        admin_message = None

        def __init__(self, disk, params=None, level=None, **kwargs):
            kwargs.update(params or {})
            self.disc = kwargs["disk"] = disk
            super(Disk.DiskError, self).__init__(
                level, self.message, self.admin_message or self.message,
                kwargs)

    class WrongDiskTypeError(DiskError):
        message = ugettext_noop("Operation can't be invoked on disk "
                                "'%(name)s' of type '%(type)s'.")

        admin_message = ugettext_noop(
            "Operation can't be invoked on disk "
            "'%(name)s' (%(pk)s) of type '%(type)s'.")

        def __init__(self, disk, params=None, **kwargs):
            super(Disk.WrongDiskTypeError, self).__init__(
                disk, params, type=disk.type, name=disk.name, pk=disk.pk)

    class DiskInUseError(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it is in use.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) because it is in use.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
139
            super(Disk.DiskInUseError, self).__init__(
140 141 142 143 144 145 146 147 148 149 150 151 152
                disk, params, name=disk.name, pk=disk.pk)

    class DiskIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because it has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
153
            super(Disk.DiskIsNotReady, self).__init__(
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename)

    class DiskBaseIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because its base has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because its base "
            "'%(b_name)s' (%(b_pk)s) [%(b_filename)s] has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
169
            base = kwargs.get('base')
Guba Sándor committed
170
            super(Disk.DiskBaseIsNotReady, self).__init__(
171 172 173
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename, b_name=base.name,
                b_pk=base.pk, b_filename=base.filename)
Guba Sándor committed
174

175 176
    @property
    def path(self):
177 178
        """The path where the files are stored.
        """
179
        return join(self.datastore.path, self.filename)
180 181

    @property
182
    def vm_format(self):
183 184
        """Returns the proper file format for different type of images.
        """
185 186 187
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
188
            'iso': 'raw',
189 190 191 192
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

193
    @property
194
    def format(self):
195 196
        """Returns the proper file format for different types of images.
        """
197 198 199 200 201 202 203 204 205
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
206
    def device_type(self):
207 208
        """Returns the proper device prefix for different types of images.
        """
209
        return {
210 211
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
212
            'iso': 'sd',
213 214 215
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
216

217
    @property
218 219 220 221 222 223 224 225 226 227 228 229
    def device_bus(self):
        """Returns the proper device prefix for different types of images.
        """
        return {
            'qcow2-norm': 'virtio',
            'qcow2-snap': 'virtio',
            'iso': 'scsi',
            'raw-ro': 'virtio',
            'raw-rw': 'virtio',
        }[self.type]

    @property
230
    def is_deletable(self):
231
        """True if the associated file can be deleted.
232
        """
233
        # Check if all children and the disk itself is destroyed.
234
        return (self.destroyed is not None) and self.children_deletable
235

236 237 238
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
239
        """
240
        return all(i.is_deletable for i in self.derivatives.all())
241

242
    @property
243
    def is_in_use(self):
244
        """True if disk is attached to an active VM.
245 246 247 248

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
249
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
250

251
    def get_appliance(self):
Bach Dániel committed
252 253
        """Return the Instance or InstanceTemplate object where the disk
        is used
254
        """
Bach Dániel committed
255 256 257 258 259
        from vm.models import Instance
        try:
            return self.instance_set.get()
        except Instance.DoesNotExist:
            return self.template_set.get()
260

261 262
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
263

264 265 266
        This method manipulates the database only.
        """
        type_mapping = {
267 268 269
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
270 271 272
        }

        if self.type not in type_mapping.keys():
273
            raise self.WrongDiskTypeError(self)
274 275

        new_type = type_mapping[self.type]
276

277 278 279
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
                           type=new_type)
280 281

    def get_vmdisk_desc(self):
282 283
        """Serialize disk object to the vmdriver.
        """
284
        return {
285
            'source': self.path,
286
            'driver_type': self.vm_format,
287
            'driver_cache': 'none',
288
            'target_device': self.device_type + self.dev_num,
289
            'target_bus': self.device_bus,
290
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
291 292
        }

293
    def get_disk_desc(self):
294 295
        """Serialize disk object to the storage driver.
        """
296 297 298 299 300 301
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
302
            'type': 'snapshot' if self.base else 'normal'
303 304
        }

305 306
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
307 308
        """Returns the proper queue name based on the datastore.
        """
309
        if self.datastore:
310 311
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
312 313 314
        else:
            return None

315
    def __unicode__(self):
316
        return u"%s (#%d)" % (self.name, self.id or 0)
317

318
    def clean(self, *args, **kwargs):
Guba Sándor committed
319
        if (self.size is None or "") and self.base:
320 321 322
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

323
    def deploy(self, user=None, task_uuid=None, timeout=15):
324 325 326 327 328
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

329 330 331 332 333 334 335
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

336 337 338 339
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
340 341 342 343
        if self.destroyed:
            self.destroyed = None
            self.save()

344
        if self.is_ready:
345
            return True
346
        if self.base and not self.base.is_ready:
347
            raise self.DiskBaseIsNotReady(self, base=self.base)
Guba Sándor committed
348 349 350 351 352 353 354 355 356 357
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
358

359 360
        self.is_ready = True
        self.save()
Guba Sándor committed
361
        return True
362

363
    @classmethod
Guba Sándor committed
364 365
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
366
        disk.clean()
367
        disk.save()
Guba Sándor committed
368
        logger.debug("Disk created: %s", params)
369
        return disk
370

371
    @classmethod
Guba Sándor committed
372 373 374 375
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
376
        return disk
377 378

    @classmethod
Guba Sándor committed
379
    def download(cls, url, task, user=None, **params):
380 381 382 383
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
384 385
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
386 387
        :param user: owner of the disk
        :type user: django.contrib.auth.User
388 389
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
390

391 392
        :return: The created Disk object
        :rtype: Disk
393
        """
Guba Sándor committed
394
        params.setdefault('name', url.split('/')[-1])
395 396 397
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
398 399
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
400
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
401 402 403 404
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
405
                result = remote.get(timeout=5)
Guba Sándor committed
406 407 408 409 410
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    raise Exception("Download aborted by user.")
411 412
        disk.size = result['size']
        disk.type = result['type']
413
        disk.is_ready = True
Guba Sándor committed
414
        disk.save()
415
        return disk
416

417
    def destroy(self, user=None, task_uuid=None):
418 419 420
        if self.destroyed:
            return False

Guba Sándor committed
421 422 423
        self.destroyed = timezone.now()
        self.save()
        return True
424

425
    def restore(self, user=None, task_uuid=None, timeout=15):
426
        """Recover destroyed disk from trash if possible.
427
        """
428 429 430 431 432 433 434
        queue_name = self.datastore.get_remote_queue_name(
            'storage', priority='slow')
        logger.info("Image: %s at Datastore: %s recovered from trash." %
                    (self.filename, self.datastore.path))
        storage_tasks.recover_from_trash.apply_async(
            args=[self.datastore.path, self.filename],
            queue=queue_name).get(timeout=timeout)
435

436
    def save_as(self, task=None, user=None, task_uuid=None, timeout=300):
437 438
        """Save VM as template.

439 440 441 442
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

443 444 445
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
446
        mapping = {
447 448 449
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
450 451
        }
        if self.type not in mapping.keys():
452
            raise self.WrongDiskTypeError(self)
453

454
        if self.is_in_use:
455 456
            raise self.DiskInUseError(self)

457
        if not self.is_ready:
Guba Sándor committed
458 459
            raise self.DiskIsNotReady(self)

460 461 462
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

463
        new_type, new_base = mapping[self.type]
464

465 466
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
467
                           name=self.name, size=self.size,
468
                           type=new_type, dev_num=self.dev_num)
469

Guba Sándor committed
470
        queue_name = self.get_remote_queue_name("storage", priority="slow")
471 472 473 474 475 476 477 478 479 480 481 482 483 484
        remote = storage_tasks.merge.apply_async(kwargs={
            "old_json": self.get_disk_desc(),
            "new_json": disk.get_disk_desc()},
            queue=queue_name
        )  # Timeout
        while True:
            try:
                remote.get(timeout=5)
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    disk.destroy()
                    raise Exception("Save as aborted by use.")
485 486
        disk.is_ready = True
        disk.save()
Guba Sándor committed
487
        return disk