models.py 17.5 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _, ugettext_noop
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

Guba Sándor committed
34
from .tasks import local_tasks, storage_tasks
35
from celery.exceptions import TimeoutError
36 37 38
from common.models import (
    WorkerNotFound, HumanReadableException, humanize_exception
)
39 40 41 42

logger = logging.getLogger(__name__)


43
class DataStore(Model):
Guba Sándor committed
44

45 46
    """Collection of virtual disks.
    """
47 48 49 50
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
51

52 53 54 55 56 57 58 59
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

60 61
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
62 63
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
64
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
65 66 67 68 69 70
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
71 72
        else:
            raise WorkerNotFound()
73

74 75 76
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
77
                    destroyed__isnull=False) if disk.is_deletable]
78

79

Bach Dániel committed
80
class Disk(TimeStampedModel):
Guba Sándor committed
81

82 83 84 85
    """A virtual disk.
    """
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
86
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
87 88
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
89 90 91
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
92
    size = FileSizeField(null=True, default=None)
93 94 95
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
96
                        verbose_name=_("device number"))
97
    destroyed = DateTimeField(blank=True, default=None, null=True)
98

Guba Sándor committed
99 100
    is_ready = BooleanField(default=False)

101 102 103 104
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')
105 106
        permissions = (
            ('create_empty_disk', _('Can create an empty disk.')),
107 108 109
            ('download_disk', _('Can download a disk.')),
            ('resize_disk', _('Can resize a disk.'))
        )
110

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
    class DiskError(HumanReadableException):
        admin_message = None

        def __init__(self, disk, params=None, level=None, **kwargs):
            kwargs.update(params or {})
            self.disc = kwargs["disk"] = disk
            super(Disk.DiskError, self).__init__(
                level, self.message, self.admin_message or self.message,
                kwargs)

    class WrongDiskTypeError(DiskError):
        message = ugettext_noop("Operation can't be invoked on disk "
                                "'%(name)s' of type '%(type)s'.")

        admin_message = ugettext_noop(
            "Operation can't be invoked on disk "
            "'%(name)s' (%(pk)s) of type '%(type)s'.")

        def __init__(self, disk, params=None, **kwargs):
            super(Disk.WrongDiskTypeError, self).__init__(
                disk, params, type=disk.type, name=disk.name, pk=disk.pk)

    class DiskInUseError(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it is in use.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) because it is in use.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
143
            super(Disk.DiskInUseError, self).__init__(
144 145 146 147 148 149 150 151 152 153 154 155 156
                disk, params, name=disk.name, pk=disk.pk)

    class DiskIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because it has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
157
            super(Disk.DiskIsNotReady, self).__init__(
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename)

    class DiskBaseIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because its base has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because its base "
            "'%(b_name)s' (%(b_pk)s) [%(b_filename)s] has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
173
            base = kwargs.get('base')
Guba Sándor committed
174
            super(Disk.DiskBaseIsNotReady, self).__init__(
175 176 177
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename, b_name=base.name,
                b_pk=base.pk, b_filename=base.filename)
Guba Sándor committed
178

179 180
    @property
    def path(self):
181 182
        """The path where the files are stored.
        """
183
        return join(self.datastore.path, self.filename)
184 185

    @property
186
    def vm_format(self):
187 188
        """Returns the proper file format for different type of images.
        """
189 190 191
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
192
            'iso': 'raw',
193 194 195 196
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

197
    @property
198
    def format(self):
199 200
        """Returns the proper file format for different types of images.
        """
201 202 203 204 205 206 207 208 209
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
210
    def device_type(self):
211 212
        """Returns the proper device prefix for different types of images.
        """
213
        return {
214 215
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
216
            'iso': 'sd',
217 218 219
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
220

221
    @property
222 223 224 225 226 227
    def device_bus(self):
        """Returns the proper device prefix for different types of images.
        """
        return {
            'qcow2-norm': 'virtio',
            'qcow2-snap': 'virtio',
228
            'iso': 'ide',
229 230 231 232 233
            'raw-ro': 'virtio',
            'raw-rw': 'virtio',
        }[self.type]

    @property
234
    def is_deletable(self):
235
        """True if the associated file can be deleted.
236
        """
237
        # Check if all children and the disk itself is destroyed.
238
        return (self.destroyed is not None) and self.children_deletable
239

240 241 242
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
243
        """
244
        return all(i.is_deletable for i in self.derivatives.all())
245

246
    @property
247
    def is_in_use(self):
248
        """True if disk is attached to an active VM.
249 250 251 252

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
253
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
254

255
    def get_appliance(self):
Bach Dániel committed
256 257
        """Return the Instance or InstanceTemplate object where the disk
        is used
258
        """
Bach Dániel committed
259 260 261 262 263
        from vm.models import Instance
        try:
            return self.instance_set.get()
        except Instance.DoesNotExist:
            return self.template_set.get()
264

265 266
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
267

268 269 270
        This method manipulates the database only.
        """
        type_mapping = {
271 272 273
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
274 275 276
        }

        if self.type not in type_mapping.keys():
277
            raise self.WrongDiskTypeError(self)
278 279

        new_type = type_mapping[self.type]
280

281 282
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
283
                           type=new_type, dev_num=self.dev_num)
284 285

    def get_vmdisk_desc(self):
286 287
        """Serialize disk object to the vmdriver.
        """
288
        return {
289
            'source': self.path,
290
            'driver_type': self.vm_format,
291
            'driver_cache': 'none',
292
            'target_device': self.device_type + self.dev_num,
293
            'target_bus': self.device_bus,
294
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
295 296
        }

297
    def get_disk_desc(self):
298 299
        """Serialize disk object to the storage driver.
        """
300 301 302 303 304 305
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
306
            'type': 'snapshot' if self.base else 'normal'
307 308
        }

309 310
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
311 312
        """Returns the proper queue name based on the datastore.
        """
313
        if self.datastore:
314 315
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
316 317 318
        else:
            return None

319
    def __unicode__(self):
320
        return u"%s (#%d)" % (self.name, self.id or 0)
321

322
    def clean(self, *args, **kwargs):
Guba Sándor committed
323
        if (self.size is None or "") and self.base:
324 325 326
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

327
    def deploy(self, user=None, task_uuid=None, timeout=15):
328 329 330 331 332
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

333 334 335 336 337 338 339
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

340 341 342 343
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
344 345 346 347
        if self.destroyed:
            self.destroyed = None
            self.save()

348
        if self.is_ready:
349
            return True
350
        if self.base and not self.base.is_ready:
351
            raise self.DiskBaseIsNotReady(self, base=self.base)
Guba Sándor committed
352 353 354 355 356 357 358 359 360 361
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
362

363 364
        self.is_ready = True
        self.save()
Guba Sándor committed
365
        return True
366

367
    @classmethod
Guba Sándor committed
368 369
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
370
        disk.clean()
371
        disk.save()
372 373
        logger.debug(u"Disk created from: %s",
                     unicode(params.get("base", "nobase")))
374
        return disk
375

376
    @classmethod
Guba Sándor committed
377 378 379 380
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
381
        return disk
382 383

    @classmethod
Guba Sándor committed
384
    def download(cls, url, task, user=None, **params):
385 386 387 388
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
389 390
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
391 392
        :param user: owner of the disk
        :type user: django.contrib.auth.User
393 394
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
395

396 397
        :return: The created Disk object
        :rtype: Disk
398
        """
Guba Sándor committed
399
        params.setdefault('name', url.split('/')[-1])
400 401 402
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
403 404
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
405
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
406 407 408 409
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
410
                result = remote.get(timeout=5)
Guba Sándor committed
411
                break
412
            except TimeoutError as e:
Guba Sándor committed
413 414
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
415 416
                    raise humanize_exception(ugettext_noop(
                        "Operation aborted by user."), e)
417 418
        disk.size = result['size']
        disk.type = result['type']
419
        disk.is_ready = True
Guba Sándor committed
420
        disk.save()
421
        return disk
422

423
    def destroy(self, user=None, task_uuid=None):
424 425 426
        if self.destroyed:
            return False

Guba Sándor committed
427 428 429
        self.destroyed = timezone.now()
        self.save()
        return True
430

431
    def restore(self, user=None, task_uuid=None, timeout=15):
432
        """Recover destroyed disk from trash if possible.
433
        """
434 435 436 437 438 439 440
        queue_name = self.datastore.get_remote_queue_name(
            'storage', priority='slow')
        logger.info("Image: %s at Datastore: %s recovered from trash." %
                    (self.filename, self.datastore.path))
        storage_tasks.recover_from_trash.apply_async(
            args=[self.datastore.path, self.filename],
            queue=queue_name).get(timeout=timeout)
441

442
    def save_as(self, task=None, user=None, task_uuid=None, timeout=300):
443 444
        """Save VM as template.

445 446 447 448
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

449 450 451
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
452
        mapping = {
453 454 455
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
456 457
        }
        if self.type not in mapping.keys():
458
            raise self.WrongDiskTypeError(self)
459

460
        if self.is_in_use:
461 462
            raise self.DiskInUseError(self)

463
        if not self.is_ready:
Guba Sándor committed
464 465
            raise self.DiskIsNotReady(self)

466 467 468
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

469
        new_type, new_base = mapping[self.type]
470

471 472
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
473
                           name=self.name, size=self.size,
474
                           type=new_type, dev_num=self.dev_num)
475

Guba Sándor committed
476
        queue_name = self.get_remote_queue_name("storage", priority="slow")
477 478
        remote = storage_tasks.merge.apply_async(kwargs={
            "old_json": self.get_disk_desc(),
479 480
            "new_json": disk.get_disk_desc(),
            "parent_id": task.request.id},
481 482 483 484 485 486
            queue=queue_name
        )  # Timeout
        while True:
            try:
                remote.get(timeout=5)
                break
487
            except TimeoutError as e:
488 489 490
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    disk.destroy()
491 492
                    raise humanize_exception(ugettext_noop(
                        "Operation aborted by user."), e)
493 494 495
            except:
                disk.destroy()
                raise
496 497
        disk.is_ready = True
        disk.save()
Guba Sándor committed
498
        return disk