models.py 17.2 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _, ugettext_noop
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

Guba Sándor committed
34
from .tasks import local_tasks, storage_tasks
35
from celery.exceptions import TimeoutError
36 37 38
from common.models import (
    WorkerNotFound, HumanReadableException, humanize_exception
)
39 40 41 42

logger = logging.getLogger(__name__)


43
class DataStore(Model):
Guba Sándor committed
44

45 46
    """Collection of virtual disks.
    """
47 48 49 50
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
51

52 53 54 55 56 57 58 59
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

60 61
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
62 63
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
64
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
65 66 67 68 69 70
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
71 72
        else:
            raise WorkerNotFound()
73

74 75 76
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
77
                    destroyed__isnull=False) if disk.is_deletable]
78

79

Bach Dániel committed
80
class Disk(TimeStampedModel):
Guba Sándor committed
81

82 83 84 85
    """A virtual disk.
    """
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
86
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
87 88
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
89 90 91
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
92
    size = FileSizeField(null=True, default=None)
93 94 95
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
96
                        verbose_name=_("device number"))
97
    destroyed = DateTimeField(blank=True, default=None, null=True)
98

Guba Sándor committed
99 100
    is_ready = BooleanField(default=False)

101 102 103 104
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')
105 106 107
        permissions = (
            ('create_empty_disk', _('Can create an empty disk.')),
            ('download_disk', _('Can download a disk.')))
108

109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
    class DiskError(HumanReadableException):
        admin_message = None

        def __init__(self, disk, params=None, level=None, **kwargs):
            kwargs.update(params or {})
            self.disc = kwargs["disk"] = disk
            super(Disk.DiskError, self).__init__(
                level, self.message, self.admin_message or self.message,
                kwargs)

    class WrongDiskTypeError(DiskError):
        message = ugettext_noop("Operation can't be invoked on disk "
                                "'%(name)s' of type '%(type)s'.")

        admin_message = ugettext_noop(
            "Operation can't be invoked on disk "
            "'%(name)s' (%(pk)s) of type '%(type)s'.")

        def __init__(self, disk, params=None, **kwargs):
            super(Disk.WrongDiskTypeError, self).__init__(
                disk, params, type=disk.type, name=disk.name, pk=disk.pk)

    class DiskInUseError(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it is in use.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) because it is in use.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
141
            super(Disk.DiskInUseError, self).__init__(
142 143 144 145 146 147 148 149 150 151 152 153 154
                disk, params, name=disk.name, pk=disk.pk)

    class DiskIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because it has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because it has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
Guba Sándor committed
155
            super(Disk.DiskIsNotReady, self).__init__(
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename)

    class DiskBaseIsNotReady(DiskError):
        message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' because its base has never been deployed.")

        admin_message = ugettext_noop(
            "The requested operation can't be performed on "
            "disk '%(name)s' (%(pk)s) [%(filename)s] because its base "
            "'%(b_name)s' (%(b_pk)s) [%(b_filename)s] has never been"
            "deployed.")

        def __init__(self, disk, params=None, **kwargs):
171
            base = kwargs.get('base')
Guba Sándor committed
172
            super(Disk.DiskBaseIsNotReady, self).__init__(
173 174 175
                disk, params, name=disk.name, pk=disk.pk,
                filename=disk.filename, b_name=base.name,
                b_pk=base.pk, b_filename=base.filename)
Guba Sándor committed
176

177 178
    @property
    def path(self):
179 180
        """The path where the files are stored.
        """
181
        return join(self.datastore.path, self.filename)
182 183

    @property
184
    def vm_format(self):
185 186
        """Returns the proper file format for different type of images.
        """
187 188 189
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
190
            'iso': 'raw',
191 192 193 194
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

195
    @property
196
    def format(self):
197 198
        """Returns the proper file format for different types of images.
        """
199 200 201 202 203 204 205 206 207
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
208
    def device_type(self):
209 210
        """Returns the proper device prefix for different types of images.
        """
211
        return {
212 213
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
214
            'iso': 'sd',
215 216 217
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
218

219
    @property
220 221 222 223 224 225
    def device_bus(self):
        """Returns the proper device prefix for different types of images.
        """
        return {
            'qcow2-norm': 'virtio',
            'qcow2-snap': 'virtio',
226
            'iso': 'ide',
227 228 229 230 231
            'raw-ro': 'virtio',
            'raw-rw': 'virtio',
        }[self.type]

    @property
232
    def is_deletable(self):
233
        """True if the associated file can be deleted.
234
        """
235
        # Check if all children and the disk itself is destroyed.
236
        return (self.destroyed is not None) and self.children_deletable
237

238 239 240
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
241
        """
242
        return all(i.is_deletable for i in self.derivatives.all())
243

244
    @property
245
    def is_in_use(self):
246
        """True if disk is attached to an active VM.
247 248 249 250

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
251
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
252

253
    def get_appliance(self):
Bach Dániel committed
254 255
        """Return the Instance or InstanceTemplate object where the disk
        is used
256
        """
Bach Dániel committed
257 258 259 260 261
        from vm.models import Instance
        try:
            return self.instance_set.get()
        except Instance.DoesNotExist:
            return self.template_set.get()
262

263 264
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
265

266 267 268
        This method manipulates the database only.
        """
        type_mapping = {
269 270 271
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
272 273 274
        }

        if self.type not in type_mapping.keys():
275
            raise self.WrongDiskTypeError(self)
276 277

        new_type = type_mapping[self.type]
278

279 280 281
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
                           type=new_type)
282 283

    def get_vmdisk_desc(self):
284 285
        """Serialize disk object to the vmdriver.
        """
286
        return {
287
            'source': self.path,
288
            'driver_type': self.vm_format,
289
            'driver_cache': 'none',
290
            'target_device': self.device_type + self.dev_num,
291
            'target_bus': self.device_bus,
292
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
293 294
        }

295
    def get_disk_desc(self):
296 297
        """Serialize disk object to the storage driver.
        """
298 299 300 301 302 303
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
304
            'type': 'snapshot' if self.base else 'normal'
305 306
        }

307 308
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
309 310
        """Returns the proper queue name based on the datastore.
        """
311
        if self.datastore:
312 313
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
314 315 316
        else:
            return None

317
    def __unicode__(self):
318
        return u"%s (#%d)" % (self.name, self.id or 0)
319

320
    def clean(self, *args, **kwargs):
Guba Sándor committed
321
        if (self.size is None or "") and self.base:
322 323 324
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

325
    def deploy(self, user=None, task_uuid=None, timeout=15):
326 327 328 329 330
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

331 332 333 334 335 336 337
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

338 339 340 341
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
342 343 344 345
        if self.destroyed:
            self.destroyed = None
            self.save()

346
        if self.is_ready:
347
            return True
348
        if self.base and not self.base.is_ready:
349
            raise self.DiskBaseIsNotReady(self, base=self.base)
Guba Sándor committed
350 351 352 353 354 355 356 357 358 359
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
360

361 362
        self.is_ready = True
        self.save()
Guba Sándor committed
363
        return True
364

365
    @classmethod
Guba Sándor committed
366 367
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
368
        disk.clean()
369
        disk.save()
Guba Sándor committed
370
        logger.debug("Disk created: %s", params)
371
        return disk
372

373
    @classmethod
Guba Sándor committed
374 375 376 377
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
378
        return disk
379 380

    @classmethod
Guba Sándor committed
381
    def download(cls, url, task, user=None, **params):
382 383 384 385
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
386 387
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
388 389
        :param user: owner of the disk
        :type user: django.contrib.auth.User
390 391
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
392

393 394
        :return: The created Disk object
        :rtype: Disk
395
        """
Guba Sándor committed
396
        params.setdefault('name', url.split('/')[-1])
397 398 399
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
400 401
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
402
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
403 404 405 406
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
407
                result = remote.get(timeout=5)
Guba Sándor committed
408
                break
409
            except TimeoutError as e:
Guba Sándor committed
410 411
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
412 413
                    raise humanize_exception(ugettext_noop(
                        "Operation aborted by user."), e)
414 415
        disk.size = result['size']
        disk.type = result['type']
416
        disk.is_ready = True
Guba Sándor committed
417
        disk.save()
418
        return disk
419

420
    def destroy(self, user=None, task_uuid=None):
421 422 423
        if self.destroyed:
            return False

Guba Sándor committed
424 425 426
        self.destroyed = timezone.now()
        self.save()
        return True
427

428
    def restore(self, user=None, task_uuid=None, timeout=15):
429
        """Recover destroyed disk from trash if possible.
430
        """
431 432 433 434 435 436 437
        queue_name = self.datastore.get_remote_queue_name(
            'storage', priority='slow')
        logger.info("Image: %s at Datastore: %s recovered from trash." %
                    (self.filename, self.datastore.path))
        storage_tasks.recover_from_trash.apply_async(
            args=[self.datastore.path, self.filename],
            queue=queue_name).get(timeout=timeout)
438

439
    def save_as(self, task=None, user=None, task_uuid=None, timeout=300):
440 441
        """Save VM as template.

442 443 444 445
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

446 447 448
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
449
        mapping = {
450 451 452
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
453 454
        }
        if self.type not in mapping.keys():
455
            raise self.WrongDiskTypeError(self)
456

457
        if self.is_in_use:
458 459
            raise self.DiskInUseError(self)

460
        if not self.is_ready:
Guba Sándor committed
461 462
            raise self.DiskIsNotReady(self)

463 464 465
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

466
        new_type, new_base = mapping[self.type]
467

468 469
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
470
                           name=self.name, size=self.size,
471
                           type=new_type, dev_num=self.dev_num)
472

Guba Sándor committed
473
        queue_name = self.get_remote_queue_name("storage", priority="slow")
474 475 476 477 478 479 480 481 482
        remote = storage_tasks.merge.apply_async(kwargs={
            "old_json": self.get_disk_desc(),
            "new_json": disk.get_disk_desc()},
            queue=queue_name
        )  # Timeout
        while True:
            try:
                remote.get(timeout=5)
                break
483
            except TimeoutError as e:
484 485 486
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    disk.destroy()
487 488
                    raise humanize_exception(ugettext_noop(
                        "Operation aborted by user."), e)
489 490
        disk.is_ready = True
        disk.save()
Guba Sándor committed
491
        return disk