models.py 15.7 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

34
from acl.models import AclBase
Guba Sándor committed
35
from .tasks import local_tasks, storage_tasks
36
from celery.exceptions import TimeoutError
Guba Sándor committed
37
from common.models import WorkerNotFound
38 39 40 41

logger = logging.getLogger(__name__)


42
class DataStore(Model):
Guba Sándor committed
43

44 45
    """Collection of virtual disks.
    """
46 47 48 49
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
50

51 52 53 54 55 56 57 58
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

59 60
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
61 62
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
63
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
64 65 66 67 68 69
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
70 71
        else:
            raise WorkerNotFound()
72

73 74 75
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
76
                    destroyed__isnull=False) if disk.is_deletable]
77

78

79
class Disk(AclBase, TimeStampedModel):
Guba Sándor committed
80

81 82
    """A virtual disk.
    """
83 84 85 86 87
    ACL_LEVELS = (
        ('user', _('user')),          # see all details
        ('operator', _('operator')),
        ('owner', _('owner')),        # superuser, can delete, delegate perms
    )
88 89
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
90
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
91 92
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
93 94 95
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
96
    size = FileSizeField(null=True, default=None)
97 98 99
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
100
                        verbose_name=_("device number"))
101
    destroyed = DateTimeField(blank=True, default=None, null=True)
102

Guba Sándor committed
103 104
    is_ready = BooleanField(default=False)

105 106 107 108
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')
109 110 111
        permissions = (
            ('create_empty_disk', _('Can create an empty disk.')),
            ('download_disk', _('Can download a disk.')))
112

113 114
    class WrongDiskTypeError(Exception):

115 116 117 118 119 120 121 122
        def __init__(self, type, message=None):
            if message is None:
                message = ("Operation can't be invoked on a disk of type '%s'."
                           % type)

            Exception.__init__(self, message)

            self.type = type
123

124 125
    class DiskInUseError(Exception):

126 127 128 129
        def __init__(self, disk, message=None):
            if message is None:
                message = ("The requested operation can't be performed on "
                           "disk '%s (%s)' because it is in use." %
Dudás Ádám committed
130
                           (disk.name, disk.filename))
131 132 133 134

            Exception.__init__(self, message)

            self.disk = disk
135

Guba Sándor committed
136
    class DiskIsNotReady(Exception):
137

Guba Sándor committed
138 139
        """ Exception for operations that need a deployed disk.
        """
Guba Sándor committed
140 141 142

        def __init__(self, disk, message=None):
            if message is None:
Guba Sándor committed
143
                message = ("The requested operation can't be performed on "
Guba Sándor committed
144 145 146 147 148 149 150
                           "disk '%s (%s)' because it has never been"
                           "deployed." % (disk.name, disk.filename))

            Exception.__init__(self, message)

            self.disk = disk

151 152
    @property
    def path(self):
153 154
        """The path where the files are stored.
        """
155
        return join(self.datastore.path, self.filename)
156 157

    @property
158
    def vm_format(self):
159 160
        """Returns the proper file format for different type of images.
        """
161 162 163
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
164
            'iso': 'raw',
165 166 167 168
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

169
    @property
170
    def format(self):
171 172
        """Returns the proper file format for different types of images.
        """
173 174 175 176 177 178 179 180 181
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
182
    def device_type(self):
183 184
        """Returns the proper device prefix for different types of images.
        """
185
        return {
186 187
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
188
            'iso': 'sd',
189 190 191
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
192

193
    @property
194 195 196 197 198 199 200 201 202 203 204 205
    def device_bus(self):
        """Returns the proper device prefix for different types of images.
        """
        return {
            'qcow2-norm': 'virtio',
            'qcow2-snap': 'virtio',
            'iso': 'scsi',
            'raw-ro': 'virtio',
            'raw-rw': 'virtio',
        }[self.type]

    @property
206
    def is_deletable(self):
207
        """True if the associated file can be deleted.
208
        """
209
        # Check if all children and the disk itself is destroyed.
210
        return (self.destroyed is not None) and self.children_deletable
211

212 213 214
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
215
        """
216
        return all(i.is_deletable for i in self.derivatives.all())
217

218
    @property
219
    def is_in_use(self):
220
        """True if disk is attached to an active VM.
221 222 223 224

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
225
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
226

227 228 229 230 231 232 233 234 235 236 237
    def get_appliance(self):
        """Return an Instance or InstanceTemplate object where the disk is used
        """
        instance = self.instance_set.all()
        template = self.template_set.all()
        app = list(instance) + list(template)
        if len(app) > 0:
            return app[0]
        else:
            return None

238 239
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
240

241 242 243
        This method manipulates the database only.
        """
        type_mapping = {
244 245 246
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
247 248 249 250 251 252
        }

        if self.type not in type_mapping.keys():
            raise self.WrongDiskTypeError(self.type)

        new_type = type_mapping[self.type]
253

254 255 256
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
                           type=new_type)
257 258

    def get_vmdisk_desc(self):
259 260
        """Serialize disk object to the vmdriver.
        """
261
        return {
262
            'source': self.path,
263
            'driver_type': self.vm_format,
264
            'driver_cache': 'none',
265
            'target_device': self.device_type + self.dev_num,
266
            'target_bus': self.device_bus,
267
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
268 269
        }

270
    def get_disk_desc(self):
271 272
        """Serialize disk object to the storage driver.
        """
273 274 275 276 277 278
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
279
            'type': 'snapshot' if self.base else 'normal'
280 281
        }

282 283
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
284 285
        """Returns the proper queue name based on the datastore.
        """
286
        if self.datastore:
287 288
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
289 290 291
        else:
            return None

292
    def __unicode__(self):
293
        return u"%s (#%d)" % (self.name, self.id or 0)
294

295
    def clean(self, *args, **kwargs):
Guba Sándor committed
296
        if (self.size is None or "") and self.base:
297 298 299
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

300
    def deploy(self, user=None, task_uuid=None, timeout=15):
301 302 303 304 305
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

306 307 308 309 310 311 312
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

313 314 315 316
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
317 318 319 320
        if self.destroyed:
            self.destroyed = None
            self.save()

321
        if self.is_ready:
322
            return True
323 324
        if self.base and not self.base.is_ready:
            raise Exception("Base image is not ready.")
Guba Sándor committed
325 326 327 328 329 330 331 332 333 334
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
335

336 337
        self.is_ready = True
        self.save()
Guba Sándor committed
338
        return True
339

340
    @classmethod
Guba Sándor committed
341 342
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
343
        disk.clean()
344
        disk.save()
Guba Sándor committed
345
        logger.debug("Disk created: %s", params)
346
        return disk
347

348
    @classmethod
Guba Sándor committed
349 350 351 352
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
353
        return disk
354 355

    @classmethod
Guba Sándor committed
356
    def download(cls, url, task, user=None, **params):
357 358 359 360
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
361 362
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
363 364
        :param user: owner of the disk
        :type user: django.contrib.auth.User
365 366
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
367

368 369
        :return: The created Disk object
        :rtype: Disk
370
        """
Guba Sándor committed
371
        params.setdefault('name', url.split('/')[-1])
372 373 374
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
375 376
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
377
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
378 379 380 381
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
382
                result = remote.get(timeout=5)
Guba Sándor committed
383 384 385 386 387
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    raise Exception("Download aborted by user.")
388 389
        disk.size = result['size']
        disk.type = result['type']
390
        disk.is_ready = True
Guba Sándor committed
391
        disk.save()
392
        return disk
393

394
    def destroy(self, user=None, task_uuid=None):
395 396 397
        if self.destroyed:
            return False

Guba Sándor committed
398 399 400
        self.destroyed = timezone.now()
        self.save()
        return True
401

402
    def restore(self, user=None, task_uuid=None, timeout=15):
403
        """Recover destroyed disk from trash if possible.
404
        """
405 406 407 408 409 410 411
        queue_name = self.datastore.get_remote_queue_name(
            'storage', priority='slow')
        logger.info("Image: %s at Datastore: %s recovered from trash." %
                    (self.filename, self.datastore.path))
        storage_tasks.recover_from_trash.apply_async(
            args=[self.datastore.path, self.filename],
            queue=queue_name).get(timeout=timeout)
412

413
    def save_as(self, task=None, user=None, task_uuid=None, timeout=300):
414 415
        """Save VM as template.

416 417 418 419
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

420 421 422
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
423
        mapping = {
424 425 426
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
427 428 429 430
        }
        if self.type not in mapping.keys():
            raise self.WrongDiskTypeError(self.type)

431
        if self.is_in_use:
432 433
            raise self.DiskInUseError(self)

434
        if not self.is_ready:
Guba Sándor committed
435 436
            raise self.DiskIsNotReady(self)

437 438 439
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

440
        new_type, new_base = mapping[self.type]
441

442 443
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
444
                           name=self.name, size=self.size,
445
                           type=new_type, dev_num=self.dev_num)
446

Guba Sándor committed
447
        queue_name = self.get_remote_queue_name("storage", priority="slow")
448 449 450 451 452 453 454 455 456 457 458 459 460 461
        remote = storage_tasks.merge.apply_async(kwargs={
            "old_json": self.get_disk_desc(),
            "new_json": disk.get_disk_desc()},
            queue=queue_name
        )  # Timeout
        while True:
            try:
                remote.get(timeout=5)
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    disk.destroy()
                    raise Exception("Save as aborted by use.")
462 463
        disk.is_ready = True
        disk.save()
Guba Sándor committed
464
        return disk