models.py 14.9 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

34
from acl.models import AclBase
Guba Sándor committed
35
from .tasks import local_tasks, storage_tasks
36
from celery.exceptions import TimeoutError
Guba Sándor committed
37
from common.models import WorkerNotFound
38 39 40 41

logger = logging.getLogger(__name__)


42
class DataStore(Model):
Guba Sándor committed
43

44 45
    """Collection of virtual disks.
    """
46 47 48 49
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
50

51 52 53 54 55 56 57 58
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

59 60
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
61 62
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
63
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
64 65 66 67 68 69
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
70 71
        else:
            raise WorkerNotFound()
72

73 74 75
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
76
                    destroyed__isnull=False) if disk.is_deletable]
77

78

79
class Disk(AclBase, TimeStampedModel):
Guba Sándor committed
80

81 82
    """A virtual disk.
    """
83 84 85 86 87
    ACL_LEVELS = (
        ('user', _('user')),          # see all details
        ('operator', _('operator')),
        ('owner', _('owner')),        # superuser, can delete, delegate perms
    )
88 89
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
90
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
91 92
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
93 94 95
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
96
    size = FileSizeField(null=True, default=None)
97 98 99
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
100
                        verbose_name=_("device number"))
101
    destroyed = DateTimeField(blank=True, default=None, null=True)
102

Guba Sándor committed
103 104
    is_ready = BooleanField(default=False)

105 106 107 108
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')
109 110 111
        permissions = (
            ('create_empty_disk', _('Can create an empty disk.')),
            ('download_disk', _('Can download a disk.')))
112

113 114
    class WrongDiskTypeError(Exception):

115 116 117 118 119 120 121 122
        def __init__(self, type, message=None):
            if message is None:
                message = ("Operation can't be invoked on a disk of type '%s'."
                           % type)

            Exception.__init__(self, message)

            self.type = type
123

124 125
    class DiskInUseError(Exception):

126 127 128 129
        def __init__(self, disk, message=None):
            if message is None:
                message = ("The requested operation can't be performed on "
                           "disk '%s (%s)' because it is in use." %
Dudás Ádám committed
130
                           (disk.name, disk.filename))
131 132 133 134

            Exception.__init__(self, message)

            self.disk = disk
135

Guba Sándor committed
136
    class DiskIsNotReady(Exception):
Guba Sándor committed
137 138
        """ Exception for operations that need a deployed disk.
        """
Guba Sándor committed
139 140 141

        def __init__(self, disk, message=None):
            if message is None:
Guba Sándor committed
142
                message = ("The requested operation can't be performed on "
Guba Sándor committed
143 144 145 146 147 148 149
                           "disk '%s (%s)' because it has never been"
                           "deployed." % (disk.name, disk.filename))

            Exception.__init__(self, message)

            self.disk = disk

150 151
    @property
    def path(self):
152 153
        """The path where the files are stored.
        """
154
        return join(self.datastore.path, self.filename)
155 156

    @property
157
    def vm_format(self):
158 159
        """Returns the proper file format for different type of images.
        """
160 161 162
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
163
            'iso': 'raw',
164 165 166 167
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

168
    @property
169
    def format(self):
170 171
        """Returns the proper file format for different types of images.
        """
172 173 174 175 176 177 178 179 180
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
181
    def device_type(self):
182 183
        """Returns the proper device prefix for different types of images.
        """
184
        return {
185 186
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
187
            'iso': 'hd',
188 189 190
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
191

192
    @property
193
    def is_deletable(self):
194
        """True if the associated file can be deleted.
195
        """
196
        # Check if all children and the disk itself is destroyed.
197
        return (self.destroyed is not None) and self.children_deletable
198

199 200 201
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
202
        """
203
        return all(i.is_deletable for i in self.derivatives.all())
204

205
    @property
206
    def is_in_use(self):
207
        """True if disk is attached to an active VM.
208 209 210 211

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
212
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
213

214 215 216 217 218 219 220 221 222 223 224
    def get_appliance(self):
        """Return an Instance or InstanceTemplate object where the disk is used
        """
        instance = self.instance_set.all()
        template = self.template_set.all()
        app = list(instance) + list(template)
        if len(app) > 0:
            return app[0]
        else:
            return None

225 226
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
227

228 229 230
        This method manipulates the database only.
        """
        type_mapping = {
231 232 233
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
234 235 236 237 238 239
        }

        if self.type not in type_mapping.keys():
            raise self.WrongDiskTypeError(self.type)

        new_type = type_mapping[self.type]
240

241 242 243
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
                           type=new_type)
244 245

    def get_vmdisk_desc(self):
246 247
        """Serialize disk object to the vmdriver.
        """
248
        return {
249
            'source': self.path,
250
            'driver_type': self.vm_format,
251
            'driver_cache': 'none',
252
            'target_device': self.device_type + self.dev_num,
253
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
254 255
        }

256
    def get_disk_desc(self):
257 258
        """Serialize disk object to the storage driver.
        """
259 260 261 262 263 264
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
265
            'type': 'snapshot' if self.base else 'normal'
266 267
        }

268 269
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
270 271
        """Returns the proper queue name based on the datastore.
        """
272
        if self.datastore:
273 274
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
275 276 277
        else:
            return None

278
    def __unicode__(self):
279
        return u"%s (#%d)" % (self.name, self.id or 0)
280

281
    def clean(self, *args, **kwargs):
Guba Sándor committed
282
        if (self.size is None or "") and self.base:
283 284 285
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

286
    def deploy(self, user=None, task_uuid=None, timeout=15):
287 288 289 290 291
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

292 293 294 295 296 297 298
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

299 300 301 302
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
303 304 305 306
        if self.destroyed:
            self.destroyed = None
            self.save()

307
        if self.is_ready:
308
            return True
Guba Sándor committed
309 310 311 312 313 314 315 316 317 318
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
319

320 321
        self.is_ready = True
        self.save()
Guba Sándor committed
322
        return True
323

324
    @classmethod
Guba Sándor committed
325 326
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
327
        disk.clean()
328
        disk.save()
Guba Sándor committed
329
        logger.debug("Disk created: %s", params)
330
        return disk
331

332
    @classmethod
Guba Sándor committed
333 334 335 336
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
337
        return disk
338 339

    @classmethod
Guba Sándor committed
340
    def download(cls, url, task, user=None, **params):
341 342 343 344
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
345 346
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
347 348
        :param user: owner of the disk
        :type user: django.contrib.auth.User
349 350
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
351

352 353
        :return: The created Disk object
        :rtype: Disk
354
        """
Guba Sándor committed
355
        params.setdefault('name', url.split('/')[-1])
356 357 358
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
359 360
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
361
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
362 363 364 365
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
366
                result = remote.get(timeout=5)
Guba Sándor committed
367 368 369 370 371
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    raise Exception("Download aborted by user.")
372 373
        disk.size = result['size']
        disk.type = result['type']
374
        disk.is_ready = True
Guba Sándor committed
375
        disk.save()
376
        return disk
377

378
    def destroy(self, user=None, task_uuid=None):
379 380 381
        if self.destroyed:
            return False

Guba Sándor committed
382 383 384
        self.destroyed = timezone.now()
        self.save()
        return True
385

386
    def restore(self, user=None, task_uuid=None, timeout=15):
387
        """Recover destroyed disk from trash if possible.
388
        """
389 390 391 392 393 394 395
        queue_name = self.datastore.get_remote_queue_name(
            'storage', priority='slow')
        logger.info("Image: %s at Datastore: %s recovered from trash." %
                    (self.filename, self.datastore.path))
        storage_tasks.recover_from_trash.apply_async(
            args=[self.datastore.path, self.filename],
            queue=queue_name).get(timeout=timeout)
396

397
    def save_as(self, user=None, task_uuid=None, timeout=300):
398 399
        """Save VM as template.

400 401 402 403
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

404 405 406
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
407
        mapping = {
408 409 410
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
411 412 413 414
        }
        if self.type not in mapping.keys():
            raise self.WrongDiskTypeError(self.type)

415
        if self.is_in_use:
416 417
            raise self.DiskInUseError(self)

418
        if not self.is_ready:
Guba Sándor committed
419 420
            raise self.DiskIsNotReady(self)

421 422 423
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

424
        new_type, new_base = mapping[self.type]
425

426 427
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
428 429
                           name=self.name, size=self.size,
                           type=new_type)
430

Guba Sándor committed
431 432 433 434 435 436 437 438
        queue_name = self.get_remote_queue_name("storage", priority="slow")
        storage_tasks.merge.apply_async(args=[self.get_disk_desc(),
                                              disk.get_disk_desc()],
                                        queue=queue_name
                                        ).get()  # Timeout
        disk.is_ready = True
        disk.save()
        return disk