models.py 14.4 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21 22

import logging
23
from os.path import join
24 25
import uuid

Guba Sándor committed
26 27
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
28
                              ForeignKey)
29
from django.utils import timezone
30
from django.utils.translation import ugettext_lazy as _
31
from model_utils.models import TimeStampedModel
32
from sizefield.models import FileSizeField
33

34
from acl.models import AclBase
Guba Sándor committed
35
from .tasks import local_tasks, storage_tasks
36
from celery.exceptions import TimeoutError
Guba Sándor committed
37
from common.models import WorkerNotFound
38 39 40 41

logger = logging.getLogger(__name__)


42
class DataStore(Model):
Guba Sándor committed
43

44 45
    """Collection of virtual disks.
    """
46 47 48 49
    name = CharField(max_length=100, unique=True, verbose_name=_('name'))
    path = CharField(max_length=200, unique=True, verbose_name=_('path'))
    hostname = CharField(max_length=40, unique=True,
                         verbose_name=_('hostname'))
Guba Sándor committed
50

51 52 53 54 55 56 57 58
    class Meta:
        ordering = ['name']
        verbose_name = _('datastore')
        verbose_name_plural = _('datastores')

    def __unicode__(self):
        return u'%s (%s)' % (self.name, self.path)

59 60
    def get_remote_queue_name(self, queue_id, priority=None,
                              check_worker=True):
61 62
        logger.debug("Checking for storage queue %s.%s",
                     self.hostname, queue_id)
63
        if not check_worker or local_tasks.check_queue(self.hostname,
Guba Sándor committed
64 65 66 67 68 69
                                                       queue_id,
                                                       priority):
            queue_name = self.hostname + '.' + queue_id
            if priority is not None:
                queue_name = queue_name + '.' + priority
            return queue_name
70 71
        else:
            raise WorkerNotFound()
72

73 74 75
    def get_deletable_disks(self):
        return [disk.filename for disk in
                self.disk_set.filter(
76
                    destroyed__isnull=False) if disk.is_deletable]
77

78

79
class Disk(AclBase, TimeStampedModel):
Guba Sándor committed
80

81 82
    """A virtual disk.
    """
83 84 85 86 87
    ACL_LEVELS = (
        ('user', _('user')),          # see all details
        ('operator', _('operator')),
        ('owner', _('owner')),        # superuser, can delete, delegate perms
    )
88 89
    TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
             ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
90
    name = CharField(blank=True, max_length=100, verbose_name=_("name"))
91 92
    filename = CharField(max_length=256, unique=True,
                         verbose_name=_("filename"))
93 94 95
    datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
                           help_text=_("The datastore that holds the disk."))
    type = CharField(max_length=10, choices=TYPES)
96
    size = FileSizeField(null=True, default=None)
97 98 99
    base = ForeignKey('self', blank=True, null=True,
                      related_name='derivatives')
    dev_num = CharField(default='a', max_length=1,
100
                        verbose_name=_("device number"))
101
    destroyed = DateTimeField(blank=True, default=None, null=True)
102

Guba Sándor committed
103 104
    is_ready = BooleanField(default=False)

105 106 107 108 109
    class Meta:
        ordering = ['name']
        verbose_name = _('disk')
        verbose_name_plural = _('disks')

110 111
    class WrongDiskTypeError(Exception):

112 113 114 115 116 117 118 119
        def __init__(self, type, message=None):
            if message is None:
                message = ("Operation can't be invoked on a disk of type '%s'."
                           % type)

            Exception.__init__(self, message)

            self.type = type
120

121 122
    class DiskInUseError(Exception):

123 124 125 126
        def __init__(self, disk, message=None):
            if message is None:
                message = ("The requested operation can't be performed on "
                           "disk '%s (%s)' because it is in use." %
Dudás Ádám committed
127
                           (disk.name, disk.filename))
128 129 130 131

            Exception.__init__(self, message)

            self.disk = disk
132

Guba Sándor committed
133
    class DiskIsNotReady(Exception):
Guba Sándor committed
134 135
        """ Exception for operations that need a deployed disk.
        """
Guba Sándor committed
136 137 138

        def __init__(self, disk, message=None):
            if message is None:
Guba Sándor committed
139
                message = ("The requested operation can't be performed on "
Guba Sándor committed
140 141 142 143 144 145 146
                           "disk '%s (%s)' because it has never been"
                           "deployed." % (disk.name, disk.filename))

            Exception.__init__(self, message)

            self.disk = disk

147 148
    @property
    def path(self):
149 150
        """The path where the files are stored.
        """
151
        return join(self.datastore.path, self.filename)
152 153

    @property
154
    def vm_format(self):
155 156
        """Returns the proper file format for different type of images.
        """
157 158 159
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
160
            'iso': 'raw',
161 162 163 164
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

165
    @property
166
    def format(self):
167 168
        """Returns the proper file format for different types of images.
        """
169 170 171 172 173 174 175 176 177
        return {
            'qcow2-norm': 'qcow2',
            'qcow2-snap': 'qcow2',
            'iso': 'iso',
            'raw-ro': 'raw',
            'raw-rw': 'raw',
        }[self.type]

    @property
178
    def device_type(self):
179 180
        """Returns the proper device prefix for different types of images.
        """
181
        return {
182 183
            'qcow2-norm': 'vd',
            'qcow2-snap': 'vd',
184
            'iso': 'hd',
185 186 187
            'raw-ro': 'vd',
            'raw-rw': 'vd',
        }[self.type]
188

189
    @property
190
    def is_deletable(self):
191
        """True if the associated file can be deleted.
192
        """
193
        # Check if all children and the disk itself is destroyed.
194
        return (self.destroyed is not None) and self.children_deletable
195

196 197 198
    @property
    def children_deletable(self):
        """True if all children of the disk are deletable.
199
        """
200
        return all(i.is_deletable for i in self.derivatives.all())
201

202
    @property
203
    def is_in_use(self):
204
        """True if disk is attached to an active VM.
205 206 207 208

        'In use' means the disk is attached to a VM which is not STOPPED, as
        any other VMs leave the disk in an inconsistent state.
        """
209
        return any(i.state != 'STOPPED' for i in self.instance_set.all())
210

211 212 213 214 215 216 217 218 219 220 221
    def get_appliance(self):
        """Return an Instance or InstanceTemplate object where the disk is used
        """
        instance = self.instance_set.all()
        template = self.template_set.all()
        app = list(instance) + list(template)
        if len(app) > 0:
            return app[0]
        else:
            return None

222 223
    def get_exclusive(self):
        """Get an instance of the disk for exclusive usage.
224

225 226 227
        This method manipulates the database only.
        """
        type_mapping = {
228 229 230
            'qcow2-norm': 'qcow2-snap',
            'iso': 'iso',
            'raw-ro': 'raw-rw',
231 232 233 234 235 236
        }

        if self.type not in type_mapping.keys():
            raise self.WrongDiskTypeError(self.type)

        new_type = type_mapping[self.type]
237

238 239 240
        return Disk.create(base=self, datastore=self.datastore,
                           name=self.name, size=self.size,
                           type=new_type)
241 242

    def get_vmdisk_desc(self):
243 244
        """Serialize disk object to the vmdriver.
        """
245
        return {
246
            'source': self.path,
247
            'driver_type': self.vm_format,
248
            'driver_cache': 'none',
249
            'target_device': self.device_type + self.dev_num,
250
            'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
251 252
        }

253
    def get_disk_desc(self):
254 255
        """Serialize disk object to the storage driver.
        """
256 257 258 259 260 261
        return {
            'name': self.filename,
            'dir': self.datastore.path,
            'format': self.format,
            'size': self.size,
            'base_name': self.base.filename if self.base else None,
262
            'type': 'snapshot' if self.base else 'normal'
263 264
        }

265 266
    def get_remote_queue_name(self, queue_id='storage', priority=None,
                              check_worker=True):
267 268
        """Returns the proper queue name based on the datastore.
        """
269
        if self.datastore:
270 271
            return self.datastore.get_remote_queue_name(queue_id, priority,
                                                        check_worker)
272 273 274
        else:
            return None

275
    def __unicode__(self):
276
        return u"%s (#%d)" % (self.name, self.id or 0)
277

278
    def clean(self, *args, **kwargs):
Guba Sándor committed
279
        if (self.size is None or "") and self.base:
280 281 282
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)

283
    def deploy(self, user=None, task_uuid=None, timeout=15):
284 285 286 287 288
        """Reify the disk model on the associated data store.

        :param self: the disk model to reify
        :type self: storage.models.Disk

289 290 291 292 293 294 295
        :param user: The user who's issuing the command.
        :type user: django.contrib.auth.models.User

        :param task_uuid: The task's UUID, if the command is being executed
                          asynchronously.
        :type task_uuid: str

296 297 298 299
        :return: True if a new reification of the disk has been created;
                 otherwise, False.
        :rtype: bool
        """
300 301 302 303
        if self.destroyed:
            self.destroyed = None
            self.save()

304
        if self.is_ready:
305
            return True
Guba Sándor committed
306 307 308 309 310 311 312 313 314 315
        queue_name = self.get_remote_queue_name('storage', priority="fast")
        disk_desc = self.get_disk_desc()
        if self.base is not None:
            storage_tasks.snapshot.apply_async(args=[disk_desc],
                                               queue=queue_name
                                               ).get(timeout=timeout)
        else:
            storage_tasks.create.apply_async(args=[disk_desc],
                                             queue=queue_name
                                             ).get(timeout=timeout)
316

317 318
        self.is_ready = True
        self.save()
Guba Sándor committed
319
        return True
320

321
    @classmethod
Guba Sándor committed
322 323
    def create(cls, user=None, **params):
        disk = cls.__create(user, params)
Guba Sándor committed
324
        disk.clean()
325
        disk.save()
Guba Sándor committed
326
        logger.debug("Disk created: %s", params)
327
        return disk
328

329
    @classmethod
Guba Sándor committed
330 331 332 333
    def __create(cls, user, params):
        datastore = params.pop('datastore', DataStore.objects.get())
        filename = params.pop('filename', str(uuid.uuid4()))
        disk = cls(filename=filename, datastore=datastore, **params)
334
        return disk
335 336

    @classmethod
Guba Sándor committed
337
    def download(cls, url, task, user=None, **params):
338 339 340 341
        """Create disk object and download data from url synchronusly.

        :param url: image url to download.
        :type url: url
342 343
        :param instance: Instance or template attach the Disk to.
        :type instance: vm.models.Instance or InstanceTemplate or NoneType
344 345
        :param user: owner of the disk
        :type user: django.contrib.auth.User
346 347
        :param task_uuid: UUID of the local task
        :param abortable_task: UUID of the remote running abortable task.
348

349 350
        :return: The created Disk object
        :rtype: Disk
351
        """
Guba Sándor committed
352
        params.setdefault('name', url.split('/')[-1])
353 354 355
        params.setdefault('type', 'iso')
        params.setdefault('size', None)
        disk = cls.__create(params=params, user=user)
Guba Sándor committed
356 357
        queue_name = disk.get_remote_queue_name('storage', priority='slow')
        remote = storage_tasks.download.apply_async(
358
            kwargs={'url': url, 'parent_id': task.request.id,
Guba Sándor committed
359 360 361 362 363 364 365 366 367 368 369
                    'disk': disk.get_disk_desc()},
            queue=queue_name)
        while True:
            try:
                size = remote.get(timeout=5)
                break
            except TimeoutError:
                if task is not None and task.is_aborted():
                    AbortableAsyncResult(remote.id).abort()
                    raise Exception("Download aborted by user.")
        disk.size = size
370
        disk.is_ready = True
Guba Sándor committed
371
        disk.save()
372
        return disk
373

374
    def destroy(self, user=None, task_uuid=None):
375 376 377
        if self.destroyed:
            return False

Guba Sándor committed
378 379 380
        self.destroyed = timezone.now()
        self.save()
        return True
381

382
    def restore(self, user=None, task_uuid=None):
383
        """Recover destroyed disk from trash if possible.
384 385 386 387
        """
        # TODO
        pass

388
    def save_as(self, user=None, task_uuid=None, timeout=300):
389 390
        """Save VM as template.

391 392 393 394
        Based on disk type:
        qcow2-norm, qcow2-snap --> qcow2-norm
        iso                    --> iso (with base)

395 396 397
        VM must be in STOPPED state to perform this action.
        The timeout parameter is not used now.
        """
398
        mapping = {
399 400 401
            'qcow2-snap': ('qcow2-norm', None),
            'qcow2-norm': ('qcow2-norm', None),
            'iso': ("iso", self),
402 403 404 405
        }
        if self.type not in mapping.keys():
            raise self.WrongDiskTypeError(self.type)

406
        if self.is_in_use:
407 408
            raise self.DiskInUseError(self)

409
        if not self.is_ready:
Guba Sándor committed
410 411
            raise self.DiskIsNotReady(self)

412 413 414
        # from this point on, the caller has to guarantee that the disk is not
        # going to be used until the operation is complete

415
        new_type, new_base = mapping[self.type]
416

417 418
        disk = Disk.create(datastore=self.datastore,
                           base=new_base,
419 420
                           name=self.name, size=self.size,
                           type=new_type)
421

Guba Sándor committed
422 423 424 425 426 427 428 429
        queue_name = self.get_remote_queue_name("storage", priority="slow")
        storage_tasks.merge.apply_async(args=[self.get_disk_desc(),
                                              disk.get_disk_desc()],
                                        queue=queue_name
                                        ).get()  # Timeout
        disk.is_ready = True
        disk.save()
        return disk