periodic_tasks.py 3.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.

18 19 20
from storage.models import DataStore
from manager.mancelery import celery
import logging
21
from storage.tasks import remote_tasks
22 23 24 25 26

logger = logging.getLogger(__name__)


@celery.task
27 28
def garbage_collector(timeout=15):
    """ Garbage collector for disk images.
Őry Máté committed
29

30 31
    If there is not enough free space on datastore (default 10%)
    deletes oldest images from trash.
32

33
    :param timeout: Seconds before TimeOut exception
Guba Sándor committed
34
    :type timeout: int
35
    """
36
    for ds in DataStore.objects.all():
37
        queue_name = ds.get_remote_queue_name('storage', priority='fast')
Guba Sándor committed
38 39 40
        files = set(remote_tasks.list_files.apply_async(
            args=[ds.path], queue=queue_name).get(timeout=timeout))
        disks = set(ds.get_deletable_disks())
41
        queue_name = ds.get_remote_queue_name('storage', priority='slow')
Guba Sándor committed
42
        for i in disks & files:
43 44 45 46 47 48 49 50 51 52
            logger.info("Image: %s at Datastore: %s moved to trash folder." %
                        (i, ds.path))
            remote_tasks.move_to_trash.apply_async(
                args=[ds.path, i], queue=queue_name).get(timeout=timeout)
        try:
            remote_tasks.make_free_space.apply_async(
                args=[ds.path], queue=queue_name).get(timeout=timeout)
        except Exception as e:
            logger.warning(str(e))

53

54 55
@celery.task
def list_orphan_disks(timeout=15):
56
    """List disk image files without Disk object in the database.
Őry Máté committed
57

58 59 60 61
    Exclude cloud-xxxxxxxx.dump format images.

    :param timeout: Seconds before TimeOut exception
    :type timeoit: int
62 63
    """
    import re
64
    for ds in DataStore.objects.all():
65
        queue_name = ds.get_remote_queue_name('storage')
Őry Máté committed
66 67 68 69
        files = set(remote_tasks.list_files.apply_async(
            args=[ds.path], queue=queue_name).get(timeout=timeout))
        disks = set([disk.filename for disk in ds.disk_set.all()])
        for i in files - disks:
70 71
            if not re.match('cloud-[0-9]*\.dump', i):
                logging.warning("Orphan disk: %s" % i)
72

73 74 75

@celery.task
def list_missing_disks(timeout=15):
76 77 78 79
    """List Disk objects without disk image files.

    :param timeout: Seconds before TimeOut exception
    :type timeoit: int
80
    """
81
    for ds in DataStore.objects.all():
82
        queue_name = ds.get_remote_queue_name('storage')
Őry Máté committed
83 84 85 86 87
        files = set(remote_tasks.list_files.apply_async(
            args=[ds.path], queue=queue_name).get(timeout=timeout))
        disks = set([disk.filename for disk in
                     ds.disk_set.filter(destroyed__isnull=True)])
        for i in disks - files:
88 89
            logging.critical("Image: %s is missing from %s datastore."
                             % (i, ds.path))