storagedriver.py 5.31 KB
Newer Older
Máhonfai Bálint committed
1
import logging
2 3
from os import path, unlink, statvfs, listdir, mkdir
from shutil import move
Máhonfai Bálint committed
4

5
from celery.contrib.abortable import AbortableTask
6

Máhonfai Bálint committed
7 8
from disk import Disk
from storagecelery import celery
9

Máhonfai Bálint committed
10
logger = logging.getLogger(__name__)
11
trash_directory = "trash"
12

Bach Dániel committed
13

14
@celery.task()
15
def list(dir):
Dudás Ádám committed
16
    return [d.get_desc() for d in Disk.list(dir)]
Bach Dániel committed
17 18


19
@celery.task()
20 21 22 23 24 25
def list_files(datastore):
    return [l for l in listdir(datastore) if
            path.isfile(path.join(datastore, l))]


@celery.task()
26
def create(disk_desc):
Dudás Ádám committed
27
    disk = Disk.deserialize(disk_desc)
28
    disk.create()
Bach Dániel committed
29 30


31 32 33 34 35 36
class download(AbortableTask):
    time_limit = 18000  # TODO: calculate proper value it's 5h now

    def run(self, **kwargs):
        disk_desc = kwargs['disk']
        url = kwargs['url']
37
        parent_id = kwargs.get("parent_id", None)
38
        disk = Disk.deserialize(disk_desc)
39
        disk.download(self, url, parent_id)
40
        return {'size': disk.size,
Guba Sándor committed
41 42
                'type': disk.format,
                'checksum': disk.checksum, }
43 44


45 46 47 48 49 50
class import_disk(AbortableTask):
    time_limit = 18000

    def run(self, **kwargs):
        disk_desc = kwargs["disk_desc"]
        url = kwargs["url"]
51 52 53 54
        if "port" in kwargs:
            port = kwargs["port"]
        else:
            port = None
55
        disk = Disk.deserialize(disk_desc)
56
        disk.import_disk(self, url, port)
57 58 59 60
        return {
            "size": disk.size,
            "checksum": disk.checksum
        }
61 62


63 64 65 66 67 68 69
class export_disk(AbortableTask):
    time_limit = 18000

    def run(self, **kwargs):
        disk_desc = kwargs["disk_desc"]
        disk_format = kwargs["disk_format"]
        upload_link = kwargs["upload_link"]
70 71 72 73
        if "port" in kwargs:
            port = kwargs["port"]
        else:
            port = None
74
        disk = Disk.deserialize(disk_desc)
75
        return disk.export(self, disk_format, upload_link, port)
76 77 78


@celery.task()
79 80
def delete(json_data):
    disk = Disk.deserialize(json_data)
81
    disk.delete()
Bach Dániel committed
82 83


84
@celery.task()
85 86
def delete_dump(disk_path):
    if disk_path.endswith(".dump") and path.isfile(disk_path):
87
        unlink(disk_path)
88 89 90


@celery.task()
Guba Sándor committed
91
def snapshot(json_data):
92
    disk = Disk.deserialize(json_data)
Guba Sándor committed
93 94 95
    disk.snapshot()


96 97 98 99 100 101
class merge(AbortableTask):
    time_limit = 18000

    def run(self, **kwargs):
        old_json = kwargs['old_json']
        new_json = kwargs['new_json']
Guba Sándor committed
102
        parent_id = kwargs.get("parent_id", None)
103 104
        disk = Disk.deserialize(old_json)
        new_disk = Disk.deserialize(new_json)
Guba Sándor committed
105
        disk.merge(self, new_disk, parent_id=parent_id)
Guba Sándor committed
106 107 108


@celery.task()
109
def get(json_data):
Guba Sándor committed
110 111
    disk = Disk.get(dir=json_data['dir'], name=json_data['name'])
    return disk.get_desc()
112 113 114


@celery.task()
115
def get_storage_stat(path):
116
    """ Return free disk space avaliable at path in bytes and percent."""
117
    s = statvfs(path)
118 119 120 121 122
    all_space = s.f_bsize * s.f_blocks
    free_space = s.f_bavail * s.f_frsize
    free_space_percent = 100.0 * free_space / all_space
    return {'free_space': free_space,
            'free_percent': free_space_percent}
123 124


125 126 127 128
@celery.task()
def get_file_statistics(datastore):
    disks = [Disk.get(datastore, name).get_desc()
             for name in listdir(datastore)
129 130
             if not name.endswith(".dump") and
             not path.isdir(path.join(datastore, name))]
131 132 133 134 135 136 137 138 139 140 141 142 143 144
    dumps = [{'name': name,
              'size': path.getsize(path.join(datastore, name))}
             for name in listdir(datastore) if name.endswith(".dump")]
    trash = [{'name': name,
              'size': path.getsize(path.join(datastore, trash_directory,
                                             name))}
             for name in listdir(path.join(datastore, trash_directory))]
    return {
        'dumps': dumps,
        'trash': trash,
        'disks': disks,
    }


145 146
@celery.task
def move_to_trash(datastore, disk_name):
147 148
    """ Move path to the trash directory.
    """
149 150 151 152
    trash_path = path.join(datastore, trash_directory)
    disk_path = path.join(datastore, disk_name)
    if not path.isdir(trash_path):
        mkdir(trash_path)
153
    # TODO: trash dir configurable?
154
    move(disk_path, trash_path)
155 156 157


@celery.task
158
def recover_from_trash(datastore, disk_name):
159 160
    """ Recover named disk from the trash directory.
    """
161 162 163
    if path.exists(path.join(datastore, disk_name)):
        return False
    disk_path = path.join(datastore, trash_directory, disk_name)
Bach Dániel committed
164
    # TODO: trash dir configurable?
165 166 167 168 169
    move(disk_path, datastore)
    return True


@celery.task
170
def make_free_space(datastore, percent=10):
171
    """ Check for free space on datastore.
172 173
        If free space is less than the given percent
        removes oldest files to satisfy the given requirement.
174
    """
175
    trash_path = path.join(datastore, trash_directory)
176 177 178 179 180 181 182 183

    def comp(filename):
        try:
            return path.getctime(path.join(trash_path, filename))
        except OSError:
            return 0

    files = sorted(listdir(trash_path), key=comp)
184 185 186 187 188 189 190 191 192 193 194
    logger.info("Free space on datastore: %s" %
                get_storage_stat(trash_path).get('free_percent'))
    while get_storage_stat(trash_path).get('free_percent') < percent:
        logger.debug(get_storage_stat(trash_path))
        try:
            f = files.pop(0)
            unlink(path.join(trash_path, f))
            logger.info('Image: %s removed.' % f)
        except IndexError:
            raise Exception("Trash folder is empty.")
    return True