Initial version, Python 2.7

.gitignore

+*.pyc
+*.swp
+.ropeproject
+.vscode
+.idea

docs/install.rst

+Installation of a storage machine
+=================================
+
+.. highlight:: bash
+
+Setting up required software
+----------------------------
+Update the package lists, and install the required system software::
+
+ $ sudo apt-get update
+ $ sudo apt-get install --yes qemu-utils virtualenvwrapper git \
+     python-pip
+
+Setting up NFS share
+--------------------
+Install the nfs-server package:: 
+ 
+ $ sudo apt-get install nfs-server
+
+Create the datastore directory::
+
+ $ sudo mkdir /datastore
+ $ sudo chown cloud:cloud /datastore
+
+Edit NFS exports::
+
+ $ sudo vim /etc/exports
+
+Restart the nfs service::
+
+ $ sudo /etc/init.d/nfs-kernel-server restart
+
+Setting up Storage itself
+-------------------------
+Clone the git repository::
+
+ $ git clone git@git.ik.bme.hu:circle/storagedriver.git storagedriver
+
+Set up *virtualenvwrapper* and the *virtual Python environment* for the
+project::
+
+  $ source /etc/bash_completion.d/virtualenvwrapper
+  $ mkvirtualenv storage
+
+Set up default Storage configuration and activate the virtual environment::
+
+  $ cat >>/home/cloud/.virtualenvs/storage/bin/postactivate <<END
+  export AMQP_URI='amqp://cloud:password@host:5672/circle'
+  END
+  $ workon storage
+  $ cd ~/storagedriver
+
+Install the required Python libraries to the virtual environment::
+
+  $ pip install -r requirements/local.txt
+
+Copy the upstart scripts for celery services::
+
+  $ sudo cp miscellaneous/storagecelery.conf /etc/init/
+
+Start celery daemons::
+
+  $ sudo start storagecelery

miscellaneous/storage.conf

+description "IK Cloud Django Development Server"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+pre-start script
+    hostname=$(hostname -s)
+    for inst in storage.fast storage.slow
+    do
+        start storagecelery NAME=$hostname.$inst || :
+    done
+end script
+
+post-stop script
+    for inst in `initctl list|grep "^storagecelery "|awk '{print $2}'|tr -d ')'|tr -d '('`
+    do
+        stop storagecelery NAME=$inst || :
+    done
+end script

miscellaneous/storage.service

+[Unit]
+Description=CIRCLE storage
+After=network.target
+BindsTo=storagecelery@storage.fast.service
+BindsTo=storagecelery@storage.slow.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/echo
+
+[Install]
+WantedBy=multi-user.target

miscellaneous/storagecelery.conf

+description	"IK Cloud Django Development Server"
+
+respawn
+respawn limit 30 30
+setuid cloud
+setgid cloud
+
+instance $NAME
+
+script
+    cd /home/cloud/storagedriver/
+    . /home/cloud/.virtualenvs/storagedriver/local/bin/activate
+    . /home/cloud/.virtualenvs/storagedriver/local/bin/postactivate
+    exec celery -A storagecelery worker --loglevel=info -n $NAME
+end script

miscellaneous/storagecelery@.service

+[Unit]
+Description=storagecelery %I
+BindsTo=storage.service
+
+[Service]
+User=cloud
+Group=cloud
+
+KillSignal=SIGTERM
+TimeoutStopSec=600
+Restart=always
+
+WorkingDirectory=/home/cloud/storagedriver
+
+ExecStart=/bin/bash -c "source /etc/profile; workon storagedriver; exec celery -A storagecelery worker --loglevel=info -n $(/bin/hostname -s).%I"

requirements/base.txt

+celery==3.1.17
+requests==2.5.3
+filemagic==1.6

requirements/local.txt

+-r base.txt

requirements/production.txt

+-r base.txt

storagecelery.py

+from celery import Celery
+from kombu import Queue, Exchange
+from os import getenv
+from argparse import ArgumentParser
+
+
+parser = ArgumentParser()
+parser.add_argument("-n", "--hostname", dest="hostname",
+                    help="Define the full queue name with"
+                    "with priority", metavar="hostname.queue.priority")
+(args, unknwon_args) = parser.parse_known_args()
+HOSTNAME = vars(args).pop("hostname")
+if HOSTNAME is None:
+    raise Exception("You must define hostname as -n <hostname> or "
+                    "--hostname=<hostname>.\n"
+                    "Hostname format must be hostname.module.priority.")
+
+AMQP_URI = getenv('AMQP_URI')
+
+celery = Celery('storagedriver',
+                broker=AMQP_URI,
+                include=['storagedriver'])
+
+celery.conf.update(
+    CELERY_RESULT_BACKEND='amqp',
+    CELERY_TASK_RESULT_EXPIRES=300,
+    CELERY_QUEUES=(
+        Queue(HOSTNAME, Exchange(
+            'storagedriver', type='direct'), routing_key='storagedriver'),
+    )
+)

storagedriver.py

+import logging
+from os import path, unlink, statvfs, listdir, mkdir
+from shutil import move
+
+from celery.contrib.abortable import AbortableTask
+
+from disk import Disk
+from storagecelery import celery
+
+logger = logging.getLogger(__name__)
+trash_directory = "trash"
+
+
+@celery.task()
+def list(dir):
+    return [d.get_desc() for d in Disk.list(dir)]
+
+
+@celery.task()
+def list_files(datastore):
+    return [l for l in listdir(datastore) if
+            path.isfile(path.join(datastore, l))]
+
+
+@celery.task()
+def create(disk_desc):
+    disk = Disk.deserialize(disk_desc)
+    disk.create()
+
+
+class download(AbortableTask):
+    time_limit = 18000  # TODO: calculate proper value it's 5h now
+
+    def run(self, **kwargs):
+        disk_desc = kwargs['disk']
+        url = kwargs['url']
+        parent_id = kwargs.get("parent_id", None)
+        disk = Disk.deserialize(disk_desc)
+        disk.download(self, url, parent_id)
+        return {'size': disk.size,
+                'type': disk.format,
+                'checksum': disk.checksum, }
+
+
+class import_disk(AbortableTask):
+    time_limit = 18000
+
+    def run(self, **kwargs):
+        disk_desc = kwargs["disk_desc"]
+        url = kwargs["url"]
+        port = kwargs["port"]
+        disk = Disk.deserialize(disk_desc)
+        disk.import_disk(self, url, port)
+        return {
+            "size": disk.size,
+            "checksum": disk.checksum
+        }
+
+
+class export_disk(AbortableTask):
+    time_limit = 18000
+
+    def run(self, **kwargs):
+        disk_desc = kwargs["disk_desc"]
+        disk_format = kwargs["disk_format"]
+        upload_link = kwargs["upload_link"]
+        port = kwargs["port"]
+        disk = Disk.deserialize(disk_desc)
+        return disk.export(self, disk_format, upload_link, port)
+
+
+@celery.task()
+def delete(json_data):
+    disk = Disk.deserialize(json_data)
+    disk.delete()
+
+
+@celery.task()
+def delete_dump(disk_path):
+    if disk_path.endswith(".dump") and path.isfile(disk_path):
+        unlink(disk_path)
+
+
+@celery.task()
+def snapshot(json_data):
+    disk = Disk.deserialize(json_data)
+    disk.snapshot()
+
+
+class merge(AbortableTask):
+    time_limit = 18000
+
+    def run(self, **kwargs):
+        old_json = kwargs['old_json']
+        new_json = kwargs['new_json']
+        parent_id = kwargs.get("parent_id", None)
+        disk = Disk.deserialize(old_json)
+        new_disk = Disk.deserialize(new_json)
+        disk.merge(self, new_disk, parent_id=parent_id)
+
+
+@celery.task()
+def get(json_data):
+    disk = Disk.get(dir=json_data['dir'], name=json_data['name'])
+    return disk.get_desc()
+
+
+@celery.task()
+def get_storage_stat(path):
+    """ Return free disk space avaliable at path in bytes and percent."""
+    s = statvfs(path)
+    all_space = s.f_bsize * s.f_blocks
+    free_space = s.f_bavail * s.f_frsize
+    free_space_percent = 100.0 * free_space / all_space
+    return {'free_space': free_space,
+            'free_percent': free_space_percent}
+
+
+@celery.task()
+def get_file_statistics(datastore):
+    disks = [Disk.get(datastore, name).get_desc()
+             for name in listdir(datastore)
+             if not name.endswith(".dump") and
+             not path.isdir(path.join(datastore, name))]
+    dumps = [{'name': name,
+              'size': path.getsize(path.join(datastore, name))}
+             for name in listdir(datastore) if name.endswith(".dump")]
+    trash = [{'name': name,
+              'size': path.getsize(path.join(datastore, trash_directory,
+                                             name))}
+             for name in listdir(path.join(datastore, trash_directory))]
+    return {
+        'dumps': dumps,
+        'trash': trash,
+        'disks': disks,
+    }
+
+
+@celery.task
+def move_to_trash(datastore, disk_name):
+    """ Move path to the trash directory.
+    """
+    trash_path = path.join(datastore, trash_directory)
+    disk_path = path.join(datastore, disk_name)
+    if not path.isdir(trash_path):
+        mkdir(trash_path)
+    # TODO: trash dir configurable?
+    move(disk_path, trash_path)
+
+
+@celery.task
+def recover_from_trash(datastore, disk_name):
+    """ Recover named disk from the trash directory.
+    """
+    if path.exists(path.join(datastore, disk_name)):
+        return False
+    disk_path = path.join(datastore, trash_directory, disk_name)
+    # TODO: trash dir configurable?
+    move(disk_path, datastore)
+    return True
+
+
+@celery.task
+def make_free_space(datastore, percent=10):
+    """ Check for free space on datastore.
+        If free space is less than the given percent
+        removes oldest files to satisfy the given requirement.
+    """
+    trash_path = path.join(datastore, trash_directory)
+
+    def comp(filename):
+        try:
+            return path.getctime(path.join(trash_path, filename))
+        except OSError:
+            return 0
+
+    files = sorted(listdir(trash_path), key=comp)
+    logger.info("Free space on datastore: %s" %
+                get_storage_stat(trash_path).get('free_percent'))
+    while get_storage_stat(trash_path).get('free_percent') < percent:
+        logger.debug(get_storage_stat(trash_path))
+        try:
+            f = files.pop(0)
+            unlink(path.join(trash_path, f))
+            logger.info('Image: %s removed.' % f)
+        except IndexError:
+            raise Exception("Trash folder is empty.")
+    return True