Merge remote-tracking branch 'origin/feature-datastore'

Conflicts:
	circle/dashboard/forms.py
	circle/dashboard/tables.py
	circle/dashboard/urls.py

circle/dashboard/forms.py

@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from datetime import timedelta
 from urlparse import urlparse
 
+from django.forms import ModelForm
 from django.contrib.auth.forms import (
     AuthenticationForm, PasswordResetForm, SetPasswordForm,
     PasswordChangeForm,
@@ -31,10 +32,12 @@ from django.core.exceptions import PermissionDenied, ValidationError
 import autocomplete_light
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import (
-    Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK,
+    Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
 )
 
 from crispy_forms.utils import render_field
+from crispy_forms.bootstrap import FormActions
+
 from django import forms
 from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
 from django.forms.widgets import TextInput, HiddenInput
@@ -51,6 +54,7 @@ from firewall.models import Vlan, Host
 from vm.models import (
     InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
 )
+from storage.models import DataStore, Disk
 from django.contrib.admin.widgets import FilteredSelectMultiple
 from django.contrib.auth.models import Permission
 from .models import Profile, GroupProfile
@@ -1544,3 +1548,36 @@ class UserListSearchForm(forms.Form):
         'class': "form-control input-tags",
         'placeholder': _("Search...")
     }))
+
+
+class DataStoreForm(ModelForm):
+
+    @property
+    def helper(self):
+        helper = FormHelper()
+        helper.layout = Layout(
+            Fieldset(
+                '',
+                'name',
+                'path',
+                'hostname',
+            ),
+            FormActions(
+                Submit('submit', _('Save')),
+            )
+        )
+        return helper
+
+    class Meta:
+        model = DataStore
+
+
+class DiskForm(ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(DiskForm, self).__init__(*args, **kwargs)
+
+        for k, v in self.fields.iteritems():
+            v.widget.attrs['readonly'] = True
+
+    class Meta:
+        model = Disk

circle/dashboard/tables.py

@@ -18,7 +18,7 @@
 from __future__ import absolute_import
 
 from django.contrib.auth.models import Group, User
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext_lazy as _, ugettext
 from django.utils.html import mark_safe
 
 from django_tables2 import Table, A
@@ -27,10 +27,36 @@ from django_tables2.columns import (
 )
 from django_sshkey.models import UserKey
 
+from storage.models import Disk
 from vm.models import Node, InstanceTemplate, Lease
 from dashboard.models import ConnectCommand
 
 
+class FileSizeColumn(Column):
+    def render(self, value):
+        from sizefield.utils import filesizeformat
+
+        size = filesizeformat(value)
+        return size
+
+
+class ApplianceColumn(TemplateColumn):
+    def render(self, *args, **kwargs):
+        value = super(ApplianceColumn, self).render(*args, **kwargs)
+
+        abbr = '<abbr title="%s">%s</abbr>'
+        appliance = kwargs['record'].get_appliance()
+        if appliance is None:
+            return value
+        elif isinstance(appliance, InstanceTemplate):
+            # Translators: [T] as Template
+            title, text = ugettext("Template"), ugettext("[T]")
+        else:
+            # Translators: [VM] as Virtual Machine
+            title, text = ugettext("Virtual machine"), ugettext("[VM]")
+        return mark_safe("%s %s" % (abbr % (title, text), value))
+
+
 class NodeListTable(Table):
 
     pk = Column(
@@ -299,3 +325,29 @@ class ConnectCommandListTable(Table):
             "You don't have any custom connection commands yet. You can "
             "specify commands to be displayed on VM detail pages instead of "
             "the defaults.")
+
+
+class DiskListTable(Table):
+    pk = LinkColumn(
+        'dashboard.views.disk-detail',
+        args=[A('pk')],
+        verbose_name=_("ID"),
+    )
+    size = FileSizeColumn()
+    appliance = ApplianceColumn(
+        template_name="dashboard/storage/column-appliance.html",
+        verbose_name=_("Appliance"),
+        orderable=False,
+    )
+    is_ready = BooleanColumn(
+        verbose_name=_("ready")
+    )
+
+    class Meta:
+        model = Disk
+        attrs = {'class': "table table-bordered table-striped table-hover",
+                 'id': "disk-list-table"}
+        fields = ("pk", "appliance", "filename", "size", "is_ready")
+        prefix = "disk-"
+        order_by = ("-pk", )
+        per_page = 65536

circle/dashboard/templates/dashboard/base.html

@@ -24,6 +24,11 @@
       <a href="/admin/"><i class="fa fa-cogs"></i> {% trans "Admin" %}</a>
     </li>
     <li>
+      <a href="{% url "dashboard.views.storage" %}"><i class="fa fa-database"></i>
+        {% trans "Storage" %}
+      </a>
+    </li>
+    <li>
       <a href="/network/"><i class="fa fa-globe"></i> {% trans "Network" %}</a>
     </li>
     {% endif %}

circle/dashboard/templates/dashboard/storage/column-appliance.html

+{% load i18n %}
+
+{% with app=record.get_appliance %}
+  {% if app %}
+    <a href="{{ app.get_absolute_url }}">{{ app.name }}</a>
+  {% else %}
+    —
+  {% endif %}
+{% endwith %}

circle/dashboard/templates/dashboard/storage/detail.html

+{% extends "dashboard/base.html" %}
+{% load staticfiles %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+{% load crispy_forms_tags %}
+
+{% block title-page %}{% trans "Storage" %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+  <div class="col-md-5">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Datastore" %}</h3>
+      </div>
+      <div class="panel-body">
+        {% crispy form %}
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+  <div class="col-md-7">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-bar-chart"></i> {% trans "Statistics" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="progress">
+          <div class="progress-bar progress-bar-success progress-bar-stripped"
+            role="progressbar" style="min-width: 30px; width: {{ stats.used_percent }}%">
+            {{ stats.used_percent }}%
+          </div>
+        </div>
+        <div class="text-muted text-center">
+          {{ stats.used_space}}/{{ stats.total_space }}
+        </div>
+
+        <h3>
+          {% trans "Missing disks" %}
+          <small>{% trans "disk objects without images files" %}</small>
+        </h3>
+        {% for m in missing_disks %}
+          <p>
+          {{ m }} - {{ m.filename }}
+          </p>
+          {% empty %}
+          {% trans "None" %}
+        {% endfor %}
+
+        <h3>
+          {% trans "Orphan disks" %}
+          <small>{% trans "image files without disk object in the database" %}</small>
+        </h3>
+        {% for o in orphan_disks %}
+          {{ o }}
+          {% empty %}
+          {% trans "None" %}
+        {% endfor %}
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-file"></i> {% trans "Disks" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="table-responsive">
+          {% render_table disk_table %}
+        </div>
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+{% endblock %}

circle/dashboard/templates/dashboard/storage/disk.html

+{% extends "dashboard/base.html" %}
+{% load staticfiles %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+{% load crispy_forms_tags %}
+
+{% block title-page %}{% trans "Disk" %} | {% trans "Storage" %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <div class="pull-right">
+          {% with app=object.get_appliance %}
+            {% if app %}
+              {% trans "Appliance" %}: <a href="{{ app.get_absolute_url }}">{{ app.name }}</a>
+            {% else %}
+              This disk is not attached to anything.
+            {% endif %}
+          {% endwith %}
+        </div>
+        <h3 class="no-margin"><i class="fa fa-file"></i> {% trans "Disk" %}</h3>
+      </div>
+      <div class="panel-body">
+        {% crispy form %}
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+{% endblock %}

circle/dashboard/urls.py

@@ -53,6 +53,7 @@ from .views import (
     OpenSearchDescriptionView,
     NodeActivityView,
     UserList,
+    StorageDetail, DiskDetail,
 )
 from .views.vm import vm_ops, vm_mass_ops
 from .views.node import node_ops
@@ -226,6 +227,12 @@ urlpatterns = patterns(
         name="dashboard.views.token-login"),
     url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
         name="dashboard.views.vm-opensearch"),
+
+
+    url(r'^storage/$', StorageDetail.as_view(),
+        name="dashboard.views.storage"),
+    url(r'^disk/(?P<pk>\d+)/$', DiskDetail.as_view(),
+        name="dashboard.views.disk-detail"),
 )
 
 urlpatterns += patterns(

circle/dashboard/views/__init__.py

@@ -12,3 +12,4 @@ from user import *
 from util import *
 from vm import *
 from graph import *
+from storage import *

circle/dashboard/views/storage.py

+# Copyright 2014 Budapest University of Technology and Economics (BME IK)
+#
+# This file is part of CIRCLE Cloud.
+#
+# CIRCLE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import unicode_literals, absolute_import
+
+
+from django.views.generic import UpdateView
+from django.core.urlresolvers import reverse
+
+from sizefield.utils import filesizeformat
+from braces.views import SuperuserRequiredMixin
+
+from storage.models import DataStore, Disk
+from ..tables import DiskListTable
+from ..forms import DataStoreForm, DiskForm
+
+
+class StorageDetail(SuperuserRequiredMixin, UpdateView):
+    model = DataStore
+    form_class = DataStoreForm
+    template_name = "dashboard/storage/detail.html"
+
+    def get_object(self):
+        return DataStore.objects.get()
+
+    def get_context_data(self, **kwargs):
+        context = super(StorageDetail, self).get_context_data(**kwargs)
+
+        ds = self.get_object()
+        context['stats'] = self._get_stats()
+        context['missing_disks'] = ds.get_missing_disks()
+        context['orphan_disks'] = ds.get_orphan_disks()
+        qs = Disk.objects.filter(datastore=ds, destroyed=None)
+        context['disk_table'] = DiskListTable(
+            qs, request=self.request,
+            template="django_tables2/table_no_page.html")
+        return context
+
+    def _get_stats(self):
+        stats = self.object.get_statistics()
+        free_space = int(stats['free_space'])
+        free_percent = float(stats['free_percent'])
+
+        total_space = free_space / (free_percent/100.0)
+        used_space = total_space - free_space
+
+        return {
+            'used_percent': int(100 - free_percent),
+            'free_space': filesizeformat(free_space),
+            'used_space': filesizeformat(used_space),
+            'total_space': filesizeformat(total_space),
+        }
+
+    def get_success_url(self):
+        return reverse("dashboard.views.storage")
+
+
+class DiskDetail(SuperuserRequiredMixin, UpdateView):
+    model = Disk
+    form_class = DiskForm
+    template_name = "dashboard/storage/disk.html"

circle/storage/models.py

@@ -22,10 +22,12 @@ from __future__ import unicode_literals
 import logging
 from os.path import join
 import uuid
+import re
 
 from celery.contrib.abortable import AbortableAsyncResult
 from django.db.models import (Model, BooleanField, CharField, DateTimeField,
                               ForeignKey)
+from django.core.exceptions import ObjectDoesNotExist
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from model_utils.models import TimeStampedModel
@@ -34,7 +36,7 @@ from sizefield.models import FileSizeField
 from .tasks import local_tasks, storage_tasks
 from celery.exceptions import TimeoutError
 from common.models import (
-    WorkerNotFound, HumanReadableException, humanize_exception
+    WorkerNotFound, HumanReadableException, humanize_exception, method_cache
 )
 
 logger = logging.getLogger(__name__)
@@ -76,6 +78,37 @@ class DataStore(Model):
                 self.disk_set.filter(
                     destroyed__isnull=False) if disk.is_deletable]
 
+    @method_cache(30)
+    def get_statistics(self, timeout=15):
+        q = self.get_remote_queue_name("storage", priority="fast")
+        return storage_tasks.get_storage_stat.apply_async(
+            args=[self.path], queue=q).get(timeout=timeout)
+
+    @method_cache(30)
+    def get_orphan_disks(self, timeout=15):
+        """Disk image files without Disk object in the database.
+        """
+        queue_name = self.get_remote_queue_name('storage', "slow")
+        files = set(storage_tasks.list_files.apply_async(
+            args=[self.path], queue=queue_name).get(timeout=timeout))
+        disks = set([disk.filename for disk in self.disk_set.all()])
+
+        orphans = []
+        for i in files - disks:
+            if not re.match('cloud-[0-9]*\.dump', i):
+                orphans.append(i)
+        return orphans
+
+    @method_cache(30)
+    def get_missing_disks(self, timeout=15):
+        """Disk objects without disk image files.
+        """
+        queue_name = self.get_remote_queue_name('storage', "slow")
+        files = set(storage_tasks.list_files.apply_async(
+            args=[self.path], queue=queue_name).get(timeout=timeout))
+        disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
+        return disks.exclude(filename__in=files)
+
 
 class Disk(TimeStampedModel):
 
@@ -261,11 +294,11 @@ class Disk(TimeStampedModel):
         """Return the Instance or InstanceTemplate object where the disk
         is used
         """
-        from vm.models import Instance
         try:
-            return self.instance_set.get()
-        except Instance.DoesNotExist:
-            return self.template_set.get()
+            app = self.template_set.all() or self.instance_set.all()
+            return app.get()
+        except ObjectDoesNotExist:
+            return None
 
     def get_exclusive(self):
         """Get an instance of the disk for exclusive usage.

circle/storage/tasks/periodic_tasks.py

@@ -62,7 +62,7 @@ def list_orphan_disks(timeout=15):
     """
     import re
     for ds in DataStore.objects.all():
-        queue_name = ds.get_remote_queue_name('storage')
+        queue_name = ds.get_remote_queue_name('storage', "slow")
         files = set(storage_tasks.list_files.apply_async(
             args=[ds.path], queue=queue_name).get(timeout=timeout))
         disks = set([disk.filename for disk in ds.disk_set.all()])
@@ -79,7 +79,7 @@ def list_missing_disks(timeout=15):
     :type timeoit: int
     """
     for ds in DataStore.objects.all():
-        queue_name = ds.get_remote_queue_name('storage')
+        queue_name = ds.get_remote_queue_name('storage', "slow")
         files = set(storage_tasks.list_files.apply_async(
             args=[ds.path], queue=queue_name).get(timeout=timeout))
         disks = set([disk.filename for disk in