dashboard: add datastore detail

circle/dashboard/forms.py

@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from datetime import timedelta
 from datetime import timedelta
 from urlparse import urlparse
 from urlparse import urlparse
 
 
+from django.forms import ModelForm
 from django.contrib.auth.forms import (
 from django.contrib.auth.forms import (
     AuthenticationForm, PasswordResetForm, SetPasswordForm,
     AuthenticationForm, PasswordResetForm, SetPasswordForm,
     PasswordChangeForm,
     PasswordChangeForm,
@@ -31,10 +32,12 @@ from django.core.exceptions import PermissionDenied, ValidationError
 import autocomplete_light
 import autocomplete_light
 from crispy_forms.helper import FormHelper
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import (
 from crispy_forms.layout import (
-    Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK,
+    Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
 )
 )
 
 
 from crispy_forms.utils import render_field
 from crispy_forms.utils import render_field
+from crispy_forms.bootstrap import FormActions
+
 from django import forms
 from django import forms
 from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
 from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
 from django.forms.widgets import TextInput, HiddenInput
 from django.forms.widgets import TextInput, HiddenInput
@@ -51,6 +54,7 @@ from firewall.models import Vlan, Host
 from vm.models import (
 from vm.models import (
     InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
     InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
 )
 )
+from storage.models import DataStore
 from django.contrib.admin.widgets import FilteredSelectMultiple
 from django.contrib.admin.widgets import FilteredSelectMultiple
 from django.contrib.auth.models import Permission
 from django.contrib.auth.models import Permission
 from .models import Profile, GroupProfile
 from .models import Profile, GroupProfile
@@ -1497,3 +1501,25 @@ class TemplateListSearchForm(forms.Form):
             data = self.data.copy()
             data = self.data.copy()
             data['stype'] = "owned"
             data['stype'] = "owned"
             self.data = data
             self.data = data
+
+
+class DataStoreForm(ModelForm):
+
+    @property
+    def helper(self):
+        helper = FormHelper()
+        helper.layout = Layout(
+            Fieldset(
+                '',
+                'name',
+                'path',
+                'hostname',
+            ),
+            FormActions(
+                Submit('submit', _('Save')),
+            )
+        )
+        return helper
+
+    class Meta:
+        model = DataStore

circle/dashboard/tables.py

@@ -23,11 +23,20 @@ from django_tables2.columns import (TemplateColumn, Column, LinkColumn,
                                     BooleanColumn)
                                     BooleanColumn)
 
 
 from vm.models import Node, InstanceTemplate, Lease
 from vm.models import Node, InstanceTemplate, Lease
+from storage.models import Disk
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 from django_sshkey.models import UserKey
 from django_sshkey.models import UserKey
 from dashboard.models import ConnectCommand
 from dashboard.models import ConnectCommand
 
 
 
 
+class FileSizeColumn(Column):
+    def render(self, value):
+        from sizefield.utils import filesizeformat
+
+        size = filesizeformat(value)
+        return size
+
+
 class NodeListTable(Table):
 class NodeListTable(Table):
 
 
     pk = Column(
     pk = Column(
@@ -292,3 +301,17 @@ class ConnectCommandListTable(Table):
             "You don't have any custom connection commands yet. You can "
             "You don't have any custom connection commands yet. You can "
             "specify commands to be displayed on VM detail pages instead of "
             "specify commands to be displayed on VM detail pages instead of "
             "the defaults.")
             "the defaults.")
+
+
+class DiskListTable(Table):
+
+    size = FileSizeColumn()
+
+    class Meta:
+        model = Disk
+        attrs = {'class': "table table-bordered table-striped table-hover",
+                 'id': "disk-list-table"}
+        fields = ("pk", "name", "filename", "size", "is_ready")
+        prefix = "disk-"
+        order_by = ("-pk", )
+        per_page = 99999999999

circle/dashboard/templates/dashboard/datastore/detail.html

+{% extends "dashboard/base.html" %}
+{% load staticfiles %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+{% load crispy_forms_tags %}
+
+{% block title-page %}{% trans "Datastores" %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+  <div class="col-md-5">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-desktop"></i> {% trans "Datastore" %}</h3>
+      </div>
+      <div class="panel-body">
+        {% crispy form %}
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+  <div class="col-md-7">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-desktop"></i> {% trans "Statistics" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="progress">
+          <div class="progress-bar progress-bar-success progress-bar-stripped"
+            role="progressbar" style="min-width: 30px; width: {{ stats.used_percent }}%">
+            {{ stats.used_percent }}%
+          </div>
+        </div>
+        <div class="text-muted text-center">
+          {{ stats.used_space}}/{{ stats.total_space }} used
+        </div>
+
+        <h3>Missing disks <small>disk objects without images files</small></h3>
+        {% for m in missing_disks %}
+          <p>
+          {{ m }} - {{ m.filename }}
+          </p>
+          {% empty %}
+          None
+        {% endfor %}
+
+        <h3>Orphan disks <small>image files without disk object in the database</small></h3>
+        {% for o in orphan_disks %}
+          {{ o }}
+          {% empty %}
+          None
+        {% endfor %}
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-desktop"></i> {% trans "Related disks" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="table-responsive">
+          {% render_table disk_table %}
+        </div>
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+{% endblock %}

circle/dashboard/templates/dashboard/datastore/list.html

+{% extends "dashboard/base.html" %}
+{% load staticfiles %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+
+{% block title-page %}{% trans "Datastores" %}{% endblock %}
+
+{% block content %}
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-desktop"></i> {% trans "Datastores" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="table-responsive">
+          {% render_table table %}
+        </div>
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <h3 class="no-margin"><i class="fa fa-desktop"></i> {% trans "Disks" %}</h3>
+      </div>
+      <div class="panel-body">
+        <div class="table-responsive">
+        </div>
+      </div><!-- .panel-body -->
+    </div>
+  </div>
+</div>
+
+
+{% endblock %}

circle/dashboard/urls.py

@@ -52,6 +52,7 @@ from .views import (
     TransferTemplateOwnershipView, TransferTemplateOwnershipConfirmView,
     TransferTemplateOwnershipView, TransferTemplateOwnershipConfirmView,
     OpenSearchDescriptionView,
     OpenSearchDescriptionView,
     NodeActivityView,
     NodeActivityView,
+    DataStoreDetail,
 )
 )
 from .views.vm import vm_ops, vm_mass_ops
 from .views.vm import vm_ops, vm_mass_ops
 from .views.node import node_ops
 from .views.node import node_ops
@@ -223,6 +224,10 @@ urlpatterns = patterns(
         name="dashboard.views.token-login"),
         name="dashboard.views.token-login"),
     url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
     url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
         name="dashboard.views.vm-opensearch"),
         name="dashboard.views.vm-opensearch"),
+
+
+    url(r'^datastore/$', DataStoreDetail.as_view(),
+        name="dashboard.views.datastore"),
 )
 )
 
 
 urlpatterns += patterns(
 urlpatterns += patterns(

circle/dashboard/views/__init__.py

@@ -12,3 +12,4 @@ from user import *
 from util import *
 from util import *
 from vm import *
 from vm import *
 from graph import *
 from graph import *
+from disk import *

circle/dashboard/views/disk.py

+# Copyright 2014 Budapest University of Technology and Economics (BME IK)
+#
+# This file is part of CIRCLE Cloud.
+#
+# CIRCLE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import unicode_literals, absolute_import
+
+
+from django.views.generic import (
+    UpdateView
+)
+from django.core.urlresolvers import reverse
+
+from sizefield.utils import filesizeformat
+
+from storage.models import DataStore, Disk
+from ..tables import DiskListTable
+from ..forms import DataStoreForm
+
+
+class DataStoreDetail(UpdateView):
+    model = DataStore
+    form_class = DataStoreForm
+    template_name = "dashboard/datastore/detail.html"
+
+    def get_object(self):
+        return DataStore.objects.get()
+
+    def get_context_data(self, **kwargs):
+        context = super(DataStoreDetail, self).get_context_data(**kwargs)
+
+        ds = self.get_object()
+        context['stats'] = self._get_stats()
+        context['missing_disks'] = ds.get_missing_disks()
+        context['orphan_disks'] = ds.get_orphan_disks()
+        qs = Disk.objects.filter(datastore=ds, destroyed=None)
+        context['disk_table'] = DiskListTable(
+            qs, request=self.request,
+            template="django_tables2/table_no_page.html")
+        return context
+
+    def _get_stats(self):
+        stats = self.object.get_statistics()
+        free_space = int(stats['free_space'])
+        free_percent = float(stats['free_percent'])
+
+        total_space = free_space / (free_percent/100.0)
+        used_space = total_space - free_space
+
+        return {
+            'used_percent': int(100 - free_percent),
+            'free_space': filesizeformat(free_space),
+            'used_space': filesizeformat(used_space),
+            'total_space': filesizeformat(total_space),
+        }
+
+    def get_success_url(self):
+        return reverse("dashboard.views.datastore")

circle/storage/models.py

@@ -22,6 +22,7 @@ from __future__ import unicode_literals
 import logging
 import logging
 from os.path import join
 from os.path import join
 import uuid
 import uuid
+import re
 
 
 from celery.contrib.abortable import AbortableAsyncResult
 from celery.contrib.abortable import AbortableAsyncResult
 from django.db.models import (Model, BooleanField, CharField, DateTimeField,
 from django.db.models import (Model, BooleanField, CharField, DateTimeField,
@@ -34,7 +35,7 @@ from sizefield.models import FileSizeField
 from .tasks import local_tasks, storage_tasks
 from .tasks import local_tasks, storage_tasks
 from celery.exceptions import TimeoutError
 from celery.exceptions import TimeoutError
 from common.models import (
 from common.models import (
-    WorkerNotFound, HumanReadableException, humanize_exception
+    WorkerNotFound, HumanReadableException, humanize_exception, method_cache
 )
 )
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
@@ -76,6 +77,37 @@ class DataStore(Model):
                 self.disk_set.filter(
                 self.disk_set.filter(
                     destroyed__isnull=False) if disk.is_deletable]
                     destroyed__isnull=False) if disk.is_deletable]
 
 
+    @method_cache(30)
+    def get_statistics(self, timeout=15):
+        q = self.get_remote_queue_name("storage", priority="fast")
+        return storage_tasks.get_storage_stat.apply_async(
+            args=[self.path], queue=q).get(timeout=timeout)
+
+    @method_cache(30)
+    def get_orphan_disks(self, timeout=15):
+        """Disk image files without Disk object in the database.
+        """
+        queue_name = self.get_remote_queue_name('storage', "slow")
+        files = set(storage_tasks.list_files.apply_async(
+            args=[self.path], queue=queue_name).get(timeout=timeout))
+        disks = set([disk.filename for disk in self.disk_set.all()])
+
+        orphans = []
+        for i in files - disks:
+            if not re.match('cloud-[0-9]*\.dump', i):
+                orphans.append(i)
+        return orphans
+
+    @method_cache(30)
+    def get_missing_disks(self, timeout=15):
+        """Disk objects without disk image files.
+        """
+        queue_name = self.get_remote_queue_name('storage', "slow")
+        files = set(storage_tasks.list_files.apply_async(
+            args=[self.path], queue=queue_name).get(timeout=timeout))
+        disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
+        return disks.exclude(filename__in=files)
+
 
 
 class Disk(TimeStampedModel):
 class Disk(TimeStampedModel):
 
 

circle/storage/tasks/periodic_tasks.py

@@ -62,7 +62,7 @@ def list_orphan_disks(timeout=15):
     """
     """
     import re
     import re
     for ds in DataStore.objects.all():
     for ds in DataStore.objects.all():
-        queue_name = ds.get_remote_queue_name('storage')
+        queue_name = ds.get_remote_queue_name('storage', "slow")
         files = set(storage_tasks.list_files.apply_async(
         files = set(storage_tasks.list_files.apply_async(
             args=[ds.path], queue=queue_name).get(timeout=timeout))
             args=[ds.path], queue=queue_name).get(timeout=timeout))
         disks = set([disk.filename for disk in ds.disk_set.all()])
         disks = set([disk.filename for disk in ds.disk_set.all()])
@@ -79,7 +79,7 @@ def list_missing_disks(timeout=15):
     :type timeoit: int
     :type timeoit: int
     """
     """
     for ds in DataStore.objects.all():
     for ds in DataStore.objects.all():
-        queue_name = ds.get_remote_queue_name('storage')
+        queue_name = ds.get_remote_queue_name('storage', "slow")
         files = set(storage_tasks.list_files.apply_async(
         files = set(storage_tasks.list_files.apply_async(
             args=[ds.path], queue=queue_name).get(timeout=timeout))
             args=[ds.path], queue=queue_name).get(timeout=timeout))
         disks = set([disk.filename for disk in
         disks = set([disk.filename for disk in