Skip to content

CIRCLE / cloud

Go to a project
Commit cbbcfb7b by Bach Dániel
Options

Merge remote-tracking branch 'origin/feature-datastore' 

Conflicts:
	circle/dashboard/forms.py
	circle/dashboard/tables.py
	circle/dashboard/urls.py
parents 84488ced 698338c2
Showing with 339 additions and 9 deletions
circle/dashboard/forms.py
...@@ -20,6 +20,7 @@ from __future__ import absolute_import ...@@ -20,6 +20,7 @@ from __future__ import absolute_import
from datetime import timedelta from datetime import timedelta
from urlparse import urlparse from urlparse import urlparse
from django.forms import ModelForm
from django.contrib.auth.forms import ( from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm, AuthenticationForm, PasswordResetForm, SetPasswordForm,
PasswordChangeForm, PasswordChangeForm,
...@@ -31,10 +32,12 @@ from django.core.exceptions import PermissionDenied, ValidationError ...@@ -31,10 +32,12 @@ from django.core.exceptions import PermissionDenied, ValidationError
import autocomplete_light import autocomplete_light
from crispy_forms.helper import FormHelper from crispy_forms.helper import FormHelper
from crispy_forms.layout import ( from crispy_forms.layout import (
Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
) )
from crispy_forms.utils import render_field from crispy_forms.utils import render_field
from crispy_forms.bootstrap import FormActions
from django import forms from django import forms
from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
from django.forms.widgets import TextInput, HiddenInput from django.forms.widgets import TextInput, HiddenInput
...@@ -51,6 +54,7 @@ from firewall.models import Vlan, Host ...@@ -51,6 +54,7 @@ from firewall.models import Vlan, Host
from vm.models import ( from vm.models import (
InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
) )
from storage.models import DataStore, Disk
from django.contrib.admin.widgets import FilteredSelectMultiple from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from .models import Profile, GroupProfile from .models import Profile, GroupProfile
...@@ -1544,3 +1548,36 @@ class UserListSearchForm(forms.Form): ...@@ -1544,3 +1548,36 @@ class UserListSearchForm(forms.Form):
'class': "form-control input-tags", 'class': "form-control input-tags",
'placeholder': _("Search...") 'placeholder': _("Search...")
})) }))
class DataStoreForm(ModelForm):
@property
def helper(self):
helper = FormHelper()
helper.layout = Layout(
Fieldset(
'',
'name',
'path',
'hostname',
),
FormActions(
Submit('submit', _('Save')),
)
)
return helper
class Meta:
model = DataStore
class DiskForm(ModelForm):
def __init__(self, *args, **kwargs):
super(DiskForm, self).__init__(*args, **kwargs)
for k, v in self.fields.iteritems():
v.widget.attrs['readonly'] = True
class Meta:
model = Disk
circle/dashboard/tables.py
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
from __future__ import absolute_import from __future__ import absolute_import
from django.contrib.auth.models import Group, User from django.contrib.auth.models import Group, User
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _, ugettext
from django.utils.html import mark_safe from django.utils.html import mark_safe
from django_tables2 import Table, A from django_tables2 import Table, A
...@@ -27,10 +27,36 @@ from django_tables2.columns import ( ...@@ -27,10 +27,36 @@ from django_tables2.columns import (
) )
from django_sshkey.models import UserKey from django_sshkey.models import UserKey
from storage.models import Disk
from vm.models import Node, InstanceTemplate, Lease from vm.models import Node, InstanceTemplate, Lease
from dashboard.models import ConnectCommand from dashboard.models import ConnectCommand
class FileSizeColumn(Column):
def render(self, value):
from sizefield.utils import filesizeformat
size = filesizeformat(value)
return size
class ApplianceColumn(TemplateColumn):
def render(self, *args, **kwargs):
value = super(ApplianceColumn, self).render(*args, **kwargs)
abbr = '<abbr title="%s">%s</abbr>'
appliance = kwargs['record'].get_appliance()
if appliance is None:
return value
elif isinstance(appliance, InstanceTemplate):
# Translators: [T] as Template
title, text = ugettext("Template"), ugettext("[T]")
else:
# Translators: [VM] as Virtual Machine
title, text = ugettext("Virtual machine"), ugettext("[VM]")
return mark_safe("%s %s" % (abbr % (title, text), value))
class NodeListTable(Table): class NodeListTable(Table):
pk = Column( pk = Column(
...@@ -299,3 +325,29 @@ class ConnectCommandListTable(Table): ...@@ -299,3 +325,29 @@ class ConnectCommandListTable(Table):
"You don't have any custom connection commands yet. You can " "You don't have any custom connection commands yet. You can "
"specify commands to be displayed on VM detail pages instead of " "specify commands to be displayed on VM detail pages instead of "
"the defaults.") "the defaults.")
class DiskListTable(Table):
pk = LinkColumn(
'dashboard.views.disk-detail',
args=[A('pk')],
verbose_name=_("ID"),
)
size = FileSizeColumn()
appliance = ApplianceColumn(
template_name="dashboard/storage/column-appliance.html",
verbose_name=_("Appliance"),
orderable=False,
)
is_ready = BooleanColumn(
verbose_name=_("ready")
)
class Meta:
model = Disk
attrs = {'class': "table table-bordered table-striped table-hover",
'id': "disk-list-table"}
fields = ("pk", "appliance", "filename", "size", "is_ready")
prefix = "disk-"
order_by = ("-pk", )
per_page = 65536
circle/dashboard/templates/dashboard/base.html
...@@ -24,6 +24,11 @@ ...@@ -24,6 +24,11 @@
<a href="/admin/"><i class="fa fa-cogs"></i> {% trans "Admin" %}</a> <a href="/admin/"><i class="fa fa-cogs"></i> {% trans "Admin" %}</a>
</li> </li>
<li> <li>
<a href="{% url "dashboard.views.storage" %}"><i class="fa fa-database"></i>
{% trans "Storage" %}
</a>
</li>
<li>
<a href="/network/"><i class="fa fa-globe"></i> {% trans "Network" %}</a> <a href="/network/"><i class="fa fa-globe"></i> {% trans "Network" %}</a>
</li> </li>
{% endif %} {% endif %}
......
circle/dashboard/templates/dashboard/storage/column-appliance.html 0 → 100644
{% load i18n %}
{% with app=record.get_appliance %}
{% if app %}
<a href="{{ app.get_absolute_url }}">{{ app.name }}</a>
{% else %}
—
{% endif %}
{% endwith %}
circle/dashboard/templates/dashboard/storage/detail.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load staticfiles %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% load crispy_forms_tags %}
{% block title-page %}{% trans "Storage" %}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-5">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Datastore" %}</h3>
</div>
<div class="panel-body">
{% crispy form %}
</div><!-- .panel-body -->
</div>
</div>
<div class="col-md-7">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-bar-chart"></i> {% trans "Statistics" %}</h3>
</div>
<div class="panel-body">
<div class="progress">
<div class="progress-bar progress-bar-success progress-bar-stripped"
role="progressbar" style="min-width: 30px; width: {{ stats.used_percent }}%">
{{ stats.used_percent }}%
</div>
</div>
<div class="text-muted text-center">
{{ stats.used_space}}/{{ stats.total_space }}
</div>
<h3>
{% trans "Missing disks" %}
<small>{% trans "disk objects without images files" %}</small>
</h3>
{% for m in missing_disks %}
<p>
{{ m }} - {{ m.filename }}
</p>
{% empty %}
{% trans "None" %}
{% endfor %}
<h3>
{% trans "Orphan disks" %}
<small>{% trans "image files without disk object in the database" %}</small>
</h3>
{% for o in orphan_disks %}
{{ o }}
{% empty %}
{% trans "None" %}
{% endfor %}
</div><!-- .panel-body -->
</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-file"></i> {% trans "Disks" %}</h3>
</div>
<div class="panel-body">
<div class="table-responsive">
{% render_table disk_table %}
</div>
</div><!-- .panel-body -->
</div>
</div>
</div>
{% endblock %}
circle/dashboard/templates/dashboard/storage/disk.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load staticfiles %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% load crispy_forms_tags %}
{% block title-page %}{% trans "Disk" %} | {% trans "Storage" %}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
{% with app=object.get_appliance %}
{% if app %}
{% trans "Appliance" %}: <a href="{{ app.get_absolute_url }}">{{ app.name }}</a>
{% else %}
This disk is not attached to anything.
{% endif %}
{% endwith %}
</div>
<h3 class="no-margin"><i class="fa fa-file"></i> {% trans "Disk" %}</h3>
</div>
<div class="panel-body">
{% crispy form %}
</div><!-- .panel-body -->
</div>
</div>
</div>
{% endblock %}
circle/dashboard/urls.py
...@@ -53,6 +53,7 @@ from .views import ( ...@@ -53,6 +53,7 @@ from .views import (
OpenSearchDescriptionView, OpenSearchDescriptionView,
NodeActivityView, NodeActivityView,
UserList, UserList,
StorageDetail, DiskDetail,
) )
from .views.vm import vm_ops, vm_mass_ops from .views.vm import vm_ops, vm_mass_ops
from .views.node import node_ops from .views.node import node_ops
...@@ -226,6 +227,12 @@ urlpatterns = patterns( ...@@ -226,6 +227,12 @@ urlpatterns = patterns(
name="dashboard.views.token-login"), name="dashboard.views.token-login"),
url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(), url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
name="dashboard.views.vm-opensearch"), name="dashboard.views.vm-opensearch"),
url(r'^storage/$', StorageDetail.as_view(),
name="dashboard.views.storage"),
url(r'^disk/(?P<pk>\d+)/$', DiskDetail.as_view(),
name="dashboard.views.disk-detail"),
) )
urlpatterns += patterns( urlpatterns += patterns(
......
circle/dashboard/views/__init__.py
...@@ -12,3 +12,4 @@ from user import * ...@@ -12,3 +12,4 @@ from user import *
from util import * from util import *
from vm import * from vm import *
from graph import * from graph import *
from storage import *
circle/dashboard/views/storage.py 0 → 100644
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals, absolute_import
from django.views.generic import UpdateView
from django.core.urlresolvers import reverse
from sizefield.utils import filesizeformat
from braces.views import SuperuserRequiredMixin
from storage.models import DataStore, Disk
from ..tables import DiskListTable
from ..forms import DataStoreForm, DiskForm
class StorageDetail(SuperuserRequiredMixin, UpdateView):
model = DataStore
form_class = DataStoreForm
template_name = "dashboard/storage/detail.html"
def get_object(self):
return DataStore.objects.get()
def get_context_data(self, **kwargs):
context = super(StorageDetail, self).get_context_data(**kwargs)
ds = self.get_object()
context['stats'] = self._get_stats()
context['missing_disks'] = ds.get_missing_disks()
context['orphan_disks'] = ds.get_orphan_disks()
qs = Disk.objects.filter(datastore=ds, destroyed=None)
context['disk_table'] = DiskListTable(
qs, request=self.request,
template="django_tables2/table_no_page.html")
return context
def _get_stats(self):
stats = self.object.get_statistics()
free_space = int(stats['free_space'])
free_percent = float(stats['free_percent'])
total_space = free_space / (free_percent/100.0)
used_space = total_space - free_space
return {
'used_percent': int(100 - free_percent),
'free_space': filesizeformat(free_space),
'used_space': filesizeformat(used_space),
'total_space': filesizeformat(total_space),
}
def get_success_url(self):
return reverse("dashboard.views.storage")
class DiskDetail(SuperuserRequiredMixin, UpdateView):
model = Disk
form_class = DiskForm
template_name = "dashboard/storage/disk.html"
circle/storage/models.py
...@@ -22,10 +22,12 @@ from __future__ import unicode_literals ...@@ -22,10 +22,12 @@ from __future__ import unicode_literals
import logging import logging
from os.path import join from os.path import join
import uuid import uuid
import re
from celery.contrib.abortable import AbortableAsyncResult from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField, from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey) ForeignKey)
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
...@@ -34,7 +36,7 @@ from sizefield.models import FileSizeField ...@@ -34,7 +36,7 @@ from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks from .tasks import local_tasks, storage_tasks
from celery.exceptions import TimeoutError from celery.exceptions import TimeoutError
from common.models import ( from common.models import (
WorkerNotFound, HumanReadableException, humanize_exception WorkerNotFound, HumanReadableException, humanize_exception, method_cache
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -76,6 +78,37 @@ class DataStore(Model): ...@@ -76,6 +78,37 @@ class DataStore(Model):
self.disk_set.filter( self.disk_set.filter(
destroyed__isnull=False) if disk.is_deletable] destroyed__isnull=False) if disk.is_deletable]
@method_cache(30)
def get_statistics(self, timeout=15):
q = self.get_remote_queue_name("storage", priority="fast")
return storage_tasks.get_storage_stat.apply_async(
args=[self.path], queue=q).get(timeout=timeout)
@method_cache(30)
def get_orphan_disks(self, timeout=15):
"""Disk image files without Disk object in the database.
"""
queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout))
disks = set([disk.filename for disk in self.disk_set.all()])
orphans = []
for i in files - disks:
if not re.match('cloud-[0-9]*\.dump', i):
orphans.append(i)
return orphans
@method_cache(30)
def get_missing_disks(self, timeout=15):
"""Disk objects without disk image files.
"""
queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout))
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
return disks.exclude(filename__in=files)
class Disk(TimeStampedModel): class Disk(TimeStampedModel):
...@@ -261,11 +294,11 @@ class Disk(TimeStampedModel): ...@@ -261,11 +294,11 @@ class Disk(TimeStampedModel):
"""Return the Instance or InstanceTemplate object where the disk """Return the Instance or InstanceTemplate object where the disk
is used is used
""" """
from vm.models import Instance
try: try:
return self.instance_set.get() app = self.template_set.all() or self.instance_set.all()
except Instance.DoesNotExist: return app.get()
return self.template_set.get() except ObjectDoesNotExist:
return None
def get_exclusive(self): def get_exclusive(self):
"""Get an instance of the disk for exclusive usage. """Get an instance of the disk for exclusive usage.
......
circle/storage/tasks/periodic_tasks.py
...@@ -62,7 +62,7 @@ def list_orphan_disks(timeout=15): ...@@ -62,7 +62,7 @@ def list_orphan_disks(timeout=15):
""" """
import re import re
for ds in DataStore.objects.all(): for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage') queue_name = ds.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async( files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout)) args=[ds.path], queue=queue_name).get(timeout=timeout))
disks = set([disk.filename for disk in ds.disk_set.all()]) disks = set([disk.filename for disk in ds.disk_set.all()])
...@@ -79,7 +79,7 @@ def list_missing_disks(timeout=15): ...@@ -79,7 +79,7 @@ def list_missing_disks(timeout=15):
:type timeoit: int :type timeoit: int
""" """
for ds in DataStore.objects.all(): for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage') queue_name = ds.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async( files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout)) args=[ds.path], queue=queue_name).get(timeout=timeout))
disks = set([disk.filename for disk in disks = set([disk.filename for disk in
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment