Commit 79aeec80 by Czémán Arnold

Merge branch 'ceph' into new_ceph

Conflicts:
	circle/dashboard/views/storage.py
	circle/storage/models.py
	circle/storage/tasks/periodic_tasks.py
	circle/storage/tasks/storage_tasks.py
	circle/vm/models/instance.py
parents 4413e841 ad35a055
......@@ -20,6 +20,8 @@
"pk": 1,
"model": "storage.datastore",
"fields": {
"type": "file",
"ceph_user": null,
"path": "/disks",
"hostname": "wut",
"name": "diszkek"
......@@ -36,6 +38,7 @@
"destroyed": null,
"base": null,
"datastore": 1,
"bus": null,
"dev_num": "a",
"type": "qcow2-norm",
"size": 8589934592,
......@@ -170,7 +173,8 @@
"priority": 10,
"template": null,
"access_method": "nx",
"lease": 1,
"lease": 1,
"datastore": 1,
"node": null,
"description": "",
"arch": "x86_64",
......@@ -200,7 +204,8 @@
"priority": 10,
"template": null,
"access_method": "nx",
"lease": 1,
"lease": 1,
"datastore": 1,
"node": null,
"description": "",
"arch": "x86_64",
......@@ -308,6 +313,7 @@
"arch": "x86_64",
"max_ram_size": 1024,
"lease": 1,
"datastore": 1,
"owner": 1
}
}
......
......@@ -36,7 +36,6 @@ from crispy_forms.layout import (
)
from crispy_forms.utils import render_field
from crispy_forms.bootstrap import FormActions
from django import forms
from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
......@@ -460,6 +459,9 @@ class NodeForm(forms.ModelForm):
class TemplateForm(forms.ModelForm):
networks = forms.ModelMultipleChoiceField(
queryset=None, required=False, label=_("Networks"))
datastore = forms.ModelChoiceField(
queryset=DataStore.objects.filter(destroyed__isnull=True),
empty_label=None)
num_cores = forms.IntegerField(widget=forms.NumberInput(attrs={
'class': "form-control input-tags cpu-count-input",
......@@ -816,10 +818,19 @@ class VmCreateDiskForm(OperationForm):
def __init__(self, *args, **kwargs):
default = kwargs.pop('default', None)
datastore_choices = kwargs.pop('datastore_choices')
super(VmCreateDiskForm, self).__init__(*args, **kwargs)
if default:
self.fields['name'].initial = default
datastore_field = forms.ModelChoiceField(
queryset=datastore_choices, required=False, initial=None,
label=_('Data store'))
if not datastore_choices:
datastore_field.widget.attrs['disabled'] = 'disabled'
datastore_field.empty_label = _('No more data stores.')
self.fields['datastore'] = datastore_field
def clean_size(self):
size_in_bytes = self.cleaned_data.get("size")
if not size_in_bytes.isdigit() and len(size_in_bytes) > 0:
......@@ -901,6 +912,18 @@ class VmDownloadDiskForm(OperationForm):
name = forms.CharField(max_length=100, label=_("Name"), required=False)
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
def __init__(self, *args, **kwargs):
datastore_choices = kwargs.pop('datastore_choices')
super(VmDownloadDiskForm, self).__init__(*args, **kwargs)
datastore_field = forms.ModelChoiceField(
queryset=datastore_choices, required=False, initial=None,
label=_('Data store'))
if not datastore_choices:
datastore_field.widget.attrs['disabled'] = 'disabled'
datastore_field.empty_label = _('No more data stores.')
self.fields['datastore'] = datastore_field
def clean(self):
cleaned_data = super(VmDownloadDiskForm, self).clean()
if not cleaned_data['name']:
......@@ -1429,6 +1452,26 @@ class RawDataForm(forms.ModelForm):
return helper
class VmDataStoreForm(forms.ModelForm):
datastore = forms.ModelChoiceField(
queryset=DataStore.objects.filter(destroyed__isnull=True),
empty_label=None)
class Meta:
model = Instance
fields = ('datastore', )
@property
def helper(self):
helper = FormHelper()
helper.form_show_labels = False
helper.form_action = reverse_lazy("dashboard.views.vm-data-store",
kwargs={'pk': self.instance.pk})
helper.add_input(Submit("submit", _("Save"),
css_class="btn btn-success", ))
return helper
class GroupPermissionForm(forms.ModelForm):
permissions = forms.ModelMultipleChoiceField(
queryset=None,
......@@ -1596,16 +1639,61 @@ class DataStoreForm(ModelForm):
'name',
'path',
'hostname',
),
FormActions(
Submit('submit', _('Save')),
)
)
return helper
class Meta:
model = DataStore
fields = ("name", "path", "hostname", )
fields = ("type", "name", "path", "hostname", )
widgets = {"type": HiddenInput()}
class CephDataStoreForm(DataStoreForm):
type = forms.CharField(widget=forms.HiddenInput())
@property
def helper(self):
helper = FormHelper()
helper.layout = Layout(
Fieldset(
'',
'ceph_user',
)
)
return helper
class Meta:
model = DataStore
fields = ("type", "name", "path", "hostname",
"ceph_user",)
class StorageListSearchForm(forms.Form):
CHOICES = (
("active", _("active")),
("destroyed", _("destroyed")),
(("all"), _("all")),
)
s = forms.CharField(widget=forms.TextInput(attrs={
'class': "form-control input-tags",
'placeholder': _("Search...")
}))
stype = forms.ChoiceField(CHOICES, widget=forms.Select(attrs={
'class': "btn btn-default input-tags",
}))
def __init__(self, *args, **kwargs):
super(StorageListSearchForm, self).__init__(*args, **kwargs)
# set initial value, otherwise it would be overwritten by request.GET
if not self.data.get("stype"):
data = self.data.copy()
data['stype'] = "active"
self.data = data
class DiskForm(ModelForm):
......
......@@ -29,17 +29,26 @@ $(function () {
return false;
});
$('.group-create, .node-create, .tx-tpl-ownership, .group-delete, .node-delete, .disk-remove, .template-delete, .delete-from-group, .lease-delete').click(function(e) {
$('.group-create, .node-create, .tx-tpl-ownership, .group-delete, .node-delete, ' +
'.disk-remove, .template-delete, .delete-from-group, .lease-delete, ' +
'.storage-delete, .storage-restore').click(function(e) {
$.ajax({
type: 'GET',
url: $(this).prop('href'),
success: function(data) {
$('body').append(data);
var modal = $('#confirmation-modal');
modal.modal('show');
modal.on('hidden.bs.modal', function() {
modal.remove();
});
success: function(data, _, xhr) {
var ctype = xhr.getResponseHeader("content-type") || "";
if(ctype.indexOf("html") > -1) {
$('body').append(data);
var modal = $('#confirmation-modal');
modal.modal('show');
modal.on('hidden.bs.modal', function() {
modal.remove();
});
}
else if(ctype.indexOf("json") > -1) {
if(data.error !== null && data.error !== undefined)
addMessage(data.error, "warning");
}
},
error: function(xhr, textStatus, error) {
if(xhr.status === 403) {
......@@ -52,7 +61,7 @@ $(function () {
return false;
});
$('.template-choose').click(function(e) {
$('.template-choose, .storage-choose').click(function(e) {
$.ajax({
type: 'GET',
url: $(this).prop('href'),
......@@ -73,6 +82,15 @@ $(function () {
}
return true;
});
$("#storage-choose-next-button").click(function() {
var radio = $('input[type="radio"]:checked', "#storage-choose-form").val();
if(!radio) {
$("#storage-choose-alert").addClass("alert-warning")
.text(gettext("Select an option to proceed!"));
return false;
}
return true;
});
}
});
return false;
......
......@@ -348,7 +348,6 @@ a.hover-black {
width: 100px;
}
.nojs-dropdown-menu
{
position:absolute;
......@@ -471,7 +470,7 @@ footer a, footer a:hover, footer a:visited {
margin-bottom: 20px;
}
.template-choose-list {
.template-choose-list, .storage-choose-list {
max-width: 600px;
}
......@@ -481,13 +480,13 @@ footer a, footer a:hover, footer a:visited {
padding-right: 50px;
}
.template-choose-list-element {
.template-choose-list-element, .storage-choose-list-element {
padding: 6px 10px;
cursor: pointer;
margin-bottom: 15px; /* bootstrap panel default is 20px */
}
.template-choose-list input[type="radio"] {
.template-choose-list input[type="radio"], .storage-choose-list input[type="radio"] {
float: right;
}
......
......@@ -27,7 +27,7 @@ from django_tables2.columns import (
)
from django_sshkey.models import UserKey
from storage.models import Disk
from storage.models import Disk, DataStore
from vm.models import Node, InstanceTemplate, Lease
from dashboard.models import ConnectCommand, Message
......@@ -371,3 +371,42 @@ class MessageListTable(Table):
order_by = ("-pk", )
fields = ('pk', 'message', 'enabled', 'effect')
empty_text = _("No messages.")
class StorageListTable(Table):
name = TemplateColumn(
verbose_name=_("Name"),
template_name="dashboard/storage-list/column-storage-name.html",
attrs={'th': {'data-sort': "string"}}
)
type = Column(
verbose_name=_("Type"),
attrs={'th': {'data-sort': "string"}}
)
path = Column(
verbose_name=_("Path or Pool name"),
attrs={'th': {'data-sort': "string"}}
)
hostname = Column(
verbose_name=_("Hostname"),
attrs={'th': {'data-sort': "string"}}
)
used_percent = TemplateColumn(
verbose_name=_("Usage"),
template_name="dashboard/storage-list/" +
"column-storage-used_percent.html",
orderable=False
)
class Meta:
model = DataStore
attrs = {'class': ('table table-bordered table-striped table-hover'
'storage-list-table')}
fields = ('name', 'type', 'path', 'hostname', 'used_percent')
prefix = "storage-"
......@@ -35,5 +35,6 @@
{% if request.user.is_superuser %}
<small>{% trans "File name" %}: {{ d.filename }}</small><br/>
<small>{% trans "Bus" %}: {{ d.device_bus }}</small>
<small>{% trans "Bus" %}: {{ d.device_bus }}</small><br/>
<small>{% trans "Data store" %}: {{ d.datastore }}</small>
{% endif %}
{% load i18n %}
<div class="alert alert-info" id="storage-choose-alert">
{% trans "Choose the type of the data store that you want to create." %}
</div>
<form action="{% url "dashboard.views.storage-choose" %}" method="POST"
id="storage-choose-form">
{% csrf_token %}
<div class="storage-choose-list">
{% for t in types %}
<div class="panel panel-default storage-choose-list-element">
<input type="radio" name="type" value="{{ t.0 }}"/>
{{ t.1 }}
<div class="clearfix"></div>
</div>
{% endfor %}
<button type="submit" id="storage-choose-next-button" class="btn btn-success pull-right">{% trans "Next" %}</button>
<div class="clearfix"></div>
</div>
</form>
<script>
$(function() {
$(".storage-choose-list-element").click(function() {
$("input", $(this)).prop("checked", true);
});
$(".storage-choose-list-element").hover(
function() {
$("small", $(this)).stop().fadeIn(200);
},
function() {
$("small", $(this)).stop().fadeOut(200);
}
);
});
</script>
{% load i18n %}
{% load crispy_forms_tags %}
<form id="storage-create-form" action="" method="POST">
{% include "dashboard/storage/form_chunk.html" %}
<fieldset>
<input type="submit" value="{% trans "Create new data store" %}" class="btn btn-success">
</fieldset>
</form>
......@@ -30,6 +30,7 @@
<fieldset>
<legend>{% trans "External resources" %}</legend>
{{ form.networks|as_crispy_field }}
{{ form.datastore|as_crispy_field }}
{{ form.lease|as_crispy_field }}
{% if show_lease_create %}
......
......@@ -38,7 +38,7 @@
</a>
</li>
<li>
<a href="{% url "dashboard.views.storage" %}">
<a href="{% url "dashboard.views.storage-list" %}">
<i class="fa fa-database"></i>
<span class="hidden-sm">{% trans "Storage" %}</span>
</a>
......
{% load i18n %}
<div class="modal fade" id="confirmation-modal" tabindex="-1" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-body">
{% blocktrans with object=object %}
Are you sure you want to restore <strong>{{ object }}</strong>?
{% endblocktrans %}
<br />
<div class="pull-right" style="margin-top: 15px;">
<form action="{{ request.path }}" method="POST">
{% csrf_token %}
<button type="button" class="btn btn-default" data-dismiss="modal">{% trans "Cancel" %}</button>
<input type="hidden" name="next" value="{{ request.GET.next }}"/>
<button class="btn btn-warning modal-accept"
{% if disable_submit %}disabled{% endif %}
>{% trans "Restore" %}</button>
</form>
</div>
<div class="clearfix"></div>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div>
{% extends "dashboard/base.html" %}
{% load i18n %}
{% block content %}
<div class="body-content">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin">
{% if title %}
{{ title }}
{% else %}
{% trans "Restore confirmation" %}
{% endif %}
</h3>
</div>
<div class="panel-body">
{% if text %}
{{ text|safe }}
{% else %}
{% blocktrans with object=object %}
Are you sure you want to restore <strong>{{ object }}</strong>?
{% endblocktrans %}
{% endif %}
<div class="pull-right">
<form action="{{ request.path }}" method="POST">
{% csrf_token %}
<a class="btn btn-default">{% trans "Cancel" %}</a>
<input type="hidden" name="next" value="{{ request.GET.next }}"/>
<button class="btn btn-warning"
{% if disable_submit %}disabled{% endif %}
>{% trans "Yes" %}</button>
</form>
</div>
</div>
</div>
</div>
{% endblock %}
{% extends "dashboard/base.html" %}
{% load staticfiles %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% block title-page %}{% trans "Data stores" %}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<a href="{% url "dashboard.views.storage-choose" %}" class="btn btn-success btn-xs storage-choose">
<i class="fa fa-plus"></i> {% trans "new data store" %}
</a>
</div>
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Data stores" %}</h3>
</div>
<div class="panel-body">
<div class="row">
<div class="col-md-offset-8 col-md-4" id="storage-list-search">
<form action="" method="GET">
<div class="input-group">
{{ search_form.s }}
<div class="input-group-btn">
{{ search_form.stype }}
<button type="submit" class="btn btn-primary input-tags">
<i class="fa fa-search"></i>
</button>
</div>
</div><!-- .input-group -->
</form>
</div><!-- .col-md-4 #storage-list-search -->
</div>
</div>
<div class="panel-body">
<div class="table-responsive">
{% render_table table %}
</div>
</div>
</div>
</div>
</div>
{% endblock %}
<a href="{% url "dashboard.views.storage-detail" pk=record.pk %}" title="{{ record.description }}">
{{ record.name }}
</a>
<div class="progress">
<div class="progress-bar progress-bar-success progress-bar-stripped"
role="progressbar" style="min-width: 30px; width: {{ record.used_percent }}%">
{{ record.used_percent }}%
</div>
</div>
......@@ -2,24 +2,49 @@
{% load staticfiles %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% load crispy_forms_tags %}
{% block title-page %}{% trans "Storage" %}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-5">
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Datastore" %}</h3>
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Data store" %}</h3>
</div>
<div class="panel-body">
{% crispy form %}
<form id="storage-create-form" action="" method="POST">
{% include "dashboard/storage/form_chunk.html" %}
<fieldset>
<input type="submit" value="{% trans "Save" %}" class="btn btn-primary">
</fieldset>
</form>
</div><!-- .panel-body -->
</div>
</div>
<div class="col-md-7">
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
{% if object.destroyed %}
<a href="{% url "dashboard.views.storage-restore" pk=object.pk %}"
class="btn btn-xs btn-warning pull-right storage-restore">
{% trans "Restore" %}
</a>
<h4 class="no-margin"><i class="fa fa-medkit"></i> {% trans "Restore data store" %}</h4>
{% else %}
<a href="{% url "dashboard.views.storage-delete" pk=object.pk %}"
class="btn btn-xs btn-danger pull-right storage-delete">
{% trans "Delete" %}
</a>
<h4 class="no-margin"><i class="fa fa-times"></i> {% trans "Delete data store" %}</h4>
{% endif %}
</div>
</div>
</div>
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-bar-chart"></i> {% trans "Statistics" %}</h3>
......@@ -104,5 +129,4 @@
</div>
</div>
</div>
{% endblock %}
{% endblock %}
{% load i18n %}
{% load crispy_forms_tags %}
{% with form=form %}
{% include "display-form-errors.html" %}
{% endwith %}
{% csrf_token %}
{{ form.type }}
<fieldset>
<legend>{% trans "General settings" %}</legend>
{{ form.name|as_crispy_field }}
{{ form.path|as_crispy_field }}
{{ form.hostname|as_crispy_field }}
</fieldset>
{% if form.type.value == "ceph_block" %}
<fieldset>
<legend>{% trans "Ceph block storage authentication settings" %}</legend>
{{ form.ceph_user|as_crispy_field }}
</fieldset>
{% endif %}
<style>
fieldset{
margin-top: 10px;
margin-bottom: 10px;
}
fieldset legend {
font-weight: bold;
}
</style>
......@@ -57,6 +57,7 @@
<legend>{% trans "External resources" %}</legend>
{{ form.networks|as_crispy_field }}
{{ form.lease|as_crispy_field }}
{{ form.datastore|as_crispy_field }}
{{ form.tags|as_crispy_field }}
</fieldset>
......
......@@ -102,6 +102,17 @@
</div>
</div>
<hr/>
<div class="row">
<div class="col-sm-12">
<h3>
{% trans "Data store" %}
</h3>
{% crispy data_store_form %}
</div>
</div>
{% endif %}
{% block extra_js %}
......
......@@ -343,8 +343,9 @@ class CircleSeleniumMixin(SeleniumMixin):
'Cannot save a vm as a template')
def create_base_template(self, name=None, architecture="x86-64",
method=None, op_system=None, lease=None,
network="vm"):
method=None, op_system=None,
datastore="default",
lease=None, network="vm"):
if name is None:
name = "new_%s" % self.conf.client_name
if op_system is None:
......@@ -370,6 +371,8 @@ class CircleSeleniumMixin(SeleniumMixin):
system_name.clear()
system_name.send_keys(op_system)
self.select_option(self.driver.find_element_by_id(
"id_datastore"), datastore)
self.select_option(self.driver.find_element_by_id(
"id_lease"), lease)
self.select_option(self.driver.find_element_by_id(
"id_networks"), network)
......
......@@ -253,7 +253,8 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
tmpl.set_level(self.u1, 'owner')
Vlan.objects.get(id=1).set_level(self.u1, 'user')
kwargs = tmpl.__dict__.copy()
kwargs.update(name='t1', lease=1, disks=1, raw_data='tst1')
kwargs.update(name='t1', lease=1, disks=1,
raw_data='tst1', datastore=1)
response = c.post('/dashboard/template/1/', kwargs)
self.assertEqual(response.status_code, 302)
self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data,
......@@ -264,7 +265,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
self.login(c, 'superuser')
kwargs = InstanceTemplate.objects.get(id=1).__dict__.copy()
kwargs.update(name='t2', lease=1, disks=1,
raw_data='<devices></devices>')
raw_data='<devices></devices>', datastore=1)
response = c.post('/dashboard/template/1/', kwargs)
self.assertEqual(response.status_code, 302)
self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data,
......
......@@ -42,7 +42,7 @@ from .views import (
ConnectCommandDelete, ConnectCommandDetail, ConnectCommandCreate,
StoreList, store_download, store_upload, store_get_upload_url, StoreRemove,
store_new_directory, store_refresh_toplist,
VmTraitsUpdate, VmRawDataUpdate,
VmTraitsUpdate, VmRawDataUpdate, VmDataStoreUpdate,
GroupPermissionsView,
LeaseAclUpdateView,
toggle_template_tutorial,
......@@ -53,7 +53,8 @@ from .views import (
OpenSearchDescriptionView,
NodeActivityView,
UserList,
StorageDetail, DiskDetail,
StorageDetail, StorageList, StorageChoose, StorageCreate, DiskDetail,
StorageDelete, StorageRestore,
MessageList, MessageDetail, MessageCreate, MessageDelete,
)
from .views.vm import vm_ops, vm_mass_ops
......@@ -113,6 +114,8 @@ urlpatterns = patterns(
name='dashboard.views.vm-traits'),
url(r'^vm/(?P<pk>\d+)/raw_data/$', VmRawDataUpdate.as_view(),
name='dashboard.views.vm-raw-data'),
url(r'^vm/(?P<pk>\d+)/data_store/$', VmDataStoreUpdate.as_view(),
name='dashboard.views.vm-data-store'),
url(r'^vm/(?P<pk>\d+)/toggle_tutorial/$', toggle_template_tutorial,
name='dashboard.views.vm-toggle-tutorial'),
......@@ -235,8 +238,19 @@ urlpatterns = patterns(
url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
name="dashboard.views.vm-opensearch"),
url(r'^storage/$', StorageDetail.as_view(),
name="dashboard.views.storage"),
url(r'^storage/create/(?P<type>.+)$', StorageCreate.as_view(),
name="dashboard.views.storage-create"),
url(r'^storage/(?P<pk>\d+)/$', StorageDetail.as_view(),
name='dashboard.views.storage-detail'),
url(r'^storage/list/$', StorageList.as_view(),
name="dashboard.views.storage-list"),
url(r'^storage/choose/$', StorageChoose.as_view(),
name="dashboard.views.storage-choose"),
url(r"^storage/delete/(?P<pk>\d+)/$", StorageDelete.as_view(),
name="dashboard.views.storage-delete"),
url(r"^storage/restore/(?P<pk>\d+)/$", StorageRestore.as_view(),
name="dashboard.views.storage-restore"),
url(r'^disk/(?P<pk>\d+)/$', DiskDetail.as_view(),
name="dashboard.views.disk-detail"),
......
......@@ -16,19 +16,182 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals, absolute_import
import logging
from django.contrib import messages
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse, reverse_lazy
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.views.generic import UpdateView
from django.utils.translation import ugettext_lazy as _, ugettext
from django.views.generic import (
UpdateView, TemplateView, CreateView, DeleteView
)
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import redirect
from django_tables2 import SingleTableView
from django.http import (
Http404, HttpResponseRedirect, JsonResponse
)
from django.core.exceptions import PermissionDenied
from braces.views import SuperuserRequiredMixin
from sizefield.utils import filesizeformat
from common.models import WorkerNotFound
from storage.models import DataStore, Disk
from ..tables import DiskListTable
from ..forms import DataStoreForm, DiskForm
from ..tables import DiskListTable, StorageListTable
from ..forms import (
DataStoreForm, CephDataStoreForm, DiskForm, StorageListSearchForm
)
from .util import FilterMixin
import json
from celery.exceptions import TimeoutError
from vm.models import Node
logger = logging.getLogger(__name__)
class StorageChoose(SuperuserRequiredMixin, TemplateView):
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/_modal.html']
else:
return ['dashboard/nojs-wrapper.html']
def get_context_data(self, *args, **kwargs):
context = super(StorageChoose, self).get_context_data(*args, **kwargs)
types = DataStore.TYPES
context.update({
'box_title': _('Choose data store type'),
'ajax_title': True,
'template': "dashboard/_storage-choose.html",
'types': types,
})
return context
def post(self, request, *args, **kwargs):
if not request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
type = request.POST.get("type")
if any(type in t for t in DataStore.TYPES):
return redirect(reverse("dashboard.views.storage-create",
kwargs={"type": type}))
else:
messages.warning(request, _("Select an option to proceed."))
return redirect(reverse("dashboard.views.storage-choose"))
class StorageCreate(SuccessMessageMixin, CreateView):
model = DataStore
form = None
def get_template_names(self):
if self.request.is_ajax():
pass
else:
return ['dashboard/nojs-wrapper.html']
def get_context_data(self, *args, **kwargs):
context = super(StorageCreate, self).get_context_data(*args, **kwargs)
context.update({
'box_title': _("Create a new data store"),
'template': "dashboard/_storage-create.html",
})
return context
def get(self, *args, **kwargs):
if not self.request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
return super(StorageCreate, self).get(*args, **kwargs)
def post(self, request, *args, **kwargs):
if not self.request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
self.form = self.form_class(request.POST)
if not self.form.is_valid():
logger.debug("invalid form")
return self.get(request, self.form, *args, **kwargs)
else:
self.form.save()
return redirect(self.get_success_url())
def get_success_url(self):
return reverse_lazy("dashboard.views.storage-list")
def get_form(self):
if self.form is not None:
return self.form
else:
type = self.kwargs.get("type")
fc = self.form_class
f = fc(initial={"type": type})
return f
@property
def form_class(self):
type = self.kwargs.get("type")
if type == "file":
fc = DataStoreForm
elif type == "ceph_block":
fc = CephDataStoreForm
else:
raise Http404(_("Invalid creation type"))
return fc
class StorageList(SuperuserRequiredMixin, FilterMixin, SingleTableView):
template_name = "dashboard/storage-list.html"
model = DataStore
table_class = StorageListTable
table_pagination = False
allowed_filters = {
'name': "name__icontains",
'type': "type__icontains",
'path': "path__icontains",
'poolname': "path__icontains",
'hostname': "hostname__iexact",
}
def get_context_data(self, *args, **kwargs):
context = super(StorageList, self).get_context_data(*args, **kwargs)
context['search_form'] = self.search_form
return context
def get(self, *args, **kwargs):
self.search_form = StorageListSearchForm(self.request.GET)
self.search_form.full_clean()
return super(StorageList, self).get(*args, **kwargs)
def get_queryset(self):
logger.debug('StorageList.get_queryset() called. User: %s',
unicode(self.request.user))
cleaned_data = self.search_form.cleaned_data
stype = cleaned_data.get('stype', "all")
qs = self.get_queryset_by_stype(stype)
self.create_fake_get()
try:
filters, excludes = self.get_queryset_filters()
qs = qs.filter(**filters).exclude(**excludes).distinct()
except ValueError:
messages.error(self.request, _("Error during filtering."))
return qs
def get_queryset_by_stype(self, stype):
if stype == "all":
return DataStore.get_all()
elif stype == "destroyed":
return DataStore.objects.filter(destroyed__isnull=False)
else:
return DataStore.objects.filter(destroyed__isnull=True)
class StorageDetail(SuperuserRequiredMixin, UpdateView):
......@@ -36,12 +199,8 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
form_class = DataStoreForm
template_name = "dashboard/storage/detail.html"
def get_object(self):
return DataStore.get_default_datastore()
def get_context_data(self, **kwargs):
context = super(StorageDetail, self).get_context_data(**kwargs)
try:
ds = self.get_object()
context['stats'] = self._get_stats()
......@@ -49,6 +208,13 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
context['orphan_disks'] = ds.get_orphan_disks()
except WorkerNotFound:
messages.error(self.request, _("The DataStore is offline."))
except TimeoutError:
messages.error(self.request, _("Operation timed out, "
"some data may insufficient."))
except Exception as e:
messages.error(self.request, _("Error occured: %s, "
"some data may insufficient."
% unicode(e)))
context['disk_table'] = DiskListTable(
self.get_table_data(), request=self.request,
......@@ -105,8 +271,108 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
'total_space': filesizeformat(total_space),
}
def get_form_class(self):
ds = self.get_object()
if ds.type == "ceph_block":
return CephDataStoreForm
else:
return DataStoreForm
def get_success_url(self):
ds = self.get_object()
return reverse("dashboard.views.storage-detail", kwargs={"pk": ds.id})
def form_valid(self, form):
# automatic credential refresh
changed = (self.object.type == "ceph_block" and
self.object.tracker.has_changed("ceph_user"))
response = super(StorageDetail, self).form_valid(form)
if changed:
nodes = Node.objects.all()
for node in nodes:
if node.get_online():
node.refresh_credential(
user=self.request.user,
username=self.object.ceph_user)
return response
class StorageDelete(SuperuserRequiredMixin, DeleteView):
model = DataStore
success_message = _("Storage successfully destroyed.")
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-delete.html']
else:
return ['dashboard/confirm/base-delete.html']
def check_destroyable(self):
object = self.get_object()
if not object.is_destroyable:
raise PermissionDenied()
def get(self, request, *args, **kwargs):
try:
self.check_destroyable()
except PermissionDenied:
message = ugettext("Another object references"
" to the selected object.")
if request.is_ajax():
return JsonResponse({"error": message})
else:
messages.warning(request, message)
return redirect(self.get_success_url())
return super(StorageDelete, self).get(request, *args, **kwargs)
def get_success_url(self):
return reverse("dashboard.views.storage")
return reverse_lazy("dashboard.views.storage-list")
def delete_obj(self, request, *args, **kwargs):
self.get_object().destroy()
def delete(self, request, *args, **kwargs):
self.check_destroyable()
self.delete_obj(request, *args, **kwargs)
if request.is_ajax():
return JsonResponse(
json.dumps({'message': self.success_message}),
)
else:
messages.success(request, self.success_message)
return HttpResponseRedirect(self.get_success_url())
class StorageRestore(SuperuserRequiredMixin, UpdateView):
model = DataStore
fields = ("destroyed",)
success_message = _("Data store successfully restored.")
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-restore.html']
else:
return ['dashboard/confirm/base-restore.html']
def form_valid(self, form):
object = self.get_object()
object.destroyed = None
object.save()
if self.request.is_ajax():
return JsonResponse(
json.dumps({'message': self.success_message}),
)
else:
messages.success(self.request, self.success_message)
return HttpResponseRedirect(self.get_success_url())
def get_success_url(self):
ds = self.get_object()
return reverse_lazy("dashboard.views.storage-detail",
kwargs={"pk": ds.id})
class DiskDetail(SuperuserRequiredMixin, UpdateView):
......
......@@ -49,7 +49,7 @@ from common.models import (
)
from firewall.models import Vlan, Host, Rule
from manager.scheduler import SchedulerError
from storage.models import Disk
from storage.models import Disk, DataStore
from vm.models import (
Instance, InstanceActivity, Node, Lease,
InstanceTemplate, InterfaceTemplate, Interface,
......@@ -66,7 +66,7 @@ from ..forms import (
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm,
VmRemoveInterfaceForm, VmDataStoreForm,
)
from request.models import TemplateAccessType, LeaseType
from request.forms import LeaseRequestForm, TemplateRequestForm
......@@ -173,6 +173,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
if self.request.user.is_superuser:
context['traits_form'] = TraitsForm(instance=instance)
context['raw_data_form'] = RawDataForm(instance=instance)
context['data_store_form'] = VmDataStoreForm(instance=instance)
# resources change perm
context['can_change_resources'] = self.request.user.has_perm(
......@@ -323,6 +324,14 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
return self.get_object().get_absolute_url() + "#resources"
class VmDataStoreUpdate(SuperuserRequiredMixin, UpdateView):
form_class = VmDataStoreForm
model = Instance
def get_success_url(self):
return self.get_object().get_absolute_url() + "#resources"
class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance
......@@ -428,6 +437,8 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
val = super(VmCreateDiskView, self).get_form_kwargs()
num = op.instance.disks.count() + 1
val['default'] = "%s %d" % (op.instance.name, num)
val['datastore_choices'] = DataStore.objects.filter(
destroyed__isnull=True)
return val
......@@ -441,6 +452,12 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
val = super(VmDownloadDiskView, self).get_form_kwargs()
val['datastore_choices'] = DataStore.objects.filter(
destroyed__isnull=True)
return val
class VmMigrateView(FormOperationMixin, VmOperationView):
......
......@@ -30,7 +30,7 @@ from dashboard.tests.test_views import LoginMixin
from vm.operations import ResourcesOperation
class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
class RequestTestBase(LoginMixin, MockCeleryMixin, TestCase):
fixtures = ['test-vm-fixture.json', 'node.json']
def setUp(self):
......@@ -57,10 +57,12 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
tat.templates.add(InstanceTemplate.objects.get(pk=1))
def tearDown(self):
super(RequestTest, self).tearDown()
super(RequestTestBase, self).tearDown()
self.u1.delete()
self.us.delete()
class ResourceRequestTest(RequestTestBase):
def test_resources_request(self):
c = Client()
self.login(c, "user1")
......@@ -98,6 +100,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
new_request = Request.objects.latest("pk")
self.assertEqual(new_request.status, "ACCEPTED")
class TemplateAccessRequestTest(RequestTestBase):
def test_template_access_request(self):
c = Client()
self.login(c, "user1")
......@@ -121,6 +125,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
self.assertEqual(new_request.status, "ACCEPTED")
self.assertTrue(template.has_level(self.u1, "user"))
class LeaseRequestTest(RequestTestBase):
def test_lease_request(self):
c = Client()
self.login(c, "user1")
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0002_disk_bus'),
]
operations = [
migrations.CreateModel(
name='DataStoreHost',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('address', models.CharField(max_length=1024, verbose_name='address')),
('port', models.IntegerField(null=True, verbose_name='port', blank=True)),
],
),
migrations.AddField(
model_name='datastore',
name='ceph_user',
field=models.CharField(max_length=255, null=True, verbose_name='Ceph username', blank=True),
),
migrations.AddField(
model_name='datastore',
name='secret_uuid',
field=models.CharField(max_length=255, null=True, verbose_name='uuid of secret', blank=True),
),
migrations.AddField(
model_name='datastore',
name='type',
field=models.CharField(default='file', max_length=10, verbose_name='type', choices=[('file', 'filesystem'), ('ceph_block', 'Ceph block device')]),
),
migrations.AlterField(
model_name='datastore',
name='hostname',
field=models.CharField(max_length=40, verbose_name='hostname'),
),
migrations.AlterField(
model_name='disk',
name='type',
field=models.CharField(max_length=10, choices=[('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'), ('ceph-norm', 'Ceph block normal'), ('ceph-snap', 'Ceph block snapshot'), ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]),
),
migrations.AddField(
model_name='datastore',
name='hosts',
field=models.ManyToManyField(to='storage.DataStoreHost', verbose_name='hosts', blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import storage.models
class Migration(migrations.Migration):
dependencies = [
('storage', '0003_auto_20151122_2104'),
]
operations = [
migrations.AddField(
model_name='datastorehost',
name='name',
field=models.CharField(default='Monitor1', unique=True, max_length=255, verbose_name='name'),
preserve_default=False,
),
migrations.AlterField(
model_name='datastore',
name='path',
field=models.CharField(unique=True, max_length=200, verbose_name='path', validators=[storage.models.validate_ascii]),
),
migrations.AlterField(
model_name='disk',
name='filename',
field=models.CharField(unique=True, max_length=256, verbose_name='filename', validators=[storage.models.validate_ascii]),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import storage.models
class Migration(migrations.Migration):
dependencies = [
('storage', '0004_auto_20151212_0402'),
]
operations = [
migrations.AlterField(
model_name='datastore',
name='path',
field=models.CharField(unique=True, max_length=200, verbose_name='path or poolname', validators=[storage.models.validate_ascii]),
),
migrations.AlterField(
model_name='datastore',
name='secret_uuid',
field=models.CharField(max_length=255, null=True, verbose_name='uuid of secret key', blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0005_auto_20160331_1823'),
]
operations = [
migrations.CreateModel(
name='Endpoint',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(unique=True, max_length=255, verbose_name='name')),
('address', models.CharField(max_length=1024, verbose_name='address')),
('port', models.IntegerField(null=True, verbose_name='port', blank=True)),
],
),
migrations.RemoveField(
model_name='datastore',
name='hosts',
),
migrations.DeleteModel(
name='DataStoreHost',
),
migrations.AddField(
model_name='datastore',
name='endpoints',
field=models.ManyToManyField(to='storage.Endpoint', verbose_name='endpoints', blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0006_auto_20160505_0824'),
]
operations = [
migrations.AddField(
model_name='datastore',
name='destroyed',
field=models.DateTimeField(default=None, null=True, blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0007_datastore_destroyed'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='secret_uuid',
),
migrations.AddField(
model_name='datastore',
name='secret',
field=models.CharField(max_length=255, null=True, verbose_name='secret key', blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0008_auto_20160609_2338'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='endpoints',
),
migrations.DeleteModel(
name='Endpoint',
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0009_auto_20160614_1125'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='secret',
),
]
......@@ -28,11 +28,12 @@ from django.conf import settings
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel
from model_utils import FieldTracker
from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks
......@@ -44,14 +45,35 @@ from common.models import (
logger = logging.getLogger(__name__)
def validate_ascii(value):
try:
str(value)
except UnicodeEncodeError:
raise ValidationError("%s is not 'ascii' string" % value)
class DataStore(Model):
"""Collection of virtual disks.
"""
TYPES = (('file', 'filesystem'), ('ceph_block', 'Ceph block device'))
type = CharField(max_length=10, verbose_name=_('type'),
default='file', choices=TYPES)
name = CharField(max_length=100, unique=True, verbose_name=_('name'))
path = CharField(max_length=200, unique=True, verbose_name=_('path'))
hostname = CharField(max_length=40, unique=True,
verbose_name=_('hostname'))
# path or pool name when use ceph block device storage
path = CharField(max_length=200, unique=True,
verbose_name=_('path or poolname'),
validators=[validate_ascii])
# hostname of storage driver
hostname = CharField(max_length=40, verbose_name=_('hostname'))
ceph_user = CharField(max_length=255, null=True, blank=True,
verbose_name=_('Ceph username'))
destroyed = DateTimeField(blank=True, default=None, null=True)
tracker = FieldTracker(fields=["ceph_user"])
class Meta:
ordering = ['name']
......@@ -82,15 +104,42 @@ class DataStore(Model):
return [disk.filename for disk in deletables]
def destroy(self):
if self.destroyed:
return False
self.destroyed = timezone.now()
self.save()
return True
@property
def used_percent(self):
stats = self.get_statistics()
free_percent = float(stats['free_percent'])
return int(100 - free_percent)
@property
def is_destroyable(self):
disk_count = self.disk_set.filter(destroyed__isnull=True).count()
template_count = self.instancetemplate_set.count()
vm_count = self.instance_set.filter(destroyed_at__isnull=True).count()
return 0 == disk_count + vm_count + template_count
@method_cache(30)
def get_statistics(self, timeout=15):
q = self.get_remote_queue_name("storage", priority="fast")
return storage_tasks.get_storage_stat.apply_async(
args=[self.path], queue=q).get(timeout=timeout)
try:
return storage_tasks.get_storage_stat.apply_async(
args=[self.type, self.path], queue=q).get(timeout=timeout)
except Exception:
return {'free_space': -1,
'free_percent': -1}
@method_cache(30)
def get_orphan_disks(self, timeout=15):
"""Disk image files without Disk object in the database.
Exclude cloud-xxxxxxxx.dump format images.
:param timeout: Seconds before TimeOut exception
......@@ -98,7 +147,8 @@ class DataStore(Model):
"""
queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout))
args=[self.type, self.path], queue=queue_name).get(
timeout=timeout))
disks = set([disk.filename for disk in self.disk_set.all()])
orphans = []
......@@ -116,8 +166,10 @@ class DataStore(Model):
"""
queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout))
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True)
args=[self.type, self.path], queue=queue_name).get(
timeout=timeout))
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True,
datastore=self)
return disks.exclude(filename__in=files)
@classmethod
......@@ -130,17 +182,26 @@ class DataStore(Model):
pass
return cls.objects.all()[0] # TODO
@classmethod
def get_all(cls):
return cls.objects.all()
class Disk(TimeStampedModel):
"""A virtual disk.
"""
TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
TYPES = (('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('ceph-norm', 'Ceph block normal'),
('ceph-snap', 'Ceph block snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw'))
BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True,
verbose_name=_("filename"))
verbose_name=_("filename"),
validators=[validate_ascii])
datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
help_text=_("The datastore that holds the disk."))
type = CharField(max_length=10, choices=TYPES)
......@@ -240,24 +301,28 @@ class Disk(TimeStampedModel):
return join(self.datastore.path, self.filename)
@property
def vm_format(self):
def format_for_vmdriver(self):
"""Returns the proper file format for different type of images.
"""
return {
'qcow2-norm': 'qcow2',
'qcow2-snap': 'qcow2',
'ceph-norm': 'raw',
'ceph-snap': 'raw',
'iso': 'raw',
'raw-ro': 'raw',
'raw-rw': 'raw',
}[self.type]
@property
def format(self):
def format_for_storagedriver(self):
"""Returns the proper file format for different types of images.
"""
return {
'qcow2-norm': 'qcow2',
'qcow2-snap': 'qcow2',
'ceph-norm': 'rbd',
'ceph-snap': 'rbd',
'iso': 'iso',
'raw-ro': 'raw',
'raw-rw': 'raw',
......@@ -270,6 +335,8 @@ class Disk(TimeStampedModel):
return {
'qcow2-norm': 'vd',
'qcow2-snap': 'vd',
'ceph-norm': 'vd',
'ceph-snap': 'vd',
'iso': 'sd',
'raw-ro': 'vd',
'raw-rw': 'vd',
......@@ -284,6 +351,8 @@ class Disk(TimeStampedModel):
return {
'qcow2-norm': 'virtio',
'qcow2-snap': 'virtio',
'ceph-norm': 'virtio',
'ceph-snap': 'virtio',
'iso': 'ide',
'raw-ro': 'virtio',
'raw-rw': 'virtio',
......@@ -328,6 +397,7 @@ class Disk(TimeStampedModel):
"""
type_mapping = {
'qcow2-norm': 'qcow2-snap',
'ceph-norm': 'ceph-snap',
'iso': 'iso',
'raw-ro': 'raw-rw',
}
......@@ -341,29 +411,55 @@ class Disk(TimeStampedModel):
name=self.name, size=self.size,
type=new_type, dev_num=self.dev_num)
def get_disk_desc(self):
"""Serialize disk object to the storage driver.
"""
if self.datastore.type == "ceph_block":
return self.get_disk_desc_for_ceph_block_device()
return self.get_disk_desc_for_filesystem()
def get_vmdisk_desc(self):
"""Serialize disk object to the vmdriver.
"""
if self.datastore.type == "ceph_block":
return self.get_vmdisk_desc_for_ceph_block_device()
return self.get_vmdisk_desc_for_filesystem()
def get_disk_desc_for_filesystem(self):
return {
'data_store_type': self.datastore.type,
'name': str(self.filename),
'dir': str(self.datastore.path),
'format': self.format_for_storagedriver,
'size': self.size,
'base_name': str(self.base.filename) if self.base else None,
'type': 'snapshot' if self.base else 'normal'
}
def get_disk_desc_for_ceph_block_device(self):
return self.get_disk_desc_for_filesystem()
def get_vmdisk_desc_for_filesystem(self):
return {
'data_store_type': self.datastore.type,
'source': self.path,
'driver_type': self.vm_format,
'driver_type': self.format_for_vmdriver,
'driver_cache': 'none',
'target_device': self.device_type + self.dev_num,
'target_bus': self.device_bus,
'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
}
def get_disk_desc(self):
"""Serialize disk object to the storage driver.
"""
return {
'name': self.filename,
'dir': self.datastore.path,
'format': self.format,
'size': self.size,
'base_name': self.base.filename if self.base else None,
'type': 'snapshot' if self.base else 'normal'
}
def get_vmdisk_desc_for_ceph_block_device(self):
desc = self.get_vmdisk_desc_for_filesystem()
desc["ceph_user"] = self.datastore.ceph_user
return desc
def get_remote_queue_name(self, queue_id='storage', priority=None,
check_worker=True):
......@@ -381,6 +477,11 @@ class Disk(TimeStampedModel):
def clean(self, *args, **kwargs):
if (self.size is None or "") and self.base:
self.size = self.base.size
if self.base is not None and\
self.datastore.type != self.base.datastore.type:
raise ValidationError(_("Forbidden inheritance of disks "
"from different type of data stores."))
super(Disk, self).clean(*args, **kwargs)
def deploy(self, user=None, task_uuid=None, timeout=15):
......@@ -423,6 +524,14 @@ class Disk(TimeStampedModel):
self.save()
return True
@staticmethod
def get_type_for_datastore(datastore):
if datastore.type == "ceph_block":
return "ceph-norm"
return "qcow2-norm"
@classmethod
def create(cls, user=None, **params):
disk = cls.__create(user, params)
......@@ -440,7 +549,7 @@ class Disk(TimeStampedModel):
return disk
@classmethod
def download(cls, url, task, user=None, **params):
def download(cls, url, datastore, task, user=None, **params):
"""Create disk object and download data from url synchronusly.
:param url: image url to download.
......@@ -458,6 +567,7 @@ class Disk(TimeStampedModel):
params.setdefault('name', url.split('/')[-1])
params.setdefault('type', 'iso')
params.setdefault('size', None)
params.setdefault('datastore', datastore)
disk = cls.__create(params=params, user=user)
queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.download.apply_async(
......@@ -493,10 +603,9 @@ class Disk(TimeStampedModel):
"""
queue_name = self.datastore.get_remote_queue_name(
'storage', priority='slow')
logger.info("Image: %s at Datastore: %s recovered from trash." %
(self.filename, self.datastore.path))
res = storage_tasks.exists.apply_async(
args=[self.datastore.path,
args=[self.datastore.type,
self.datastore.path,
self.filename],
queue=queue_name).get(timeout=timeout)
if res:
......@@ -512,6 +621,7 @@ class Disk(TimeStampedModel):
Based on disk type:
qcow2-norm, qcow2-snap --> qcow2-norm
iso --> iso (with base)
ceph-norm, ceph-snap --> ceph-norm
VM must be in STOPPED state to perform this action.
The timeout parameter is not used now.
......@@ -519,6 +629,8 @@ class Disk(TimeStampedModel):
mapping = {
'qcow2-snap': ('qcow2-norm', None),
'qcow2-norm': ('qcow2-norm', None),
'ceph-snap': ('ceph-norm', None),
'ceph-norm': ('ceph-norm', None),
'iso': ("iso", self),
}
if self.type not in mapping.keys():
......
......@@ -36,7 +36,7 @@ def garbage_collector(timeout=15, percent=10):
for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage', priority='fast')
files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout))
args=[ds.type, ds.path], queue=queue_name).get(timeout=timeout))
disks = ds.get_deletable_disks()
queue_name = ds.get_remote_queue_name('storage', priority='slow')
......@@ -46,7 +46,7 @@ def garbage_collector(timeout=15, percent=10):
(i, ds.path))
try:
success = storage_tasks.make_free_space.apply_async(
args=[ds.path, deletable_disks, percent],
args=[ds.type, ds.path, deletable_disks, percent],
queue=queue_name).get(timeout=timeout)
if not success:
logger.warning("Has no deletable disk.")
......
......@@ -19,12 +19,12 @@ from manager.mancelery import celery
@celery.task(name='storagedriver.list')
def list(dir):
def list(data_store_type, dir):
pass
@celery.task(name='storagedriver.list_files')
def list_files(dir):
def list_files(data_store_type, dir):
pass
......@@ -39,12 +39,12 @@ def download(disk_desc, url):
@celery.task(name='storagedriver.delete')
def delete(path):
def delete(disk_desc):
pass
@celery.task(name='storagedriver.delete_dump')
def delete_dump(path):
def delete_dump(data_store_type, dir, filename):
pass
......@@ -54,7 +54,7 @@ def snapshot(disk_desc):
@celery.task(name='storagedriver.get')
def get(path):
def get(disk_desc):
pass
......@@ -64,15 +64,15 @@ def merge(src_disk_desc, dst_disk_desc):
@celery.task(name='storagedriver.exists')
def exists(path, disk_name):
def exists(data_store_type, path, disk_name):
pass
@celery.task(name='storagedriver.make_free_space')
def make_free_space(path, deletable_disks, percent):
def make_free_space(datastore, path, deletable_disks, percent):
pass
@celery.task(name='storagedriver.get_storage_stat')
def get_storage_stat(path):
def get_storage_stat(data_store_type, path):
pass
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0005_auto_20160331_1823'),
('vm', '0002_interface_model'),
]
operations = [
migrations.AddField(
model_name='instance',
name='datastore',
field=models.ForeignKey(default=1, verbose_name='Data store', to='storage.DataStore', help_text="The target of VM's dump."),
preserve_default=False,
),
migrations.AddField(
model_name='instancetemplate',
name='datastore',
field=models.ForeignKey(default=1, verbose_name='Data store', to='storage.DataStore', help_text="The target of VM's dump."),
preserve_default=False,
),
]
......@@ -50,6 +50,7 @@ from .activity import (ActivityInProgressError, InstanceActivity)
from .common import BaseResourceConfigModel, Lease
from .network import Interface
from .node import Node, Trait
from storage.models import DataStore
logger = getLogger(__name__)
pre_state_changed = Signal(providing_args=["new_state"])
......@@ -93,6 +94,7 @@ class VirtualMachineDescModel(BaseResourceConfigModel):
"""Abstract base for virtual machine describing models.
"""
access_method = CharField(max_length=10, choices=ACCESS_METHODS,
verbose_name=_('access method'),
help_text=_('Primary remote access method.'))
......@@ -116,6 +118,8 @@ class VirtualMachineDescModel(BaseResourceConfigModel):
help_text=_(
'If the machine has agent installed, and '
'the manager should wait for its start.'))
datastore = ForeignKey(DataStore, verbose_name=_("Data store"),
help_text=_("The target of VM's dump."))
class Meta:
abstract = True
......@@ -425,7 +429,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
common_fields = ['name', 'description', 'num_cores', 'ram_size',
'max_ram_size', 'arch', 'priority', 'boot_menu',
'raw_data', 'lease', 'access_method', 'system',
'has_agent']
'has_agent', 'datastore']
params = dict(template=template, owner=owner, pw=pwgen())
params.update([(f, getattr(template, f)) for f in common_fields])
params.update(kwargs) # override defaults w/ user supplied values
......@@ -487,16 +491,10 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
def mem_dump(self):
"""Return the path and datastore for the memory dump.
It is always on the first hard drive storage named cloud-<id>.dump
It named cloud-<id>.dump
"""
try:
datastore = self.disks.all()[0].datastore
except IndexError:
from storage.models import DataStore
datastore = DataStore.get_default_datastore()
path = datastore.path + '/' + self.vm_name + '.dump'
return {'datastore': datastore, 'path': path}
filename = self.vm_name + '.dump'
return {'datastore': self.datastore, 'filename': filename}
@property
def primary_host(self):
......@@ -878,3 +876,18 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
user=user, concurrency_check=concurrency_check,
readable_name=readable_name, resultant_state=resultant_state)
return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
def get_most_used_datastore(self):
disks = self.disks.all()
if not disks:
return None
freqs = dict()
for disk in disks:
datastore = disk.datastore
freqs[datastore] = freqs.get(datastore, 0) + 1
datastore = max(freqs.items(), key=lambda x: x[1])
return datastore[0]
......@@ -335,7 +335,6 @@ class Node(OperatedMixin, TimeStampedModel):
try:
logger.info('%s %s', settings.GRAPHITE_URL, params)
response = requests.get(settings.GRAPHITE_URL, params=params)
retval = {}
for target in response.json():
# Example:
......@@ -366,12 +365,12 @@ class Node(OperatedMixin, TimeStampedModel):
@property
@node_available
def cpu_usage(self):
return self.monitor_info.get('cpu.percent') / 100
return self.monitor_info.get('cpu.percent', 0) / 100
@property
@node_available
def ram_usage(self):
return self.monitor_info.get('memory.usage') / 100
return self.monitor_info.get('memory.usage', 0) / 100
@property
@node_available
......
......@@ -255,12 +255,20 @@ class CreateDiskOperation(InstanceOperation):
required_perms = ('storage.create_empty_disk', )
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, size, activity, name=None):
from storage.models import Disk
def _operation(self, user, size, datastore, activity, name=None):
from storage.models import Disk, DataStore
if not datastore:
datastore = self.instance.get_most_used_datastore()
if not datastore:
datastore = DataStore.get_default_datastore()
type = Disk.get_type_for_datastore(datastore)
if not name:
name = "new disk"
disk = Disk.create(size=size, name=name, type="qcow2-norm")
disk = Disk.create(size=size, name=name,
datastore=datastore, type=type)
disk.full_clean()
devnums = list(ascii_lowercase)
for d in self.instance.disks.all():
......@@ -328,10 +336,16 @@ class DownloadDiskOperation(InstanceOperation):
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = "localhost.man.slow"
def _operation(self, user, url, task, activity, name=None):
from storage.models import Disk
def _operation(self, user, url, datastore, task, activity, name=None):
from storage.models import Disk, DataStore
if not datastore:
datastore = self.instance.get_most_used_datastore()
if not datastore:
datastore = DataStore.get_default_datastore()
disk = Disk.download(url=url, name=name, task=task)
disk = Disk.download(url=url, name=name, task=task,
datastore=datastore)
devnums = list(ascii_lowercase)
for d in self.instance.disks.all():
devnums.remove(d.dev_num)
......@@ -519,7 +533,9 @@ class DestroyOperation(InstanceOperation):
"storage", "fast")
def _get_remote_args(self, **kwargs):
return [self.instance.mem_dump['path']]
return [self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']]
@register_operation
......@@ -546,7 +562,7 @@ class MigrateOperation(RemoteInstanceOperation):
"redeploy network (rollback)")):
self.instance.deploy_net()
def _operation(self, activity, to_node=None, live_migration=True):
def _operation(self, activity, user, to_node=None, live_migration=True):
if not to_node:
with activity.sub_activity('scheduling',
readable_name=ugettext_noop(
......@@ -556,6 +572,16 @@ class MigrateOperation(RemoteInstanceOperation):
try:
with activity.sub_activity(
'refresh_credential', readable_name=create_readable(
ugettext_noop("refresh credential on %(node)s"),
node=to_node)):
ceph_blocks = self.instance.disks.filter(
datastore__type="ceph_block")
if ceph_blocks.exists():
ds = ceph_blocks[0].datastore
to_node.refresh_credential(user=user,
username=ds.ceph_user)
with activity.sub_activity(
'migrate_vm', readable_name=create_readable(
ugettext_noop("migrate to %(node)s"), node=to_node)):
super(MigrateOperation, self)._operation(
......@@ -763,6 +789,7 @@ class SaveAsTemplateOperation(InstanceOperation):
'ram_size': self.instance.ram_size,
'raw_data': self.instance.raw_data,
'system': self.instance.system,
'datastore': self.instance.datastore,
}
params.update(kwargs)
params.pop("parent_activity", None)
......@@ -916,7 +943,9 @@ class SleepOperation(InstanceOperation):
def _get_remote_args(self, **kwargs):
return (super(SleepOperation.SuspendVmOperation, self)
._get_remote_args(**kwargs) +
[self.instance.mem_dump['path']])
[self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']])
@register_operation
......@@ -970,7 +999,9 @@ class WakeUpOperation(InstanceOperation):
def _get_remote_args(self, **kwargs):
return (super(WakeUpOperation.WakeUpVmOperation, self)
._get_remote_args(**kwargs) +
[self.instance.mem_dump['path']])
[self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']])
@register_operation
......@@ -1122,6 +1153,14 @@ class NodeOperation(Operation):
user=user, readable_name=name)
class RemoteNodeOperation(RemoteOperationMixin, NodeOperation):
remote_queue = ('vm', 'fast')
def _get_remote_queue(self):
return self.node.get_remote_queue_name(*self.remote_queue)
@register_operation
class ResetNodeOperation(NodeOperation):
id = 'reset'
......@@ -1312,6 +1351,22 @@ class UpdateNodeOperation(NodeOperation):
@register_operation
class RefreshCredentialOperation(RemoteNodeOperation):
id = 'refresh_credential'
name = _("refresh credential")
description = _("Refresh credential.")
required_perms = ()
task = vm_tasks.refresh_credential
def _get_remote_args(self, **kwargs):
return [kwargs["username"]]
def _operation(self, activity, username):
super(RefreshCredentialOperation, self)._operation(
username=username)
@register_operation
class ScreenshotOperation(RemoteInstanceOperation):
id = 'screenshot'
name = _("screenshot")
......
......@@ -185,3 +185,8 @@ def get_node_metrics(params):
@celery.task(name='vmdriver.screenshot')
def screenshot(params):
pass
@celery.task(name='vmdriver.refresh_secret')
def refresh_credential(user):
pass
......@@ -36,6 +36,15 @@ from ..operations import (
)
class DiskQuerySet(list):
def filter(self, *args, **kwargs):
return DiskQuerySet()
def exists(self):
return False
class PortFinderTestCase(TestCase):
def test_find_unused_port_without_used_ports(self):
......@@ -106,6 +115,7 @@ class InstanceTestCase(TestCase):
inst = Mock(destroyed_at=None, spec=Instance)
inst.interface_set.all.return_value = []
inst.node = MagicMock(spec=Node)
inst.disks = DiskQuerySet()
inst.status = 'RUNNING'
migrate_op = MigrateOperation(inst)
with patch('vm.operations.vm_tasks.migrate') as migr, \
......@@ -124,6 +134,7 @@ class InstanceTestCase(TestCase):
inst.interface_set.all.return_value = []
inst.node = MagicMock(spec=Node)
inst.status = 'RUNNING'
inst.disks = DiskQuerySet()
migrate_op = MigrateOperation(inst)
with patch('vm.operations.vm_tasks.migrate') as migr, \
patch.object(RemoteOperationMixin, "_operation"):
......
......@@ -25,6 +25,7 @@ from vm.operations import (
RebootOperation, ResetOperation, SaveAsTemplateOperation,
ShutdownOperation, ShutOffOperation, SleepOperation, WakeUpOperation,
)
from test_models import DiskQuerySet
class DeployOperationTestCase(TestCase):
......@@ -55,9 +56,10 @@ class MigrateOperationTestCase(TestCase):
op = MigrateOperation(inst)
op._get_remote_args = MagicMock(side_effect=MigrateException())
inst.select_node = MagicMock(return_value='test')
inst.disks = DiskQuerySet()
self.assertRaises(
MigrateException, op._operation,
act, to_node=None)
act, user=None, to_node=None)
assert inst.select_node.called
op._get_remote_args.assert_called_once_with(
to_node='test', live_migration=True)
......
......@@ -11,7 +11,7 @@ django-braces==1.8.0
django-crispy-forms==1.6.0
django-model-utils==2.2
djangosaml2==0.13.0
django-sizefield==0.7
django-sizefield==0.9.1
git+https://git.ik.bme.hu/circle/django-sshkey.git
django-statici18n==1.1.3
django-tables2==0.16.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment