Skip to content

Gelencsér Szabolcs / cloud

Go to a project
Commit 79aeec80 by Czémán Arnold
Options

Merge branch 'ceph' into new_ceph 

Conflicts:
	circle/dashboard/views/storage.py
	circle/storage/models.py
	circle/storage/tasks/periodic_tasks.py
	circle/storage/tasks/storage_tasks.py
	circle/vm/models/instance.py
parents 4413e841 ad35a055
Showing with 1254 additions and 117 deletions
circle/dashboard/fixtures/test-vm-fixture.json
...@@ -20,6 +20,8 @@ ...@@ -20,6 +20,8 @@
"pk": 1, "pk": 1,
"model": "storage.datastore", "model": "storage.datastore",
"fields": { "fields": {
"type": "file",
"ceph_user": null,
"path": "/disks", "path": "/disks",
"hostname": "wut", "hostname": "wut",
"name": "diszkek" "name": "diszkek"
...@@ -36,6 +38,7 @@ ...@@ -36,6 +38,7 @@
"destroyed": null, "destroyed": null,
"base": null, "base": null,
"datastore": 1, "datastore": 1,
"bus": null,
"dev_num": "a", "dev_num": "a",
"type": "qcow2-norm", "type": "qcow2-norm",
"size": 8589934592, "size": 8589934592,
...@@ -170,7 +173,8 @@ ...@@ -170,7 +173,8 @@
"priority": 10, "priority": 10,
"template": null, "template": null,
"access_method": "nx", "access_method": "nx",
"lease": 1, "lease": 1,
"datastore": 1,
"node": null, "node": null,
"description": "", "description": "",
"arch": "x86_64", "arch": "x86_64",
...@@ -200,7 +204,8 @@ ...@@ -200,7 +204,8 @@
"priority": 10, "priority": 10,
"template": null, "template": null,
"access_method": "nx", "access_method": "nx",
"lease": 1, "lease": 1,
"datastore": 1,
"node": null, "node": null,
"description": "", "description": "",
"arch": "x86_64", "arch": "x86_64",
...@@ -308,6 +313,7 @@ ...@@ -308,6 +313,7 @@
"arch": "x86_64", "arch": "x86_64",
"max_ram_size": 1024, "max_ram_size": 1024,
"lease": 1, "lease": 1,
"datastore": 1,
"owner": 1 "owner": 1
} }
} }
......
circle/dashboard/forms.py
...@@ -36,7 +36,6 @@ from crispy_forms.layout import ( ...@@ -36,7 +36,6 @@ from crispy_forms.layout import (
) )
from crispy_forms.utils import render_field from crispy_forms.utils import render_field
from crispy_forms.bootstrap import FormActions
from django import forms from django import forms
from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
...@@ -460,6 +459,9 @@ class NodeForm(forms.ModelForm): ...@@ -460,6 +459,9 @@ class NodeForm(forms.ModelForm):
class TemplateForm(forms.ModelForm): class TemplateForm(forms.ModelForm):
networks = forms.ModelMultipleChoiceField( networks = forms.ModelMultipleChoiceField(
queryset=None, required=False, label=_("Networks")) queryset=None, required=False, label=_("Networks"))
datastore = forms.ModelChoiceField(
queryset=DataStore.objects.filter(destroyed__isnull=True),
empty_label=None)
num_cores = forms.IntegerField(widget=forms.NumberInput(attrs={ num_cores = forms.IntegerField(widget=forms.NumberInput(attrs={
'class': "form-control input-tags cpu-count-input", 'class': "form-control input-tags cpu-count-input",
...@@ -816,10 +818,19 @@ class VmCreateDiskForm(OperationForm): ...@@ -816,10 +818,19 @@ class VmCreateDiskForm(OperationForm):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
default = kwargs.pop('default', None) default = kwargs.pop('default', None)
datastore_choices = kwargs.pop('datastore_choices')
super(VmCreateDiskForm, self).__init__(*args, **kwargs) super(VmCreateDiskForm, self).__init__(*args, **kwargs)
if default: if default:
self.fields['name'].initial = default self.fields['name'].initial = default
datastore_field = forms.ModelChoiceField(
queryset=datastore_choices, required=False, initial=None,
label=_('Data store'))
if not datastore_choices:
datastore_field.widget.attrs['disabled'] = 'disabled'
datastore_field.empty_label = _('No more data stores.')
self.fields['datastore'] = datastore_field
def clean_size(self): def clean_size(self):
size_in_bytes = self.cleaned_data.get("size") size_in_bytes = self.cleaned_data.get("size")
if not size_in_bytes.isdigit() and len(size_in_bytes) > 0: if not size_in_bytes.isdigit() and len(size_in_bytes) > 0:
...@@ -901,6 +912,18 @@ class VmDownloadDiskForm(OperationForm): ...@@ -901,6 +912,18 @@ class VmDownloadDiskForm(OperationForm):
name = forms.CharField(max_length=100, label=_("Name"), required=False) name = forms.CharField(max_length=100, label=_("Name"), required=False)
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ]) url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
def __init__(self, *args, **kwargs):
datastore_choices = kwargs.pop('datastore_choices')
super(VmDownloadDiskForm, self).__init__(*args, **kwargs)
datastore_field = forms.ModelChoiceField(
queryset=datastore_choices, required=False, initial=None,
label=_('Data store'))
if not datastore_choices:
datastore_field.widget.attrs['disabled'] = 'disabled'
datastore_field.empty_label = _('No more data stores.')
self.fields['datastore'] = datastore_field
def clean(self): def clean(self):
cleaned_data = super(VmDownloadDiskForm, self).clean() cleaned_data = super(VmDownloadDiskForm, self).clean()
if not cleaned_data['name']: if not cleaned_data['name']:
...@@ -1429,6 +1452,26 @@ class RawDataForm(forms.ModelForm): ...@@ -1429,6 +1452,26 @@ class RawDataForm(forms.ModelForm):
return helper return helper
class VmDataStoreForm(forms.ModelForm):
datastore = forms.ModelChoiceField(
queryset=DataStore.objects.filter(destroyed__isnull=True),
empty_label=None)
class Meta:
model = Instance
fields = ('datastore', )
@property
def helper(self):
helper = FormHelper()
helper.form_show_labels = False
helper.form_action = reverse_lazy("dashboard.views.vm-data-store",
kwargs={'pk': self.instance.pk})
helper.add_input(Submit("submit", _("Save"),
css_class="btn btn-success", ))
return helper
class GroupPermissionForm(forms.ModelForm): class GroupPermissionForm(forms.ModelForm):
permissions = forms.ModelMultipleChoiceField( permissions = forms.ModelMultipleChoiceField(
queryset=None, queryset=None,
...@@ -1596,16 +1639,61 @@ class DataStoreForm(ModelForm): ...@@ -1596,16 +1639,61 @@ class DataStoreForm(ModelForm):
'name', 'name',
'path', 'path',
'hostname', 'hostname',
),
FormActions(
Submit('submit', _('Save')),
) )
) )
return helper return helper
class Meta: class Meta:
model = DataStore model = DataStore
fields = ("name", "path", "hostname", ) fields = ("type", "name", "path", "hostname", )
widgets = {"type": HiddenInput()}
class CephDataStoreForm(DataStoreForm):
type = forms.CharField(widget=forms.HiddenInput())
@property
def helper(self):
helper = FormHelper()
helper.layout = Layout(
Fieldset(
'',
'ceph_user',
)
)
return helper
class Meta:
model = DataStore
fields = ("type", "name", "path", "hostname",
"ceph_user",)
class StorageListSearchForm(forms.Form):
CHOICES = (
("active", _("active")),
("destroyed", _("destroyed")),
(("all"), _("all")),
)
s = forms.CharField(widget=forms.TextInput(attrs={
'class': "form-control input-tags",
'placeholder': _("Search...")
}))
stype = forms.ChoiceField(CHOICES, widget=forms.Select(attrs={
'class': "btn btn-default input-tags",
}))
def __init__(self, *args, **kwargs):
super(StorageListSearchForm, self).__init__(*args, **kwargs)
# set initial value, otherwise it would be overwritten by request.GET
if not self.data.get("stype"):
data = self.data.copy()
data['stype'] = "active"
self.data = data
class DiskForm(ModelForm): class DiskForm(ModelForm):
......
circle/dashboard/static/dashboard/dashboard.js
...@@ -29,17 +29,26 @@ $(function () { ...@@ -29,17 +29,26 @@ $(function () {
return false; return false;
}); });
$('.group-create, .node-create, .tx-tpl-ownership, .group-delete, .node-delete, .disk-remove, .template-delete, .delete-from-group, .lease-delete').click(function(e) { $('.group-create, .node-create, .tx-tpl-ownership, .group-delete, .node-delete, ' +
'.disk-remove, .template-delete, .delete-from-group, .lease-delete, ' +
'.storage-delete, .storage-restore').click(function(e) {
$.ajax({ $.ajax({
type: 'GET', type: 'GET',
url: $(this).prop('href'), url: $(this).prop('href'),
success: function(data) { success: function(data, _, xhr) {
$('body').append(data); var ctype = xhr.getResponseHeader("content-type") || "";
var modal = $('#confirmation-modal'); if(ctype.indexOf("html") > -1) {
modal.modal('show'); $('body').append(data);
modal.on('hidden.bs.modal', function() { var modal = $('#confirmation-modal');
modal.remove(); modal.modal('show');
}); modal.on('hidden.bs.modal', function() {
modal.remove();
});
}
else if(ctype.indexOf("json") > -1) {
if(data.error !== null && data.error !== undefined)
addMessage(data.error, "warning");
}
}, },
error: function(xhr, textStatus, error) { error: function(xhr, textStatus, error) {
if(xhr.status === 403) { if(xhr.status === 403) {
...@@ -52,7 +61,7 @@ $(function () { ...@@ -52,7 +61,7 @@ $(function () {
return false; return false;
}); });
$('.template-choose').click(function(e) { $('.template-choose, .storage-choose').click(function(e) {
$.ajax({ $.ajax({
type: 'GET', type: 'GET',
url: $(this).prop('href'), url: $(this).prop('href'),
...@@ -73,6 +82,15 @@ $(function () { ...@@ -73,6 +82,15 @@ $(function () {
} }
return true; return true;
}); });
$("#storage-choose-next-button").click(function() {
var radio = $('input[type="radio"]:checked', "#storage-choose-form").val();
if(!radio) {
$("#storage-choose-alert").addClass("alert-warning")
.text(gettext("Select an option to proceed!"));
return false;
}
return true;
});
} }
}); });
return false; return false;
......
circle/dashboard/static/dashboard/dashboard.less
...@@ -348,7 +348,6 @@ a.hover-black { ...@@ -348,7 +348,6 @@ a.hover-black {
width: 100px; width: 100px;
} }
.nojs-dropdown-menu .nojs-dropdown-menu
{ {
position:absolute; position:absolute;
...@@ -471,7 +470,7 @@ footer a, footer a:hover, footer a:visited { ...@@ -471,7 +470,7 @@ footer a, footer a:hover, footer a:visited {
margin-bottom: 20px; margin-bottom: 20px;
} }
.template-choose-list { .template-choose-list, .storage-choose-list {
max-width: 600px; max-width: 600px;
} }
...@@ -481,13 +480,13 @@ footer a, footer a:hover, footer a:visited { ...@@ -481,13 +480,13 @@ footer a, footer a:hover, footer a:visited {
padding-right: 50px; padding-right: 50px;
} }
.template-choose-list-element { .template-choose-list-element, .storage-choose-list-element {
padding: 6px 10px; padding: 6px 10px;
cursor: pointer; cursor: pointer;
margin-bottom: 15px; /* bootstrap panel default is 20px */ margin-bottom: 15px; /* bootstrap panel default is 20px */
} }
.template-choose-list input[type="radio"] { .template-choose-list input[type="radio"], .storage-choose-list input[type="radio"] {
float: right; float: right;
} }
......
circle/dashboard/tables.py
...@@ -27,7 +27,7 @@ from django_tables2.columns import ( ...@@ -27,7 +27,7 @@ from django_tables2.columns import (
) )
from django_sshkey.models import UserKey from django_sshkey.models import UserKey
from storage.models import Disk from storage.models import Disk, DataStore
from vm.models import Node, InstanceTemplate, Lease from vm.models import Node, InstanceTemplate, Lease
from dashboard.models import ConnectCommand, Message from dashboard.models import ConnectCommand, Message
...@@ -371,3 +371,42 @@ class MessageListTable(Table): ...@@ -371,3 +371,42 @@ class MessageListTable(Table):
order_by = ("-pk", ) order_by = ("-pk", )
fields = ('pk', 'message', 'enabled', 'effect') fields = ('pk', 'message', 'enabled', 'effect')
empty_text = _("No messages.") empty_text = _("No messages.")
class StorageListTable(Table):
name = TemplateColumn(
verbose_name=_("Name"),
template_name="dashboard/storage-list/column-storage-name.html",
attrs={'th': {'data-sort': "string"}}
)
type = Column(
verbose_name=_("Type"),
attrs={'th': {'data-sort': "string"}}
)
path = Column(
verbose_name=_("Path or Pool name"),
attrs={'th': {'data-sort': "string"}}
)
hostname = Column(
verbose_name=_("Hostname"),
attrs={'th': {'data-sort': "string"}}
)
used_percent = TemplateColumn(
verbose_name=_("Usage"),
template_name="dashboard/storage-list/" +
"column-storage-used_percent.html",
orderable=False
)
class Meta:
model = DataStore
attrs = {'class': ('table table-bordered table-striped table-hover'
'storage-list-table')}
fields = ('name', 'type', 'path', 'hostname', 'used_percent')
prefix = "storage-"
circle/dashboard/templates/dashboard/_disk-list-element.html
...@@ -35,5 +35,6 @@ ...@@ -35,5 +35,6 @@
{% if request.user.is_superuser %} {% if request.user.is_superuser %}
<small>{% trans "File name" %}: {{ d.filename }}</small><br/> <small>{% trans "File name" %}: {{ d.filename }}</small><br/>
<small>{% trans "Bus" %}: {{ d.device_bus }}</small> <small>{% trans "Bus" %}: {{ d.device_bus }}</small><br/>
<small>{% trans "Data store" %}: {{ d.datastore }}</small>
{% endif %} {% endif %}
circle/dashboard/templates/dashboard/_storage-choose.html 0 → 100644
{% load i18n %}
<div class="alert alert-info" id="storage-choose-alert">
{% trans "Choose the type of the data store that you want to create." %}
</div>
<form action="{% url "dashboard.views.storage-choose" %}" method="POST"
id="storage-choose-form">
{% csrf_token %}
<div class="storage-choose-list">
{% for t in types %}
<div class="panel panel-default storage-choose-list-element">
<input type="radio" name="type" value="{{ t.0 }}"/>
{{ t.1 }}
<div class="clearfix"></div>
</div>
{% endfor %}
<button type="submit" id="storage-choose-next-button" class="btn btn-success pull-right">{% trans "Next" %}</button>
<div class="clearfix"></div>
</div>
</form>
<script>
$(function() {
$(".storage-choose-list-element").click(function() {
$("input", $(this)).prop("checked", true);
});
$(".storage-choose-list-element").hover(
function() {
$("small", $(this)).stop().fadeIn(200);
},
function() {
$("small", $(this)).stop().fadeOut(200);
}
);
});
</script>
circle/dashboard/templates/dashboard/_storage-create.html 0 → 100644
{% load i18n %}
{% load crispy_forms_tags %}
<form id="storage-create-form" action="" method="POST">
{% include "dashboard/storage/form_chunk.html" %}
<fieldset>
<input type="submit" value="{% trans "Create new data store" %}" class="btn btn-success">
</fieldset>
</form>
circle/dashboard/templates/dashboard/_template-create.html
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
<fieldset> <fieldset>
<legend>{% trans "External resources" %}</legend> <legend>{% trans "External resources" %}</legend>
{{ form.networks|as_crispy_field }} {{ form.networks|as_crispy_field }}
{{ form.datastore|as_crispy_field }}
{{ form.lease|as_crispy_field }} {{ form.lease|as_crispy_field }}
{% if show_lease_create %} {% if show_lease_create %}
......
circle/dashboard/templates/dashboard/base.html
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
</a> </a>
</li> </li>
<li> <li>
<a href="{% url "dashboard.views.storage" %}"> <a href="{% url "dashboard.views.storage-list" %}">
<i class="fa fa-database"></i> <i class="fa fa-database"></i>
<span class="hidden-sm">{% trans "Storage" %}</span> <span class="hidden-sm">{% trans "Storage" %}</span>
</a> </a>
......
circle/dashboard/templates/dashboard/confirm/ajax-restore.html 0 → 100644
{% load i18n %}
<div class="modal fade" id="confirmation-modal" tabindex="-1" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-body">
{% blocktrans with object=object %}
Are you sure you want to restore <strong>{{ object }}</strong>?
{% endblocktrans %}
<br />
<div class="pull-right" style="margin-top: 15px;">
<form action="{{ request.path }}" method="POST">
{% csrf_token %}
<button type="button" class="btn btn-default" data-dismiss="modal">{% trans "Cancel" %}</button>
<input type="hidden" name="next" value="{{ request.GET.next }}"/>
<button class="btn btn-warning modal-accept"
{% if disable_submit %}disabled{% endif %}
>{% trans "Restore" %}</button>
</form>
</div>
<div class="clearfix"></div>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div>
circle/dashboard/templates/dashboard/confirm/base-restore.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load i18n %}
{% block content %}
<div class="body-content">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin">
{% if title %}
{{ title }}
{% else %}
{% trans "Restore confirmation" %}
{% endif %}
</h3>
</div>
<div class="panel-body">
{% if text %}
{{ text|safe }}
{% else %}
{% blocktrans with object=object %}
Are you sure you want to restore <strong>{{ object }}</strong>?
{% endblocktrans %}
{% endif %}
<div class="pull-right">
<form action="{{ request.path }}" method="POST">
{% csrf_token %}
<a class="btn btn-default">{% trans "Cancel" %}</a>
<input type="hidden" name="next" value="{{ request.GET.next }}"/>
<button class="btn btn-warning"
{% if disable_submit %}disabled{% endif %}
>{% trans "Yes" %}</button>
</form>
</div>
</div>
</div>
</div>
{% endblock %}
circle/dashboard/templates/dashboard/storage-list.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load staticfiles %}
{% load i18n %}
{% load render_table from django_tables2 %}
{% block title-page %}{% trans "Data stores" %}{% endblock %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<div class="pull-right">
<a href="{% url "dashboard.views.storage-choose" %}" class="btn btn-success btn-xs storage-choose">
<i class="fa fa-plus"></i> {% trans "new data store" %}
</a>
</div>
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Data stores" %}</h3>
</div>
<div class="panel-body">
<div class="row">
<div class="col-md-offset-8 col-md-4" id="storage-list-search">
<form action="" method="GET">
<div class="input-group">
{{ search_form.s }}
<div class="input-group-btn">
{{ search_form.stype }}
<button type="submit" class="btn btn-primary input-tags">
<i class="fa fa-search"></i>
</button>
</div>
</div><!-- .input-group -->
</form>
</div><!-- .col-md-4 #storage-list-search -->
</div>
</div>
<div class="panel-body">
<div class="table-responsive">
{% render_table table %}
</div>
</div>
</div>
</div>
</div>
{% endblock %}
circle/dashboard/templates/dashboard/storage-list/column-storage-name.html 0 → 100644
<a href="{% url "dashboard.views.storage-detail" pk=record.pk %}" title="{{ record.description }}">
{{ record.name }}
</a>
circle/dashboard/templates/dashboard/storage-list/column-storage-used_percent.html 0 → 100644
<div class="progress">
<div class="progress-bar progress-bar-success progress-bar-stripped"
role="progressbar" style="min-width: 30px; width: {{ record.used_percent }}%">
{{ record.used_percent }}%
</div>
</div>
circle/dashboard/templates/dashboard/storage/detail.html
...@@ -2,24 +2,49 @@ ...@@ -2,24 +2,49 @@
{% load staticfiles %} {% load staticfiles %}
{% load i18n %} {% load i18n %}
{% load render_table from django_tables2 %} {% load render_table from django_tables2 %}
{% load crispy_forms_tags %}
{% block title-page %}{% trans "Storage" %}{% endblock %} {% block title-page %}{% trans "Storage" %}{% endblock %}
{% block content %} {% block content %}
<div class="row"> <div class="row">
<div class="col-md-5"> <div class="col-md-6">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Datastore" %}</h3> <h3 class="no-margin"><i class="fa fa-database"></i> {% trans "Data store" %}</h3>
</div> </div>
<div class="panel-body"> <div class="panel-body">
{% crispy form %} <form id="storage-create-form" action="" method="POST">
{% include "dashboard/storage/form_chunk.html" %}
<fieldset>
<input type="submit" value="{% trans "Save" %}" class="btn btn-primary">
</fieldset>
</form>
</div><!-- .panel-body --> </div><!-- .panel-body -->
</div> </div>
</div> </div>
<div class="col-md-7">
<div class="col-md-6">
<div class="panel panel-default">
<div class="panel-heading">
{% if object.destroyed %}
<a href="{% url "dashboard.views.storage-restore" pk=object.pk %}"
class="btn btn-xs btn-warning pull-right storage-restore">
{% trans "Restore" %}
</a>
<h4 class="no-margin"><i class="fa fa-medkit"></i> {% trans "Restore data store" %}</h4>
{% else %}
<a href="{% url "dashboard.views.storage-delete" pk=object.pk %}"
class="btn btn-xs btn-danger pull-right storage-delete">
{% trans "Delete" %}
</a>
<h4 class="no-margin"><i class="fa fa-times"></i> {% trans "Delete data store" %}</h4>
{% endif %}
</div>
</div>
</div>
<div class="col-md-6">
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-bar-chart"></i> {% trans "Statistics" %}</h3> <h3 class="no-margin"><i class="fa fa-bar-chart"></i> {% trans "Statistics" %}</h3>
...@@ -104,5 +129,4 @@ ...@@ -104,5 +129,4 @@
</div> </div>
</div> </div>
</div> </div>
{% endblock %}
{% endblock %}
circle/dashboard/templates/dashboard/storage/form_chunk.html 0 → 100644
{% load i18n %}
{% load crispy_forms_tags %}
{% with form=form %}
{% include "display-form-errors.html" %}
{% endwith %}
{% csrf_token %}
{{ form.type }}
<fieldset>
<legend>{% trans "General settings" %}</legend>
{{ form.name|as_crispy_field }}
{{ form.path|as_crispy_field }}
{{ form.hostname|as_crispy_field }}
</fieldset>
{% if form.type.value == "ceph_block" %}
<fieldset>
<legend>{% trans "Ceph block storage authentication settings" %}</legend>
{{ form.ceph_user|as_crispy_field }}
</fieldset>
{% endif %}
<style>
fieldset{
margin-top: 10px;
margin-bottom: 10px;
}
fieldset legend {
font-weight: bold;
}
</style>
circle/dashboard/templates/dashboard/template-edit.html
...@@ -57,6 +57,7 @@ ...@@ -57,6 +57,7 @@
<legend>{% trans "External resources" %}</legend> <legend>{% trans "External resources" %}</legend>
{{ form.networks|as_crispy_field }} {{ form.networks|as_crispy_field }}
{{ form.lease|as_crispy_field }} {{ form.lease|as_crispy_field }}
{{ form.datastore|as_crispy_field }}
{{ form.tags|as_crispy_field }} {{ form.tags|as_crispy_field }}
</fieldset> </fieldset>
......
circle/dashboard/templates/dashboard/vm-detail/resources.html
...@@ -102,6 +102,17 @@ ...@@ -102,6 +102,17 @@
</div> </div>
</div> </div>
<hr/>
<div class="row">
<div class="col-sm-12">
<h3>
{% trans "Data store" %}
</h3>
{% crispy data_store_form %}
</div>
</div>
{% endif %} {% endif %}
{% block extra_js %} {% block extra_js %}
......
circle/dashboard/tests/selenium/util.py
...@@ -343,8 +343,9 @@ class CircleSeleniumMixin(SeleniumMixin): ...@@ -343,8 +343,9 @@ class CircleSeleniumMixin(SeleniumMixin):
'Cannot save a vm as a template') 'Cannot save a vm as a template')
def create_base_template(self, name=None, architecture="x86-64", def create_base_template(self, name=None, architecture="x86-64",
method=None, op_system=None, lease=None, method=None, op_system=None,
network="vm"): datastore="default",
lease=None, network="vm"):
if name is None: if name is None:
name = "new_%s" % self.conf.client_name name = "new_%s" % self.conf.client_name
if op_system is None: if op_system is None:
...@@ -370,6 +371,8 @@ class CircleSeleniumMixin(SeleniumMixin): ...@@ -370,6 +371,8 @@ class CircleSeleniumMixin(SeleniumMixin):
system_name.clear() system_name.clear()
system_name.send_keys(op_system) system_name.send_keys(op_system)
self.select_option(self.driver.find_element_by_id( self.select_option(self.driver.find_element_by_id(
"id_datastore"), datastore)
self.select_option(self.driver.find_element_by_id(
"id_lease"), lease) "id_lease"), lease)
self.select_option(self.driver.find_element_by_id( self.select_option(self.driver.find_element_by_id(
"id_networks"), network) "id_networks"), network)
......
circle/dashboard/tests/test_views.py
...@@ -253,7 +253,8 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase): ...@@ -253,7 +253,8 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
tmpl.set_level(self.u1, 'owner') tmpl.set_level(self.u1, 'owner')
Vlan.objects.get(id=1).set_level(self.u1, 'user') Vlan.objects.get(id=1).set_level(self.u1, 'user')
kwargs = tmpl.__dict__.copy() kwargs = tmpl.__dict__.copy()
kwargs.update(name='t1', lease=1, disks=1, raw_data='tst1') kwargs.update(name='t1', lease=1, disks=1,
raw_data='tst1', datastore=1)
response = c.post('/dashboard/template/1/', kwargs) response = c.post('/dashboard/template/1/', kwargs)
self.assertEqual(response.status_code, 302) self.assertEqual(response.status_code, 302)
self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data, self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data,
...@@ -264,7 +265,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase): ...@@ -264,7 +265,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
self.login(c, 'superuser') self.login(c, 'superuser')
kwargs = InstanceTemplate.objects.get(id=1).__dict__.copy() kwargs = InstanceTemplate.objects.get(id=1).__dict__.copy()
kwargs.update(name='t2', lease=1, disks=1, kwargs.update(name='t2', lease=1, disks=1,
raw_data='<devices></devices>') raw_data='<devices></devices>', datastore=1)
response = c.post('/dashboard/template/1/', kwargs) response = c.post('/dashboard/template/1/', kwargs)
self.assertEqual(response.status_code, 302) self.assertEqual(response.status_code, 302)
self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data, self.assertEqual(InstanceTemplate.objects.get(id=1).raw_data,
......
circle/dashboard/urls.py
...@@ -42,7 +42,7 @@ from .views import ( ...@@ -42,7 +42,7 @@ from .views import (
ConnectCommandDelete, ConnectCommandDetail, ConnectCommandCreate, ConnectCommandDelete, ConnectCommandDetail, ConnectCommandCreate,
StoreList, store_download, store_upload, store_get_upload_url, StoreRemove, StoreList, store_download, store_upload, store_get_upload_url, StoreRemove,
store_new_directory, store_refresh_toplist, store_new_directory, store_refresh_toplist,
VmTraitsUpdate, VmRawDataUpdate, VmTraitsUpdate, VmRawDataUpdate, VmDataStoreUpdate,
GroupPermissionsView, GroupPermissionsView,
LeaseAclUpdateView, LeaseAclUpdateView,
toggle_template_tutorial, toggle_template_tutorial,
...@@ -53,7 +53,8 @@ from .views import ( ...@@ -53,7 +53,8 @@ from .views import (
OpenSearchDescriptionView, OpenSearchDescriptionView,
NodeActivityView, NodeActivityView,
UserList, UserList,
StorageDetail, DiskDetail, StorageDetail, StorageList, StorageChoose, StorageCreate, DiskDetail,
StorageDelete, StorageRestore,
MessageList, MessageDetail, MessageCreate, MessageDelete, MessageList, MessageDetail, MessageCreate, MessageDelete,
) )
from .views.vm import vm_ops, vm_mass_ops from .views.vm import vm_ops, vm_mass_ops
...@@ -113,6 +114,8 @@ urlpatterns = patterns( ...@@ -113,6 +114,8 @@ urlpatterns = patterns(
name='dashboard.views.vm-traits'), name='dashboard.views.vm-traits'),
url(r'^vm/(?P<pk>\d+)/raw_data/$', VmRawDataUpdate.as_view(), url(r'^vm/(?P<pk>\d+)/raw_data/$', VmRawDataUpdate.as_view(),
name='dashboard.views.vm-raw-data'), name='dashboard.views.vm-raw-data'),
url(r'^vm/(?P<pk>\d+)/data_store/$', VmDataStoreUpdate.as_view(),
name='dashboard.views.vm-data-store'),
url(r'^vm/(?P<pk>\d+)/toggle_tutorial/$', toggle_template_tutorial, url(r'^vm/(?P<pk>\d+)/toggle_tutorial/$', toggle_template_tutorial,
name='dashboard.views.vm-toggle-tutorial'), name='dashboard.views.vm-toggle-tutorial'),
...@@ -235,8 +238,19 @@ urlpatterns = patterns( ...@@ -235,8 +238,19 @@ urlpatterns = patterns(
url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(), url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
name="dashboard.views.vm-opensearch"), name="dashboard.views.vm-opensearch"),
url(r'^storage/$', StorageDetail.as_view(), url(r'^storage/create/(?P<type>.+)$', StorageCreate.as_view(),
name="dashboard.views.storage"), name="dashboard.views.storage-create"),
url(r'^storage/(?P<pk>\d+)/$', StorageDetail.as_view(),
name='dashboard.views.storage-detail'),
url(r'^storage/list/$', StorageList.as_view(),
name="dashboard.views.storage-list"),
url(r'^storage/choose/$', StorageChoose.as_view(),
name="dashboard.views.storage-choose"),
url(r"^storage/delete/(?P<pk>\d+)/$", StorageDelete.as_view(),
name="dashboard.views.storage-delete"),
url(r"^storage/restore/(?P<pk>\d+)/$", StorageRestore.as_view(),
name="dashboard.views.storage-restore"),
url(r'^disk/(?P<pk>\d+)/$', DiskDetail.as_view(), url(r'^disk/(?P<pk>\d+)/$', DiskDetail.as_view(),
name="dashboard.views.disk-detail"), name="dashboard.views.disk-detail"),
......
circle/dashboard/views/storage.py
...@@ -16,19 +16,182 @@ ...@@ -16,19 +16,182 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals, absolute_import from __future__ import unicode_literals, absolute_import
import logging
from django.contrib import messages from django.contrib import messages
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse, reverse_lazy
from django.db.models import Q from django.db.models import Q
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _, ugettext
from django.views.generic import UpdateView from django.views.generic import (
UpdateView, TemplateView, CreateView, DeleteView
)
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import redirect
from django_tables2 import SingleTableView
from django.http import (
Http404, HttpResponseRedirect, JsonResponse
)
from django.core.exceptions import PermissionDenied
from braces.views import SuperuserRequiredMixin from braces.views import SuperuserRequiredMixin
from sizefield.utils import filesizeformat from sizefield.utils import filesizeformat
from common.models import WorkerNotFound from common.models import WorkerNotFound
from storage.models import DataStore, Disk from storage.models import DataStore, Disk
from ..tables import DiskListTable from ..tables import DiskListTable, StorageListTable
from ..forms import DataStoreForm, DiskForm from ..forms import (
DataStoreForm, CephDataStoreForm, DiskForm, StorageListSearchForm
)
from .util import FilterMixin
import json
from celery.exceptions import TimeoutError
from vm.models import Node
logger = logging.getLogger(__name__)
class StorageChoose(SuperuserRequiredMixin, TemplateView):
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/_modal.html']
else:
return ['dashboard/nojs-wrapper.html']
def get_context_data(self, *args, **kwargs):
context = super(StorageChoose, self).get_context_data(*args, **kwargs)
types = DataStore.TYPES
context.update({
'box_title': _('Choose data store type'),
'ajax_title': True,
'template': "dashboard/_storage-choose.html",
'types': types,
})
return context
def post(self, request, *args, **kwargs):
if not request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
type = request.POST.get("type")
if any(type in t for t in DataStore.TYPES):
return redirect(reverse("dashboard.views.storage-create",
kwargs={"type": type}))
else:
messages.warning(request, _("Select an option to proceed."))
return redirect(reverse("dashboard.views.storage-choose"))
class StorageCreate(SuccessMessageMixin, CreateView):
model = DataStore
form = None
def get_template_names(self):
if self.request.is_ajax():
pass
else:
return ['dashboard/nojs-wrapper.html']
def get_context_data(self, *args, **kwargs):
context = super(StorageCreate, self).get_context_data(*args, **kwargs)
context.update({
'box_title': _("Create a new data store"),
'template': "dashboard/_storage-create.html",
})
return context
def get(self, *args, **kwargs):
if not self.request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
return super(StorageCreate, self).get(*args, **kwargs)
def post(self, request, *args, **kwargs):
if not self.request.user.has_perm('storage.add_datastore'):
raise PermissionDenied()
self.form = self.form_class(request.POST)
if not self.form.is_valid():
logger.debug("invalid form")
return self.get(request, self.form, *args, **kwargs)
else:
self.form.save()
return redirect(self.get_success_url())
def get_success_url(self):
return reverse_lazy("dashboard.views.storage-list")
def get_form(self):
if self.form is not None:
return self.form
else:
type = self.kwargs.get("type")
fc = self.form_class
f = fc(initial={"type": type})
return f
@property
def form_class(self):
type = self.kwargs.get("type")
if type == "file":
fc = DataStoreForm
elif type == "ceph_block":
fc = CephDataStoreForm
else:
raise Http404(_("Invalid creation type"))
return fc
class StorageList(SuperuserRequiredMixin, FilterMixin, SingleTableView):
template_name = "dashboard/storage-list.html"
model = DataStore
table_class = StorageListTable
table_pagination = False
allowed_filters = {
'name': "name__icontains",
'type': "type__icontains",
'path': "path__icontains",
'poolname': "path__icontains",
'hostname': "hostname__iexact",
}
def get_context_data(self, *args, **kwargs):
context = super(StorageList, self).get_context_data(*args, **kwargs)
context['search_form'] = self.search_form
return context
def get(self, *args, **kwargs):
self.search_form = StorageListSearchForm(self.request.GET)
self.search_form.full_clean()
return super(StorageList, self).get(*args, **kwargs)
def get_queryset(self):
logger.debug('StorageList.get_queryset() called. User: %s',
unicode(self.request.user))
cleaned_data = self.search_form.cleaned_data
stype = cleaned_data.get('stype', "all")
qs = self.get_queryset_by_stype(stype)
self.create_fake_get()
try:
filters, excludes = self.get_queryset_filters()
qs = qs.filter(**filters).exclude(**excludes).distinct()
except ValueError:
messages.error(self.request, _("Error during filtering."))
return qs
def get_queryset_by_stype(self, stype):
if stype == "all":
return DataStore.get_all()
elif stype == "destroyed":
return DataStore.objects.filter(destroyed__isnull=False)
else:
return DataStore.objects.filter(destroyed__isnull=True)
class StorageDetail(SuperuserRequiredMixin, UpdateView): class StorageDetail(SuperuserRequiredMixin, UpdateView):
...@@ -36,12 +199,8 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView): ...@@ -36,12 +199,8 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
form_class = DataStoreForm form_class = DataStoreForm
template_name = "dashboard/storage/detail.html" template_name = "dashboard/storage/detail.html"
def get_object(self):
return DataStore.get_default_datastore()
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
context = super(StorageDetail, self).get_context_data(**kwargs) context = super(StorageDetail, self).get_context_data(**kwargs)
try: try:
ds = self.get_object() ds = self.get_object()
context['stats'] = self._get_stats() context['stats'] = self._get_stats()
...@@ -49,6 +208,13 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView): ...@@ -49,6 +208,13 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
context['orphan_disks'] = ds.get_orphan_disks() context['orphan_disks'] = ds.get_orphan_disks()
except WorkerNotFound: except WorkerNotFound:
messages.error(self.request, _("The DataStore is offline.")) messages.error(self.request, _("The DataStore is offline."))
except TimeoutError:
messages.error(self.request, _("Operation timed out, "
"some data may insufficient."))
except Exception as e:
messages.error(self.request, _("Error occured: %s, "
"some data may insufficient."
% unicode(e)))
context['disk_table'] = DiskListTable( context['disk_table'] = DiskListTable(
self.get_table_data(), request=self.request, self.get_table_data(), request=self.request,
...@@ -105,8 +271,108 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView): ...@@ -105,8 +271,108 @@ class StorageDetail(SuperuserRequiredMixin, UpdateView):
'total_space': filesizeformat(total_space), 'total_space': filesizeformat(total_space),
} }
def get_form_class(self):
ds = self.get_object()
if ds.type == "ceph_block":
return CephDataStoreForm
else:
return DataStoreForm
def get_success_url(self):
ds = self.get_object()
return reverse("dashboard.views.storage-detail", kwargs={"pk": ds.id})
def form_valid(self, form):
# automatic credential refresh
changed = (self.object.type == "ceph_block" and
self.object.tracker.has_changed("ceph_user"))
response = super(StorageDetail, self).form_valid(form)
if changed:
nodes = Node.objects.all()
for node in nodes:
if node.get_online():
node.refresh_credential(
user=self.request.user,
username=self.object.ceph_user)
return response
class StorageDelete(SuperuserRequiredMixin, DeleteView):
model = DataStore
success_message = _("Storage successfully destroyed.")
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-delete.html']
else:
return ['dashboard/confirm/base-delete.html']
def check_destroyable(self):
object = self.get_object()
if not object.is_destroyable:
raise PermissionDenied()
def get(self, request, *args, **kwargs):
try:
self.check_destroyable()
except PermissionDenied:
message = ugettext("Another object references"
" to the selected object.")
if request.is_ajax():
return JsonResponse({"error": message})
else:
messages.warning(request, message)
return redirect(self.get_success_url())
return super(StorageDelete, self).get(request, *args, **kwargs)
def get_success_url(self): def get_success_url(self):
return reverse("dashboard.views.storage") return reverse_lazy("dashboard.views.storage-list")
def delete_obj(self, request, *args, **kwargs):
self.get_object().destroy()
def delete(self, request, *args, **kwargs):
self.check_destroyable()
self.delete_obj(request, *args, **kwargs)
if request.is_ajax():
return JsonResponse(
json.dumps({'message': self.success_message}),
)
else:
messages.success(request, self.success_message)
return HttpResponseRedirect(self.get_success_url())
class StorageRestore(SuperuserRequiredMixin, UpdateView):
model = DataStore
fields = ("destroyed",)
success_message = _("Data store successfully restored.")
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-restore.html']
else:
return ['dashboard/confirm/base-restore.html']
def form_valid(self, form):
object = self.get_object()
object.destroyed = None
object.save()
if self.request.is_ajax():
return JsonResponse(
json.dumps({'message': self.success_message}),
)
else:
messages.success(self.request, self.success_message)
return HttpResponseRedirect(self.get_success_url())
def get_success_url(self):
ds = self.get_object()
return reverse_lazy("dashboard.views.storage-detail",
kwargs={"pk": ds.id})
class DiskDetail(SuperuserRequiredMixin, UpdateView): class DiskDetail(SuperuserRequiredMixin, UpdateView):
......
circle/dashboard/views/vm.py
...@@ -49,7 +49,7 @@ from common.models import ( ...@@ -49,7 +49,7 @@ from common.models import (
) )
from firewall.models import Vlan, Host, Rule from firewall.models import Vlan, Host, Rule
from manager.scheduler import SchedulerError from manager.scheduler import SchedulerError
from storage.models import Disk from storage.models import Disk, DataStore
from vm.models import ( from vm.models import (
Instance, InstanceActivity, Node, Lease, Instance, InstanceActivity, Node, Lease,
InstanceTemplate, InterfaceTemplate, Interface, InstanceTemplate, InterfaceTemplate, Interface,
...@@ -66,7 +66,7 @@ from ..forms import ( ...@@ -66,7 +66,7 @@ from ..forms import (
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm, VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm, VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm, VmRemoveInterfaceForm, VmDataStoreForm,
) )
from request.models import TemplateAccessType, LeaseType from request.models import TemplateAccessType, LeaseType
from request.forms import LeaseRequestForm, TemplateRequestForm from request.forms import LeaseRequestForm, TemplateRequestForm
...@@ -173,6 +173,7 @@ class VmDetailView(GraphMixin, CheckedDetailView): ...@@ -173,6 +173,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
if self.request.user.is_superuser: if self.request.user.is_superuser:
context['traits_form'] = TraitsForm(instance=instance) context['traits_form'] = TraitsForm(instance=instance)
context['raw_data_form'] = RawDataForm(instance=instance) context['raw_data_form'] = RawDataForm(instance=instance)
context['data_store_form'] = VmDataStoreForm(instance=instance)
# resources change perm # resources change perm
context['can_change_resources'] = self.request.user.has_perm( context['can_change_resources'] = self.request.user.has_perm(
...@@ -323,6 +324,14 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView): ...@@ -323,6 +324,14 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
return self.get_object().get_absolute_url() + "#resources" return self.get_object().get_absolute_url() + "#resources"
class VmDataStoreUpdate(SuperuserRequiredMixin, UpdateView):
form_class = VmDataStoreForm
model = Instance
def get_success_url(self):
return self.get_object().get_absolute_url() + "#resources"
class VmOperationView(AjaxOperationMixin, OperationView): class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance model = Instance
...@@ -428,6 +437,8 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView): ...@@ -428,6 +437,8 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
val = super(VmCreateDiskView, self).get_form_kwargs() val = super(VmCreateDiskView, self).get_form_kwargs()
num = op.instance.disks.count() + 1 num = op.instance.disks.count() + 1
val['default'] = "%s %d" % (op.instance.name, num) val['default'] = "%s %d" % (op.instance.name, num)
val['datastore_choices'] = DataStore.objects.filter(
destroyed__isnull=True)
return val return val
...@@ -441,6 +452,12 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView): ...@@ -441,6 +452,12 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
is_disk_operation = True is_disk_operation = True
with_reload = True with_reload = True
def get_form_kwargs(self):
val = super(VmDownloadDiskView, self).get_form_kwargs()
val['datastore_choices'] = DataStore.objects.filter(
destroyed__isnull=True)
return val
class VmMigrateView(FormOperationMixin, VmOperationView): class VmMigrateView(FormOperationMixin, VmOperationView):
......
circle/request/tests.py
...@@ -30,7 +30,7 @@ from dashboard.tests.test_views import LoginMixin ...@@ -30,7 +30,7 @@ from dashboard.tests.test_views import LoginMixin
from vm.operations import ResourcesOperation from vm.operations import ResourcesOperation
class RequestTest(LoginMixin, MockCeleryMixin, TestCase): class RequestTestBase(LoginMixin, MockCeleryMixin, TestCase):
fixtures = ['test-vm-fixture.json', 'node.json'] fixtures = ['test-vm-fixture.json', 'node.json']
def setUp(self): def setUp(self):
...@@ -57,10 +57,12 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase): ...@@ -57,10 +57,12 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
tat.templates.add(InstanceTemplate.objects.get(pk=1)) tat.templates.add(InstanceTemplate.objects.get(pk=1))
def tearDown(self): def tearDown(self):
super(RequestTest, self).tearDown() super(RequestTestBase, self).tearDown()
self.u1.delete() self.u1.delete()
self.us.delete() self.us.delete()
class ResourceRequestTest(RequestTestBase):
def test_resources_request(self): def test_resources_request(self):
c = Client() c = Client()
self.login(c, "user1") self.login(c, "user1")
...@@ -98,6 +100,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase): ...@@ -98,6 +100,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
new_request = Request.objects.latest("pk") new_request = Request.objects.latest("pk")
self.assertEqual(new_request.status, "ACCEPTED") self.assertEqual(new_request.status, "ACCEPTED")
class TemplateAccessRequestTest(RequestTestBase):
def test_template_access_request(self): def test_template_access_request(self):
c = Client() c = Client()
self.login(c, "user1") self.login(c, "user1")
...@@ -121,6 +125,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase): ...@@ -121,6 +125,8 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
self.assertEqual(new_request.status, "ACCEPTED") self.assertEqual(new_request.status, "ACCEPTED")
self.assertTrue(template.has_level(self.u1, "user")) self.assertTrue(template.has_level(self.u1, "user"))
class LeaseRequestTest(RequestTestBase):
def test_lease_request(self): def test_lease_request(self):
c = Client() c = Client()
self.login(c, "user1") self.login(c, "user1")
......
circle/storage/migrations/0003_auto_20151122_2104.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0002_disk_bus'),
]
operations = [
migrations.CreateModel(
name='DataStoreHost',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('address', models.CharField(max_length=1024, verbose_name='address')),
('port', models.IntegerField(null=True, verbose_name='port', blank=True)),
],
),
migrations.AddField(
model_name='datastore',
name='ceph_user',
field=models.CharField(max_length=255, null=True, verbose_name='Ceph username', blank=True),
),
migrations.AddField(
model_name='datastore',
name='secret_uuid',
field=models.CharField(max_length=255, null=True, verbose_name='uuid of secret', blank=True),
),
migrations.AddField(
model_name='datastore',
name='type',
field=models.CharField(default='file', max_length=10, verbose_name='type', choices=[('file', 'filesystem'), ('ceph_block', 'Ceph block device')]),
),
migrations.AlterField(
model_name='datastore',
name='hostname',
field=models.CharField(max_length=40, verbose_name='hostname'),
),
migrations.AlterField(
model_name='disk',
name='type',
field=models.CharField(max_length=10, choices=[('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'), ('ceph-norm', 'Ceph block normal'), ('ceph-snap', 'Ceph block snapshot'), ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]),
),
migrations.AddField(
model_name='datastore',
name='hosts',
field=models.ManyToManyField(to='storage.DataStoreHost', verbose_name='hosts', blank=True),
),
]
circle/storage/migrations/0004_auto_20151212_0402.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import storage.models
class Migration(migrations.Migration):
dependencies = [
('storage', '0003_auto_20151122_2104'),
]
operations = [
migrations.AddField(
model_name='datastorehost',
name='name',
field=models.CharField(default='Monitor1', unique=True, max_length=255, verbose_name='name'),
preserve_default=False,
),
migrations.AlterField(
model_name='datastore',
name='path',
field=models.CharField(unique=True, max_length=200, verbose_name='path', validators=[storage.models.validate_ascii]),
),
migrations.AlterField(
model_name='disk',
name='filename',
field=models.CharField(unique=True, max_length=256, verbose_name='filename', validators=[storage.models.validate_ascii]),
),
]
circle/storage/migrations/0005_auto_20160331_1823.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import storage.models
class Migration(migrations.Migration):
dependencies = [
('storage', '0004_auto_20151212_0402'),
]
operations = [
migrations.AlterField(
model_name='datastore',
name='path',
field=models.CharField(unique=True, max_length=200, verbose_name='path or poolname', validators=[storage.models.validate_ascii]),
),
migrations.AlterField(
model_name='datastore',
name='secret_uuid',
field=models.CharField(max_length=255, null=True, verbose_name='uuid of secret key', blank=True),
),
]
circle/storage/migrations/0006_auto_20160505_0824.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0005_auto_20160331_1823'),
]
operations = [
migrations.CreateModel(
name='Endpoint',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(unique=True, max_length=255, verbose_name='name')),
('address', models.CharField(max_length=1024, verbose_name='address')),
('port', models.IntegerField(null=True, verbose_name='port', blank=True)),
],
),
migrations.RemoveField(
model_name='datastore',
name='hosts',
),
migrations.DeleteModel(
name='DataStoreHost',
),
migrations.AddField(
model_name='datastore',
name='endpoints',
field=models.ManyToManyField(to='storage.Endpoint', verbose_name='endpoints', blank=True),
),
]
circle/storage/migrations/0007_datastore_destroyed.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0006_auto_20160505_0824'),
]
operations = [
migrations.AddField(
model_name='datastore',
name='destroyed',
field=models.DateTimeField(default=None, null=True, blank=True),
),
]
circle/storage/migrations/0008_auto_20160609_2338.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0007_datastore_destroyed'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='secret_uuid',
),
migrations.AddField(
model_name='datastore',
name='secret',
field=models.CharField(max_length=255, null=True, verbose_name='secret key', blank=True),
),
]
circle/storage/migrations/0009_auto_20160614_1125.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0008_auto_20160609_2338'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='endpoints',
),
migrations.DeleteModel(
name='Endpoint',
),
]
circle/storage/migrations/0010_remove_datastore_secret.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('storage', '0009_auto_20160614_1125'),
]
operations = [
migrations.RemoveField(
model_name='datastore',
name='secret',
),
]
circle/storage/models.py
...@@ -28,11 +28,12 @@ from django.conf import settings ...@@ -28,11 +28,12 @@ from django.conf import settings
from celery.contrib.abortable import AbortableAsyncResult from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField, from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey) ForeignKey)
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
from django.utils import timezone from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
from model_utils import FieldTracker
from sizefield.models import FileSizeField from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks from .tasks import local_tasks, storage_tasks
...@@ -44,14 +45,35 @@ from common.models import ( ...@@ -44,14 +45,35 @@ from common.models import (
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def validate_ascii(value):
try:
str(value)
except UnicodeEncodeError:
raise ValidationError("%s is not 'ascii' string" % value)
class DataStore(Model): class DataStore(Model):
"""Collection of virtual disks. """Collection of virtual disks.
""" """
TYPES = (('file', 'filesystem'), ('ceph_block', 'Ceph block device'))
type = CharField(max_length=10, verbose_name=_('type'),
default='file', choices=TYPES)
name = CharField(max_length=100, unique=True, verbose_name=_('name')) name = CharField(max_length=100, unique=True, verbose_name=_('name'))
path = CharField(max_length=200, unique=True, verbose_name=_('path')) # path or pool name when use ceph block device storage
hostname = CharField(max_length=40, unique=True, path = CharField(max_length=200, unique=True,
verbose_name=_('hostname')) verbose_name=_('path or poolname'),
validators=[validate_ascii])
# hostname of storage driver
hostname = CharField(max_length=40, verbose_name=_('hostname'))
ceph_user = CharField(max_length=255, null=True, blank=True,
verbose_name=_('Ceph username'))
destroyed = DateTimeField(blank=True, default=None, null=True)
tracker = FieldTracker(fields=["ceph_user"])
class Meta: class Meta:
ordering = ['name'] ordering = ['name']
...@@ -82,15 +104,42 @@ class DataStore(Model): ...@@ -82,15 +104,42 @@ class DataStore(Model):
return [disk.filename for disk in deletables] return [disk.filename for disk in deletables]
def destroy(self):
if self.destroyed:
return False
self.destroyed = timezone.now()
self.save()
return True
@property
def used_percent(self):
stats = self.get_statistics()
free_percent = float(stats['free_percent'])
return int(100 - free_percent)
@property
def is_destroyable(self):
disk_count = self.disk_set.filter(destroyed__isnull=True).count()
template_count = self.instancetemplate_set.count()
vm_count = self.instance_set.filter(destroyed_at__isnull=True).count()
return 0 == disk_count + vm_count + template_count
@method_cache(30) @method_cache(30)
def get_statistics(self, timeout=15): def get_statistics(self, timeout=15):
q = self.get_remote_queue_name("storage", priority="fast") q = self.get_remote_queue_name("storage", priority="fast")
return storage_tasks.get_storage_stat.apply_async( try:
args=[self.path], queue=q).get(timeout=timeout) return storage_tasks.get_storage_stat.apply_async(
args=[self.type, self.path], queue=q).get(timeout=timeout)
except Exception:
return {'free_space': -1,
'free_percent': -1}
@method_cache(30) @method_cache(30)
def get_orphan_disks(self, timeout=15): def get_orphan_disks(self, timeout=15):
"""Disk image files without Disk object in the database. """Disk image files without Disk object in the database.
Exclude cloud-xxxxxxxx.dump format images. Exclude cloud-xxxxxxxx.dump format images.
:param timeout: Seconds before TimeOut exception :param timeout: Seconds before TimeOut exception
...@@ -98,7 +147,8 @@ class DataStore(Model): ...@@ -98,7 +147,8 @@ class DataStore(Model):
""" """
queue_name = self.get_remote_queue_name('storage', "slow") queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async( files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout)) args=[self.type, self.path], queue=queue_name).get(
timeout=timeout))
disks = set([disk.filename for disk in self.disk_set.all()]) disks = set([disk.filename for disk in self.disk_set.all()])
orphans = [] orphans = []
...@@ -116,8 +166,10 @@ class DataStore(Model): ...@@ -116,8 +166,10 @@ class DataStore(Model):
""" """
queue_name = self.get_remote_queue_name('storage', "slow") queue_name = self.get_remote_queue_name('storage', "slow")
files = set(storage_tasks.list_files.apply_async( files = set(storage_tasks.list_files.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout)) args=[self.type, self.path], queue=queue_name).get(
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True) timeout=timeout))
disks = Disk.objects.filter(destroyed__isnull=True, is_ready=True,
datastore=self)
return disks.exclude(filename__in=files) return disks.exclude(filename__in=files)
@classmethod @classmethod
...@@ -130,17 +182,26 @@ class DataStore(Model): ...@@ -130,17 +182,26 @@ class DataStore(Model):
pass pass
return cls.objects.all()[0] # TODO return cls.objects.all()[0] # TODO
@classmethod
def get_all(cls):
return cls.objects.all()
class Disk(TimeStampedModel): class Disk(TimeStampedModel):
"""A virtual disk. """A virtual disk.
""" """
TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'), TYPES = (('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')] ('ceph-norm', 'Ceph block normal'),
('ceph-snap', 'Ceph block snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw'))
BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi')) BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
name = CharField(blank=True, max_length=100, verbose_name=_("name")) name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True, filename = CharField(max_length=256, unique=True,
verbose_name=_("filename")) verbose_name=_("filename"),
validators=[validate_ascii])
datastore = ForeignKey(DataStore, verbose_name=_("datastore"), datastore = ForeignKey(DataStore, verbose_name=_("datastore"),
help_text=_("The datastore that holds the disk.")) help_text=_("The datastore that holds the disk."))
type = CharField(max_length=10, choices=TYPES) type = CharField(max_length=10, choices=TYPES)
...@@ -240,24 +301,28 @@ class Disk(TimeStampedModel): ...@@ -240,24 +301,28 @@ class Disk(TimeStampedModel):
return join(self.datastore.path, self.filename) return join(self.datastore.path, self.filename)
@property @property
def vm_format(self): def format_for_vmdriver(self):
"""Returns the proper file format for different type of images. """Returns the proper file format for different type of images.
""" """
return { return {
'qcow2-norm': 'qcow2', 'qcow2-norm': 'qcow2',
'qcow2-snap': 'qcow2', 'qcow2-snap': 'qcow2',
'ceph-norm': 'raw',
'ceph-snap': 'raw',
'iso': 'raw', 'iso': 'raw',
'raw-ro': 'raw', 'raw-ro': 'raw',
'raw-rw': 'raw', 'raw-rw': 'raw',
}[self.type] }[self.type]
@property @property
def format(self): def format_for_storagedriver(self):
"""Returns the proper file format for different types of images. """Returns the proper file format for different types of images.
""" """
return { return {
'qcow2-norm': 'qcow2', 'qcow2-norm': 'qcow2',
'qcow2-snap': 'qcow2', 'qcow2-snap': 'qcow2',
'ceph-norm': 'rbd',
'ceph-snap': 'rbd',
'iso': 'iso', 'iso': 'iso',
'raw-ro': 'raw', 'raw-ro': 'raw',
'raw-rw': 'raw', 'raw-rw': 'raw',
...@@ -270,6 +335,8 @@ class Disk(TimeStampedModel): ...@@ -270,6 +335,8 @@ class Disk(TimeStampedModel):
return { return {
'qcow2-norm': 'vd', 'qcow2-norm': 'vd',
'qcow2-snap': 'vd', 'qcow2-snap': 'vd',
'ceph-norm': 'vd',
'ceph-snap': 'vd',
'iso': 'sd', 'iso': 'sd',
'raw-ro': 'vd', 'raw-ro': 'vd',
'raw-rw': 'vd', 'raw-rw': 'vd',
...@@ -284,6 +351,8 @@ class Disk(TimeStampedModel): ...@@ -284,6 +351,8 @@ class Disk(TimeStampedModel):
return { return {
'qcow2-norm': 'virtio', 'qcow2-norm': 'virtio',
'qcow2-snap': 'virtio', 'qcow2-snap': 'virtio',
'ceph-norm': 'virtio',
'ceph-snap': 'virtio',
'iso': 'ide', 'iso': 'ide',
'raw-ro': 'virtio', 'raw-ro': 'virtio',
'raw-rw': 'virtio', 'raw-rw': 'virtio',
...@@ -328,6 +397,7 @@ class Disk(TimeStampedModel): ...@@ -328,6 +397,7 @@ class Disk(TimeStampedModel):
""" """
type_mapping = { type_mapping = {
'qcow2-norm': 'qcow2-snap', 'qcow2-norm': 'qcow2-snap',
'ceph-norm': 'ceph-snap',
'iso': 'iso', 'iso': 'iso',
'raw-ro': 'raw-rw', 'raw-ro': 'raw-rw',
} }
...@@ -341,29 +411,55 @@ class Disk(TimeStampedModel): ...@@ -341,29 +411,55 @@ class Disk(TimeStampedModel):
name=self.name, size=self.size, name=self.name, size=self.size,
type=new_type, dev_num=self.dev_num) type=new_type, dev_num=self.dev_num)
def get_disk_desc(self):
"""Serialize disk object to the storage driver.
"""
if self.datastore.type == "ceph_block":
return self.get_disk_desc_for_ceph_block_device()
return self.get_disk_desc_for_filesystem()
def get_vmdisk_desc(self): def get_vmdisk_desc(self):
"""Serialize disk object to the vmdriver. """Serialize disk object to the vmdriver.
""" """
if self.datastore.type == "ceph_block":
return self.get_vmdisk_desc_for_ceph_block_device()
return self.get_vmdisk_desc_for_filesystem()
def get_disk_desc_for_filesystem(self):
return {
'data_store_type': self.datastore.type,
'name': str(self.filename),
'dir': str(self.datastore.path),
'format': self.format_for_storagedriver,
'size': self.size,
'base_name': str(self.base.filename) if self.base else None,
'type': 'snapshot' if self.base else 'normal'
}
def get_disk_desc_for_ceph_block_device(self):
return self.get_disk_desc_for_filesystem()
def get_vmdisk_desc_for_filesystem(self):
return { return {
'data_store_type': self.datastore.type,
'source': self.path, 'source': self.path,
'driver_type': self.vm_format, 'driver_type': self.format_for_vmdriver,
'driver_cache': 'none', 'driver_cache': 'none',
'target_device': self.device_type + self.dev_num, 'target_device': self.device_type + self.dev_num,
'target_bus': self.device_bus, 'target_bus': self.device_bus,
'disk_device': 'cdrom' if self.type == 'iso' else 'disk' 'disk_device': 'cdrom' if self.type == 'iso' else 'disk'
} }
def get_disk_desc(self): def get_vmdisk_desc_for_ceph_block_device(self):
"""Serialize disk object to the storage driver.
""" desc = self.get_vmdisk_desc_for_filesystem()
return { desc["ceph_user"] = self.datastore.ceph_user
'name': self.filename,
'dir': self.datastore.path, return desc
'format': self.format,
'size': self.size,
'base_name': self.base.filename if self.base else None,
'type': 'snapshot' if self.base else 'normal'
}
def get_remote_queue_name(self, queue_id='storage', priority=None, def get_remote_queue_name(self, queue_id='storage', priority=None,
check_worker=True): check_worker=True):
...@@ -381,6 +477,11 @@ class Disk(TimeStampedModel): ...@@ -381,6 +477,11 @@ class Disk(TimeStampedModel):
def clean(self, *args, **kwargs): def clean(self, *args, **kwargs):
if (self.size is None or "") and self.base: if (self.size is None or "") and self.base:
self.size = self.base.size self.size = self.base.size
if self.base is not None and\
self.datastore.type != self.base.datastore.type:
raise ValidationError(_("Forbidden inheritance of disks "
"from different type of data stores."))
super(Disk, self).clean(*args, **kwargs) super(Disk, self).clean(*args, **kwargs)
def deploy(self, user=None, task_uuid=None, timeout=15): def deploy(self, user=None, task_uuid=None, timeout=15):
...@@ -423,6 +524,14 @@ class Disk(TimeStampedModel): ...@@ -423,6 +524,14 @@ class Disk(TimeStampedModel):
self.save() self.save()
return True return True
@staticmethod
def get_type_for_datastore(datastore):
if datastore.type == "ceph_block":
return "ceph-norm"
return "qcow2-norm"
@classmethod @classmethod
def create(cls, user=None, **params): def create(cls, user=None, **params):
disk = cls.__create(user, params) disk = cls.__create(user, params)
...@@ -440,7 +549,7 @@ class Disk(TimeStampedModel): ...@@ -440,7 +549,7 @@ class Disk(TimeStampedModel):
return disk return disk
@classmethod @classmethod
def download(cls, url, task, user=None, **params): def download(cls, url, datastore, task, user=None, **params):
"""Create disk object and download data from url synchronusly. """Create disk object and download data from url synchronusly.
:param url: image url to download. :param url: image url to download.
...@@ -458,6 +567,7 @@ class Disk(TimeStampedModel): ...@@ -458,6 +567,7 @@ class Disk(TimeStampedModel):
params.setdefault('name', url.split('/')[-1]) params.setdefault('name', url.split('/')[-1])
params.setdefault('type', 'iso') params.setdefault('type', 'iso')
params.setdefault('size', None) params.setdefault('size', None)
params.setdefault('datastore', datastore)
disk = cls.__create(params=params, user=user) disk = cls.__create(params=params, user=user)
queue_name = disk.get_remote_queue_name('storage', priority='slow') queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.download.apply_async( remote = storage_tasks.download.apply_async(
...@@ -493,10 +603,9 @@ class Disk(TimeStampedModel): ...@@ -493,10 +603,9 @@ class Disk(TimeStampedModel):
""" """
queue_name = self.datastore.get_remote_queue_name( queue_name = self.datastore.get_remote_queue_name(
'storage', priority='slow') 'storage', priority='slow')
logger.info("Image: %s at Datastore: %s recovered from trash." %
(self.filename, self.datastore.path))
res = storage_tasks.exists.apply_async( res = storage_tasks.exists.apply_async(
args=[self.datastore.path, args=[self.datastore.type,
self.datastore.path,
self.filename], self.filename],
queue=queue_name).get(timeout=timeout) queue=queue_name).get(timeout=timeout)
if res: if res:
...@@ -512,6 +621,7 @@ class Disk(TimeStampedModel): ...@@ -512,6 +621,7 @@ class Disk(TimeStampedModel):
Based on disk type: Based on disk type:
qcow2-norm, qcow2-snap --> qcow2-norm qcow2-norm, qcow2-snap --> qcow2-norm
iso --> iso (with base) iso --> iso (with base)
ceph-norm, ceph-snap --> ceph-norm
VM must be in STOPPED state to perform this action. VM must be in STOPPED state to perform this action.
The timeout parameter is not used now. The timeout parameter is not used now.
...@@ -519,6 +629,8 @@ class Disk(TimeStampedModel): ...@@ -519,6 +629,8 @@ class Disk(TimeStampedModel):
mapping = { mapping = {
'qcow2-snap': ('qcow2-norm', None), 'qcow2-snap': ('qcow2-norm', None),
'qcow2-norm': ('qcow2-norm', None), 'qcow2-norm': ('qcow2-norm', None),
'ceph-snap': ('ceph-norm', None),
'ceph-norm': ('ceph-norm', None),
'iso': ("iso", self), 'iso': ("iso", self),
} }
if self.type not in mapping.keys(): if self.type not in mapping.keys():
......
circle/storage/tasks/periodic_tasks.py
...@@ -36,7 +36,7 @@ def garbage_collector(timeout=15, percent=10): ...@@ -36,7 +36,7 @@ def garbage_collector(timeout=15, percent=10):
for ds in DataStore.objects.all(): for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage', priority='fast') queue_name = ds.get_remote_queue_name('storage', priority='fast')
files = set(storage_tasks.list_files.apply_async( files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout)) args=[ds.type, ds.path], queue=queue_name).get(timeout=timeout))
disks = ds.get_deletable_disks() disks = ds.get_deletable_disks()
queue_name = ds.get_remote_queue_name('storage', priority='slow') queue_name = ds.get_remote_queue_name('storage', priority='slow')
...@@ -46,7 +46,7 @@ def garbage_collector(timeout=15, percent=10): ...@@ -46,7 +46,7 @@ def garbage_collector(timeout=15, percent=10):
(i, ds.path)) (i, ds.path))
try: try:
success = storage_tasks.make_free_space.apply_async( success = storage_tasks.make_free_space.apply_async(
args=[ds.path, deletable_disks, percent], args=[ds.type, ds.path, deletable_disks, percent],
queue=queue_name).get(timeout=timeout) queue=queue_name).get(timeout=timeout)
if not success: if not success:
logger.warning("Has no deletable disk.") logger.warning("Has no deletable disk.")
......
circle/storage/tasks/storage_tasks.py
...@@ -19,12 +19,12 @@ from manager.mancelery import celery ...@@ -19,12 +19,12 @@ from manager.mancelery import celery
@celery.task(name='storagedriver.list') @celery.task(name='storagedriver.list')
def list(dir): def list(data_store_type, dir):
pass pass
@celery.task(name='storagedriver.list_files') @celery.task(name='storagedriver.list_files')
def list_files(dir): def list_files(data_store_type, dir):
pass pass
...@@ -39,12 +39,12 @@ def download(disk_desc, url): ...@@ -39,12 +39,12 @@ def download(disk_desc, url):
@celery.task(name='storagedriver.delete') @celery.task(name='storagedriver.delete')
def delete(path): def delete(disk_desc):
pass pass
@celery.task(name='storagedriver.delete_dump') @celery.task(name='storagedriver.delete_dump')
def delete_dump(path): def delete_dump(data_store_type, dir, filename):
pass pass
...@@ -54,7 +54,7 @@ def snapshot(disk_desc): ...@@ -54,7 +54,7 @@ def snapshot(disk_desc):
@celery.task(name='storagedriver.get') @celery.task(name='storagedriver.get')
def get(path): def get(disk_desc):
pass pass
...@@ -64,15 +64,15 @@ def merge(src_disk_desc, dst_disk_desc): ...@@ -64,15 +64,15 @@ def merge(src_disk_desc, dst_disk_desc):
@celery.task(name='storagedriver.exists') @celery.task(name='storagedriver.exists')
def exists(path, disk_name): def exists(data_store_type, path, disk_name):
pass pass
@celery.task(name='storagedriver.make_free_space') @celery.task(name='storagedriver.make_free_space')
def make_free_space(path, deletable_disks, percent): def make_free_space(datastore, path, deletable_disks, percent):
pass pass
@celery.task(name='storagedriver.get_storage_stat') @celery.task(name='storagedriver.get_storage_stat')
def get_storage_stat(path): def get_storage_stat(data_store_type, path):
pass pass
circle/vm/migrations/0003_auto_20160404_1525.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0005_auto_20160331_1823'),
('vm', '0002_interface_model'),
]
operations = [
migrations.AddField(
model_name='instance',
name='datastore',
field=models.ForeignKey(default=1, verbose_name='Data store', to='storage.DataStore', help_text="The target of VM's dump."),
preserve_default=False,
),
migrations.AddField(
model_name='instancetemplate',
name='datastore',
field=models.ForeignKey(default=1, verbose_name='Data store', to='storage.DataStore', help_text="The target of VM's dump."),
preserve_default=False,
),
]
circle/vm/models/instance.py
...@@ -50,6 +50,7 @@ from .activity import (ActivityInProgressError, InstanceActivity) ...@@ -50,6 +50,7 @@ from .activity import (ActivityInProgressError, InstanceActivity)
from .common import BaseResourceConfigModel, Lease from .common import BaseResourceConfigModel, Lease
from .network import Interface from .network import Interface
from .node import Node, Trait from .node import Node, Trait
from storage.models import DataStore
logger = getLogger(__name__) logger = getLogger(__name__)
pre_state_changed = Signal(providing_args=["new_state"]) pre_state_changed = Signal(providing_args=["new_state"])
...@@ -93,6 +94,7 @@ class VirtualMachineDescModel(BaseResourceConfigModel): ...@@ -93,6 +94,7 @@ class VirtualMachineDescModel(BaseResourceConfigModel):
"""Abstract base for virtual machine describing models. """Abstract base for virtual machine describing models.
""" """
access_method = CharField(max_length=10, choices=ACCESS_METHODS, access_method = CharField(max_length=10, choices=ACCESS_METHODS,
verbose_name=_('access method'), verbose_name=_('access method'),
help_text=_('Primary remote access method.')) help_text=_('Primary remote access method.'))
...@@ -116,6 +118,8 @@ class VirtualMachineDescModel(BaseResourceConfigModel): ...@@ -116,6 +118,8 @@ class VirtualMachineDescModel(BaseResourceConfigModel):
help_text=_( help_text=_(
'If the machine has agent installed, and ' 'If the machine has agent installed, and '
'the manager should wait for its start.')) 'the manager should wait for its start.'))
datastore = ForeignKey(DataStore, verbose_name=_("Data store"),
help_text=_("The target of VM's dump."))
class Meta: class Meta:
abstract = True abstract = True
...@@ -425,7 +429,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -425,7 +429,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
common_fields = ['name', 'description', 'num_cores', 'ram_size', common_fields = ['name', 'description', 'num_cores', 'ram_size',
'max_ram_size', 'arch', 'priority', 'boot_menu', 'max_ram_size', 'arch', 'priority', 'boot_menu',
'raw_data', 'lease', 'access_method', 'system', 'raw_data', 'lease', 'access_method', 'system',
'has_agent'] 'has_agent', 'datastore']
params = dict(template=template, owner=owner, pw=pwgen()) params = dict(template=template, owner=owner, pw=pwgen())
params.update([(f, getattr(template, f)) for f in common_fields]) params.update([(f, getattr(template, f)) for f in common_fields])
params.update(kwargs) # override defaults w/ user supplied values params.update(kwargs) # override defaults w/ user supplied values
...@@ -487,16 +491,10 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -487,16 +491,10 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
def mem_dump(self): def mem_dump(self):
"""Return the path and datastore for the memory dump. """Return the path and datastore for the memory dump.
It is always on the first hard drive storage named cloud-<id>.dump It named cloud-<id>.dump
""" """
try: filename = self.vm_name + '.dump'
datastore = self.disks.all()[0].datastore return {'datastore': self.datastore, 'filename': filename}
except IndexError:
from storage.models import DataStore
datastore = DataStore.get_default_datastore()
path = datastore.path + '/' + self.vm_name + '.dump'
return {'datastore': datastore, 'path': path}
@property @property
def primary_host(self): def primary_host(self):
...@@ -878,3 +876,18 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -878,3 +876,18 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
user=user, concurrency_check=concurrency_check, user=user, concurrency_check=concurrency_check,
readable_name=readable_name, resultant_state=resultant_state) readable_name=readable_name, resultant_state=resultant_state)
return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit) return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
def get_most_used_datastore(self):
disks = self.disks.all()
if not disks:
return None
freqs = dict()
for disk in disks:
datastore = disk.datastore
freqs[datastore] = freqs.get(datastore, 0) + 1
datastore = max(freqs.items(), key=lambda x: x[1])
return datastore[0]
circle/vm/models/node.py
...@@ -335,7 +335,6 @@ class Node(OperatedMixin, TimeStampedModel): ...@@ -335,7 +335,6 @@ class Node(OperatedMixin, TimeStampedModel):
try: try:
logger.info('%s %s', settings.GRAPHITE_URL, params) logger.info('%s %s', settings.GRAPHITE_URL, params)
response = requests.get(settings.GRAPHITE_URL, params=params) response = requests.get(settings.GRAPHITE_URL, params=params)
retval = {} retval = {}
for target in response.json(): for target in response.json():
# Example: # Example:
...@@ -366,12 +365,12 @@ class Node(OperatedMixin, TimeStampedModel): ...@@ -366,12 +365,12 @@ class Node(OperatedMixin, TimeStampedModel):
@property @property
@node_available @node_available
def cpu_usage(self): def cpu_usage(self):
return self.monitor_info.get('cpu.percent') / 100 return self.monitor_info.get('cpu.percent', 0) / 100
@property @property
@node_available @node_available
def ram_usage(self): def ram_usage(self):
return self.monitor_info.get('memory.usage') / 100 return self.monitor_info.get('memory.usage', 0) / 100
@property @property
@node_available @node_available
......
circle/vm/operations.py
...@@ -255,12 +255,20 @@ class CreateDiskOperation(InstanceOperation): ...@@ -255,12 +255,20 @@ class CreateDiskOperation(InstanceOperation):
required_perms = ('storage.create_empty_disk', ) required_perms = ('storage.create_empty_disk', )
accept_states = ('STOPPED', 'PENDING', 'RUNNING') accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, size, activity, name=None): def _operation(self, user, size, datastore, activity, name=None):
from storage.models import Disk from storage.models import Disk, DataStore
if not datastore:
datastore = self.instance.get_most_used_datastore()
if not datastore:
datastore = DataStore.get_default_datastore()
type = Disk.get_type_for_datastore(datastore)
if not name: if not name:
name = "new disk" name = "new disk"
disk = Disk.create(size=size, name=name, type="qcow2-norm") disk = Disk.create(size=size, name=name,
datastore=datastore, type=type)
disk.full_clean() disk.full_clean()
devnums = list(ascii_lowercase) devnums = list(ascii_lowercase)
for d in self.instance.disks.all(): for d in self.instance.disks.all():
...@@ -328,10 +336,16 @@ class DownloadDiskOperation(InstanceOperation): ...@@ -328,10 +336,16 @@ class DownloadDiskOperation(InstanceOperation):
accept_states = ('STOPPED', 'PENDING', 'RUNNING') accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = "localhost.man.slow" async_queue = "localhost.man.slow"
def _operation(self, user, url, task, activity, name=None): def _operation(self, user, url, datastore, task, activity, name=None):
from storage.models import Disk from storage.models import Disk, DataStore
if not datastore:
datastore = self.instance.get_most_used_datastore()
if not datastore:
datastore = DataStore.get_default_datastore()
disk = Disk.download(url=url, name=name, task=task) disk = Disk.download(url=url, name=name, task=task,
datastore=datastore)
devnums = list(ascii_lowercase) devnums = list(ascii_lowercase)
for d in self.instance.disks.all(): for d in self.instance.disks.all():
devnums.remove(d.dev_num) devnums.remove(d.dev_num)
...@@ -519,7 +533,9 @@ class DestroyOperation(InstanceOperation): ...@@ -519,7 +533,9 @@ class DestroyOperation(InstanceOperation):
"storage", "fast") "storage", "fast")
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return [self.instance.mem_dump['path']] return [self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']]
@register_operation @register_operation
...@@ -546,7 +562,7 @@ class MigrateOperation(RemoteInstanceOperation): ...@@ -546,7 +562,7 @@ class MigrateOperation(RemoteInstanceOperation):
"redeploy network (rollback)")): "redeploy network (rollback)")):
self.instance.deploy_net() self.instance.deploy_net()
def _operation(self, activity, to_node=None, live_migration=True): def _operation(self, activity, user, to_node=None, live_migration=True):
if not to_node: if not to_node:
with activity.sub_activity('scheduling', with activity.sub_activity('scheduling',
readable_name=ugettext_noop( readable_name=ugettext_noop(
...@@ -556,6 +572,16 @@ class MigrateOperation(RemoteInstanceOperation): ...@@ -556,6 +572,16 @@ class MigrateOperation(RemoteInstanceOperation):
try: try:
with activity.sub_activity( with activity.sub_activity(
'refresh_credential', readable_name=create_readable(
ugettext_noop("refresh credential on %(node)s"),
node=to_node)):
ceph_blocks = self.instance.disks.filter(
datastore__type="ceph_block")
if ceph_blocks.exists():
ds = ceph_blocks[0].datastore
to_node.refresh_credential(user=user,
username=ds.ceph_user)
with activity.sub_activity(
'migrate_vm', readable_name=create_readable( 'migrate_vm', readable_name=create_readable(
ugettext_noop("migrate to %(node)s"), node=to_node)): ugettext_noop("migrate to %(node)s"), node=to_node)):
super(MigrateOperation, self)._operation( super(MigrateOperation, self)._operation(
...@@ -763,6 +789,7 @@ class SaveAsTemplateOperation(InstanceOperation): ...@@ -763,6 +789,7 @@ class SaveAsTemplateOperation(InstanceOperation):
'ram_size': self.instance.ram_size, 'ram_size': self.instance.ram_size,
'raw_data': self.instance.raw_data, 'raw_data': self.instance.raw_data,
'system': self.instance.system, 'system': self.instance.system,
'datastore': self.instance.datastore,
} }
params.update(kwargs) params.update(kwargs)
params.pop("parent_activity", None) params.pop("parent_activity", None)
...@@ -916,7 +943,9 @@ class SleepOperation(InstanceOperation): ...@@ -916,7 +943,9 @@ class SleepOperation(InstanceOperation):
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return (super(SleepOperation.SuspendVmOperation, self) return (super(SleepOperation.SuspendVmOperation, self)
._get_remote_args(**kwargs) + ._get_remote_args(**kwargs) +
[self.instance.mem_dump['path']]) [self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']])
@register_operation @register_operation
...@@ -970,7 +999,9 @@ class WakeUpOperation(InstanceOperation): ...@@ -970,7 +999,9 @@ class WakeUpOperation(InstanceOperation):
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return (super(WakeUpOperation.WakeUpVmOperation, self) return (super(WakeUpOperation.WakeUpVmOperation, self)
._get_remote_args(**kwargs) + ._get_remote_args(**kwargs) +
[self.instance.mem_dump['path']]) [self.instance.mem_dump['datastore'].type,
self.instance.mem_dump['datastore'].path,
self.instance.mem_dump['filename']])
@register_operation @register_operation
...@@ -1122,6 +1153,14 @@ class NodeOperation(Operation): ...@@ -1122,6 +1153,14 @@ class NodeOperation(Operation):
user=user, readable_name=name) user=user, readable_name=name)
class RemoteNodeOperation(RemoteOperationMixin, NodeOperation):
remote_queue = ('vm', 'fast')
def _get_remote_queue(self):
return self.node.get_remote_queue_name(*self.remote_queue)
@register_operation @register_operation
class ResetNodeOperation(NodeOperation): class ResetNodeOperation(NodeOperation):
id = 'reset' id = 'reset'
...@@ -1312,6 +1351,22 @@ class UpdateNodeOperation(NodeOperation): ...@@ -1312,6 +1351,22 @@ class UpdateNodeOperation(NodeOperation):
@register_operation @register_operation
class RefreshCredentialOperation(RemoteNodeOperation):
id = 'refresh_credential'
name = _("refresh credential")
description = _("Refresh credential.")
required_perms = ()
task = vm_tasks.refresh_credential
def _get_remote_args(self, **kwargs):
return [kwargs["username"]]
def _operation(self, activity, username):
super(RefreshCredentialOperation, self)._operation(
username=username)
@register_operation
class ScreenshotOperation(RemoteInstanceOperation): class ScreenshotOperation(RemoteInstanceOperation):
id = 'screenshot' id = 'screenshot'
name = _("screenshot") name = _("screenshot")
......
circle/vm/tasks/vm_tasks.py
...@@ -185,3 +185,8 @@ def get_node_metrics(params): ...@@ -185,3 +185,8 @@ def get_node_metrics(params):
@celery.task(name='vmdriver.screenshot') @celery.task(name='vmdriver.screenshot')
def screenshot(params): def screenshot(params):
pass pass
@celery.task(name='vmdriver.refresh_secret')
def refresh_credential(user):
pass
circle/vm/tests/test_models.py
...@@ -36,6 +36,15 @@ from ..operations import ( ...@@ -36,6 +36,15 @@ from ..operations import (
) )
class DiskQuerySet(list):
def filter(self, *args, **kwargs):
return DiskQuerySet()
def exists(self):
return False
class PortFinderTestCase(TestCase): class PortFinderTestCase(TestCase):
def test_find_unused_port_without_used_ports(self): def test_find_unused_port_without_used_ports(self):
...@@ -106,6 +115,7 @@ class InstanceTestCase(TestCase): ...@@ -106,6 +115,7 @@ class InstanceTestCase(TestCase):
inst = Mock(destroyed_at=None, spec=Instance) inst = Mock(destroyed_at=None, spec=Instance)
inst.interface_set.all.return_value = [] inst.interface_set.all.return_value = []
inst.node = MagicMock(spec=Node) inst.node = MagicMock(spec=Node)
inst.disks = DiskQuerySet()
inst.status = 'RUNNING' inst.status = 'RUNNING'
migrate_op = MigrateOperation(inst) migrate_op = MigrateOperation(inst)
with patch('vm.operations.vm_tasks.migrate') as migr, \ with patch('vm.operations.vm_tasks.migrate') as migr, \
...@@ -124,6 +134,7 @@ class InstanceTestCase(TestCase): ...@@ -124,6 +134,7 @@ class InstanceTestCase(TestCase):
inst.interface_set.all.return_value = [] inst.interface_set.all.return_value = []
inst.node = MagicMock(spec=Node) inst.node = MagicMock(spec=Node)
inst.status = 'RUNNING' inst.status = 'RUNNING'
inst.disks = DiskQuerySet()
migrate_op = MigrateOperation(inst) migrate_op = MigrateOperation(inst)
with patch('vm.operations.vm_tasks.migrate') as migr, \ with patch('vm.operations.vm_tasks.migrate') as migr, \
patch.object(RemoteOperationMixin, "_operation"): patch.object(RemoteOperationMixin, "_operation"):
......
circle/vm/tests/test_operations.py
...@@ -25,6 +25,7 @@ from vm.operations import ( ...@@ -25,6 +25,7 @@ from vm.operations import (
RebootOperation, ResetOperation, SaveAsTemplateOperation, RebootOperation, ResetOperation, SaveAsTemplateOperation,
ShutdownOperation, ShutOffOperation, SleepOperation, WakeUpOperation, ShutdownOperation, ShutOffOperation, SleepOperation, WakeUpOperation,
) )
from test_models import DiskQuerySet
class DeployOperationTestCase(TestCase): class DeployOperationTestCase(TestCase):
...@@ -55,9 +56,10 @@ class MigrateOperationTestCase(TestCase): ...@@ -55,9 +56,10 @@ class MigrateOperationTestCase(TestCase):
op = MigrateOperation(inst) op = MigrateOperation(inst)
op._get_remote_args = MagicMock(side_effect=MigrateException()) op._get_remote_args = MagicMock(side_effect=MigrateException())
inst.select_node = MagicMock(return_value='test') inst.select_node = MagicMock(return_value='test')
inst.disks = DiskQuerySet()
self.assertRaises( self.assertRaises(
MigrateException, op._operation, MigrateException, op._operation,
act, to_node=None) act, user=None, to_node=None)
assert inst.select_node.called assert inst.select_node.called
op._get_remote_args.assert_called_once_with( op._get_remote_args.assert_called_once_with(
to_node='test', live_migration=True) to_node='test', live_migration=True)
......
requirements/base.txt
...@@ -11,7 +11,7 @@ django-braces==1.8.0 ...@@ -11,7 +11,7 @@ django-braces==1.8.0
django-crispy-forms==1.6.0 django-crispy-forms==1.6.0
django-model-utils==2.2 django-model-utils==2.2
djangosaml2==0.13.0 djangosaml2==0.13.0
django-sizefield==0.7 django-sizefield==0.9.1
git+https://git.ik.bme.hu/circle/django-sshkey.git git+https://git.ik.bme.hu/circle/django-sshkey.git
django-statici18n==1.1.3 django-statici18n==1.1.3
django-tables2==0.16.0 django-tables2==0.16.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment