Skip to content

Gelencsér Szabolcs / cloud

Go to a project
Commit a5be38e8 by Bach Dániel
Options

Merge remote-tracking branch 'origin/master' into issue-248 

Conflicts:
	circle/vm/tasks/local_agent_tasks.py
parents ab68f725 0bfdb2a3
Showing with 734 additions and 240 deletions
circle/circle/settings/base.py
...@@ -431,9 +431,18 @@ LOGIN_REDIRECT_URL = "/" ...@@ -431,9 +431,18 @@ LOGIN_REDIRECT_URL = "/"
AGENT_DIR = get_env_variable( AGENT_DIR = get_env_variable(
'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent')) 'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent'))
# AGENT_DIR is the root directory for the agent.
# The directory structure SHOULD be:
# /home/username/agent
# |-- agent-linux
# | |-- .git
# | +-- ...
# |-- agent-win
# | +-- agent-win-%(version).exe
#
try: try:
git_env = {'GIT_DIR': join(AGENT_DIR, '.git')} git_env = {'GIT_DIR': join(join(AGENT_DIR, "agent-linux"), '.git')}
AGENT_VERSION = check_output( AGENT_VERSION = check_output(
('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env) ('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env)
except: except:
......
circle/dashboard/forms.py
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
from __future__ import absolute_import from __future__ import absolute_import
from datetime import timedelta from datetime import timedelta
from urlparse import urlparse
from django.contrib.auth.forms import ( from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm, AuthenticationForm, PasswordResetForm, SetPasswordForm,
...@@ -39,6 +40,7 @@ from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm ...@@ -39,6 +40,7 @@ from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
from django.forms.widgets import TextInput, HiddenInput from django.forms.widgets import TextInput, HiddenInput
from django.template import Context from django.template import Context
from django.template.loader import render_to_string from django.template.loader import render_to_string
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from sizefield.widgets import FileSizeWidget from sizefield.widgets import FileSizeWidget
from django.core.urlresolvers import reverse_lazy from django.core.urlresolvers import reverse_lazy
...@@ -79,6 +81,12 @@ class VmSaveForm(forms.Form): ...@@ -79,6 +81,12 @@ class VmSaveForm(forms.Form):
helper.form_tag = False helper.form_tag = False
return helper return helper
def __init__(self, *args, **kwargs):
default = kwargs.pop('default', None)
super(VmSaveForm, self).__init__(*args, **kwargs)
if default:
self.fields['name'].initial = default
class VmCustomizeForm(forms.Form): class VmCustomizeForm(forms.Form):
name = forms.CharField(widget=forms.TextInput(attrs={ name = forms.CharField(widget=forms.TextInput(attrs={
...@@ -744,6 +752,20 @@ class VmRenewForm(forms.Form): ...@@ -744,6 +752,20 @@ class VmRenewForm(forms.Form):
return helper return helper
class VmMigrateForm(forms.Form):
live_migration = forms.BooleanField(
required=False, initial=True, label=_("live migration"))
def __init__(self, *args, **kwargs):
choices = kwargs.pop('choices')
default = kwargs.pop('default')
super(VmMigrateForm, self).__init__(*args, **kwargs)
self.fields.insert(0, 'to_node', forms.ModelChoiceField(
queryset=choices, initial=default, required=False,
widget=forms.RadioSelect(), label=_("Node")))
class VmStateChangeForm(forms.Form): class VmStateChangeForm(forms.Form):
interrupt = forms.BooleanField(required=False, label=_( interrupt = forms.BooleanField(required=False, label=_(
...@@ -788,6 +810,12 @@ class VmCreateDiskForm(forms.Form): ...@@ -788,6 +810,12 @@ class VmCreateDiskForm(forms.Form):
help_text=_('Size of disk to create in bytes or with units ' help_text=_('Size of disk to create in bytes or with units '
'like MB or GB.')) 'like MB or GB.'))
def __init__(self, *args, **kwargs):
default = kwargs.pop('default', None)
super(VmCreateDiskForm, self).__init__(*args, **kwargs)
if default:
self.fields['name'].initial = default
def clean_size(self): def clean_size(self):
size_in_bytes = self.cleaned_data.get("size") size_in_bytes = self.cleaned_data.get("size")
if not size_in_bytes.isdigit() and len(size_in_bytes) > 0: if not size_in_bytes.isdigit() and len(size_in_bytes) > 0:
...@@ -839,13 +867,42 @@ class VmDiskResizeForm(forms.Form): ...@@ -839,13 +867,42 @@ class VmDiskResizeForm(forms.Form):
helper.form_tag = False helper.form_tag = False
if self.disk: if self.disk:
helper.layout = Layout( helper.layout = Layout(
HTML(_("<label>Disk:</label> %s") % self.disk), HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
Field('disk'), Field('size')) Field('disk'), Field('size'))
return helper return helper
class VmDiskRemoveForm(forms.Form):
def __init__(self, *args, **kwargs):
choices = kwargs.pop('choices')
self.disk = kwargs.pop('default')
super(VmDiskRemoveForm, self).__init__(*args, **kwargs)
self.fields.insert(0, 'disk', forms.ModelChoiceField(
queryset=choices, initial=self.disk, required=True,
empty_label=None, label=_('Disk')))
if self.disk:
self.fields['disk'].widget = HiddenInput()
@property
def helper(self):
helper = FormHelper(self)
helper.form_tag = False
if self.disk:
helper.layout = Layout(
AnyTag(
"div",
HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
css_class="form-group",
),
Field("disk"),
)
return helper
class VmDownloadDiskForm(forms.Form): class VmDownloadDiskForm(forms.Form):
name = forms.CharField(max_length=100, label=_("Name")) name = forms.CharField(max_length=100, label=_("Name"), required=False)
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ]) url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
@property @property
...@@ -854,6 +911,18 @@ class VmDownloadDiskForm(forms.Form): ...@@ -854,6 +911,18 @@ class VmDownloadDiskForm(forms.Form):
helper.form_tag = False helper.form_tag = False
return helper return helper
def clean(self):
cleaned_data = super(VmDownloadDiskForm, self).clean()
if not cleaned_data['name']:
if cleaned_data['url']:
cleaned_data['name'] = urlparse(
cleaned_data['url']).path.split('/')[-1]
if not cleaned_data['name']:
raise forms.ValidationError(
_("Could not find filename in URL, "
"please specify a name explicitly."))
return cleaned_data
class VmAddInterfaceForm(forms.Form): class VmAddInterfaceForm(forms.Form):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
......
circle/dashboard/static/dashboard/dashboard.css
...@@ -528,7 +528,7 @@ footer a, footer a:hover, footer a:visited { ...@@ -528,7 +528,7 @@ footer a, footer a:hover, footer a:visited {
} }
#dashboard-template-list a small { #dashboard-template-list a small {
max-width: 50%; max-width: 45%;
float: left; float: left;
padding-top: 2px; padding-top: 2px;
text-overflow: ellipsis; text-overflow: ellipsis;
...@@ -1012,3 +1012,7 @@ textarea[name="new_members"] { ...@@ -1012,3 +1012,7 @@ textarea[name="new_members"] {
.disk-resize-btn { .disk-resize-btn {
margin-right: 5px; margin-right: 5px;
} }
#vm-migrate-node-list li {
cursor: pointer;
}
circle/dashboard/static/dashboard/dashboard.js
...@@ -411,6 +411,17 @@ $(function () { ...@@ -411,6 +411,17 @@ $(function () {
$(this).removeClass("btn-default").addClass("btn-primary"); $(this).removeClass("btn-default").addClass("btn-primary");
return false; return false;
}); });
// vm migrate select for node
$(document).on("click", "#vm-migrate-node-list li", function(e) {
var li = $(this).closest('li');
if (li.find('input').attr('disabled'))
return true;
$('#vm-migrate-node-list li').removeClass('panel-primary');
li.addClass('panel-primary').find('input').prop("checked", true);
return true;
});
}); });
function generateVmHTML(pk, name, host, icon, _status, fav, is_last) { function generateVmHTML(pk, name, host, icon, _status, fav, is_last) {
...@@ -445,7 +456,7 @@ function generateNodeHTML(name, icon, _status, url, is_last) { ...@@ -445,7 +456,7 @@ function generateNodeHTML(name, icon, _status, url, is_last) {
function generateNodeTagHTML(name, icon, _status, label , url) { function generateNodeTagHTML(name, icon, _status, label , url) {
return '<a href="' + url + '" class="label ' + label + '" >' + return '<a href="' + url + '" class="label ' + label + '" >' +
'<i class="' + icon + '" title="' + _status + '"></i> ' + name + '<i class="fa ' + icon + '" title="' + _status + '"></i> ' + name +
'</a> '; '</a> ';
} }
......
circle/dashboard/static/dashboard/vm-common.js
...@@ -16,15 +16,6 @@ $(function() { ...@@ -16,15 +16,6 @@ $(function() {
$('#confirmation-modal').on('hidden.bs.modal', function() { $('#confirmation-modal').on('hidden.bs.modal', function() {
$('#confirmation-modal').remove(); $('#confirmation-modal').remove();
}); });
$('#vm-migrate-node-list li').click(function(e) {
var li = $(this).closest('li');
if (li.find('input').attr('disabled'))
return true;
$('#vm-migrate-node-list li').removeClass('panel-primary');
li.addClass('panel-primary').find('input').attr('checked', true);
return false;
});
$('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary'); $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
} }
}); });
...@@ -51,7 +42,8 @@ $(function() { ...@@ -51,7 +42,8 @@ $(function() {
if(data.success) { if(data.success) {
$('a[href="#activity"]').trigger("click"); $('a[href="#activity"]').trigger("click");
if(data.with_reload) { if(data.with_reload) {
location.reload(); // when the activity check stops the page will reload
reload_vm_detail = true;
} }
/* if there are messages display them */ /* if there are messages display them */
......
circle/dashboard/static/dashboard/vm-details.js
var show_all = false; var show_all = false;
var in_progress = false; var in_progress = false;
var activity_hash = 5; var activity_hash = 5;
var reload_vm_detail = false;
$(function() { $(function() {
/* do we need to check for new activities */ /* do we need to check for new activities */
...@@ -404,6 +405,7 @@ function checkNewActivity(runs) { ...@@ -404,6 +405,7 @@ function checkNewActivity(runs) {
); );
} else { } else {
in_progress = false; in_progress = false;
if(reload_vm_detail) location.reload();
} }
$('a[href="#activity"] i').removeClass('fa-spin'); $('a[href="#activity"] i').removeClass('fa-spin');
}, },
......
circle/dashboard/templates/dashboard/_disk-list-element.html
{% load i18n %} {% load i18n %}
{% load sizefieldtags %} {% load sizefieldtags %}
<i class="fa {% if d.is_downloading %}fa-refresh fa-spin{% else %}fa-file{% if d.failed %}" style="color: #d9534f;{% endif %}{% endif %}"></i> <i class="fa fa-file"></i>
{{ d.name }} (#{{ d.id }}) - {{ d.name }} (#{{ d.id }}) - {{ d.size|filesize }}
{% if not d.is_downloading %}
{% if not d.failed %} {% if op.remove_disk %}
{% if d.size %}{{ d.size|filesize }}{% endif %} <span class="operation-wrapper">
{% else %} <a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
<div class="label label-danger"{% if user.is_superuser %} title="{{ d.get_latest_activity_result }}"{% endif %}>{% trans "failed" %}</div> class="btn btn-xs btn-{{ op.remove_disk.effect}} pull-right operation disk-remove-btn
{% endif %} {% if op.resize_disk.disabled %}disabled{% endif %}">
{% else %}<span class="disk-list-disk-percentage" data-disk-pk="{{ d.pk }}">{{ d.get_download_percentage }}</span>%{% endif %} <i class="fa fa-{{ op.remove_disk.icon }}"></i> {% trans "Remove" %}
{% if is_owner != False %} </a>
<a href="{% url "dashboard.views.disk-remove" pk=d.pk %}?next={{ request.path }}" </span>
data-disk-pk="{{ d.pk }}" class="btn btn-xs btn-danger pull-right disk-remove" {% endif %}
{% if not long_remove %}title="{% trans "Remove" %}"{% endif %}> {% if op.resize_disk %}
<i class="fa fa-times"></i>{% if long_remove %} {% trans "Remove" %}{% endif %} <span class="operation-wrapper">
</a> <a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
{% if op.resize_disk %} class="btn btn-xs btn-{{ op.resize_disk.effect }} pull-right operation disk-resize-btn
<span class="operation-wrapper"> {% if op.resize_disk.disabled %}disabled{% endif %}">
<a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}" <i class="fa fa-{{ op.resize_disk.icon }}"></i> {% trans "Resize" %}
class="btn btn-xs btn-warning pull-right operation disk-resize-btn"> </a>
<i class="fa fa-arrows-alt"></i> {% trans "Resize" %} </span>
</a>
</span>
{% endif %}
{% endif %} {% endif %}
<div style="clear: both;"></div> <div style="clear: both;"></div>
{% if request.user.is_superuser %}
<small>{% trans "File name" %}: {{ d.filename }}</small>
{% endif %}
circle/dashboard/templates/dashboard/_vm-mass-migrate.html
{% extends "dashboard/mass-operate.html" %} {% extends "dashboard/mass-operate.html" %}
{% load i18n %} {% load i18n %}
{% load sizefieldtags %} {% load sizefieldtags %}
{% load crispy_forms_tags %}
{% block formfields %} {% block formfields %}
...@@ -11,20 +12,20 @@ ...@@ -11,20 +12,20 @@
<label for="migrate-to-none"> <label for="migrate-to-none">
<strong>{% trans "Reschedule" %}</strong> <strong>{% trans "Reschedule" %}</strong>
</label> </label>
<input id="migrate-to-none" type="radio" name="node" value="" style="float: right;" checked="checked"> <input id="migrate-to-none" type="radio" name="to_node" value="" style="float: right;" checked="checked">
<span class="vm-migrate-node-property"> <span class="vm-migrate-node-property">
{% trans "This option will reschedule each virtual machine to the optimal node." %} {% trans "This option will reschedule each virtual machine to the optimal node." %}
</span> </span>
<div style="clear: both;"></div> <div style="clear: both;"></div>
</div> </div>
</li> </li>
{% for n in nodes %} {% for n in form.fields.to_node.queryset.all %}
<li class="panel panel-default mass-migrate-node"> <li class="panel panel-default mass-migrate-node">
<div class="panel-body"> <div class="panel-body">
<label for="migrate-to-{{n.pk}}"> <label for="migrate-to-{{n.pk}}">
<strong>{{ n }}</strong> <strong>{{ n }}</strong>
</label> </label>
<input id="migrate-to-{{n.pk}}" type="radio" name="node" value="{{ n.pk }}" style="float: right;"/> <input id="migrate-to-{{n.pk}}" type="radio" name="to_node" value="{{ n.pk }}" style="float: right;"/>
<span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span> <span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span>
<span class="vm-migrate-node-property">{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span> <span class="vm-migrate-node-property">{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span>
<div style="clear: both;"></div> <div style="clear: both;"></div>
...@@ -32,5 +33,6 @@ ...@@ -32,5 +33,6 @@
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
{{ form.live_migration|as_crispy_field }}
<hr /> <hr />
{% endblock %} {% endblock %}
circle/dashboard/templates/dashboard/_vm-migrate.html
{% extends "dashboard/operate.html" %} {% extends "dashboard/operate.html" %}
{% load i18n %} {% load i18n %}
{% load sizefieldtags %} {% load sizefieldtags %}
{% load crispy_forms_tags %}
{% block question %} {% block question %}
<p> <p>
...@@ -13,24 +14,27 @@ Choose a compute node to migrate {{obj}} to. ...@@ -13,24 +14,27 @@ Choose a compute node to migrate {{obj}} to.
{% block formfields %} {% block formfields %}
<ul id="vm-migrate-node-list" class="list-unstyled"> <ul id="vm-migrate-node-list" class="list-unstyled">
{% with current=object.node.pk %} {% with current=object.node.pk recommended=form.fields.to_node.initial.pk %}
{% for n in nodes %} {% for n in form.fields.to_node.queryset.all %}
<li class="panel panel-default"><div class="panel-body"> <li class="panel panel-default"><div class="panel-body">
<label for="migrate-to-{{n.pk}}"> <label for="migrate-to-{{n.pk}}">
<strong>{{ n }}</strong> <strong>{{ n }}</strong>
<div class="label label-primary"><i class="fa {{n.get_status_icon}}"></i> <div class="label label-primary">
{{n.get_status_display}}</div> <i class="fa {{n.get_status_icon}}"></i> {{n.get_status_display}}</div>
{% if current == n.pk %}<div class="label label-info">{% trans "current" %}</div>{% endif %} {% if current == n.pk %}<div class="label label-info">{% trans "current" %}</div>{% endif %}
{% if recommended == n.pk %}<div class="label label-success">{% trans "recommended" %}</div>{% endif %} {% if recommended == n.pk %}<div class="label label-success">{% trans "recommended" %}</div>{% endif %}
</label> </label>
<input id="migrate-to-{{n.pk}}" type="radio" name="node" value="{{ n.pk }}" style="float: right;" <input id="migrate-to-{{n.pk}}" type="radio" name="to_node" value="{{ n.pk }}" style="float: right;"
{% if current == n.pk %}disabled="disabled"{% endif %} {% if current == n.pk %}disabled="disabled"{% endif %}
{% if recommended == n.pk %}checked="checked"{% endif %} /> {% if recommended == n.pk %}checked="checked"{% endif %}
/>
<span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span> <span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span>
<span class="vm-migrate-node-property">{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span> <span class="vm-migrate-node-property">
{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span>
<div style="clear: both;"></div> <div style="clear: both;"></div>
</div></li> </li>
{% endfor %} {% endfor %}
{% endwith %} {% endwith %}
</ul> </ul>
{{ form.live_migration|as_crispy_field }}
{% endblock %} {% endblock %}
circle/dashboard/templates/dashboard/index-nodes.html
{% load i18n %} {% load i18n %}
<div class="panel panel-default"> <div class="panel panel-default">
<div class="panel-heading"> <div class="panel-heading">
<div class="pull-right toolbar"> <div class="pull-right toolbar">
<div class="btn-group"> <div class="btn-group">
...@@ -7,9 +7,10 @@ ...@@ -7,9 +7,10 @@
data-container="body"><i class="fa fa-dashboard"></i></a> data-container="body"><i class="fa fa-dashboard"></i></a>
<a href="#index-list-view" data-index-box="node" class="btn btn-default btn-xs disabled" <a href="#index-list-view" data-index-box="node" class="btn btn-default btn-xs disabled"
data-container="body"><i class="fa fa-list"></i></a> data-container="body"><i class="fa fa-list"></i></a>
</div> </div>
<span class="btn btn-default btn-xs infobtn" title="{% trans "List of compute nodes, also called worker nodes or hypervisors, which run the virtual machines." %}"><i class="fa fa-info-circle"></i></span> <span class="btn btn-default btn-xs infobtn" title="{% trans "List of compute nodes, also called worker nodes or hypervisors, which run the virtual machines." %}">
<i class="fa fa-info-circle"></i>
</span>
</div> </div>
<h3 class="no-margin"> <h3 class="no-margin">
<i class="fa fa-sitemap"></i> {% trans "Nodes" %} <i class="fa fa-sitemap"></i> {% trans "Nodes" %}
...@@ -28,50 +29,55 @@ ...@@ -28,50 +29,55 @@
</a> </a>
{% endfor %} {% endfor %}
</div> </div>
<div href="#" class="list-group-item list-group-footer"> </div><!-- #node-list-view -->
<div class="row">
<div class="col-sm-6 col-xs-6 input-group input-group-sm">
<input id="dashboard-node-search-input" type="text" class="form-control" placeholder="{% trans "Search..." %}" />
<div class="input-group-btn">
<button type="submit" class="form-control btn btn-primary" title="search"><i class="fa fa-search"></i></button>
</div>
</div>
<div class="col-sm-6 text-right">
<a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}">
<i class="fa fa-chevron-circle-right"></i>
{% if more_nodes > 0 %}
{% blocktrans with count=more_nodes %}<strong>{{count}}</strong> more{% endblocktrans %}
{% else %}
{% trans "list" %}
{% endif %}
</a>
<a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}"><i class="fa fa-plus-circle"></i> {% trans "new" %}</a>
</div>
</div>
</div>
</div>
<div class="panel-body" id="node-graph-view" style="display: none"> <div class="panel-body" id="node-graph-view" style="display: none; min-height: 204px;">
<p class="pull-right"> <input class="knob" data-fgColor="chartreuse" data-thickness=".4" data-width="60" data-height="60" data-readOnly="true" value="{% widthratio node_num.running sum_node_num 100 %}"></p> <p class="pull-right">
<p><span class="big"><big>{{ node_num.running }}</big> running </span> <input class="knob" data-fgColor="chartreuse"
+ <big>{{ node_num.missing }}</big> missing + <br><big>{{ node_num.disabled }}</big> disabled + <big>{{ node_num.offline }}</big> offline</p> data-thickness=".4" data-width="60" data-height="60" data-readOnly="true"
<ul class="list-inline" id="dashboard-node-taglist"> value="{% widthratio node_num.running sum_node_num 100 %}">
{% for i in nodes %} </p>
<a href="{{ i.get_absolute_url }}" class="label {{i.get_status_label}}" > <p>
<i class="fa {{ i.get_status_icon }}" title="{{ i.get_status_display }}"></i> {{ i.name }}</a> <span class="big">
{% endfor %} <big>{{ node_num.running }}</big> running
</ul> </span>
+ <big>{{ node_num.missing }}</big>
missing + <br><big>{{ node_num.disabled }}</big> disabled + <big>{{ node_num.offline }}</big> offline
</p>
<ul class="list-inline" id="dashboard-node-taglist">
{% for i in nodes %}
<a href="{{ i.get_absolute_url }}" class="label {{i.get_status_label}}" >
<i class="fa {{ i.get_status_icon }}" title="{{ i.get_status_display }}"></i> {{ i.name }}</a>
{% endfor %}
</ul>
<div class="clearfix"></div> <div class="clearfix"></div>
<div class="row"> </div>
<div class="col-sm-6 text-right pull-right">
{% if more_nodes >= 0 %} <div href="#" class="list-group-item list-group-footer">
<a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}"> <div class="row">
<i class="fa fa-chevron-circle-right"></i> {% blocktrans with count=more_nodes %}<strong>{{count}}</strong> more{% endblocktrans %} <div class="col-sm-6 col-xs-6 input-group input-group-sm">
</a> <input id="dashboard-node-search-input" type="text" class="form-control"
{% endif %} placeholder="{% trans "Search..." %}" />
<a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}"><i class="fa fa-plus-circle"></i> {% trans "new" %}</a> <div class="input-group-btn">
<button type="submit" class="btn btn-primary" title="{% trans "Search" %}" data-container="body">
<i class="fa fa-search"></i>
</button>
</div> </div>
</div> </div>
</div> <div class="col-sm-6 text-right">
<a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}">
<i class="fa fa-chevron-circle-right"></i>
{% if more_nodes > 0 %}
{% blocktrans with count=more_nodes %}<strong>{{count}}</strong> more{% endblocktrans %}
{% else %}
{% trans "list" %}
{% endif %}
</a>
<a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}">
<i class="fa fa-plus-circle"></i> {% trans "new" %}
</a>
</div>
</div>
</div>
</div> </div>
circle/dashboard/templates/dashboard/instanceactivity_detail.html
...@@ -58,6 +58,26 @@ ...@@ -58,6 +58,26 @@
<dt>{% trans "resultant state" %}</dt> <dt>{% trans "resultant state" %}</dt>
<dd>{{object.resultant_state|default:'n/a'}}</dd> <dd>{{object.resultant_state|default:'n/a'}}</dd>
<dt>{% trans "subactivities" %}</dt>
{% for s in object.children.all %}
<dd>
<span{% if s.result %} title="{{ s.result|get_text:user }}"{% endif %}>
<a href="{{ s.get_absolute_url }}">
{{ s.readable_name|get_text:user|capfirst }}</a></span> &ndash;
{% if s.finished %}
{{ s.finished|time:"H:i:s" }}
{% else %}
<i class="fa fa-refresh fa-spin" class="sub-activity-loading-icon"></i>
{% endif %}
{% if s.has_failed %}
<div class="label label-danger">{% trans "failed" %}</div>
{% endif %}
</dd>
{% empty %}
<dd>{% trans "none" %}</dd>
{% endfor %}
</div>
</div> </div>
</div> </div>
</div> </div>
......
circle/dashboard/templates/dashboard/node-list/column-vm.html
{% load i18n %} {% load i18n %}
<div id="node-list-column-vm"> <div id="node-list-column-vm">
<a class="real-link" href="{% url "dashboard.views.vm-list" %}?s=node:{{ record.name }}"> <a class="real-link" href="{% url "dashboard.views.vm-list" %}?s=node_exact:{{ record.name }}">
{{ value }} {{ value }}
</a> </a>
</div> </div>
circle/dashboard/templates/dashboard/template-edit.html
...@@ -86,7 +86,13 @@ ...@@ -86,7 +86,13 @@
{% endif %} {% endif %}
{% for d in disks %} {% for d in disks %}
<li> <li>
{% include "dashboard/_disk-list-element.html" %} <i class="fa fa-file"></i>
{{ d.name }} (#{{ d.id }}) -
<a href="{% url "dashboard.views.disk-remove" pk=d.pk %}?next={{ request.path }}"
data-disk-pk="{{ d.pk }}" class="btn btn-xs btn-danger pull-right disk-remove"
{% if not long_remove %}title="{% trans "Remove" %}"{% endif %}>
<i class="fa fa-times"></i>{% if long_remove %} {% trans "Remove" %}{% endif %}
</a>
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
......
circle/dashboard/tests/test_mockedviews.py
...@@ -34,6 +34,13 @@ from ..views import AclUpdateView ...@@ -34,6 +34,13 @@ from ..views import AclUpdateView
from .. import views from .. import views
class QuerySet(list):
model = MagicMock()
def get(self, *args, **kwargs):
return self.pop()
class ViewUserTestCase(unittest.TestCase): class ViewUserTestCase(unittest.TestCase):
def test_404(self): def test_404(self):
...@@ -145,58 +152,66 @@ class VmOperationViewTestCase(unittest.TestCase): ...@@ -145,58 +152,66 @@ class VmOperationViewTestCase(unittest.TestCase):
view.as_view()(request, pk=1234).render() view.as_view()(request, pk=1234).render()
def test_migrate(self): def test_migrate(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=True) request = FakeRequestFactory(
POST={'to_node': 1, 'live_migration': True}, superuser=True)
view = vm_ops['migrate'] view = vm_ops['migrate']
node = MagicMock(pk=1, name='node1')
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
patch('dashboard.views.util.messages') as msg, \ patch('dashboard.views.util.messages') as msg, \
patch('dashboard.views.vm.get_object_or_404') as go4: patch.object(view, 'get_form_kwargs') as form_kwargs:
inst = MagicMock(spec=Instance) inst = MagicMock(spec=Instance)
inst._meta.object_name = "Instance" inst._meta.object_name = "Instance"
inst.migrate = Instance._ops['migrate'](inst) inst.migrate = Instance._ops['migrate'](inst)
inst.migrate.async = MagicMock() inst.migrate.async = MagicMock()
inst.has_level.return_value = True inst.has_level.return_value = True
form_kwargs.return_value = {
'default': 100, 'choices': QuerySet([node])}
go.return_value = inst go.return_value = inst
go4.return_value = MagicMock()
assert view.as_view()(request, pk=1234)['location'] assert view.as_view()(request, pk=1234)['location']
assert not msg.error.called assert not msg.error.called
assert go4.called inst.migrate.async.assert_called_once_with(
to_node=node, live_migration=True, user=request.user)
def test_migrate_failed(self): def test_migrate_failed(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=True) request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
view = vm_ops['migrate'] view = vm_ops['migrate']
node = MagicMock(pk=1, name='node1')
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
patch('dashboard.views.util.messages') as msg, \ patch('dashboard.views.util.messages') as msg, \
patch('dashboard.views.vm.get_object_or_404') as go4: patch.object(view, 'get_form_kwargs') as form_kwargs:
inst = MagicMock(spec=Instance) inst = MagicMock(spec=Instance)
inst._meta.object_name = "Instance" inst._meta.object_name = "Instance"
inst.migrate = Instance._ops['migrate'](inst) inst.migrate = Instance._ops['migrate'](inst)
inst.migrate.async = MagicMock() inst.migrate.async = MagicMock()
inst.migrate.async.side_effect = Exception inst.migrate.async.side_effect = Exception
inst.has_level.return_value = True inst.has_level.return_value = True
form_kwargs.return_value = {
'default': 100, 'choices': QuerySet([node])}
go.return_value = inst go.return_value = inst
go4.return_value = MagicMock()
assert view.as_view()(request, pk=1234)['location'] assert view.as_view()(request, pk=1234)['location']
assert inst.migrate.async.called
assert msg.error.called assert msg.error.called
assert go4.called
def test_migrate_wo_permission(self): def test_migrate_wo_permission(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=False) request = FakeRequestFactory(POST={'to_node': 1}, superuser=False)
view = vm_ops['migrate'] view = vm_ops['migrate']
node = MagicMock(pk=1, name='node1')
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
patch('dashboard.views.vm.get_object_or_404') as go4: patch.object(view, 'get_form_kwargs') as form_kwargs:
inst = MagicMock(spec=Instance) inst = MagicMock(spec=Instance)
inst._meta.object_name = "Instance" inst._meta.object_name = "Instance"
inst.migrate = Instance._ops['migrate'](inst) inst.migrate = Instance._ops['migrate'](inst)
inst.migrate.async = MagicMock() inst.migrate.async = MagicMock()
inst.has_level.return_value = True inst.has_level.return_value = True
form_kwargs.return_value = {
'default': 100, 'choices': QuerySet([node])}
go.return_value = inst go.return_value = inst
go4.return_value = MagicMock()
with self.assertRaises(PermissionDenied): with self.assertRaises(PermissionDenied):
assert view.as_view()(request, pk=1234)['location'] assert view.as_view()(request, pk=1234)['location']
assert go4.called assert not inst.migrate.async.called
def test_migrate_template(self): def test_migrate_template(self):
"""check if GET dialog's template can be rendered""" """check if GET dialog's template can be rendered"""
...@@ -219,6 +234,7 @@ class VmOperationViewTestCase(unittest.TestCase): ...@@ -219,6 +234,7 @@ class VmOperationViewTestCase(unittest.TestCase):
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
patch('dashboard.views.util.messages') as msg: patch('dashboard.views.util.messages') as msg:
inst = MagicMock(spec=Instance) inst = MagicMock(spec=Instance)
inst.name = "asd"
inst._meta.object_name = "Instance" inst._meta.object_name = "Instance"
inst.save_as_template = Instance._ops['save_as_template'](inst) inst.save_as_template = Instance._ops['save_as_template'](inst)
inst.save_as_template.async = MagicMock() inst.save_as_template.async = MagicMock()
...@@ -235,6 +251,7 @@ class VmOperationViewTestCase(unittest.TestCase): ...@@ -235,6 +251,7 @@ class VmOperationViewTestCase(unittest.TestCase):
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
patch('dashboard.views.util.messages') as msg: patch('dashboard.views.util.messages') as msg:
inst = MagicMock(spec=Instance) inst = MagicMock(spec=Instance)
inst.name = "asd"
inst._meta.object_name = "Instance" inst._meta.object_name = "Instance"
inst.save_as_template = Instance._ops['save_as_template'](inst) inst.save_as_template = Instance._ops['save_as_template'](inst)
inst.save_as_template.async = MagicMock() inst.save_as_template.async = MagicMock()
...@@ -301,7 +318,7 @@ class VmMassOperationViewTestCase(unittest.TestCase): ...@@ -301,7 +318,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
view.as_view()(request, pk=1234).render() view.as_view()(request, pk=1234).render()
def test_migrate(self): def test_migrate(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=True) request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
view = vm_mass_ops['migrate'] view = vm_mass_ops['migrate']
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
...@@ -318,7 +335,7 @@ class VmMassOperationViewTestCase(unittest.TestCase): ...@@ -318,7 +335,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
assert not msg2.error.called assert not msg2.error.called
def test_migrate_failed(self): def test_migrate_failed(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=True) request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
view = vm_mass_ops['migrate'] view = vm_mass_ops['migrate']
with patch.object(view, 'get_object') as go, \ with patch.object(view, 'get_object') as go, \
...@@ -334,7 +351,7 @@ class VmMassOperationViewTestCase(unittest.TestCase): ...@@ -334,7 +351,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
assert msg.error.called assert msg.error.called
def test_migrate_wo_permission(self): def test_migrate_wo_permission(self):
request = FakeRequestFactory(POST={'node': 1}, superuser=False) request = FakeRequestFactory(POST={'to_node': 1}, superuser=False)
view = vm_mass_ops['migrate'] view = vm_mass_ops['migrate']
with patch.object(view, 'get_object') as go: with patch.object(view, 'get_object') as go:
......
circle/dashboard/views/template.py
...@@ -37,6 +37,7 @@ from braces.views import ( ...@@ -37,6 +37,7 @@ from braces.views import (
from django_tables2 import SingleTableView from django_tables2 import SingleTableView
from vm.models import InstanceTemplate, InterfaceTemplate, Instance, Lease from vm.models import InstanceTemplate, InterfaceTemplate, Instance, Lease
from storage.models import Disk
from ..forms import ( from ..forms import (
TemplateForm, TemplateListSearchForm, AclUserOrGroupAddForm, LeaseForm, TemplateForm, TemplateListSearchForm, AclUserOrGroupAddForm, LeaseForm,
...@@ -319,6 +320,57 @@ class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView): ...@@ -319,6 +320,57 @@ class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView):
return kwargs return kwargs
class DiskRemoveView(DeleteView):
model = Disk
def get_queryset(self):
qs = super(DiskRemoveView, self).get_queryset()
return qs.exclude(template_set=None)
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-delete.html']
else:
return ['dashboard/confirm/base-delete.html']
def get_context_data(self, **kwargs):
context = super(DiskRemoveView, self).get_context_data(**kwargs)
disk = self.get_object()
template = disk.template_set.get()
if not template.has_level(self.request.user, 'owner'):
raise PermissionDenied()
context['title'] = _("Disk remove confirmation")
context['text'] = _("Are you sure you want to remove "
"<strong>%(disk)s</strong> from "
"<strong>%(app)s</strong>?" % {'disk': disk,
'app': template}
)
return context
def delete(self, request, *args, **kwargs):
disk = self.get_object()
template = disk.template_set.get()
if not template.has_level(request.user, 'owner'):
raise PermissionDenied()
template.remove_disk(disk=disk, user=request.user)
disk.destroy()
next_url = request.POST.get("next")
success_url = next_url if next_url else template.get_absolute_url()
success_message = _("Disk successfully removed.")
if request.is_ajax():
return HttpResponse(
json.dumps({'message': success_message}),
content_type="application/json",
)
else:
messages.success(request, success_message)
return HttpResponseRedirect("%s#resources" % success_url)
class LeaseCreate(LoginRequiredMixin, PermissionRequiredMixin, class LeaseCreate(LoginRequiredMixin, PermissionRequiredMixin,
SuccessMessageMixin, CreateView): SuccessMessageMixin, CreateView):
model = Lease model = Lease
......
circle/dashboard/views/vm.py
...@@ -59,7 +59,8 @@ from ..forms import ( ...@@ -59,7 +59,8 @@ from ..forms import (
AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm, AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm, VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm, VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
TransferOwnershipForm, VmDiskResizeForm, RedeployForm, TransferOwnershipForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm,
) )
from ..models import Favourite, Profile from ..models import Favourite, Profile
...@@ -370,11 +371,9 @@ class VmAddInterfaceView(FormOperationMixin, VmOperationView): ...@@ -370,11 +371,9 @@ class VmAddInterfaceView(FormOperationMixin, VmOperationView):
return val return val
class VmDiskResizeView(FormOperationMixin, VmOperationView): class VmDiskModifyView(FormOperationMixin, VmOperationView):
op = 'resize_disk'
form_class = VmDiskResizeForm
show_in_toolbar = False show_in_toolbar = False
with_reload = True
icon = 'arrows-alt' icon = 'arrows-alt'
effect = "success" effect = "success"
...@@ -389,7 +388,7 @@ class VmDiskResizeView(FormOperationMixin, VmOperationView): ...@@ -389,7 +388,7 @@ class VmDiskResizeView(FormOperationMixin, VmOperationView):
else: else:
default = None default = None
val = super(VmDiskResizeView, self).get_form_kwargs() val = super(VmDiskModifyView, self).get_form_kwargs()
val.update({'choices': choices, 'default': default}) val.update({'choices': choices, 'default': default})
return val return val
...@@ -402,6 +401,14 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView): ...@@ -402,6 +401,14 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
icon = 'hdd-o' icon = 'hdd-o'
effect = "success" effect = "success"
is_disk_operation = True is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
op = self.get_op()
val = super(VmCreateDiskView, self).get_form_kwargs()
num = op.instance.disks.count() + 1
val['default'] = "%s %d" % (op.instance.name, num)
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView): class VmDownloadDiskView(FormOperationMixin, VmOperationView):
...@@ -412,38 +419,31 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView): ...@@ -412,38 +419,31 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
icon = 'download' icon = 'download'
effect = "success" effect = "success"
is_disk_operation = True is_disk_operation = True
with_reload = True
class VmMigrateView(VmOperationView): class VmMigrateView(FormOperationMixin, VmOperationView):
op = 'migrate' op = 'migrate'
icon = 'truck' icon = 'truck'
effect = 'info' effect = 'info'
template_name = 'dashboard/_vm-migrate.html' template_name = 'dashboard/_vm-migrate.html'
form_class = VmMigrateForm
def get_context_data(self, **kwargs): def get_form_kwargs(self):
ctx = super(VmMigrateView, self).get_context_data(**kwargs) online = (n.pk for n in Node.objects.filter(enabled=True) if n.online)
ctx['nodes'] = [n for n in Node.objects.filter(enabled=True) choices = Node.objects.filter(pk__in=online)
if n.online] default = None
inst = self.get_object() inst = self.get_object()
ctx["recommended"] = None
try: try:
if isinstance(inst, Instance): if isinstance(inst, Instance):
ctx["recommended"] = inst.select_node().pk default = inst.select_node()
except SchedulerError: except SchedulerError:
logger.exception("scheduler error:") logger.exception("scheduler error:")
return ctx val = super(VmMigrateView, self).get_form_kwargs()
val.update({'choices': choices, 'default': default})
def post(self, request, extra=None, *args, **kwargs): return val
if extra is None:
extra = {}
node = self.request.POST.get("node")
if node:
node = get_object_or_404(Node, pk=node)
extra["to_node"] = node
return super(VmMigrateView, self).post(request, extra, *args, **kwargs)
class VmSaveView(FormOperationMixin, VmOperationView): class VmSaveView(FormOperationMixin, VmOperationView):
...@@ -453,6 +453,12 @@ class VmSaveView(FormOperationMixin, VmOperationView): ...@@ -453,6 +453,12 @@ class VmSaveView(FormOperationMixin, VmOperationView):
effect = 'info' effect = 'info'
form_class = VmSaveForm form_class = VmSaveForm
def get_form_kwargs(self):
op = self.get_op()
val = super(VmSaveView, self).get_form_kwargs()
val['default'] = op._rename(op.instance.name)
return val
class VmResourcesChangeView(VmOperationView): class VmResourcesChangeView(VmOperationView):
op = 'resources_change' op = 'resources_change'
...@@ -649,7 +655,12 @@ vm_ops = OrderedDict([ ...@@ -649,7 +655,12 @@ vm_ops = OrderedDict([
op='destroy', icon='times', effect='danger')), op='destroy', icon='times', effect='danger')),
('create_disk', VmCreateDiskView), ('create_disk', VmCreateDiskView),
('download_disk', VmDownloadDiskView), ('download_disk', VmDownloadDiskView),
('resize_disk', VmDiskResizeView), ('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")),
('remove_disk', VmDiskModifyView.factory(
op='remove_disk', form_class=VmDiskRemoveForm,
icon='times', effect="danger")),
('add_interface', VmAddInterfaceView), ('add_interface', VmAddInterfaceView),
('renew', VmRenewView), ('renew', VmRenewView),
('resources_change', VmResourcesChangeView), ('resources_change', VmResourcesChangeView),
...@@ -751,6 +762,12 @@ class MassOperationView(OperationView): ...@@ -751,6 +762,12 @@ class MassOperationView(OperationView):
self.check_auth() self.check_auth()
if extra is None: if extra is None:
extra = {} extra = {}
if hasattr(self, 'form_class'):
form = self.form_class(self.request.POST, **self.get_form_kwargs())
if form.is_valid():
extra.update(form.cleaned_data)
self._call_operations(extra) self._call_operations(extra)
if request.is_ajax(): if request.is_ajax():
store = messages.get_messages(request) store = messages.get_messages(request)
...@@ -789,6 +806,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView): ...@@ -789,6 +806,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
allowed_filters = { allowed_filters = {
'name': "name__icontains", 'name': "name__icontains",
'node': "node__name__icontains", 'node': "node__name__icontains",
'node_exact': "node__name",
'status': "status__iexact", 'status': "status__iexact",
'tags[]': "tags__name__in", 'tags[]': "tags__name__in",
'tags': "tags__name__in", # for search string 'tags': "tags__name__in", # for search string
...@@ -1112,51 +1130,6 @@ class InstanceActivityDetail(CheckedDetailView): ...@@ -1112,51 +1130,6 @@ class InstanceActivityDetail(CheckedDetailView):
return ctx return ctx
class DiskRemoveView(DeleteView):
model = Disk
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/confirm/ajax-delete.html']
else:
return ['dashboard/confirm/base-delete.html']
def get_context_data(self, **kwargs):
context = super(DiskRemoveView, self).get_context_data(**kwargs)
disk = self.get_object()
app = disk.get_appliance()
context['title'] = _("Disk remove confirmation")
context['text'] = _("Are you sure you want to remove "
"<strong>%(disk)s</strong> from "
"<strong>%(app)s</strong>?" % {'disk': disk,
'app': app}
)
return context
def delete(self, request, *args, **kwargs):
disk = self.get_object()
app = disk.get_appliance()
if not app.has_level(request.user, 'owner'):
raise PermissionDenied()
app.remove_disk(disk=disk, user=request.user)
disk.destroy()
next_url = request.POST.get("next")
success_url = next_url if next_url else app.get_absolute_url()
success_message = _("Disk successfully removed.")
if request.is_ajax():
return HttpResponse(
json.dumps({'message': success_message}),
content_type="application/json",
)
else:
messages.success(request, success_message)
return HttpResponseRedirect("%s#resources" % success_url)
@require_GET @require_GET
def get_disk_download_status(request, pk): def get_disk_download_status(request, pk):
disk = Disk.objects.get(pk=pk) disk = Disk.objects.get(pk=pk)
......
circle/firewall/migrations/0052_auto__chg_field_record_address.py 0 → 100644
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Changing field 'Record.address'
db.alter_column(u'firewall_record', 'address', self.gf('django.db.models.fields.CharField')(max_length=400))
def backwards(self, orm):
# Changing field 'Record.address'
db.alter_column(u'firewall_record', 'address', self.gf('django.db.models.fields.CharField')(max_length=200))
models = {
u'auth.group': {
'Meta': {'object_name': 'Group'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
u'auth.permission': {
'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
u'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
u'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
u'firewall.blacklistitem': {
'Meta': {'object_name': 'BlacklistItem'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'host': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Host']", 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ipv4': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'reason': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'snort_message': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'type': ('django.db.models.fields.CharField', [], {'default': "'tempban'", 'max_length': '10'})
},
u'firewall.domain': {
'Meta': {'object_name': 'Domain'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
'ttl': ('django.db.models.fields.IntegerField', [], {'default': '600'})
},
u'firewall.ethernetdevice': {
'Meta': {'object_name': 'EthernetDevice'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
'switch_port': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ethernet_devices'", 'to': u"orm['firewall.SwitchPort']"})
},
u'firewall.firewall': {
'Meta': {'object_name': 'Firewall'},
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'})
},
u'firewall.group': {
'Meta': {'object_name': 'Group'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'})
},
u'firewall.host': {
'Meta': {'ordering': "('normalized_hostname', 'vlan')", 'unique_together': "(('hostname', 'vlan'),)", 'object_name': 'Host'},
'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'external_ipv4': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Group']", 'null': 'True', 'blank': 'True'}),
'hostname': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ipv4': ('firewall.fields.IPAddressField', [], {'unique': 'True', 'max_length': '100'}),
'ipv6': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'location': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'mac': ('firewall.fields.MACAddressField', [], {'unique': 'True', 'max_length': '17'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'normalized_hostname': ('common.models.HumanSortField', [], {'default': "''", 'maximum_number_length': '4', 'max_length': '80', 'monitor': "'hostname'", 'blank': 'True'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
'reverse': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
'shared_ip': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'vlan': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Vlan']"})
},
u'firewall.record': {
'Meta': {'ordering': "('domain', 'name')", 'object_name': 'Record'},
'address': ('django.db.models.fields.CharField', [], {'max_length': '400'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Domain']"}),
'host': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Host']", 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
'ttl': ('django.db.models.fields.IntegerField', [], {'default': '600'}),
'type': ('django.db.models.fields.CharField', [], {'max_length': '6'})
},
u'firewall.rule': {
'Meta': {'ordering': "('direction', 'proto', 'sport', 'dport', 'nat_external_port', 'host')", 'object_name': 'Rule'},
'action': ('django.db.models.fields.CharField', [], {'default': "'drop'", 'max_length': '10'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'direction': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
'dport': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'extra': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'firewall': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Firewall']"}),
'foreign_network': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ForeignRules'", 'to': u"orm['firewall.VlanGroup']"}),
'host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Host']"}),
'hostgroup': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Group']"}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'nat': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'nat_external_ipv4': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'nat_external_port': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
'proto': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
'sport': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
'vlan': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Vlan']"}),
'vlangroup': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.VlanGroup']"}),
'weight': ('django.db.models.fields.IntegerField', [], {'default': '30000'})
},
u'firewall.switchport': {
'Meta': {'object_name': 'SwitchPort'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'tagged_vlans': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'tagged_ports'", 'null': 'True', 'to': u"orm['firewall.VlanGroup']"}),
'untagged_vlan': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'untagged_ports'", 'to': u"orm['firewall.Vlan']"})
},
u'firewall.vlan': {
'Meta': {'object_name': 'Vlan'},
'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'dhcp_pool': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Domain']"}),
'host_ipv6_prefixlen': ('django.db.models.fields.IntegerField', [], {'default': '112'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ipv6_template': ('django.db.models.fields.TextField', [], {'default': "'2001:738:2001:4031:%(b)d:%(c)d:%(d)d:0'"}),
'managed': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
'network4': ('firewall.fields.IPNetworkField', [], {'max_length': '100'}),
'network6': ('firewall.fields.IPNetworkField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'network_type': ('django.db.models.fields.CharField', [], {'default': "'portforward'", 'max_length': '20'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
'reverse_domain': ('django.db.models.fields.TextField', [], {'default': "'%(d)d.%(c)d.%(b)d.%(a)d.in-addr.arpa'"}),
'snat_ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}),
'snat_to': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Vlan']", 'null': 'True', 'blank': 'True'}),
'vid': ('django.db.models.fields.IntegerField', [], {'unique': 'True'})
},
u'firewall.vlangroup': {
'Meta': {'object_name': 'VlanGroup'},
'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
'vlans': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Vlan']", 'null': 'True', 'blank': 'True'})
}
}
complete_apps = ['firewall']
\ No newline at end of file
circle/firewall/models.py
...@@ -874,7 +874,7 @@ class Record(models.Model): ...@@ -874,7 +874,7 @@ class Record(models.Model):
verbose_name=_('host')) verbose_name=_('host'))
type = models.CharField(max_length=6, choices=CHOICES_type, type = models.CharField(max_length=6, choices=CHOICES_type,
verbose_name=_('type')) verbose_name=_('type'))
address = models.CharField(max_length=200, address = models.CharField(max_length=400,
verbose_name=_('address')) verbose_name=_('address'))
ttl = models.IntegerField(default=600, verbose_name=_('ttl')) ttl = models.IntegerField(default=600, verbose_name=_('ttl'))
owner = models.ForeignKey(User, verbose_name=_('owner')) owner = models.ForeignKey(User, verbose_name=_('owner'))
......
circle/firewall/tasks/local_tasks.py
...@@ -29,26 +29,24 @@ settings = django.conf.settings.FIREWALL_SETTINGS ...@@ -29,26 +29,24 @@ settings = django.conf.settings.FIREWALL_SETTINGS
logger = getLogger(__name__) logger = getLogger(__name__)
def _apply_once(name, queues, task, data): def _apply_once(name, tasks, queues, task, data):
"""Reload given networking component if needed. """Reload given networking component if needed.
""" """
lockname = "%s_lock" % name if name not in tasks:
if not cache.get(lockname):
return return
cache.delete(lockname)
data = data() data = data()
for queue in queues: for queue in queues:
try: try:
task.apply_async(args=data, queue=queue, expires=60).get(timeout=5) task.apply_async(args=data, queue=queue, expires=60).get(timeout=2)
logger.info("%s configuration is reloaded. (queue: %s)", logger.info("%s configuration is reloaded. (queue: %s)",
name, queue) name, queue)
except TimeoutError as e: except TimeoutError as e:
logger.critical('%s (queue: %s)', e, queue) logger.critical('%s (queue: %s, task: %s)', e, queue, name)
except: except:
logger.critical('Unhandled exception: queue: %s data: %s', logger.critical('Unhandled exception: queue: %s data: %s task: %s',
queue, data, exc_info=True) queue, data, name, exc_info=True)
def get_firewall_queues(): def get_firewall_queues():
...@@ -68,19 +66,28 @@ def reloadtask_worker(): ...@@ -68,19 +66,28 @@ def reloadtask_worker():
from remote_tasks import (reload_dns, reload_dhcp, reload_firewall, from remote_tasks import (reload_dns, reload_dhcp, reload_firewall,
reload_firewall_vlan, reload_blacklist) reload_firewall_vlan, reload_blacklist)
tasks = []
for i in ('dns', 'dhcp', 'firewall', 'firewall_vlan', 'blacklist'):
lockname = "%s_lock" % i
if cache.get(lockname):
tasks.append(i)
cache.delete(lockname)
logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
firewall_queues = get_firewall_queues() firewall_queues = get_firewall_queues()
dns_queues = [("%s.dns" % i) for i in dns_queues = [("%s.dns" % i) for i in
settings.get('dns_queues', [gethostname()])] settings.get('dns_queues', [gethostname()])]
_apply_once('dns', dns_queues, reload_dns, _apply_once('dns', tasks, dns_queues, reload_dns,
lambda: (dns(), )) lambda: (dns(), ))
_apply_once('dhcp', firewall_queues, reload_dhcp, _apply_once('dhcp', tasks, firewall_queues, reload_dhcp,
lambda: (dhcp(), )) lambda: (dhcp(), ))
_apply_once('firewall', firewall_queues, reload_firewall, _apply_once('firewall', tasks, firewall_queues, reload_firewall,
lambda: (BuildFirewall().build_ipt())) lambda: (BuildFirewall().build_ipt()))
_apply_once('firewall_vlan', firewall_queues, reload_firewall_vlan, _apply_once('firewall_vlan', tasks, firewall_queues, reload_firewall_vlan,
lambda: (vlan(), )) lambda: (vlan(), ))
_apply_once('blacklist', firewall_queues, reload_blacklist, _apply_once('blacklist', tasks, firewall_queues, reload_blacklist,
lambda: (list(ipset()), )) lambda: (list(ipset()), ))
......
circle/manager/mancelery.py
...@@ -16,12 +16,15 @@ ...@@ -16,12 +16,15 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta from datetime import timedelta
from kombu import Queue, Exchange from kombu import Queue, Exchange
from os import getenv from os import getenv
HOSTNAME = "localhost" HOSTNAME = "localhost"
CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/") CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
QUEUE_NAME = HOSTNAME + '.man'
celery = Celery('manager', celery = Celery('manager',
broker=getenv("AMQP_URI"), broker=getenv("AMQP_URI"),
...@@ -57,3 +60,10 @@ celery.conf.update( ...@@ -57,3 +60,10 @@ celery.conf.update(
} }
) )
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
circle/manager/moncelery.py
...@@ -16,12 +16,14 @@ ...@@ -16,12 +16,14 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta from datetime import timedelta
from kombu import Queue, Exchange from kombu import Queue, Exchange
from os import getenv from os import getenv
HOSTNAME = "localhost" HOSTNAME = "localhost"
CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/") CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
QUEUE_NAME = HOSTNAME + '.monitor'
celery = Celery('monitor', celery = Celery('monitor',
broker=getenv("AMQP_URI"), broker=getenv("AMQP_URI"),
...@@ -34,7 +36,7 @@ celery.conf.update( ...@@ -34,7 +36,7 @@ celery.conf.update(
CELERY_CACHE_BACKEND=CACHE_URI, CELERY_CACHE_BACKEND=CACHE_URI,
CELERY_TASK_RESULT_EXPIRES=300, CELERY_TASK_RESULT_EXPIRES=300,
CELERY_QUEUES=( CELERY_QUEUES=(
Queue(HOSTNAME + '.monitor', Exchange('monitor', type='direct'), Queue(QUEUE_NAME, Exchange('monitor', type='direct'),
routing_key="monitor"), routing_key="monitor"),
), ),
CELERYBEAT_SCHEDULE={ CELERYBEAT_SCHEDULE={
...@@ -70,3 +72,10 @@ celery.conf.update( ...@@ -70,3 +72,10 @@ celery.conf.update(
} }
) )
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
circle/manager/slowcelery.py
...@@ -16,12 +16,14 @@ ...@@ -16,12 +16,14 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta from datetime import timedelta
from kombu import Queue, Exchange from kombu import Queue, Exchange
from os import getenv from os import getenv
HOSTNAME = "localhost" HOSTNAME = "localhost"
CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/") CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
QUEUE_NAME = HOSTNAME + '.man.slow'
celery = Celery('manager.slow', celery = Celery('manager.slow',
broker=getenv("AMQP_URI"), broker=getenv("AMQP_URI"),
...@@ -36,7 +38,7 @@ celery.conf.update( ...@@ -36,7 +38,7 @@ celery.conf.update(
CELERY_CACHE_BACKEND=CACHE_URI, CELERY_CACHE_BACKEND=CACHE_URI,
CELERY_TASK_RESULT_EXPIRES=300, CELERY_TASK_RESULT_EXPIRES=300,
CELERY_QUEUES=( CELERY_QUEUES=(
Queue(HOSTNAME + '.man.slow', Exchange('manager.slow', type='direct'), Queue(QUEUE_NAME, Exchange('manager.slow', type='direct'),
routing_key="manager.slow"), routing_key="manager.slow"),
), ),
CELERYBEAT_SCHEDULE={ CELERYBEAT_SCHEDULE={
...@@ -48,3 +50,10 @@ celery.conf.update( ...@@ -48,3 +50,10 @@ celery.conf.update(
} }
) )
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
circle/storage/models.py
...@@ -490,6 +490,9 @@ class Disk(TimeStampedModel): ...@@ -490,6 +490,9 @@ class Disk(TimeStampedModel):
disk.destroy() disk.destroy()
raise humanize_exception(ugettext_noop( raise humanize_exception(ugettext_noop(
"Operation aborted by user."), e) "Operation aborted by user."), e)
except:
disk.destroy()
raise
disk.is_ready = True disk.is_ready = True
disk.save() disk.save()
return disk return disk
circle/vm/models/activity.py
...@@ -20,7 +20,6 @@ from contextlib import contextmanager ...@@ -20,7 +20,6 @@ from contextlib import contextmanager
from logging import getLogger from logging import getLogger
from warnings import warn from warnings import warn
from celery.signals import worker_ready
from celery.contrib.abortable import AbortableAsyncResult from celery.contrib.abortable import AbortableAsyncResult
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
...@@ -263,17 +262,17 @@ def node_activity(code_suffix, node, task_uuid=None, user=None, ...@@ -263,17 +262,17 @@ def node_activity(code_suffix, node, task_uuid=None, user=None,
return activitycontextimpl(act) return activitycontextimpl(act)
@worker_ready.connect()
def cleanup(conf=None, **kwargs): def cleanup(conf=None, **kwargs):
# TODO check if other manager workers are running # TODO check if other manager workers are running
from celery.task.control import discard_all
discard_all()
msg_txt = ugettext_noop("Manager is restarted, activity is cleaned up. " msg_txt = ugettext_noop("Manager is restarted, activity is cleaned up. "
"You can try again now.") "You can try again now.")
message = create_readable(msg_txt, msg_txt) message = create_readable(msg_txt, msg_txt)
queue_name = kwargs.get('queue_name', None)
for i in InstanceActivity.objects.filter(finished__isnull=True): for i in InstanceActivity.objects.filter(finished__isnull=True):
i.finish(False, result=message) op = i.get_operation()
logger.error('Forced finishing stale activity %s', i) if op and op.async_queue == queue_name:
i.finish(False, result=message)
logger.error('Forced finishing stale activity %s', i)
for i in NodeActivity.objects.filter(finished__isnull=True): for i in NodeActivity.objects.filter(finished__isnull=True):
i.finish(False, result=message) i.finish(False, result=message)
logger.error('Forced finishing stale activity %s', i) logger.error('Forced finishing stale activity %s', i)
circle/vm/models/node.py
...@@ -313,10 +313,11 @@ class Node(OperatedMixin, TimeStampedModel): ...@@ -313,10 +313,11 @@ class Node(OperatedMixin, TimeStampedModel):
def get_status_label(self): def get_status_label(self):
return { return {
'OFFLINE': 'label-warning', 'OFFLINE': 'label-warning',
'DISABLED': 'label-warning', 'DISABLED': 'label-danger',
'MISSING': 'label-danger', 'MISSING': 'label-danger',
'ONLINE': 'label-success'}.get(self.get_state(), 'ACTIVE': 'label-success',
'label-danger') 'PASSIVE': 'label-warning',
}.get(self.get_state(), 'label-danger')
@node_available @node_available
def update_vm_states(self): def update_vm_states(self):
......
circle/vm/operations.py
...@@ -324,7 +324,7 @@ class DeployOperation(InstanceOperation): ...@@ -324,7 +324,7 @@ class DeployOperation(InstanceOperation):
"deployed to node: %(node)s"), "deployed to node: %(node)s"),
node=self.instance.node) node=self.instance.node)
def _operation(self, activity, timeout=15): def _operation(self, activity):
# Allocate VNC port and host node # Allocate VNC port and host node
self.instance.allocate_vnc_port() self.instance.allocate_vnc_port()
self.instance.allocate_node() self.instance.allocate_node()
...@@ -405,7 +405,7 @@ class DestroyOperation(InstanceOperation): ...@@ -405,7 +405,7 @@ class DestroyOperation(InstanceOperation):
required_perms = () required_perms = ()
resultant_state = 'DESTROYED' resultant_state = 'DESTROYED'
def _operation(self, activity): def _operation(self, activity, system):
# Destroy networks # Destroy networks
with activity.sub_activity( with activity.sub_activity(
'destroying_net', 'destroying_net',
...@@ -415,7 +415,7 @@ class DestroyOperation(InstanceOperation): ...@@ -415,7 +415,7 @@ class DestroyOperation(InstanceOperation):
self.instance.destroy_net() self.instance.destroy_net()
if self.instance.node: if self.instance.node:
self.instance._delete_vm(parent_activity=activity) self.instance._delete_vm(parent_activity=activity, system=system)
# Destroy disks # Destroy disks
with activity.sub_activity( with activity.sub_activity(
...@@ -425,7 +425,8 @@ class DestroyOperation(InstanceOperation): ...@@ -425,7 +425,8 @@ class DestroyOperation(InstanceOperation):
# Delete mem. dump if exists # Delete mem. dump if exists
try: try:
self.instance._delete_mem_dump(parent_activity=activity) self.instance._delete_mem_dump(parent_activity=activity,
system=system)
except: except:
pass pass
...@@ -470,12 +471,11 @@ class MigrateOperation(RemoteInstanceOperation): ...@@ -470,12 +471,11 @@ class MigrateOperation(RemoteInstanceOperation):
async_queue = "localhost.man.slow" async_queue = "localhost.man.slow"
task = vm_tasks.migrate task = vm_tasks.migrate
remote_queue = ("vm", "slow") remote_queue = ("vm", "slow")
timeout = 600 remote_timeout = 1000
def _get_remote_args(self, to_node, **kwargs): def _get_remote_args(self, to_node, live_migration, **kwargs):
return (super(MigrateOperation, self)._get_remote_args(**kwargs) return (super(MigrateOperation, self)._get_remote_args(**kwargs)
+ [to_node.host.hostname, True]) + [to_node.host.hostname, live_migration])
# TODO handle non-live migration
def rollback(self, activity): def rollback(self, activity):
with activity.sub_activity( with activity.sub_activity(
...@@ -483,7 +483,7 @@ class MigrateOperation(RemoteInstanceOperation): ...@@ -483,7 +483,7 @@ class MigrateOperation(RemoteInstanceOperation):
"redeploy network (rollback)")): "redeploy network (rollback)")):
self.instance.deploy_net() self.instance.deploy_net()
def _operation(self, activity, to_node=None): def _operation(self, activity, to_node=None, live_migration=True):
if not to_node: if not to_node:
with activity.sub_activity('scheduling', with activity.sub_activity('scheduling',
readable_name=ugettext_noop( readable_name=ugettext_noop(
...@@ -495,7 +495,8 @@ class MigrateOperation(RemoteInstanceOperation): ...@@ -495,7 +495,8 @@ class MigrateOperation(RemoteInstanceOperation):
with activity.sub_activity( with activity.sub_activity(
'migrate_vm', readable_name=create_readable( 'migrate_vm', readable_name=create_readable(
ugettext_noop("migrate to %(node)s"), node=to_node)): ugettext_noop("migrate to %(node)s"), node=to_node)):
super(MigrateOperation, self)._operation(to_node=to_node) super(MigrateOperation, self)._operation(
to_node=to_node, live_migration=live_migration)
except Exception as e: except Exception as e:
if hasattr(e, 'libvirtError'): if hasattr(e, 'libvirtError'):
self.rollback(activity) self.rollback(activity)
...@@ -575,6 +576,7 @@ class RemoveDiskOperation(InstanceOperation): ...@@ -575,6 +576,7 @@ class RemoveDiskOperation(InstanceOperation):
'destroy_disk', 'destroy_disk',
readable_name=ugettext_noop('destroy disk') readable_name=ugettext_noop('destroy disk')
): ):
disk.destroy()
return self.instance.disks.remove(disk) return self.instance.disks.remove(disk)
def get_activity_name(self, kwargs): def get_activity_name(self, kwargs):
...@@ -631,7 +633,7 @@ class SaveAsTemplateOperation(InstanceOperation): ...@@ -631,7 +633,7 @@ class SaveAsTemplateOperation(InstanceOperation):
for disk in self.disks: for disk in self.disks:
disk.destroy() disk.destroy()
def _operation(self, activity, user, system, timeout=300, name=None, def _operation(self, activity, user, system, name=None,
with_shutdown=True, task=None, **kwargs): with_shutdown=True, task=None, **kwargs):
if with_shutdown: if with_shutdown:
try: try:
...@@ -709,7 +711,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin, ...@@ -709,7 +711,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin,
resultant_state = 'STOPPED' resultant_state = 'STOPPED'
task = vm_tasks.shutdown task = vm_tasks.shutdown
remote_queue = ("vm", "slow") remote_queue = ("vm", "slow")
timeout = 120 remote_timeout = 120
def _operation(self, task): def _operation(self, task):
super(ShutdownOperation, self)._operation(task=task) super(ShutdownOperation, self)._operation(task=task)
...@@ -778,12 +780,12 @@ class SleepOperation(InstanceOperation): ...@@ -778,12 +780,12 @@ class SleepOperation(InstanceOperation):
else: else:
activity.resultant_state = 'ERROR' activity.resultant_state = 'ERROR'
def _operation(self, activity): def _operation(self, activity, system):
with activity.sub_activity('shutdown_net', with activity.sub_activity('shutdown_net',
readable_name=ugettext_noop( readable_name=ugettext_noop(
"shutdown network")): "shutdown network")):
self.instance.shutdown_net() self.instance.shutdown_net()
self.instance._suspend_vm(parent_activity=activity) self.instance._suspend_vm(parent_activity=activity, system=system)
self.instance.yield_node() self.instance.yield_node()
@register_operation @register_operation
...@@ -792,7 +794,7 @@ class SleepOperation(InstanceOperation): ...@@ -792,7 +794,7 @@ class SleepOperation(InstanceOperation):
name = _("suspend virtual machine") name = _("suspend virtual machine")
task = vm_tasks.sleep task = vm_tasks.sleep
remote_queue = ("vm", "slow") remote_queue = ("vm", "slow")
timeout = 600 remote_timeout = 1000
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return (super(SleepOperation.SuspendVmOperation, self) return (super(SleepOperation.SuspendVmOperation, self)
...@@ -845,7 +847,7 @@ class WakeUpOperation(InstanceOperation): ...@@ -845,7 +847,7 @@ class WakeUpOperation(InstanceOperation):
name = _("resume virtual machine") name = _("resume virtual machine")
task = vm_tasks.wake_up task = vm_tasks.wake_up
remote_queue = ("vm", "slow") remote_queue = ("vm", "slow")
timeout = 600 remote_timeout = 1000
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return (super(WakeUpOperation.WakeUpVmOperation, self) return (super(WakeUpOperation.WakeUpVmOperation, self)
......
circle/vm/tasks/agent_tasks.py
...@@ -53,8 +53,18 @@ def start_access_server(vm): ...@@ -53,8 +53,18 @@ def start_access_server(vm):
pass pass
@celery.task(name='agent.update_legacy')
def update_legacy(vm, data, executable=None):
pass
@celery.task(name='agent.append')
def append(vm, data, filename, chunk_number):
pass
@celery.task(name='agent.update') @celery.task(name='agent.update')
def update(vm, data): def update(vm, filename, executable, checksum):
pass pass
......
circle/vm/tasks/local_agent_tasks.py
...@@ -19,11 +19,14 @@ from common.models import create_readable ...@@ -19,11 +19,14 @@ from common.models import create_readable
from manager.mancelery import celery from manager.mancelery import celery
from vm.tasks.agent_tasks import (restart_networking, change_password, from vm.tasks.agent_tasks import (restart_networking, change_password,
set_time, set_hostname, start_access_server, set_time, set_hostname, start_access_server,
cleanup, update, change_ip) cleanup, update, append,
change_ip, update_legacy)
from firewall.models import Host from firewall.models import Host
import time import time
import os
from base64 import encodestring from base64 import encodestring
from hashlib import md5
from StringIO import StringIO from StringIO import StringIO
from tarfile import TarFile, TarInfo from tarfile import TarFile, TarInfo
from django.conf import settings from django.conf import settings
...@@ -61,17 +64,34 @@ def send_networking_commands(instance, act): ...@@ -61,17 +64,34 @@ def send_networking_commands(instance, act):
restart_networking.apply_async(queue=queue, args=(instance.vm_name, )) restart_networking.apply_async(queue=queue, args=(instance.vm_name, ))
def create_agent_tar(): def create_linux_tar():
def exclude(tarinfo): def exclude(tarinfo):
if tarinfo.name.startswith('./.git'): ignored = ('./.', './misc', './windows')
if any(tarinfo.name.startswith(x) for x in ignored):
return None return None
else: else:
return tarinfo return tarinfo
f = StringIO() f = StringIO()
with TarFile.open(fileobj=f, mode='w:gz') as tar:
agent_path = os.path.join(settings.AGENT_DIR, "agent-linux")
tar.add(agent_path, arcname='.', filter=exclude)
version_fileobj = StringIO(settings.AGENT_VERSION)
version_info = TarInfo(name='version.txt')
version_info.size = len(version_fileobj.buf)
tar.addfile(version_info, version_fileobj)
return encodestring(f.getvalue()).replace('\n', '')
def create_windows_tar():
f = StringIO()
agent_path = os.path.join(settings.AGENT_DIR, "agent-win")
with TarFile.open(fileobj=f, mode='w|gz') as tar: with TarFile.open(fileobj=f, mode='w|gz') as tar:
tar.add(settings.AGENT_DIR, arcname='.', filter=exclude) tar.add(agent_path, arcname='.')
version_fileobj = StringIO(settings.AGENT_VERSION) version_fileobj = StringIO(settings.AGENT_VERSION)
version_info = TarInfo(name='version.txt') version_info = TarInfo(name='version.txt')
...@@ -82,7 +102,7 @@ def create_agent_tar(): ...@@ -82,7 +102,7 @@ def create_agent_tar():
@celery.task @celery.task
def agent_started(vm, version=None): def agent_started(vm, version=None, system=None):
from vm.models import Instance, InstanceActivity from vm.models import Instance, InstanceActivity
instance = Instance.objects.get(id=int(vm.split('-')[-1])) instance = Instance.objects.get(id=int(vm.split('-')[-1]))
queue = instance.get_remote_queue_name("agent") queue = instance.get_remote_queue_name("agent")
...@@ -104,7 +124,7 @@ def agent_started(vm, version=None): ...@@ -104,7 +124,7 @@ def agent_started(vm, version=None):
if version and version != settings.AGENT_VERSION: if version and version != settings.AGENT_VERSION:
try: try:
update_agent(instance, act) update_agent(instance, act, system, settings.AGENT_VERSION)
except TimeoutError: except TimeoutError:
pass pass
else: else:
...@@ -146,11 +166,16 @@ def measure_boot_time(instance): ...@@ -146,11 +166,16 @@ def measure_boot_time(instance):
@celery.task @celery.task
def agent_stopped(vm): def agent_stopped(vm):
from vm.models import Instance, InstanceActivity from vm.models import Instance, InstanceActivity
from vm.models.activity import ActivityInProgressError
instance = Instance.objects.get(id=int(vm.split('-')[-1])) instance = Instance.objects.get(id=int(vm.split('-')[-1]))
qs = InstanceActivity.objects.filter(instance=instance, qs = InstanceActivity.objects.filter(instance=instance,
activity_code='vm.Instance.agent') activity_code='vm.Instance.agent')
act = qs.latest('id') act = qs.latest('id')
with act.sub_activity('stopping', readable_name=ugettext_noop('stopping')): try:
with act.sub_activity('stopping',
readable_name=ugettext_noop('stopping')):
pass
except ActivityInProgressError:
pass pass
...@@ -161,7 +186,7 @@ def get_network_configs(instance): ...@@ -161,7 +186,7 @@ def get_network_configs(instance):
return (interfaces, settings.FIREWALL_SETTINGS['rdns_ip']) return (interfaces, settings.FIREWALL_SETTINGS['rdns_ip'])
def update_agent(instance, act=None): def update_agent(instance, act=None, system=None, version=None):
if act: if act:
act = act.sub_activity( act = act.sub_activity(
'update', 'update',
...@@ -176,6 +201,40 @@ def update_agent(instance, act=None): ...@@ -176,6 +201,40 @@ def update_agent(instance, act=None):
version=settings.AGENT_VERSION)) version=settings.AGENT_VERSION))
with act: with act:
queue = instance.get_remote_queue_name("agent") queue = instance.get_remote_queue_name("agent")
update.apply_async( if system == "Windows":
queue=queue, executable = os.listdir(os.path.join(settings.AGENT_DIR,
args=(instance.vm_name, create_agent_tar())).get(timeout=10) "agent-win"))[0]
# executable = "agent-winservice-%(version)s.exe" % {
# 'version': version}
data = create_windows_tar()
elif system == "Linux":
executable = ""
data = create_linux_tar()
else:
executable = ""
# Legacy update method
return update_legacy.apply_async(
queue=queue,
args=(instance.vm_name, create_linux_tar())
).get(timeout=60)
checksum = md5(data).hexdigest()
chunk_size = 1024 * 1024
chunk_number = 0
index = 0
filename = version + ".tar"
while True:
chunk = data[index:index+chunk_size]
if chunk:
append.apply_async(
queue=queue,
args=(instance.vm_name, chunk,
filename, chunk_number)).get(timeout=60)
index = index + chunk_size
chunk_number = chunk_number + 1
else:
update.apply_async(
queue=queue,
args=(instance.vm_name, filename, executable, checksum)
).get(timeout=60)
break
circle/vm/tests/test_operations.py
...@@ -59,7 +59,8 @@ class MigrateOperationTestCase(TestCase): ...@@ -59,7 +59,8 @@ class MigrateOperationTestCase(TestCase):
MigrateException, op._operation, MigrateException, op._operation,
act, to_node=None) act, to_node=None)
assert inst.select_node.called assert inst.select_node.called
op._get_remote_args.assert_called_once_with(to_node='test') op._get_remote_args.assert_called_once_with(
to_node='test', live_migration=True)
class RebootOperationTestCase(TestCase): class RebootOperationTestCase(TestCase):
......
miscellaneous/mancelery.conf
...@@ -6,9 +6,14 @@ respawn limit 30 30 ...@@ -6,9 +6,14 @@ respawn limit 30 30
setgid cloud setgid cloud
setuid cloud setuid cloud
kill timeout 360
kill signal SIGTERM
script script
cd /home/cloud/circle/circle cd /home/cloud/circle/circle
. /home/cloud/.virtualenvs/circle/bin/activate . /home/cloud/.virtualenvs/circle/bin/activate
. /home/cloud/.virtualenvs/circle/bin/postactivate . /home/cloud/.virtualenvs/circle/bin/postactivate
exec ./manage.py celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 10 ./manage.py celery -f --app=manager.mancelery purge
exec ./manage.py celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 3
end script end script
miscellaneous/moncelery.conf
...@@ -3,6 +3,7 @@ description "CIRCLE moncelery for monitoring jobs" ...@@ -3,6 +3,7 @@ description "CIRCLE moncelery for monitoring jobs"
respawn respawn
respawn limit 30 30 respawn limit 30 30
setgid cloud setgid cloud
setuid cloud setuid cloud
...@@ -10,5 +11,7 @@ script ...@@ -10,5 +11,7 @@ script
cd /home/cloud/circle/circle cd /home/cloud/circle/circle
. /home/cloud/.virtualenvs/circle/bin/activate . /home/cloud/.virtualenvs/circle/bin/activate
. /home/cloud/.virtualenvs/circle/bin/postactivate . /home/cloud/.virtualenvs/circle/bin/postactivate
exec ./manage.py celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 3 ./manage.py celery -f --app=manager.moncelery purge
exec ./manage.py celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 2
end script end script
miscellaneous/slowcelery.conf
description "CIRCLE mancelery for slow jobs" description "CIRCLE slowcelery for resource intensive or long jobs"
respawn respawn
respawn limit 30 30 respawn limit 30 30
...@@ -6,9 +6,15 @@ respawn limit 30 30 ...@@ -6,9 +6,15 @@ respawn limit 30 30
setgid cloud setgid cloud
setuid cloud setuid cloud
kill timeout 360
kill signal INT
script script
cd /home/cloud/circle/circle cd /home/cloud/circle/circle
. /home/cloud/.virtualenvs/circle/bin/activate . /home/cloud/.virtualenvs/circle/bin/activate
. /home/cloud/.virtualenvs/circle/bin/postactivate . /home/cloud/.virtualenvs/circle/bin/postactivate
exec ./manage.py celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 5 ./manage.py celery -f --app=manager.slowcelery purge
exec ./manage.py celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 1
end script end script
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment