Skip to content

CIRCLE / cloud

Go to a project
Commit 51a45b5a by Kálmán Viktor
Options

Merge branch 'master' into feature-profile-rework

parents 287cedde e561b9eb
Showing with 619 additions and 619 deletions
circle/acl/management/__init__.py
......@@ -31,13 +31,13 @@ def create_levels(app, created_models, verbosity, db=DEFAULT_DB_ALIAS,
for klass in app_models:
# Force looking up the content types in the current database
# before creating foreign keys to them.
ctype = ContentType.objects.db_manager(db).get_for_model(klass)
ctypes.add(ctype)
ctype1 = ContentType.objects.db_manager(db).get_for_model(klass)
ctypes.add(ctype1)
weight = 0
try:
for codename, name in klass.ACL_LEVELS:
searched_levels.append((ctype, (codename, name)))
level_weights.append((ctype, codename, weight))
searched_levels.append((ctype1, (codename, name)))
level_weights.append((ctype1, codename, weight))
weight += 1
except AttributeError:
raise ImproperlyConfigured(
......
circle/circle/settings/base.py
......@@ -18,8 +18,10 @@
"""Common settings and globals."""
# flake8: noqa
from os import environ
from os.path import abspath, basename, dirname, join, normpath, isfile
from os.path import (abspath, basename, dirname, join, normpath, isfile,
expanduser)
from sys import path
from subprocess import check_output
from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import ugettext_lazy as _
......@@ -418,6 +420,16 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
LOGIN_REDIRECT_URL = "/"
AGENT_DIR = get_env_variable(
'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent'))
try:
git_env = {'GIT_DIR': join(AGENT_DIR, '.git')}
AGENT_VERSION = check_output(
('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env)
except:
AGENT_VERSION = None
LOCALE_PATHS = (join(SITE_ROOT, 'locale'), )
COMPANY_NAME = "BME IK 2014"
SOUTH_MIGRATION_MODULES = {
......
circle/circle/settings/local.py
......@@ -16,6 +16,7 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
"""Development settings and globals."""
# flake8: noqa
from base import * # noqa
......
circle/circle/settings/production.py
......@@ -16,6 +16,7 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
"""Production settings and globals."""
# flake8: noqa
from os import environ
......
circle/circle/settings/test.py
......@@ -17,6 +17,8 @@
from .base import * # noqa
# flake8: noqa
########## IN-MEMORY TEST DATABASE
DATABASES = {
"default": {
......
circle/circle/urls.py
......@@ -30,7 +30,7 @@ admin.autodiscover()
urlpatterns = patterns(
'',
#url(r'^$', TemplateView.as_view(template_name='base.html')),
# url(r'^$', TemplateView.as_view(template_name='base.html')),
# Examples:
# url(r'^$', 'circle.views.home', name='home'),
......
circle/common/operations.py
......@@ -33,6 +33,7 @@ class Operation(object):
required_perms = ()
do_not_call_in_templates = True
abortable = False
has_percentage = False
def __call__(self, **kwargs):
return self.call(**kwargs)
......
circle/dashboard/fixtures/test-vm-fixture.json
......@@ -38,25 +38,9 @@
"datastore": 1,
"dev_num": "a",
"type": "qcow2-norm",
"size": 8589934592
}
},
{
"pk": 1,
"model": "storage.diskactivity",
"fields":{
"activity_code": "storage.Disk.create",
"succeeded": true,
"parent": null,
"created": "2014-03-18T15:44:37.671Z",
"started": "2014-03-18T15:44:37.671Z",
"finished": "2014-03-18T15:44:37.677Z",
"modified": "2014-03-18T15:44:37.679Z",
"task_uuid": null,
"user": 1,
"disk": 1,
"result":null
}
"size": 8589934592,
"is_ready": true
}
},
{
"pk": 1,
......
circle/dashboard/forms.py
......@@ -24,6 +24,7 @@ from django.contrib.auth.forms import (
PasswordChangeForm,
)
from django.contrib.auth.models import User, Group
from django.core.validators import URLValidator
from crispy_forms.helper import FormHelper
from crispy_forms.layout import (
......@@ -40,9 +41,9 @@ from django.utils.translation import ugettext as _
from sizefield.widgets import FileSizeWidget
from firewall.models import Vlan, Host
from storage.models import Disk, DataStore
from storage.models import Disk
from vm.models import (
InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
InstanceTemplate, Lease, InterfaceTemplate, Node, Trait
)
from .models import Profile, GroupProfile
from circle.settings.base import LANGUAGES
......@@ -852,20 +853,12 @@ class LeaseForm(forms.ModelForm):
model = Lease
class DiskAddForm(forms.Form):
name = forms.CharField()
size = forms.CharField(widget=FileSizeWidget, required=False)
url = forms.CharField(required=False)
is_template = forms.CharField()
object_pk = forms.CharField()
def __init__(self, *args, **kwargs):
self.is_template = kwargs.pop("is_template")
self.object_pk = kwargs.pop("object_pk")
self.user = kwargs.pop("user")
super(DiskAddForm, self).__init__(*args, **kwargs)
self.initial['is_template'] = 1 if self.is_template else 0
self.initial['object_pk'] = self.object_pk
class VmCreateDiskForm(forms.Form):
name = forms.CharField(max_length=100, label=_("Name"))
size = forms.CharField(
widget=FileSizeWidget, initial=(10 << 30), label=_('Size'),
help_text=_('Size of disk to create in bytes or with units '
'like MB or GB.'))
def clean_size(self):
size_in_bytes = self.cleaned_data.get("size")
......@@ -874,66 +867,23 @@ class DiskAddForm(forms.Form):
" GB or MB!"))
return size_in_bytes
def clean(self):
cleaned_data = self.cleaned_data
size = cleaned_data.get("size")
url = cleaned_data.get("url")
if not size and not url:
msg = _("You have to either specify size or URL")
self._errors[_("Global")] = self.error_class([msg])
return cleaned_data
def save(self, commit=True):
data = self.cleaned_data
@property
def helper(self):
helper = FormHelper(self)
helper.form_tag = False
return helper
if self.is_template:
inst = InstanceTemplate.objects.get(pk=self.object_pk)
else:
inst = Instance.objects.get(pk=self.object_pk)
if data['size']:
kwargs = {
'name': data['name'],
'type': "qcow2-norm",
'datastore': DataStore.objects.all()[0],
'size': data['size'],
}
d = Disk.create_empty(instance=inst, user=self.user, **kwargs)
else:
kwargs = {
'name': data['name'],
'url': data['url'],
}
Disk.create_from_url_async(instance=inst, user=self.user,
**kwargs)
d = None
return d
class VmDownloadDiskForm(forms.Form):
name = forms.CharField(max_length=100, label=_("Name"))
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
@property
def helper(self):
helper = FormHelper()
helper.form_show_labels = False
helper.layout = Layout(
Field("is_template", type="hidden"),
Field("object_pk", type="hidden"),
Field("name", placeholder=_("Name")),
Field("size", placeholder=_("Disk size (for example: 20GB, "
"1500MB)")),
Field("url", placeholder=_("URL to an ISO image")),
AnyTag(
"div",
HTML(
_("Either specify the size for an empty disk or a URL "
"to an ISO image!")
),
css_class="alert alert-info",
style="padding: 5px; text-align: justify;",
),
)
helper.add_input(Submit("submit", _("Add"),
helper = FormHelper(self)
helper.add_input(Submit("submit", _("Create"),
css_class="btn btn-success"))
helper.form_tag = False
return helper
......
circle/dashboard/static/dashboard/dashboard.css
......@@ -705,6 +705,7 @@ textarea[name="list-new-namelist"] {
#group-detail-user-table td:nth-child(2) a,
#group-detail-perm-table td:nth-child(2) a,
#template-access-table td:nth-child(2) a,
#vm-access-table td:nth-child(2) a,
.no-style-link, .no-style-link:hover {
color: #555 !important;
......
circle/dashboard/static/dashboard/vm-common.js
......@@ -3,7 +3,7 @@
$(function() {
/* vm operations */
$('#ops').on('click', '.operation.btn', function(e) {
$('#ops, #vm-details-resources-disk').on('click', '.operation.btn', function(e) {
var icon = $(this).children("i").addClass('icon-spinner icon-spin');
$.ajax({
......
circle/dashboard/templates/dashboard/index-templates.html
......@@ -11,7 +11,7 @@
<div id="dashboard-template-list">
{% for t in templates %}
<a href="{% url "dashboard.views.template-detail" pk=t.pk %}" class="list-group-item
{% if forloop.last and nodes|length < 5 %} list-group-item-last{% endif %}">
{% if forloop.last and templates|length < 5 %} list-group-item-last{% endif %}">
<span class="index-template-list-name">
<i class="icon-{{ t.os_type }}"></i> {{ t.name }}
</span>
......
circle/dashboard/templates/dashboard/template-edit.html
......@@ -37,48 +37,62 @@
<th>{% trans "Who" %}</th>
<th>{% trans "What" %}</th>
<th><i class="icon-remove"></i></th>
</tr></thead>
</tr>
</thead>
<tbody>
{% for i in acl.users %}
<tr>
<td><i class="icon-user"></i></td><td>{{i.user}}</td>
<td>
<select class="form-control" name="perm-u-{{i.user.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-u-{{i.user.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
{% for i in acl.groups %}
<tr>
<td><i class="icon-group"></i></td><td>{{i.group}}</td>
<td>
<select class="form-control" name="perm-g-{{i.group.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-g-{{i.group.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
<tr><td><i class="icon-plus"></i></td>
<td><input type="text" class="form-control" name="perm-new-name"
placeholder="{% trans "Name of group or user" %}"></td>
<td><select class="form-control" name="perm-new">
{% for id, name in acl.levels %}
<option value="{{id}}">{{name}}</option>
{% endfor %}
</select></td><td></td>
</tr>
</tbody>
{% for i in acl.users %}
<tr>
<td>
<i class="icon-user"></i>
</td>
<td>
<a href="{% url "dashboard.views.profile" username=i.user.username %}"
title="{{ i.user.username }}">
{% include "dashboard/_display-name.html" with user=i.user show_org=True %}
</a>
</td>
<td>
<select class="form-control" name="perm-u-{{i.user.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-u-{{i.user.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
{% for i in acl.groups %}
<tr>
<td><i class="icon-group"></i></td>
<td>
<a href="{% url "dashboard.views.group-detail" pk=i.group.pk %}">
{{i.group}}
</a>
</td>
<td>
<select class="form-control" name="perm-g-{{i.group.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-g-{{i.group.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
<tr><td><i class="icon-plus"></i></td>
<td><input type="text" class="form-control" name="perm-new-name"
placeholder="{% trans "Name of group or user" %}"></td>
<td><select class="form-control" name="perm-new">
{% for id, name in acl.levels %}
<option value="{{id}}">{{name}}</option>
{% endfor %}
</select></td><td></td>
</tr>
</tbody>
</table>
<div class="form-actions">
<button type="submit" class="btn btn-success">{% trans "Save" %}</button>
......@@ -105,17 +119,6 @@
</ul>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="no-margin"><i class="icon-folder-open"></i> {% trans "Create new disk" %}</h4>
</div>
<div class="panel-body">
<form action="{% url "dashboard.views.disk-add" %}" method="POST">
{% crispy disk_add_form %}
</form>
</div>
</div>
</div><!-- .col-md-4 -->
</div><!-- .row -->
......
circle/dashboard/templates/dashboard/vm-detail/_activity-timeline.html
......@@ -13,6 +13,9 @@
{% include "dashboard/_display-name.html" with user=a.user show_org=True %}
</a>
{% endif %}
{% if a.has_percent %}
{{ a.percentage }}%
{% endif %}
{% if a.is_abortable_for_user %}
<form action="{{ a.instance.get_absolute_url }}" method="POST" class="pull-right">
{% csrf_token %}
......
circle/dashboard/templates/dashboard/vm-detail/_operations.html
{% load i18n %}
{% for op in ops %}
{% if op.show_in_toolbar %}
<a href="{{op.get_url}}" class="operation operation-{{op.op}} btn btn-default btn-xs"
title="{{op.name}}: {{op.description}}">
<i class="icon-{{op.icon}}"></i>
<span class="sr-only">{{op.name}}</span>
</a>
{% endif %}
{% endfor %}
circle/dashboard/templates/dashboard/vm-detail/resources.html
......@@ -47,9 +47,18 @@
<h3>
{% trans "Disks" %}
<div class="pull-right">
<a href="#" id="vm-details-disk-add" class="btn btn-success btn-xs">
<i class="icon-plus"></i> {% trans "Add new disk" %}
</a>
{% if op.download_disk %}
<a href="{{op.download_disk.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.download_disk.op}} btn btn-default">
<i class="icon-{{op.download_disk.icon}}"></i>
{{op.download_disk.name}} </a>
{% endif %}
{% if op.create_disk %}
<a href="{{op.create_disk.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.create_disk.op}} btn btn-default">
<i class="icon-{{op.create_disk.icon}}"></i>
{{op.create_disk.name}} </a>
{% endif %}
</div>
</h3>
......@@ -68,17 +77,6 @@
</div>
</div>
<div class="js-hidden row" id="vm-details-disk-add-form">
<div class="col-md-12">
<div>
<hr />
<form method="POST" action="{% url "dashboard.views.disk-add" %}" style="max-width: 350px;">
{% crispy forms.disk_add_form %}
</form>
<hr />
</div>
</div>
</div>
{% block extra_js %}
<style>
......
circle/dashboard/tests/test_views.py
......@@ -17,7 +17,7 @@
import json
from unittest import skip
# from unittest import skip
from django.test import TestCase
from django.test.client import Client
from django.contrib.auth.models import User, Group
......@@ -333,38 +333,6 @@ class VmDetailTest(LoginMixin, TestCase):
self.assertEqual(response.status_code, 302)
self.assertEqual(leases, Lease.objects.count())
def test_unpermitted_vm_disk_add(self):
c = Client()
self.login(c, "user2")
inst = Instance.objects.get(pk=1)
inst.set_level(self.u1, 'owner')
disks = inst.disks.count()
response = c.post("/dashboard/disk/add/", {
'disk-name': "a",
'disk-size': 1,
'disk-is_template': 0,
'disk-object_pk': 1,
})
self.assertEqual(response.status_code, 403)
self.assertEqual(disks, inst.disks.count())
@skip("until fix merged")
def test_permitted_vm_disk_add(self):
c = Client()
self.login(c, "user1")
inst = Instance.objects.get(pk=1)
inst.set_level(self.u1, 'owner')
# disks = inst.disks.count()
response = c.post("/dashboard/disk/add/", {
'disk-name': "a",
'disk-size': 1,
'disk-is_template': 0,
'disk-object_pk': 1,
})
self.assertEqual(response.status_code, 302)
# mancelery is needed TODO
# self.assertEqual(disks + 1, inst.disks.count())
def test_notification_read(self):
c = Client()
self.login(c, "user1")
......
circle/dashboard/urls.py
......@@ -20,7 +20,7 @@ from django.conf.urls import patterns, url, include
from vm.models import Instance
from .views import (
AclUpdateView, DiskAddView, FavouriteView, GroupAclUpdateView, GroupDelete,
AclUpdateView, FavouriteView, GroupAclUpdateView, GroupDelete,
GroupDetailView, GroupList, IndexView,
InstanceActivityDetail, LeaseCreate, LeaseDelete, LeaseDetail,
MyPreferencesView, NodeAddTraitView, NodeCreate, NodeDelete,
......@@ -128,8 +128,6 @@ urlpatterns = patterns(
url(r'^notifications/$', NotificationView.as_view(),
name="dashboard.views.notifications"),
url(r'^disk/add/$', DiskAddView.as_view(),
name="dashboard.views.disk-add"),
url(r'^disk/(?P<pk>\d+)/remove/$', DiskRemoveView.as_view(),
name="dashboard.views.disk-remove"),
url(r'^disk/(?P<pk>\d+)/status/$', get_disk_download_status,
......
circle/dashboard/views.py
......@@ -44,7 +44,6 @@ from django.views.generic import (TemplateView, DetailView, View, DeleteView,
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.utils.translation import ungettext as __
from django.template.defaultfilters import title as title_filter
from django.template.loader import render_to_string
from django.template import RequestContext
......@@ -55,10 +54,11 @@ from braces.views import (LoginRequiredMixin, SuperuserRequiredMixin,
from braces.views._access import AccessMixin
from .forms import (
CircleAuthenticationForm, DiskAddForm, HostForm, LeaseForm, MyProfileForm,
CircleAuthenticationForm, HostForm, LeaseForm, MyProfileForm,
NodeForm, TemplateForm, TraitForm, VmCustomizeForm, GroupCreateForm,
UserCreationForm, GroupProfileUpdateForm, UnsubscribeForm,
CirclePasswordChangeForm, VmSaveForm,
VmSaveForm,
CirclePasswordChangeForm, VmCreateDiskForm, VmDownloadDiskForm,
)
from .tables import (
......@@ -126,7 +126,11 @@ class FilterMixin(object):
filters = {}
for item in self.allowed_filters:
if item in self.request.GET:
filters[self.allowed_filters[item]] = self.request.GET[item]
filters[self.allowed_filters[item]] = (
self.request.GET[item].split(",")
if self.allowed_filters[item].endswith("__in") else
self.request.GET[item])
return filters
def get_queryset(self):
......@@ -275,12 +279,6 @@ class VmDetailView(CheckedDetailView):
instance=self.get_object()).values_list("vlan", flat=True)
).all()
context['acl'] = get_vm_acl_data(instance)
context['forms'] = {
'disk_add_form': DiskAddForm(
user=self.request.user,
is_template=False, object_pk=self.get_object().pk,
prefix="disk"),
}
context['os_type_icon'] = instance.os_type.replace("unknown",
"question")
# ipv6 infos
......@@ -600,6 +598,22 @@ class FormOperationMixin(object):
return self.get(request)
class VmCreateDiskView(FormOperationMixin, VmOperationView):
op = 'create_disk'
form_class = VmCreateDiskForm
show_in_toolbar = False
icon = 'hdd'
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
op = 'download_disk'
form_class = VmDownloadDiskForm
show_in_toolbar = False
icon = 'download'
class VmMigrateView(VmOperationView):
op = 'migrate'
......@@ -639,6 +653,8 @@ vm_ops = {
'destroy': VmOperationView.factory(op='destroy', icon='remove'),
'sleep': VmOperationView.factory(op='sleep', icon='moon'),
'wake_up': VmOperationView.factory(op='wake_up', icon='sun'),
'create_disk': VmCreateDiskView,
'download_disk': VmDownloadDiskView,
}
......@@ -897,7 +913,9 @@ class TemplateAclUpdateView(AclUpdateView):
post_for_disk['perm-new'] = 'user'
request.POST = post_for_disk
for d in template.disks.all():
self.set_levels(request, d)
self.add_levels(request, d)
self.remove_levels(request, d)
return redirect(template)
......@@ -1064,12 +1082,6 @@ class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView):
context = super(TemplateDetail, self).get_context_data(**kwargs)
context['acl'] = get_vm_acl_data(obj)
context['disks'] = obj.disks.all()
context['disk_add_form'] = DiskAddForm(
user=self.request.user,
is_template=True,
object_pk=obj.pk,
prefix="disk",
)
return context
def get_success_url(self):
......@@ -1150,7 +1162,8 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
'name': "name__icontains",
'node': "node__name__icontains",
'status': "status__iexact",
'tags': "tags__name__in", # note: use it as ?tags[]=a,b
'tags[]': "tags__name__in",
'tags': "tags__name__in", # for search string
'owner': "owner__username",
}
......@@ -1189,8 +1202,10 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
(sort[1:] if sort[0] == "-" else sort)
in [i.name for i in Instance._meta.fields] + ["pk"]):
queryset = queryset.order_by(sort)
return queryset.filter(**self.get_queryset_filters()
).select_related('owner', 'node')
return queryset.filter(
**self.get_queryset_filters()).select_related('owner', 'node'
).distinct()
def create_fake_get(self):
"""
......@@ -1222,8 +1237,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
# generate a new GET request, that is kinda fake
fake = self.request.GET.copy()
for k, v in got.iteritems():
fake["%s%s" % (
k, "[]" if len(v.split(",")) > 1 else "")] = v
fake[k] = v
self.request.GET = fake
......@@ -2523,47 +2537,6 @@ def circle_login(request):
return response
class DiskAddView(TemplateView):
def post(self, *args, **kwargs):
is_template = self.request.POST.get("disk-is_template")
object_pk = self.request.POST.get("disk-object_pk")
is_template = int(is_template) == 1
if is_template:
obj = InstanceTemplate.objects.get(pk=object_pk)
else:
obj = Instance.objects.get(pk=object_pk)
if not obj.has_level(self.request.user, 'owner'):
raise PermissionDenied()
form = DiskAddForm(
self.request.POST,
user=self.request.user,
is_template=is_template, object_pk=object_pk,
prefix="disk"
)
if form.is_valid():
if form.cleaned_data.get("size"):
messages.success(self.request, _("Disk successfully added."))
else:
messages.success(self.request, _("Disk download started."))
form.save()
else:
error = "<br /> ".join(["<strong>%s</strong>: %s" %
(title_filter(i[0]), i[1][0])
for i in form.errors.items()])
messages.error(self.request, error)
if is_template:
r = obj.get_absolute_url()
else:
r = obj.get_absolute_url()
r = "%s#resources" % r
return redirect(r)
class MyPreferencesView(UpdateView):
model = Profile
......
circle/firewall/fields.py
......@@ -201,7 +201,7 @@ class IPNetworkField(models.Field):
return super(IPNetworkField, self).formfield(**defaults)
add_introspection_rules([], ["firewall\.fields\.MACAddressField"])
add_introspection_rules([], ["firewall\.fields\."])
def val_alfanum(value):
......
circle/firewall/models.py
......@@ -52,7 +52,7 @@ class Rule(models.Model):
Others set address translation or other free-form iptables parameters.
"""
CHOICES_type = (('host', 'host'), ('firewall', 'firewall'),
('vlan', 'vlan'))
('vlan', 'vlan'))
CHOICES_proto = (('tcp', 'tcp'), ('udp', 'udp'), ('icmp', 'icmp'))
CHOICES_dir = (('out', _('out')), ('in', _('in')))
CHOICES_action = (('accept', _('accept')), ('drop', _('drop')),
......@@ -820,7 +820,7 @@ class Domain(models.Model):
class Record(models.Model):
CHOICES_type = (('A', 'A'), ('CNAME', 'CNAME'), ('AAAA', 'AAAA'),
('MX', 'MX'), ('NS', 'NS'), ('PTR', 'PTR'), ('TXT', 'TXT'))
('MX', 'MX'), ('NS', 'NS'), ('PTR', 'PTR'), ('TXT', 'TXT'))
name = models.CharField(max_length=40, validators=[val_domain],
blank=True, null=True, verbose_name=_('name'))
domain = models.ForeignKey('Domain', verbose_name=_('domain'))
......
circle/firewall/tests/test_firewall.py
......@@ -319,7 +319,7 @@ class ReloadTestCase(TestCase):
'tcp', public=1000, private=22)
def test_periodic_task(self):
#TODO
# TODO
with patch('firewall.tasks.local_tasks.cache') as cache:
self.test_host_add_port()
self.test_host_add_port2()
......
circle/network/static/js/network.js
......@@ -27,3 +27,7 @@ function getURLParameter(name) {
(RegExp(name + '=' + '(.+?)(&|$)').exec(location.search)||[,null])[1]
);
}
$(function() {
$("[title]").tooltip();
});
circle/network/static/network/network.css 0 → 100644
.table-with-form-fields tbody tr td {
line-height: 34px;
}
#vlan-access-table th:last-child, #vlan-access-table td:last-child {
text-align: center;
}
circle/network/tables.py
......@@ -120,7 +120,7 @@ class RecordTable(Table):
fields = ('type', 'fqdn', 'host', 'address', 'ttl', 'host',
'owner', )
sequence = ('type', 'fqdn', )
#order_by = 'name'
# order_by = 'name'
class SmallRecordTable(Table):
......@@ -131,7 +131,7 @@ class SmallRecordTable(Table):
attrs = {'class': 'table table-striped table-bordered'}
fields = ('type', 'fqdn', 'host', 'address', )
sequence = ('type', 'fqdn', )
#order_by = '-type'
# order_by = '-type'
orderable = False
......
circle/network/templates/network/base.html
......@@ -12,6 +12,7 @@
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.min.css">
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap-theme.min.css">
<link href="//netdna.bootstrapcdn.com/font-awesome/3.2.1/css/font-awesome.css" rel="stylesheet" />
<link href="{% static "network/network.css" %}" rel="stylesheet">
<style type="text/css">
body {
padding-top:40px;
......@@ -35,7 +36,6 @@
<!--[if lt IE 9]>
<script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<!--<link href="{% static "css/network.css" %}" rel="stylesheet">-->
{% block extra_css %}{% endblock %}
</head>
<body>
......
circle/network/templates/network/vlan-edit.html
......@@ -12,11 +12,72 @@
</div>
<div class="row">
<div class="col-sm-8">
{% crispy form %}
<div class="col-sm-6">
{% crispy form %}
</div>
<div class="col-sm-6">
<div class="page-header">
<h3>{% trans "Host list" %}</h3>
</div>
<div class="col-sm-4">
{% render_table host_list %}
{% render_table host_list %}
<div class="page-header">
<h3>{% trans "Manage access" %}</h3>
</div>
<form action="{% url "network.vlan-acl" vid=vlan_vid %}" method="post">{% csrf_token %}
<table class="table table-striped table-with-form-fields" id="vlan-access-table">
<thead>
<tr>
<th></th>
<th>{% trans "Who" %}</th>
<th>{% trans "What" %}</th>
<th><i class="icon-remove"></i></th>
</tr></thead>
<tbody>
{% for i in acl.users %}
<tr>
<td><i class="icon-user"></i></td><td>{{i.user}}</td>
<td>
<select class="form-control" name="perm-u-{{i.user.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-u-{{i.user.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
{% for i in acl.groups %}
<tr>
<td><i class="icon-group"></i></td><td>{{i.group}}</td>
<td>
<select class="form-control" name="perm-g-{{i.group.id}}">
{% for id, name in acl.levels %}
<option{%if id = i.level%} selected="selected"{%endif%} value="{{id}}">{{name}}</option>
{% endfor %}
</select>
</td>
<td>
<input type="checkbox" name="remove-g-{{i.group.id}}" title="{% trans "Remove" %}"/>
</td>
</tr>
{% endfor %}
<tr><td><i class="icon-plus"></i></td>
<td><input type="text" class="form-control" name="perm-new-name"
placeholder="{% trans "Name of group or user" %}"></td>
<td><select class="form-control" name="perm-new">
{% for id, name in acl.levels %}
<option value="{{id}}">{{name}}</option>
{% endfor %}
</select></td><td></td>
</tr>
</tbody>
</table>
<div class="form-actions">
<button type="submit" class="btn btn-success">{% trans "Save" %}</button>
</div>
</form>
</div>
</div>
{% endblock %}
circle/network/tests.py deleted 100644 → 0
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
circle/network/tests/test_views.py 0 → 100644
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from django.test import TestCase
from django.test.client import Client
from django.contrib.auth.models import User, Group
from mock import Mock
from dashboard.tests.test_views import LoginMixin
from vm.models import Instance
from firewall.models import Vlan, VlanGroup
import django.conf
settings = django.conf.settings.FIREWALL_SETTINGS
class VlanAclTest(LoginMixin, TestCase):
fixtures = ['test-vm-fixture.json', 'node.json']
def setUp(self):
Instance.get_remote_queue_name = Mock(return_value='test')
self.u1 = User.objects.create(username='user1')
self.u1.set_password('password')
self.u1.save()
self.u2 = User.objects.create(username='user2', is_staff=True)
self.u2.set_password('password')
self.u2.save()
self.us = User.objects.create(username='superuser', is_superuser=True)
self.us.set_password('password')
self.us.save()
self.g1 = Group.objects.create(name='group1')
self.g1.user_set.add(self.u1)
self.g1.user_set.add(self.u2)
self.g1.save()
settings["default_vlangroup"] = 'public'
VlanGroup.objects.create(name='public')
def tearDown(self):
super(VlanAclTest, self).tearDown()
self.u1.delete()
self.u2.delete()
self.us.delete()
self.g1.delete()
def test_add_new_user_permission(self):
c = Client()
self.login(c, "superuser")
vlan = Vlan.objects.get(vid=1)
self.assertEqual([], vlan.get_users_with_level())
resp = c.post("/network/vlans/1/acl/", {
'perm-new-name': "user1",
'perm-new': "user",
})
vlan = Vlan.objects.get(vid=1)
self.assertTrue((self.u1, "user") in vlan.get_users_with_level())
self.assertEqual(resp.status_code, 302)
def test_make_user_operator(self):
c = Client()
self.login(c, "superuser")
vlan = Vlan.objects.get(vid=1)
vlan.set_level(self.u1, "user")
self.assertTrue((self.u1, "user") in vlan.get_users_with_level())
resp = c.post("/network/vlans/1/acl/", {
'perm-u-%d' % self.u1.pk: "operator",
'perm-new': "",
'perm-new-name': "",
})
self.assertTrue((self.u1, "operator") in vlan.get_users_with_level())
self.assertEqual(resp.status_code, 302)
def test_remove_user_permission(self):
c = Client()
self.login(c, "superuser")
vlan = Vlan.objects.get(vid=1)
vlan.set_level(self.u1, "user")
self.assertTrue((self.u1, "user") in vlan.get_users_with_level())
resp = c.post("/network/vlans/1/acl/", {
'remove-u-%d' % self.u1.pk: "",
'perm-new': "",
'perm-new-name': "",
})
self.assertTrue((self.u1, "user") not in vlan.get_users_with_level())
self.assertEqual(resp.status_code, 302)
circle/network/urls.py
......@@ -30,7 +30,8 @@ from .views import (IndexView,
VlanGroupList, VlanGroupDetail, VlanGroupDelete,
VlanGroupCreate,
remove_host_group, add_host_group,
remove_switch_port_device, add_switch_port_device)
remove_switch_port_device, add_switch_port_device,
VlanAclUpdateView)
urlpatterns = patterns(
'',
......@@ -83,6 +84,8 @@ urlpatterns = patterns(
url('^vlans/$', VlanList.as_view(), name='network.vlan_list'),
url('^vlans/create$', VlanCreate.as_view(), name='network.vlan_create'),
url('^vlans/(?P<vid>\d+)/$', VlanDetail.as_view(), name='network.vlan'),
url('^vlans/(?P<vid>\d+)/acl/$', VlanAclUpdateView.as_view(),
name='network.vlan-acl'),
url('^vlans/delete/(?P<vid>\d+)/$', VlanDelete.as_view(),
name="network.vlan_delete"),
url('^vlangroups/$', VlanGroupList.as_view(),
......
circle/network/views.py
......@@ -41,6 +41,7 @@ from braces.views import LoginRequiredMixin, SuperuserRequiredMixin
from operator import itemgetter
from itertools import chain
import json
from dashboard.views import AclUpdateView
class SuccessMessageMixin(FormMixin):
......@@ -628,6 +629,21 @@ class VlanList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
table_pagination = False
def get_vlan_acl_data(obj):
levels = obj.ACL_LEVELS
users = obj.get_users_with_level()
users = [{'user': u, 'level': l} for u, l in users]
groups = obj.get_groups_with_level()
groups = [{'group': g, 'level': l} for g, l in groups]
return {'users': users, 'groups': groups, 'levels': levels}
class VlanAclUpdateView(AclUpdateView):
model = Vlan
slug_field = "vid"
slug_url_kwarg = "vid"
class VlanDetail(LoginRequiredMixin, SuperuserRequiredMixin,
SuccessMessageMixin, UpdateView):
model = Vlan
......@@ -646,6 +662,7 @@ class VlanDetail(LoginRequiredMixin, SuperuserRequiredMixin,
context['host_list'] = SmallHostTable(q)
context['vlan_vid'] = self.kwargs.get('vid')
context['acl'] = get_vlan_acl_data(self.get_object())
return context
success_url = reverse_lazy('network.vlan_list')
......
circle/storage/admin.py
......@@ -18,7 +18,7 @@
from django import contrib
# from django.utils.translation import ugettext_lazy as _
from .models import Disk, DataStore, DiskActivity
from .models import Disk, DataStore
class DiskAdmin(contrib.admin.ModelAdmin):
......@@ -31,5 +31,4 @@ class DataStoreAdmin(contrib.admin.ModelAdmin):
contrib.admin.site.register(Disk, DiskAdmin)
contrib.admin.site.register(DiskActivity)
contrib.admin.site.register(DataStore, DataStoreAdmin)
circle/storage/migrations/0014_auto__del_diskactivity__add_field_disk_is_ready__chg_field_disk_size.py 0 → 100644
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'DiskActivity'
db.delete_table(u'storage_diskactivity')
# Adding field 'Disk.is_ready'
db.add_column(u'storage_disk', 'is_ready',
self.gf('django.db.models.fields.BooleanField')(default=False),
keep_default=False)
# Changing field 'Disk.size'
db.alter_column(u'storage_disk', 'size', self.gf('sizefield.models.FileSizeField')(null=True))
def backwards(self, orm):
# Adding model 'DiskActivity'
db.create_table(u'storage_diskactivity', (
('task_uuid', self.gf('django.db.models.fields.CharField')(unique=True, max_length=50, null=True, blank=True)),
('parent', self.gf('django.db.models.fields.related.ForeignKey')(related_name='children', null=True, to=orm['storage.DiskActivity'], blank=True)),
('started', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
('finished', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
('result', self.gf('django.db.models.fields.TextField')(null=True, blank=True)),
('disk', self.gf('django.db.models.fields.related.ForeignKey')(related_name='activity_log', to=orm['storage.Disk'])),
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('activity_code', self.gf('django.db.models.fields.CharField')(max_length=100)),
('succeeded', self.gf('django.db.models.fields.NullBooleanField')(null=True, blank=True)),
('created', self.gf('model_utils.fields.AutoCreatedField')(default=datetime.datetime.now)),
('modified', self.gf('model_utils.fields.AutoLastModifiedField')(default=datetime.datetime.now)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, blank=True)),
))
db.send_create_signal(u'storage', ['DiskActivity'])
# Deleting field 'Disk.is_ready'
db.delete_column(u'storage_disk', 'is_ready')
# Changing field 'Disk.size'
db.alter_column(u'storage_disk', 'size', self.gf('sizefield.models.FileSizeField')(default=None))
models = {
u'storage.datastore': {
'Meta': {'ordering': "[u'name']", 'object_name': 'DataStore'},
'hostname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
'path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'})
},
u'storage.disk': {
'Meta': {'ordering': "[u'name']", 'object_name': 'Disk'},
'base': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "u'derivatives'", 'null': 'True', 'to': u"orm['storage.Disk']"}),
'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
'datastore': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['storage.DataStore']"}),
'destroyed': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
'dev_num': ('django.db.models.fields.CharField', [], {'default': "u'a'", 'max_length': '1'}),
'filename': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_ready': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'size': ('sizefield.models.FileSizeField', [], {'default': 'None', 'null': 'True'}),
'type': ('django.db.models.fields.CharField', [], {'max_length': '10'})
}
}
complete_apps = ['storage']
\ No newline at end of file
circle/storage/models.py
......@@ -19,13 +19,12 @@
from __future__ import unicode_literals
from contextlib import contextmanager
import logging
from os.path import join
import uuid
from celery.signals import worker_ready
from django.db.models import (Model, CharField, DateTimeField,
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
......@@ -33,11 +32,9 @@ from model_utils.models import TimeStampedModel
from sizefield.models import FileSizeField
from acl.models import AclBase
from .tasks import local_tasks, remote_tasks
from .tasks import local_tasks, storage_tasks
from celery.exceptions import TimeoutError
from manager.mancelery import celery
from common.models import (ActivityModel, activitycontextimpl,
WorkerNotFound)
from common.models import WorkerNotFound
logger = logging.getLogger(__name__)
......@@ -64,8 +61,12 @@ class DataStore(Model):
logger.debug("Checking for storage queue %s.%s",
self.hostname, queue_id)
if not check_worker or local_tasks.check_queue(self.hostname,
queue_id, priority):
return self.hostname + '.' + queue_id
queue_id,
priority):
queue_name = self.hostname + '.' + queue_id
if priority is not None:
queue_name = queue_name + '.' + priority
return queue_name
else:
raise WorkerNotFound()
......@@ -99,6 +100,8 @@ class Disk(AclBase, TimeStampedModel):
verbose_name=_("device number"))
destroyed = DateTimeField(blank=True, default=None, null=True)
is_ready = BooleanField(default=False)
class Meta:
ordering = ['name']
verbose_name = _('disk')
......@@ -142,22 +145,6 @@ class Disk(AclBase, TimeStampedModel):
self.disk = disk
@property
def is_ready(self):
""" Returns True if the disk is physically ready on the storage.
It needs at least 1 successfull deploy action.
"""
return self.activity_log.filter(activity_code__endswith="deploy",
succeeded=True)
@property
def failed(self):
""" Returns True if the last activity on the disk is failed.
"""
result = self.activity_log.all().order_by('-id')[0].succeeded
return not (result is None) and not result
@property
def path(self):
"""The path where the files are stored.
"""
......@@ -199,24 +186,6 @@ class Disk(AclBase, TimeStampedModel):
'raw-rw': 'vd',
}[self.type]
def is_downloading(self):
return self.size is None and not self.failed
def get_download_percentage(self):
if not self.is_downloading():
return None
try:
task = self.activity_log.filter(
activity_code__endswith="deploy",
succeeded__isnull=True)[0].task_uuid
result = celery.AsyncResult(id=task)
return result.info.get("percent")
except:
return 0
def get_latest_activity_result(self):
return self.activity_log.latest("pk").result
@property
def is_deletable(self):
"""True if the associated file can be deleted.
......@@ -334,92 +303,38 @@ class Disk(AclBase, TimeStampedModel):
if self.is_ready:
return True
with disk_activity(code_suffix='deploy', disk=self,
task_uuid=task_uuid, user=user) as act:
# Delegate create / snapshot jobs
queue_name = self.get_remote_queue_name('storage')
disk_desc = self.get_disk_desc()
if self.base is not None:
with act.sub_activity('creating_snapshot'):
remote_tasks.snapshot.apply_async(args=[disk_desc],
queue=queue_name
).get(timeout=timeout)
else:
with act.sub_activity('creating_disk'):
remote_tasks.create.apply_async(args=[disk_desc],
queue=queue_name
).get(timeout=timeout)
queue_name = self.get_remote_queue_name('storage', priority="fast")
disk_desc = self.get_disk_desc()
if self.base is not None:
storage_tasks.snapshot.apply_async(args=[disk_desc],
queue=queue_name
).get(timeout=timeout)
else:
storage_tasks.create.apply_async(args=[disk_desc],
queue=queue_name
).get(timeout=timeout)
return True
def deploy_async(self, user=None):
"""Execute deploy asynchronously.
"""
return local_tasks.deploy.apply_async(args=[self, user],
queue="localhost.man")
self.is_ready = True
self.save()
return True
@classmethod
def create(cls, instance=None, user=None, **params):
"""Create disk with activity.
"""
datastore = params.pop('datastore', DataStore.objects.get())
filename = params.pop('filename', str(uuid.uuid4()))
disk = cls(filename=filename, datastore=datastore, **params)
def create(cls, user=None, **params):
disk = cls.__create(user, params)
disk.clean()
disk.save()
logger.debug("Disk created: %s", params)
with disk_activity(code_suffix="create",
user=user,
disk=disk):
if instance:
instance.disks.add(disk)
return disk
@classmethod
def create_empty_async(cls, instance=None, user=None, **kwargs):
"""Execute deploy asynchronously.
"""
return local_tasks.create_empty.apply_async(
args=[cls, instance, user, kwargs], queue="localhost.man")
@classmethod
def create_empty(cls, instance=None, user=None, task_uuid=None, **kwargs):
"""Create empty Disk object.
:param instance: Instance or template attach the Disk to.
:type instance: vm.models.Instance or InstanceTemplate or NoneType
:param user: Creator of the disk.
:type user: django.contrib.auth.User
:return: Disk object without a real image, to be .deploy()ed later.
"""
disk = Disk.create(instance, user, **kwargs)
disk.deploy(user=user, task_uuid=task_uuid)
def __create(cls, user, params):
datastore = params.pop('datastore', DataStore.objects.get())
filename = params.pop('filename', str(uuid.uuid4()))
disk = cls(filename=filename, datastore=datastore, **params)
return disk
@classmethod
def create_from_url_async(cls, url, instance=None, user=None, **kwargs):
"""Create disk object and download data from url asynchrnously.
:param url: URL of image to download.
:type url: string
:param instance: Instance or template attach the Disk to.
:type instance: vm.models.Instance or InstanceTemplate or NoneType
:param user: owner of the disk
:type user: django.contrib.auth.User
:return: Task
:rtype: AsyncResult
"""
kwargs.update({'cls': cls, 'url': url,
'instance': instance, 'user': user})
return local_tasks.create_from_url.apply_async(
kwargs=kwargs, queue='localhost.man')
@classmethod
def create_from_url(cls, url, instance=None, user=None,
task_uuid=None, abortable_task=None, **kwargs):
def download(cls, url, task, user=None, **params):
"""Create disk object and download data from url synchronusly.
:param url: image url to download.
......@@ -434,57 +349,35 @@ class Disk(AclBase, TimeStampedModel):
:return: The created Disk object
:rtype: Disk
"""
kwargs.setdefault('name', url.split('/')[-1])
disk = Disk.create(type="iso", instance=instance, user=user,
size=None, **kwargs)
queue_name = disk.get_remote_queue_name('storage')
def __on_abort(activity, error):
activity.disk.destroyed = timezone.now()
activity.disk.save()
if abortable_task:
from celery.contrib.abortable import AbortableAsyncResult
class AbortException(Exception):
pass
with disk_activity(code_suffix='deploy', disk=disk,
task_uuid=task_uuid, user=user,
on_abort=__on_abort) as act:
with act.sub_activity('downloading_disk'):
result = remote_tasks.download.apply_async(
kwargs={'url': url, 'parent_id': task_uuid,
'disk': disk.get_disk_desc()},
queue=queue_name)
while True:
try:
size = result.get(timeout=5)
break
except TimeoutError:
if abortable_task and abortable_task.is_aborted():
AbortableAsyncResult(result.id).abort()
raise AbortException("Download aborted by user.")
disk.size = size
disk.save()
params.setdefault('name', url.split('/')[-1])
params.setdefault('type', 'iso')
params.setdefault('size', None)
disk = cls.__create(params=params, user=user)
queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.download.apply_async(
kwargs={'url': url, 'parent_id': task.request.id,
'disk': disk.get_disk_desc()},
queue=queue_name)
while True:
try:
size = remote.get(timeout=5)
break
except TimeoutError:
if task is not None and task.is_aborted():
AbortableAsyncResult(remote.id).abort()
raise Exception("Download aborted by user.")
disk.size = size
disk.is_ready = True
disk.save()
return disk
def destroy(self, user=None, task_uuid=None):
if self.destroyed:
return False
with disk_activity(code_suffix='destroy', disk=self,
task_uuid=task_uuid, user=user):
self.destroyed = timezone.now()
self.save()
return True
def destroy_async(self, user=None):
"""Execute destroy asynchronously.
"""
return local_tasks.destroy.apply_async(args=[self, user],
queue='localhost.man')
self.destroyed = timezone.now()
self.save()
return True
def restore(self, user=None, task_uuid=None):
"""Recover destroyed disk from trash if possible.
......@@ -492,62 +385,6 @@ class Disk(AclBase, TimeStampedModel):
# TODO
pass
def restore_async(self, user=None):
local_tasks.restore.apply_async(args=[self, user],
queue='localhost.man')
def clone_async(self, new_disk=None, timeout=300, user=None):
"""Clone a Disk to another Disk
:param new_disk: optional, the new Disk object to clone in
:type new_disk: storage.models.Disk
:param user: Creator of the disk.
:type user: django.contrib.auth.User
:return: AsyncResult
"""
return local_tasks.clone.apply_async(args=[self, new_disk,
timeout, user],
queue="localhost.man")
def clone(self, disk=None, user=None, task_uuid=None, timeout=300):
"""Cloning Disk into another Disk.
The Disk.type can'T be snapshot.
:param new_disk: optional, the new Disk object to clone in
:type new_disk: storage.models.Disk
:param user: Creator of the disk.
:type user: django.contrib.auth.User
:return: the cloned Disk object.
"""
banned_types = ['qcow2-snap']
if self.type in banned_types:
raise self.WrongDiskTypeError(self.type)
if self.is_in_use:
raise self.DiskInUseError(self)
if not self.is_ready:
raise self.DiskIsNotReady(self)
if not disk:
base = None
if self.type == "iso":
base = self
disk = Disk.create(datastore=self.datastore,
name=self.name, size=self.size,
type=self.type, base=base)
with disk_activity(code_suffix="clone", disk=self,
user=user, task_uuid=task_uuid):
with disk_activity(code_suffix="deploy", disk=disk,
user=user, task_uuid=task_uuid):
queue_name = self.get_remote_queue_name('storage')
remote_tasks.merge.apply_async(args=[self.get_disk_desc(),
disk.get_disk_desc()],
queue=queue_name
).get() # Timeout
return disk
def save_as(self, user=None, task_uuid=None, timeout=300):
"""Save VM as template.
......@@ -582,65 +419,11 @@ class Disk(AclBase, TimeStampedModel):
name=self.name, size=self.size,
type=new_type)
with disk_activity(code_suffix="save_as", disk=self,
user=user, task_uuid=task_uuid):
with disk_activity(code_suffix="deploy", disk=disk,
user=user, task_uuid=task_uuid):
queue_name = self.get_remote_queue_name('storage')
remote_tasks.merge.apply_async(args=[self.get_disk_desc(),
disk.get_disk_desc()],
queue=queue_name
).get() # Timeout
return disk
class DiskActivity(ActivityModel):
disk = ForeignKey(Disk, related_name='activity_log',
help_text=_('Disk this activity works on.'),
verbose_name=_('disk'))
@classmethod
def create(cls, code_suffix, disk, task_uuid=None, user=None):
act = cls(activity_code='storage.Disk.' + code_suffix,
disk=disk, parent=None, started=timezone.now(),
task_uuid=task_uuid, user=user)
act.save()
return act
def __unicode__(self):
if self.parent:
return '{}({})->{}'.format(self.parent.activity_code,
self.disk,
self.activity_code)
else:
return '{}({})'.format(self.activity_code,
self.disk)
def create_sub(self, code_suffix, task_uuid=None):
act = DiskActivity(
activity_code=self.activity_code + '.' + code_suffix,
disk=self.disk, parent=self, started=timezone.now(),
task_uuid=task_uuid, user=self.user)
act.save()
return act
@contextmanager
def sub_activity(self, code_suffix, task_uuid=None):
act = self.create_sub(code_suffix, task_uuid)
return activitycontextimpl(act)
@contextmanager
def disk_activity(code_suffix, disk, task_uuid=None, user=None,
on_abort=None, on_commit=None):
act = DiskActivity.create(code_suffix, disk, task_uuid, user)
return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
@worker_ready.connect()
def cleanup(conf=None, **kwargs):
# TODO check if other manager workers are running
for i in DiskActivity.objects.filter(finished__isnull=True):
i.finish(False, "Manager is restarted, activity is cleaned up. "
"You can try again now.")
logger.error('Forced finishing stale activity %s', i)
queue_name = self.get_remote_queue_name("storage", priority="slow")
storage_tasks.merge.apply_async(args=[self.get_disk_desc(),
disk.get_disk_desc()],
queue=queue_name
).get() # Timeout
disk.is_ready = True
disk.save()
return disk
circle/storage/tasks/periodic_tasks.py
......@@ -18,7 +18,7 @@
from storage.models import DataStore
from manager.mancelery import celery
import logging
from storage.tasks import remote_tasks
from storage.tasks import storage_tasks
logger = logging.getLogger(__name__)
......@@ -34,18 +34,18 @@ def garbage_collector(timeout=15):
:type timeout: int
"""
for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage')
files = set(remote_tasks.list_files.apply_async(
queue_name = ds.get_remote_queue_name('storage', priority='fast')
files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout))
disks = set(ds.get_deletable_disks())
queue_name = ds.get_remote_queue_name('storage')
queue_name = ds.get_remote_queue_name('storage', priority='slow')
for i in disks & files:
logger.info("Image: %s at Datastore: %s moved to trash folder." %
(i, ds.path))
remote_tasks.move_to_trash.apply_async(
storage_tasks.move_to_trash.apply_async(
args=[ds.path, i], queue=queue_name).get(timeout=timeout)
try:
remote_tasks.make_free_space.apply_async(
storage_tasks.make_free_space.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout)
except Exception as e:
logger.warning(str(e))
......@@ -63,7 +63,7 @@ def list_orphan_disks(timeout=15):
import re
for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage')
files = set(remote_tasks.list_files.apply_async(
files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout))
disks = set([disk.filename for disk in ds.disk_set.all()])
for i in files - disks:
......@@ -80,7 +80,7 @@ def list_missing_disks(timeout=15):
"""
for ds in DataStore.objects.all():
queue_name = ds.get_remote_queue_name('storage')
files = set(remote_tasks.list_files.apply_async(
files = set(storage_tasks.list_files.apply_async(
args=[ds.path], queue=queue_name).get(timeout=timeout))
disks = set([disk.filename for disk in
ds.disk_set.filter(destroyed__isnull=True)])
......
circle/storage/tests/test_models.py
......@@ -19,7 +19,7 @@ from datetime import timedelta
from django.test import TestCase
from django.utils import timezone
from mock import MagicMock, Mock
from mock import MagicMock
from ..models import Disk, DataStore
......@@ -99,11 +99,6 @@ class DiskTestCase(TestCase):
with self.assertRaises(MockException):
Disk.save_as(d)
def test_download_percentage_no_download(self):
d = MagicMock(spec=Disk)
d.is_downloading = Mock(return_value=False)
assert Disk.get_download_percentage(d) is None
def test_undeployed_disk_ready(self):
d = self._disk()
assert not d.is_ready
circle/vm/models/activity.py
......@@ -116,6 +116,20 @@ class InstanceActivity(ActivityModel):
else:
return 'failed'
def has_percentage(self):
op = self.instance.get_operation_from_activity_code(self.activity_code)
return (self.task_uuid and op and op.has_percentage
and not self.finished)
def get_percentage(self):
"""Returns the percentage of the running operation if available.
"""
result = celery.AsyncResult(id=self.task_uuid)
if self.has_percentage() and result.info is not None:
return result.info.get("percent")
else:
return 0
@property
def is_abortable(self):
"""Can the activity be aborted?
......
circle/vm/models/common.py
......@@ -55,8 +55,8 @@ class NamedBaseResourceConfig(BaseResourceConfigModel, TimeStampedModel):
"""Pre-created, named base resource configurations.
"""
name = CharField(max_length=50, unique=True,
verbose_name=_('name'), help_text=
_('Name of base resource configuration.'))
verbose_name=_('name'),
help_text=_('Name of base resource configuration.'))
class Meta:
app_label = 'vm'
......
circle/vm/models/instance.py
......@@ -910,6 +910,11 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
acts = (self.activity_log.filter(parent=None).
order_by('-started').
select_related('user').prefetch_related('children'))
# Check latest activity for percentage
for i in acts:
if i.has_percentage():
i.has_percent = True
i.percentage = i.get_percentage()
if user is not None:
for i in acts:
i.is_abortable_for_user = partial(i.is_abortable_for,
......
circle/vm/operations.py
......@@ -26,6 +26,7 @@ from django.utils.translation import ugettext_lazy as _
from celery.exceptions import TimeLimitExceeded
from common.operations import Operation, register_operation
from storage.models import Disk
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
......@@ -99,24 +100,48 @@ class AddInterfaceOperation(InstanceOperation):
register_operation(AddInterfaceOperation)
class AddDiskOperation(InstanceOperation):
activity_code_suffix = 'add_disk'
id = 'add_disk'
name = _("add disk")
description = _("Add the specified disk to the VM.")
class CreateDiskOperation(InstanceOperation):
activity_code_suffix = 'create_disk'
id = 'create_disk'
name = _("create disk")
description = _("Create empty disk for the VM.")
def check_precond(self):
super(AddDiskOperation, self).check_precond()
super(CreateDiskOperation, self).check_precond()
# TODO remove check when hot-attach is implemented
if self.instance.status not in ['STOPPED']:
raise self.instance.WrongStateError(self.instance)
def _operation(self, activity, user, system, disk):
def _operation(self, user, size, name=None):
# TODO implement with hot-attach when it'll be available
return self.instance.disks.add(disk)
if not name:
name = "new disk"
disk = Disk.create(size=size, name=name, type="qcow2-norm")
self.instance.disks.add(disk)
register_operation(CreateDiskOperation)
class DownloadDiskOperation(InstanceOperation):
activity_code_suffix = 'download_disk'
id = 'download_disk'
name = _("download disk")
description = _("Download disk for the VM.")
abortable = True
has_percentage = True
def check_precond(self):
super(DownloadDiskOperation, self).check_precond()
# TODO remove check when hot-attach is implemented
if self.instance.status not in ['STOPPED']:
raise self.instance.WrongStateError(self.instance)
def _operation(self, user, url, task, name=None):
# TODO implement with hot-attach when it'll be available
disk = Disk.download(url=url, name=name, task=task)
self.instance.disks.add(disk)
register_operation(AddDiskOperation)
register_operation(DownloadDiskOperation)
class DeployOperation(InstanceOperation):
......
circle/vm/tasks/agent_tasks.py
......@@ -51,3 +51,8 @@ def cleanup(vm):
@celery.task(name='agent.start_access_server')
def start_access_server(vm):
pass
@celery.task(name='agent.update')
def update(vm, data):
pass
circle/vm/tasks/local_agent_tasks.py
......@@ -18,8 +18,13 @@
from manager.mancelery import celery
from vm.tasks.agent_tasks import (restart_networking, change_password,
set_time, set_hostname, start_access_server,
cleanup)
cleanup, update)
import time
from base64 import encodestring
from StringIO import StringIO
from tarfile import TarFile, TarInfo
from django.conf import settings
from celery.result import TimeoutError
def send_init_commands(instance, act, vm):
......@@ -38,22 +43,53 @@ def send_init_commands(instance, act, vm):
queue=queue, args=(vm, instance.primary_host.hostname))
def create_agent_tar():
def exclude(tarinfo):
if tarinfo.name.startswith('./.git'):
return None
else:
return tarinfo
f = StringIO()
with TarFile.open(fileobj=f, mode='w|gz') as tar:
tar.add(settings.AGENT_DIR, arcname='.', filter=exclude)
version_fileobj = StringIO(settings.AGENT_VERSION)
version_info = TarInfo(name='version.txt')
version_info.size = len(version_fileobj.buf)
tar.addfile(version_info, version_fileobj)
return encodestring(f.getvalue()).replace('\n', '')
@celery.task
def agent_started(vm):
def agent_started(vm, version=None):
from vm.models import Instance, instance_activity, InstanceActivity
instance = Instance.objects.get(id=int(vm.split('-')[-1]))
queue = instance.get_remote_queue_name("agent")
initialized = InstanceActivity.objects.filter(
instance=instance, activity_code='vm.Instance.agent').exists()
instance=instance, activity_code='vm.Instance.agent.cleanup').exists()
with instance_activity(code_suffix='agent', instance=instance) as act:
with act.sub_activity('starting'):
pass
if version and version != settings.AGENT_VERSION:
try:
with act.sub_activity('update'):
update.apply_async(
queue=queue,
args=(vm, create_agent_tar())).get(timeout=10)
return
except TimeoutError:
pass
if not initialized:
send_init_commands(instance, act, vm)
with act.sub_activity('start_access_server'):
queue = instance.get_remote_queue_name("agent")
start_access_server.apply_async(
queue=queue, args=(vm, ))
start_access_server.apply_async(queue=queue, args=(vm, ))
@celery.task
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment