Commit 6d44029c by Őry Máté

Merge branch 'master' into feature-pipeline

parents 43de3dcb 1ca7edbb
......@@ -108,9 +108,9 @@ $(function () {
return false;
$('.title-favourite').tooltip({'placement': 'right'});
$(':input[title]').tooltip({trigger: 'focus', placement: 'auto right'});
$('body [title]:not(.title-favourite)').tooltip();
$('body .title-favourite').tooltip({'placement': 'right'});
$('body :input[title]').tooltip({trigger: 'focus', placement: 'auto right'});
$('[data-toggle="pill"]').click(function() {
......@@ -132,7 +132,7 @@ $(function () {
/* favourite star */
$("#dashboard-vm-list").on('click', '.dashboard-vm-favourite', function(e) {
$("#dashboard-vm-list, .page-header").on('click', '.dashboard-vm-favourite', function(e) {
var star = $(this).children("i");
var pk = $(this).data("vm");
if(star.hasClass("fa-star-o")) {
......@@ -5,6 +5,11 @@
{% block title-site %}Dashboard | CIRCLE{% endblock %}
{% block extra_link %}
{% block extra_link_2 %}{% endblock %}
{% endblock %}
{% block navbar-brand %}
<a class="navbar-brand" href="{% url "dashboard.index" %}" style="padding: 10px 15px;">
{% include "branding.html" %}
......@@ -4,6 +4,13 @@
{% block title-page %}{% trans "Index" %}{% endblock %}
{% block extra_link_2 %}
<link rel="search"
href="{% url "dashboard.views.vm-opensearch" %}"
title="{% blocktrans with name=COMPANY_NAME %}{{name}} virtual machines{% endblocktrans %}" />
{% endblock %}
{% block content %}
<div class="body-content dashboard-index">
<div class="row">
......@@ -50,7 +50,7 @@
<dd>{{ object.task_uuid|default:'n/a' }}</dd>
<dt>{% trans "status" %}</dt>
<dd>{{ object.get_status_id }}</dd>
<dd id="activity_status">{{ object.get_status_id }}</dd>
<dt>{% trans "result" %}</dt>
......@@ -72,6 +72,13 @@
{{ }}
<small>{{ instance.primary_host.get_fqdn }}</small>
<small class="dashboard-vm-favourite" style="line-height: 39.6px;" data-vm="{{ }}">
{% if fav %}
<i class="fa fa-star text-primary title-favourite" title="{% trans "Unfavourite" %}"></i>
{% else %}
<i class="fa fa-star-o text-primary title-favourite" title="{% trans "Mark as favorite" %}"></i>
{% endif %}
<div style="clear: both;"></div>
{% load i18n %}<?xml version="1.0" encoding="UTF-8"?>
<OpenSearchDescription xmlns="">
<ShortName>{% blocktrans with name=COMPANY_NAME %}{{name}} virtual machines{% endblocktrans %}</ShortName>
<Url type="text/html"
template="{{ url }}?s={searchTerms}&amp;stype=shared" />
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
# This file is part of CIRCLE Cloud.
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <>.
from os import listdir
from os.path import isfile, isdir, join
import unittest
from django.conf import settings
from django.template import Template, Context, VariableDoesNotExist
from django.template.loader import find_template_loader
from django.core.urlresolvers import NoReverseMatch
class TemplateSyntaxTestCase(unittest.TestCase):
def test_templates(self):
"""Test all templates for syntax errors."""
for loader_name in settings.TEMPLATE_LOADERS:
print loader_name
loader = find_template_loader(loader_name)
def _test_dir(self, dir, path="/"):
for i in dir:
i = join(path, i)
if isfile(i):
self._test_template(join(path, i))
elif isdir(i):
print "%s:" % i
self._test_dir(listdir(i), i)
def _test_template(self, path):
print path
except (NoReverseMatch, VariableDoesNotExist, KeyError, AttributeError,
ValueError, ) as e:
print e
......@@ -50,6 +50,7 @@ from .views import (
VmGraphView, NodeGraphView, NodeListGraphView,
TransferInstanceOwnershipView, TransferInstanceOwnershipConfirmView,
TransferTemplateOwnershipView, TransferTemplateOwnershipConfirmView,
from .views.vm import vm_ops, vm_mass_ops
from .views.node import node_ops
......@@ -221,6 +222,8 @@ urlpatterns = patterns(
url(r'^token-login/(?P<token>.*)/$', TokenLogin.as_view(),
url(r'^vm/opensearch.xml$', OpenSearchDescriptionView.as_view(),
urlpatterns += patterns(
......@@ -19,6 +19,7 @@ from __future__ import unicode_literals, absolute_import
import logging
from django.core.cache import get_cache
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.auth.models import Group
from django.views.generic import TemplateView
......@@ -121,3 +122,15 @@ class HelpView(TemplateView):
ctx.update({"saml": hasattr(settings, "SAML_CONFIG"),
"store": settings.STORE_URL})
return ctx
class OpenSearchDescriptionView(TemplateView):
template_name = "dashboard/vm-opensearch.xml"
content_type = "application/opensearchdescription+xml"
def get_context_data(self, **kwargs):
context = super(OpenSearchDescriptionView, self).get_context_data(
context['url'] = self.request.build_absolute_uri(
return context
......@@ -115,6 +115,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
'op': {i.op: i for i in ops},
'connect_commands': user.profile.get_connect_commands(instance),
'hide_tutorial': hide_tutorial,
'fav': instance.favourite_set.filter(user=user).exists(),
# activity data
......@@ -112,10 +112,12 @@ def pull(dir="~/circle/circle"):
def update_portal(test=False):
def update_portal(test=False, git=True):
"Update and restart portal+manager"
with _stopped("portal", "manager"):
if git:
pip("circle", "~/circle/requirements.txt")
......@@ -125,6 +127,12 @@ def update_portal(test=False):
def build_portal():
"Update portal without pulling from git"
return update_portal(False, False)
def stop_portal(test=False):
"Stop portal and manager"
_stop_services("portal", "manager")
......@@ -136,10 +144,15 @@ def update_node():
with _stopped("node", "agentdriver", "monitor-client"):
pip("vmdriver", "~/vmdriver/requirements/production.txt")
pip("agentdriver", "~/agentdriver/requirements.txt")
pip("monitor-client", "~/monitor-client/requirements.txt")
......@@ -161,6 +174,18 @@ def checkout(vmdriver="master", agent="master"):
run("git checkout %s" % agent)
def cleanup():
"Clean pyc files of portal"
def _cleanup(dir="~/circle/circle"):
"Clean pyc files"
with cd("~/circle/circle"):
run("find -name '*.py[co]' -exec rm -f {} +")
def _stop_services(*services):
"Stop given services (warn only if not running)"
with settings(warn_only=True):
......@@ -189,3 +214,12 @@ def _stopped(*services):
def _workon(name):
return prefix("source ~/.virtualenvs/%s/bin/activate && "
"source ~/.virtualenvs/%s/bin/postactivate" % (name, name))
def install_bash_completion_script():
"autocompletion/48baf5735bafbb2be5be8787d2c2c04a44b6cdb0/fab "
"-O /etc/bash_completion.d/fab")
print("To have bash completion instantly, run\n"
" source /etc/bash_completion.d/fab")
......@@ -15,6 +15,7 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <>.
from string import ascii_letters
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.translation import ugettext_lazy as _
......@@ -22,7 +23,7 @@ from django.utils.ipv6 import is_valid_ipv6_address
from south.modelsinspector import add_introspection_rules
from django import forms
from netaddr import (IPAddress, IPNetwork, AddrFormatError, ZEROFILL,
EUI, mac_unix)
EUI, mac_unix, AddrConversionError)
import re
......@@ -31,7 +32,6 @@ domain_re = re.compile(r'^([A-Za-z0-9_-]\.?)+$')
domain_wildcard_re = re.compile(r'^(\*\.)?([A-Za-z0-9_-]\.?)+$')
ipv4_re = re.compile('^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)$')
reverse_domain_re = re.compile(r'^(%\([abcd]\)d|[a-z0-9.-])+$')
ipv6_template_re = re.compile(r'^(%\([abcd]\)[dxX]|[A-Za-z0-9:-])+$')
class mac_custom(mac_unix):
......@@ -246,15 +246,60 @@ def val_reverse_domain(value):
raise ValidationError(u'%s - invalid reverse domain name' % value)
def is_valid_ipv6_template(value):
"""Check whether the parameter is a valid ipv6 template."""
return ipv6_template_re.match(value) is not None
def val_ipv6_template(value):
"""Validate whether the parameter is a valid ipv6 template."""
if not is_valid_ipv6_template(value):
raise ValidationError(u'%s - invalid reverse ipv6 template' % value)
"""Validate whether the parameter is a valid ipv6 template.
Normal use:
>>> val_ipv6_template("123::%(a)d:%(b)d:%(c)d:%(d)d")
>>> val_ipv6_template("::%(a)x:%(b)x:%(c)d:%(d)d")
Don't have to use all bytes from the left (no a):
>>> val_ipv6_template("::%(b)x:%(c)d:%(d)d")
But have to use all ones to the right (a, but no b):
>>> val_ipv6_template("::%(a)x:%(c)d:%(d)d")
Traceback (most recent call last):
ValidationError: [u"template doesn't use parameter b"]
Detects valid templates building invalid ips:
>>> val_ipv6_template("xxx::%(a)d:%(b)d:%(c)d:%(d)d")
Traceback (most recent call last):
ValidationError: [u'template renders invalid IPv6 address']
Also IPv4-compatible addresses are invalid:
>>> val_ipv6_template("::%(a)02x%(b)02x:%(c)d:%(d)d")
Traceback (most recent call last):
ValidationError: [u'template results in IPv4 address']
tpl = {ascii_letters[i]: 255 for i in range(4)}
v6 = value % tpl
raise ValidationError(_('%s: invalid template') % value)
used = False
for i in ascii_letters[:4]:
value % {k: tpl[k] for k in tpl if k != i}
except KeyError:
used = True # ok, it misses this key
if used:
raise ValidationError(
_("template doesn't use parameter %s") % i)
v6 = IPAddress(v6, 6)
raise ValidationError(_('template renders invalid IPv6 address'))
except (AddrConversionError, AddrFormatError):
pass # can't converted to ipv4 == it's real ipv6
raise ValidationError(_('template results in IPv4 address'))
def is_valid_ipv4_address(value):
......@@ -284,12 +329,3 @@ def val_mx(value):
raise ValidationError(_("Bad MX address format. "
"Should be: <priority>:<hostname>"))
def convert_ipv4_to_ipv6(ipv6_template, ipv4):
"""Convert IPv4 address string to IPv6 address string."""
m = ipv4.words
return IPAddress(ipv6_template % {'a': int(m[0]),
'b': int(m[1]),
'c': int(m[2]),
'd': int(m[3])})
......@@ -17,9 +17,11 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <>.
from itertools import islice, ifilter
from string import ascii_letters
from itertools import islice, ifilter, chain
from math import ceil
import logging
from netaddr import IPSet, EUI, IPNetwork
import random
from django.contrib.auth.models import User
from django.db import models
......@@ -28,15 +30,17 @@ from django.utils.translation import ugettext_lazy as _
from firewall.fields import (MACAddressField, val_alfanum, val_reverse_domain,
val_ipv6_template, val_domain, val_ipv4,
val_ipv6, val_mx, convert_ipv4_to_ipv6,
val_ipv6, val_mx,
IPNetworkField, IPAddressField)
from django.core.validators import MinValueValidator, MaxValueValidator
import django.conf
from django.db.models.signals import post_save, post_delete
import random
from celery.exceptions import TimeoutError
from netaddr import IPSet, EUI, IPNetwork, IPAddress, ipv6_full
from common.models import method_cache, WorkerNotFound, HumanSortField
from firewall.tasks.local_tasks import reloadtask
from firewall.tasks.remote_tasks import get_dhcp_clients
from .iptables import IptRule
from acl.models import AclBase
......@@ -360,9 +364,19 @@ class Vlan(AclBase, models.Model):
'address is: "%(d)d.%(c)d.%(b)d.%(a)".'),
ipv6_template = models.TextField(
verbose_name=_('ipv6 template'),
help_text=_('Template for translating IPv4 addresses to IPv6. '
'Automatically generated hosts in dual-stack networks '
'will get this address. The template '
'can contain four tokens: "%(a)d", "%(b)d", '
'"%(c)d", and "%(d)d", representing the four bytes '
'of the IPv4 address, respectively, in decimal notation. '
'Moreover you can use any standard printf format '
'specification like %(a)02x to get the first byte as two '
'hexadecimal digits. Usual choices for mapping '
' to 2001:0DB8:1:1::/64 would be '
'"2001:db8:1:1:%(d)d::" and "2001:db8:1:1:%(d)02x00::".'),
validators=[val_ipv6_template], verbose_name=_('ipv6 template'))
dhcp_pool = models.TextField(blank=True, verbose_name=_('DHCP pool'),
'The address range of the DHCP pool: '
......@@ -377,6 +391,87 @@ class Vlan(AclBase, models.Model):
modified_at = models.DateTimeField(auto_now=True,
verbose_name=_('modified at'))
def clean(self):
super(Vlan, self).clean()
if self.ipv6_template:
if not self.network6:
raise ValidationError(
_("You cannot specify an IPv6 template if there is no "
"IPv6 network set."))
for i in (self.network4[1], self.network4[-1]):
i6 = self.convert_ipv4_to_ipv6(i)
if i6 not in self.network6:
raise ValidationError(
_("%(ip6)s (translated from %(ip4)s) is outside of "
"the IPv6 network.") % {"ip4": i, "ip6": i6})
if self.network6:
tpl, prefixlen = self._magic_ipv6_template(self.network4,
if not self.ipv6_template:
self.ipv6_template = tpl
if not self.host_ipv6_prefixlen:
self.host_ipv6_prefixlen = prefixlen
def _host_bytes(prefixlen, maxbytes):
return int(ceil((maxbytes - prefixlen / 8.0)))
def _append_hexa(s, v, lasthalf):
if lasthalf: # can use last half word
assert s[-1] == "0" or s[-1].endswith("00")
if s[-1].endswith("00"):
s[-1] = s[-1][:-2]
s[-1] += "%({})02x".format(v)
def _magic_ipv6_template(cls, network4, network6, verbose=None):
"""Offer a sensible ipv6_template value.
Based on prefix lengths the method magically selects verbose (decimal)
>>> Vlan._magic_ipv6_template(IPNetwork(""),
... IPNetwork("2001:0DB8:1:1::/64"))
('2001:db8:1:1:%(d)d::', 80)
However you can explicitly select non-verbose, i.e. hexa format:
>>> Vlan._magic_ipv6_template(IPNetwork(""),
... IPNetwork("2001:0DB8:1:1::/64"), False)
('2001:db8:1:1:%(d)02x00::', 72)
host4_bytes = cls._host_bytes(network4.prefixlen, 4)
host6_bytes = cls._host_bytes(network6.prefixlen, 16)
if host4_bytes > host6_bytes:
raise ValidationError(
_("IPv6 network is too small to map IPv4 addresses to it."))
letters = ascii_letters[4-host4_bytes:4]
remove = host6_bytes // 2
ipstr =
s = ipstr.split(":")[0:-remove]
if verbose is None: # use verbose format if net6 much wider
verbose = 2 * (host4_bytes + 1) < host6_bytes
if verbose:
for i in letters:
remain = host6_bytes
for i in letters:
cls._append_hexa(s, i, remain % 2 == 1)
remain -= 1
if host6_bytes > host4_bytes:
tpl = ":".join(s)
# compute prefix length
mask = int(IPAddress(tpl % {"a": 1, "b": 1, "c": 1, "d": 1}))
prefixlen = 128
while mask % 2 == 0:
mask /= 2
prefixlen -= 1
return (tpl, prefixlen)
def __unicode__(self):
return "%s - %s" % ("managed" if self.managed else "unmanaged",
......@@ -402,7 +497,7 @@ class Vlan(AclBase, models.Model):
logger.debug("Found unused IPv4 address %s.", ipv4)
ipv6 = None
if self.network6 is not None:
ipv6 = convert_ipv4_to_ipv6(self.ipv6_template, ipv4)
ipv6 = self.convert_ipv4_to_ipv6(ipv4)
if ipv6 in used_v6:
......@@ -411,6 +506,20 @@ class Vlan(AclBase, models.Model):
raise ValidationError(_("All IP addresses are already in use."))
def convert_ipv4_to_ipv6(self, ipv4):
"""Convert IPv4 address string to IPv6 address string."""
if isinstance(ipv4, basestring):
ipv4 = IPAddress(ipv4, 4)
nums = {ascii_letters[i]: int(ipv4.words[i]) for i in range(4)}
return IPAddress(self.ipv6_template % nums)
def get_dhcp_clients(self):
macs = set(i.mac for i in self.host_set.all())
return [{"mac": k, "ip": v["ip"], "hostname": v["hostname"]}
for k, v in chain(*(fw.get_dhcp_clients().iteritems()
for fw in Firewall.objects.all() if fw))
if v["interface"] == and EUI(k) not in macs]
class VlanGroup(models.Model):
......@@ -581,8 +690,7 @@ class Host(models.Model):
def save(self, *args, **kwargs):
if not and self.ipv6 == "auto":
self.ipv6 = convert_ipv4_to_ipv6(self.vlan.ipv6_template,
self.ipv6 = self.vlan.convert_ipv4_to_ipv6(self.ipv4)
super(Host, self).save(*args, **kwargs)
......@@ -838,7 +946,7 @@ class Firewall(models.Model):
def get_remote_queue_name(self, queue_id):
def get_remote_queue_name(self, queue_id="firewall"):
"""Returns the name of the remote celery queue for this node.
Throws Exception if there is no worker on the queue.
......@@ -851,6 +959,20 @@ class Firewall(models.Model):
raise WorkerNotFound()
def get_dhcp_clients(self):
return get_dhcp_clients.apply_async(
queue=self.get_remote_queue_name(), expires=60).get(timeout=2)
except TimeoutError:"get_dhcp_clients task timed out")
except IOError:
logger.exception("get_dhcp_clients failed. "
"maybe syslog isn't readble by firewall worker")
logger.exception("get_dhcp_clients failed")
return {}
class Domain(models.Model):
name = models.CharField(max_length=40, validators=[val_domain],
......@@ -62,5 +62,6 @@ def reload_blacklist(data):
def get_dhcp_clients(data):
def get_dhcp_clients():
# {'00:21:5a:73:72:cd': {'interface': 'OFF', 'ip': None, 'hostname': None}}
......@@ -78,6 +78,7 @@ class GetNewAddressTestCase(TestCase):
self.vlan = Vlan(vid=1, name='test', network4='',
network6='2001:738:2001:4031::/80', domain=d,
for i in [1] + range(3, 6):
......@@ -85,6 +86,9 @@ class GetNewAddressTestCase(TestCase):
ipv4='10.0.0.%d' % i, vlan=self.vlan,
def tearDown(self):
def test_new_addr_w_empty_vlan(self):
......@@ -96,12 +100,6 @@ class GetNewAddressTestCase(TestCase):
self.assertRaises(ValidationError, self.vlan.get_new_address)
def test_all_addr_in_use_w_ipv6(self):
Host(hostname='h-x', mac='01:02:03:04:05:06',
ipv4='', ipv6='2001:738:2001:4031:0:0:2:0',
vlan=self.vlan, owner=self.u1).save()
self.assertRaises(ValidationError, self.vlan.get_new_address)
def test_new_addr(self):
used_v4 = IPSet(self.vlan.host_set.values_list('ipv4', flat=True))
assert self.vlan.get_new_address()['ipv4'] not in used_v4
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -15,13 +15,13 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <>.
from django.forms import ModelForm
from django.forms import ModelForm, widgets
from django.core.urlresolvers import reverse_lazy
from django.utils.translation import ugettext_lazy as _
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Fieldset, Div, Submit, BaseInput
from crispy_forms.bootstrap import FormActions
from crispy_forms.bootstrap import FormActions, FieldWithButtons, StrictButton
from firewall.models import (Host, Vlan, Domain, Group, Record, BlacklistItem,
Rule, VlanGroup, SwitchPort)
......@@ -122,8 +122,12 @@ class HostForm(ModelForm):
FieldWithButtons('ipv4', StrictButton(
'<i class="fa fa-magic"></i>', css_id="ipv4-magic",
title=_("Generate random address."))),
FieldWithButtons('ipv6', StrictButton(
'<i class="fa fa-magic"></i>', css_id="ipv6-magic",
title=_("Generate IPv6 pair of IPv4 address."))),
......@@ -252,7 +256,9 @@ class VlanForm(ModelForm):
FieldWithButtons('ipv6_template', StrictButton(
'<i class="fa fa-magic"></i>', css_id="ipv6-tpl-magic",
title=_("Generate sensible template."))),
......@@ -277,6 +283,9 @@ class VlanForm(ModelForm):
class Meta:
model = Vlan
widgets = {
'ipv6_template': widgets.TextInput,
class VlanGroupForm(ModelForm):
......@@ -28,6 +28,49 @@ function getURLParameter(name) {
function doBlink(id, count) {
if (count > 0) {
$(id).parent().delay(200).queue(function() {
$(this).delay(200).queue(function() {
doBlink(id, count-1);});
$(function() {
$("#ipv6-magic").click(function() {
$.ajax({url: window.location,
data: {ipv4: $("[name=ipv4]").val(),
vlan: $("[name=vlan]").val()},
success: function(data) {
$("#ipv4-magic").click(function() {
$.ajax({url: window.location,
data: {vlan: $("[name=vlan]").val()},
success: function(data) {
if ($("[name=ipv6]").val() != data.ipv6) {
doBlink("[name=ipv6]", 3);
$("#ipv6-tpl-magic").click(function() {
$.ajax({url: window.location,
data: {network4: $("[name=network4]").val(),
network6: $("[name=network6]").val()},
success: function(data) {
if ($("[name=host_ipv6_prefixlen]").val() != data.host_ipv6_prefixlen) {
doBlink("[name=host_ipv6_prefixlen]", 3);
......@@ -15,14 +15,30 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <>.
from netaddr import EUI
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html
from django_tables2 import Table, A
from django_tables2.columns import LinkColumn, TemplateColumn
from django_tables2.columns import LinkColumn, TemplateColumn, Column
from firewall.models import Host, Vlan, Domain, Group, Record, Rule, SwitchPort
class MACColumn(Column):
def render(self, value):
if isinstance(value, basestring):
value = EUI(value)