Merge remote-tracking branch 'origin/master' into feature-setty

circle/acl/models.py

@@ -246,8 +246,8 @@ class AclBase(Model):
 
     def save(self, *args, **kwargs):
         super(AclBase, self).save(*args, **kwargs)
-        if 'owner' in dict(self.ACL_LEVELS) and (hasattr(self, 'owner')
-                                                 and self.owner):
+        if 'owner' in dict(self.ACL_LEVELS) and (hasattr(self, 'owner') and
+                                                 self.owner):
             self.set_user_level(self.owner, 'owner')
 
     class Meta:

circle/circle/settings/base.py

@@ -508,6 +508,8 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
     if get_env_variable('DJANGO_SAML_ORG_ID_ATTRIBUTE', False) is not False:
         SAML_ORG_ID_ATTRIBUTE = get_env_variable(
             'DJANGO_SAML_ORG_ID_ATTRIBUTE')
+    SAML_MAIN_ATTRIBUTE_MAX_LENGTH = int(get_env_variable(
+        "DJANGO_SAML_MAIN_ATTRIBUTE_MAX_LENGTH", 0))
 
 LOGIN_REDIRECT_URL = "/"
 

circle/circle/settings/test.py

@@ -71,3 +71,5 @@ STORE_URL = ""
 
 # buildbot doesn't love pipeline
 STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
+
+SAML_MAIN_ATTRIBUTE_MAX_LENGTH=0  # doctest on SAML2 backend runs either way

circle/circle/wsgi.py

@@ -46,7 +46,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "circle.settings.production")
 # This application object is used by any WSGI server configured to use this
 # file. This includes Django's development server, if the WSGI_APPLICATION
 # setting points here.
-from django.core.wsgi import get_wsgi_application
+from django.core.wsgi import get_wsgi_application  # noqa
 
 _application = get_wsgi_application()
 

circle/common/backends.py

@@ -17,9 +17,14 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
+import logging
+import sha
 
+from django.conf import settings
 from djangosaml2.backends import Saml2Backend as Saml2BackendBase
 
+logger = logging.getLogger(__name__)
+
 
 class Saml2Backend(Saml2BackendBase):
     u"""
@@ -41,7 +46,14 @@ class Saml2Backend(Saml2BackendBase):
         if isinstance(main_attribute, str):
             main_attribute = main_attribute.decode('UTF-8')
         assert isinstance(main_attribute, unicode)
-        return re.sub(r'[^\w.@-]', replace, main_attribute)
+        attr = re.sub(r'[^\w.@-]', replace, main_attribute)
+        max_length = settings.SAML_MAIN_ATTRIBUTE_MAX_LENGTH
+        if max_length > 0 and len(attr) > max_length:
+            logger.info("Main attribute '%s' is too long." % attr)
+            hashed = sha.new(attr).hexdigest()
+            attr = hashed[:max_length]
+            logger.info("New main attribute: %s" % attr)
+        return attr
 
     def _set_attribute(self, obj, attr, value):
         if attr == 'username':

circle/common/models.py

@@ -97,7 +97,7 @@ def has_prefix(activity_code, *prefixes):
     >>> assert has_prefix('foo.bar.buz', 'foo', 'bar', 'buz')
     >>> assert not has_prefix('foo.bar.buz', 'foo', 'buz')
     """
-    equal = lambda a, b: a == b
+    def equal(a, b): return a == b
     act_code_parts = split_activity_code(activity_code)
     prefixes = chain(*imap(split_activity_code, prefixes))
     return all(imap(equal, act_code_parts, prefixes))
@@ -112,7 +112,7 @@ def has_suffix(activity_code, *suffixes):
     >>> assert has_suffix('foo.bar.buz', 'foo', 'bar', 'buz')
     >>> assert not has_suffix('foo.bar.buz', 'foo', 'buz')
     """
-    equal = lambda a, b: a == b
+    def equal(a, b): return a == b
     act_code_parts = split_activity_code(activity_code)
     suffixes = list(chain(*imap(split_activity_code, suffixes)))
     return all(imap(equal, reversed(act_code_parts), reversed(suffixes)))
@@ -441,8 +441,8 @@ class HumanReadableObject(object):
     @classmethod
     def create(cls, user_text_template, admin_text_template=None, **params):
         return cls(user_text_template=user_text_template,
-                   admin_text_template=(admin_text_template
-                                        or user_text_template), params=params)
+                   admin_text_template=(admin_text_template or
+                                        user_text_template), params=params)
 
     def set(self, user_text_template, admin_text_template=None, **params):
         self._set_values(user_text_template,

circle/dashboard/arrow_local.py

-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Budapest University of Technology and Economics (BME IK)
-#
-# This file is part of CIRCLE Cloud.
-#
-# CIRCLE is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along
-# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
-
-from arrow import locales
-
-
-class HungarianLocale(locales.Locale):
-
-    names = ['hu', 'HU']
-
-    past = '{0} ezelőtt'
-    future = '{0} múlva'
-
-    timeframes = {
-        'now': 'éppen most',
-        'seconds': {
-            'past': 'másodpercekkel',
-            'future': 'pár másodperc'},
-        'minute': {'past': 'egy perccel', 'future': 'egy perc'},
-        'minutes': {'past': '{0} perccel', 'future': '{0} perc'},
-        'hour': {'past': 'egy órával', 'future': 'egy óra'},
-        'hours': {'past': '{0} órával', 'future': '{0} óra'},
-        'day': {
-            'past': 'egy nappal',
-            'future': 'egy nap'
-        },
-        'days': {
-            'past': '{0} nappal',
-            'future': '{0} nap'
-        },
-        'month': {'past': 'egy hónappal', 'future': 'egy hónap'},
-        'months': {'past': '{0} hónappal', 'future': '{0} hónap'},
-        'year': {'past': 'egy évvel', 'future': 'egy év'},
-        'years': {'past': '{0} évvel', 'future': '{0} év'},
-    }
-
-    month_names = ['', 'Január', 'Február', 'Március', 'Április', 'Május',
-                   'Június', 'Július', 'Augusztus', 'Szeptember',
-                   'Október', 'November', 'December']
-    month_abbreviations = ['', 'Jan', 'Febr', 'Márc', 'Ápr', 'Máj', 'Jún',
-                           'Júl', 'Aug', 'Szept', 'Okt', 'Nov', 'Dec']
-
-    day_names = ['', 'Hétfő', 'Kedd', 'Szerda', 'Csütörtök', 'Péntek',
-                 'Szombat', 'Vasárnap']
-    day_abbreviations = ['', 'Hét', 'Kedd', 'Szer', 'Csüt', 'Pént',
-                         'Szom', 'Vas']
-
-    meridians = {
-        'am': 'de',
-        'pm': 'du',
-        'AM': 'DE',
-        'PM': 'DU',
-    }
-
-    def _format_timeframe(self, timeframe, delta):
-        form = self.timeframes[timeframe]
-        if isinstance(form, dict):
-            if delta > 0:
-                form = form['future']
-            else:
-                form = form['past']
-        delta = abs(delta)
-
-        return form.format(delta)

circle/dashboard/autocomplete_light_registry.py

@@ -38,10 +38,10 @@ def highlight(field, q, none_wo_match=True):
         match = None
     if q and match is not None:
         match_end = match + len(q)
-        return (escape(field[:match])
-                + '<span class="autocomplete-hl">'
-                + escape(field[match:match_end])
-                + '</span>' + escape(field[match_end:]))
+        return (escape(field[:match]) +
+                '<span class="autocomplete-hl">' +
+                escape(field[match:match_end]) +
+                '</span>' + escape(field[match_end:]))
     elif none_wo_match:
         return None
     else:

circle/dashboard/forms.py

@@ -506,8 +506,8 @@ class TemplateForm(forms.ModelForm):
             self.allowed_fields = (
                 'name', 'access_method', 'description', 'system', 'tags',
                 'arch', 'lease', 'has_agent')
-        if (self.user.has_perm('vm.change_template_resources')
-                or not self.instance.pk):
+        if (self.user.has_perm('vm.change_template_resources') or
+                not self.instance.pk):
             self.allowed_fields += tuple(set(self.fields.keys()) -
                                          set(['raw_data']))
         if self.user.is_superuser:
@@ -523,8 +523,8 @@ class TemplateForm(forms.ModelForm):
             self.initial['max_ram_size'] = 512
 
         lease_queryset = (
-            Lease.get_objects_with_level("operator", self.user).distinct()
-            | Lease.objects.filter(pk=self.instance.lease_id).distinct())
+            Lease.get_objects_with_level("operator", self.user).distinct() |
+            Lease.objects.filter(pk=self.instance.lease_id).distinct())
 
         self.fields["lease"].queryset = lease_queryset
 

circle/dashboard/management/commands/init.py

@@ -64,8 +64,8 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         self.changed = False
 
-        if (DataStore.objects.exists() and Vlan.objects.exists()
-                and not options['force']):
+        if (DataStore.objects.exists() and Vlan.objects.exists() and
+                not options['force']):
             return self.print_state()
 
         admin = self.create(User, 'username', username=options['admin_user'],

circle/dashboard/models.py

@@ -438,10 +438,14 @@ def add_ssh_keys(sender, **kwargs):
 
     userkey = kwargs.get('instance')
     instances = Instance.get_objects_with_level(
-        'user', userkey.user).filter(status='RUNNING')
+        'user', userkey.user, disregard_superuser=True
+    ).filter(status='RUNNING')
     for i in instances:
         logger.info('called add_keys(%s, %s)', i, userkey)
-        i.install_keys(user=userkey.user, keys=[userkey.key])
+        try:
+            i.install_keys(user=userkey.user, keys=[userkey.key])
+        except Instance.NoAgentError:
+            logger.info("%s has no agent running", i)
 
 
 def del_ssh_keys(sender, **kwargs):
@@ -449,10 +453,14 @@ def del_ssh_keys(sender, **kwargs):
 
     userkey = kwargs.get('instance')
     instances = Instance.get_objects_with_level(
-        'user', userkey.user).filter(status='RUNNING')
+        'user', userkey.user, disregard_superuser=True
+    ).filter(status='RUNNING')
     for i in instances:
         logger.info('called del_keys(%s, %s)', i, userkey)
-        i.remove_keys(user=userkey.user, keys=[userkey.key])
+        try:
+            i.remove_keys(user=userkey.user, keys=[userkey.key])
+        except Instance.NoAgentError:
+            logger.info("%s has no agent running", i)
 
 
 post_save.connect(add_ssh_keys, sender=UserKey)

circle/dashboard/templates/dashboard/_vm-renew.html

@@ -8,7 +8,7 @@
   <a class="btn btn-default" href="{{object.get_absolute_url}}" data-dismiss="modal">
     {% trans "Cancel" %}
   </a>
-  {% if lease_types %}
+  {% if lease_types and not request.token_user %}
   <a class="btn btn-primary" id="vm-renew-request-lease-button"
    href="{% url "request.views.request-lease" vm_pk=object.pk %}">
     <i class="fa fa-forward"></i>

circle/dashboard/templates/dashboard/profile.html

@@ -42,7 +42,7 @@
                 {% trans "Email address" %}: {{ profile.email }}
               {% endif %}
             </p>
-            <p>{% trans "Last login" %}: <span title="{{ request.user.last_login  }}">{{ request.user.last_login|arrowfilter:LANGUAGE_CODE}}</span></p>
+            <p>{% trans "Last login" %}: <span title="{{ profile.last_login  }}">{{ profile.last_login|arrowfilter:LANGUAGE_CODE}}</span></p>
             {% if request.user == profile %}
             <p>
               {% trans "Use email address as Gravatar profile image" %}:

circle/dashboard/templates/dashboard/template-edit.html

@@ -166,6 +166,28 @@
         </ul>
       </div>
     </div>
+
+    {% if show_graph %}
+      <div class="panel panel-default">
+        <div class="panel-heading">
+          <h3 class="no-margin"><i class="fa fa-area-chart"></i> {% trans "Graphs" %}</h3>
+        </div>
+        <div class="text-center panel-body">
+          <div class="graph-buttons">
+            {% include "dashboard/_graph-time-buttons.html" %}
+          </div>
+          <div class="text-center graph-images">
+            <img src="{% url "dashboard.views.template-graph" object.pk "instances" graph_time %}"/>
+          </div>
+
+          {% if request.user.is_superuser %}
+            <a  href="{% url "dashboard.views.vm-list" %}?s=template:{{object.pk}}&stype=all">
+              {% trans "List all template instances" %}
+            </a>
+          {% endif %}
+        </div>
+      </div>
+    {% endif %}
   </div><!-- .col-md-4 -->
 </div><!-- .row -->
 

circle/dashboard/templatetags/arrowfilter.py

@@ -18,9 +18,6 @@
 from django.template import Library
 
 import arrow
-from dashboard.arrow_local import HungarianLocale
-for name in HungarianLocale.names:
-    arrow.locales._locales[name] = HungarianLocale
 
 
 register = Library()

circle/dashboard/tests/selenium/util.py

@@ -601,8 +601,8 @@ class CircleSeleniumMixin(SeleniumMixin):
             choices = self.driver.find_elements_by_css_selector(
                 "input[type='radio']")
             choice_list = [item for item in choices if (
-                'test' not in item.get_attribute('value')
-                and item.get_attribute('value') != 'base_vm')]
+                'test' not in item.get_attribute('value') and
+                item.get_attribute('value') != 'base_vm')]
             chosen = random.randint(0, len(choice_list) - 1)
             choice_list[chosen].click()
             self.driver.find_element_by_id(

circle/dashboard/urls.py

@@ -47,7 +47,7 @@ from .views import (
     LeaseAclUpdateView,
     toggle_template_tutorial,
     ClientCheck, TokenLogin,
-    VmGraphView, NodeGraphView, NodeListGraphView,
+    VmGraphView, NodeGraphView, NodeListGraphView, TemplateGraphView,
     TransferInstanceOwnershipView, TransferInstanceOwnershipConfirmView,
     TransferTemplateOwnershipView, TransferTemplateOwnershipConfirmView,
     OpenSearchDescriptionView,
@@ -152,6 +152,10 @@ urlpatterns = patterns(
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
         NodeListGraphView.as_view(),
         name='dashboard.views.node-list-graph'),
+    url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
+         r'(?P<time>[0-9]{1,2}[hdwy])$'),
+        TemplateGraphView.as_view(),
+        name='dashboard.views.template-graph'),
     url(r'^group/(?P<pk>\d+)/$', GroupDetailView.as_view(),
         name='dashboard.views.group-detail'),
     url(r'^group/(?P<pk>\d+)/update/$', GroupProfileUpdate.as_view(),

circle/dashboard/views/graph.py

@@ -28,7 +28,7 @@ from django.views.generic import View
 
 from braces.views import LoginRequiredMixin
 
-from vm.models import Instance, Node
+from vm.models import Instance, Node, InstanceTemplate
 
 
 logger = logging.getLogger(__name__)
@@ -152,6 +152,28 @@ class NodeGraphView(GraphViewBase):
         return self.model.objects.get(id=pk)
 
 
+class TemplateGraphView(GraphViewBase):
+    model = InstanceTemplate
+    base = Metric
+
+    def get_object(self, request, pk):
+        instance = super(TemplateGraphView, self).get_object(request, pk)
+        if not instance.has_level(request.user, 'operator'):
+            raise PermissionDenied()
+        return instance
+
+
+class TemplateVms(object):
+    metric_name = "instances.running"
+    title = _("Instance count")
+    label = _("instance count")
+
+    def get_minmax(self):
+        return (0, None)
+
+register_graph(TemplateVms, 'instances', TemplateGraphView)
+
+
 class NodeListGraphView(GraphViewBase):
     model = Node
     base = Metric

circle/dashboard/views/template.py

@@ -47,7 +47,8 @@ from ..tables import TemplateListTable, LeaseListTable
 from .util import (
     AclUpdateView, FilterMixin,
     TransferOwnershipConfirmView, TransferOwnershipView,
-    DeleteViewBase
+    DeleteViewBase,
+    GraphMixin
 )
 
 logger = logging.getLogger(__name__)
@@ -258,7 +259,8 @@ class TemplateDelete(DeleteViewBase):
         object.delete()
 
 
-class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView):
+class TemplateDetail(LoginRequiredMixin, GraphMixin,
+                     SuccessMessageMixin, UpdateView):
     model = InstanceTemplate
     template_name = "dashboard/template-edit.html"
     form_class = TemplateForm
@@ -300,6 +302,7 @@ class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView):
         context['is_owner'] = obj.has_level(self.request.user, 'owner')
         context['aclform'] = AclUserOrGroupAddForm()
         context['parent'] = obj.parent
+        context['show_graph'] = obj.has_level(self.request.user, 'operator')
         return context
 
     def get_success_url(self):

circle/dashboard/views/user.py

@@ -545,8 +545,8 @@ class UserList(LoginRequiredMixin, PermissionRequiredMixin, SingleTableView):
 
         q = self.search_form.cleaned_data.get('s')
         if q:
-            filters = (Q(username__icontains=q) | Q(email__icontains=q)
-                       | Q(profile__org_id__icontains=q))
+            filters = (Q(username__icontains=q) | Q(email__icontains=q) |
+                       Q(profile__org_id__icontains=q))
             for w in q.split()[:3]:
                 filters |= (
                     Q(first_name__icontains=w) | Q(last_name__icontains=w))

circle/dashboard/views/vm.py

@@ -150,8 +150,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
         # resources forms
         can_edit = (
-            instance.has_level(user, "owner")
-            and self.request.user.has_perm("vm.change_resources"))
+            instance.has_level(user, "owner") and
+            self.request.user.has_perm("vm.change_resources"))
         context['resources_form'] = VmResourcesForm(
             can_edit=can_edit, instance=instance)
 
@@ -174,8 +174,10 @@ class VmDetailView(GraphMixin, CheckedDetailView):
         context['is_owner'] = is_owner
 
         # operation also allows RUNNING (if with_shutdown is present)
-        context['save_resources_enabled'] = instance.status not in ("RUNNING",
-                                                                    "PENDING")
+        context['save_resources_enabled'] = instance.status in (
+            "STOPPED",
+            "PENDING",
+        )
 
         return context
 
@@ -567,8 +569,8 @@ class VmResourcesChangeView(VmOperationView):
                     content_type="application=json"
                 )
             else:
-                return HttpResponseRedirect(instance.get_absolute_url()
-                                            + "#resources")
+                return HttpResponseRedirect(instance.get_absolute_url() +
+                                            "#resources")
         else:
             extra = form.cleaned_data
             extra['max_ram_size'] = extra['ram_size']
@@ -1259,8 +1261,9 @@ def vm_activity(request, pk):
     response['status'] = instance.status
     response['icon'] = instance.get_status_icon()
     latest = instance.get_latest_activity_in_progress()
-    response['is_new_state'] = (latest and latest.resultant_state is not None
-                                and instance.status != latest.resultant_state)
+    response['is_new_state'] = (latest and
+                                latest.resultant_state is not None and
+                                instance.status != latest.resultant_state)
 
     context = {
         'instance': instance,

circle/fabfile.py

@@ -58,9 +58,9 @@ def bower(component=None):
     "Install bower component"
     with cd("~/circle/circle"):
         if component:
-            run("bower install %s" % component)
+            run("bower install %s --config.interactive=false" % component)
         else:
-            run("bower install")
+            run("bower install --config.interactive=false")
 
 
 @roles('portal')

circle/firewall/fields.py

@@ -188,11 +188,11 @@ class IPNetworkField(models.Field):
 
         if isinstance(value, IPNetwork):
             if self.version == 4:
-                return ('.'.join("%03d" % x for x in value.ip.words)
-                        + '/%02d' % value.prefixlen)
+                return ('.'.join("%03d" % x for x in value.ip.words) +
+                        '/%02d' % value.prefixlen)
             else:
-                return (':'.join("%04X" % x for x in value.ip.words)
-                        + '/%03d' % value.prefixlen)
+                return (':'.join("%04X" % x for x in value.ip.words) +
+                        '/%03d' % value.prefixlen)
         return value
 
     def formfield(self, **kwargs):

circle/firewall/management/commands/reload_firewall.py

@@ -21,7 +21,34 @@ from django.core.management.base import BaseCommand
 
 from firewall.tasks.local_tasks import reloadtask
 
+from argparse import ArgumentTypeError
+
 
 class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+
+        parser.add_argument('--sync',
+                            action='store_const',
+                            dest='sync',
+                            const=True,
+                            default=False,
+                            help='synchronous reload')
+
+        parser.add_argument('--timeout',
+                            action='store',
+                            dest='timeout',
+                            default=15,
+                            type=self.positive_int,
+                            help='timeout for synchronous reload')
+
     def handle(self, *args, **options):
-        reloadtask('Vlan')
+
+        reloadtask('Vlan', sync=options["sync"], timeout=options["timeout"])
+
+    def positive_int(self, val):
+
+        if not val.isdigit():
+            raise ArgumentTypeError("'%s' is not a valid positive int" % val)
+
+        return int(val)

circle/firewall/models.py

@@ -157,6 +157,10 @@ class Rule(models.Model):
         selected_fields = [field for field in fields if field]
         if len(selected_fields) > 1:
             raise ValidationError(_('Only one field can be selected.'))
+        elif len(selected_fields) < 1:
+            raise ValidationError(
+                _('One of the following fields must be selected: '
+                  'vlan, vlan group, host, host group, firewall.'))
 
     def get_external_ipv4(self):
         return (self.nat_external_ipv4
@@ -696,8 +700,8 @@ class Host(models.Model):
         return self.vlan.network_type != 'public'
 
     def clean(self):
-        if (self.external_ipv4 and not self.shared_ip and self.behind_nat
-                and Host.objects.exclude(id=self.id).filter(
+        if (self.external_ipv4 and not self.shared_ip and self.behind_nat and
+                Host.objects.exclude(id=self.id).filter(
                     external_ipv4=self.external_ipv4)):
             raise ValidationError(_("If shared_ip has been checked, "
                                     "external_ipv4 has to be unique."))

circle/firewall/tasks/local_tasks.py

@@ -60,7 +60,7 @@ def get_firewall_queues():
     return list(retval)
 
 
-@celery.task(ignore_result=True)
+@celery.task
 def reloadtask_worker():
     from firewall.fw import BuildFirewall, dhcp, dns, ipset, vlan
     from remote_tasks import (reload_dns, reload_dhcp, reload_firewall,
@@ -92,7 +92,7 @@ def reloadtask_worker():
 
 
 @celery.task
-def reloadtask(type='Host', timeout=15):
+def reloadtask(type='Host', timeout=15, sync=False):
     reload = {
         'Host': ['dns', 'dhcp', 'firewall'],
         'Record': ['dns'],
@@ -107,4 +107,6 @@ def reloadtask(type='Host', timeout=15):
     logger.info("Reload %s on next periodic iteration applying change to %s.",
                 ", ".join(reload), type)
     if all([cache.add("%s_lock" % i, 'true', 30) for i in reload]):
-        reloadtask_worker.apply_async(queue='localhost.man', countdown=5)
+        res = reloadtask_worker.apply_async(queue='localhost.man', countdown=5)
+        if sync:
+            res.get(timeout)

circle/manager/scheduler.py

@@ -55,8 +55,8 @@ def select_node(instance, nodes):
     '''
     # check required traits
     nodes = [n for n in nodes
-             if n.schedule_enabled and n.online
-             and has_traits(instance.req_traits.all(), n)]
+             if n.schedule_enabled and n.online and
+             has_traits(instance.req_traits.all(), n)]
     if not nodes:
         logger.warning('select_node: no usable node for %s', unicode(instance))
         raise TraitsUnsatisfiableException()

circle/monitor/tasks/local_periodic_tasks.py

@@ -54,7 +54,7 @@ def measure_response_time():
 
 @celery.task(ignore_result=True)
 def check_celery_queues():
-    graphite_string = lambda component, hostname, celery, is_alive, time: (
+    def graphite_string(component, hostname, celery, is_alive, time): return (
         "%s.%s.celery-queues.%s %d %s" % (
             component, hostname, celery, 1 if is_alive else 0, time)
     )
@@ -92,7 +92,7 @@ def check_celery_queues():
 
 @celery.task(ignore_result=True)
 def instance_per_template():
-    graphite_string = lambda pk, state, val, time: (
+    def graphite_string(pk, state, val, time): return (
         "template.%d.instances.%s %d %s" % (
             pk, state, val, time)
     )
@@ -111,7 +111,7 @@ def instance_per_template():
 
 @celery.task(ignore_result=True)
 def allocated_memory():
-    graphite_string = lambda hostname, val, time: (
+    def graphite_string(hostname, val, time): return (
         "circle.%s.memory.allocated %d %s" % (
             hostname, val, time)
     )

circle/network/views.py

@@ -979,8 +979,8 @@ def remove_switch_port_device(request, **kwargs):
 def add_switch_port_device(request, **kwargs):
     device_name = request.POST.get('device_name')
 
-    if (request.method == "POST" and device_name and len(device_name) > 0
-       and EthernetDevice.objects.filter(name=device_name).count() == 0):
+    if (request.method == "POST" and device_name and len(device_name) > 0 and
+            EthernetDevice.objects.filter(name=device_name).count() == 0):
 
         switch_port = SwitchPort.objects.get(pk=kwargs['pk'])
         new_device = EthernetDevice(name=device_name, switch_port=switch_port)

circle/templates/info/help/faq.html

@@ -29,7 +29,7 @@
 </blockquote>
 
 
-<h3 id="how-can-i-portforward">{% trans "How can I open ports?" %}</h3>
+<h3 id="how-can-i-open-ports">{% trans "How can I open ports?" %}</h3>
 
 <blockquote>
   <ol>

circle/templates/info/help/faq_toc.html

@@ -4,7 +4,7 @@
 
   <li><a href="#how-can-i-create-a-vm-and-give-to-another-user">{% trans "How can I create a VM and give to another user?" %}</a></li>
 
-  <li><a href="#how-can-i-portforward">{% trans "How can I portforward?" %}</a></li>
+  <li><a href="#how-can-i-open-ports">{% trans "How can I open ports?" %}</a></li>
   <li><a href="#my-machines-lease-is-short-how-can-i-extend-it">{% trans "My machine’s lease is short. How can I extend it?" %}</a></li>
 
   <li><a href="#how-can-i-have-more-cpumemory">{% trans "How can I have more CPU/memory?" %}</a></li>

circle/templates/info/help/overview.html

@@ -348,32 +348,34 @@
   <blockquote>
     <p>
       <h4>{% trans "Architecture" %}</h4>
-      The user can choose the template's architecture (x86 or x86-64).
+      {% trans "The user can choose the template's architecture (x86 or x86-64)." %}
     </p>
     <p>
       <h4>{% trans "Access method" %}</h4>
-      The default access method is modifiable. Currently SSH, RDP and NX are supported.
+      {% trans "The default access method is modifiable. Currently SSH, RDP and NX are supported." %}
     </p>
     <p>
       <h4>{% trans "Boot menu" %} </h4>
-      Check it to turn on the boot menu.
+      {% trans "Check it to turn on the boot menu." %}
     </p>
     <p>
       <h4>{% trans "Traits" %}</h4>
-      By adding or removing traits we can guarantee specific features the host node will have (like <em>GPU</em>) for the virtual machine.
+      {% trans "By adding or removing traits we can guarantee specific features the host node will have (like <em>GPU</em>) for the virtual machine." %}
     </p>
     <p>
       <h4>{% trans "Operating system" %}</h4>
-      The name of the operating system.
+      {% trans "The name of the operating system." %}
     </p>
     <p>
       <h4>{% trans "Agent" %}</h4>
-      Check this if the machine has agent installed and the manager should wait for its start.
+      {% trans "Check this if the machine has agent installed and the manager should wait for its start." %}
     </p>
     <p>
       <h4>{% trans "Raw data" %}</h4>
-      The CIRCLE Cloud is using libvirt, so the owner can customize the running VM's options here by
-      <a href="https://libvirt.org/formatdomain.html">libvirt domain parameters</a>.
+      {% blocktrans %}
+        The CIRCLE Cloud is using libvirt, so the owner can customize the running VM's options here by
+        <a href="https://libvirt.org/formatdomain.html">libvirt domain parameters</a>.
+      {% endblocktrans %}
     </p>
   </blockquote>
 
@@ -387,7 +389,7 @@
 
 
 
-  <h4 id="how-can-i-give-access-to-users-or-groups-to-the-template">{% trans "How can I grant access for users or groups to the template?" %}</h4>
+  <h4 id="how-can-i-grant-access-for-users-or-groups-to-the-template">{% trans "How can I grant access for users or groups to the template?" %}</h4>
   <blockquote>
     <p>
       {% blocktrans %}

circle/templates/info/help/overview_toc.html

@@ -10,10 +10,8 @@
     <ul>
     <li><a href="#how-can-i-create-a-vm">{% trans "How can I create a VM?" %}</a></li>
     <li><a href="#how-can-i-mark-frequently-used-vms">{% trans "How can I mark frequently used VMs?" %}</a></li>
-    <li><a href="#how-can-i-search-for-vms">{% trans "How can I search for VMs?" %}</a></li>
     </ul>
     </li>
-    <li><a href="#templates-box">{% trans "Templates box" %}</a></li>
   </ul>
   </li>
 
@@ -37,8 +35,7 @@
     <li>
     <ul><a href="#home">{% trans "Home" %}</a>
       <li><a href="#expiration">{% trans "Expiration" %}</a></li>
-      <li><a href="#how-can-i-expand-the-vms-expiration-date">{% trans "How can I expand the VM’s expiration date?" %}</a></li>
-      <li><a href="#file-management">{% trans "File management" %}</a></li>
+      <li><a href="#how-can-i-extend-the-vms-expiration-date">{% trans "How can I extend the VM's expiration date?" %}</a></li>
       <li><a href="#how-can-i-share-previously-uploaded-files-with-the-vm">{% trans "How can I share previously uploaded files with the VM?" %}</a></li>
     </ul>
     </li>
@@ -68,9 +65,7 @@
   <ul><a href="#templates">{% trans "Templates" %}</a>
     <li><a href="#how-can-i-create-templates">{% trans "How can I create templates?" %}</a></li>
     <li><a href="#what-kind-of-options-are-customizable-in-the-template">{% trans "What kind of options are customizable in the template?" %}</a></li>
-    <li><a href="#how-can-i-change-the-expiration-of-the-templates-vms">{% trans "How can I change the expiration of the tempalte's VMs" %}</a></li>
-    <li><a href="#how-can-i-give-the-template-to-other-user">{% trans "How can I give the template to other user?" %}</a></li>
-    <li><a href="#how-can-i-give-access-to-users-or-groups-to-the-template">{% trans "How can I give access to users or groups to the template?"%}</a></li>
+    <li><a href="#how-can-i-grant-access-for-users-or-groups-to-the-template">{% trans "How can I grant access for users or groups to the template?"%}</a></li>
   </ul>
   </li>
 
@@ -92,7 +87,6 @@
   <ul><a href="#profile">{% trans "Profile" %}</a>
   <li><a href="#how-can-i-change-my-password">{% trans "How can I change my password?" %}</a></li>
   <li><a href="#how-can-i-store-public-keys-on-the-vms">{% trans "How can I store public keys on the VMs?" %}</a></li>
-  <li><a href="#how-can-i-change-connection-template">{% trans "How can I change connection template?" %}</a></li>
   </ul>
   </li>
 

circle/vm/models/activity.py

@@ -145,8 +145,8 @@ class InstanceActivity(ActivityModel):
 
     def has_percentage(self):
         op = self.instance.get_operation_from_activity_code(self.activity_code)
-        return (self.task_uuid and op and op.has_percentage
-                and not self.finished)
+        return (self.task_uuid and op and op.has_percentage and
+                not self.finished)
 
     def get_percentage(self):
         """Returns the percentage of the running operation if available.

circle/vm/models/common.py

@@ -174,6 +174,6 @@ class Trait(Model):
     @property
     def in_use(self):
         return (
-            self.instance_set.exists() or self.node_set.exists()
-            or self.instancetemplate_set.exists()
+            self.instance_set.exists() or self.node_set.exists() or
+            self.instancetemplate_set.exists()
         )

circle/vm/models/instance.py

@@ -200,6 +200,10 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
     def get_running_instances(self):
         return Instance.active.filter(template=self, status="RUNNING")
 
+    @property
+    def metric_prefix(self):
+        return 'template.%d' % self.pk
+
 
 class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
                TimeStampedModel):
@@ -848,8 +852,8 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
 
     def is_in_status_change(self):
         latest = self.get_latest_activity_in_progress()
-        return (latest and latest.resultant_state is not None
-                and self.status != latest.resultant_state)
+        return (latest and latest.resultant_state is not None and
+                self.status != latest.resultant_state)
 
     @property
     def metric_prefix(self):

circle/vm/operations.py

@@ -133,8 +133,8 @@ class InstanceOperation(Operation):
 
         super(InstanceOperation, self).check_auth(user=user)
 
-        if (self.instance.node and not self.instance.node.online
-                and not user.is_superuser):
+        if (self.instance.node and not self.instance.node.online and
+                not user.is_superuser):
             raise self.instance.WrongStateError(self.instance)
 
     def create_activity(self, parent, user, kwargs):
@@ -191,7 +191,8 @@ class EnsureAgentMixin(object):
         try:
             InstanceActivity.objects.filter(
                 activity_code="vm.Instance.agent.starting",
-                started__gt=last_boot_time).latest("started")
+                started__gt=last_boot_time, instance=self.instance
+            ).latest("started")
         except InstanceActivity.DoesNotExist:  # no agent since last boot
             raise self.instance.NoAgentError(self.instance)
 
@@ -533,8 +534,8 @@ class MigrateOperation(RemoteInstanceOperation):
     remote_timeout = 1000
 
     def _get_remote_args(self, to_node, live_migration, **kwargs):
-        return (super(MigrateOperation, self)._get_remote_args(**kwargs)
-                + [to_node.host.hostname, live_migration])
+        return (super(MigrateOperation, self)._get_remote_args(**kwargs) +
+                [to_node.host.hostname, live_migration])
 
     def rollback(self, activity):
         with activity.sub_activity(
@@ -907,8 +908,8 @@ class SleepOperation(InstanceOperation):
 
         def _get_remote_args(self, **kwargs):
             return (super(SleepOperation.SuspendVmOperation, self)
-                    ._get_remote_args(**kwargs)
-                    + [self.instance.mem_dump['path']])
+                    ._get_remote_args(**kwargs) +
+                    [self.instance.mem_dump['path']])
 
 
 @register_operation
@@ -961,8 +962,8 @@ class WakeUpOperation(InstanceOperation):
 
         def _get_remote_args(self, **kwargs):
             return (super(WakeUpOperation.WakeUpVmOperation, self)
-                    ._get_remote_args(**kwargs)
-                    + [self.instance.mem_dump['path']])
+                    ._get_remote_args(**kwargs) +
+                    [self.instance.mem_dump['path']])
 
 
 @register_operation
@@ -1407,9 +1408,9 @@ class PasswordResetOperation(RemoteAgentOperation):
     task = agent_tasks.change_password
     required_perms = ()
 
-    def _get_remote_args(self, password, **kwargs):
-        return (super(PasswordResetOperation, self)._get_remote_args(**kwargs)
-                + [password])
+    def _get_remote_args(self, password, **kwrgs):
+        return (super(PasswordResetOperation, self)._get_remote_args(**kwrgs) +
+                [password])
 
     def _operation(self, password=None):
         if not password:
@@ -1432,8 +1433,8 @@ class InstallKeysOperation(RemoteAgentOperation):
     def _get_remote_args(self, user, keys=None, **kwargs):
         if keys is None:
             keys = list(user.userkey_set.values_list('key', flat=True))
-        return (super(InstallKeysOperation, self)._get_remote_args(**kwargs)
-                + [keys])
+        return (super(InstallKeysOperation, self)._get_remote_args(**kwargs) +
+                [keys])
 
 
 @register_operation
@@ -1445,8 +1446,8 @@ class RemoveKeysOperation(RemoteAgentOperation):
     required_perms = ()
 
     def _get_remote_args(self, user, keys, **kwargs):
-        return (super(RemoveKeysOperation, self)._get_remote_args(**kwargs)
-                + [keys])
+        return (super(RemoveKeysOperation, self)._get_remote_args(**kwargs) +
+                [keys])
 
 
 @register_operation
@@ -1540,8 +1541,8 @@ class AgentStartedOperation(InstanceOperation):
 
         def _get_remote_args(self, **kwargs):
             cls = AgentStartedOperation.SetTimeOperation
-            return (super(cls, self)._get_remote_args(**kwargs)
-                    + [time.time()])
+            return (super(cls, self)._get_remote_args(**kwargs) +
+                    [time.time()])
 
     @register_operation
     class SetHostnameOperation(SubOperationMixin, RemoteAgentOperation):
@@ -1551,8 +1552,8 @@ class AgentStartedOperation(InstanceOperation):
 
         def _get_remote_args(self, **kwargs):
             cls = AgentStartedOperation.SetHostnameOperation
-            return (super(cls, self)._get_remote_args(**kwargs)
-                    + [self.instance.short_hostname])
+            return (super(cls, self)._get_remote_args(**kwargs) +
+                    [self.instance.short_hostname])
 
     @register_operation
     class RestartNetworkingOperation(SubOperationMixin, RemoteAgentOperation):
@@ -1571,8 +1572,8 @@ class AgentStartedOperation(InstanceOperation):
             interfaces = {str(host.mac): host.get_network_config()
                           for host in hosts}
             cls = AgentStartedOperation.ChangeIpOperation
-            return (super(cls, self)._get_remote_args(**kwargs)
-                    + [interfaces, settings.FIREWALL_SETTINGS['rdns_ip']])
+            return (super(cls, self)._get_remote_args(**kwargs) +
+                    [interfaces, settings.FIREWALL_SETTINGS['rdns_ip']])
 
 
 @register_operation
@@ -1696,8 +1697,8 @@ class AbstractDiskOperation(SubOperationMixin, RemoteInstanceOperation):
     required_perms = ()
 
     def _get_remote_args(self, disk, **kwargs):
-        return (super(AbstractDiskOperation, self)._get_remote_args(**kwargs)
-                + [disk.get_vmdisk_desc()])
+        return (super(AbstractDiskOperation, self)._get_remote_args(**kwargs) +
+                [disk.get_vmdisk_desc()])
 
 
 @register_operation

docs/deploy.rst

-Deploy
-======
+Deploying CIRCLE
+================
 
 This tutorial describes the installation of a production environment. To
 have a fully working environment, you have to set up the other components
@@ -145,4 +145,4 @@ the portal application server::
   sudo cp miscellaneous/mancelery.conf /etc/init/
   sudo start mancelery
   sudo cp miscellaneous/portal-uwsgi.conf /etc/init/
-  sudo start portal-uwsgi
+  sudo start portal-uwsgi
\ No newline at end of file

requirements/base.txt

 amqp==1.4.6
 anyjson==0.3.3
-arrow==0.5.4
+arrow==0.6.0
 billiard==3.3.0.20
 bpython==0.14.1
 celery==3.1.18
@@ -31,7 +31,7 @@ python-dateutil==2.4.2
 pyinotify==0.9.5
 pytz==2015.4
 requests==2.7.0
-salt==2015.5.1
+salt==2014.7.1
 shutilwhich==1.1.0
 simplejson==3.7.2
 six==1.9.0