Merge branch 'master' into issue-218

.gitignore

@@ -23,6 +23,7 @@ celerybeat-schedule
 .coverage
 *,cover
 coverage.xml
+.noseids
 
 # Gettext object file:
 *.mo

circle/acl/models.py

@@ -229,7 +229,7 @@ class AclBase(Model):
             levelfilter,
             content_type=ct, level__weight__gte=level.weight).distinct()
         clsfilter = Q(object_level_set__in=ols.all())
-        return cls.objects.filter(clsfilter)
+        return cls.objects.filter(clsfilter).distinct()
 
     def save(self, *args, **kwargs):
         super(AclBase, self).save(*args, **kwargs)

circle/circle/settings/base.py

@@ -431,9 +431,18 @@ LOGIN_REDIRECT_URL = "/"
 
 AGENT_DIR = get_env_variable(
     'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent'))
+    # AGENT_DIR is the root directory for the agent.
+    # The directory structure SHOULD be:
+    # /home/username/agent
+    # |-- agent-linux
+    # |    |-- .git
+    # |    +-- ...
+    # |-- agent-win
+    # |    +-- agent-win-%(version).exe
+    #
 
 try:
-    git_env = {'GIT_DIR': join(AGENT_DIR, '.git')}
+    git_env = {'GIT_DIR': join(join(AGENT_DIR, "agent-linux"), '.git')}
     AGENT_VERSION = check_output(
         ('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env)
 except:

circle/common/operations.py

@@ -26,6 +26,17 @@ from .models import activity_context, has_suffix, humanize_exception
 logger = getLogger(__name__)
 
 
+class SubOperationMixin(object):
+    required_perms = ()
+
+    def create_activity(self, parent, user, kwargs):
+        if not parent:
+            raise TypeError("SubOperation can only be called with "
+                            "parent_activity specified.")
+        return super(SubOperationMixin, self).create_activity(
+            parent, user, kwargs)
+
+
 class Operation(object):
     """Base class for VM operations.
     """
@@ -36,6 +47,10 @@ class Operation(object):
     abortable = False
     has_percentage = False
 
+    @classmethod
+    def get_activity_code_suffix(cls):
+        return cls.id
+
     def __call__(self, **kwargs):
         return self.call(**kwargs)
 
@@ -62,6 +77,8 @@ class Operation(object):
         parent_activity = auxargs.pop('parent_activity')
         if parent_activity and user is None and not skip_auth_check:
             user = parent_activity.user
+            if user is None:  # parent was a system call
+                skip_auth_check = True
 
         # check for unexpected keyword arguments
         argspec = getargspec(self._operation)
@@ -232,7 +249,7 @@ class OperatedMixin(object):
                   operation could be found.
         """
         for op in getattr(self, operation_registry_name, {}).itervalues():
-            if has_suffix(activity_code, op.activity_code_suffix):
+            if has_suffix(activity_code, op.get_activity_code_suffix()):
                 return op(self)
         else:
             return None

circle/common/tests/test_operations.py

@@ -27,9 +27,7 @@ class OperationTestCase(TestCase):
         class AbortEx(Exception):
             pass
 
-        op = Operation(MagicMock())
-        op.activity_code_suffix = 'test'
-        op.id = 'test'
+        op = TestOp(MagicMock())
         op.async_operation = MagicMock(
             apply_async=MagicMock(side_effect=AbortEx))
 
@@ -44,9 +42,7 @@ class OperationTestCase(TestCase):
         class AbortEx(Exception):
             pass
 
-        op = Operation(MagicMock())
-        op.activity_code_suffix = 'test'
-        op.id = 'test'
+        op = TestOp(MagicMock())
         with patch.object(Operation, 'create_activity', side_effect=AbortEx):
             with patch.object(Operation, 'check_precond') as chk_pre:
                 try:
@@ -55,9 +51,7 @@ class OperationTestCase(TestCase):
                     self.assertTrue(chk_pre.called)
 
     def test_auth_check_on_non_system_call(self):
-        op = Operation(MagicMock())
-        op.activity_code_suffix = 'test'
-        op.id = 'test'
+        op = TestOp(MagicMock())
         user = MagicMock()
         with patch.object(Operation, 'check_auth') as check_auth:
             with patch.object(Operation, 'check_precond'), \
@@ -67,9 +61,7 @@ class OperationTestCase(TestCase):
             check_auth.assert_called_with(user)
 
     def test_no_auth_check_on_system_call(self):
-        op = Operation(MagicMock())
-        op.activity_code_suffix = 'test'
-        op.id = 'test'
+        op = TestOp(MagicMock())
         with patch.object(Operation, 'check_auth', side_effect=AssertionError):
             with patch.object(Operation, 'check_precond'), \
                     patch.object(Operation, 'create_activity'), \
@@ -77,39 +69,25 @@ class OperationTestCase(TestCase):
                 op.call(system=True)
 
     def test_no_exception_for_more_arguments_when_operation_takes_kwargs(self):
-        class KwargOp(Operation):
-            activity_code_suffix = 'test'
-            id = 'test'
-
-            def _operation(self, **kwargs):
-                pass
-
-        op = KwargOp(MagicMock())
-        with patch.object(KwargOp, 'create_activity'), \
-                patch.object(KwargOp, '_exec_op'):
+        op = TestOp(MagicMock())
+        with patch.object(TestOp, 'create_activity'), \
+                patch.object(TestOp, '_exec_op'):
             op.call(system=True, foo=42)
 
     def test_exception_for_unexpected_arguments(self):
-        class TestOp(Operation):
-            activity_code_suffix = 'test'
-            id = 'test'
-
-            def _operation(self):
-                pass
-
         op = TestOp(MagicMock())
         with patch.object(TestOp, 'create_activity'), \
                 patch.object(TestOp, '_exec_op'):
-            self.assertRaises(TypeError, op.call, system=True, foo=42)
+            self.assertRaises(TypeError, op.call, system=True, bar=42)
 
     def test_exception_for_missing_arguments(self):
-        class TestOp(Operation):
-            activity_code_suffix = 'test'
-            id = 'test'
-
-            def _operation(self, foo):
-                pass
-
         op = TestOp(MagicMock())
         with patch.object(TestOp, 'create_activity'):
             self.assertRaises(TypeError, op.call, system=True)
+
+
+class TestOp(Operation):
+    id = 'test'
+
+    def _operation(self, foo):
+        pass

circle/dashboard/forms.py

@@ -18,6 +18,7 @@
 from __future__ import absolute_import
 
 from datetime import timedelta
+from urlparse import urlparse
 
 from django.contrib.auth.forms import (
     AuthenticationForm, PasswordResetForm, SetPasswordForm,
@@ -39,6 +40,7 @@ from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
 from django.forms.widgets import TextInput, HiddenInput
 from django.template import Context
 from django.template.loader import render_to_string
+from django.utils.html import escape
 from django.utils.translation import ugettext_lazy as _
 from sizefield.widgets import FileSizeWidget
 from django.core.urlresolvers import reverse_lazy
@@ -79,6 +81,12 @@ class VmSaveForm(forms.Form):
         helper.form_tag = False
         return helper
 
+    def __init__(self, *args, **kwargs):
+        default = kwargs.pop('default', None)
+        super(VmSaveForm, self).__init__(*args, **kwargs)
+        if default:
+            self.fields['name'].initial = default
+
 
 class VmCustomizeForm(forms.Form):
     name = forms.CharField(widget=forms.TextInput(attrs={
@@ -744,6 +752,20 @@ class VmRenewForm(forms.Form):
         return helper
 
 
+class VmMigrateForm(forms.Form):
+    live_migration = forms.BooleanField(
+        required=False, initial=True, label=_("live migration"))
+
+    def __init__(self, *args, **kwargs):
+        choices = kwargs.pop('choices')
+        default = kwargs.pop('default')
+        super(VmMigrateForm, self).__init__(*args, **kwargs)
+
+        self.fields.insert(0, 'to_node', forms.ModelChoiceField(
+            queryset=choices, initial=default, required=False,
+            widget=forms.RadioSelect(), label=_("Node")))
+
+
 class VmStateChangeForm(forms.Form):
 
     interrupt = forms.BooleanField(required=False, label=_(
@@ -752,6 +774,7 @@ class VmStateChangeForm(forms.Form):
                     "but don't interrupt any tasks."))
     new_state = forms.ChoiceField(Instance.STATUS, label=_(
         "New status"))
+    reset_node = forms.BooleanField(required=False, label=_("Reset node"))
 
     def __init__(self, *args, **kwargs):
         show_interrupt = kwargs.pop('show_interrupt')
@@ -769,6 +792,17 @@ class VmStateChangeForm(forms.Form):
         return helper
 
 
+class RedeployForm(forms.Form):
+    with_emergency_change_state = forms.BooleanField(
+        required=False, initial=True, label=_("use emergency state change"))
+
+    @property
+    def helper(self):
+        helper = FormHelper(self)
+        helper.form_tag = False
+        return helper
+
+
 class VmCreateDiskForm(forms.Form):
     name = forms.CharField(max_length=100, label=_("Name"))
     size = forms.CharField(
@@ -776,6 +810,12 @@ class VmCreateDiskForm(forms.Form):
         help_text=_('Size of disk to create in bytes or with units '
                     'like MB or GB.'))
 
+    def __init__(self, *args, **kwargs):
+        default = kwargs.pop('default', None)
+        super(VmCreateDiskForm, self).__init__(*args, **kwargs)
+        if default:
+            self.fields['name'].initial = default
+
     def clean_size(self):
         size_in_bytes = self.cleaned_data.get("size")
         if not size_in_bytes.isdigit() and len(size_in_bytes) > 0:
@@ -827,13 +867,42 @@ class VmDiskResizeForm(forms.Form):
         helper.form_tag = False
         if self.disk:
             helper.layout = Layout(
-                HTML(_("<label>Disk:</label> %s") % self.disk),
+                HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
                 Field('disk'), Field('size'))
         return helper
 
 
+class VmDiskRemoveForm(forms.Form):
+    def __init__(self, *args, **kwargs):
+        choices = kwargs.pop('choices')
+        self.disk = kwargs.pop('default')
+
+        super(VmDiskRemoveForm, self).__init__(*args, **kwargs)
+
+        self.fields.insert(0, 'disk', forms.ModelChoiceField(
+            queryset=choices, initial=self.disk, required=True,
+            empty_label=None, label=_('Disk')))
+        if self.disk:
+            self.fields['disk'].widget = HiddenInput()
+
+    @property
+    def helper(self):
+        helper = FormHelper(self)
+        helper.form_tag = False
+        if self.disk:
+            helper.layout = Layout(
+                AnyTag(
+                    "div",
+                    HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
+                    css_class="form-group",
+                ),
+                Field("disk"),
+            )
+        return helper
+
+
 class VmDownloadDiskForm(forms.Form):
-    name = forms.CharField(max_length=100, label=_("Name"))
+    name = forms.CharField(max_length=100, label=_("Name"), required=False)
     url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
 
     @property
@@ -842,6 +911,18 @@ class VmDownloadDiskForm(forms.Form):
         helper.form_tag = False
         return helper
 
+    def clean(self):
+        cleaned_data = super(VmDownloadDiskForm, self).clean()
+        if not cleaned_data['name']:
+            if cleaned_data['url']:
+                cleaned_data['name'] = urlparse(
+                    cleaned_data['url']).path.split('/')[-1]
+            if not cleaned_data['name']:
+                raise forms.ValidationError(
+                    _("Could not find filename in URL, "
+                      "please specify a name explicitly."))
+        return cleaned_data
+
 
 class VmAddInterfaceForm(forms.Form):
     def __init__(self, *args, **kwargs):

circle/dashboard/models.py

@@ -236,6 +236,9 @@ class GroupProfile(AclBase):
         help_text=_('Unique identifier of the group at the organization.'))
     description = TextField(blank=True)
 
+    def __unicode__(self):
+        return self.group.name
+
     def save(self, *args, **kwargs):
         if not self.org_id:
             self.org_id = None

circle/dashboard/static/dashboard/dashboard.css

@@ -528,7 +528,7 @@ footer a, footer a:hover, footer a:visited {
 }
 
 #dashboard-template-list a small {
-  max-width: 50%;
+  max-width: 45%;
   float: left;
   padding-top: 2px;
   text-overflow: ellipsis;
@@ -974,6 +974,10 @@ textarea[name="new_members"] {
   color: orange;
 }
 
+#vm-info-pane {
+  margin-bottom: 20px;
+}
+
 .node-list-table tbody>tr>td, .node-list-table thead>tr>th {
   vertical-align: middle;
 }
@@ -996,10 +1000,19 @@ textarea[name="new_members"] {
   max-width: 100%;
 }
 
-#vm-list-table tbody td:nth-child(3) {
+#vm-list-table td.state,
+#vm-list-table td.memory {
   white-space: nowrap;
 }
 
 #vm-list-table td {
   vertical-align: middle;
 }
+
+.disk-resize-btn {
+  margin-right: 5px;
+}
+
+#vm-migrate-node-list li {
+  cursor: pointer;
+}

circle/dashboard/static/dashboard/dashboard.js

@@ -411,6 +411,17 @@ $(function () {
     $(this).removeClass("btn-default").addClass("btn-primary");
     return false;
   });
+
+  // vm migrate select for node
+  $(document).on("click", "#vm-migrate-node-list li", function(e) {
+    var li = $(this).closest('li');
+    if (li.find('input').attr('disabled'))
+      return true;
+    $('#vm-migrate-node-list li').removeClass('panel-primary');
+    li.addClass('panel-primary').find('input').prop("checked", true);
+    return true;
+  });
+
 });
 
 function generateVmHTML(pk, name, host, icon, _status, fav, is_last) {
@@ -445,7 +456,7 @@ function generateNodeHTML(name, icon, _status, url, is_last) {
 
 function generateNodeTagHTML(name, icon, _status, label , url) {
   return '<a href="' + url + '" class="label ' + label + '" >' +
-        '<i class="' + icon + '" title="' + _status + '"></i> ' + name +
+        '<i class="fa ' + icon + '" title="' + _status + '"></i> ' + name +
         '</a> ';
 }
 
@@ -618,7 +629,7 @@ function addModalConfirmation(func, data) {
 }
 
 function clientInstalledAction(location) {   
-  setCookie('downloaded_client', true, 365 * 24 * 60 * 60, "/");
+  setCookie('downloaded_client', true, 365 * 24 * 60 * 60 * 1000, "/");
   window.location.href = location;
   $('#confirmation-modal').modal("hide");
 }

circle/dashboard/static/dashboard/vm-common.js

@@ -16,15 +16,6 @@ $(function() {
         $('#confirmation-modal').on('hidden.bs.modal', function() {
           $('#confirmation-modal').remove();
         });
-
-        $('#vm-migrate-node-list li').click(function(e) {
-          var li = $(this).closest('li');
-          if (li.find('input').attr('disabled'))
-            return true;
-          $('#vm-migrate-node-list li').removeClass('panel-primary');
-          li.addClass('panel-primary').find('input').attr('checked', true);
-          return false;
-        });
         $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
       }
     });
@@ -51,7 +42,8 @@ $(function() {
         if(data.success) {
           $('a[href="#activity"]').trigger("click");
           if(data.with_reload) {
-            location.reload();
+            // when the activity check stops the page will reload
+            reload_vm_detail = true;
           }
 
           /* if there are messages display them */

circle/dashboard/static/dashboard/vm-details.js

 var show_all = false;
 var in_progress = false;
 var activity_hash = 5;
+var reload_vm_detail = false;
 
 $(function() {
   /* do we need to check for new activities */
@@ -404,6 +405,7 @@ function checkNewActivity(runs) {
         );
       } else {
         in_progress = false;
+        if(reload_vm_detail) location.reload();
       }
       $('a[href="#activity"] i').removeClass('fa-spin');
     },

circle/dashboard/templates/branding.html

+<img src="{{ STATIC_URL}}dashboard/img/logo.png" style="height: 25px;"/>
+<img src="{{ STATIC_URL}}local-logo.png" style="padding-left: 2px; height: 25px;"/>

circle/dashboard/templates/dashboard/_disk-list-element.html

 {% load i18n %}
 {% load sizefieldtags %}
 
-<i class="fa {% if d.is_downloading %}fa-refresh fa-spin{% else %}fa-file{% if d.failed %}" style="color: #d9534f;{% endif %}{% endif %}"></i>
-{{ d.name }} (#{{ d.id }}) -
-{% if not d.is_downloading %}
-  {% if not d.failed %}
-    {% if d.size %}{{ d.size|filesize }}{% endif %}
-  {% else %}
-  <div class="label label-danger"{% if user.is_superuser %} title="{{ d.get_latest_activity_result }}"{% endif %}>{% trans "failed" %}</div>
-  {% endif %}
-{% else %}<span class="disk-list-disk-percentage" data-disk-pk="{{ d.pk }}">{{ d.get_download_percentage }}</span>%{% endif %}
-{% if is_owner != False %}
-  <a href="{% url "dashboard.views.disk-remove" pk=d.pk %}?next={{ request.path }}"
-    data-disk-pk="{{ d.pk }}" class="btn btn-xs btn-danger pull-right disk-remove"
-    {% if not long_remove %}title="{% trans "Remove" %}"{% endif %}>
-    <i class="fa fa-times"></i>{% if long_remove %} {% trans "Remove" %}{% endif %}
-  </a>
-  {% if op.resize_disk %}
-    <span class="operation-wrapper">
-      <a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}" class="btn btn-xs btn-warning pull-right operation">
-        <i class="fa fa-arrows-alt"></i> {% trans "Resize" %}
-      </a>
-    </span>
-  {% endif %}
+<i class="fa fa-file"></i>
+{{ d.name }} (#{{ d.id }}) - {{ d.size|filesize }}
+
+{% if op.remove_disk %}
+  <span class="operation-wrapper">
+    <a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
+      class="btn btn-xs btn-{{ op.remove_disk.effect}} pull-right operation disk-remove-btn
+      {% if op.resize_disk.disabled %}disabled{% endif %}">
+      <i class="fa fa-{{ op.remove_disk.icon }}"></i> {% trans "Remove" %}
+    </a>
+  </span>
+{% endif %}
+{% if op.resize_disk %}
+  <span class="operation-wrapper">
+    <a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
+      class="btn btn-xs btn-{{ op.resize_disk.effect }} pull-right operation disk-resize-btn
+      {% if op.resize_disk.disabled %}disabled{% endif %}">
+      <i class="fa fa-{{ op.resize_disk.icon }}"></i> {% trans "Resize" %}
+    </a>
+  </span>
 {% endif %}
 <div style="clear: both;"></div>
+
+{% if request.user.is_superuser %}
+  <small>{% trans "File name" %}: {{ d.filename }}</small>
+{% endif %}

circle/dashboard/templates/dashboard/_vm-mass-migrate.html

 {% extends "dashboard/mass-operate.html" %}
 {% load i18n %}
 {% load sizefieldtags %}
+{% load crispy_forms_tags %}
 
 
 {% block formfields %}
@@ -11,20 +12,20 @@
         <label for="migrate-to-none">
           <strong>{% trans "Reschedule" %}</strong>
         </label>
-        <input id="migrate-to-none" type="radio" name="node" value="" style="float: right;" checked="checked">
+        <input id="migrate-to-none" type="radio" name="to_node" value="" style="float: right;" checked="checked">
         <span class="vm-migrate-node-property">
           {% trans "This option will reschedule each virtual machine to the optimal node." %}
         </span>
         <div style="clear: both;"></div>
       </div>
     </li>
-    {% for n in nodes %}
+    {% for n in form.fields.to_node.queryset.all %}
     <li class="panel panel-default mass-migrate-node">
       <div class="panel-body">
         <label for="migrate-to-{{n.pk}}">
           <strong>{{ n }}</strong>
         </label>
-        <input id="migrate-to-{{n.pk}}" type="radio" name="node" value="{{ n.pk }}" style="float: right;"/>
+        <input id="migrate-to-{{n.pk}}" type="radio" name="to_node" value="{{ n.pk }}" style="float: right;"/>
         <span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span>
         <span class="vm-migrate-node-property">{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span>
         <div style="clear: both;"></div>
@@ -32,5 +33,6 @@
     </li>
     {% endfor %}
   </ul>
+  {{ form.live_migration|as_crispy_field }}
   <hr />
 {% endblock %}

circle/dashboard/templates/dashboard/_vm-migrate.html

 {% extends "dashboard/operate.html" %}
 {% load i18n %}
 {% load sizefieldtags %}
+{% load crispy_forms_tags %}
 
 {% block question %}
 <p>
@@ -13,24 +14,27 @@ Choose a compute node to migrate {{obj}} to.
 
 {% block formfields %}
   <ul id="vm-migrate-node-list" class="list-unstyled">
-  {% with current=object.node.pk selected=object.select_node.pk %}
-    {% for n in nodes %}
+  {% with current=object.node.pk recommended=form.fields.to_node.initial.pk %}
+    {% for n in form.fields.to_node.queryset.all %}
       <li class="panel panel-default"><div class="panel-body">
         <label for="migrate-to-{{n.pk}}">
           <strong>{{ n }}</strong>
-          <div class="label label-primary"><i class="fa {{n.get_status_icon}}"></i>
-              {{n.get_status_display}}</div>
+          <div class="label label-primary">
+            <i class="fa {{n.get_status_icon}}"></i> {{n.get_status_display}}</div>
           {% if current == n.pk %}<div class="label label-info">{% trans "current" %}</div>{% endif %}
-          {% if selected == n.pk %}<div class="label label-success">{% trans "recommended" %}</div>{% endif %}
+          {% if recommended == n.pk %}<div class="label label-success">{% trans "recommended" %}</div>{% endif %}
         </label>
-        <input id="migrate-to-{{n.pk}}" type="radio" name="node" value="{{ n.pk }}" style="float: right;"
-        {% if current == n.pk %}disabled="disabled"{% endif %}
-        {% if selected == n.pk %}checked="checked"{% endif %} />
+        <input id="migrate-to-{{n.pk}}" type="radio" name="to_node" value="{{ n.pk }}" style="float: right;"
+          {% if current == n.pk %}disabled="disabled"{% endif %}
+          {% if recommended == n.pk %}checked="checked"{% endif %}
+        />
         <span class="vm-migrate-node-property">{% trans "CPU load" %}: {{ n.cpu_usage }}</span>
-        <span class="vm-migrate-node-property">{% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span>
+        <span class="vm-migrate-node-property">
+          {% trans "RAM usage" %}: {{ n.byte_ram_usage|filesize }}/{{ n.ram_size|filesize }}</span>
         <div style="clear: both;"></div>
-      </div></li>
+      </li>
     {% endfor %}
   {% endwith %}
   </ul>
+  {{ form.live_migration|as_crispy_field }}
 {% endblock %}

circle/dashboard/templates/dashboard/base.html

@@ -13,7 +13,7 @@
 
 {% block navbar-brand %}
   <a class="navbar-brand" href="{% url "dashboard.index" %}" style="padding: 10px 15px;">
-    <img src="{{ STATIC_URL}}dashboard/img/logo.png" style="height: 25px;"/>
+    {% include "branding.html" %}
   </a>
 {% endblock %}
 

circle/dashboard/templates/dashboard/group-detail.html

@@ -52,6 +52,36 @@
 
 <hr />
 
+<h3>{% trans "Available objects for this group" %}</h3>
+<ul class="dashboard-profile-vm-list fa-ul">
+  {% for i in vm_objects %}
+    <li>
+      <a href="{{ i.get_absolute_url }}">
+        <i class="fa fa-li {{ i.get_status_icon }}"></i>
+        {{ i }}
+      </a>
+    </li>
+  {% endfor %}
+  {% for t in template_objects %}
+    <li>
+      <a href="{{ t.get_absolute_url }}">
+        <i class="fa fa-li fa-puzzle-piece"></i>
+        {{ t }}
+      </a>
+    </li>
+  {% endfor %}
+  {% for g in group_objects %}
+    <li>
+      <a href="{{ g.get_absolute_url }}">
+        <i class="fa fa-li fa-users"></i>
+        {{ g }}
+      </a>
+    </li>
+  {% endfor %}
+</ul>
+
+<hr />
+
 <h3>{% trans "User list"|capfirst %}
     {% if perms.auth.add_user %}
         <a href="{% url "dashboard.views.create-user" group.pk %}" class="btn btn-success pull-right">{% trans "Create user" %}</a>

circle/dashboard/templates/dashboard/index-nodes.html

 {% load i18n %}
-  <div class="panel panel-default">
+<div class="panel panel-default">
   <div class="panel-heading">
     <div class="pull-right toolbar">
       <div class="btn-group">
@@ -7,9 +7,10 @@
           data-container="body"><i class="fa fa-dashboard"></i></a>
         <a href="#index-list-view" data-index-box="node" class="btn btn-default btn-xs disabled"
           data-container="body"><i class="fa fa-list"></i></a>
-
       </div>
-      <span class="btn btn-default btn-xs infobtn" title="{% trans "List of compute nodes, also called worker nodes or hypervisors, which run the virtual machines." %}"><i class="fa fa-info-circle"></i></span>
+      <span class="btn btn-default btn-xs infobtn" title="{% trans "List of compute nodes, also called worker nodes or hypervisors, which run the virtual machines." %}">
+        <i class="fa fa-info-circle"></i>
+      </span>
     </div>
     <h3 class="no-margin">
       <i class="fa fa-sitemap"></i> {% trans "Nodes" %}
@@ -28,50 +29,55 @@
       </a>
       {% endfor %}
     </div>
-    <div href="#" class="list-group-item list-group-footer">
-      <div class="row">
-        <div class="col-sm-6 col-xs-6 input-group input-group-sm">
-          <input id="dashboard-node-search-input" type="text" class="form-control" placeholder="{% trans "Search..." %}" />
-          <div class="input-group-btn">
-            <button type="submit" class="form-control btn btn-primary" title="search"><i class="fa fa-search"></i></button>
-          </div>
-        </div>
-        <div class="col-sm-6 text-right">
-          <a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}">
-            <i class="fa fa-chevron-circle-right"></i>
-            {% if more_nodes > 0 %}
-                {% blocktrans with count=more_nodes %}<strong>{{count}}</strong>  more{% endblocktrans %}
-            {% else %}
-                {% trans "list" %}
-            {% endif %}
-          </a>
-          <a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}"><i class="fa fa-plus-circle"></i> {% trans "new" %}</a>
-        </div>
-      </div>
-    </div>
-  </div>
+  </div><!-- #node-list-view -->
 
-  <div class="panel-body" id="node-graph-view" style="display: none">
-    <p class="pull-right">  <input class="knob" data-fgColor="chartreuse" data-thickness=".4" data-width="60" data-height="60" data-readOnly="true" value="{% widthratio node_num.running sum_node_num 100 %}"></p>
-    <p><span class="big"><big>{{ node_num.running }}</big> running </span>
-    + <big>{{ node_num.missing }}</big> missing + <br><big>{{ node_num.disabled }}</big> disabled + <big>{{ node_num.offline }}</big> offline</p>
-      <ul class="list-inline" id="dashboard-node-taglist">
-        {% for i in nodes %}
-        <a href="{{ i.get_absolute_url }}" class="label {{i.get_status_label}}" >
-        <i class="fa {{ i.get_status_icon }}" title="{{ i.get_status_display }}"></i> {{ i.name }}</a>
-        {% endfor %}
-      </ul>
+  <div class="panel-body" id="node-graph-view" style="display: none; min-height: 204px;">
+    <p class="pull-right">
+      <input class="knob" data-fgColor="chartreuse"
+      data-thickness=".4" data-width="60" data-height="60" data-readOnly="true"
+      value="{% widthratio node_num.running sum_node_num 100 %}">
+    </p>
+    <p>
+      <span class="big">
+        <big>{{ node_num.running }}</big> running
+      </span>
+      + <big>{{ node_num.missing }}</big>
+      missing + <br><big>{{ node_num.disabled }}</big> disabled + <big>{{ node_num.offline }}</big> offline
+    </p>
+    <ul class="list-inline" id="dashboard-node-taglist">
+      {% for i in nodes %}
+      <a href="{{ i.get_absolute_url }}" class="label {{i.get_status_label}}" >
+      <i class="fa {{ i.get_status_icon }}" title="{{ i.get_status_display }}"></i> {{ i.name }}</a>
+      {% endfor %}
+    </ul>
 
     <div class="clearfix"></div>
- <div class="row">
-        <div class="col-sm-6 text-right pull-right">
-         {% if more_nodes >= 0  %}
-          <a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}">
-            <i class="fa fa-chevron-circle-right"></i> {% blocktrans with count=more_nodes %}<strong>{{count}}</strong>  more{% endblocktrans %}
-          </a>
-         {% endif %}
-         <a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}"><i class="fa fa-plus-circle"></i> {% trans "new" %}</a>
+  </div>
+
+  <div href="#" class="list-group-item list-group-footer">
+    <div class="row">
+      <div class="col-sm-6 col-xs-6 input-group input-group-sm">
+        <input id="dashboard-node-search-input" type="text" class="form-control"
+         placeholder="{% trans "Search..." %}" />
+        <div class="input-group-btn">
+          <button type="submit" class="btn btn-primary" title="{% trans "Search" %}" data-container="body">
+            <i class="fa fa-search"></i>
+          </button>
         </div>
-</div>
-</div>
+      </div>
+      <div class="col-sm-6 text-right">
+        <a class="btn btn-primary btn-xs" href="{% url "dashboard.views.node-list" %}">
+          <i class="fa fa-chevron-circle-right"></i>
+          {% if more_nodes > 0 %}
+              {% blocktrans with count=more_nodes %}<strong>{{count}}</strong>  more{% endblocktrans %}
+          {% else %}
+              {% trans "list" %}
+          {% endif %}
+        </a>
+        <a class="btn btn-success btn-xs node-create" href="{% url "dashboard.views.node-create" %}">
+          <i class="fa fa-plus-circle"></i> {% trans "new" %}
+        </a>
+      </div>
+    </div>
+  </div>
 </div>

circle/dashboard/templates/dashboard/instanceactivity_detail.html

@@ -58,6 +58,26 @@
 
             <dt>{% trans "resultant state" %}</dt>
             <dd>{{object.resultant_state|default:'n/a'}}</dd>
+
+            <dt>{% trans "subactivities" %}</dt>
+            {% for s in object.children.all %}
+              <dd>
+                <span{% if s.result %} title="{{ s.result|get_text:user }}"{% endif %}>
+                  <a href="{{ s.get_absolute_url }}">
+                      {{ s.readable_name|get_text:user|capfirst }}</a></span> &ndash;
+                {% if s.finished %}
+                  {{ s.finished|time:"H:i:s" }}
+                {% else %}
+                  <i class="fa fa-refresh fa-spin" class="sub-activity-loading-icon"></i>
+                {% endif %}
+                {% if s.has_failed %}
+                  <div class="label label-danger">{% trans "failed" %}</div>
+                {% endif %}
+              </dd>
+            {% empty %}
+              <dd>{% trans "none" %}</dd>
+            {% endfor %}
+          </div>
         </div>
       </div>
     </div>

circle/dashboard/templates/dashboard/node-list/column-vm.html

 {% load i18n %}
 
 <div id="node-list-column-vm">
-  <a class="real-link" href="{% url "dashboard.views.vm-list" %}?s=node:{{ record.name }}">
+  <a class="real-link" href="{% url "dashboard.views.vm-list" %}?s=node_exact:{{ record.name }}">
     {{ value }}
   </a>
 </div>

circle/dashboard/templates/dashboard/template-edit.html

@@ -86,7 +86,13 @@
         {% endif %}
         {% for d in disks %}
           <li>
-            {% include "dashboard/_disk-list-element.html" %}
+            <i class="fa fa-file"></i>
+            {{ d.name }} (#{{ d.id }}) -
+            <a href="{% url "dashboard.views.disk-remove" pk=d.pk %}?next={{ request.path }}"
+              data-disk-pk="{{ d.pk }}" class="btn btn-xs btn-danger pull-right disk-remove"
+              {% if not long_remove %}title="{% trans "Remove" %}"{% endif %}>
+              <i class="fa fa-times"></i>{% if long_remove %} {% trans "Remove" %}{% endif %}
+            </a>
           </li>
         {% endfor %}
         </ul>

circle/dashboard/templates/dashboard/vm-detail.html

@@ -47,7 +47,10 @@
           <div class="input-group vm-details-home-name">
             <input id="vm-details-rename-name" class="form-control input-sm" name="new_name" type="text" value="{{ instance.name }}"/>
             <span class="input-group-btn">
-              <button type="submit" class="btn btn-sm vm-details-rename-submit">{% trans "Rename" %}</button>
+              <button type="submit" class="btn btn-sm vm-details-rename-submit
+                {% if not is_operator %}disabled{% endif %}">
+                {% trans "Rename" %}
+              </button>
             </span>
           </div>
         </form>

circle/dashboard/templates/dashboard/vm-detail/_network-port-add.html

@@ -11,7 +11,8 @@
       <span class="input-group-addon">/</span>
       <select class="form-control" name="proto" style="width: 70px;"><option>tcp</option><option>udp</option></select>
       <div class="input-group-btn">
-        <button type="submit" class="btn btn-success btn-sm">{% trans "Add" %}</button>
+        <button type="submit" class="btn btn-success btn-sm
+          {% if not is_operator %}disabled{% endif %}">{% trans "Add" %}</button>
       </div>
     </div>
   </form>

circle/dashboard/templates/dashboard/vm-detail/home.html

@@ -6,7 +6,9 @@
       <dd><i class="fa fa-{{ os_type_icon }}"></i> {{ instance.system }}</dd>
       <dt style="margin-top: 5px;">
         {% trans "Name" %}:
-        <a href="#" class="vm-details-home-edit-name-click"><i class="fa fa-pencil"></i></a>
+        {% if is_operator %}
+          <a href="#" class="vm-details-home-edit-name-click"><i class="fa fa-pencil"></i></a>
+        {% endif %}
       </dt>
       <dd>
         <div class="vm-details-home-edit-name-click">
@@ -18,8 +20,9 @@
             <div class="input-group">
               <input type="text" name="new_name" value="{{ instance.name }}" class="form-control input-sm"/>
               <span class="input-group-btn">
-                <button type="submit" class="btn btn-success btn-sm vm-details-rename-submit">
-                  <i class="fa fa-pencil"></i> {% trans "Rename" %}
+                <button type="submit" class="btn btn-success btn-sm vm-details-rename-submit
+                  {% if not is_operator %}disabled{% endif %}" title="{% trans "Rename" %}">
+                  <i class="fa fa-pencil"></i>
                 </button>
               </span>
             </div>
@@ -28,7 +31,9 @@
       </dd>
       <dt style="margin-top: 5px;">
         {% trans "Description" %}:
-        <a href="#" class="vm-details-home-edit-description-click"><i class="fa fa-pencil"></i></a>
+        {% if is_operator %}
+          <a href="#" class="vm-details-home-edit-description-click"><i class="fa fa-pencil"></i></a>
+        {% endif %}
       </dt>
       <dd>
         {% csrf_token %}
@@ -38,7 +43,8 @@
         <div id="vm-details-home-description" class="js-hidden">
           <form method="POST">
             <textarea name="new_description" class="form-control">{{ instance.description }}</textarea>
-            <button type="submit" class="btn btn-xs btn-success vm-details-description-submit">
+            <button type="submit" class="btn btn-xs btn-success vm-details-description-submit
+              {% if not is_operator %}disabled{% endif %}">
               <i class="fa fa-pencil"></i> {% trans "Update" %}
             </button>
           </form>
@@ -58,9 +64,17 @@
     </h4>
     <dl>
       <dt>{% trans "Suspended at:" %}</dt>
-      <dd><i class="fa fa-moon-o"></i> {{ instance.time_of_suspend|timeuntil }}</dd>
+      <dd>
+        <span title="{{ instance.time_of_suspend }}">
+          <i class="fa fa-moon-o"></i> {{ instance.time_of_suspend|timeuntil }}
+        </span>
+      </dd>
       <dt>{% trans "Destroyed at:" %}</dt>
-      <dd><i class="fa fa-times"></i> {{ instance.time_of_delete|timeuntil }}</dd>
+      <dd>
+        <span title="{{ instance.time_of_delete }}">
+          <i class="fa fa-times"></i> {{ instance.time_of_delete|timeuntil }}
+        </span>
+      </dd>
     </dl>
 
     <div style="font-weight: bold;">{% trans "Tags" %}</div>
@@ -70,11 +84,13 @@
           {% for t in instance.tags.all %}
             <div class="label label-primary label-tag" style="display: inline-block">
               {{ t }}
-              <a href="#" class="vm-details-remove-tag"><i class="fa fa-times"></i></a>
+              {% if is_operator %}
+                <a href="#" class="vm-details-remove-tag"><i class="fa fa-times"></i></a>
+              {% endif %}
             </div>
           {% endfor %}
         {% else %}
-          <small>{% trans "No tag added!" %}</small>
+          <small>{% trans "No tag added." %}</small>
         {% endif %}
       </div>
       <form action="" method="POST">
@@ -85,11 +101,26 @@
             <i class="fa fa-question"></i>
           </div>-->
           <div class="input-group-btn">
-            <input type="submit" class="btn btn-default btn-sm input-tags" value="{% trans "Add tag" %}"/>
+            <input type="submit" class="btn btn-default btn-sm input-tags
+            {% if not is_operator %}disabled{% endif %}" value="{% trans "Add tag" %}"/>
           </div>
         </div>
       </form>
     </div><!-- id:vm-details-tags -->
+    {% if request.user.is_superuser %}
+    <dl>
+      <dt>{% trans "Node" %}:</dt>
+      <dd>
+        {% if instance.node %}
+          <a href="{{ instance.node.get_absolute_url }}">
+            {{ instance.node.name }}
+          </a>
+        {% else %}
+          -
+        {% endif %}
+      </dd>
+    {% endif %}
+    </dl>
     <dl>
       <dt>{% trans "Template" %}:</dt>
       <dd>

circle/dashboard/templates/dashboard/vm-detail/network.html

@@ -21,11 +21,13 @@
       <a href="{{ i.host.get_absolute_url }}"
         class="btn btn-default btn-xs">{% trans "edit" %}</a>
     {% endif %}
-    <a href="{% url "dashboard.views.interface-delete" pk=i.pk %}?next={{ request.path }}"
-      class="btn btn-danger btn-xs interface-remove"
-      data-interface-pk="{{ i.pk }}">
-      {% trans "remove" %}
-    </a>
+    {% if is_owner %}
+      <a href="{% url "dashboard.views.interface-delete" pk=i.pk %}?next={{ request.path }}"
+        class="btn btn-danger btn-xs interface-remove"
+        data-interface-pk="{{ i.pk }}">
+        {% trans "remove" %}
+      </a>
+    {% endif %}
   </h3>
   {% if i.host %}
     <div class="row">

circle/dashboard/templates/dashboard/vm-list.html

@@ -72,6 +72,10 @@
                 {% trans "Lease" as t %}
                 {% include "dashboard/vm-list/header-link.html" with name=t sort="lease" %}
               </th>
+              <th data-sort="string" class="orderable sortable">
+                {% trans "Memory" as t %}
+                {% include "dashboard/vm-list/header-link.html" with name=t sort="ram_size" %}
+              </th>
               {% if user.is_superuser %}
                 <th data-sort="string" class="orderable sortable">
                   {% trans "IP address" as t %}
@@ -86,7 +90,9 @@
           {% for i in object_list %}
           <tr class="{% cycle 'odd' 'even' %}" data-vm-pk="{{ i.pk }}">
             <td class="pk"><div id="vm-{{i.pk}}">{{i.pk}}</div> </td>
-            <td class="name"><a class="real-link" href="{% url "dashboard.views.detail" i.pk %}">{{ i.name }}</a>  </td>
+            <td class="name"><a class="real-link" href="{% url "dashboard.views.detail" i.pk %}">
+                {{ i.name }}</a>
+            </td>
             <td class="state">
               <i class="fa fa-fw
                 {% if show_acts_in_progress and i.is_in_status_change %}
@@ -104,7 +110,12 @@
               {# include "dashboard/_display-name.html" with user=i.owner show_org=True #}
             </td>
             <td class="lease "data-sort-value="{{ i.lease.name }}">
-              {{ i.lease.name }}
+              <span title="{{ i.time_of_suspend|timeuntil }} | {{ i.time_of_delete|timeuntil }}">
+                {{ i.lease.name }}
+              </span>
+            </td>
+            <td class="memory "data-sort-value="{{ i.ram_size }}">
+              {{ i.ram_size }} MiB
             </td>
             {% if user.is_superuser %}
             <td class="ip_addr "data-sort-value="{{ i.ipv4 }}">

circle/dashboard/tests/test_mockedviews.py

@@ -34,6 +34,13 @@ from ..views import AclUpdateView
 from .. import views
 
 
+class QuerySet(list):
+    model = MagicMock()
+
+    def get(self, *args, **kwargs):
+        return self.pop()
+
+
 class ViewUserTestCase(unittest.TestCase):
 
     def test_404(self):
@@ -145,58 +152,66 @@ class VmOperationViewTestCase(unittest.TestCase):
                 view.as_view()(request, pk=1234).render()
 
     def test_migrate(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=True)
+        request = FakeRequestFactory(
+            POST={'to_node': 1, 'live_migration': True}, superuser=True)
         view = vm_ops['migrate']
+        node = MagicMock(pk=1, name='node1')
 
         with patch.object(view, 'get_object') as go, \
                 patch('dashboard.views.util.messages') as msg, \
-                patch('dashboard.views.vm.get_object_or_404') as go4:
+                patch.object(view, 'get_form_kwargs') as form_kwargs:
             inst = MagicMock(spec=Instance)
             inst._meta.object_name = "Instance"
             inst.migrate = Instance._ops['migrate'](inst)
             inst.migrate.async = MagicMock()
             inst.has_level.return_value = True
+            form_kwargs.return_value = {
+                'default': 100, 'choices': QuerySet([node])}
             go.return_value = inst
-            go4.return_value = MagicMock()
             assert view.as_view()(request, pk=1234)['location']
             assert not msg.error.called
-            assert go4.called
+            inst.migrate.async.assert_called_once_with(
+                to_node=node, live_migration=True, user=request.user)
 
     def test_migrate_failed(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=True)
+        request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
         view = vm_ops['migrate']
+        node = MagicMock(pk=1, name='node1')
 
         with patch.object(view, 'get_object') as go, \
                 patch('dashboard.views.util.messages') as msg, \
-                patch('dashboard.views.vm.get_object_or_404') as go4:
+                patch.object(view, 'get_form_kwargs') as form_kwargs:
             inst = MagicMock(spec=Instance)
             inst._meta.object_name = "Instance"
             inst.migrate = Instance._ops['migrate'](inst)
             inst.migrate.async = MagicMock()
             inst.migrate.async.side_effect = Exception
             inst.has_level.return_value = True
+            form_kwargs.return_value = {
+                'default': 100, 'choices': QuerySet([node])}
             go.return_value = inst
-            go4.return_value = MagicMock()
             assert view.as_view()(request, pk=1234)['location']
+            assert inst.migrate.async.called
             assert msg.error.called
-            assert go4.called
 
     def test_migrate_wo_permission(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=False)
+        request = FakeRequestFactory(POST={'to_node': 1}, superuser=False)
         view = vm_ops['migrate']
+        node = MagicMock(pk=1, name='node1')
 
         with patch.object(view, 'get_object') as go, \
-                patch('dashboard.views.vm.get_object_or_404') as go4:
+                patch.object(view, 'get_form_kwargs') as form_kwargs:
             inst = MagicMock(spec=Instance)
             inst._meta.object_name = "Instance"
             inst.migrate = Instance._ops['migrate'](inst)
             inst.migrate.async = MagicMock()
             inst.has_level.return_value = True
+            form_kwargs.return_value = {
+                'default': 100, 'choices': QuerySet([node])}
             go.return_value = inst
-            go4.return_value = MagicMock()
             with self.assertRaises(PermissionDenied):
                 assert view.as_view()(request, pk=1234)['location']
-            assert go4.called
+            assert not inst.migrate.async.called
 
     def test_migrate_template(self):
         """check if GET dialog's template can be rendered"""
@@ -219,6 +234,7 @@ class VmOperationViewTestCase(unittest.TestCase):
         with patch.object(view, 'get_object') as go, \
                 patch('dashboard.views.util.messages') as msg:
             inst = MagicMock(spec=Instance)
+            inst.name = "asd"
             inst._meta.object_name = "Instance"
             inst.save_as_template = Instance._ops['save_as_template'](inst)
             inst.save_as_template.async = MagicMock()
@@ -235,6 +251,7 @@ class VmOperationViewTestCase(unittest.TestCase):
         with patch.object(view, 'get_object') as go, \
                 patch('dashboard.views.util.messages') as msg:
             inst = MagicMock(spec=Instance)
+            inst.name = "asd"
             inst._meta.object_name = "Instance"
             inst.save_as_template = Instance._ops['save_as_template'](inst)
             inst.save_as_template.async = MagicMock()
@@ -301,7 +318,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
                 view.as_view()(request, pk=1234).render()
 
     def test_migrate(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=True)
+        request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
         view = vm_mass_ops['migrate']
 
         with patch.object(view, 'get_object') as go, \
@@ -318,7 +335,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
             assert not msg2.error.called
 
     def test_migrate_failed(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=True)
+        request = FakeRequestFactory(POST={'to_node': 1}, superuser=True)
         view = vm_mass_ops['migrate']
 
         with patch.object(view, 'get_object') as go, \
@@ -334,7 +351,7 @@ class VmMassOperationViewTestCase(unittest.TestCase):
             assert msg.error.called
 
     def test_migrate_wo_permission(self):
-        request = FakeRequestFactory(POST={'node': 1}, superuser=False)
+        request = FakeRequestFactory(POST={'to_node': 1}, superuser=False)
         view = vm_mass_ops['migrate']
 
         with patch.object(view, 'get_object') as go:

circle/dashboard/tests/test_views.py

@@ -512,20 +512,20 @@ class VmDetailTest(LoginMixin, TestCase):
         self.login(c, "user2")
         with patch.object(Instance, 'select_node', return_value=None), \
                 patch.object(WakeUpOperation, 'async') as new_wake_up, \
-                patch('vm.tasks.vm_tasks.wake_up.apply_async') as wuaa, \
                 patch.object(Instance.WrongStateError, 'send_message') as wro:
             inst = Instance.objects.get(pk=1)
             new_wake_up.side_effect = inst.wake_up
+            inst._wake_up_vm = Mock()
             inst.get_remote_queue_name = Mock(return_value='test')
             inst.status = 'SUSPENDED'
             inst.set_level(self.u2, 'owner')
             with patch('dashboard.views.messages') as msg:
                 response = c.post("/dashboard/vm/1/op/wake_up/")
                 assert not msg.error.called
+            assert inst._wake_up_vm.called
             self.assertEqual(response.status_code, 302)
             self.assertEqual(inst.status, 'RUNNING')
             assert new_wake_up.called
-            assert wuaa.called
             assert not wro.called
 
     def test_unpermitted_wake_up(self):
@@ -1210,7 +1210,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         gp = self.g1.profile
         acl_users = len(gp.get_users_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'user3', 'level': 'owner'})
         self.assertEqual(acl_users, len(gp.get_users_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1221,7 +1221,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         gp = self.g1.profile
         acl_users = len(gp.get_users_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'user3', 'level': 'owner'})
         self.assertEqual(acl_users, len(gp.get_users_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1232,7 +1232,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         self.login(c, 'superuser')
         acl_users = len(gp.get_users_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'user3', 'level': 'owner'})
         self.assertEqual(acl_users + 1, len(gp.get_users_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1243,7 +1243,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         self.login(c, 'user0')
         acl_users = len(gp.get_users_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'user3', 'level': 'owner'})
         self.assertEqual(acl_users + 1, len(gp.get_users_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1253,7 +1253,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         gp = self.g1.profile
         acl_groups = len(gp.get_groups_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'group2', 'level': 'owner'})
         self.assertEqual(acl_groups, len(gp.get_groups_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1264,7 +1264,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         self.login(c, 'user3')
         acl_groups = len(gp.get_groups_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'group2', 'level': 'owner'})
         self.assertEqual(acl_groups, len(gp.get_groups_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1275,7 +1275,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         self.login(c, 'superuser')
         acl_groups = len(gp.get_groups_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'group2', 'level': 'owner'})
         self.assertEqual(acl_groups + 1, len(gp.get_groups_with_level()))
         self.assertEqual(response.status_code, 302)
@@ -1286,7 +1286,7 @@ class GroupDetailTest(LoginMixin, TestCase):
         self.login(c, 'user0')
         acl_groups = len(gp.get_groups_with_level())
         response = c.post('/dashboard/group/' +
-                          str(self.g1.pk) + '/acl/',
+                          str(gp.pk) + '/acl/',
                           {'name': 'group2', 'level': 'owner'})
         self.assertEqual(acl_groups + 1, len(gp.get_groups_with_level()))
         self.assertEqual(response.status_code, 302)

circle/dashboard/views/group.py

@@ -39,6 +39,7 @@ from ..forms import (
     GroupCreateForm, GroupProfileUpdateForm,
 )
 from ..models import FutureMember, GroupProfile
+from vm.models import Instance, InstanceTemplate
 from ..tables import GroupListTable
 from .util import CheckedDetailView, AclUpdateView, search_user, saml_available
 
@@ -100,6 +101,15 @@ class GroupDetailView(CheckedDetailView):
         context['group_profile_form'] = GroupProfileUpdate.get_form_object(
             self.request, self.object.profile)
 
+        context.update({
+            'group_objects': GroupProfile.get_objects_with_group_level(
+                "operator", self.get_object()),
+            'vm_objects': Instance.get_objects_with_group_level(
+                "user", self.get_object()),
+            'template_objects': InstanceTemplate.get_objects_with_group_level(
+                "user", self.get_object()),
+        })
+
         if self.request.user.is_superuser:
             context['group_perm_form'] = GroupPermissionForm(
                 instance=self.object)
@@ -180,10 +190,7 @@ class GroupPermissionsView(SuperuserRequiredMixin, UpdateView):
 
 
 class GroupAclUpdateView(AclUpdateView):
-    model = Group
-
-    def get_object(self):
-        return super(GroupAclUpdateView, self).get_object().profile
+    model = GroupProfile
 
 
 class GroupList(LoginRequiredMixin, SingleTableView):

circle/dashboard/views/node.py

@@ -68,6 +68,8 @@ node_ops = OrderedDict([
         op='passivate', icon='play-circle-o', effect='info')),
     ('disable', NodeOperationView.factory(
         op='disable', icon='times-circle-o', effect='danger')),
+    ('reset', NodeOperationView.factory(
+        op='reset', icon='stethoscope', effect='danger')),
     ('flush', NodeOperationView.factory(
         op='flush', icon='paint-brush', effect='danger')),
 ])

circle/dashboard/views/template.py

@@ -37,6 +37,7 @@ from braces.views import (
 from django_tables2 import SingleTableView
 
 from vm.models import InstanceTemplate, InterfaceTemplate, Instance, Lease
+from storage.models import Disk
 
 from ..forms import (
     TemplateForm, TemplateListSearchForm, AclUserOrGroupAddForm, LeaseForm,
@@ -319,6 +320,57 @@ class TemplateDetail(LoginRequiredMixin, SuccessMessageMixin, UpdateView):
         return kwargs
 
 
+class DiskRemoveView(DeleteView):
+    model = Disk
+
+    def get_queryset(self):
+        qs = super(DiskRemoveView, self).get_queryset()
+        return qs.exclude(template_set=None)
+
+    def get_template_names(self):
+        if self.request.is_ajax():
+            return ['dashboard/confirm/ajax-delete.html']
+        else:
+            return ['dashboard/confirm/base-delete.html']
+
+    def get_context_data(self, **kwargs):
+        context = super(DiskRemoveView, self).get_context_data(**kwargs)
+        disk = self.get_object()
+        template = disk.template_set.get()
+        if not template.has_level(self.request.user, 'owner'):
+            raise PermissionDenied()
+        context['title'] = _("Disk remove confirmation")
+        context['text'] = _("Are you sure you want to remove "
+                            "<strong>%(disk)s</strong> from "
+                            "<strong>%(app)s</strong>?" % {'disk': disk,
+                                                           'app': template}
+                            )
+        return context
+
+    def delete(self, request, *args, **kwargs):
+        disk = self.get_object()
+        template = disk.template_set.get()
+
+        if not template.has_level(request.user, 'owner'):
+            raise PermissionDenied()
+
+        template.remove_disk(disk=disk, user=request.user)
+        disk.destroy()
+
+        next_url = request.POST.get("next")
+        success_url = next_url if next_url else template.get_absolute_url()
+        success_message = _("Disk successfully removed.")
+
+        if request.is_ajax():
+            return HttpResponse(
+                json.dumps({'message': success_message}),
+                content_type="application/json",
+            )
+        else:
+            messages.success(request, success_message)
+            return HttpResponseRedirect("%s#resources" % success_url)
+
+
 class LeaseCreate(LoginRequiredMixin, PermissionRequiredMixin,
                   SuccessMessageMixin, CreateView):
     model = Lease

circle/dashboard/views/util.py

@@ -184,7 +184,7 @@ class OperationView(RedirectToLoginMixin, DetailView):
 
     @classmethod
     def get_urlname(cls):
-        return 'dashboard.vm.op.%s' % cls.op
+        return 'dashboard.%s.op.%s' % (cls.model._meta.model_name, cls.op)
 
     @classmethod
     def get_instance_url(cls, pk, key=None, *args, **kwargs):

circle/dashboard/views/vm.py

@@ -45,9 +45,10 @@ from common.models import (
     create_readable, HumanReadableException, fetch_human_exception,
 )
 from firewall.models import Vlan, Host, Rule
+from manager.scheduler import SchedulerError
 from storage.models import Disk
 from vm.models import (
-    Instance, instance_activity, InstanceActivity, Node, Lease,
+    Instance, InstanceActivity, Node, Lease,
     InstanceTemplate, InterfaceTemplate, Interface,
 )
 from .util import (
@@ -58,7 +59,8 @@ from ..forms import (
     AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
     VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
     VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
-    TransferOwnershipForm, VmDiskResizeForm,
+    TransferOwnershipForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
+    VmMigrateForm,
 )
 from ..models import Favourite, Profile
 
@@ -76,10 +78,10 @@ class VmDetailVncTokenView(CheckedDetailView):
         if not request.user.has_perm('vm.access_console'):
             raise PermissionDenied()
         if self.object.node:
-            with instance_activity(
-                    code_suffix='console-accessed', instance=self.object,
-                    user=request.user, readable_name=ugettext_noop(
-                        "console access"), concurrency_check=False):
+            with self.object.activity(
+                    code_suffix='console-accessed', user=request.user,
+                    readable_name=ugettext_noop("console access"),
+                    concurrency_check=False):
                 port = self.object.vnc_port
                 host = str(self.object.node.host.ipv4)
                 value = signing.dumps({'host': host, 'port': port},
@@ -97,6 +99,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
         context = super(VmDetailView, self).get_context_data(**kwargs)
         instance = context['instance']
         user = self.request.user
+        is_operator = instance.has_level(user, "operator")
+        is_owner = instance.has_level(user, "owner")
         ops = get_operations(instance, user)
         context.update({
             'graphite_enabled': settings.GRAPHITE_URL is not None,
@@ -152,9 +156,11 @@ class VmDetailView(GraphMixin, CheckedDetailView):
         context['client_download'] = self.request.COOKIES.get(
             'downloaded_client')
         # can link template
-        context['can_link_template'] = (
-            instance.template and instance.template.has_level(user, "operator")
-        )
+        context['can_link_template'] = instance.template and is_operator
+
+        # is operator/owner
+        context['is_operator'] = is_operator
+        context['is_owner'] = is_owner
 
         return context
 
@@ -174,7 +180,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
     def __set_name(self, request):
         self.object = self.get_object()
-        if not self.object.has_level(request.user, 'owner'):
+        if not self.object.has_level(request.user, "operator"):
             raise PermissionDenied()
         new_name = request.POST.get("new_name")
         Instance.objects.filter(pk=self.object.pk).update(
@@ -197,7 +203,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
     def __set_description(self, request):
         self.object = self.get_object()
-        if not self.object.has_level(request.user, 'owner'):
+        if not self.object.has_level(request.user, "operator"):
             raise PermissionDenied()
 
         new_description = request.POST.get("new_description")
@@ -221,7 +227,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
     def __add_tag(self, request):
         new_tag = request.POST.get('new_tag')
         self.object = self.get_object()
-        if not self.object.has_level(request.user, 'owner'):
+        if not self.object.has_level(request.user, "operator"):
             raise PermissionDenied()
 
         if len(new_tag) < 1:
@@ -243,7 +249,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
         try:
             to_remove = request.POST.get('to_remove')
             self.object = self.get_object()
-            if not self.object.has_level(request.user, 'owner'):
+            if not self.object.has_level(request.user, "operator"):
                 raise PermissionDenied()
 
             self.object.tags.remove(to_remove)
@@ -262,8 +268,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
     def __add_port(self, request):
         object = self.get_object()
-        if (not object.has_level(request.user, 'owner') or
-                not request.user.has_perm('vm.config_ports')):
+        if not (object.has_level(request.user, "operator") and
+                request.user.has_perm('vm.config_ports')):
             raise PermissionDenied()
 
         port = request.POST.get("port")
@@ -365,11 +371,9 @@ class VmAddInterfaceView(FormOperationMixin, VmOperationView):
         return val
 
 
-class VmDiskResizeView(FormOperationMixin, VmOperationView):
-
-    op = 'resize_disk'
-    form_class = VmDiskResizeForm
+class VmDiskModifyView(FormOperationMixin, VmOperationView):
     show_in_toolbar = False
+    with_reload = True
     icon = 'arrows-alt'
     effect = "success"
 
@@ -384,7 +388,7 @@ class VmDiskResizeView(FormOperationMixin, VmOperationView):
         else:
             default = None
 
-        val = super(VmDiskResizeView, self).get_form_kwargs()
+        val = super(VmDiskModifyView, self).get_form_kwargs()
         val.update({'choices': choices, 'default': default})
         return val
 
@@ -397,6 +401,14 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
     icon = 'hdd-o'
     effect = "success"
     is_disk_operation = True
+    with_reload = True
+
+    def get_form_kwargs(self):
+        op = self.get_op()
+        val = super(VmCreateDiskView, self).get_form_kwargs()
+        num = op.instance.disks.count() + 1
+        val['default'] = "%s %d" % (op.instance.name, num)
+        return val
 
 
 class VmDownloadDiskView(FormOperationMixin, VmOperationView):
@@ -407,29 +419,31 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
     icon = 'download'
     effect = "success"
     is_disk_operation = True
+    with_reload = True
 
 
-class VmMigrateView(VmOperationView):
+class VmMigrateView(FormOperationMixin, VmOperationView):
 
     op = 'migrate'
     icon = 'truck'
     effect = 'info'
     template_name = 'dashboard/_vm-migrate.html'
+    form_class = VmMigrateForm
 
-    def get_context_data(self, **kwargs):
-        ctx = super(VmMigrateView, self).get_context_data(**kwargs)
-        ctx['nodes'] = [n for n in Node.objects.filter(enabled=True)
-                        if n.online]
-        return ctx
+    def get_form_kwargs(self):
+        online = (n.pk for n in Node.objects.filter(enabled=True) if n.online)
+        choices = Node.objects.filter(pk__in=online)
+        default = None
+        inst = self.get_object()
+        try:
+            if isinstance(inst, Instance):
+                default = inst.select_node()
+        except SchedulerError:
+            logger.exception("scheduler error:")
 
-    def post(self, request, extra=None, *args, **kwargs):
-        if extra is None:
-            extra = {}
-        node = self.request.POST.get("node")
-        if node:
-            node = get_object_or_404(Node, pk=node)
-            extra["to_node"] = node
-        return super(VmMigrateView, self).post(request, extra, *args, **kwargs)
+        val = super(VmMigrateView, self).get_form_kwargs()
+        val.update({'choices': choices, 'default': default})
+        return val
 
 
 class VmSaveView(FormOperationMixin, VmOperationView):
@@ -439,6 +453,12 @@ class VmSaveView(FormOperationMixin, VmOperationView):
     effect = 'info'
     form_class = VmSaveForm
 
+    def get_form_kwargs(self):
+        op = self.get_op()
+        val = super(VmSaveView, self).get_form_kwargs()
+        val['default'] = op._rename(op.instance.name)
+        return val
+
 
 class VmResourcesChangeView(VmOperationView):
     op = 'resources_change'
@@ -599,6 +619,15 @@ class VmStateChangeView(FormOperationMixin, VmOperationView):
         return val
 
 
+class RedeployView(FormOperationMixin, VmOperationView):
+    op = 'redeploy'
+    icon = 'stethoscope'
+    effect = 'danger'
+    show_in_toolbar = True
+    form_class = RedeployForm
+    wait_for_result = 0.5
+
+
 vm_ops = OrderedDict([
     ('deploy', VmOperationView.factory(
         op='deploy', icon='play', effect='success')),
@@ -620,12 +649,18 @@ vm_ops = OrderedDict([
     ('recover', VmOperationView.factory(
         op='recover', icon='medkit', effect='warning')),
     ('nostate', VmStateChangeView),
+    ('redeploy', RedeployView),
     ('destroy', VmOperationView.factory(
         extra_bases=[TokenOperationView],
         op='destroy', icon='times', effect='danger')),
     ('create_disk', VmCreateDiskView),
     ('download_disk', VmDownloadDiskView),
-    ('resize_disk', VmDiskResizeView),
+    ('resize_disk', VmDiskModifyView.factory(
+        op='resize_disk', form_class=VmDiskResizeForm,
+        icon='arrows-alt', effect="warning")),
+    ('remove_disk', VmDiskModifyView.factory(
+        op='remove_disk', form_class=VmDiskRemoveForm,
+        icon='times', effect="danger")),
     ('add_interface', VmAddInterfaceView),
     ('renew', VmRenewView),
     ('resources_change', VmResourcesChangeView),
@@ -727,6 +762,12 @@ class MassOperationView(OperationView):
         self.check_auth()
         if extra is None:
             extra = {}
+
+        if hasattr(self, 'form_class'):
+            form = self.form_class(self.request.POST, **self.get_form_kwargs())
+            if form.is_valid():
+                extra.update(form.cleaned_data)
+
         self._call_operations(extra)
         if request.is_ajax():
             store = messages.get_messages(request)
@@ -765,6 +806,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
     allowed_filters = {
         'name': "name__icontains",
         'node': "node__name__icontains",
+        'node_exact': "node__name",
         'status': "status__iexact",
         'tags[]': "tags__name__in",
         'tags': "tags__name__in",  # for search string
@@ -850,10 +892,9 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
                 in [i.name for i in Instance._meta.fields] + ["pk"]):
             queryset = queryset.order_by(sort)
 
-        return queryset.filter(
-            **self.get_queryset_filters()).prefetch_related(
-                "owner", "node", "owner__profile", "interface_set", "lease",
-                "interface_set__host").distinct()
+        return queryset.filter(**self.get_queryset_filters()).prefetch_related(
+            "owner", "node", "owner__profile", "interface_set", "lease",
+            "interface_set__host").distinct()
 
 
 class VmCreate(LoginRequiredMixin, TemplateView):
@@ -1089,51 +1130,6 @@ class InstanceActivityDetail(CheckedDetailView):
         return ctx
 
 
-class DiskRemoveView(DeleteView):
-    model = Disk
-
-    def get_template_names(self):
-        if self.request.is_ajax():
-            return ['dashboard/confirm/ajax-delete.html']
-        else:
-            return ['dashboard/confirm/base-delete.html']
-
-    def get_context_data(self, **kwargs):
-        context = super(DiskRemoveView, self).get_context_data(**kwargs)
-        disk = self.get_object()
-        app = disk.get_appliance()
-        context['title'] = _("Disk remove confirmation")
-        context['text'] = _("Are you sure you want to remove "
-                            "<strong>%(disk)s</strong> from "
-                            "<strong>%(app)s</strong>?" % {'disk': disk,
-                                                           'app': app}
-                            )
-        return context
-
-    def delete(self, request, *args, **kwargs):
-        disk = self.get_object()
-        app = disk.get_appliance()
-
-        if not app.has_level(request.user, 'owner'):
-            raise PermissionDenied()
-
-        app.remove_disk(disk=disk, user=request.user)
-        disk.destroy()
-
-        next_url = request.POST.get("next")
-        success_url = next_url if next_url else app.get_absolute_url()
-        success_message = _("Disk successfully removed.")
-
-        if request.is_ajax():
-            return HttpResponse(
-                json.dumps({'message': success_message}),
-                content_type="application/json",
-            )
-        else:
-            messages.success(request, success_message)
-            return HttpResponseRedirect("%s#resources" % success_url)
-
-
 @require_GET
 def get_disk_download_status(request, pk):
     disk = Disk.objects.get(pk=pk)
@@ -1381,9 +1377,8 @@ class TransferOwnershipConfirmView(LoginRequiredMixin, View):
         instance, owner = self.get_instance(key, request.user)
 
         old = instance.owner
-        with instance_activity(code_suffix='ownership-transferred',
-                               concurrency_check=False,
-                               instance=instance, user=request.user):
+        with instance.activity(code_suffix='ownership-transferred',
+                               concurrency_check=False, user=request.user):
             instance.owner = request.user
             instance.clean()
             instance.save()

circle/fabfile.py

@@ -84,6 +84,10 @@ def make_messages():
 def test(test=""):
     "Run portal tests"
     with _workon("circle"), cd("~/circle/circle"):
+        if test == "f":
+            test = "--failed"
+        else:
+            test += " --with-id"
         run("./manage.py test --settings=circle.settings.test %s" % test)
 
 

circle/firewall/migrations/0052_auto__chg_field_record_address.py

+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Changing field 'Record.address'
+        db.alter_column(u'firewall_record', 'address', self.gf('django.db.models.fields.CharField')(max_length=400))
+
+    def backwards(self, orm):
+
+        # Changing field 'Record.address'
+        db.alter_column(u'firewall_record', 'address', self.gf('django.db.models.fields.CharField')(max_length=200))
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'firewall.blacklistitem': {
+            'Meta': {'object_name': 'BlacklistItem'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'host': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Host']", 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ipv4': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'reason': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'snort_message': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'default': "'tempban'", 'max_length': '10'})
+        },
+        u'firewall.domain': {
+            'Meta': {'object_name': 'Domain'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'ttl': ('django.db.models.fields.IntegerField', [], {'default': '600'})
+        },
+        u'firewall.ethernetdevice': {
+            'Meta': {'object_name': 'EthernetDevice'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
+            'switch_port': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ethernet_devices'", 'to': u"orm['firewall.SwitchPort']"})
+        },
+        u'firewall.firewall': {
+            'Meta': {'object_name': 'Firewall'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'})
+        },
+        u'firewall.group': {
+            'Meta': {'object_name': 'Group'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        u'firewall.host': {
+            'Meta': {'ordering': "('normalized_hostname', 'vlan')", 'unique_together': "(('hostname', 'vlan'),)", 'object_name': 'Host'},
+            'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'external_ipv4': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Group']", 'null': 'True', 'blank': 'True'}),
+            'hostname': ('django.db.models.fields.CharField', [], {'max_length': '40'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ipv4': ('firewall.fields.IPAddressField', [], {'unique': 'True', 'max_length': '100'}),
+            'ipv6': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'mac': ('firewall.fields.MACAddressField', [], {'unique': 'True', 'max_length': '17'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'normalized_hostname': ('common.models.HumanSortField', [], {'default': "''", 'maximum_number_length': '4', 'max_length': '80', 'monitor': "'hostname'", 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'reverse': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'shared_ip': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'vlan': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Vlan']"})
+        },
+        u'firewall.record': {
+            'Meta': {'ordering': "('domain', 'name')", 'object_name': 'Record'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '400'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Domain']"}),
+            'host': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Host']", 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'ttl': ('django.db.models.fields.IntegerField', [], {'default': '600'}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '6'})
+        },
+        u'firewall.rule': {
+            'Meta': {'ordering': "('direction', 'proto', 'sport', 'dport', 'nat_external_port', 'host')", 'object_name': 'Rule'},
+            'action': ('django.db.models.fields.CharField', [], {'default': "'drop'", 'max_length': '10'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'direction': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
+            'dport': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'extra': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'firewall': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Firewall']"}),
+            'foreign_network': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ForeignRules'", 'to': u"orm['firewall.VlanGroup']"}),
+            'host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Host']"}),
+            'hostgroup': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'nat': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'nat_external_ipv4': ('firewall.fields.IPAddressField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'nat_external_port': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+            'proto': ('django.db.models.fields.CharField', [], {'max_length': '10', 'null': 'True', 'blank': 'True'}),
+            'sport': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'vlan': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.Vlan']"}),
+            'vlangroup': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'rules'", 'null': 'True', 'to': u"orm['firewall.VlanGroup']"}),
+            'weight': ('django.db.models.fields.IntegerField', [], {'default': '30000'})
+        },
+        u'firewall.switchport': {
+            'Meta': {'object_name': 'SwitchPort'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'tagged_vlans': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'tagged_ports'", 'null': 'True', 'to': u"orm['firewall.VlanGroup']"}),
+            'untagged_vlan': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'untagged_ports'", 'to': u"orm['firewall.Vlan']"})
+        },
+        u'firewall.vlan': {
+            'Meta': {'object_name': 'Vlan'},
+            'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'dhcp_pool': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'domain': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['firewall.Domain']"}),
+            'host_ipv6_prefixlen': ('django.db.models.fields.IntegerField', [], {'default': '112'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ipv6_template': ('django.db.models.fields.TextField', [], {'default': "'2001:738:2001:4031:%(b)d:%(c)d:%(d)d:0'"}),
+            'managed': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
+            'network4': ('firewall.fields.IPNetworkField', [], {'max_length': '100'}),
+            'network6': ('firewall.fields.IPNetworkField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'network_type': ('django.db.models.fields.CharField', [], {'default': "'portforward'", 'max_length': '20'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+            'reverse_domain': ('django.db.models.fields.TextField', [], {'default': "'%(d)d.%(c)d.%(b)d.%(a)d.in-addr.arpa'"}),
+            'snat_ip': ('django.db.models.fields.GenericIPAddressField', [], {'max_length': '39', 'null': 'True', 'blank': 'True'}),
+            'snat_to': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Vlan']", 'null': 'True', 'blank': 'True'}),
+            'vid': ('django.db.models.fields.IntegerField', [], {'unique': 'True'})
+        },
+        u'firewall.vlangroup': {
+            'Meta': {'object_name': 'VlanGroup'},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '20'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}),
+            'vlans': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['firewall.Vlan']", 'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['firewall']
\ No newline at end of file

circle/firewall/models.py

@@ -874,7 +874,7 @@ class Record(models.Model):
                              verbose_name=_('host'))
     type = models.CharField(max_length=6, choices=CHOICES_type,
                             verbose_name=_('type'))
-    address = models.CharField(max_length=200,
+    address = models.CharField(max_length=400,
                                verbose_name=_('address'))
     ttl = models.IntegerField(default=600, verbose_name=_('ttl'))
     owner = models.ForeignKey(User, verbose_name=_('owner'))

circle/firewall/tasks/local_tasks.py

@@ -29,26 +29,24 @@ settings = django.conf.settings.FIREWALL_SETTINGS
 logger = getLogger(__name__)
 
 
-def _apply_once(name, queues, task, data):
+def _apply_once(name, tasks, queues, task, data):
     """Reload given networking component if needed.
     """
 
-    lockname = "%s_lock" % name
-    if not cache.get(lockname):
+    if name not in tasks:
         return
-    cache.delete(lockname)
 
     data = data()
     for queue in queues:
         try:
-            task.apply_async(args=data, queue=queue, expires=60).get(timeout=5)
+            task.apply_async(args=data, queue=queue, expires=60).get(timeout=2)
             logger.info("%s configuration is reloaded. (queue: %s)",
                         name, queue)
         except TimeoutError as e:
-            logger.critical('%s (queue: %s)', e, queue)
+            logger.critical('%s (queue: %s, task: %s)', e, queue, name)
         except:
-            logger.critical('Unhandled exception: queue: %s data: %s',
-                            queue, data, exc_info=True)
+            logger.critical('Unhandled exception: queue: %s data: %s task: %s',
+                            queue, data, name, exc_info=True)
 
 
 def get_firewall_queues():
@@ -68,19 +66,28 @@ def reloadtask_worker():
     from remote_tasks import (reload_dns, reload_dhcp, reload_firewall,
                               reload_firewall_vlan, reload_blacklist)
 
+    tasks = []
+    for i in ('dns', 'dhcp', 'firewall', 'firewall_vlan', 'blacklist'):
+        lockname = "%s_lock" % i
+        if cache.get(lockname):
+            tasks.append(i)
+        cache.delete(lockname)
+
+    logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
+
     firewall_queues = get_firewall_queues()
     dns_queues = [("%s.dns" % i) for i in
                   settings.get('dns_queues', [gethostname()])]
 
-    _apply_once('dns', dns_queues, reload_dns,
+    _apply_once('dns', tasks, dns_queues, reload_dns,
                 lambda: (dns(), ))
-    _apply_once('dhcp', firewall_queues, reload_dhcp,
+    _apply_once('dhcp', tasks, firewall_queues, reload_dhcp,
                 lambda: (dhcp(), ))
-    _apply_once('firewall', firewall_queues, reload_firewall,
+    _apply_once('firewall', tasks, firewall_queues, reload_firewall,
                 lambda: (BuildFirewall().build_ipt()))
-    _apply_once('firewall_vlan', firewall_queues, reload_firewall_vlan,
+    _apply_once('firewall_vlan', tasks, firewall_queues, reload_firewall_vlan,
                 lambda: (vlan(), ))
-    _apply_once('blacklist', firewall_queues, reload_blacklist,
+    _apply_once('blacklist', tasks, firewall_queues, reload_blacklist,
                 lambda: (list(ipset()), ))
 
 

circle/manager/mancelery.py

@@ -16,12 +16,15 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 from celery import Celery
+from celery.signals import worker_ready
 from datetime import timedelta
 from kombu import Queue, Exchange
 from os import getenv
 
 HOSTNAME = "localhost"
 CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
+QUEUE_NAME = HOSTNAME + '.man'
+
 
 celery = Celery('manager',
                 broker=getenv("AMQP_URI"),
@@ -57,3 +60,10 @@ celery.conf.update(
     }
 
 )
+
+
+@worker_ready.connect()
+def cleanup_tasks(conf=None, **kwargs):
+    '''Discard all task and clean up activity.'''
+    from vm.models.activity import cleanup
+    cleanup(queue_name=QUEUE_NAME)

circle/manager/moncelery.py

@@ -16,12 +16,14 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 from celery import Celery
+from celery.signals import worker_ready
 from datetime import timedelta
 from kombu import Queue, Exchange
 from os import getenv
 
 HOSTNAME = "localhost"
 CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
+QUEUE_NAME = HOSTNAME + '.monitor'
 
 celery = Celery('monitor',
                 broker=getenv("AMQP_URI"),
@@ -34,7 +36,7 @@ celery.conf.update(
     CELERY_CACHE_BACKEND=CACHE_URI,
     CELERY_TASK_RESULT_EXPIRES=300,
     CELERY_QUEUES=(
-        Queue(HOSTNAME + '.monitor', Exchange('monitor', type='direct'),
+        Queue(QUEUE_NAME, Exchange('monitor', type='direct'),
               routing_key="monitor"),
     ),
     CELERYBEAT_SCHEDULE={
@@ -70,3 +72,10 @@ celery.conf.update(
     }
 
 )
+
+
+@worker_ready.connect()
+def cleanup_tasks(conf=None, **kwargs):
+    '''Discard all task and clean up activity.'''
+    from vm.models.activity import cleanup
+    cleanup(queue_name=QUEUE_NAME)

circle/manager/slowcelery.py

@@ -16,12 +16,14 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 from celery import Celery
+from celery.signals import worker_ready
 from datetime import timedelta
 from kombu import Queue, Exchange
 from os import getenv
 
 HOSTNAME = "localhost"
 CACHE_URI = getenv("CACHE_URI", "pylibmc://127.0.0.1:11211/")
+QUEUE_NAME = HOSTNAME + '.man.slow'
 
 celery = Celery('manager.slow',
                 broker=getenv("AMQP_URI"),
@@ -36,7 +38,7 @@ celery.conf.update(
     CELERY_CACHE_BACKEND=CACHE_URI,
     CELERY_TASK_RESULT_EXPIRES=300,
     CELERY_QUEUES=(
-        Queue(HOSTNAME + '.man.slow', Exchange('manager.slow', type='direct'),
+        Queue(QUEUE_NAME, Exchange('manager.slow', type='direct'),
               routing_key="manager.slow"),
     ),
     CELERYBEAT_SCHEDULE={
@@ -48,3 +50,10 @@ celery.conf.update(
     }
 
 )
+
+
+@worker_ready.connect()
+def cleanup_tasks(conf=None, **kwargs):
+    '''Discard all task and clean up activity.'''
+    from vm.models.activity import cleanup
+    cleanup(queue_name=QUEUE_NAME)

circle/network/views.py

@@ -657,7 +657,7 @@ class VlanDetail(LoginRequiredMixin, SuperuserRequiredMixin,
         context = super(VlanDetail, self).get_context_data(**kwargs)
 
         q = Host.objects.filter(interface__in=Interface.objects.filter(
-            vlan=self.object, instance__destroyed_at=None
+            vlan=self.object
         ))
 
         context['host_list'] = SmallHostTable(q)

circle/storage/models.py

@@ -490,6 +490,9 @@ class Disk(TimeStampedModel):
                     disk.destroy()
                     raise humanize_exception(ugettext_noop(
                         "Operation aborted by user."), e)
+            except:
+                disk.destroy()
+                raise
         disk.is_ready = True
         disk.save()
         return disk

circle/vm/models/__init__.py

 # flake8: noqa
 from .activity import InstanceActivity
-from .activity import instance_activity
 from .activity import NodeActivity
 from .activity import node_activity
 from .common import BaseResourceConfigModel
 from .common import Lease
 from .common import NamedBaseResourceConfig
 from .common import Trait
-from .instance import InstanceActiveManager
 from .instance import VirtualMachineDescModel
 from .instance import InstanceTemplate
 from .instance import Instance
@@ -19,9 +17,9 @@ from .network import Interface
 from .node import Node
 
 __all__ = [
-    'InstanceActivity', 'InstanceActiveManager', 'BaseResourceConfigModel',
+    'InstanceActivity', 'BaseResourceConfigModel',
     'NamedBaseResourceConfig', 'VirtualMachineDescModel', 'InstanceTemplate',
-    'Instance', 'instance_activity', 'post_state_changed', 'pre_state_changed',
-    'InterfaceTemplate', 'Interface', 'Trait', 'Node', 'NodeActivity', 'Lease',
-    'node_activity', 'pwgen'
+    'Instance', 'post_state_changed', 'pre_state_changed', 'InterfaceTemplate',
+    'Interface', 'Trait', 'Node', 'NodeActivity', 'Lease', 'node_activity',
+    'pwgen'
 ]

circle/vm/models/activity.py

@@ -20,7 +20,6 @@ from contextlib import contextmanager
 from logging import getLogger
 from warnings import warn
 
-from celery.signals import worker_ready
 from celery.contrib.abortable import AbortableAsyncResult
 
 from django.core.urlresolvers import reverse
@@ -206,21 +205,6 @@ class InstanceActivity(ActivityModel):
             self.activity_code)
 
 
-@contextmanager
-def instance_activity(code_suffix, instance, on_abort=None, on_commit=None,
-                      task_uuid=None, user=None, concurrency_check=True,
-                      readable_name=None, resultant_state=None):
-    """Create a transactional context for an instance activity.
-    """
-    if not readable_name:
-        warn("Set readable_name", stacklevel=3)
-    act = InstanceActivity.create(code_suffix, instance, task_uuid, user,
-                                  concurrency_check,
-                                  readable_name=readable_name,
-                                  resultant_state=resultant_state)
-    return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
-
-
 class NodeActivity(ActivityModel):
     ACTIVITY_CODE_BASE = join_activity_code('vm', 'Node')
     node = ForeignKey('Node', related_name='activity_log',
@@ -278,17 +262,17 @@ def node_activity(code_suffix, node, task_uuid=None, user=None,
     return activitycontextimpl(act)
 
 
-@worker_ready.connect()
 def cleanup(conf=None, **kwargs):
     # TODO check if other manager workers are running
-    from celery.task.control import discard_all
-    discard_all()
     msg_txt = ugettext_noop("Manager is restarted, activity is cleaned up. "
                             "You can try again now.")
     message = create_readable(msg_txt, msg_txt)
+    queue_name = kwargs.get('queue_name', None)
     for i in InstanceActivity.objects.filter(finished__isnull=True):
-        i.finish(False, result=message)
-        logger.error('Forced finishing stale activity %s', i)
+        op = i.get_operation()
+        if op and op.async_queue == queue_name:
+            i.finish(False, result=message)
+            logger.error('Forced finishing stale activity %s', i)
     for i in NodeActivity.objects.filter(finished__isnull=True):
         i.finish(False, result=message)
         logger.error('Forced finishing stale activity %s', i)

circle/vm/models/instance.py

@@ -16,15 +16,13 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import absolute_import, unicode_literals
+from contextlib import contextmanager
 from datetime import timedelta
 from functools import partial
 from importlib import import_module
 from logging import getLogger
-from string import ascii_lowercase
 from warnings import warn
 
-from celery.exceptions import TimeoutError
-from celery.contrib.abortable import AbortableAsyncResult
 import django.conf
 from django.contrib.auth.models import User
 from django.core import signing
@@ -38,17 +36,17 @@ from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 
 from model_utils import Choices
+from model_utils.managers import QueryManager
 from model_utils.models import TimeStampedModel, StatusModel
 from taggit.managers import TaggableManager
 
 from acl.models import AclBase
 from common.models import (
-    create_readable, HumanReadableException, humanize_exception
+    activitycontextimpl, create_readable, HumanReadableException,
 )
 from common.operations import OperatedMixin
-from ..tasks import vm_tasks, agent_tasks
-from .activity import (ActivityInProgressError, instance_activity,
-                       InstanceActivity)
+from ..tasks import agent_tasks
+from .activity import (ActivityInProgressError, InstanceActivity)
 from .common import BaseResourceConfigModel, Lease
 from .network import Interface
 from .node import Node, Trait
@@ -92,13 +90,6 @@ def find_unused_vnc_port():
         return port
 
 
-class InstanceActiveManager(Manager):
-
-    def get_query_set(self):
-        return super(InstanceActiveManager,
-                     self).get_query_set().filter(destroyed_at=None)
-
-
 class VirtualMachineDescModel(BaseResourceConfigModel):
 
     """Abstract base for virtual machine describing models.
@@ -264,7 +255,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
                                  help_text=_("The virtual machine's time of "
                                              "destruction."))
     objects = Manager()
-    active = InstanceActiveManager()
+    active = QueryManager(destroyed_at=None)
 
     class Meta:
         app_label = 'vm'
@@ -275,6 +266,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
             ('change_resources', _('Can change resources of a running VM.')),
             ('set_resources', _('Can change resources of a new VM.')),
             ('create_vm', _('Can create a new VM.')),
+            ('redeploy', _('Can redeploy a VM.')),
             ('config_ports', _('Can configure port forwards.')),
             ('recover', _('Can recover a destroyed VM.')),
             ('emergency_change_state', _('Can change VM state to NOSTATE.')),
@@ -373,9 +365,9 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         def __on_commit(activity):
             activity.resultant_state = 'PENDING'
 
-        with instance_activity(code_suffix='create', instance=inst,
-                               readable_name=ugettext_noop("create instance"),
-                               on_commit=__on_commit, user=inst.owner) as act:
+        with inst.activity(code_suffix='create',
+                           readable_name=ugettext_noop("create instance"),
+                           on_commit=__on_commit, user=inst.owner) as act:
             # create related entities
             inst.disks.add(*[disk.get_exclusive() for disk in disks])
 
@@ -676,10 +668,10 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
                     "%(success)s notifications succeeded."),
                     success=len(success), successes=success)
 
-        with instance_activity('notification_about_expiration', instance=self,
-                               readable_name=ugettext_noop(
-                                   "notify owner about expiration"),
-                               on_commit=on_commit):
+        with self.activity('notification_about_expiration',
+                           readable_name=ugettext_noop(
+                               "notify owner about expiration"),
+                           on_commit=on_commit):
             from dashboard.views import VmRenewView, absolute_url
             level = self.get_level_object("owner")
             for u, ulevel in self.get_users_with_level(level__pk=level.pk):
@@ -744,75 +736,6 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         """
         return scheduler.select_node(self, Node.objects.all())
 
-    def attach_disk(self, disk, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'fast')
-        return vm_tasks.attach_disk.apply_async(
-            args=[self.vm_name,
-                  disk.get_vmdisk_desc()],
-            queue=queue_name
-        ).get(timeout=timeout)
-
-    def detach_disk(self, disk, timeout=15):
-        try:
-            queue_name = self.get_remote_queue_name('vm', 'fast')
-            return vm_tasks.detach_disk.apply_async(
-                args=[self.vm_name,
-                      disk.get_vmdisk_desc()],
-                queue=queue_name
-            ).get(timeout=timeout)
-        except Exception as e:
-            if e.libvirtError and "not found" in str(e):
-                logger.debug("Disk %s was not found."
-                             % disk.name)
-            else:
-                raise
-
-    def attach_network(self, network, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'fast')
-        return vm_tasks.attach_network.apply_async(
-            args=[self.vm_name,
-                  network.get_vmnetwork_desc()],
-            queue=queue_name
-        ).get(timeout=timeout)
-
-    def detach_network(self, network, timeout=15):
-        try:
-            queue_name = self.get_remote_queue_name('vm', 'fast')
-            return vm_tasks.detach_network.apply_async(
-                args=[self.vm_name,
-                      network.get_vmnetwork_desc()],
-                queue=queue_name
-            ).get(timeout=timeout)
-        except Exception as e:
-            if e.libvirtError and "not found" in str(e):
-                logger.debug("Interface %s was not found."
-                             % (network.__unicode__()))
-            else:
-                raise
-
-    def resize_disk_live(self, disk, size, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        result = vm_tasks.resize_disk.apply_async(
-            args=[self.vm_name, disk.path, size],
-            queue=queue_name).get(timeout=timeout)
-        disk.size = size
-        disk.save()
-        return result
-
-    def deploy_disks(self):
-        """Deploy all associated disks.
-        """
-        devnums = list(ascii_lowercase)  # a-z
-        for disk in self.disks.all():
-            # assign device numbers
-            if disk.dev_num in devnums:
-                devnums.remove(disk.dev_num)
-            else:
-                disk.dev_num = devnums.pop(0)
-                disk.save()
-            # deploy disk
-            disk.deploy()
-
     def destroy_disks(self):
         """Destroy all associated disks.
         """
@@ -837,92 +760,11 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         for net in self.interface_set.all():
             net.shutdown()
 
-    def delete_vm(self, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'fast')
-        try:
-            return vm_tasks.destroy.apply_async(args=[self.vm_name],
-                                                queue=queue_name
-                                                ).get(timeout=timeout)
-        except Exception as e:
-            if e.libvirtError and "Domain not found" in str(e):
-                logger.debug("Domain %s was not found at %s"
-                             % (self.vm_name, queue_name))
-            else:
-                raise
-
-    def deploy_vm(self, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        return vm_tasks.deploy.apply_async(args=[self.get_vm_desc()],
-                                           queue=queue_name
-                                           ).get(timeout=timeout)
-
-    def migrate_vm(self, to_node, timeout=120):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        return vm_tasks.migrate.apply_async(args=[self.vm_name,
-                                                  to_node.host.hostname,
-                                                  True],
-                                            queue=queue_name
-                                            ).get(timeout=timeout)
-
-    def reboot_vm(self, timeout=5):
-        queue_name = self.get_remote_queue_name('vm', 'fast')
-        return vm_tasks.reboot.apply_async(args=[self.vm_name],
-                                           queue=queue_name
-                                           ).get(timeout=timeout)
-
-    def reset_vm(self, timeout=5):
-        queue_name = self.get_remote_queue_name('vm', 'fast')
-        return vm_tasks.reset.apply_async(args=[self.vm_name],
-                                          queue=queue_name
-                                          ).get(timeout=timeout)
-
-    def resume_vm(self, timeout=15):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        return vm_tasks.resume.apply_async(args=[self.vm_name],
-                                           queue=queue_name
-                                           ).get(timeout=timeout)
-
-    def shutdown_vm(self, task=None, step=5):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        logger.debug("RPC Shutdown at queue: %s, for vm: %s.", queue_name,
-                     self.vm_name)
-        remote = vm_tasks.shutdown.apply_async(kwargs={'name': self.vm_name},
-                                               queue=queue_name)
-
-        while True:
-            try:
-                return remote.get(timeout=step)
-            except TimeoutError as e:
-                if task is not None and task.is_aborted():
-                    AbortableAsyncResult(remote.id).abort()
-                    raise humanize_exception(ugettext_noop(
-                        "Operation aborted by user."), e)
-
-    def suspend_vm(self, timeout=230):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        return vm_tasks.sleep.apply_async(args=[self.vm_name,
-                                                self.mem_dump['path']],
-                                          queue=queue_name
-                                          ).get(timeout=timeout)
-
-    def wake_up_vm(self, timeout=60):
-        queue_name = self.get_remote_queue_name('vm', 'slow')
-        return vm_tasks.wake_up.apply_async(args=[self.vm_name,
-                                                  self.mem_dump['path']],
-                                            queue=queue_name
-                                            ).get(timeout=timeout)
-
-    def delete_mem_dump(self, timeout=15):
-        queue_name = self.mem_dump['datastore'].get_remote_queue_name(
-            'storage', 'fast')
-        from storage.tasks.storage_tasks import delete_dump
-        delete_dump.apply_async(args=[self.mem_dump['path']],
-                                queue=queue_name).get(timeout=timeout)
-
     def allocate_node(self):
         if self.node is None:
             self.node = self.select_node()
             self.save()
+            return self.node
 
     def yield_node(self):
         if self.node is not None:
@@ -995,12 +837,6 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
 
         return merged_acts
 
-    def get_screenshot(self, timeout=5):
-        queue_name = self.get_remote_queue_name("vm", "fast")
-        return vm_tasks.screenshot.apply_async(args=[self.vm_name],
-                                               queue=queue_name
-                                               ).get(timeout=timeout)
-
     def get_latest_activity_in_progress(self):
         try:
             return InstanceActivity.objects.filter(
@@ -1016,3 +852,17 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
     @property
     def metric_prefix(self):
         return 'vm.%s' % self.vm_name
+
+    @contextmanager
+    def activity(self, code_suffix, readable_name, on_abort=None,
+                 on_commit=None, task_uuid=None, user=None,
+                 concurrency_check=True, resultant_state=None):
+        """Create a transactional context for an instance activity.
+        """
+        if not readable_name:
+            warn("Set readable_name", stacklevel=3)
+        act = InstanceActivity.create(
+            code_suffix=code_suffix, instance=self, task_uuid=task_uuid,
+            user=user, concurrency_check=concurrency_check,
+            readable_name=readable_name, resultant_state=resultant_state)
+        return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)

circle/vm/models/network.py

@@ -26,7 +26,6 @@ from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from common.models import create_readable
 from firewall.models import Vlan, Host
 from ..tasks import net_tasks
-from .activity import instance_activity
 
 logger = getLogger(__name__)
 
@@ -120,10 +119,10 @@ class Interface(Model):
             host.hostname = instance.vm_name
             # Get addresses from firewall
             if base_activity is None:
-                act_ctx = instance_activity(
+                act_ctx = instance.activity(
                     code_suffix='allocating_ip',
                     readable_name=ugettext_noop("allocate IP address"),
-                    instance=instance, user=owner)
+                    user=owner)
             else:
                 act_ctx = base_activity.sub_activity(
                     'allocating_ip',

circle/vm/models/node.py

@@ -114,8 +114,8 @@ class Node(OperatedMixin, TimeStampedModel):
     def get_info(self):
         return self.remote_query(vm_tasks.get_info,
                                  priority='fast',
-                                 default={'core_num': '',
-                                          'ram_size': '0',
+                                 default={'core_num': 0,
+                                          'ram_size': 0,
                                           'architecture': ''})
 
     info = property(get_info)
@@ -313,10 +313,11 @@ class Node(OperatedMixin, TimeStampedModel):
     def get_status_label(self):
         return {
             'OFFLINE': 'label-warning',
-            'DISABLED': 'label-warning',
+            'DISABLED': 'label-danger',
             'MISSING': 'label-danger',
-            'ONLINE': 'label-success'}.get(self.get_state(),
-                                           'label-danger')
+            'ACTIVE': 'label-success',
+            'PASSIVE': 'label-warning',
+        }.get(self.get_state(), 'label-danger')
 
     @node_available
     def update_vm_states(self):

circle/vm/operations.py

@@ -28,12 +28,13 @@ from django.conf import settings
 
 from sizefield.utils import filesizeformat
 
-from celery.exceptions import TimeLimitExceeded
+from celery.contrib.abortable import AbortableAsyncResult
+from celery.exceptions import TimeLimitExceeded, TimeoutError
 
 from common.models import (
     create_readable, humanize_exception, HumanReadableException
 )
-from common.operations import Operation, register_operation
+from common.operations import Operation, register_operation, SubOperationMixin
 from manager.scheduler import SchedulerError
 from .tasks.local_tasks import (
     abortable_async_instance_operation, abortable_async_node_operation,
@@ -42,13 +43,46 @@ from .models import (
     Instance, InstanceActivity, InstanceTemplate, Interface, Node,
     NodeActivity, pwgen
 )
-from .tasks import agent_tasks, local_agent_tasks
+from .tasks import agent_tasks, local_agent_tasks, vm_tasks
 
 from dashboard.store_api import Store, NoStoreException
+from storage.tasks import storage_tasks
 
 logger = getLogger(__name__)
 
 
+class RemoteOperationMixin(object):
+
+    remote_timeout = 30
+
+    def _operation(self, **kwargs):
+        args = self._get_remote_args(**kwargs)
+        return self.task.apply_async(
+            args=args, queue=self._get_remote_queue()
+        ).get(timeout=self.remote_timeout)
+
+    def check_precond(self):
+        super(RemoteOperationMixin, self).check_precond()
+        self._get_remote_queue()
+
+
+class AbortableRemoteOperationMixin(object):
+    remote_step = property(lambda self: self.remote_timeout / 10)
+
+    def _operation(self, task, **kwargs):
+        args = self._get_remote_args(**kwargs),
+        remote = self.task.apply_async(
+            args=args, queue=self._get_remote_queue())
+        for i in xrange(0, self.remote_timeout, self.remote_step):
+            try:
+                return remote.get(timeout=self.remote_step)
+            except TimeoutError as e:
+                if task is not None and task.is_aborted():
+                    AbortableAsyncResult(remote.id).abort()
+                    raise humanize_exception(ugettext_noop(
+                        "Operation aborted by user."), e)
+
+
 class InstanceOperation(Operation):
     acl_level = 'owner'
     async_operation = abortable_async_instance_operation
@@ -100,12 +134,13 @@ class InstanceOperation(Operation):
                                  "parent activity does not match the user "
                                  "provided as parameter.")
 
-            return parent.create_sub(code_suffix=self.activity_code_suffix,
-                                     readable_name=name,
-                                     resultant_state=self.resultant_state)
+            return parent.create_sub(
+                code_suffix=self.get_activity_code_suffix(),
+                readable_name=name, resultant_state=self.resultant_state)
         else:
             return InstanceActivity.create(
-                code_suffix=self.activity_code_suffix, instance=self.instance,
+                code_suffix=self.get_activity_code_suffix(),
+                instance=self.instance,
                 readable_name=name, user=user,
                 concurrency_check=self.concurrency_check,
                 resultant_state=self.resultant_state)
@@ -116,9 +151,19 @@ class InstanceOperation(Operation):
         return False
 
 
+class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
+
+    remote_queue = ('vm', 'fast')
+
+    def _get_remote_queue(self):
+        return self.instance.get_remote_queue_name(*self.remote_queue)
+
+    def _get_remote_args(self, **kwargs):
+        return [self.instance.vm_name]
+
+
 @register_operation
 class AddInterfaceOperation(InstanceOperation):
-    activity_code_suffix = 'add_interface'
     id = 'add_interface'
     name = _("add interface")
     description = _("Add a new network interface for the specified VLAN to "
@@ -146,10 +191,8 @@ class AddInterfaceOperation(InstanceOperation):
 
         if self.instance.is_running:
             try:
-                with activity.sub_activity(
-                    'attach_network',
-                        readable_name=ugettext_noop("attach network")):
-                    self.instance.attach_network(net)
+                self.instance._attach_network(
+                    interface=net, parent_activity=activity)
             except Exception as e:
                 if hasattr(e, 'libvirtError'):
                     self.rollback(net, activity)
@@ -165,7 +208,6 @@ class AddInterfaceOperation(InstanceOperation):
 @register_operation
 class CreateDiskOperation(InstanceOperation):
 
-    activity_code_suffix = 'create_disk'
     id = 'create_disk'
     name = _("create disk")
     description = _("Create and attach empty disk to the virtual machine.")
@@ -192,11 +234,7 @@ class CreateDiskOperation(InstanceOperation):
                 readable_name=ugettext_noop("deploying disk")
             ):
                 disk.deploy()
-            with activity.sub_activity(
-                'attach_disk',
-                readable_name=ugettext_noop("attach disk")
-            ):
-                self.instance.attach_disk(disk)
+            self.instance._attach_disk(parent_activity=activity, disk=disk)
 
     def get_activity_name(self, kwargs):
         return create_readable(
@@ -205,9 +243,8 @@ class CreateDiskOperation(InstanceOperation):
 
 
 @register_operation
-class ResizeDiskOperation(InstanceOperation):
+class ResizeDiskOperation(RemoteInstanceOperation):
 
-    activity_code_suffix = 'resize_disk'
     id = 'resize_disk'
     name = _("resize disk")
     description = _("Resize the virtual disk image. "
@@ -215,9 +252,12 @@ class ResizeDiskOperation(InstanceOperation):
     required_perms = ('storage.resize_disk', )
     accept_states = ('RUNNING', )
     async_queue = "localhost.man.slow"
+    remote_queue = ('vm', 'slow')
+    task = vm_tasks.resize_disk
 
-    def _operation(self, user, disk, size, activity):
-        self.instance.resize_disk_live(disk, size)
+    def _get_remote_args(self, disk, size, **kwargs):
+        return (super(ResizeDiskOperation, self)
+                ._get_remote_args(**kwargs) + [disk.path, size])
 
     def get_activity_name(self, kwargs):
         return create_readable(
@@ -227,7 +267,6 @@ class ResizeDiskOperation(InstanceOperation):
 
 @register_operation
 class DownloadDiskOperation(InstanceOperation):
-    activity_code_suffix = 'download_disk'
     id = 'download_disk'
     name = _("download disk")
     description = _("Download and attach disk image (ISO file) for the "
@@ -257,16 +296,11 @@ class DownloadDiskOperation(InstanceOperation):
 
         # TODO iso (cd) hot-plug is not supported by kvm/guests
         if self.instance.is_running and disk.type not in ["iso"]:
-            with activity.sub_activity(
-                'attach_disk',
-                readable_name=ugettext_noop("attach disk")
-            ):
-                self.instance.attach_disk(disk)
+            self.instance._attach_disk(parent_activity=activity, disk=disk)
 
 
 @register_operation
 class DeployOperation(InstanceOperation):
-    activity_code_suffix = 'deploy'
     id = 'deploy'
     name = _("deploy")
     description = _("Deploy and start the virtual machine (including storage "
@@ -290,25 +324,17 @@ class DeployOperation(InstanceOperation):
                           "deployed to node: %(node)s"),
             node=self.instance.node)
 
-    def _operation(self, activity, timeout=15):
+    def _operation(self, activity):
         # Allocate VNC port and host node
         self.instance.allocate_vnc_port()
         self.instance.allocate_node()
 
         # Deploy virtual images
-        with activity.sub_activity(
-            'deploying_disks', readable_name=ugettext_noop(
-                "deploy disks")):
-            self.instance.deploy_disks()
+        self.instance._deploy_disks(parent_activity=activity)
 
         # Deploy VM on remote machine
         if self.instance.state not in ['PAUSED']:
-            rn = create_readable(ugettext_noop("deploy virtual machine"),
-                                 ugettext_noop("deploy vm to %(node)s"),
-                                 node=self.instance.node)
-            with activity.sub_activity(
-                    'deploying_vm', readable_name=rn) as deploy_act:
-                deploy_act.result = self.instance.deploy_vm(timeout=timeout)
+            self.instance._deploy_vm(parent_activity=activity)
 
         # Establish network connection (vmdriver)
         with activity.sub_activity(
@@ -321,20 +347,57 @@ class DeployOperation(InstanceOperation):
         except:
             pass
 
-        # Resume vm
-        with activity.sub_activity(
-            'booting', readable_name=ugettext_noop(
-                "boot virtual machine")):
-            self.instance.resume_vm(timeout=timeout)
+        self.instance._resume_vm(parent_activity=activity)
 
         if self.instance.has_agent:
             activity.sub_activity('os_boot', readable_name=ugettext_noop(
                 "wait operating system loading"), interruptible=True)
 
+    @register_operation
+    class DeployVmOperation(SubOperationMixin, RemoteInstanceOperation):
+        id = "_deploy_vm"
+        name = _("deploy vm")
+        description = _("Deploy virtual machine.")
+        remote_queue = ("vm", "slow")
+        task = vm_tasks.deploy
+
+        def _get_remote_args(self, **kwargs):
+            return [self.instance.get_vm_desc()]
+            # intentionally not calling super
+
+        def get_activity_name(self, kwargs):
+            return create_readable(ugettext_noop("deploy virtual machine"),
+                                   ugettext_noop("deploy vm to %(node)s"),
+                                   node=self.instance.node)
+
+    @register_operation
+    class DeployDisksOperation(SubOperationMixin, InstanceOperation):
+        id = "_deploy_disks"
+        name = _("deploy disks")
+        description = _("Deploy all associated disks.")
+
+        def _operation(self):
+            devnums = list(ascii_lowercase)  # a-z
+            for disk in self.instance.disks.all():
+                # assign device numbers
+                if disk.dev_num in devnums:
+                    devnums.remove(disk.dev_num)
+                else:
+                    disk.dev_num = devnums.pop(0)
+                    disk.save()
+                # deploy disk
+                disk.deploy()
+
+    @register_operation
+    class ResumeVmOperation(SubOperationMixin, RemoteInstanceOperation):
+        id = "_resume_vm"
+        name = _("boot virtual machine")
+        remote_queue = ("vm", "slow")
+        task = vm_tasks.resume
+
 
 @register_operation
 class DestroyOperation(InstanceOperation):
-    activity_code_suffix = 'destroy'
     id = 'destroy'
     name = _("destroy")
     description = _("Permanently destroy virtual machine, its network "
@@ -342,7 +405,7 @@ class DestroyOperation(InstanceOperation):
     required_perms = ()
     resultant_state = 'DESTROYED'
 
-    def _operation(self, activity):
+    def _operation(self, activity, system):
         # Destroy networks
         with activity.sub_activity(
                 'destroying_net',
@@ -352,11 +415,7 @@ class DestroyOperation(InstanceOperation):
             self.instance.destroy_net()
 
         if self.instance.node:
-            # Delete virtual machine
-            with activity.sub_activity(
-                    'destroying_vm',
-                    readable_name=ugettext_noop("destroy virtual machine")):
-                self.instance.delete_vm()
+            self.instance._delete_vm(parent_activity=activity)
 
         # Destroy disks
         with activity.sub_activity(
@@ -366,7 +425,7 @@ class DestroyOperation(InstanceOperation):
 
         # Delete mem. dump if exists
         try:
-            self.instance.delete_mem_dump()
+            self.instance._delete_mem_dump(parent_activity=activity)
         except:
             pass
 
@@ -377,10 +436,30 @@ class DestroyOperation(InstanceOperation):
         self.instance.destroyed_at = timezone.now()
         self.instance.save()
 
+    @register_operation
+    class DeleteVmOperation(SubOperationMixin, RemoteInstanceOperation):
+        id = "_delete_vm"
+        name = _("destroy virtual machine")
+        task = vm_tasks.destroy
+        # if e.libvirtError and "Domain not found" in str(e):
+
+    @register_operation
+    class DeleteMemDumpOperation(RemoteOperationMixin, SubOperationMixin,
+                                 InstanceOperation):
+        id = "_delete_mem_dump"
+        name = _("removing memory dump")
+        task = storage_tasks.delete_dump
+
+        def _get_remote_queue(self):
+            return self.instance.mem_dump['datastore'].get_remote_queue_name(
+                "storage", "fast")
+
+        def _get_remote_args(self, **kwargs):
+            return [self.instance.mem_dump['path']]
+
 
 @register_operation
-class MigrateOperation(InstanceOperation):
-    activity_code_suffix = 'migrate'
+class MigrateOperation(RemoteInstanceOperation):
     id = 'migrate'
     name = _("migrate")
     description = _("Move virtual machine to an other worker node with a few "
@@ -389,6 +468,13 @@ class MigrateOperation(InstanceOperation):
     superuser_required = True
     accept_states = ('RUNNING', )
     async_queue = "localhost.man.slow"
+    task = vm_tasks.migrate
+    remote_queue = ("vm", "slow")
+    remote_timeout = 1000
+
+    def _get_remote_args(self, to_node, live_migration, **kwargs):
+        return (super(MigrateOperation, self)._get_remote_args(**kwargs)
+                + [to_node.host.hostname, live_migration])
 
     def rollback(self, activity):
         with activity.sub_activity(
@@ -396,7 +482,7 @@ class MigrateOperation(InstanceOperation):
                 "redeploy network (rollback)")):
             self.instance.deploy_net()
 
-    def _operation(self, activity, to_node=None, timeout=120):
+    def _operation(self, activity, to_node=None, live_migration=True):
         if not to_node:
             with activity.sub_activity('scheduling',
                                        readable_name=ugettext_noop(
@@ -408,7 +494,8 @@ class MigrateOperation(InstanceOperation):
             with activity.sub_activity(
                 'migrate_vm', readable_name=create_readable(
                     ugettext_noop("migrate to %(node)s"), node=to_node)):
-                self.instance.migrate_vm(to_node=to_node, timeout=timeout)
+                super(MigrateOperation, self)._operation(
+                    to_node=to_node, live_migration=live_migration)
         except Exception as e:
             if hasattr(e, 'libvirtError'):
                 self.rollback(activity)
@@ -423,6 +510,7 @@ class MigrateOperation(InstanceOperation):
         # Refresh node information
         self.instance.node = to_node
         self.instance.save()
+
         # Estabilish network connection (vmdriver)
         with activity.sub_activity(
             'deploying_net', readable_name=ugettext_noop(
@@ -431,17 +519,17 @@ class MigrateOperation(InstanceOperation):
 
 
 @register_operation
-class RebootOperation(InstanceOperation):
-    activity_code_suffix = 'reboot'
+class RebootOperation(RemoteInstanceOperation):
     id = 'reboot'
     name = _("reboot")
     description = _("Warm reboot virtual machine by sending Ctrl+Alt+Del "
                     "signal to its console.")
     required_perms = ()
     accept_states = ('RUNNING', )
+    task = vm_tasks.reboot
 
-    def _operation(self, activity, timeout=5):
-        self.instance.reboot_vm(timeout=timeout)
+    def _operation(self, activity):
+        super(RebootOperation, self)._operation()
         if self.instance.has_agent:
             activity.sub_activity('os_boot', readable_name=ugettext_noop(
                 "wait operating system loading"), interruptible=True)
@@ -449,7 +537,6 @@ class RebootOperation(InstanceOperation):
 
 @register_operation
 class RemoveInterfaceOperation(InstanceOperation):
-    activity_code_suffix = 'remove_interface'
     id = 'remove_interface'
     name = _("remove interface")
     description = _("Remove the specified network interface and erase IP "
@@ -460,11 +547,8 @@ class RemoveInterfaceOperation(InstanceOperation):
 
     def _operation(self, activity, user, system, interface):
         if self.instance.is_running:
-            with activity.sub_activity(
-                'detach_network',
-                readable_name=ugettext_noop("detach network")
-            ):
-                self.instance.detach_network(interface)
+            self.instance._detach_network(interface=interface,
+                                          parent_activity=activity)
             interface.shutdown()
 
         interface.destroy()
@@ -477,7 +561,6 @@ class RemoveInterfaceOperation(InstanceOperation):
 
 @register_operation
 class RemoveDiskOperation(InstanceOperation):
-    activity_code_suffix = 'remove_disk'
     id = 'remove_disk'
     name = _("remove disk")
     description = _("Remove the specified disk from the virtual machine, and "
@@ -487,15 +570,12 @@ class RemoveDiskOperation(InstanceOperation):
 
     def _operation(self, activity, user, system, disk):
         if self.instance.is_running and disk.type not in ["iso"]:
-            with activity.sub_activity(
-                'detach_disk',
-                readable_name=ugettext_noop('detach disk')
-            ):
-                self.instance.detach_disk(disk)
+            self.instance._detach_disk(disk=disk, parent_activity=activity)
         with activity.sub_activity(
             'destroy_disk',
             readable_name=ugettext_noop('destroy disk')
         ):
+            disk.destroy()
             return self.instance.disks.remove(disk)
 
     def get_activity_name(self, kwargs):
@@ -504,16 +584,16 @@ class RemoveDiskOperation(InstanceOperation):
 
 
 @register_operation
-class ResetOperation(InstanceOperation):
-    activity_code_suffix = 'reset'
+class ResetOperation(RemoteInstanceOperation):
     id = 'reset'
     name = _("reset")
     description = _("Cold reboot virtual machine (power cycle).")
     required_perms = ()
     accept_states = ('RUNNING', )
+    task = vm_tasks.reset
 
-    def _operation(self, activity, timeout=5):
-        self.instance.reset_vm(timeout=timeout)
+    def _operation(self, activity):
+        super(ResetOperation, self)._operation()
         if self.instance.has_agent:
             activity.sub_activity('os_boot', readable_name=ugettext_noop(
                 "wait operating system loading"), interruptible=True)
@@ -521,7 +601,6 @@ class ResetOperation(InstanceOperation):
 
 @register_operation
 class SaveAsTemplateOperation(InstanceOperation):
-    activity_code_suffix = 'save_as_template'
     id = 'save_as_template'
     name = _("save as template")
     description = _("Save virtual machine as a template so they can be shared "
@@ -553,7 +632,7 @@ class SaveAsTemplateOperation(InstanceOperation):
             for disk in self.disks:
                 disk.destroy()
 
-    def _operation(self, activity, user, system, timeout=300, name=None,
+    def _operation(self, activity, user, system, name=None,
                    with_shutdown=True, task=None, **kwargs):
         if with_shutdown:
             try:
@@ -617,8 +696,8 @@ class SaveAsTemplateOperation(InstanceOperation):
 
 
 @register_operation
-class ShutdownOperation(InstanceOperation):
-    activity_code_suffix = 'shutdown'
+class ShutdownOperation(AbortableRemoteOperationMixin,
+                        RemoteInstanceOperation):
     id = 'shutdown'
     name = _("shutdown")
     description = _("Try to halt virtual machine by a standard ACPI signal, "
@@ -629,9 +708,12 @@ class ShutdownOperation(InstanceOperation):
     required_perms = ()
     accept_states = ('RUNNING', )
     resultant_state = 'STOPPED'
+    task = vm_tasks.shutdown
+    remote_queue = ("vm", "slow")
+    remote_timeout = 120
 
-    def _operation(self, task=None):
-        self.instance.shutdown_vm(task=task)
+    def _operation(self, task):
+        super(ShutdownOperation, self)._operation(task=task)
         self.instance.yield_node()
 
     def on_abort(self, activity, error):
@@ -648,7 +730,6 @@ class ShutdownOperation(InstanceOperation):
 
 @register_operation
 class ShutOffOperation(InstanceOperation):
-    activity_code_suffix = 'shut_off'
     id = 'shut_off'
     name = _("shut off")
     description = _("Forcibly halt a virtual machine without notifying the "
@@ -667,16 +748,12 @@ class ShutOffOperation(InstanceOperation):
         with activity.sub_activity('shutdown_net'):
             self.instance.shutdown_net()
 
-        # Delete virtual machine
-        with activity.sub_activity('delete_vm'):
-            self.instance.delete_vm()
-
+        self.instance._delete_vm(parent_activity=activity)
         self.instance.yield_node()
 
 
 @register_operation
 class SleepOperation(InstanceOperation):
-    activity_code_suffix = 'sleep'
     id = 'sleep'
     name = _("sleep")
     description = _("Suspend virtual machine. This means the machine is "
@@ -702,25 +779,30 @@ class SleepOperation(InstanceOperation):
         else:
             activity.resultant_state = 'ERROR'
 
-    def _operation(self, activity, timeout=240):
-        # Destroy networks
-        with activity.sub_activity('shutdown_net', readable_name=ugettext_noop(
-                "shutdown network")):
+    def _operation(self, activity, system):
+        with activity.sub_activity('shutdown_net',
+                                   readable_name=ugettext_noop(
+                                       "shutdown network")):
             self.instance.shutdown_net()
+        self.instance._suspend_vm(parent_activity=activity)
+        self.instance.yield_node()
 
-        # Suspend vm
-        with activity.sub_activity('suspending',
-                                   readable_name=ugettext_noop(
-                                       "suspend virtual machine")):
-            self.instance.suspend_vm(timeout=timeout)
+    @register_operation
+    class SuspendVmOperation(SubOperationMixin, RemoteInstanceOperation):
+        id = "_suspend_vm"
+        name = _("suspend virtual machine")
+        task = vm_tasks.sleep
+        remote_queue = ("vm", "slow")
+        remote_timeout = 1000
 
-        self.instance.yield_node()
-        # VNC port needs to be kept
+        def _get_remote_args(self, **kwargs):
+            return (super(SleepOperation.SuspendVmOperation, self)
+                    ._get_remote_args(**kwargs)
+                    + [self.instance.mem_dump['path']])
 
 
 @register_operation
 class WakeUpOperation(InstanceOperation):
-    activity_code_suffix = 'wake_up'
     id = 'wake_up'
     name = _("wake up")
     description = _("Wake up sleeping (suspended) virtual machine. This will "
@@ -739,16 +821,13 @@ class WakeUpOperation(InstanceOperation):
         else:
             activity.resultant_state = 'ERROR'
 
-    def _operation(self, activity, timeout=60):
+    def _operation(self, activity):
         # Schedule vm
         self.instance.allocate_vnc_port()
         self.instance.allocate_node()
 
         # Resume vm
-        with activity.sub_activity(
-            'resuming', readable_name=ugettext_noop(
-                "resume virtual machine")):
-            self.instance.wake_up_vm(timeout=timeout)
+        self.instance._wake_up_vm(parent_activity=activity)
 
         # Estabilish network connection (vmdriver)
         with activity.sub_activity(
@@ -761,10 +840,22 @@ class WakeUpOperation(InstanceOperation):
         except:
             pass
 
+    @register_operation
+    class WakeUpVmOperation(SubOperationMixin, RemoteInstanceOperation):
+        id = "_wake_up_vm"
+        name = _("resume virtual machine")
+        task = vm_tasks.wake_up
+        remote_queue = ("vm", "slow")
+        remote_timeout = 1000
+
+        def _get_remote_args(self, **kwargs):
+            return (super(WakeUpOperation.WakeUpVmOperation, self)
+                    ._get_remote_args(**kwargs)
+                    + [self.instance.mem_dump['path']])
+
 
 @register_operation
 class RenewOperation(InstanceOperation):
-    activity_code_suffix = 'renew'
     id = 'renew'
     name = _("renew")
     description = _("Virtual machines are suspended and destroyed after they "
@@ -799,7 +890,6 @@ class RenewOperation(InstanceOperation):
 
 @register_operation
 class ChangeStateOperation(InstanceOperation):
-    activity_code_suffix = 'emergency_change_state'
     id = 'emergency_change_state'
     name = _("emergency state change")
     description = _("Change the virtual machine state to NOSTATE. This "
@@ -811,7 +901,8 @@ class ChangeStateOperation(InstanceOperation):
     required_perms = ('vm.emergency_change_state', )
     concurrency_check = False
 
-    def _operation(self, user, activity, new_state="NOSTATE", interrupt=False):
+    def _operation(self, user, activity, new_state="NOSTATE", interrupt=False,
+                   reset_node=False):
         activity.resultant_state = new_state
         if interrupt:
             msg_txt = ugettext_noop("Activity is forcibly interrupted.")
@@ -821,6 +912,36 @@ class ChangeStateOperation(InstanceOperation):
                 i.finish(False, result=message)
                 logger.error('Forced finishing activity %s', i)
 
+        if reset_node:
+            self.instance.node = None
+            self.instance.save()
+
+
+@register_operation
+class RedeployOperation(InstanceOperation):
+    id = 'redeploy'
+    name = _("redeploy")
+    description = _("Change the virtual machine state to NOSTATE "
+                    "and redeploy the VM. This operation allows starting "
+                    "machines formerly running on a failed node.")
+    acl_level = "owner"
+    required_perms = ('vm.redeploy', )
+    concurrency_check = False
+
+    def _operation(self, user, activity, with_emergency_change_state=True):
+        if with_emergency_change_state:
+            ChangeStateOperation(self.instance).call(
+                parent_activity=activity, user=user,
+                new_state='NOSTATE', interrupt=False, reset_node=True)
+        else:
+            ShutOffOperation(self.instance).call(
+                parent_activity=activity, user=user)
+
+        self.instance._update_status()
+
+        DeployOperation(self.instance).call(
+            parent_activity=activity, user=user)
+
 
 class NodeOperation(Operation):
     async_operation = abortable_async_node_operation
@@ -851,17 +972,45 @@ class NodeOperation(Operation):
                                  "parent activity does not match the user "
                                  "provided as parameter.")
 
-            return parent.create_sub(code_suffix=self.activity_code_suffix,
-                                     readable_name=name)
+            return parent.create_sub(
+                code_suffix=self.get_activity_code_suffix(),
+                readable_name=name)
         else:
-            return NodeActivity.create(code_suffix=self.activity_code_suffix,
-                                       node=self.node, user=user,
-                                       readable_name=name)
+            return NodeActivity.create(
+                code_suffix=self.get_activity_code_suffix(), node=self.node,
+                user=user, readable_name=name)
+
+
+@register_operation
+class ResetNodeOperation(NodeOperation):
+    id = 'reset'
+    name = _("reset")
+    description = _("Disable missing node and redeploy all instances "
+                    "on other ones.")
+    required_perms = ()
+    online_required = False
+    async_queue = "localhost.man.slow"
+
+    def check_precond(self):
+        super(ResetNodeOperation, self).check_precond()
+        if not self.node.enabled or self.node.online:
+            raise humanize_exception(ugettext_noop(
+                "You cannot reset a disabled or online node."), Exception())
+
+    def _operation(self, activity, user):
+        if self.node.enabled:
+            DisableOperation(self.node).call(parent_activity=activity,
+                                             user=user)
+        for i in self.node.instance_set.all():
+            name = create_readable(ugettext_noop(
+                "migrate %(instance)s (%(pk)s)"), instance=i.name, pk=i.pk)
+            with activity.sub_activity('migrate_instance_%d' % i.pk,
+                                       readable_name=name):
+                i.redeploy(user=user)
 
 
 @register_operation
 class FlushOperation(NodeOperation):
-    activity_code_suffix = 'flush'
     id = 'flush'
     name = _("flush")
     description = _("Passivate node and move all instances to other ones.")
@@ -882,7 +1031,6 @@ class FlushOperation(NodeOperation):
 
 @register_operation
 class ActivateOperation(NodeOperation):
-    activity_code_suffix = 'activate'
     id = 'activate'
     name = _("activate")
     description = _("Make node active, i.e. scheduler is allowed to deploy "
@@ -903,7 +1051,6 @@ class ActivateOperation(NodeOperation):
 
 @register_operation
 class PassivateOperation(NodeOperation):
-    activity_code_suffix = 'passivate'
     id = 'passivate'
     name = _("passivate")
     description = _("Make node passive, i.e. scheduler is denied to deploy "
@@ -925,7 +1072,6 @@ class PassivateOperation(NodeOperation):
 
 @register_operation
 class DisableOperation(NodeOperation):
-    activity_code_suffix = 'disable'
     id = 'disable'
     name = _("disable")
     description = _("Disable node.")
@@ -949,8 +1095,7 @@ class DisableOperation(NodeOperation):
 
 
 @register_operation
-class ScreenshotOperation(InstanceOperation):
-    activity_code_suffix = 'screenshot'
+class ScreenshotOperation(RemoteInstanceOperation):
     id = 'screenshot'
     name = _("screenshot")
     description = _("Get a screenshot about the virtual machine's console. A "
@@ -959,14 +1104,11 @@ class ScreenshotOperation(InstanceOperation):
     acl_level = "owner"
     required_perms = ()
     accept_states = ('RUNNING', )
-
-    def _operation(self):
-        return self.instance.get_screenshot(timeout=20)
+    task = vm_tasks.screenshot
 
 
 @register_operation
 class RecoverOperation(InstanceOperation):
-    activity_code_suffix = 'recover'
     id = 'recover'
     name = _("recover")
     description = _("Try to recover virtual machine disks from destroyed "
@@ -994,7 +1136,6 @@ class RecoverOperation(InstanceOperation):
 
 @register_operation
 class ResourcesOperation(InstanceOperation):
-    activity_code_suffix = 'Resources change'
     id = 'resources_change'
     name = _("resources change")
     description = _("Change resources of a stopped virtual machine.")
@@ -1041,7 +1182,6 @@ class EnsureAgentMixin(object):
 
 @register_operation
 class PasswordResetOperation(EnsureAgentMixin, InstanceOperation):
-    activity_code_suffix = 'password_reset'
     id = 'password_reset'
     name = _("password reset")
     description = _("Generate and set a new login password on the virtual "
@@ -1062,7 +1202,6 @@ class PasswordResetOperation(EnsureAgentMixin, InstanceOperation):
 
 @register_operation
 class MountStoreOperation(EnsureAgentMixin, InstanceOperation):
-    activity_code_suffix = 'mount_store'
     id = 'mount_store'
     name = _("mount store")
     description = _(
@@ -1087,3 +1226,61 @@ class MountStoreOperation(EnsureAgentMixin, InstanceOperation):
         password = user.profile.smb_password
         agent_tasks.mount_store.apply_async(
             queue=queue, args=(inst.vm_name, host, username, password))
+
+
+class AbstractDiskOperation(SubOperationMixin, RemoteInstanceOperation):
+    required_perms = ()
+
+    def _get_remote_args(self, disk, **kwargs):
+        return (super(AbstractDiskOperation, self)._get_remote_args(**kwargs)
+                + [disk.get_vmdisk_desc()])
+
+
+@register_operation
+class AttachDisk(AbstractDiskOperation):
+    id = "_attach_disk"
+    name = _("attach disk")
+    task = vm_tasks.attach_disk
+
+
+class DetachMixin(object):
+    def _operation(self, activity, **kwargs):
+        try:
+            super(DetachMixin, self)._operation(**kwargs)
+        except Exception as e:
+            if hasattr(e, "libvirtError") and "not found" in unicode(e):
+                activity.result = create_readable(
+                    ugettext_noop("Resource was not found."),
+                    ugettext_noop("Resource was not found. %(exception)s"),
+                    exception=unicode(e))
+            else:
+                raise
+
+
+@register_operation
+class DetachDisk(DetachMixin, AbstractDiskOperation):
+    id = "_detach_disk"
+    name = _("detach disk")
+    task = vm_tasks.detach_disk
+
+
+class AbstractNetworkOperation(SubOperationMixin, RemoteInstanceOperation):
+    required_perms = ()
+
+    def _get_remote_args(self, interface, **kwargs):
+        return (super(AbstractNetworkOperation, self)
+                ._get_remote_args(**kwargs) + [interface.get_vmnetwork_desc()])
+
+
+@register_operation
+class AttachNetwork(AbstractNetworkOperation):
+    id = "_attach_network"
+    name = _("attach network")
+    task = vm_tasks.attach_network
+
+
+@register_operation
+class DetachNetwork(DetachMixin, AbstractNetworkOperation):
+    id = "_detach_network"
+    name = _("detach network")
+    task = vm_tasks.detach_network

circle/vm/tasks/agent_tasks.py

@@ -53,8 +53,18 @@ def start_access_server(vm):
     pass
 
 
+@celery.task(name='agent.update_legacy')
+def update_legacy(vm, data, executable=None):
+    pass
+
+
+@celery.task(name='agent.append')
+def append(vm, data, filename, chunk_number):
+    pass
+
+
 @celery.task(name='agent.update')
-def update(vm, data):
+def update(vm, filename, executable, checksum):
     pass
 
 

circle/vm/tasks/local_agent_tasks.py

@@ -19,11 +19,14 @@ from common.models import create_readable
 from manager.mancelery import celery
 from vm.tasks.agent_tasks import (restart_networking, change_password,
                                   set_time, set_hostname, start_access_server,
-                                  cleanup, update, change_ip)
+                                  cleanup, update, append,
+                                  change_ip, update_legacy)
 from firewall.models import Host
 
 import time
+import os
 from base64 import encodestring
+from hashlib import md5
 from StringIO import StringIO
 from tarfile import TarFile, TarInfo
 from django.conf import settings
@@ -61,17 +64,34 @@ def send_networking_commands(instance, act):
         restart_networking.apply_async(queue=queue, args=(instance.vm_name, ))
 
 
-def create_agent_tar():
+def create_linux_tar():
     def exclude(tarinfo):
-        if tarinfo.name.startswith('./.git'):
+        ignored = ('./.', './misc', './windows')
+        if any(tarinfo.name.startswith(x) for x in ignored):
             return None
         else:
             return tarinfo
 
     f = StringIO()
 
+    with TarFile.open(fileobj=f, mode='w:gz') as tar:
+        agent_path = os.path.join(settings.AGENT_DIR, "agent-linux")
+        tar.add(agent_path, arcname='.', filter=exclude)
+
+        version_fileobj = StringIO(settings.AGENT_VERSION)
+        version_info = TarInfo(name='version.txt')
+        version_info.size = len(version_fileobj.buf)
+        tar.addfile(version_info, version_fileobj)
+
+    return encodestring(f.getvalue()).replace('\n', '')
+
+
+def create_windows_tar():
+    f = StringIO()
+
+    agent_path = os.path.join(settings.AGENT_DIR, "agent-win")
     with TarFile.open(fileobj=f, mode='w|gz') as tar:
-        tar.add(settings.AGENT_DIR, arcname='.', filter=exclude)
+        tar.add(agent_path, arcname='.')
 
         version_fileobj = StringIO(settings.AGENT_VERSION)
         version_info = TarInfo(name='version.txt')
@@ -82,17 +102,16 @@ def create_agent_tar():
 
 
 @celery.task
-def agent_started(vm, version=None):
-    from vm.models import Instance, instance_activity, InstanceActivity
+def agent_started(vm, version=None, system=None):
+    from vm.models import Instance, InstanceActivity
     instance = Instance.objects.get(id=int(vm.split('-')[-1]))
     queue = instance.get_remote_queue_name("agent")
     initialized = instance.activity_log.filter(
         activity_code='vm.Instance.agent.cleanup').exists()
 
-    with instance_activity(code_suffix='agent',
+    with instance.activity(code_suffix='agent',
                            readable_name=ugettext_noop('agent'),
-                           concurrency_check=False,
-                           instance=instance) as act:
+                           concurrency_check=False) as act:
         with act.sub_activity('starting',
                               readable_name=ugettext_noop('starting')):
             pass
@@ -105,7 +124,7 @@ def agent_started(vm, version=None):
 
         if version and version != settings.AGENT_VERSION:
             try:
-                update_agent(instance, act)
+                update_agent(instance, act, system, settings.AGENT_VERSION)
             except TimeoutError:
                 pass
             else:
@@ -147,11 +166,16 @@ def measure_boot_time(instance):
 @celery.task
 def agent_stopped(vm):
     from vm.models import Instance, InstanceActivity
+    from vm.models.activity import ActivityInProgressError
     instance = Instance.objects.get(id=int(vm.split('-')[-1]))
     qs = InstanceActivity.objects.filter(instance=instance,
                                          activity_code='vm.Instance.agent')
     act = qs.latest('id')
-    with act.sub_activity('stopping', readable_name=ugettext_noop('stopping')):
+    try:
+        with act.sub_activity('stopping',
+                              readable_name=ugettext_noop('stopping')):
+            pass
+    except ActivityInProgressError:
         pass
 
 
@@ -162,7 +186,7 @@ def get_network_configs(instance):
     return (interfaces, settings.FIREWALL_SETTINGS['rdns_ip'])
 
 
-def update_agent(instance, act=None):
+def update_agent(instance, act=None, system=None, version=None):
     if act:
         act = act.sub_activity(
             'update',
@@ -170,14 +194,47 @@ def update_agent(instance, act=None):
                 ugettext_noop('update to %(version)s'),
                 version=settings.AGENT_VERSION))
     else:
-        from vm.models import instance_activity
-        act = instance_activity(
-            code_suffix='agent.update', instance=instance,
+        act = instance.activity(
+            code_suffix='agent.update',
             readable_name=create_readable(
                 ugettext_noop('update agent to %(version)s'),
                 version=settings.AGENT_VERSION))
     with act:
         queue = instance.get_remote_queue_name("agent")
-        update.apply_async(
-            queue=queue,
-            args=(instance.vm_name, create_agent_tar())).get(timeout=10)
+        if system == "Windows":
+            executable = os.listdir(os.path.join(settings.AGENT_DIR,
+                                                 "agent-win"))[0]
+            # executable = "agent-winservice-%(version)s.exe" % {
+            #   'version': version}
+            data = create_windows_tar()
+        elif system == "Linux":
+            executable = ""
+            data = create_linux_tar()
+        else:
+            executable = ""
+            # Legacy update method
+            return update_legacy.apply_async(
+                queue=queue,
+                args=(instance.vm_name, create_linux_tar())
+            ).get(timeout=60)
+
+        checksum = md5(data).hexdigest()
+        chunk_size = 1024 * 1024
+        chunk_number = 0
+        index = 0
+        filename = version + ".tar"
+        while True:
+            chunk = data[index:index+chunk_size]
+            if chunk:
+                append.apply_async(
+                    queue=queue,
+                    args=(instance.vm_name, chunk,
+                          filename, chunk_number)).get(timeout=60)
+                index = index + chunk_size
+                chunk_number = chunk_number + 1
+            else:
+                update.apply_async(
+                    queue=queue,
+                    args=(instance.vm_name, filename, executable, checksum)
+                ).get(timeout=60)
+                break

circle/vm/tests/test_models.py

@@ -29,7 +29,8 @@ from ..models import (
 )
 from ..models.instance import find_unused_port, ActivityInProgressError
 from ..operations import (
-    DeployOperation, DestroyOperation, FlushOperation, MigrateOperation,
+    RemoteOperationMixin, DeployOperation, DestroyOperation, FlushOperation,
+    MigrateOperation,
 )
 
 
@@ -89,7 +90,7 @@ class InstanceTestCase(TestCase):
         self.assertFalse(inst.save.called)
 
     def test_destroy_sets_destroyed(self):
-        inst = Mock(destroyed_at=None, spec=Instance,
+        inst = Mock(destroyed_at=None, spec=Instance, _delete_vm=Mock(),
                     InstanceDestroyedError=Instance.InstanceDestroyedError)
         inst.node = MagicMock(spec=Node)
         inst.disks.all.return_value = []
@@ -105,15 +106,15 @@ class InstanceTestCase(TestCase):
         inst.node = MagicMock(spec=Node)
         inst.status = 'RUNNING'
         migrate_op = MigrateOperation(inst)
-        with patch('vm.models.instance.vm_tasks.migrate') as migr:
+        with patch('vm.operations.vm_tasks.migrate') as migr, \
+                patch.object(RemoteOperationMixin, "_operation"):
             act = MagicMock()
             with patch.object(MigrateOperation, 'create_activity',
                               return_value=act):
                 migrate_op(system=True)
 
             migr.apply_async.assert_called()
-            self.assertIn(call.sub_activity(
-                u'scheduling', readable_name=u'schedule'), act.mock_calls)
+            inst.allocate_node.assert_called()
             inst.select_node.assert_called()
 
     def test_migrate_wo_scheduling(self):
@@ -122,7 +123,8 @@ class InstanceTestCase(TestCase):
         inst.node = MagicMock(spec=Node)
         inst.status = 'RUNNING'
         migrate_op = MigrateOperation(inst)
-        with patch('vm.models.instance.vm_tasks.migrate') as migr:
+        with patch('vm.operations.vm_tasks.migrate') as migr, \
+                patch.object(RemoteOperationMixin, "_operation"):
             inst.select_node.side_effect = AssertionError
             act = MagicMock()
             with patch.object(MigrateOperation, 'create_activity',
@@ -130,7 +132,7 @@ class InstanceTestCase(TestCase):
                 migrate_op(to_node=inst.node, system=True)
 
             migr.apply_async.assert_called()
-            self.assertNotIn(call.sub_activity(u'scheduling'), act.mock_calls)
+            inst.allocate_node.assert_called()
 
     def test_migrate_with_error(self):
         inst = Mock(destroyed_at=None, spec=Instance)
@@ -139,20 +141,21 @@ class InstanceTestCase(TestCase):
         inst.status = 'RUNNING'
         e = Exception('abc')
         setattr(e, 'libvirtError', '')
-        inst.migrate_vm.side_effect = e
         migrate_op = MigrateOperation(inst)
-        with patch('vm.models.instance.vm_tasks.migrate') as migr:
+        migrate_op.rollback = Mock()
+        with patch('vm.operations.vm_tasks.migrate') as migr, \
+                patch.object(RemoteOperationMixin, '_operation') as remop:
             act = MagicMock()
+            remop.side_effect = e
             with patch.object(MigrateOperation, 'create_activity',
                               return_value=act):
                 self.assertRaises(Exception, migrate_op, system=True)
 
+            remop.assert_called()
             migr.apply_async.assert_called()
             self.assertIn(call.sub_activity(
                 u'scheduling', readable_name=u'schedule'), act.mock_calls)
-            self.assertIn(call.sub_activity(
-                u'rollback_net', readable_name=u'redeploy network (rollback)'),
-                act.mock_calls)
+            migrate_op.rollback.assert_called()
             inst.select_node.assert_called()
 
     def test_status_icon(self):

circle/vm/tests/test_operations.py

@@ -16,9 +16,10 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.test import TestCase
+from mock import MagicMock
 
 from common.operations import operation_registry_name as op_reg_name
-from vm.models import Instance, Node
+from vm.models import Instance, InstanceActivity, Node
 from vm.operations import (
     DeployOperation, DestroyOperation, FlushOperation, MigrateOperation,
     RebootOperation, ResetOperation, SaveAsTemplateOperation,
@@ -45,6 +46,22 @@ class MigrateOperationTestCase(TestCase):
     def test_operation_registered(self):
         assert MigrateOperation.id in getattr(Instance, op_reg_name)
 
+    def test_operation_wo_to_node_param(self):
+        class MigrateException(Exception):
+            pass
+
+        inst = MagicMock(spec=Instance)
+        act = MagicMock(spec=InstanceActivity)
+        op = MigrateOperation(inst)
+        op._get_remote_args = MagicMock(side_effect=MigrateException())
+        inst.select_node = MagicMock(return_value='test')
+        self.assertRaises(
+            MigrateException, op._operation,
+            act, to_node=None)
+        assert inst.select_node.called
+        op._get_remote_args.assert_called_once_with(
+            to_node='test', live_migration=True)
+
 
 class RebootOperationTestCase(TestCase):
     def test_operation_registered(self):

miscellaneous/mancelery.conf

@@ -6,9 +6,14 @@ respawn limit 30 30
 setgid cloud
 setuid cloud
 
+kill timeout 360
+kill signal SIGTERM
+
 script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    exec ./manage.py celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 10
+    ./manage.py celery -f --app=manager.mancelery purge
+    exec ./manage.py celery --app=manager.mancelery worker --autoreload --loglevel=info --hostname=mancelery -B -c 3
 end script
+

miscellaneous/moncelery.conf

@@ -3,6 +3,7 @@ description     "CIRCLE moncelery for monitoring jobs"
 respawn
 respawn limit 30 30
 
+
 setgid cloud
 setuid cloud
 
@@ -10,5 +11,7 @@ script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    exec ./manage.py celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 3
+    ./manage.py celery -f --app=manager.moncelery purge
+    exec ./manage.py celery --app=manager.moncelery worker --autoreload --loglevel=info --hostname=moncelery -B -c 2
 end script
+

miscellaneous/slowcelery.conf

-description     "CIRCLE mancelery for slow jobs"
+description     "CIRCLE slowcelery for resource intensive or long  jobs"
 
 respawn
 respawn limit 30 30
@@ -6,9 +6,15 @@ respawn limit 30 30
 setgid cloud
 setuid cloud
 
+kill timeout 360                                                                                      
+kill signal INT 
+                                          
+
 script
     cd /home/cloud/circle/circle
     . /home/cloud/.virtualenvs/circle/bin/activate
     . /home/cloud/.virtualenvs/circle/bin/postactivate
-    exec ./manage.py celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 5
+    ./manage.py celery -f --app=manager.slowcelery purge
+    exec ./manage.py celery --app=manager.slowcelery worker --autoreload --loglevel=info --hostname=slowcelery -B -c 1
 end script
+

requirements/base.txt

@@ -9,7 +9,7 @@ django-braces==1.4.0
 django-celery==3.1.10
 django-crispy-forms==1.4.0
 django-model-utils==2.0.3
-django-sizefield==0.5
+django-sizefield==0.6
 django-sshkey==2.2.0
 django-statici18n==1.1
 django-tables2==0.15.0