Merge branch 'master' into 'smallville_fix'

# Conflicts:
#   circle/vm/tasks/local_periodic_tasks.py

.gitignore

@@ -8,6 +8,8 @@
 *.swp
 *.swp
 *.swo
 *.swo
 *~
 *~
+.vscode
+.idea
 
 
 # Sphinx docs:
 # Sphinx docs:
 build
 build

circle/circle/settings/base.py

@@ -495,6 +495,7 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
                 },
                 },
                 'required_attributes': required_attrs,
                 'required_attributes': required_attrs,
                 'optional_attributes': optional_attrs,
                 'optional_attributes': optional_attrs,
+                'want_response_signed': False,
             },
             },
         },
         },
         'metadata': {'local': [remote_metadata], },
         'metadata': {'local': [remote_metadata], },
@@ -576,7 +577,7 @@ SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
 MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
 MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
 MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
 MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
 
 
-SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'random')
+SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'advanced')
 
 
 # Url to download the client: (e.g. http://circlecloud.org/client/download/)
 # Url to download the client: (e.g. http://circlecloud.org/client/download/)
 CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
 CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
@@ -590,3 +591,12 @@ REQUEST_HOOK_URL = get_env_variable("REQUEST_HOOK_URL", "")
 SSHKEY_EMAIL_ADD_KEY = False
 SSHKEY_EMAIL_ADD_KEY = False
 
 
 TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
 TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
+
+# Default value is every day at midnight
+AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *")
+AUTO_MIGRATION_TIME_LIMIT_IN_HOURS = (
+    get_env_variable("AUTO_MIGRATION_TIME_LIMIT_IN_HOURS", "2"))
+
+# Maximum time difference until the monitor's values get valid
+SCHEDULER_TIME_SENSITIVITY_IN_SECONDS = (
+    get_env_variable("SCHEDULER_TIME_SENSITIVITY_IN_SECONDS", "60"))

circle/dashboard/fixtures/node.json

@@ -65,7 +65,10 @@
     "modified": "2014-02-19T21:11:34.671Z", 
     "modified": "2014-02-19T21:11:34.671Z", 
     "priority": 1, 
     "priority": 1, 
     "traits": [], 
     "traits": [], 
-    "host": 1
+    "host": 1,
+    "ram_weight": 1.0,
+    "cpu_weight": 1.0,
+    "time_stamp":  "2017-12-13T21:08:08.819Z"
   }
   }
 }
 }
 ]
 ]

circle/dashboard/forms.py

@@ -17,56 +17,51 @@
 
 
 from __future__ import absolute_import
 from __future__ import absolute_import
 
 
-from datetime import timedelta
 from urlparse import urlparse
 from urlparse import urlparse
 
 
+import os
 import pyotp
 import pyotp
-
-from django.forms import ModelForm
-from django.contrib.auth.forms import (
-    AuthenticationForm, PasswordResetForm, SetPasswordForm,
-    PasswordChangeForm,
-)
-from django.contrib.auth.models import User, Group
-from django.core.validators import URLValidator
-from django.core.exceptions import PermissionDenied, ValidationError
-
-from dal import autocomplete
+from crispy_forms.bootstrap import FormActions
 from crispy_forms.helper import FormHelper
 from crispy_forms.helper import FormHelper
 from crispy_forms.layout import (
 from crispy_forms.layout import (
     Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
     Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
 )
 )
-
 from crispy_forms.utils import render_field
 from crispy_forms.utils import render_field
-from crispy_forms.bootstrap import FormActions
-
+from dal import autocomplete
+from datetime import timedelta
 from django import forms
 from django import forms
+from django.contrib.admin.widgets import FilteredSelectMultiple
+from django.contrib.auth.forms import (
+    AuthenticationForm, PasswordResetForm, SetPasswordForm,
+    PasswordChangeForm,
+)
 from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
 from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
+from django.contrib.auth.models import Permission
+from django.contrib.auth.models import User, Group
+from django.core.exceptions import PermissionDenied, ValidationError
+from django.core.urlresolvers import reverse_lazy
+from django.core.validators import URLValidator
+from django.forms import ModelForm
 from django.forms.widgets import TextInput, HiddenInput
 from django.forms.widgets import TextInput, HiddenInput
 from django.template.loader import render_to_string
 from django.template.loader import render_to_string
 from django.utils.html import escape, format_html
 from django.utils.html import escape, format_html
 from django.utils.safestring import mark_safe
 from django.utils.safestring import mark_safe
+from django.utils.translation import string_concat
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
+from django_sshkey.models import UserKey
 from sizefield.widgets import FileSizeWidget
 from sizefield.widgets import FileSizeWidget
-from django.core.urlresolvers import reverse_lazy
 
 
-from django_sshkey.models import UserKey
+from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
+from dashboard.models import ConnectCommand, create_profile
+from dashboard.store_api import Store
 from firewall.models import Vlan, Host
 from firewall.models import Vlan, Host
+from storage.models import DataStore, Disk
 from vm.models import (
 from vm.models import (
     InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
     InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
 )
 )
-from storage.models import DataStore, Disk
-from django.contrib.admin.widgets import FilteredSelectMultiple
-from django.contrib.auth.models import Permission
 from .models import Profile, GroupProfile, Message
 from .models import Profile, GroupProfile, Message
-from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
-from django.utils.translation import string_concat
-
 from .validators import domain_validator
 from .validators import domain_validator
 
 
-from dashboard.models import ConnectCommand, create_profile
-
-
 LANGUAGES_WITH_CODE = ((l[0], string_concat(l[1], " (", l[0], ")"))
 LANGUAGES_WITH_CODE = ((l[0], string_concat(l[1], " (", l[0], ")"))
                        for l in LANGUAGES)
                        for l in LANGUAGES)
 
 
@@ -189,7 +184,7 @@ class VmCustomizeForm(forms.Form):
             self.initial['ram_size'] = self.template.ram_size
             self.initial['ram_size'] = self.template.ram_size
 
 
         else:
         else:
-            self.allowed_fields = ("name", "template", "customized", )
+            self.allowed_fields = ("name", "template", "customized",)
 
 
         # initial name and template pk
         # initial name and template pk
         self.initial['name'] = self.template.name
         self.initial['name'] = self.template.name
@@ -214,7 +209,6 @@ class VmCustomizeForm(forms.Form):
 
 
 
 
 class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
 class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
-
     description = forms.CharField(label=_("Description"), required=False,
     description = forms.CharField(label=_("Description"), required=False,
                                   widget=forms.Textarea(attrs={'rows': 3}))
                                   widget=forms.Textarea(attrs={'rows': 3}))
 
 
@@ -258,7 +252,7 @@ class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = Group
         model = Group
-        fields = ('name', )
+        fields = ('name',)
 
 
 
 
 class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
 class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
@@ -276,6 +270,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
                 label=_('Directory identifier'))
                 label=_('Directory identifier'))
             if not new_groups:
             if not new_groups:
                 self.fields['org_id'].widget = HiddenInput()
                 self.fields['org_id'].widget = HiddenInput()
+            self.fields['disk_quota'].widget = HiddenInput()
         self.fields['description'].widget = forms.Textarea(attrs={'rows': 3})
         self.fields['description'].widget = forms.Textarea(attrs={'rows': 3})
 
 
     @property
     @property
@@ -293,7 +288,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = GroupProfile
         model = GroupProfile
-        fields = ('description', 'org_id')
+        fields = ('description', 'org_id', 'disk_quota')
 
 
 
 
 class HostForm(NoFormTagMixin, forms.ModelForm):
 class HostForm(NoFormTagMixin, forms.ModelForm):
@@ -513,7 +508,7 @@ class TemplateForm(forms.ModelForm):
             self.allowed_fields += tuple(set(self.fields.keys()) -
             self.allowed_fields += tuple(set(self.fields.keys()) -
                                          set(['raw_data']))
                                          set(['raw_data']))
         if self.user.is_superuser:
         if self.user.is_superuser:
-            self.allowed_fields += ('raw_data', )
+            self.allowed_fields += ('raw_data',)
         for name, field in self.fields.items():
         for name, field in self.fields.items():
             if name not in self.allowed_fields:
             if name not in self.allowed_fields:
                 field.widget.attrs['disabled'] = 'disabled'
                 field.widget.attrs['disabled'] = 'disabled'
@@ -525,8 +520,8 @@ class TemplateForm(forms.ModelForm):
             self.initial['max_ram_size'] = 512
             self.initial['max_ram_size'] = 512
 
 
         lease_queryset = (
         lease_queryset = (
-            Lease.get_objects_with_level("operator", self.user).distinct() |
-            Lease.objects.filter(pk=self.instance.lease_id).distinct())
+                Lease.get_objects_with_level("operator", self.user).distinct() |
+                Lease.objects.filter(pk=self.instance.lease_id).distinct())
 
 
         self.fields["lease"].queryset = lease_queryset
         self.fields["lease"].queryset = lease_queryset
 
 
@@ -602,7 +597,7 @@ class TemplateForm(forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = InstanceTemplate
         model = InstanceTemplate
-        exclude = ('state', 'disks', )
+        exclude = ('state', 'disks',)
         widgets = {
         widgets = {
             'system': forms.TextInput,
             'system': forms.TextInput,
             'max_ram_size': forms.HiddenInput,
             'max_ram_size': forms.HiddenInput,
@@ -745,7 +740,6 @@ class LeaseForm(forms.ModelForm):
 
 
 
 
 class VmRenewForm(OperationForm):
 class VmRenewForm(OperationForm):
-
     force = forms.BooleanField(required=False, label=_(
     force = forms.BooleanField(required=False, label=_(
         "Set expiration times even if they are shorter than "
         "Set expiration times even if they are shorter than "
         "the current value."))
         "the current value."))
@@ -785,11 +779,10 @@ class VmMigrateForm(forms.Form):
 
 
 
 
 class VmStateChangeForm(OperationForm):
 class VmStateChangeForm(OperationForm):
-
     interrupt = forms.BooleanField(required=False, label=_(
     interrupt = forms.BooleanField(required=False, label=_(
         "Forcibly interrupt all running activities."),
         "Forcibly interrupt all running activities."),
-        help_text=_("Set all activities to finished state, "
-                    "but don't interrupt any tasks."))
+                                   help_text=_("Set all activities to finished state, "
+                                               "but don't interrupt any tasks."))
     new_state = forms.ChoiceField(Instance.STATUS, label=_(
     new_state = forms.ChoiceField(Instance.STATUS, label=_(
         "New status"))
         "New status"))
     reset_node = forms.BooleanField(required=False, label=_("Reset node"))
     reset_node = forms.BooleanField(required=False, label=_("Reset node"))
@@ -830,6 +823,41 @@ class VmCreateDiskForm(OperationForm):
         return size_in_bytes
         return size_in_bytes
 
 
 
 
+class VmDiskExportForm(OperationForm):
+    exported_name = forms.CharField(max_length=100, label=_('Filename'))
+    disk_format = forms.ChoiceField(
+        choices=Disk.EXPORT_FORMATS,
+        label=_('Format'))
+
+    def __init__(self, *args, **kwargs):
+        choices = kwargs.pop('choices')
+        self.disk = kwargs.pop('default')
+
+        super(VmDiskExportForm, self).__init__(*args, **kwargs)
+
+        self.fields['disk'] = forms.ModelChoiceField(
+            queryset=choices, initial=self.disk, required=True,
+            empty_label=None, label=_('Disk'))
+        if self.disk:
+            self.fields['disk'].widget = HiddenInput()
+
+    @property
+    def helper(self):
+        helper = super(VmDiskExportForm, self).helper
+        if self.disk:
+            helper.layout = Layout(
+                AnyTag(
+                    "div",
+                    HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
+                    css_class="form-group",
+                ),
+                Field('disk'),
+                Field('exported_name'),
+                Field('disk_format')
+            )
+        return helper
+
+
 class VmDiskResizeForm(OperationForm):
 class VmDiskResizeForm(OperationForm):
     size = forms.CharField(
     size = forms.CharField(
         widget=FileSizeWidget, initial=(10 << 30), label=_('Size'),
         widget=FileSizeWidget, initial=(10 << 30), label=_('Size'),
@@ -858,7 +886,7 @@ class VmDiskResizeForm(OperationForm):
                                           " GB or MB!"))
                                           " GB or MB!"))
         if int(size_in_bytes) < int(disk.size):
         if int(size_in_bytes) < int(disk.size):
             raise forms.ValidationError(_("Disk size must be greater than the "
             raise forms.ValidationError(_("Disk size must be greater than the "
-                                        "actual size."))
+                                          "actual size."))
         return cleaned_data
         return cleaned_data
 
 
     @property
     @property
@@ -899,6 +927,20 @@ class VmDiskRemoveForm(OperationForm):
         return helper
         return helper
 
 
 
 
+class VmImportDiskForm(OperationForm):
+    def __init__(self, *args, **kwargs):
+        self.user = kwargs.pop('user')
+        super(VmImportDiskForm, self).__init__(*args, **kwargs)
+
+        disk_paths = Store(self.user).get_disk_images()
+        disk_filenames = [os.path.basename(item) for item in disk_paths]
+        self.choices = zip(disk_paths, disk_filenames)
+
+        self.fields['name'] = forms.CharField(max_length=100, label=_('Name'))
+        self.fields['disk_path'] = forms.ChoiceField(label=_('Disk image'),
+                                                     choices=self.choices)
+
+
 class VmDownloadDiskForm(OperationForm):
 class VmDownloadDiskForm(OperationForm):
     name = forms.CharField(max_length=100, label=_("Name"), required=False)
     name = forms.CharField(max_length=100, label=_("Name"), required=False)
     url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
     url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
@@ -1138,7 +1180,6 @@ class CircleSetPasswordForm(SetPasswordForm):
 
 
 
 
 class LinkButton(BaseInput):
 class LinkButton(BaseInput):
-
     """
     """
     Used to create a link button descriptor for the {% crispy %} template tag::
     Used to create a link button descriptor for the {% crispy %} template tag::
 
 
@@ -1224,7 +1265,7 @@ class MyProfileForm(forms.ModelForm):
 
 
     class Meta:
     class Meta:
         fields = ('preferred_language', 'email_notifications',
         fields = ('preferred_language', 'email_notifications',
-                  'desktop_notifications', 'use_gravatar', )
+                  'desktop_notifications', 'use_gravatar',)
         model = Profile
         model = Profile
 
 
     @property
     @property
@@ -1239,9 +1280,8 @@ class MyProfileForm(forms.ModelForm):
 
 
 
 
 class UnsubscribeForm(forms.ModelForm):
 class UnsubscribeForm(forms.ModelForm):
-
     class Meta:
     class Meta:
-        fields = ('email_notifications', )
+        fields = ('email_notifications',)
         model = Profile
         model = Profile
 
 
     @property
     @property
@@ -1299,6 +1339,9 @@ class UserEditForm(forms.ModelForm):
     instance_limit = forms.IntegerField(
     instance_limit = forms.IntegerField(
         label=_('Instance limit'),
         label=_('Instance limit'),
         min_value=0, widget=NumberInput)
         min_value=0, widget=NumberInput)
+    template_instance_limit = forms.IntegerField(
+        label=_('Template instance limit'),
+        min_value=0, widget=NumberInput)
     two_factor_secret = forms.CharField(
     two_factor_secret = forms.CharField(
         label=_('Two-factor authentication secret'),
         label=_('Two-factor authentication secret'),
         help_text=_("Remove the secret key to disable two-factor "
         help_text=_("Remove the secret key to disable two-factor "
@@ -1308,20 +1351,25 @@ class UserEditForm(forms.ModelForm):
         super(UserEditForm, self).__init__(*args, **kwargs)
         super(UserEditForm, self).__init__(*args, **kwargs)
         self.fields["instance_limit"].initial = (
         self.fields["instance_limit"].initial = (
             self.instance.profile.instance_limit)
             self.instance.profile.instance_limit)
+        self.fields["template_instance_limit"].initial = (
+            self.instance.profile.template_instance_limit)
         self.fields["two_factor_secret"].initial = (
         self.fields["two_factor_secret"].initial = (
             self.instance.profile.two_factor_secret)
             self.instance.profile.two_factor_secret)
 
 
     class Meta:
     class Meta:
         model = User
         model = User
-        fields = ('email', 'first_name', 'last_name', 'instance_limit',
-                  'is_active', "two_factor_secret", )
+        fields = ('email', 'first_name', 'last_name',
+                  'instance_limit', 'template_instance_limit',
+                  'is_active', 'two_factor_secret',)
 
 
     def save(self, commit=True):
     def save(self, commit=True):
         user = super(UserEditForm, self).save()
         user = super(UserEditForm, self).save()
         user.profile.instance_limit = (
         user.profile.instance_limit = (
-            self.cleaned_data['instance_limit'] or None)
+                self.cleaned_data['instance_limit'] or None)
+        user.profile.template_instance_limit = (
+                self.cleaned_data['template_instance_limit'] or None)
         user.profile.two_factor_secret = (
         user.profile.two_factor_secret = (
-            self.cleaned_data['two_factor_secret'] or None)
+                self.cleaned_data['two_factor_secret'] or None)
         user.profile.save()
         user.profile.save()
         return user
         return user
 
 
@@ -1405,10 +1453,9 @@ class ConnectCommandForm(forms.ModelForm):
 
 
 
 
 class TraitsForm(forms.ModelForm):
 class TraitsForm(forms.ModelForm):
-
     class Meta:
     class Meta:
         model = Instance
         model = Instance
-        fields = ('req_traits', )
+        fields = ('req_traits',)
 
 
     @property
     @property
     def helper(self):
     def helper(self):
@@ -1428,7 +1475,7 @@ class RawDataForm(forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = Instance
         model = Instance
-        fields = ('raw_data', )
+        fields = ('raw_data',)
 
 
     @property
     @property
     def helper(self):
     def helper(self):
@@ -1490,7 +1537,7 @@ class GroupPermissionForm(forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = Group
         model = Group
-        fields = ('permissions', )
+        fields = ('permissions',)
 
 
     @property
     @property
     def helper(self):
     def helper(self):
@@ -1538,7 +1585,7 @@ class VmResourcesForm(forms.ModelForm):
 
 
     class Meta:
     class Meta:
         model = Instance
         model = Instance
-        fields = ('num_cores', 'priority', 'ram_size', )
+        fields = ('num_cores', 'priority', 'ram_size',)
 
 
 
 
 class VmRenameForm(forms.Form):
 class VmRenameForm(forms.Form):
@@ -1629,7 +1676,7 @@ class DataStoreForm(ModelForm):
 
 
     class Meta:
     class Meta:
         model = DataStore
         model = DataStore
-        fields = ("name", "path", "hostname", )
+        fields = ("name", "path", "hostname",)
 
 
 
 
 class DiskForm(ModelForm):
 class DiskForm(ModelForm):
@@ -1647,7 +1694,7 @@ class DiskForm(ModelForm):
     class Meta:
     class Meta:
         model = Disk
         model = Disk
         fields = ("name", "filename", "datastore", "type", "bus", "size",
         fields = ("name", "filename", "datastore", "type", "bus", "size",
-                  "base", "dev_num", "destroyed", "is_ready", )
+                  "base", "dev_num", "destroyed", "is_ready",)
 
 
 
 
 class MessageForm(ModelForm):
 class MessageForm(ModelForm):
@@ -1697,3 +1744,11 @@ class TwoFactorConfirmationForm(forms.Form):
         totp = pyotp.TOTP(self.user.profile.two_factor_secret)
         totp = pyotp.TOTP(self.user.profile.two_factor_secret)
         if not totp.verify(self.cleaned_data.get('confirmation_code')):
         if not totp.verify(self.cleaned_data.get('confirmation_code')):
             raise ValidationError(_("Invalid confirmation code."))
             raise ValidationError(_("Invalid confirmation code."))
+
+
+class AutoMigrationForm(forms.Form):
+    minute = forms.CharField()
+    hour = forms.CharField()
+    day_of_month = forms.CharField()
+    month_of_year = forms.CharField()
+    day_of_week = forms.CharField()

circle/dashboard/migrations/0007_groupprofile_disk_quota.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-04-24 20:00
+from __future__ import unicode_literals
+
+from django.db import migrations
+import sizefield.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0006_auto_20170707_1909'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='groupprofile',
+            name='disk_quota',
+            field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
+        ),
+    ]

circle/dashboard/migrations/0008_profile_template_instance_limit.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-11-06 13:33
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0007_groupprofile_disk_quota'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='profile',
+            name='template_instance_limit',
+            field=models.IntegerField(default=1),
+        ),
+    ]

circle/dashboard/models.py

@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
 
 
 from vm.models.instance import ACCESS_METHODS
 from vm.models.instance import ACCESS_METHODS
 
 
-from .store_api import Store, NoStoreException, NotOkException, Timeout
+from .store_api import Store, NoStoreException, NotOkException
 from .validators import connect_command_template_validator
 from .validators import connect_command_template_validator
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
@@ -162,7 +162,7 @@ class ConnectCommand(Model):
                          validators=[connect_command_template_validator])
                          validators=[connect_command_template_validator])
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.template
         return self.template
@@ -178,6 +178,7 @@ class Profile(Model):
         unique=True, blank=True, null=True, max_length=64,
         unique=True, blank=True, null=True, max_length=64,
         help_text=_('Unique identifier of the person, e.g. a student number.'))
         help_text=_('Unique identifier of the person, e.g. a student number.'))
     instance_limit = IntegerField(default=5)
     instance_limit = IntegerField(default=5)
+    template_instance_limit = IntegerField(default=1)
     use_gravatar = BooleanField(
     use_gravatar = BooleanField(
         verbose_name=_("Use Gravatar"), default=True,
         verbose_name=_("Use Gravatar"), default=True,
         help_text=_("Whether to use email address as Gravatar profile image"))
         help_text=_("Whether to use email address as Gravatar profile image"))
@@ -218,7 +219,7 @@ class Profile(Model):
                     'id': command.id,
                     'id': command.id,
                     'cmd': command.template % {
                     'cmd': command.template % {
                         'port': instance.get_connect_port(use_ipv6=use_ipv6),
                         'port': instance.get_connect_port(use_ipv6=use_ipv6),
-                        'host':  instance.get_connect_host(use_ipv6=use_ipv6),
+                        'host': instance.get_connect_host(use_ipv6=use_ipv6),
                         'password': instance.pw,
                         'password': instance.pw,
                         'username': 'cloud',
                         'username': 'cloud',
                     }} for command in commands]
                     }} for command in commands]
@@ -263,7 +264,7 @@ class Profile(Model):
         super(Profile, self).save(*args, **kwargs)
         super(Profile, self).save(*args, **kwargs)
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
         permissions = (
         permissions = (
             ('use_autocomplete', _('Can use autocomplete.')),
             ('use_autocomplete', _('Can use autocomplete.')),
         )
         )
@@ -275,7 +276,7 @@ class FutureMember(Model):
     group = ForeignKey(Group)
     group = ForeignKey(Group)
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
         unique_together = ('org_id', 'group')
         unique_together = ('org_id', 'group')
 
 
     def __unicode__(self):
     def __unicode__(self):
@@ -293,9 +294,13 @@ class GroupProfile(AclBase):
         unique=True, blank=True, null=True, max_length=64,
         unique=True, blank=True, null=True, max_length=64,
         help_text=_('Unique identifier of the group at the organization.'))
         help_text=_('Unique identifier of the group at the organization.'))
     description = TextField(blank=True)
     description = TextField(blank=True)
+    disk_quota = FileSizeField(
+        verbose_name=_('disk quota'),
+        default=2048 * 1024 * 1024,
+        help_text=_('Disk quota in mebibytes.'))
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.group.name
         return self.group.name
@@ -331,7 +336,11 @@ def create_profile(user):
     profile, created = Profile.objects.get_or_create(user=user)
     profile, created = Profile.objects.get_or_create(user=user)
 
 
     try:
     try:
-        Store(user).create_user(profile.smb_password, None, profile.disk_quota)
+        store = Store(user)
+        quotas = [profile.disk_quota]
+        quotas += [group.profile.disk_quota for group in user.groups.all()]
+        max_quota = max(quotas)
+        store.create_user(profile.smb_password, None, max_quota)
     except:
     except:
         logger.exception("Can't create user %s", unicode(user))
         logger.exception("Can't create user %s", unicode(user))
     return created
     return created
@@ -347,6 +356,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
     logger.debug("Register save_org_id to djangosaml2 pre_user_save")
     logger.debug("Register save_org_id to djangosaml2 pre_user_save")
     from djangosaml2.signals import pre_user_save
     from djangosaml2.signals import pre_user_save
 
 
+
     def save_org_id(sender, instance, attributes, **kwargs):
     def save_org_id(sender, instance, attributes, **kwargs):
         logger.debug("save_org_id called by %s", instance.username)
         logger.debug("save_org_id called by %s", instance.username)
         atr = settings.SAML_ORG_ID_ATTRIBUTE
         atr = settings.SAML_ORG_ID_ATTRIBUTE
@@ -399,6 +409,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
 
 
         return False  # User did not change
         return False  # User did not change
 
 
+
     pre_user_save.connect(save_org_id)
     pre_user_save.connect(save_org_id)
 
 
 
 
@@ -411,7 +422,7 @@ def update_store_profile(sender, **kwargs):
                       profile.disk_quota)
                       profile.disk_quota)
     except NoStoreException:
     except NoStoreException:
         logger.debug("Store is not available.")
         logger.debug("Store is not available.")
-    except (NotOkException, Timeout):
+    except NotOkException:
         logger.critical("Store is not accepting connections.")
         logger.critical("Store is not accepting connections.")
 
 
 
 

circle/dashboard/static/dashboard/activity.js

@@ -41,12 +41,14 @@ $(function() {
         $('#confirmation-modal').on('hidden.bs.modal', function() {
         $('#confirmation-modal').on('hidden.bs.modal', function() {
           $('#confirmation-modal').remove();
           $('#confirmation-modal').remove();
         });
         });
+
         $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
         $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
       }
       }
     });
     });
     e.preventDefault();
     e.preventDefault();
   });
   });
 
 
+
   /* if the operation fails show the modal again */
   /* if the operation fails show the modal again */
   $("body").on("click", "#confirmation-modal #op-form-send", function() {
   $("body").on("click", "#confirmation-modal #op-form-send", function() {
     var url = $(this).closest("form").prop("action");
     var url = $(this).closest("form").prop("action");
@@ -237,4 +239,3 @@ String.prototype.hashCode = function() {
   }
   }
   return hash;
   return hash;
 };
 };
-

circle/dashboard/static/dashboard/dashboard.js

@@ -558,3 +558,5 @@ $(function() {
     inputs.prop("checked", !inputs.prop("checked"));
     inputs.prop("checked", !inputs.prop("checked"));
   });
   });
 });
 });
+
+$.fn.modal.Constructor.prototype.enforceFocus = function() {};

circle/dashboard/static/dashboard/dashboard.less

@@ -1079,6 +1079,10 @@ textarea[name="new_members"] {
   max-width: 100%;
   max-width: 100%;
 }
 }
 
 
+#node-list-auto-migration-body {
+  padding: 20px;
+}
+
 #vm-list-table td.state,
 #vm-list-table td.state,
 #vm-list-table td.memory {
 #vm-list-table td.memory {
   white-space: nowrap;
   white-space: nowrap;
@@ -1088,7 +1092,7 @@ textarea[name="new_members"] {
   vertical-align: middle;
   vertical-align: middle;
 }
 }
 
 
-.disk-resize-btn {
+.disk-resize-btn, .disk-export-btn {
   margin-right: 5px;
   margin-right: 5px;
 }
 }
 
 

circle/dashboard/static/dashboard/node-list.js

@@ -3,4 +3,11 @@ $(function() {
     // find disabled nodes, set danger (red) on the rows
     // find disabled nodes, set danger (red) on the rows
     $('.node-disabled').closest("tr").addClass('danger');
     $('.node-disabled').closest("tr").addClass('danger');
   });
   });
+  $('#reschedule-now').click(function() {
+    $.get($(this).attr('href'), function(data){
+      highlight = data.result === 'ok' ? 'success' : 'danger';
+      addMessage(data.message, highlight);
+    });
+    return false;
+  });
 });
 });

circle/dashboard/store_api.py

@@ -14,19 +14,20 @@
 #
 #
 # You should have received a copy of the GNU General Public License along
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
-
-from os.path import splitext
 import json
 import json
 import logging
 import logging
 from urlparse import urljoin
 from urlparse import urljoin
-from datetime import datetime
 
 
-from django.http import Http404
+import os
+from datetime import datetime
 from django.conf import settings
 from django.conf import settings
+from django.http import Http404
+from os.path import splitext
 from requests import get, post, codes
 from requests import get, post, codes
-from requests.exceptions import Timeout  # noqa
 from sizefield.utils import filesizeformat
 from sizefield.utils import filesizeformat
 
 
+from storage.models import Disk
+
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 
 
 
@@ -47,6 +48,17 @@ class NoStoreException(StoreApiException):
 class Store(object):
 class Store(object):
 
 
     def __init__(self, user, default_timeout=0.5):
     def __init__(self, user, default_timeout=0.5):
+        self.store_url = settings.STORE_URL
+        if not self.store_url:
+            raise NoStoreException
+
+        if user.is_superuser and not user.profile.org_id:
+            self.username = 'u-admin'
+        elif not user.profile.org_id:
+            raise NoStoreException
+        else:
+            self.username = 'u-%s' % user.profile.org_id
+
         self.request_args = {'verify': settings.STORE_VERIFY_SSL}
         self.request_args = {'verify': settings.STORE_VERIFY_SSL}
         if settings.STORE_SSL_AUTH:
         if settings.STORE_SSL_AUTH:
             self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
             self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
@@ -54,18 +66,15 @@ class Store(object):
         if settings.STORE_BASIC_AUTH:
         if settings.STORE_BASIC_AUTH:
             self.request_args['auth'] = (settings.STORE_CLIENT_USER,
             self.request_args['auth'] = (settings.STORE_CLIENT_USER,
                                          settings.STORE_CLIENT_PASSWORD)
                                          settings.STORE_CLIENT_PASSWORD)
-        self.username = "u-%d" % user.pk
         self.default_timeout = default_timeout
         self.default_timeout = default_timeout
-        self.store_url = settings.STORE_URL
-        if not self.store_url:
-            raise NoStoreException
 
 
     def _request(self, url, method=get, timeout=None,
     def _request(self, url, method=get, timeout=None,
                  raise_status_code=True, **kwargs):
                  raise_status_code=True, **kwargs):
         url = urljoin(self.store_url, url)
         url = urljoin(self.store_url, url)
         if timeout is None:
         if timeout is None:
             timeout = self.default_timeout
             timeout = self.default_timeout
-        payload = json.dumps(kwargs) if kwargs else None
+        kwargs['USER'] = self.username
+        payload = json.dumps(kwargs)
         try:
         try:
             headers = {'content-type': 'application/json'}
             headers = {'content-type': 'application/json'}
             response = method(url, data=payload, headers=headers,
             response = method(url, data=payload, headers=headers,
@@ -83,7 +92,7 @@ class Store(object):
             return response
             return response
 
 
     def _request_cmd(self, cmd, **kwargs):
     def _request_cmd(self, cmd, **kwargs):
-        return self._request(self.username, post, CMD=cmd, **kwargs)
+        return self._request("/user/", post, CMD=cmd, **kwargs)
 
 
     def list(self, path, process=True):
     def list(self, path, process=True):
         r = self._request_cmd("LIST", PATH=path)
         r = self._request_cmd("LIST", PATH=path)
@@ -101,13 +110,22 @@ class Store(object):
         else:
         else:
             return result
             return result
 
 
+    def get_disk_images(self, path='/'):
+        images = []
+        file_list = self.list(path, process=False)
+        export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
+        for item in file_list:
+            if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
+                images.append(os.path.join(path, item['NAME']))
+        return images
+
     def request_download(self, path):
     def request_download(self, path):
         r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
         r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
         return r.json()['LINK']
         return r.json()['LINK']
 
 
     def request_upload(self, path):
     def request_upload(self, path):
-            r = self._request_cmd("UPLOAD", PATH=path)
-            return r.json()['LINK']
+        r = self._request_cmd("UPLOAD", PATH=path)
+        return r.json()['LINK']
 
 
     def remove(self, path):
     def remove(self, path):
         self._request_cmd("REMOVE", PATH=path)
         self._request_cmd("REMOVE", PATH=path)
@@ -119,7 +137,7 @@ class Store(object):
         self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
         self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
 
 
     def get_quota(self):  # no CMD? :o
     def get_quota(self):  # no CMD? :o
-        r = self._request(self.username)
+        r = self._request("/user/")
         quota = r.json()
         quota = r.json()
         quota.update({
         quota.update({
             'readable_used': filesizeformat(float(quota['used'])),
             'readable_used': filesizeformat(float(quota['used'])),
@@ -129,17 +147,17 @@ class Store(object):
         return quota
         return quota
 
 
     def set_quota(self, quota):
     def set_quota(self, quota):
-        self._request("/quota/" + self.username, post, QUOTA=quota)
+        self._request("/quota/", post, QUOTA=quota)
 
 
     def user_exist(self):
     def user_exist(self):
         try:
         try:
-            self._request(self.username)
+            self._request("/user/")
             return True
             return True
         except NotOkException:
         except NotOkException:
             return False
             return False
 
 
     def create_user(self, password, keys, quota):
     def create_user(self, password, keys, quota):
-        self._request("/new/" + self.username, method=post,
+        self._request("/new/", method=post,
                       SMBPASSWD=password, KEYS=keys, QUOTA=quota)
                       SMBPASSWD=password, KEYS=keys, QUOTA=quota)
 
 
     @staticmethod
     @staticmethod

circle/dashboard/templates/dashboard/_disk-list-element.html

@@ -6,15 +6,29 @@
 
 
 
 
 <span class="operation-wrapper pull-right">
 <span class="operation-wrapper pull-right">
+  {% if d.is_exportable %}
+    {% if op.export_disk %}
+      <a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
+         class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
+      {% if op.export_disk.disabled %}disabled{% endif %}">
+      <i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
+    </a>
+    {% endif %}
+  {% else %}
+    <small class="btn-xs">
+      {% trans "Not exportable" %}
+    </small>
+  {% endif %}
   {% if d.is_resizable %}
   {% if d.is_resizable %}
     {% if op.resize_disk %}
     {% if op.resize_disk %}
-      <a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
-        class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
+      <a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
+         class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
         {% if op.resize_disk.disabled %}disabled{% endif %}">
         {% if op.resize_disk.disabled %}disabled{% endif %}">
         <i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
         <i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
       </a>
       </a>
     {% else %}
     {% else %}
-      <a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
+      <a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
+         class="btn btn-xs btn-primary operation">
         <i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
         <i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
       </a>
       </a>
     {% endif %}
     {% endif %}
@@ -24,8 +38,8 @@
     </small>
     </small>
   {% endif %}
   {% endif %}
   {% if op.remove_disk %}
   {% if op.remove_disk %}
-    <a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
-      class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
+    <a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
+       class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
       {% if op.remove_disk.disabled %}disabled{% endif %}">
       {% if op.remove_disk.disabled %}disabled{% endif %}">
       <i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
       <i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
     </a>
     </a>

circle/dashboard/templates/dashboard/node-list.html

@@ -41,4 +41,23 @@
   </div><!-- -col-md-12 -->
   </div><!-- -col-md-12 -->
 </div><!-- .row -->
 </div><!-- .row -->
 
 
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <a id="reschedule-now" class="btn btn-danger pull-right" href="{% url "dashboard.views.reschedule" %}">
+          <i class="fa fa-magic"></i> {% trans "Reschedule now" %}
+        </a>
+        <h3 class="no-margin"><i class="fa fa-truck"></i> {% trans "Virtual machine auto migration" %}</h3>
+      </div>
+      <div id="node-list-auto-migration-body">
+        <h1>Crontab</h1>
+        <form>
+          {{ auto_migration_form.as_p }}
+        </form>
+      </div>
+    </div>
+  </div><!-- -col-md-12 -->
+</div><!-- .row -->
+
 {% endblock %}
 {% endblock %}

circle/dashboard/templates/dashboard/vm-detail/_disk-operations.html

 {% load i18n %}
 {% load i18n %}
 
 
 {% for op in ops %}
 {% for op in ops %}
-{% if op.is_disk_operation %}
-    <a href="{{op.get_url}}" class="btn btn-success btn-xs
-            operation operation-{{op.op}}">
-      <i class="fa fa-{{op.icon}} fa-fw-12"></i>
-            {{op.name}} </a>
-{% endif %}
+  {% if op.is_disk_operation %}
+    <a href="{{ op.get_url }}" class="btn btn-success btn-xs
+            operation operation-{{ op.op }}">
+      <i class="fa fa-{{ op.icon }} fa-fw-12"></i>
+      {{ op.name }} </a>
+  {% endif %}
 {% endfor %}
 {% endfor %}

circle/dashboard/templates/dashboard/vm-detail/network.html

@@ -37,6 +37,9 @@
         <dl>
         <dl>
           <dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
           <dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
           <dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
           <dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
+          {% if request.user.is_superuser %}
+          <dt>{% trans "MAC address" %}:</dt> <dd>{{ i.host.mac }}</dd>
+          {% endif %}
           <dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
           <dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
           <dt>{% trans "Groups" %}:</dt>
           <dt>{% trans "Groups" %}:</dt>
           <dd>
           <dd>
@@ -114,7 +117,9 @@
                   {% if l.ipv6 %}
                   {% if l.ipv6 %}
                     <tr>
                     <tr>
                       <td>
                       <td>
-                        {% display_portforward6 l %}
+                        {% autoescape off %}
+                          {% display_portforward6 l %}
+                        {% endautoescape %}
                       </td>
                       </td>
                       <td><i class="fa fa-long-arrow-right"></i></td>
                       <td><i class="fa fa-long-arrow-right"></i></td>
                       <td>
                       <td>

circle/dashboard/tests/test_views.py

@@ -534,7 +534,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
         with patch.object(DeployOperation, 'async') as async:
         with patch.object(DeployOperation, 'async') as async:
             response = c.post("/dashboard/vm/create/", {
             response = c.post("/dashboard/vm/create/", {
                 'name': 'vm',
                 'name': 'vm',
-                'amount': 2,
+                'amount': 1,
                 'customized': 1,
                 'customized': 1,
                 'template': 1,
                 'template': 1,
                 'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
                 'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
@@ -543,7 +543,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
 
 
         assert async.called
         assert async.called
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response.status_code, 302)
-        self.assertEqual(instance_count + 2, Instance.objects.all().count())
+        self.assertEqual(instance_count + 1, Instance.objects.all().count())
 
 
     def test_unpermitted_description_update(self):
     def test_unpermitted_description_update(self):
         c = Client()
         c = Client()

circle/dashboard/urls.py

@@ -56,6 +56,7 @@ from .views import (
     MessageList, MessageDetail, MessageCreate, MessageDelete,
     MessageList, MessageDetail, MessageCreate, MessageDelete,
     EnableTwoFactorView, DisableTwoFactorView,
     EnableTwoFactorView, DisableTwoFactorView,
     AclUserGroupAutocomplete, AclUserAutocomplete,
     AclUserGroupAutocomplete, AclUserAutocomplete,
+    RescheduleView,
 )
 )
 from .views.vm import vm_ops, vm_mass_ops
 from .views.vm import vm_ops, vm_mass_ops
 from .views.node import node_ops
 from .views.node import node_ops
@@ -153,6 +154,8 @@ urlpatterns = [
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
         NodeListGraphView.as_view(),
         NodeListGraphView.as_view(),
         name='dashboard.views.node-list-graph'),
         name='dashboard.views.node-list-graph'),
+    url(r'^node/reschedule/$', RescheduleView.as_view(),
+        name="dashboard.views.reschedule"),
     url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
     url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
         TemplateGraphView.as_view(),
         TemplateGraphView.as_view(),

circle/dashboard/views/node.py

@@ -25,7 +25,7 @@ from django.core.exceptions import PermissionDenied
 from django.core.urlresolvers import reverse_lazy
 from django.core.urlresolvers import reverse_lazy
 from django.db.models import Count
 from django.db.models import Count
 from django.forms.models import inlineformset_factory
 from django.forms.models import inlineformset_factory
-from django.http import HttpResponse
+from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect
 from django.shortcuts import redirect
 from django.template.loader import render_to_string
 from django.template.loader import render_to_string
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
@@ -37,11 +37,14 @@ from django_tables2 import SingleTableView
 from firewall.models import Host
 from firewall.models import Host
 from vm.models import Node, NodeActivity, Trait
 from vm.models import Node, NodeActivity, Trait
 from vm.tasks.vm_tasks import check_queue
 from vm.tasks.vm_tasks import check_queue
+from vm.tasks.local_periodic_tasks import auto_migrate
 
 
-from ..forms import TraitForm, HostForm, NodeForm
+from ..forms import TraitForm, HostForm, NodeForm, AutoMigrationForm
 from ..tables import NodeListTable
 from ..tables import NodeListTable
 from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
 from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
 
 
+from manager.mancelery import crontab_parser
+
 
 
 def get_operations(instance, user):
 def get_operations(instance, user):
     ops = []
     ops = []
@@ -190,6 +193,14 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
     table_class = NodeListTable
     table_class = NodeListTable
     table_pagination = False
     table_pagination = False
 
 
+    def get_crontab(self):
+        return crontab_parser(settings.AUTO_MIGRATION_CRONTAB)
+
+    def get_context_data(self):
+        context = super(NodeList, self).get_context_data()
+        context["auto_migration_form"] = AutoMigrationForm(self.get_crontab())
+        return context
+
     def get(self, *args, **kwargs):
     def get(self, *args, **kwargs):
         if not self.request.user.has_perm('vm.view_statistics'):
         if not self.request.user.has_perm('vm.view_statistics'):
             raise PermissionDenied()
             raise PermissionDenied()
@@ -210,9 +221,20 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
             return super(NodeList, self).get(*args, **kwargs)
             return super(NodeList, self).get(*args, **kwargs)
 
 
     def get_queryset(self):
     def get_queryset(self):
+        self.wrong_nodes_message()
         return Node.objects.annotate(
         return Node.objects.annotate(
             number_of_VMs=Count('instance_set')).select_related('host')
             number_of_VMs=Count('instance_set')).select_related('host')
 
 
+    def wrong_nodes_message(self):
+        wrong_nodes = []
+        for node in Node.objects.all():
+            if node.monitor_info is None:
+                wrong_nodes.append(node.name)
+        message = ', '.join(wrong_nodes)
+        if wrong_nodes:
+            messages.error(self.request,
+                           "Can't reach " + message + " monitor info")
+
 
 
 class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
 class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
 
 
@@ -356,3 +378,23 @@ class NodeActivityDetail(LoginRequiredMixin, SuperuserRequiredMixin,
         ).order_by('-started').select_related())
         ).order_by('-started').select_related())
         ctx['icon'] = _get_activity_icon(self.object)
         ctx['icon'] = _get_activity_icon(self.object)
         return ctx
         return ctx
+
+
+class RescheduleView(SuperuserRequiredMixin, View):
+    def get(self, *args, **kwargs):
+        try:
+            auto_migrate.apply_async(queue='localhost.man.slow')
+        except Exception as e:
+            msg = str(e)
+            result = 'error'
+        else:
+            result = 'ok'
+            msg = _('Reschedule has started.')
+        if self.request.is_ajax():
+            return JsonResponse({'result': result, 'message': msg})
+        else:
+            if result == 'ok':
+                messages.success(self.request, msg)
+            else:
+                messages.error(self.request, msg)
+        return redirect('dashboard.views.node-list')

circle/dashboard/views/store.py

@@ -35,7 +35,8 @@ from django.views.generic import TemplateView
 
 
 from braces.views import LoginRequiredMixin
 from braces.views import LoginRequiredMixin
 
 
-from ..store_api import Store, NoStoreException, NotOkException
+from ..store_api import (Store, NoStoreException,
+                         NotOkException)
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 

circle/dashboard/views/util.py

@@ -682,7 +682,7 @@ class TransferOwnershipConfirmView(LoginRequiredMixin, View):
             messages.error(request, _('This token is invalid or has expired.'))
             messages.error(request, _('This token is invalid or has expired.'))
             raise PermissionDenied()
             raise PermissionDenied()
         return render(request, self.template,
         return render(request, self.template,
-                      dictionary={'instance': instance, 'key': key})
+                      {'instance': instance, 'key': key})
 
 
     def change_owner(self, instance, new_owner):
     def change_owner(self, instance, new_owner):
         instance.owner = new_owner
         instance.owner = new_owner

circle/dashboard/views/vm.py

@@ -61,9 +61,10 @@ from .util import (
 )
 )
 from ..forms import (
 from ..forms import (
     AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
     AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
-    VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
+    VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
+    VmImportDiskForm, VmSaveForm,
     VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
     VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
-    VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
+    VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
     VmMigrateForm, VmDeployForm,
     VmMigrateForm, VmDeployForm,
     VmPortRemoveForm, VmPortAddForm,
     VmPortRemoveForm, VmPortAddForm,
     VmRemoveInterfaceForm,
     VmRemoveInterfaceForm,
@@ -166,8 +167,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
 
         # resources forms
         # resources forms
         can_edit = (
         can_edit = (
-            instance.has_level(user, "owner") and
-            self.request.user.has_perm("vm.change_resources"))
+                instance.has_level(user, "owner") and
+                self.request.user.has_perm("vm.change_resources"))
         context['resources_form'] = VmResourcesForm(
         context['resources_form'] = VmResourcesForm(
             can_edit=can_edit, instance=instance)
             can_edit=can_edit, instance=instance)
 
 
@@ -269,7 +270,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
             return JsonResponse({'message': message})
             return JsonResponse({'message': message})
         else:
         else:
             return redirect(reverse_lazy("dashboard.views.detail",
             return redirect(reverse_lazy("dashboard.views.detail",
-                            kwargs={'pk': self.object.pk}))
+                                         kwargs={'pk': self.object.pk}))
 
 
     def __abort_operation(self, request):
     def __abort_operation(self, request):
         self.object = self.get_object()
         self.object = self.get_object()
@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
 
 
 
 
 class VmOperationView(AjaxOperationMixin, OperationView):
 class VmOperationView(AjaxOperationMixin, OperationView):
-
     model = Instance
     model = Instance
     context_object_name = 'instance'  # much simpler to mock object
     context_object_name = 'instance'  # much simpler to mock object
 
 
@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmAddInterfaceView(FormOperationMixin, VmOperationView):
 class VmAddInterfaceView(FormOperationMixin, VmOperationView):
-
     op = 'add_interface'
     op = 'add_interface'
     form_class = VmAddInterfaceForm
     form_class = VmAddInterfaceForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmCreateDiskView(FormOperationMixin, VmOperationView):
 class VmCreateDiskView(FormOperationMixin, VmOperationView):
-
     op = 'create_disk'
     op = 'create_disk'
     form_class = VmCreateDiskForm
     form_class = VmCreateDiskForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
         return val
         return val
 
 
 
 
-class VmDownloadDiskView(FormOperationMixin, VmOperationView):
+class VmImportDiskView(FormOperationMixin, VmOperationView):
+    op = 'import_disk'
+    form_class = VmImportDiskForm
+    show_in_toolbar = False
+    icon = 'upload'
+    effect = "success"
+    is_disk_operation = True
+    with_reload = True
+
+    def get_form_kwargs(self):
+        val = super(VmImportDiskView, self).get_form_kwargs()
+        val.update({'user': self.request.user})
+        return val
+
 
 
+class VmDownloadDiskView(FormOperationMixin, VmOperationView):
     op = 'download_disk'
     op = 'download_disk'
     form_class = VmDownloadDiskForm
     form_class = VmDownloadDiskForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmMigrateView(FormOperationMixin, VmOperationView):
 class VmMigrateView(FormOperationMixin, VmOperationView):
-
     op = 'migrate'
     op = 'migrate'
     icon = 'truck'
     icon = 'truck'
     effect = 'info'
     effect = 'info'
@@ -449,8 +460,7 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
         if isinstance(inst, Instance):
         if isinstance(inst, Instance):
             nodes_w_traits = [
             nodes_w_traits = [
                 n.pk for n in Node.objects.filter(enabled=True)
                 n.pk for n in Node.objects.filter(enabled=True)
-                if n.online and
-                has_traits(inst.req_traits.all(), n)
+                if n.online and has_traits(inst.req_traits.all(), n)
             ]
             ]
             ctx['nodes_w_traits'] = nodes_w_traits
             ctx['nodes_w_traits'] = nodes_w_traits
 
 
@@ -458,7 +468,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmPortRemoveView(FormOperationMixin, VmOperationView):
 class VmPortRemoveView(FormOperationMixin, VmOperationView):
-
     template_name = 'dashboard/_vm-remove-port.html'
     template_name = 'dashboard/_vm-remove-port.html'
     op = 'remove_port'
     op = 'remove_port'
     show_in_toolbar = False
     show_in_toolbar = False
@@ -487,7 +496,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmPortAddView(FormOperationMixin, VmOperationView):
 class VmPortAddView(FormOperationMixin, VmOperationView):
-
     op = 'add_port'
     op = 'add_port'
     show_in_toolbar = False
     show_in_toolbar = False
     with_reload = True
     with_reload = True
@@ -514,7 +522,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmSaveView(FormOperationMixin, VmOperationView):
 class VmSaveView(FormOperationMixin, VmOperationView):
-
     op = 'save_as_template'
     op = 'save_as_template'
     icon = 'save'
     icon = 'save'
     effect = 'info'
     effect = 'info'
@@ -570,7 +577,7 @@ class TokenOperationView(OperationView):
     User can do the action with a valid token instead of logging in.
     User can do the action with a valid token instead of logging in.
     """
     """
     token_max_age = 3 * 24 * 3600
     token_max_age = 3 * 24 * 3600
-    redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
+    redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
 
 
     @classmethod
     @classmethod
     def get_salt(cls):
     def get_salt(cls):
@@ -642,7 +649,6 @@ class TokenOperationView(OperationView):
 
 
 
 
 class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
 class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
-
     op = 'renew'
     op = 'renew'
     icon = 'calendar'
     icon = 'calendar'
     effect = 'success'
     effect = 'success'
@@ -769,7 +775,11 @@ vm_ops = OrderedDict([
         extra_bases=[TokenOperationView],
         extra_bases=[TokenOperationView],
         op='destroy', icon='times', effect='danger')),
         op='destroy', icon='times', effect='danger')),
     ('create_disk', VmCreateDiskView),
     ('create_disk', VmCreateDiskView),
+    ('import_disk', VmImportDiskView),
     ('download_disk', VmDownloadDiskView),
     ('download_disk', VmDownloadDiskView),
+    ('export_disk', VmDiskModifyView.factory(
+        op='export_disk', form_class=VmDiskExportForm,
+        icon='download', effect='info')),
     ('resize_disk', VmDiskModifyView.factory(
     ('resize_disk', VmDiskModifyView.factory(
         op='resize_disk', form_class=VmDiskResizeForm,
         op='resize_disk', form_class=VmDiskResizeForm,
         icon='arrows-alt', effect="warning")),
         icon='arrows-alt', effect="warning")),
@@ -1015,7 +1025,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
         # remove "-" that means descending order
         # remove "-" that means descending order
         # also check if the column name is valid
         # also check if the column name is valid
         if (sort and
         if (sort and
-            (sort[1:] if sort[0] == "-" else sort)
+                (sort[1:] if sort[0] == "-" else sort)
                 in [i.name for i in Instance._meta.fields] + ["pk"]):
                 in [i.name for i in Instance._meta.fields] + ["pk"]):
             queryset = queryset.order_by(sort)
             queryset = queryset.order_by(sort)
 
 
@@ -1030,7 +1040,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
 
 
 
 
 class VmCreate(LoginRequiredMixin, TemplateView):
 class VmCreate(LoginRequiredMixin, TemplateView):
-
     form_class = VmCustomizeForm
     form_class = VmCustomizeForm
     form = None
     form = None
 
 
@@ -1139,7 +1148,7 @@ class VmCreate(LoginRequiredMixin, TemplateView):
             messages.success(request, ungettext_lazy(
             messages.success(request, ungettext_lazy(
                 "Successfully created %(count)d VM.",  # this should not happen
                 "Successfully created %(count)d VM.",  # this should not happen
                 "Successfully created %(count)d VMs.", len(instances)) % {
                 "Successfully created %(count)d VMs.", len(instances)) % {
-                'count': len(instances)})
+                                 'count': len(instances)})
             path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
             path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
         else:
         else:
             messages.success(request, _("VM successfully created."))
             messages.success(request, _("VM successfully created."))
@@ -1161,24 +1170,28 @@ class VmCreate(LoginRequiredMixin, TemplateView):
 
 
         # limit chekcs
         # limit chekcs
         try:
         try:
-            limit = user.profile.instance_limit
+            instance_limit = user.profile.instance_limit
+            template_instance_limit = user.profile.template_instance_limit
         except Exception as e:
         except Exception as e:
             logger.debug('No profile or instance limit: %s', e)
             logger.debug('No profile or instance limit: %s', e)
         else:
         else:
             try:
             try:
                 amount = int(request.POST.get("amount", 1))
                 amount = int(request.POST.get("amount", 1))
             except:
             except:
-                amount = limit  # TODO this should definitely use a Form
-            current = Instance.active.filter(owner=user).count()
-            logger.debug('current use: %d, limit: %d', current, limit)
-            if current + amount > limit:
-                messages.error(request,
-                               _('Instance limit (%d) exceeded.') % limit)
-                if request.is_ajax():
-                    return HttpResponse(json.dumps({'redirect': '/'}),
-                                        content_type="application/json")
-                else:
-                    return redirect('/')
+                # TODO this should definitely use a Form
+                amount = instance_limit
+            instances = Instance.active.filter(owner=user).count()
+            template_instances = template.get_user_instances(user).count()
+
+            logger.debug('current instance use: %d, limit: %d',
+                         instances, instance_limit)
+            logger.debug('current template instance use: %d, limit: %d',
+                         template_instances, template_instance_limit)
+
+            if instances + amount > instance_limit:
+                return self._limit_exceeded(instance_limit, request)
+            if template_instances + amount > template_instance_limit:
+                return self._limit_exceeded(template_instance_limit, request)
 
 
         create_func = (self.__create_normal if
         create_func = (self.__create_normal if
                        request.POST.get("customized") is None else
                        request.POST.get("customized") is None else
@@ -1186,6 +1199,16 @@ class VmCreate(LoginRequiredMixin, TemplateView):
 
 
         return create_func(request, template, *args, **kwargs)
         return create_func(request, template, *args, **kwargs)
 
 
+    def _limit_exceeded(self, limit, request):
+        messages.error(request,
+                       _('Instance limit (%d) exceeded.')
+                       % limit)
+        if request.is_ajax():
+            return HttpResponse(json.dumps({'redirect': '/'}),
+                                content_type="application/json")
+        else:
+            return redirect('/')
+
 
 
 @require_GET
 @require_GET
 def get_vm_screenshot(request, pk):
 def get_vm_screenshot(request, pk):
@@ -1331,7 +1354,7 @@ class TransferInstanceOwnershipConfirmView(TransferOwnershipConfirmView):
 
 
     def change_owner(self, instance, new_owner):
     def change_owner(self, instance, new_owner):
         with instance.activity(
         with instance.activity(
-            code_suffix='ownership-transferred',
+                code_suffix='ownership-transferred',
                 readable_name=ugettext_noop("transfer ownership"),
                 readable_name=ugettext_noop("transfer ownership"),
                 concurrency_check=False, user=new_owner):
                 concurrency_check=False, user=new_owner):
             super(TransferInstanceOwnershipConfirmView, self).change_owner(
             super(TransferInstanceOwnershipConfirmView, self).change_owner(

circle/manager/mancelery.py

@@ -17,13 +17,27 @@
 
 
 from celery import Celery
 from celery import Celery
 from celery.signals import worker_ready
 from celery.signals import worker_ready
+from celery.schedules import crontab
 from datetime import timedelta
 from datetime import timedelta
 from celery.schedules import crontab
 from celery.schedules import crontab
 from kombu import Queue, Exchange
 from kombu import Queue, Exchange
 from os import getenv
 from os import getenv
 
 
+
 HOSTNAME = "localhost"
 HOSTNAME = "localhost"
 QUEUE_NAME = HOSTNAME + '.man'
 QUEUE_NAME = HOSTNAME + '.man'
+AUTO_MIGRATION_CRONTAB = getenv('AUTO_MIGRATION_CRONTAB', '0 0 * * *')
+
+
+def crontab_parser(crontab):
+    fields = crontab.split(' ')
+    return dict(
+        minute=fields[0],
+        hour=fields[1],
+        day_of_month=fields[2],
+        month_of_year=fields[3],
+        day_of_week=fields[4],
+    )
 
 
 
 
 celery = Celery('manager',
 celery = Celery('manager',
@@ -56,6 +70,11 @@ celery.conf.update(
             'schedule': crontab(minute=10, hour=1),
             'schedule': crontab(minute=10, hour=1),
             'options': {'queue': 'localhost.man'}
             'options': {'queue': 'localhost.man'}
         },
         },
+#        'vm.local_periodic_tasks': {
+#            'task': 'vm.tasks.local_periodic_tasks.auto_migrate',
+#            'schedule': crontab(**crontab_parser(AUTO_MIGRATION_CRONTAB)),
+#            'options': {'queue': 'localhost.man.slow'},
+#        },
     }
     }
 
 
 )
 )

circle/manager/scheduler.py

@@ -15,15 +15,18 @@
 # You should have received a copy of the GNU General Public License along
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import datetime
+import json
+import random
 from logging import getLogger
 from logging import getLogger
 
 
+from django.conf import settings
+from django.core.cache import cache
+from django.utils import timezone
 from django.utils.translation import ugettext_noop
 from django.utils.translation import ugettext_noop
 
 
-from common.models import HumanReadableException
-
 from circle.settings.base import SCHEDULER_METHOD
 from circle.settings.base import SCHEDULER_METHOD
-
-import random
+from common.models import HumanReadableException
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
 
 
@@ -69,14 +72,14 @@ def common_select(instance, nodes):
         logger.warning('select_node: no enough RAM for %s', unicode(instance))
         logger.warning('select_node: no enough RAM for %s', unicode(instance))
         raise NotEnoughMemoryException()
         raise NotEnoughMemoryException()
 
 
-    # sort nodes first by processor usage, then priority
+    # sort nodes first by priority
     nodes.sort(key=lambda n: n.priority, reverse=True)
     nodes.sort(key=lambda n: n.priority, reverse=True)
-    nodes.sort(key=free_cpu_time, reverse=True)
     return nodes
     return nodes
 
 
 
 
 def common_evenly(instance, nodes):
 def common_evenly(instance, nodes):
     nodes = common_select(instance, nodes)
     nodes = common_select(instance, nodes)
+    nodes.sort(key=free_cpu_time, reverse=True)
     result = nodes[0]
     result = nodes[0]
     return result
     return result
 
 
@@ -87,6 +90,16 @@ def common_random(instance, nodes):
     return result
     return result
 
 
 
 
+def advanced_with_time_stamp(instance, nodes):
+    nodes = common_select(instance, nodes)
+    nodes.sort(key=sorting_key, reverse=True)
+    logger.info("SCHEDLOG: {}".format(json.dumps({
+        "event": "after_sort", 
+        "list": map(lambda node: unicode(node), nodes)})))
+    result = nodes[0]
+    return result
+
+
 def select_node(instance, nodes):
 def select_node(instance, nodes):
     ''' Select a node for hosting an instance based on its requirements.
     ''' Select a node for hosting an instance based on its requirements.
     '''
     '''
@@ -94,14 +107,72 @@ def select_node(instance, nodes):
         result = common_evenly(instance, nodes)
         result = common_evenly(instance, nodes)
     elif SCHEDULER_METHOD == 'random':
     elif SCHEDULER_METHOD == 'random':
         result = common_random(instance, nodes)
         result = common_random(instance, nodes)
+    elif SCHEDULER_METHOD == 'advanced':
+        result = advanced_with_time_stamp(instance, nodes)
     else:  # Default method is the random
     else:  # Default method is the random
         result = common_random(instance, nodes)
         result = common_random(instance, nodes)
 
 
-    logger.info('Scheduler method: %s selected', unicode(SCHEDULER_METHOD))
-    logger.info('select_node: %s for %s', unicode(result), unicode(instance))
+    logger.info("SCHEDLOG: {}".format(json.dumps(
+        {"event": "select",
+         "node": unicode(result),
+          "vm": unicode(instance)})))
+
+    set_time_stamp(result)
+
     return result
     return result
 
 
 
 
+def sorting_key(node):
+    """Determines how valuable a node is for scheduling.
+    """
+    key = 0
+    corr = last_scheduled_correction_factor(node)
+    if free_cpu_time(node) < free_ram(node):
+        key = free_cpu_time(node) * corr
+    else:
+        key = free_ram(node) * corr
+    logger.info("SCHEDLOG: {}".format(json.dumps({
+        "event": "sort",
+        "node": unicode(node),
+        "sorting_key": unicode(key),
+        "free_cpu_time": unicode(free_cpu_time(node)),
+        "free_ram": unicode(free_ram(node)),
+        "last_scheduled_correction_factor": unicode(last_scheduled_correction_factor(node))})))
+    return key
+
+
+def set_time_stamp(node):
+    cache.set('time_stamp{}'.format(node.id), timezone.now())
+
+
+def get_time_stamp(node):
+    time_stamp = cache.get('time_stamp{}'.format(node.id))
+    if time_stamp:
+        return time_stamp
+    return datetime.datetime(1970, 1, 1, tzinfo=timezone.get_current_timezone())
+
+
+def last_scheduled_correction_factor(node):
+    """Returns the time correction factor for a node.
+
+    The monitor data may be outdated, because of recent scheduling for a given node.
+    The return value is between 0 and 1, higher value indicates more time since the
+    last scheduling for the given node.
+    """
+    factor = 0
+    max_time_diff = settings.SCHEDULER_TIME_SENSITIVITY_IN_SECONDS
+    current_time = timezone.now()
+    time_difference_in_seconds = (
+        current_time - get_time_stamp(node)).total_seconds()
+    factor = time_difference_in_seconds/float(max_time_diff)
+    if factor > 1:
+        factor = 1
+    elif factor < 0:
+        factor = 1
+    logger.info('Scheduler set factor to %s', unicode(factor))
+    return factor
+
+
 def has_traits(traits, node):
 def has_traits(traits, node):
     """True, if the node has all specified traits; otherwise, false.
     """True, if the node has all specified traits; otherwise, false.
     """
     """
@@ -142,11 +213,27 @@ def free_cpu_time(node):
     Higher values indicate more idle time.
     Higher values indicate more idle time.
     """
     """
     try:
     try:
-        activity = node.cpu_usage / 100
-        inactivity = 1 - activity
-        cores = node.num_cores
-        return cores * inactivity
+        free_cpu_percent = 1 - node.cpu_usage
+        weight = node.cpu_weight
+        weighted_value = free_cpu_percent * weight
+        return weighted_value
+    except TypeError as e:
+        logger.exception('Got incorrect monitoring data for node %s. %s',
+                         unicode(node), unicode(e))
+        return 0  # will result lowest priority
+
+
+def free_ram(node):
+    """Get an indicator number for free RAM on the node.
+
+    Higher value indicates more RAM.
+    """
+    try:
+        free_ram_percent = 1 - node.ram_usage
+        weight = node.ram_weight
+        weighted_value = free_ram_percent * weight
+        return weighted_value
     except TypeError as e:
     except TypeError as e:
-        logger.warning('Got incorrect monitoring data for node %s. %s',
-                       unicode(node), unicode(e))
-        return False  # monitoring data is incorrect
+        logger.exception('Got incorrect monitoring data for node %s. %s',
+                         unicode(node), unicode(e))
+        return 0  # will result lowest priority

circle/request/forms.py

@@ -16,7 +16,7 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 from django.forms import (
 from django.forms import (
     ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
     ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
-    Textarea, ValidationError
+    Textarea, ValidationError, TextInput, IntegerField, EmailField
 )
 )
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 from django.template.loader import render_to_string
 from django.template.loader import render_to_string
@@ -27,11 +27,78 @@ from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Submit
 from crispy_forms.layout import Submit
 
 
 from request.models import (
 from request.models import (
-    LeaseType, TemplateAccessType, TemplateAccessAction,
+    LeaseType, TemplateAccessType, TemplateAccessAction, RequestField
 )
 )
 from dashboard.forms import VmResourcesForm
 from dashboard.forms import VmResourcesForm
 
 
 
 
+class RequestFieldModelForm(ModelForm):
+    class Meta:
+        model = RequestField
+        fields = '__all__'
+        widgets = {
+            'choices': TextInput(attrs={'placeholder': 'Optional'}),
+            }
+
+    def __init__(self, *args, **kwargs):
+        super(RequestFieldModelForm, self).__init__(*args, **kwargs)
+        self.fields['choices'].required = False
+
+    @property
+    def helper(self):
+        helper = FormHelper()
+        return helper
+
+    def clean(self):
+        cleaned_data = super(RequestFieldModelForm, self).clean()
+        if cleaned_data['type'] == 'Email' and cleaned_data['choices']:
+            raise ValidationError(_("Email field can't have choices!"))
+        if cleaned_data['type'] == 'Integer':
+            for choice in cleaned_data['choices'].split(','):
+                try:
+                    int(choice)
+                except:
+                    raise ValidationError(_("IntegerField choices must be \
+                                            integers"))
+
+
+class EditableForm(Form):
+    def __init__(self, *args, **kwargs):
+        type = kwargs.pop('type', None)
+        kwargs.pop("request", None)
+        super(EditableForm, self).__init__(*args, **kwargs)
+        fields = RequestField.objects.filter(request_type=type)
+        n = 0
+        if fields:
+            for field in fields:
+                n = n+1
+                type = field.type
+                if(type == 'Char'):
+                    self.fields['field'+str(n)] = CharField(
+                                                   max_length=30,
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                elif (type == 'Integer'):
+                    self.fields['field'+str(n)] = IntegerField(
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                elif (type == 'Email'):
+                    self.fields['field'+str(n)] = EmailField(
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                if(field.choices):
+                    choices = [(ch, ch)for ch in field.choices.split(',')]
+                    self.fields['field'+str(n)] = ChoiceField(
+                                                  choices=choices,
+                                                  required=field.required,
+                                                  label=field.fieldname)
+
+    def get_dynamic_fields(self):
+        for field_name in self.fields:
+            if field_name.startswith("field"):
+                yield self[field_name]
+
+
 class LeaseTypeForm(ModelForm):
 class LeaseTypeForm(ModelForm):
     @property
     @property
     def helper(self):
     def helper(self):
@@ -80,27 +147,18 @@ class InitialFromFileMixin(object):
         return message.strip()
         return message.strip()
 
 
 
 
-class TemplateRequestForm(InitialFromFileMixin, Form):
-    message = CharField(widget=Textarea, label=_("Message"))
+class TemplateRequestForm(EditableForm):
     template = ModelChoiceField(TemplateAccessType.objects.all(),
     template = ModelChoiceField(TemplateAccessType.objects.all(),
                                 label=_("Template share"))
                                 label=_("Template share"))
     level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
     level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
                         initial=TemplateAccessAction.LEVELS.user)
                         initial=TemplateAccessAction.LEVELS.user)
 
 
-    initial_template = "request/initials/template.html"
-
 
 
-class LeaseRequestForm(InitialFromFileMixin, Form):
+class LeaseRequestForm(EditableForm):
     lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
     lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
-    message = CharField(widget=Textarea, label=_("Message"))
-
-    initial_template = "request/initials/lease.html"
 
 
 
 
-class ResourceRequestForm(InitialFromFileMixin, VmResourcesForm):
-    message = CharField(widget=Textarea, label=_("Message"))
-
-    initial_template = "request/initials/resources.html"
+class ResourceRequestForm(EditableForm, VmResourcesForm):
 
 
     def clean(self):
     def clean(self):
         cleaned_data = super(ResourceRequestForm, self).clean()
         cleaned_data = super(ResourceRequestForm, self).clean()

circle/request/migrations/0005_requestfield.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-12 15:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0004_auto_20150629_1605'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RequestField',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('fieldname', models.CharField(max_length=50, unique=True)),
+                ('type', models.CharField(choices=[(b'Char', b'CharField'), (b'Integer', b'IntegerField'), (b'Email', b'EmailField')], default=b'Char', max_length=20)),
+                ('choices', models.CharField(max_length=100, null=True)),
+                ('required', models.BooleanField(default=True)),
+            ],
+        ),
+    ]

circle/request/migrations/0006_auto_20181115_1548.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-15 15:48
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0005_requestfield'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requestfield',
+            name='choices',
+            field=models.CharField(max_length=300, null=True),
+        ),
+    ]

circle/request/migrations/0007_requestfield_request_type.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-12-12 14:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0006_auto_20181115_1548'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='requestfield',
+            name='request_type',
+            field=models.CharField(choices=[(b'resource', 'resource request'), (b'lease', 'lease request'), (b'template', 'template access request'), (b'resize', 'disk resize request')], default=b'template', max_length=20),
+        ),
+    ]

circle/request/models.py

@@ -19,6 +19,7 @@ import logging
 
 
 from django.db.models import (
 from django.db.models import (
     Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
     Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
+    BooleanField,
 )
 )
 from django.db.models.signals import post_save
 from django.db.models.signals import post_save
 from django.conf import settings
 from django.conf import settings
@@ -157,6 +158,27 @@ class Request(TimeStampedModel):
         return self.action.is_acceptable()
         return self.action.is_acceptable()
 
 
 
 
+class RequestField(Model):
+    TYPES = (
+        ('Char', 'CharField'),
+        ('Integer', 'IntegerField'),
+        ('Email', 'EmailField')
+        )
+
+    fieldname = CharField(max_length=50, blank=False, unique=True)
+    type = CharField(choices=TYPES, default='Char', max_length=20)
+    request_type = CharField(choices=Request.TYPES, default='template',
+                             max_length=20)
+    choices = CharField(max_length=300, null=True)
+    required = BooleanField(default=True)
+
+    def __unicode__(self):
+        return self.fieldname
+
+    def get_absolute_url(self):
+        return reverse('fields_detail', kwargs={'pk': self.pk})
+
+
 class LeaseType(RequestType):
 class LeaseType(RequestType):
     lease = ForeignKey(Lease, verbose_name=_("Lease"))
     lease = ForeignKey(Lease, verbose_name=_("Lease"))
 
 

circle/request/templates/request/_request-field-form.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block content %}
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading"></div>
+      <div class="panel-body">
+          <form action={% url 'request.views.request-field-add' %} method="post">
+            {% include "display-form-errors.html" %}
+            {% csrf_token %}
+            {% for field in form %}
+              {{ field|as_crispy_field}}
+            {% endfor %}
+            <button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
+          </form>
+      </div> <!-- .panel-body -->
+    </div>
+  </div>
+</div>
+{% endblock %}

circle/request/templates/request/_request-lease-form.html

@@ -6,5 +6,8 @@
   {% csrf_token %}
   {% csrf_token %}
   {{ form.lease|as_crispy_field }}
   {{ form.lease|as_crispy_field }}
   {{ form.message|as_crispy_field }}
   {{ form.message|as_crispy_field }}
+  {% for fields in form.get_dynamic_fields %}
+    {{ field|as_crispy_field }}
+  {% endfor %}
   <input type="submit" class="btn btn-primary"/>
   <input type="submit" class="btn btn-primary"/>
 </form>
 </form>

circle/request/templates/request/_request-template-form.html

@@ -21,6 +21,8 @@
     </label>
     </label>
   </div>
   </div>
   {% endfor %}
   {% endfor %}
-  {{ form.message|as_crispy_field }}
+  {% for field in form.get_dynamic_fields %}
+      {{ field|as_crispy_field }}
+  {% endfor %}
   <input type="submit" class="btn btn-primary"/>
   <input type="submit" class="btn btn-primary"/>
 </form>
 </form>

circle/request/templates/request/field-list.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+{% load crispy_forms_field %}
+
+{% block content %}
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+          <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Request fields" %}</h3>
+         </div>
+            <div class="panel-body">
+              <div class="table-responsive">
+                <div class="table-container">
+                  <table class="text-center table table-bordered table-striped table-hover" >
+                    <thead class"align-center">
+                        <tr>
+                        <th class="text-center">{% trans "Fieldname" %}</th>
+                        <th class="text-center">{% trans "Type" %}</th>
+                        <th class="text-center">{% trans "RequestType" %}</th>
+                        <th class="text-center">{% trans "Choices" %}</th>
+                        <th class="text-center">{% trans "Required" %}</th>
+                        <th class="text-center">{% trans "Delete" %}</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for field in object_list %}
+                        <tr>
+                          <td>{{field.fieldname}}</td>
+                          <td>{{field.type}}</td>
+                          <td>{{field.request_type}}</td>
+                          <td>{{field.choices}}</td>
+                          <td>{{field.required}}</td>
+                          <td><a href={% url "request.views.field-delete" pk=field.pk %}>
+                            <i class="fa fa-times"></i></a>
+                          </td>
+                        </tr>
+                        {% endfor %}
+                        <tr>
+                        <form action={% url 'request.views.request-field-add' %} method="post">
+                          {% csrf_token %}
+                          {% for field in add_form %}
+                              <td> {% crispy_field field %}</td>
+                          {% endfor %}
+                        <td><button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
+                      </form>
+                      </tr>
+                      </tbody>
+                  </table>
+                </div>
+              </div> <!-- .table-responsive -->
+            </div> <!-- .panel-body -->
+          </div>
+        </div>
+      </div>
+
+{% endblock %}

circle/request/templates/request/list.html

@@ -11,9 +11,14 @@
   <div class="col-md-12">
   <div class="col-md-12">
     <div class="panel panel-default">
     <div class="panel panel-default">
       <div class="panel-heading">
       <div class="panel-heading">
-        <a class="btn btn-xs btn-primary pull-right "href="{% url "request.views.type-list" %}">
-          {% trans "Request types" %}
-        </a>
+        <div class="pull-right">
+            <a class="btn btn-xs btn-primary"href="{% url "request.views.type-list" %}">
+              {% trans "Request types" %}
+            </a>
+            <a class="btn btn-xs btn-success" href="{% url "request.views.field-list" %}">
+              {% trans "Request fields" %}
+            </a>
+          </div>
         <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Requests" %}</h3>
         <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Requests" %}</h3>
       </div>
       </div>
       <div class="panel-body">
       <div class="panel-body">

circle/request/templates/request/request-resource.html

@@ -24,7 +24,9 @@
           </div>
           </div>
           {% include "display-form-errors.html" %}
           {% include "display-form-errors.html" %}
           {% include "dashboard/_resources-sliders.html" with field_priority=form.priority field_num_cores=form.num_cores field_ram_size=form.ram_size %}
           {% include "dashboard/_resources-sliders.html" with field_priority=form.priority field_num_cores=form.num_cores field_ram_size=form.ram_size %}
-          {{ form.message|as_crispy_field }}
+          {% for field in form.get_dynamic_fields %}
+              {{ field|as_crispy_field }}
+          {% endfor %}
           <button type="submit" class="btn btn-success">
           <button type="submit" class="btn btn-success">
             {% trans "Request new resources" %}
             {% trans "Request new resources" %}
           </button>
           </button>

circle/request/tests.py

@@ -25,7 +25,7 @@ from mock import Mock, patch
 from common.tests.celery_mock import MockCeleryMixin
 from common.tests.celery_mock import MockCeleryMixin
 from vm.models import Instance, InstanceTemplate, Lease
 from vm.models import Instance, InstanceTemplate, Lease
 from dashboard.models import Profile
 from dashboard.models import Profile
-from request.models import Request, LeaseType, TemplateAccessType
+from request.models import Request, LeaseType, TemplateAccessType, RequestField
 from dashboard.tests.test_views import LoginMixin
 from dashboard.tests.test_views import LoginMixin
 from vm.operations import ResourcesOperation
 from vm.operations import ResourcesOperation
 
 
@@ -67,12 +67,16 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
         inst = Instance.objects.get(pk=1)
         inst = Instance.objects.get(pk=1)
         inst.set_level(self.u1, 'owner')
         inst.set_level(self.u1, 'owner')
 
 
+        field = RequestField(fieldname="Oka", type="Char",
+                             request_type="resource", required=True)
+        field.save()
+
         req_count = Request.objects.count()
         req_count = Request.objects.count()
         resp = c.post("/request/resource/1/", {
         resp = c.post("/request/resource/1/", {
             'num_cores': 5,
             'num_cores': 5,
             'ram_size': 512,
             'ram_size': 512,
             'priority': 30,
             'priority': 30,
-            'message': "szia",
+            'field1': "szia",
         })
         })
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(req_count + 1, Request.objects.count())
         self.assertEqual(req_count + 1, Request.objects.count())
@@ -104,17 +108,25 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
         template = InstanceTemplate.objects.get(pk=1)
         template = InstanceTemplate.objects.get(pk=1)
         self.assertFalse(template.has_level(self.u1, "user"))
         self.assertFalse(template.has_level(self.u1, "user"))
 
 
+        field = RequestField(fieldname="Tanszek", type="Char",
+                             request_type="template", required=True)
+        field.save()
+        field = RequestField(fieldname="Szobaszam", type="Integer",
+                             request_type="template", required=False)
+        field.save()
+
         req_count = Request.objects.count()
         req_count = Request.objects.count()
         resp = c.post("/request/template/", {
         resp = c.post("/request/template/", {
             'template': 1,
             'template': 1,
             'level': "user",
             'level': "user",
-            'message': "szia",
+            'field1': "IIT",
+            'field2': 10,
         })
         })
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(req_count + 1, Request.objects.count())
         self.assertEqual(req_count + 1, Request.objects.count())
         new_request = Request.objects.latest("pk")
         new_request = Request.objects.latest("pk")
         self.assertEqual(new_request.status, "PENDING")
         self.assertEqual(new_request.status, "PENDING")
-
+        self.assertEqual(new_request.message, "Tanszek: IIT\nSzobaszam: 10\n")
         new_request.accept(self.us)
         new_request.accept(self.us)
 
 
         new_request = Request.objects.latest("pk")
         new_request = Request.objects.latest("pk")

circle/request/urls.py

@@ -24,6 +24,8 @@ from .views import (
     TemplateAccessTypeCreate, TemplateAccessTypeDetail,
     TemplateAccessTypeCreate, TemplateAccessTypeDetail,
     TemplateRequestView, LeaseRequestView, ResourceRequestView,
     TemplateRequestView, LeaseRequestView, ResourceRequestView,
     LeaseTypeDelete, TemplateAccessTypeDelete, ResizeRequestView,
     LeaseTypeDelete, TemplateAccessTypeDelete, ResizeRequestView,
+    RequestFieldFormView, RequestFieldListView, RequestFieldDetailView,
+    RequestFieldDeleteView,
 )
 )
 
 
 urlpatterns = [
 urlpatterns = [
@@ -35,6 +37,15 @@ urlpatterns = [
     url(r'^type/list/$', RequestTypeList.as_view(),
     url(r'^type/list/$', RequestTypeList.as_view(),
         name="request.views.type-list"),
         name="request.views.type-list"),
 
 
+    url(r'fields/add/$', RequestFieldFormView.as_view(),
+        name='request.views.request-field-add'),
+    url(r'fields/$', RequestFieldListView.as_view(),
+        name='request.views.field-list'),
+    url(r'fields/field/(?P<pk>[0-9]+)/$', RequestFieldDetailView.as_view(),
+        name='request.views.fields-detail'),
+    url(r'^field/delete/(?P<pk>\d+)/$', RequestFieldDeleteView.as_view(),
+        name='request.views.field-delete'),
+
     # request types
     # request types
     url(r'^type/lease/create/$', LeaseTypeCreate.as_view(),
     url(r'^type/lease/create/$', LeaseTypeCreate.as_view(),
         name="request.views.lease-type-create"),
         name="request.views.lease-type-create"),

circle/request/views.py

@@ -18,6 +18,7 @@ from __future__ import unicode_literals, absolute_import
 
 
 from django.views.generic import (
 from django.views.generic import (
     UpdateView, TemplateView, DetailView, CreateView, FormView, DeleteView,
     UpdateView, TemplateView, DetailView, CreateView, FormView, DeleteView,
+    ListView
 )
 )
 from django.contrib import messages
 from django.contrib import messages
 from django.contrib.messages.views import SuccessMessageMixin
 from django.contrib.messages.views import SuccessMessageMixin
@@ -32,7 +33,7 @@ from django_tables2 import SingleTableView
 
 
 from request.models import (
 from request.models import (
     Request, TemplateAccessType, LeaseType, TemplateAccessAction,
     Request, TemplateAccessType, LeaseType, TemplateAccessAction,
-    ExtendLeaseAction, ResourceChangeAction, DiskResizeAction
+    ExtendLeaseAction, ResourceChangeAction, DiskResizeAction, RequestField
 )
 )
 from storage.models import Disk
 from storage.models import Disk
 from vm.models import Instance
 from vm.models import Instance
@@ -42,7 +43,39 @@ from request.tables import (
 from request.forms import (
 from request.forms import (
     LeaseTypeForm, TemplateAccessTypeForm, TemplateRequestForm,
     LeaseTypeForm, TemplateAccessTypeForm, TemplateRequestForm,
     LeaseRequestForm, ResourceRequestForm, ResizeRequestForm,
     LeaseRequestForm, ResourceRequestForm, ResizeRequestForm,
+    RequestFieldModelForm
 )
 )
+from django.urls import reverse_lazy
+
+
+class RequestFieldFormView(LoginRequiredMixin, CreateView):
+    template_name = 'request/_request-field-form.html'
+    model = RequestField
+    form_class = RequestFieldModelForm
+    success_url = reverse_lazy('request.views.field-list')
+
+
+class RequestFieldListView(LoginRequiredMixin, ListView):
+    template_name = 'request/field-list.html'
+    model = RequestField
+
+    def get_context_data(self, *args, **kwargs):
+        ctx = super(RequestFieldListView, self).get_context_data(*args,
+                                                                 **kwargs)
+        ctx['add_form'] = RequestFieldModelForm()
+        ctx['add_form'].helper.form_show_labels = False
+
+        return ctx
+
+
+class RequestFieldDetailView(LoginRequiredMixin, DetailView):
+    model = RequestField
+
+
+class RequestFieldDeleteView(LoginRequiredMixin, DeleteView):
+    model = RequestField
+    template_name = "dashboard/confirm/base-delete.html"
+    success_url = reverse_lazy('request.views.field-list')
 
 
 
 
 class RequestList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
 class RequestList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
@@ -173,7 +206,7 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
 
 
     def get_form_kwargs(self):
     def get_form_kwargs(self):
         kwargs = super(TemplateRequestView, self).get_form_kwargs()
         kwargs = super(TemplateRequestView, self).get_form_kwargs()
-        kwargs['request'] = self.request
+        kwargs['type'] = 'template'
         return kwargs
         return kwargs
 
 
     def form_valid(self, form):
     def form_valid(self, form):
@@ -187,9 +220,13 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
         )
         )
         ta.save()
         ta.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.template,
             type=Request.TYPES.template,
             action=ta
             action=ta
         )
         )
@@ -236,6 +273,12 @@ class LeaseRequestView(VmRequestMixin, FormView):
     user_level = "operator"
     user_level = "operator"
     success_message = _("Request successfully sent.")
     success_message = _("Request successfully sent.")
 
 
+    def get_form_kwargs(self):
+        kwargs = super(LeaseRequestView, self).get_form_kwargs()
+        kwargs['type'] = 'resource'
+
+        return kwargs
+
     def form_valid(self, form):
     def form_valid(self, form):
         data = form.cleaned_data
         data = form.cleaned_data
         user = self.request.user
         user = self.request.user
@@ -247,9 +290,14 @@ class LeaseRequestView(VmRequestMixin, FormView):
         )
         )
         el.save()
         el.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
+
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.lease,
             type=Request.TYPES.lease,
             action=el
             action=el
         )
         )
@@ -267,6 +315,7 @@ class ResourceRequestView(VmRequestMixin, FormView):
 
 
     def get_form_kwargs(self):
     def get_form_kwargs(self):
         kwargs = super(ResourceRequestView, self).get_form_kwargs()
         kwargs = super(ResourceRequestView, self).get_form_kwargs()
+        kwargs['type'] = 'resource'
         kwargs['can_edit'] = True
         kwargs['can_edit'] = True
         kwargs['instance'] = self.get_vm()
         kwargs['instance'] = self.get_vm()
         return kwargs
         return kwargs
@@ -292,9 +341,14 @@ class ResourceRequestView(VmRequestMixin, FormView):
         )
         )
         rc.save()
         rc.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
+
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.resource,
             type=Request.TYPES.resource,
             action=rc
             action=rc
         )
         )

circle/storage/migrations/0003_auto_20200424_2000.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-04-24 20:00
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storage', '0002_disk_bus'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='disk',
+            options={'ordering': ['name'], 'permissions': (('create_empty_disk', 'Can create an empty disk.'), ('download_disk', 'Can download a disk.'), ('resize_disk', 'Can resize a disk.'), ('import_disk', 'Can import a disk.'), ('export_disk', 'Can export a disk.')), 'verbose_name': 'disk', 'verbose_name_plural': 'disks'},
+        ),
+    ]

circle/storage/models.py

@@ -20,31 +20,30 @@
 from __future__ import unicode_literals
 from __future__ import unicode_literals
 
 
 import logging
 import logging
-from os.path import join
 import uuid
 import uuid
-import re
 
 
+import re
 from celery.contrib.abortable import AbortableAsyncResult
 from celery.contrib.abortable import AbortableAsyncResult
-from django.db.models import (Model, BooleanField, CharField, DateTimeField,
-                              ForeignKey)
+from celery.exceptions import TimeoutError
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
+from django.db.models import (Model, BooleanField, CharField, DateTimeField,
+                              ForeignKey)
 from django.utils import timezone
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from model_utils.models import TimeStampedModel
 from model_utils.models import TimeStampedModel
+from os.path import join
 from sizefield.models import FileSizeField
 from sizefield.models import FileSizeField
 
 
-from .tasks import local_tasks, storage_tasks
-from celery.exceptions import TimeoutError
 from common.models import (
 from common.models import (
     WorkerNotFound, HumanReadableException, humanize_exception, method_cache
     WorkerNotFound, HumanReadableException, humanize_exception, method_cache
 )
 )
+from .tasks import local_tasks, storage_tasks
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 
 
 
 class DataStore(Model):
 class DataStore(Model):
-
     """Collection of virtual disks.
     """Collection of virtual disks.
     """
     """
     name = CharField(max_length=100, unique=True, verbose_name=_('name'))
     name = CharField(max_length=100, unique=True, verbose_name=_('name'))
@@ -119,12 +118,15 @@ class DataStore(Model):
 
 
 
 
 class Disk(TimeStampedModel):
 class Disk(TimeStampedModel):
-
     """A virtual disk.
     """A virtual disk.
     """
     """
     TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
     TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
              ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
              ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
     BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
     BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
+    EXPORT_FORMATS = (('qcow2', _('QEMU disk image')),
+                      ('vmdk', _('VMware disk image')),
+                      ('vdi', _('VirtualBox disk image')),
+                      ('vpc', _('HyperV disk image')))
     name = CharField(blank=True, max_length=100, verbose_name=_("name"))
     name = CharField(blank=True, max_length=100, verbose_name=_("name"))
     filename = CharField(max_length=256, unique=True,
     filename = CharField(max_length=256, unique=True,
                          verbose_name=_("filename"))
                          verbose_name=_("filename"))
@@ -149,7 +151,9 @@ class Disk(TimeStampedModel):
         permissions = (
         permissions = (
             ('create_empty_disk', _('Can create an empty disk.')),
             ('create_empty_disk', _('Can create an empty disk.')),
             ('download_disk', _('Can download a disk.')),
             ('download_disk', _('Can download a disk.')),
-            ('resize_disk', _('Can resize a disk.'))
+            ('resize_disk', _('Can resize a disk.')),
+            ('import_disk', _('Can import a disk.')),
+            ('export_disk', _('Can export a disk.'))
         )
         )
 
 
     class DiskError(HumanReadableException):
     class DiskError(HumanReadableException):
@@ -427,6 +431,19 @@ class Disk(TimeStampedModel):
         return disk
         return disk
 
 
     @classmethod
     @classmethod
+    def _run_abortable_task(cls, remote, task):
+        while True:
+            try:
+                result = remote.get(timeout=5)
+                break
+            except TimeoutError as e:
+                if task is not None and task.is_aborted():
+                    AbortableAsyncResult(remote.id).abort()
+                    raise humanize_exception(ugettext_noop(
+                        "Operation aborted by user."), e)
+        return result
+
+    @classmethod
     def download(cls, url, task, user=None, **params):
     def download(cls, url, task, user=None, **params):
         """Create disk object and download data from url synchronusly.
         """Create disk object and download data from url synchronusly.
 
 
@@ -451,15 +468,7 @@ class Disk(TimeStampedModel):
             kwargs={'url': url, 'parent_id': task.request.id,
             kwargs={'url': url, 'parent_id': task.request.id,
                     'disk': disk.get_disk_desc()},
                     'disk': disk.get_disk_desc()},
             queue=queue_name)
             queue=queue_name)
-        while True:
-            try:
-                result = remote.get(timeout=5)
-                break
-            except TimeoutError as e:
-                if task is not None and task.is_aborted():
-                    AbortableAsyncResult(remote.id).abort()
-                    raise humanize_exception(ugettext_noop(
-                        "Operation aborted by user."), e)
+        result = cls._run_abortable_task(remote, task)
         disk.size = result['size']
         disk.size = result['size']
         disk.type = result['type']
         disk.type = result['type']
         disk.checksum = result.get('checksum', None)
         disk.checksum = result.get('checksum', None)
@@ -467,6 +476,40 @@ class Disk(TimeStampedModel):
         disk.save()
         disk.save()
         return disk
         return disk
 
 
+    @classmethod
+    def import_disk(cls, user, name, download_link, task):
+        params = {'name': name,
+                  'type': 'qcow2-norm'}
+        disk = cls.__create(user=user, params=params)
+        queue_name = disk.get_remote_queue_name('storage', priority='slow')
+        remote = storage_tasks.import_disk.apply_async(
+            kwargs={
+                "disk_desc": disk.get_disk_desc(),
+                "url": download_link,
+                "task": task.request.id
+            },
+            queue=queue_name
+        )
+        result = cls._run_abortable_task(remote, task)
+        disk.size = result["size"]
+        disk.checksum = result["checksum"]
+        disk.is_ready = True
+        disk.save()
+        return disk
+
+    def export(self, exported_name, disk_format, upload_link, task):
+        queue_name = self.get_remote_queue_name('storage', priority='slow')
+        remote = storage_tasks.export_disk.apply_async(
+            kwargs={
+                "disk_desc": self.get_disk_desc(),
+                "disk_format": disk_format,
+                "exported_name": exported_name,
+                "upload_link": upload_link,
+                "task": task.request.id
+            },
+            queue=queue_name)
+        self._run_abortable_task(remote, task)
+
     def destroy(self, user=None, task_uuid=None):
     def destroy(self, user=None, task_uuid=None):
         if self.destroyed:
         if self.destroyed:
             return False
             return False
@@ -549,4 +592,8 @@ class Disk(TimeStampedModel):
 
 
     @property
     @property
     def is_resizable(self):
     def is_resizable(self):
-        return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap', )
+        return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap',)
+
+    @property
+    def is_exportable(self):
+        return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')

circle/storage/tasks/storage_tasks.py

@@ -38,6 +38,16 @@ def download(disk_desc, url):
     pass
     pass
 
 
 
 
+@celery.task(name='storagedriver.import_disk')
+def import_disk(disk_desc, url):
+    pass
+
+
+@celery.task(name='storagedriver.export_disk')
+def export_disk(disk_desc, format):
+    pass
+
+
 @celery.task(name='storagedriver.delete')
 @celery.task(name='storagedriver.delete')
 def delete(path):
 def delete(path):
     pass
     pass

circle/vm/management/commands/mass_create_vms.py

+# Copyright 2014 Budapest University of Technology and Economics (BME IK)
+#
+# This file is part of CIRCLE Cloud.
+#
+# CIRCLE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.management.base import BaseCommand
+from vm.models import Instance, InstanceTemplate
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('-t', '--template', type=int, required=True)
+        parser.add_argument('-u', '--users', required=True)
+        parser.add_argument('-a', '--admin')
+        parser.add_argument('-o', '--operator')
+
+    def handle(self, *args, **options):
+        template = InstanceTemplate.objects.get(id=options['template'])
+
+        with open(options['users']) as f:
+            users = f.read().splitlines()
+
+        missing_users = Instance.mass_create_for_users(
+            template, users, options['admin'],  options['operator']
+        )
+
+        if len(missing_users) > 0:
+            self.stdout.write('These users do not exist:')
+            for user in missing_users:
+                self.stdout.write(user)

circle/vm/migrations/0003_auto_20171213_2018.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2017-12-13 20:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0002_interface_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='node',
+            name='cpu_weight',
+            field=models.FloatField(default=1.0, help_text='Indicates the relative CPU power of this node.', verbose_name='CPU Weight'),
+        ),
+        migrations.AddField(
+            model_name='node',
+            name='ram_weight',
+            field=models.FloatField(default=1.0, help_text='Indicates the relative RAM quantity of this node.', verbose_name='RAM Weight'),
+        ),
+        migrations.AddField(
+            model_name='node',
+            name='time_stamp',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, help_text='A timestamp for the node, used by the scheduler.', verbose_name='Last Scheduled Time Stamp'),
+            preserve_default=False,
+        ),
+    ]

circle/vm/models/instance.py

@@ -200,6 +200,9 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
     def get_running_instances(self):
     def get_running_instances(self):
         return Instance.active.filter(template=self, status="RUNNING")
         return Instance.active.filter(template=self, status="RUNNING")
 
 
+    def get_user_instances(self, user):
+        return Instance.active.filter(template=self, owner=user)
+
     @property
     @property
     def metric_prefix(self):
     def metric_prefix(self):
         return 'template.%d' % self.pk
         return 'template.%d' % self.pk
@@ -439,6 +442,30 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         return [cls.create(cps, disks, networks, req_traits, tags)
         return [cls.create(cps, disks, networks, req_traits, tags)
                 for cps in customized_params]
                 for cps in customized_params]
 
 
+    @classmethod
+    def mass_create_for_users(cls, template, users, admin=None, operator=None, **kwargs):
+        """
+        Create and deploy an instance of a template for each user
+        in a list of users. Returns the user IDs of missing users.
+        """
+        user_instances = []
+        missing_users = []
+        for user_id in users:
+            try:
+                user_instances.append(User.objects.get(profile__org_id=user_id))
+            except User.DoesNotExist:
+                missing_users.append(user_id)
+
+        for user in user_instances:
+            instance = cls.create_from_template(template, user, **kwargs)
+            if admin:
+                instance.set_level(User.objects.get(username=admin), 'owner')
+            if operator:
+                instance.set_level(User.objects.get(username=operator), 'operator')
+            instance.deploy(user=user)
+
+        return missing_users
+
     def clean(self, *args, **kwargs):
     def clean(self, *args, **kwargs):
         self.time_of_suspend, self.time_of_delete = self.get_renew_times()
         self.time_of_suspend, self.time_of_delete = self.get_renew_times()
         super(Instance, self).clean(*args, **kwargs)
         super(Instance, self).clean(*args, **kwargs)
@@ -576,7 +603,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
             host = self.get_connect_host(use_ipv6=use_ipv6)
             host = self.get_connect_host(use_ipv6=use_ipv6)
             proto = self.access_method
             proto = self.access_method
             if proto == 'rdp':
             if proto == 'rdp':
-                return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s' % {
+                return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s -f' % {
                     'port': port, 'proto': proto, 'pw': self.pw,
                     'port': port, 'proto': proto, 'pw': self.pw,
                     'host': host}
                     'host': host}
             elif proto == 'ssh':
             elif proto == 'ssh':
@@ -865,6 +892,53 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
     def metric_prefix(self):
     def metric_prefix(self):
         return 'vm.%s' % self.vm_name
         return 'vm.%s' % self.vm_name
 
 
+
+    class MonitorUnavailableException(Exception):
+        """Exception for monitor_info()
+
+            Indicates the unavailability of the monitoring server.
+        """
+        pass
+
+    def monitor_info(self):
+        metrics = ('cpu.percent', 'memory.usage')
+        prefix = self.metric_prefix
+        params = [('target', '%s.%s' % (prefix, metric))
+                  for metric in metrics]
+        params.append(('from', '-5min'))
+        params.append(('format', 'json'))
+
+        try:
+            logger.info('%s %s', settings.GRAPHITE_URL, params)
+            response = requests.get(settings.GRAPHITE_URL, params=params)
+
+            retval = {}
+            for target in response.json():
+                # Example:
+                # {"target": "circle.vm.{name}.cpu.usage",
+                #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
+                try:
+                    metric = target['target']
+                    if metric.startswith(prefix):
+                        metric = metric[len(prefix):]
+                    else:
+                        continue
+                    value = target['datapoints'][-2][0]
+                    retval[metric] = float(value)
+                except (KeyError, IndexError, ValueError):
+                    continue
+
+            return retval
+        except Exception:
+            logger.exception('Monitor server unavailable: ')
+            raise Instance.MonitorUnavailableException()
+
+    def cpu_usage(self):
+        return self.monitor_info().get('cpu.percent')
+
+    def ram_usage(self):
+        return self.monitor_info().get('memory.usage')
+
     @contextmanager
     @contextmanager
     def activity(self, code_suffix, readable_name, on_abort=None,
     def activity(self, code_suffix, readable_name, on_abort=None,
                  on_commit=None, task_uuid=None, user=None,
                  on_commit=None, task_uuid=None, user=None,

circle/vm/models/node.py

@@ -30,12 +30,12 @@ from time import time, sleep
 from django.conf import settings
 from django.conf import settings
 from django.db.models import (
 from django.db.models import (
     CharField, IntegerField, ForeignKey, BooleanField, ManyToManyField,
     CharField, IntegerField, ForeignKey, BooleanField, ManyToManyField,
-    FloatField, permalink, Sum
+    FloatField, DateTimeField, permalink, Sum
 )
 )
 from django.utils import timezone
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 
 
-from celery.exceptions import TimeoutError
+from celery.exceptions import TimeoutError, TaskRevokedError
 from model_utils.models import TimeStampedModel
 from model_utils.models import TimeStampedModel
 from taggit.managers import TaggableManager
 from taggit.managers import TaggableManager
 
 
@@ -128,11 +128,15 @@ class Node(OperatedMixin, TimeStampedModel):
     enabled = BooleanField(verbose_name=_('enabled'), default=False,
     enabled = BooleanField(verbose_name=_('enabled'), default=False,
                            help_text=_('Indicates whether the node can '
                            help_text=_('Indicates whether the node can '
                                        'be used for hosting.'))
                                        'be used for hosting.'))
-    schedule_enabled = BooleanField(verbose_name=_('schedule enabled'),
-                                    default=False, help_text=_(
-                                        'Indicates whether a vm can be '
-                                        'automatically scheduled to this '
-                                        'node.'))
+    schedule_enabled = BooleanField(
+        verbose_name=_('schedule enabled'),
+        default=False,
+        help_text=_(
+            'Indicates whether a vm can be '
+            'automatically scheduled to this '
+            'node.'
+        )
+    )
     traits = ManyToManyField(Trait, blank=True,
     traits = ManyToManyField(Trait, blank=True,
                              help_text=_("Declared traits."),
                              help_text=_("Declared traits."),
                              verbose_name=_('traits'))
                              verbose_name=_('traits'))
@@ -140,6 +144,21 @@ class Node(OperatedMixin, TimeStampedModel):
     overcommit = FloatField(default=1.0, verbose_name=_("overcommit ratio"),
     overcommit = FloatField(default=1.0, verbose_name=_("overcommit ratio"),
                             help_text=_("The ratio of total memory with "
                             help_text=_("The ratio of total memory with "
                                         "to without overcommit."))
                                         "to without overcommit."))
+    ram_weight = FloatField(
+        default=1.0,
+        help_text=_("Indicates the relative RAM quantity of this node."),
+        verbose_name=_("RAM Weight")
+    )
+    cpu_weight = FloatField(
+        default=1.0,
+        help_text=_("Indicates the relative CPU power of this node."),
+        verbose_name=_("CPU Weight")
+    )
+    time_stamp = DateTimeField(
+        auto_now_add=True,
+        help_text=_("A timestamp for the node, used by the scheduler."),
+        verbose_name=_("Last Scheduled Time Stamp")
+    )
 
 
     class Meta:
     class Meta:
         app_label = 'vm'
         app_label = 'vm'
@@ -162,7 +181,7 @@ class Node(OperatedMixin, TimeStampedModel):
             self.get_remote_queue_name("vm", "fast")
             self.get_remote_queue_name("vm", "fast")
             self.get_remote_queue_name("vm", "slow")
             self.get_remote_queue_name("vm", "slow")
             self.get_remote_queue_name("net", "fast")
             self.get_remote_queue_name("net", "fast")
-        except:
+        except Exception:
             return False
             return False
         else:
         else:
             return True
             return True
@@ -315,7 +334,7 @@ class Node(OperatedMixin, TimeStampedModel):
                 queue=self.get_remote_queue_name('vm', priority),
                 queue=self.get_remote_queue_name('vm', priority),
                 expires=timeout + 60)
                 expires=timeout + 60)
             return r.get(timeout=timeout)
             return r.get(timeout=timeout)
-        except (TimeoutError, WorkerNotFound):
+        except (TimeoutError, WorkerNotFound, TaskRevokedError):
             if raise_:
             if raise_:
                 raise
                 raise
             else:
             else:
@@ -341,19 +360,25 @@ class Node(OperatedMixin, TimeStampedModel):
                 # Example:
                 # Example:
                 # {"target": "circle.szianode.cpu.usage",
                 # {"target": "circle.szianode.cpu.usage",
                 #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
                 #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
+                logger.info('MONITOR_TARGET: %s', target)
                 try:
                 try:
                     metric = target['target']
                     metric = target['target']
                     if metric.startswith(prefix):
                     if metric.startswith(prefix):
                         metric = metric[len(prefix):]
                         metric = metric[len(prefix):]
                     else:
                     else:
+            	        logger.info('MONITOR_MET: %s %s', target, metric)
                         continue
                         continue
-                    value = target['datapoints'][-2][0]
+                    value = target['datapoints'][-1][0]
+                    if value is None: 
+                        value = target['datapoints'][-2][0]
                     retval[metric] = float(value)
                     retval[metric] = float(value)
+                    logger.info('MONITOR_RETVAL: %s %s, %s', target['target'], metric, retval[metric])
                 except (KeyError, IndexError, ValueError, TypeError):
                 except (KeyError, IndexError, ValueError, TypeError):
+                    logger.info('MONITOR_ERR: %s %s', metric, value)
                     continue
                     continue
 
 
             return retval
             return retval
-        except:
+        except Exception:
             logger.exception('Unhandled exception: ')
             logger.exception('Unhandled exception: ')
             return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
             return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
                                      priority="fast")
                                      priority="fast")
@@ -363,15 +388,27 @@ class Node(OperatedMixin, TimeStampedModel):
     def driver_version(self):
     def driver_version(self):
         return self.info.get('driver_version')
         return self.info.get('driver_version')
 
 
+    def get_monitor_info(self, metric):
+        # return with the metric value if the monitor info not none
+        # or return 0 if its None or the metric unreachable
+        if self.monitor_info is None:
+            logger.warning('Monitor info is None')
+            return 0
+        elif self.monitor_info.get(metric) is None:
+            logger.warning('Unreachable monitor info of: ' + metric)
+            return 0
+        else:
+            return self.monitor_info.get(metric)
+
     @property
     @property
     @node_available
     @node_available
     def cpu_usage(self):
     def cpu_usage(self):
-        return self.monitor_info.get('cpu.percent') / 100
+        return self.get_monitor_info('cpu.percent') / 100
 
 
     @property
     @property
     @node_available
     @node_available
     def ram_usage(self):
     def ram_usage(self):
-        return self.monitor_info.get('memory.usage') / 100
+        return self.get_monitor_info('memory.usage') / 100
 
 
     @property
     @property
     @node_available
     @node_available
@@ -404,7 +441,7 @@ class Node(OperatedMixin, TimeStampedModel):
         vm_state_changed hook.
         vm_state_changed hook.
         """
         """
         domains = {}
         domains = {}
-        domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5,
+        domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=10,
                                         priority="fast")
                                         priority="fast")
         if domain_list is None:
         if domain_list is None:
             logger.info("Monitoring failed at: %s", self.name)
             logger.info("Monitoring failed at: %s", self.name)
@@ -413,7 +450,7 @@ class Node(OperatedMixin, TimeStampedModel):
             # [{'name': 'cloud-1234', 'state': 'RUNNING', ...}, ...]
             # [{'name': 'cloud-1234', 'state': 'RUNNING', ...}, ...]
             try:
             try:
                 id = int(i['name'].split('-')[1])
                 id = int(i['name'].split('-')[1])
-            except:
+            except Exception:
                 pass  # name format doesn't match
                 pass  # name format doesn't match
             else:
             else:
                 domains[id] = i['state']
                 domains[id] = i['state']

circle/vm/operations.py

@@ -16,53 +16,51 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 
 from __future__ import absolute_import, unicode_literals
 from __future__ import absolute_import, unicode_literals
+
+from StringIO import StringIO
 from base64 import encodestring
 from base64 import encodestring
 from hashlib import md5
 from hashlib import md5
 from logging import getLogger
 from logging import getLogger
-import os
-from re import search
 from string import ascii_lowercase
 from string import ascii_lowercase
-from StringIO import StringIO
 from tarfile import TarFile, TarInfo
 from tarfile import TarFile, TarInfo
-import time
 from urlparse import urlsplit
 from urlparse import urlsplit
 
 
+import os
+import time
+from celery.contrib.abortable import AbortableAsyncResult
+from celery.exceptions import TimeLimitExceeded, TimeoutError
+from django.conf import settings
 from django.core.exceptions import PermissionDenied, SuspiciousOperation
 from django.core.exceptions import PermissionDenied, SuspiciousOperation
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
+from django.db.models import Q
 from django.utils import timezone
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
-from django.conf import settings
-from django.db.models import Q
-
+from re import search
 from sizefield.utils import filesizeformat
 from sizefield.utils import filesizeformat
 
 
-from celery.contrib.abortable import AbortableAsyncResult
-from celery.exceptions import TimeLimitExceeded, TimeoutError
-
 from common.models import (
 from common.models import (
     create_readable, humanize_exception, HumanReadableException
     create_readable, humanize_exception, HumanReadableException
 )
 )
 from common.operations import Operation, register_operation, SubOperationMixin
 from common.operations import Operation, register_operation, SubOperationMixin
+from dashboard.store_api import Store, NoStoreException
+from firewall.models import Host
 from manager.scheduler import SchedulerError
 from manager.scheduler import SchedulerError
-from .tasks.local_tasks import (
-    abortable_async_instance_operation, abortable_async_node_operation,
-)
+from monitor.client import Client
+from storage.models import Disk
+from storage.tasks import storage_tasks
 from .models import (
 from .models import (
     Instance, InstanceActivity, InstanceTemplate, Interface, Node,
     Instance, InstanceActivity, InstanceTemplate, Interface, Node,
     NodeActivity, pwgen
     NodeActivity, pwgen
 )
 )
 from .tasks import agent_tasks, vm_tasks
 from .tasks import agent_tasks, vm_tasks
-
-from dashboard.store_api import Store, NoStoreException
-from firewall.models import Host
-from monitor.client import Client
-from storage.tasks import storage_tasks
+from .tasks.local_tasks import (
+    abortable_async_instance_operation, abortable_async_node_operation,
+)
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
 
 
 
 
 class RemoteOperationMixin(object):
 class RemoteOperationMixin(object):
-
     remote_timeout = 30
     remote_timeout = 30
 
 
     def _operation(self, **kwargs):
     def _operation(self, **kwargs):
@@ -167,7 +165,6 @@ class InstanceOperation(Operation):
 
 
 
 
 class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
 class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
-
     remote_queue = ('vm', 'fast')
     remote_queue = ('vm', 'fast')
 
 
     def _get_remote_queue(self):
     def _get_remote_queue(self):
@@ -178,7 +175,7 @@ class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
 
 
 
 
 class EnsureAgentMixin(object):
 class EnsureAgentMixin(object):
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
 
 
     def check_precond(self):
     def check_precond(self):
         super(EnsureAgentMixin, self).check_precond()
         super(EnsureAgentMixin, self).check_precond()
@@ -198,7 +195,7 @@ class EnsureAgentMixin(object):
 
 
 
 
 class RemoteAgentOperation(EnsureAgentMixin, RemoteInstanceOperation):
 class RemoteAgentOperation(EnsureAgentMixin, RemoteInstanceOperation):
-    remote_queue = ('agent', )
+    remote_queue = ('agent',)
     concurrency_check = False
     concurrency_check = False
 
 
 
 
@@ -213,7 +210,7 @@ class AddInterfaceOperation(InstanceOperation):
 
 
     def rollback(self, net, activity):
     def rollback(self, net, activity):
         with activity.sub_activity(
         with activity.sub_activity(
-            'destroying_net',
+                'destroying_net',
                 readable_name=ugettext_noop("destroy network (rollback)")):
                 readable_name=ugettext_noop("destroy network (rollback)")):
             net.destroy()
             net.destroy()
             net.delete()
             net.delete()
@@ -248,11 +245,10 @@ class AddInterfaceOperation(InstanceOperation):
 
 
 @register_operation
 @register_operation
 class CreateDiskOperation(InstanceOperation):
 class CreateDiskOperation(InstanceOperation):
-
     id = 'create_disk'
     id = 'create_disk'
     name = _("create disk")
     name = _("create disk")
     description = _("Create and attach empty disk to the virtual machine.")
     description = _("Create and attach empty disk to the virtual machine.")
-    required_perms = ('storage.create_empty_disk', )
+    required_perms = ('storage.create_empty_disk',)
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
 
 
     def _operation(self, user, size, activity, name=None):
     def _operation(self, user, size, activity, name=None):
@@ -271,8 +267,8 @@ class CreateDiskOperation(InstanceOperation):
 
 
         if self.instance.is_running:
         if self.instance.is_running:
             with activity.sub_activity(
             with activity.sub_activity(
-                'deploying_disk',
-                readable_name=ugettext_noop("deploying disk")
+                    'deploying_disk',
+                    readable_name=ugettext_noop("deploying disk")
             ):
             ):
                 disk.deploy()
                 disk.deploy()
             self.instance._attach_disk(parent_activity=activity, disk=disk)
             self.instance._attach_disk(parent_activity=activity, disk=disk)
@@ -285,13 +281,12 @@ class CreateDiskOperation(InstanceOperation):
 
 
 @register_operation
 @register_operation
 class ResizeDiskOperation(RemoteInstanceOperation):
 class ResizeDiskOperation(RemoteInstanceOperation):
-
     id = 'resize_disk'
     id = 'resize_disk'
     name = _("resize disk")
     name = _("resize disk")
     description = _("Resize the virtual disk image. "
     description = _("Resize the virtual disk image. "
                     "Size must be greater value than the actual size.")
                     "Size must be greater value than the actual size.")
-    required_perms = ('storage.resize_disk', )
-    accept_states = ('RUNNING', )
+    required_perms = ('storage.resize_disk',)
+    accept_states = ('RUNNING',)
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
     remote_queue = ('vm', 'slow')
     remote_queue = ('vm', 'slow')
     task = vm_tasks.resize_disk
     task = vm_tasks.resize_disk
@@ -324,13 +319,11 @@ class DownloadDiskOperation(InstanceOperation):
                     "machine.")
                     "machine.")
     abortable = True
     abortable = True
     has_percentage = True
     has_percentage = True
-    required_perms = ('storage.download_disk', )
+    required_perms = ('storage.download_disk',)
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
 
 
     def _operation(self, user, url, task, activity, name=None):
     def _operation(self, user, url, task, activity, name=None):
-        from storage.models import Disk
-
         disk = Disk.download(url=url, name=name, task=task)
         disk = Disk.download(url=url, name=name, task=task)
         devnums = list(ascii_lowercase)
         devnums = list(ascii_lowercase)
         for d in self.instance.disks.all():
         for d in self.instance.disks.all():
@@ -352,6 +345,57 @@ class DownloadDiskOperation(InstanceOperation):
 
 
 
 
 @register_operation
 @register_operation
+class ImportDiskOperation(InstanceOperation):
+    id = 'import_disk'
+    name = _('import disk')
+    description = _('Import and attach a disk image to the virtual machine '
+                    'from the user store. The disk image has to be in the '
+                    'root directory of the store.')
+    abortable = True
+    has_percentage = True
+    required_perms = ('storage.import_disk',)
+    accept_states = ('STOPPED', 'PENDING', 'RUNNING')
+    async_queue = 'localhost.man.slow'
+
+    def check_auth(self, user):
+        super(ImportDiskOperation, self).check_auth(user)
+        try:
+            Store(user)
+        except NoStoreException:
+            raise PermissionDenied
+
+    def _operation(self, user, name, disk_path, task):
+        store = Store(user)
+        download_link = store.request_download(disk_path)
+        disk = Disk.import_disk(user, name, download_link, task)
+        self.instance.disks.add(disk)
+
+
+@register_operation
+class ExportDiskOperation(InstanceOperation):
+    id = 'export_disk'
+    name = _('export disk')
+    description = _('Export disk to the selected format.')
+    abortable = True
+    has_percentage = True
+    required_perms = ('storage.export_disk',)
+    accept_states = ('STOPPED',)
+    async_queue = 'localhost.man.slow'
+
+    def check_auth(self, user):
+        super(ExportDiskOperation, self).check_auth(user)
+        try:
+            Store(user)
+        except NoStoreException:
+            raise PermissionDenied
+
+    def _operation(self, user, disk, exported_name, disk_format, task):
+        store = Store(user)
+        upload_link = store.request_upload('/')
+        disk.export(exported_name, disk_format, upload_link, task)
+
+
+@register_operation
 class DeployOperation(InstanceOperation):
 class DeployOperation(InstanceOperation):
     id = 'deploy'
     id = 'deploy'
     name = _("deploy")
     name = _("deploy")
@@ -399,8 +443,8 @@ class DeployOperation(InstanceOperation):
 
 
         # Establish network connection (vmdriver)
         # Establish network connection (vmdriver)
         with activity.sub_activity(
         with activity.sub_activity(
-            'deploying_net', readable_name=ugettext_noop(
-                "deploy network")):
+                'deploying_net', readable_name=ugettext_noop(
+                    "deploy network")):
             self.instance.deploy_net()
             self.instance.deploy_net()
 
 
         try:
         try:
@@ -421,7 +465,8 @@ class DeployOperation(InstanceOperation):
         description = _("Deploy virtual machine.")
         description = _("Deploy virtual machine.")
         remote_queue = ("vm", "slow")
         remote_queue = ("vm", "slow")
         task = vm_tasks.deploy
         task = vm_tasks.deploy
-	remote_timeout = 120
+        remote_timeout = 120
+
         def _get_remote_args(self, **kwargs):
         def _get_remote_args(self, **kwargs):
             return [self.instance.get_vm_desc()]
             return [self.instance.get_vm_desc()]
             # intentionally not calling super
             # intentionally not calling super
@@ -530,7 +575,7 @@ class MigrateOperation(RemoteInstanceOperation):
                     "keeping its full state.")
                     "keeping its full state.")
     required_perms = ()
     required_perms = ()
     superuser_required = True
     superuser_required = True
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
     task = vm_tasks.migrate
     task = vm_tasks.migrate
     remote_queue = ("vm", "slow")
     remote_queue = ("vm", "slow")
@@ -542,8 +587,8 @@ class MigrateOperation(RemoteInstanceOperation):
 
 
     def rollback(self, activity):
     def rollback(self, activity):
         with activity.sub_activity(
         with activity.sub_activity(
-            'rollback_net', readable_name=ugettext_noop(
-                "redeploy network (rollback)")):
+                'rollback_net', readable_name=ugettext_noop(
+                    "redeploy network (rollback)")):
             self.instance.deploy_net()
             self.instance.deploy_net()
 
 
     def _operation(self, activity, to_node=None, live_migration=True):
     def _operation(self, activity, to_node=None, live_migration=True):
@@ -556,8 +601,8 @@ class MigrateOperation(RemoteInstanceOperation):
 
 
         try:
         try:
             with activity.sub_activity(
             with activity.sub_activity(
-                'migrate_vm', readable_name=create_readable(
-                    ugettext_noop("migrate to %(node)s"), node=to_node)):
+                    'migrate_vm', readable_name=create_readable(
+                        ugettext_noop("migrate to %(node)s"), node=to_node)):
                 super(MigrateOperation, self)._operation(
                 super(MigrateOperation, self)._operation(
                     to_node=to_node, live_migration=live_migration)
                     to_node=to_node, live_migration=live_migration)
         except Exception as e:
         except Exception as e:
@@ -567,8 +612,8 @@ class MigrateOperation(RemoteInstanceOperation):
 
 
         # Shutdown networks
         # Shutdown networks
         with activity.sub_activity(
         with activity.sub_activity(
-            'shutdown_net', readable_name=ugettext_noop(
-                "shutdown network")):
+                'shutdown_net', readable_name=ugettext_noop(
+                    "shutdown network")):
             self.instance.shutdown_net()
             self.instance.shutdown_net()
 
 
         # Refresh node information
         # Refresh node information
@@ -577,8 +622,8 @@ class MigrateOperation(RemoteInstanceOperation):
 
 
         # Estabilish network connection (vmdriver)
         # Estabilish network connection (vmdriver)
         with activity.sub_activity(
         with activity.sub_activity(
-            'deploying_net', readable_name=ugettext_noop(
-                "deploy network")):
+                'deploying_net', readable_name=ugettext_noop(
+                    "deploy network")):
             self.instance.deploy_net()
             self.instance.deploy_net()
 
 
 
 
@@ -589,7 +634,7 @@ class RebootOperation(RemoteInstanceOperation):
     description = _("Warm reboot virtual machine by sending Ctrl+Alt+Del "
     description = _("Warm reboot virtual machine by sending Ctrl+Alt+Del "
                     "signal to its console.")
                     "signal to its console.")
     required_perms = ()
     required_perms = ()
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     task = vm_tasks.reboot
     task = vm_tasks.reboot
 
 
     def _operation(self, activity):
     def _operation(self, activity):
@@ -630,7 +675,7 @@ class RemovePortOperation(InstanceOperation):
     description = _("Close the specified port.")
     description = _("Close the specified port.")
     concurrency_check = False
     concurrency_check = False
     acl_level = "operator"
     acl_level = "operator"
-    required_perms = ('vm.config_ports', )
+    required_perms = ('vm.config_ports',)
 
 
     def _operation(self, activity, rule):
     def _operation(self, activity, rule):
         interface = rule.host.interface_set.get()
         interface = rule.host.interface_set.get()
@@ -649,7 +694,7 @@ class AddPortOperation(InstanceOperation):
     description = _("Open the specified port.")
     description = _("Open the specified port.")
     concurrency_check = False
     concurrency_check = False
     acl_level = "operator"
     acl_level = "operator"
-    required_perms = ('vm.config_ports', )
+    required_perms = ('vm.config_ports',)
 
 
     def _operation(self, activity, host, proto, port):
     def _operation(self, activity, host, proto, port):
         if host.interface_set.get().instance != self.instance:
         if host.interface_set.get().instance != self.instance:
@@ -673,8 +718,8 @@ class RemoveDiskOperation(InstanceOperation):
         if self.instance.is_running and disk.type not in ["iso"]:
         if self.instance.is_running and disk.type not in ["iso"]:
             self.instance._detach_disk(disk=disk, parent_activity=activity)
             self.instance._detach_disk(disk=disk, parent_activity=activity)
         with activity.sub_activity(
         with activity.sub_activity(
-            'destroy_disk',
-            readable_name=ugettext_noop('destroy disk')
+                'destroy_disk',
+                readable_name=ugettext_noop('destroy disk')
         ):
         ):
             disk.destroy()
             disk.destroy()
             return self.instance.disks.remove(disk)
             return self.instance.disks.remove(disk)
@@ -690,7 +735,7 @@ class ResetOperation(RemoteInstanceOperation):
     name = _("reset")
     name = _("reset")
     description = _("Cold reboot virtual machine (power cycle).")
     description = _("Cold reboot virtual machine (power cycle).")
     required_perms = ()
     required_perms = ()
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     task = vm_tasks.reset
     task = vm_tasks.reset
 
 
     def _operation(self, activity):
     def _operation(self, activity):
@@ -710,7 +755,7 @@ class SaveAsTemplateOperation(InstanceOperation):
                     "start an instance of it.")
                     "start an instance of it.")
     has_percentage = True
     has_percentage = True
     abortable = True
     abortable = True
-    required_perms = ('vm.create_template', )
+    required_perms = ('vm.create_template',)
     accept_states = ('RUNNING', 'STOPPED')
     accept_states = ('RUNNING', 'STOPPED')
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
 
 
@@ -778,10 +823,10 @@ class SaveAsTemplateOperation(InstanceOperation):
         self.disks = []
         self.disks = []
         for disk in self.instance.disks.all():
         for disk in self.instance.disks.all():
             with activity.sub_activity(
             with activity.sub_activity(
-                'saving_disk',
-                readable_name=create_readable(
-                    ugettext_noop("saving disk %(name)s"),
-                    name=disk.name)
+                    'saving_disk',
+                    readable_name=create_readable(
+                        ugettext_noop("saving disk %(name)s"),
+                        name=disk.name)
             ):
             ):
                 self.disks.append(__try_save_disk(disk))
                 self.disks.append(__try_save_disk(disk))
 
 
@@ -822,7 +867,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin,
                     "turn itself off in a period.")
                     "turn itself off in a period.")
     abortable = True
     abortable = True
     required_perms = ()
     required_perms = ()
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     resultant_state = 'STOPPED'
     resultant_state = 'STOPPED'
     task = vm_tasks.shutdown
     task = vm_tasks.shutdown
     remote_queue = ("vm", "slow")
     remote_queue = ("vm", "slow")
@@ -883,7 +928,7 @@ class SleepOperation(InstanceOperation):
                     "resumed. In the meantime, the machine will only use "
                     "resumed. In the meantime, the machine will only use "
                     "storage resources, and keep network resources allocated.")
                     "storage resources, and keep network resources allocated.")
     required_perms = ()
     required_perms = ()
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     resultant_state = 'SUSPENDED'
     resultant_state = 'SUSPENDED'
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
 
 
@@ -927,7 +972,7 @@ class WakeUpOperation(InstanceOperation):
                     "load the saved memory of the system and start the "
                     "load the saved memory of the system and start the "
                     "virtual machine from this state.")
                     "virtual machine from this state.")
     required_perms = ()
     required_perms = ()
-    accept_states = ('SUSPENDED', )
+    accept_states = ('SUSPENDED',)
     resultant_state = 'RUNNING'
     resultant_state = 'RUNNING'
     async_queue = "localhost.man.slow"
     async_queue = "localhost.man.slow"
 
 
@@ -950,8 +995,8 @@ class WakeUpOperation(InstanceOperation):
 
 
         # Estabilish network connection (vmdriver)
         # Estabilish network connection (vmdriver)
         with activity.sub_activity(
         with activity.sub_activity(
-            'deploying_net', readable_name=ugettext_noop(
-                "deploy network")):
+                'deploying_net', readable_name=ugettext_noop(
+                    "deploy network")):
             self.instance.deploy_net()
             self.instance.deploy_net()
 
 
         try:
         try:
@@ -987,7 +1032,7 @@ class RenewOperation(InstanceOperation):
 
 
     def set_time_of_suspend(self, activity, suspend, force):
     def set_time_of_suspend(self, activity, suspend, force):
         with activity.sub_activity(
         with activity.sub_activity(
-            'renew_suspend', concurrency_check=False,
+                'renew_suspend', concurrency_check=False,
                 readable_name=ugettext_noop('set time of suspend')):
                 readable_name=ugettext_noop('set time of suspend')):
             if (not force and suspend and self.instance.time_of_suspend and
             if (not force and suspend and self.instance.time_of_suspend and
                     suspend < self.instance.time_of_suspend):
                     suspend < self.instance.time_of_suspend):
@@ -998,7 +1043,7 @@ class RenewOperation(InstanceOperation):
 
 
     def set_time_of_delete(self, activity, delete, force):
     def set_time_of_delete(self, activity, delete, force):
         with activity.sub_activity(
         with activity.sub_activity(
-            'renew_delete', concurrency_check=False,
+                'renew_delete', concurrency_check=False,
                 readable_name=ugettext_noop('set time of delete')):
                 readable_name=ugettext_noop('set time of delete')):
             if (not force and delete and self.instance.time_of_delete and
             if (not force and delete and self.instance.time_of_delete and
                     delete < self.instance.time_of_delete):
                     delete < self.instance.time_of_delete):
@@ -1039,7 +1084,7 @@ class ChangeStateOperation(InstanceOperation):
                     "redeployed without losing its storage and network "
                     "redeployed without losing its storage and network "
                     "resources.")
                     "resources.")
     acl_level = "owner"
     acl_level = "owner"
-    required_perms = ('vm.emergency_change_state', )
+    required_perms = ('vm.emergency_change_state',)
     concurrency_check = False
     concurrency_check = False
 
 
     def _operation(self, user, activity, new_state="NOSTATE", interrupt=False,
     def _operation(self, user, activity, new_state="NOSTATE", interrupt=False,
@@ -1066,7 +1111,7 @@ class RedeployOperation(InstanceOperation):
                     "and redeploy the VM. This operation allows starting "
                     "and redeploy the VM. This operation allows starting "
                     "machines formerly running on a failed node.")
                     "machines formerly running on a failed node.")
     acl_level = "owner"
     acl_level = "owner"
-    required_perms = ('vm.redeploy', )
+    required_perms = ('vm.redeploy',)
     concurrency_check = False
     concurrency_check = False
 
 
     def _operation(self, user, activity, with_emergency_change_state=True):
     def _operation(self, user, activity, with_emergency_change_state=True):
@@ -1320,7 +1365,7 @@ class ScreenshotOperation(RemoteInstanceOperation):
                     "screensaver.")
                     "screensaver.")
     acl_level = "owner"
     acl_level = "owner"
     required_perms = ()
     required_perms = ()
-    accept_states = ('RUNNING', )
+    accept_states = ('RUNNING',)
     task = vm_tasks.screenshot
     task = vm_tasks.screenshot
 
 
 
 
@@ -1332,8 +1377,8 @@ class RecoverOperation(InstanceOperation):
                     "state. Network resources (allocations) are already lost, "
                     "state. Network resources (allocations) are already lost, "
                     "so you will have to manually add interfaces afterwards.")
                     "so you will have to manually add interfaces afterwards.")
     acl_level = "owner"
     acl_level = "owner"
-    required_perms = ('vm.recover', )
-    accept_states = ('DESTROYED', )
+    required_perms = ('vm.recover',)
+    accept_states = ('DESTROYED',)
     resultant_state = 'PENDING'
     resultant_state = 'PENDING'
 
 
     def check_precond(self):
     def check_precond(self):
@@ -1344,7 +1389,7 @@ class RecoverOperation(InstanceOperation):
 
 
     def _operation(self, user, activity):
     def _operation(self, user, activity):
         with activity.sub_activity(
         with activity.sub_activity(
-            'recover_instance',
+                'recover_instance',
                 readable_name=ugettext_noop("recover instance")):
                 readable_name=ugettext_noop("recover instance")):
             self.instance.destroyed_at = None
             self.instance.destroyed_at = None
             for disk in self.instance.disks.all():
             for disk in self.instance.disks.all():
@@ -1371,7 +1416,7 @@ class ResourcesOperation(InstanceOperation):
     name = _("resources change")
     name = _("resources change")
     description = _("Change resources of a stopped virtual machine.")
     description = _("Change resources of a stopped virtual machine.")
     acl_level = "owner"
     acl_level = "owner"
-    required_perms = ('vm.change_resources', )
+    required_perms = ('vm.change_resources',)
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
     accept_states = ('STOPPED', 'PENDING', 'RUNNING')
 
 
     def _operation(self, user, activity,
     def _operation(self, user, activity,
@@ -1677,7 +1722,7 @@ class UpdateAgentOperation(RemoteAgentOperation):
         index = 0
         index = 0
         filename = settings.AGENT_VERSION + ".tar"
         filename = settings.AGENT_VERSION + ".tar"
         while True:
         while True:
-            chunk = data[index:index+chunk_size]
+            chunk = data[index:index + chunk_size]
             if chunk:
             if chunk:
                 agent_tasks.append.apply_async(
                 agent_tasks.append.apply_async(
                     queue=queue,
                     queue=queue,

circle/vm/tasks/local_periodic_tasks.py

@@ -17,7 +17,9 @@
 
 
 import logging
 import logging
 from django.utils import timezone
 from django.utils import timezone
+from datetime import timedelta
 from django.utils.translation import ugettext_noop
 from django.utils.translation import ugettext_noop
+from django.conf import settings
 
 
 from manager.mancelery import celery
 from manager.mancelery import celery
 from vm.models import Node, Instance
 from vm.models import Node, Instance
@@ -33,7 +35,7 @@ def update_domain_states():
 
 
 
 
 @celery.task(ignore_result=True)
 @celery.task(ignore_result=True)
-def garbage_collector(timeout=15):
+def garbage_collector(offset=timezone.timedelta(seconds=20)):
     """Garbage collector for instances.
     """Garbage collector for instances.
 
 
     Suspends and destroys expired instances.
     Suspends and destroys expired instances.
@@ -64,7 +66,7 @@ def garbage_collector(timeout=15):
             work_package -= 1
             work_package -= 1
             logger.info("Expired instance %d suspended." % i.pk)
             logger.info("Expired instance %d suspended." % i.pk)
             try:
             try:
-            	i.sleep.async(system=True)
+                i.sleep.async(system=True)
                 i.owner.profile.notify(
                 i.owner.profile.notify(
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop(
                     ugettext_noop(
@@ -72,8 +74,8 @@ def garbage_collector(timeout=15):
                         'has been suspended due to expiration. '
                         'has been suspended due to expiration. '
                         'You can resume or destroy it.'),
                         'You can resume or destroy it.'),
                     instance=i.name, url=i.get_absolute_url())
                     instance=i.name, url=i.get_absolute_url())
- 	    except ActivityInProgressError:
-            	logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
+            except ActivityInProgressError:
+                logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
             except Exception as e:
             except Exception as e:
                 logger.info('Could not notify owner of instance %d .%s',
                 logger.info('Could not notify owner of instance %d .%s',
                              i.pk, unicode(e))
                              i.pk, unicode(e))
@@ -81,4 +83,44 @@ def garbage_collector(timeout=15):
             logger.debug("Instance %d expires soon." % i.pk)
             logger.debug("Instance %d expires soon." % i.pk)
             i.notify_owners_about_expiration()
             i.notify_owners_about_expiration()
         else:
         else:
-            logger.debug("Instance %d didn't expire." % i.pk)
+            logger.debug("Instance %d didn't expire. bw:%d", i.pk, bw)
+
+
+@celery.task(ignore_result=True)
+def auto_migrate():
+    """Auto migration task for runtime scaling
+    """
+    time_limit = settings.AUTO_MIGRATION_TIME_LIMIT_IN_HOURS
+    available_time = timedelta(hours=int(time_limit))
+    deadline = timezone.now() + available_time
+    while timezone.now() < deadline:
+        migrate_one()
+
+
+def migrate_one():
+    """Migrate a VM syncronously.
+
+        The target node chosen by the scheduler.
+    """
+    nodes = [n for n in Node.objects.filter(enabled=True) if n.online]
+    node_max_cpu = max(nodes, key=lambda x: x.cpu_usage / x.cpu_weight)
+    node_max_ram = max(nodes, key=lambda x: x.ram_usage / x.ram_weight)
+    if node_max_cpu.cpu_usage > node_max_ram.ram_usage:
+        try:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=lambda x: x.cpu_usage())
+            instance_to_migrate.migrate(system=True)
+        except Instance.MonitorUnavailableException:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=(lambda x: x.get_vm_desc()["vcpu"] *
+                                           x.get_vm_desc()["cpu_share"]))
+            instance_to_migrate.migrate(system=True)
+    else:
+        try:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_ram.pk),
+                                      key=lambda x: x.ram_usage())
+            instance_to_migrate.migrate(system=True)
+        except Instance.MonitorUnavailableException:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=lambda x: x.get_vm_desc()["memory"])
+            instance_to_migrate.migrate(system=True)

requirements/base.txt

-cryptography==2.0
+cryptography==2.7
 amqp==1.4.7
 amqp==1.4.7
 anyjson==0.3.3
 anyjson==0.3.3
 arrow==0.7.0
 arrow==0.7.0
 billiard==3.3.0.20
 billiard==3.3.0.20
 bpython==0.14.1
 bpython==0.14.1
 celery==3.1.18
 celery==3.1.18
-Django==1.11.6
+Django==1.11.25
 django-appconf==1.0.2
 django-appconf==1.0.2
 django-autocomplete-light==3.2.9
 django-autocomplete-light==3.2.9
 django-braces==1.11.0
 django-braces==1.11.0
@@ -16,7 +16,7 @@ django-sizefield==0.9.1
 django-statici18n==1.4.0
 django-statici18n==1.4.0
 django-tables2==1.10.0
 django-tables2==1.10.0
 django-taggit==0.22.1
 django-taggit==0.22.1
-djangosaml2==0.16.10
+djangosaml2==0.17.1
 git+https://git.ik.bme.hu/circle/django-sshkey.git
 git+https://git.ik.bme.hu/circle/django-sshkey.git
 docutils==0.12
 docutils==0.12
 Jinja2==2.7.3
 Jinja2==2.7.3
@@ -26,9 +26,9 @@ logutils==0.3.3
 MarkupSafe==0.23
 MarkupSafe==0.23
 netaddr==0.7.14
 netaddr==0.7.14
 pip-tools==0.3.6
 pip-tools==0.3.6
-psycopg2==2.6
+psycopg2==2.8.3
 Pygments==2.0.2
 Pygments==2.0.2
-pylibmc==1.4.3
+pylibmc==1.6.0
 python-dateutil==2.4.2
 python-dateutil==2.4.2
 pyinotify==0.9.5
 pyinotify==0.9.5
 pyotp==2.1.1
 pyotp==2.1.1

requirements/production.txt

 # Pro-tip: Try not to put anything here. There should be no dependency in
 # Pro-tip: Try not to put anything here. There should be no dependency in
 # production that isn't in development.
 # production that isn't in development.
 -r base.txt
 -r base.txt
-uWSGI==2.0.13.1
+uWSGI==2.0.18