Merge branch 'master' into 'smallville_fix'

# Conflicts:
#   circle/vm/tasks/local_periodic_tasks.py

.gitignore

@@ -8,6 +8,8 @@
 *.swp
 *.swp
 *.swo
 *.swo
 *~
 *~
+.vscode
+.idea
 
 
 # Sphinx docs:
 # Sphinx docs:
 build
 build

circle/circle/settings/base.py

@@ -495,6 +495,7 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
                 },
                 },
                 'required_attributes': required_attrs,
                 'required_attributes': required_attrs,
                 'optional_attributes': optional_attrs,
                 'optional_attributes': optional_attrs,
+                'want_response_signed': False,
             },
             },
         },
         },
         'metadata': {'local': [remote_metadata], },
         'metadata': {'local': [remote_metadata], },
@@ -576,7 +577,7 @@ SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
 MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
 MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
 MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
 MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
 
 
-SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'random')
+SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'advanced')
 
 
 # Url to download the client: (e.g. http://circlecloud.org/client/download/)
 # Url to download the client: (e.g. http://circlecloud.org/client/download/)
 CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
 CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
@@ -590,3 +591,12 @@ REQUEST_HOOK_URL = get_env_variable("REQUEST_HOOK_URL", "")
 SSHKEY_EMAIL_ADD_KEY = False
 SSHKEY_EMAIL_ADD_KEY = False
 
 
 TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
 TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
+
+# Default value is every day at midnight
+AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *")
+AUTO_MIGRATION_TIME_LIMIT_IN_HOURS = (
+    get_env_variable("AUTO_MIGRATION_TIME_LIMIT_IN_HOURS", "2"))
+
+# Maximum time difference until the monitor's values get valid
+SCHEDULER_TIME_SENSITIVITY_IN_SECONDS = (
+    get_env_variable("SCHEDULER_TIME_SENSITIVITY_IN_SECONDS", "60"))

circle/dashboard/fixtures/node.json

@@ -65,7 +65,10 @@
     "modified": "2014-02-19T21:11:34.671Z", 
     "modified": "2014-02-19T21:11:34.671Z", 
     "priority": 1, 
     "priority": 1, 
     "traits": [], 
     "traits": [], 
-    "host": 1
+    "host": 1,
+    "ram_weight": 1.0,
+    "cpu_weight": 1.0,
+    "time_stamp":  "2017-12-13T21:08:08.819Z"
   }
   }
 }
 }
 ]
 ]

circle/dashboard/migrations/0007_groupprofile_disk_quota.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-04-24 20:00
+from __future__ import unicode_literals
+
+from django.db import migrations
+import sizefield.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0006_auto_20170707_1909'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='groupprofile',
+            name='disk_quota',
+            field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
+        ),
+    ]

circle/dashboard/migrations/0008_profile_template_instance_limit.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-11-06 13:33
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0007_groupprofile_disk_quota'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='profile',
+            name='template_instance_limit',
+            field=models.IntegerField(default=1),
+        ),
+    ]

circle/dashboard/models.py

@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
 
 
 from vm.models.instance import ACCESS_METHODS
 from vm.models.instance import ACCESS_METHODS
 
 
-from .store_api import Store, NoStoreException, NotOkException, Timeout
+from .store_api import Store, NoStoreException, NotOkException
 from .validators import connect_command_template_validator
 from .validators import connect_command_template_validator
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
@@ -162,7 +162,7 @@ class ConnectCommand(Model):
                          validators=[connect_command_template_validator])
                          validators=[connect_command_template_validator])
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.template
         return self.template
@@ -178,6 +178,7 @@ class Profile(Model):
         unique=True, blank=True, null=True, max_length=64,
         unique=True, blank=True, null=True, max_length=64,
         help_text=_('Unique identifier of the person, e.g. a student number.'))
         help_text=_('Unique identifier of the person, e.g. a student number.'))
     instance_limit = IntegerField(default=5)
     instance_limit = IntegerField(default=5)
+    template_instance_limit = IntegerField(default=1)
     use_gravatar = BooleanField(
     use_gravatar = BooleanField(
         verbose_name=_("Use Gravatar"), default=True,
         verbose_name=_("Use Gravatar"), default=True,
         help_text=_("Whether to use email address as Gravatar profile image"))
         help_text=_("Whether to use email address as Gravatar profile image"))
@@ -218,7 +219,7 @@ class Profile(Model):
                     'id': command.id,
                     'id': command.id,
                     'cmd': command.template % {
                     'cmd': command.template % {
                         'port': instance.get_connect_port(use_ipv6=use_ipv6),
                         'port': instance.get_connect_port(use_ipv6=use_ipv6),
-                        'host':  instance.get_connect_host(use_ipv6=use_ipv6),
+                        'host': instance.get_connect_host(use_ipv6=use_ipv6),
                         'password': instance.pw,
                         'password': instance.pw,
                         'username': 'cloud',
                         'username': 'cloud',
                     }} for command in commands]
                     }} for command in commands]
@@ -263,7 +264,7 @@ class Profile(Model):
         super(Profile, self).save(*args, **kwargs)
         super(Profile, self).save(*args, **kwargs)
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
         permissions = (
         permissions = (
             ('use_autocomplete', _('Can use autocomplete.')),
             ('use_autocomplete', _('Can use autocomplete.')),
         )
         )
@@ -275,7 +276,7 @@ class FutureMember(Model):
     group = ForeignKey(Group)
     group = ForeignKey(Group)
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
         unique_together = ('org_id', 'group')
         unique_together = ('org_id', 'group')
 
 
     def __unicode__(self):
     def __unicode__(self):
@@ -293,9 +294,13 @@ class GroupProfile(AclBase):
         unique=True, blank=True, null=True, max_length=64,
         unique=True, blank=True, null=True, max_length=64,
         help_text=_('Unique identifier of the group at the organization.'))
         help_text=_('Unique identifier of the group at the organization.'))
     description = TextField(blank=True)
     description = TextField(blank=True)
+    disk_quota = FileSizeField(
+        verbose_name=_('disk quota'),
+        default=2048 * 1024 * 1024,
+        help_text=_('Disk quota in mebibytes.'))
 
 
     class Meta:
     class Meta:
-        ordering = ('id', )
+        ordering = ('id',)
 
 
     def __unicode__(self):
     def __unicode__(self):
         return self.group.name
         return self.group.name
@@ -331,7 +336,11 @@ def create_profile(user):
     profile, created = Profile.objects.get_or_create(user=user)
     profile, created = Profile.objects.get_or_create(user=user)
 
 
     try:
     try:
-        Store(user).create_user(profile.smb_password, None, profile.disk_quota)
+        store = Store(user)
+        quotas = [profile.disk_quota]
+        quotas += [group.profile.disk_quota for group in user.groups.all()]
+        max_quota = max(quotas)
+        store.create_user(profile.smb_password, None, max_quota)
     except:
     except:
         logger.exception("Can't create user %s", unicode(user))
         logger.exception("Can't create user %s", unicode(user))
     return created
     return created
@@ -347,6 +356,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
     logger.debug("Register save_org_id to djangosaml2 pre_user_save")
     logger.debug("Register save_org_id to djangosaml2 pre_user_save")
     from djangosaml2.signals import pre_user_save
     from djangosaml2.signals import pre_user_save
 
 
+
     def save_org_id(sender, instance, attributes, **kwargs):
     def save_org_id(sender, instance, attributes, **kwargs):
         logger.debug("save_org_id called by %s", instance.username)
         logger.debug("save_org_id called by %s", instance.username)
         atr = settings.SAML_ORG_ID_ATTRIBUTE
         atr = settings.SAML_ORG_ID_ATTRIBUTE
@@ -399,6 +409,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
 
 
         return False  # User did not change
         return False  # User did not change
 
 
+
     pre_user_save.connect(save_org_id)
     pre_user_save.connect(save_org_id)
 
 
 
 
@@ -411,7 +422,7 @@ def update_store_profile(sender, **kwargs):
                       profile.disk_quota)
                       profile.disk_quota)
     except NoStoreException:
     except NoStoreException:
         logger.debug("Store is not available.")
         logger.debug("Store is not available.")
-    except (NotOkException, Timeout):
+    except NotOkException:
         logger.critical("Store is not accepting connections.")
         logger.critical("Store is not accepting connections.")
 
 
 
 

circle/dashboard/static/dashboard/activity.js

@@ -41,12 +41,14 @@ $(function() {
         $('#confirmation-modal').on('hidden.bs.modal', function() {
         $('#confirmation-modal').on('hidden.bs.modal', function() {
           $('#confirmation-modal').remove();
           $('#confirmation-modal').remove();
         });
         });
+
         $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
         $('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
       }
       }
     });
     });
     e.preventDefault();
     e.preventDefault();
   });
   });
 
 
+
   /* if the operation fails show the modal again */
   /* if the operation fails show the modal again */
   $("body").on("click", "#confirmation-modal #op-form-send", function() {
   $("body").on("click", "#confirmation-modal #op-form-send", function() {
     var url = $(this).closest("form").prop("action");
     var url = $(this).closest("form").prop("action");
@@ -237,4 +239,3 @@ String.prototype.hashCode = function() {
   }
   }
   return hash;
   return hash;
 };
 };
-

circle/dashboard/static/dashboard/dashboard.js

@@ -558,3 +558,5 @@ $(function() {
     inputs.prop("checked", !inputs.prop("checked"));
     inputs.prop("checked", !inputs.prop("checked"));
   });
   });
 });
 });
+
+$.fn.modal.Constructor.prototype.enforceFocus = function() {};

circle/dashboard/static/dashboard/dashboard.less

@@ -1079,6 +1079,10 @@ textarea[name="new_members"] {
   max-width: 100%;
   max-width: 100%;
 }
 }
 
 
+#node-list-auto-migration-body {
+  padding: 20px;
+}
+
 #vm-list-table td.state,
 #vm-list-table td.state,
 #vm-list-table td.memory {
 #vm-list-table td.memory {
   white-space: nowrap;
   white-space: nowrap;
@@ -1088,7 +1092,7 @@ textarea[name="new_members"] {
   vertical-align: middle;
   vertical-align: middle;
 }
 }
 
 
-.disk-resize-btn {
+.disk-resize-btn, .disk-export-btn {
   margin-right: 5px;
   margin-right: 5px;
 }
 }
 
 

circle/dashboard/static/dashboard/node-list.js

@@ -3,4 +3,11 @@ $(function() {
     // find disabled nodes, set danger (red) on the rows
     // find disabled nodes, set danger (red) on the rows
     $('.node-disabled').closest("tr").addClass('danger');
     $('.node-disabled').closest("tr").addClass('danger');
   });
   });
+  $('#reschedule-now').click(function() {
+    $.get($(this).attr('href'), function(data){
+      highlight = data.result === 'ok' ? 'success' : 'danger';
+      addMessage(data.message, highlight);
+    });
+    return false;
+  });
 });
 });

circle/dashboard/store_api.py

@@ -14,19 +14,20 @@
 #
 #
 # You should have received a copy of the GNU General Public License along
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
-
-from os.path import splitext
 import json
 import json
 import logging
 import logging
 from urlparse import urljoin
 from urlparse import urljoin
-from datetime import datetime
 
 
-from django.http import Http404
+import os
+from datetime import datetime
 from django.conf import settings
 from django.conf import settings
+from django.http import Http404
+from os.path import splitext
 from requests import get, post, codes
 from requests import get, post, codes
-from requests.exceptions import Timeout  # noqa
 from sizefield.utils import filesizeformat
 from sizefield.utils import filesizeformat
 
 
+from storage.models import Disk
+
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 
 
 
@@ -47,6 +48,17 @@ class NoStoreException(StoreApiException):
 class Store(object):
 class Store(object):
 
 
     def __init__(self, user, default_timeout=0.5):
     def __init__(self, user, default_timeout=0.5):
+        self.store_url = settings.STORE_URL
+        if not self.store_url:
+            raise NoStoreException
+
+        if user.is_superuser and not user.profile.org_id:
+            self.username = 'u-admin'
+        elif not user.profile.org_id:
+            raise NoStoreException
+        else:
+            self.username = 'u-%s' % user.profile.org_id
+
         self.request_args = {'verify': settings.STORE_VERIFY_SSL}
         self.request_args = {'verify': settings.STORE_VERIFY_SSL}
         if settings.STORE_SSL_AUTH:
         if settings.STORE_SSL_AUTH:
             self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
             self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
@@ -54,18 +66,15 @@ class Store(object):
         if settings.STORE_BASIC_AUTH:
         if settings.STORE_BASIC_AUTH:
             self.request_args['auth'] = (settings.STORE_CLIENT_USER,
             self.request_args['auth'] = (settings.STORE_CLIENT_USER,
                                          settings.STORE_CLIENT_PASSWORD)
                                          settings.STORE_CLIENT_PASSWORD)
-        self.username = "u-%d" % user.pk
         self.default_timeout = default_timeout
         self.default_timeout = default_timeout
-        self.store_url = settings.STORE_URL
-        if not self.store_url:
-            raise NoStoreException
 
 
     def _request(self, url, method=get, timeout=None,
     def _request(self, url, method=get, timeout=None,
                  raise_status_code=True, **kwargs):
                  raise_status_code=True, **kwargs):
         url = urljoin(self.store_url, url)
         url = urljoin(self.store_url, url)
         if timeout is None:
         if timeout is None:
             timeout = self.default_timeout
             timeout = self.default_timeout
-        payload = json.dumps(kwargs) if kwargs else None
+        kwargs['USER'] = self.username
+        payload = json.dumps(kwargs)
         try:
         try:
             headers = {'content-type': 'application/json'}
             headers = {'content-type': 'application/json'}
             response = method(url, data=payload, headers=headers,
             response = method(url, data=payload, headers=headers,
@@ -83,7 +92,7 @@ class Store(object):
             return response
             return response
 
 
     def _request_cmd(self, cmd, **kwargs):
     def _request_cmd(self, cmd, **kwargs):
-        return self._request(self.username, post, CMD=cmd, **kwargs)
+        return self._request("/user/", post, CMD=cmd, **kwargs)
 
 
     def list(self, path, process=True):
     def list(self, path, process=True):
         r = self._request_cmd("LIST", PATH=path)
         r = self._request_cmd("LIST", PATH=path)
@@ -101,13 +110,22 @@ class Store(object):
         else:
         else:
             return result
             return result
 
 
+    def get_disk_images(self, path='/'):
+        images = []
+        file_list = self.list(path, process=False)
+        export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
+        for item in file_list:
+            if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
+                images.append(os.path.join(path, item['NAME']))
+        return images
+
     def request_download(self, path):
     def request_download(self, path):
         r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
         r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
         return r.json()['LINK']
         return r.json()['LINK']
 
 
     def request_upload(self, path):
     def request_upload(self, path):
-            r = self._request_cmd("UPLOAD", PATH=path)
-            return r.json()['LINK']
+        r = self._request_cmd("UPLOAD", PATH=path)
+        return r.json()['LINK']
 
 
     def remove(self, path):
     def remove(self, path):
         self._request_cmd("REMOVE", PATH=path)
         self._request_cmd("REMOVE", PATH=path)
@@ -119,7 +137,7 @@ class Store(object):
         self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
         self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
 
 
     def get_quota(self):  # no CMD? :o
     def get_quota(self):  # no CMD? :o
-        r = self._request(self.username)
+        r = self._request("/user/")
         quota = r.json()
         quota = r.json()
         quota.update({
         quota.update({
             'readable_used': filesizeformat(float(quota['used'])),
             'readable_used': filesizeformat(float(quota['used'])),
@@ -129,17 +147,17 @@ class Store(object):
         return quota
         return quota
 
 
     def set_quota(self, quota):
     def set_quota(self, quota):
-        self._request("/quota/" + self.username, post, QUOTA=quota)
+        self._request("/quota/", post, QUOTA=quota)
 
 
     def user_exist(self):
     def user_exist(self):
         try:
         try:
-            self._request(self.username)
+            self._request("/user/")
             return True
             return True
         except NotOkException:
         except NotOkException:
             return False
             return False
 
 
     def create_user(self, password, keys, quota):
     def create_user(self, password, keys, quota):
-        self._request("/new/" + self.username, method=post,
+        self._request("/new/", method=post,
                       SMBPASSWD=password, KEYS=keys, QUOTA=quota)
                       SMBPASSWD=password, KEYS=keys, QUOTA=quota)
 
 
     @staticmethod
     @staticmethod

circle/dashboard/templates/dashboard/_disk-list-element.html

@@ -6,15 +6,29 @@
 
 
 
 
 <span class="operation-wrapper pull-right">
 <span class="operation-wrapper pull-right">
+  {% if d.is_exportable %}
+    {% if op.export_disk %}
+      <a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
+         class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
+      {% if op.export_disk.disabled %}disabled{% endif %}">
+      <i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
+    </a>
+    {% endif %}
+  {% else %}
+    <small class="btn-xs">
+      {% trans "Not exportable" %}
+    </small>
+  {% endif %}
   {% if d.is_resizable %}
   {% if d.is_resizable %}
     {% if op.resize_disk %}
     {% if op.resize_disk %}
-      <a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
-        class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
+      <a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
+         class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
         {% if op.resize_disk.disabled %}disabled{% endif %}">
         {% if op.resize_disk.disabled %}disabled{% endif %}">
         <i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
         <i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
       </a>
       </a>
     {% else %}
     {% else %}
-      <a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
+      <a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
+         class="btn btn-xs btn-primary operation">
         <i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
         <i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
       </a>
       </a>
     {% endif %}
     {% endif %}
@@ -24,8 +38,8 @@
     </small>
     </small>
   {% endif %}
   {% endif %}
   {% if op.remove_disk %}
   {% if op.remove_disk %}
-    <a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
-      class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
+    <a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
+       class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
       {% if op.remove_disk.disabled %}disabled{% endif %}">
       {% if op.remove_disk.disabled %}disabled{% endif %}">
       <i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
       <i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
     </a>
     </a>

circle/dashboard/templates/dashboard/node-list.html

@@ -41,4 +41,23 @@
   </div><!-- -col-md-12 -->
   </div><!-- -col-md-12 -->
 </div><!-- .row -->
 </div><!-- .row -->
 
 
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+        <a id="reschedule-now" class="btn btn-danger pull-right" href="{% url "dashboard.views.reschedule" %}">
+          <i class="fa fa-magic"></i> {% trans "Reschedule now" %}
+        </a>
+        <h3 class="no-margin"><i class="fa fa-truck"></i> {% trans "Virtual machine auto migration" %}</h3>
+      </div>
+      <div id="node-list-auto-migration-body">
+        <h1>Crontab</h1>
+        <form>
+          {{ auto_migration_form.as_p }}
+        </form>
+      </div>
+    </div>
+  </div><!-- -col-md-12 -->
+</div><!-- .row -->
+
 {% endblock %}
 {% endblock %}

circle/dashboard/templates/dashboard/vm-detail/_disk-operations.html

 {% load i18n %}
 {% load i18n %}
 
 
 {% for op in ops %}
 {% for op in ops %}
-{% if op.is_disk_operation %}
-    <a href="{{op.get_url}}" class="btn btn-success btn-xs
-            operation operation-{{op.op}}">
-      <i class="fa fa-{{op.icon}} fa-fw-12"></i>
-            {{op.name}} </a>
-{% endif %}
+  {% if op.is_disk_operation %}
+    <a href="{{ op.get_url }}" class="btn btn-success btn-xs
+            operation operation-{{ op.op }}">
+      <i class="fa fa-{{ op.icon }} fa-fw-12"></i>
+      {{ op.name }} </a>
+  {% endif %}
 {% endfor %}
 {% endfor %}

circle/dashboard/templates/dashboard/vm-detail/network.html

@@ -37,6 +37,9 @@
         <dl>
         <dl>
           <dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
           <dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
           <dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
           <dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
+          {% if request.user.is_superuser %}
+          <dt>{% trans "MAC address" %}:</dt> <dd>{{ i.host.mac }}</dd>
+          {% endif %}
           <dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
           <dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
           <dt>{% trans "Groups" %}:</dt>
           <dt>{% trans "Groups" %}:</dt>
           <dd>
           <dd>
@@ -114,7 +117,9 @@
                   {% if l.ipv6 %}
                   {% if l.ipv6 %}
                     <tr>
                     <tr>
                       <td>
                       <td>
-                        {% display_portforward6 l %}
+                        {% autoescape off %}
+                          {% display_portforward6 l %}
+                        {% endautoescape %}
                       </td>
                       </td>
                       <td><i class="fa fa-long-arrow-right"></i></td>
                       <td><i class="fa fa-long-arrow-right"></i></td>
                       <td>
                       <td>

circle/dashboard/tests/test_views.py

@@ -534,7 +534,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
         with patch.object(DeployOperation, 'async') as async:
         with patch.object(DeployOperation, 'async') as async:
             response = c.post("/dashboard/vm/create/", {
             response = c.post("/dashboard/vm/create/", {
                 'name': 'vm',
                 'name': 'vm',
-                'amount': 2,
+                'amount': 1,
                 'customized': 1,
                 'customized': 1,
                 'template': 1,
                 'template': 1,
                 'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
                 'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
@@ -543,7 +543,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
 
 
         assert async.called
         assert async.called
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response.status_code, 302)
-        self.assertEqual(instance_count + 2, Instance.objects.all().count())
+        self.assertEqual(instance_count + 1, Instance.objects.all().count())
 
 
     def test_unpermitted_description_update(self):
     def test_unpermitted_description_update(self):
         c = Client()
         c = Client()

circle/dashboard/urls.py

@@ -56,6 +56,7 @@ from .views import (
     MessageList, MessageDetail, MessageCreate, MessageDelete,
     MessageList, MessageDetail, MessageCreate, MessageDelete,
     EnableTwoFactorView, DisableTwoFactorView,
     EnableTwoFactorView, DisableTwoFactorView,
     AclUserGroupAutocomplete, AclUserAutocomplete,
     AclUserGroupAutocomplete, AclUserAutocomplete,
+    RescheduleView,
 )
 )
 from .views.vm import vm_ops, vm_mass_ops
 from .views.vm import vm_ops, vm_mass_ops
 from .views.node import node_ops
 from .views.node import node_ops
@@ -153,6 +154,8 @@ urlpatterns = [
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
         NodeListGraphView.as_view(),
         NodeListGraphView.as_view(),
         name='dashboard.views.node-list-graph'),
         name='dashboard.views.node-list-graph'),
+    url(r'^node/reschedule/$', RescheduleView.as_view(),
+        name="dashboard.views.reschedule"),
     url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
     url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
          r'(?P<time>[0-9]{1,2}[hdwy])$'),
         TemplateGraphView.as_view(),
         TemplateGraphView.as_view(),

circle/dashboard/views/node.py

@@ -25,7 +25,7 @@ from django.core.exceptions import PermissionDenied
 from django.core.urlresolvers import reverse_lazy
 from django.core.urlresolvers import reverse_lazy
 from django.db.models import Count
 from django.db.models import Count
 from django.forms.models import inlineformset_factory
 from django.forms.models import inlineformset_factory
-from django.http import HttpResponse
+from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect
 from django.shortcuts import redirect
 from django.template.loader import render_to_string
 from django.template.loader import render_to_string
 from django.utils.translation import ugettext as _
 from django.utils.translation import ugettext as _
@@ -37,11 +37,14 @@ from django_tables2 import SingleTableView
 from firewall.models import Host
 from firewall.models import Host
 from vm.models import Node, NodeActivity, Trait
 from vm.models import Node, NodeActivity, Trait
 from vm.tasks.vm_tasks import check_queue
 from vm.tasks.vm_tasks import check_queue
+from vm.tasks.local_periodic_tasks import auto_migrate
 
 
-from ..forms import TraitForm, HostForm, NodeForm
+from ..forms import TraitForm, HostForm, NodeForm, AutoMigrationForm
 from ..tables import NodeListTable
 from ..tables import NodeListTable
 from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
 from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
 
 
+from manager.mancelery import crontab_parser
+
 
 
 def get_operations(instance, user):
 def get_operations(instance, user):
     ops = []
     ops = []
@@ -190,6 +193,14 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
     table_class = NodeListTable
     table_class = NodeListTable
     table_pagination = False
     table_pagination = False
 
 
+    def get_crontab(self):
+        return crontab_parser(settings.AUTO_MIGRATION_CRONTAB)
+
+    def get_context_data(self):
+        context = super(NodeList, self).get_context_data()
+        context["auto_migration_form"] = AutoMigrationForm(self.get_crontab())
+        return context
+
     def get(self, *args, **kwargs):
     def get(self, *args, **kwargs):
         if not self.request.user.has_perm('vm.view_statistics'):
         if not self.request.user.has_perm('vm.view_statistics'):
             raise PermissionDenied()
             raise PermissionDenied()
@@ -210,9 +221,20 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
             return super(NodeList, self).get(*args, **kwargs)
             return super(NodeList, self).get(*args, **kwargs)
 
 
     def get_queryset(self):
     def get_queryset(self):
+        self.wrong_nodes_message()
         return Node.objects.annotate(
         return Node.objects.annotate(
             number_of_VMs=Count('instance_set')).select_related('host')
             number_of_VMs=Count('instance_set')).select_related('host')
 
 
+    def wrong_nodes_message(self):
+        wrong_nodes = []
+        for node in Node.objects.all():
+            if node.monitor_info is None:
+                wrong_nodes.append(node.name)
+        message = ', '.join(wrong_nodes)
+        if wrong_nodes:
+            messages.error(self.request,
+                           "Can't reach " + message + " monitor info")
+
 
 
 class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
 class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
 
 
@@ -356,3 +378,23 @@ class NodeActivityDetail(LoginRequiredMixin, SuperuserRequiredMixin,
         ).order_by('-started').select_related())
         ).order_by('-started').select_related())
         ctx['icon'] = _get_activity_icon(self.object)
         ctx['icon'] = _get_activity_icon(self.object)
         return ctx
         return ctx
+
+
+class RescheduleView(SuperuserRequiredMixin, View):
+    def get(self, *args, **kwargs):
+        try:
+            auto_migrate.apply_async(queue='localhost.man.slow')
+        except Exception as e:
+            msg = str(e)
+            result = 'error'
+        else:
+            result = 'ok'
+            msg = _('Reschedule has started.')
+        if self.request.is_ajax():
+            return JsonResponse({'result': result, 'message': msg})
+        else:
+            if result == 'ok':
+                messages.success(self.request, msg)
+            else:
+                messages.error(self.request, msg)
+        return redirect('dashboard.views.node-list')

circle/dashboard/views/store.py

@@ -35,7 +35,8 @@ from django.views.generic import TemplateView
 
 
 from braces.views import LoginRequiredMixin
 from braces.views import LoginRequiredMixin
 
 
-from ..store_api import Store, NoStoreException, NotOkException
+from ..store_api import (Store, NoStoreException,
+                         NotOkException)
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 

circle/dashboard/views/util.py

@@ -682,7 +682,7 @@ class TransferOwnershipConfirmView(LoginRequiredMixin, View):
             messages.error(request, _('This token is invalid or has expired.'))
             messages.error(request, _('This token is invalid or has expired.'))
             raise PermissionDenied()
             raise PermissionDenied()
         return render(request, self.template,
         return render(request, self.template,
-                      dictionary={'instance': instance, 'key': key})
+                      {'instance': instance, 'key': key})
 
 
     def change_owner(self, instance, new_owner):
     def change_owner(self, instance, new_owner):
         instance.owner = new_owner
         instance.owner = new_owner

circle/dashboard/views/vm.py

@@ -61,9 +61,10 @@ from .util import (
 )
 )
 from ..forms import (
 from ..forms import (
     AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
     AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
-    VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
+    VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
+    VmImportDiskForm, VmSaveForm,
     VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
     VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
-    VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
+    VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
     VmMigrateForm, VmDeployForm,
     VmMigrateForm, VmDeployForm,
     VmPortRemoveForm, VmPortAddForm,
     VmPortRemoveForm, VmPortAddForm,
     VmRemoveInterfaceForm,
     VmRemoveInterfaceForm,
@@ -166,8 +167,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
 
 
         # resources forms
         # resources forms
         can_edit = (
         can_edit = (
-            instance.has_level(user, "owner") and
-            self.request.user.has_perm("vm.change_resources"))
+                instance.has_level(user, "owner") and
+                self.request.user.has_perm("vm.change_resources"))
         context['resources_form'] = VmResourcesForm(
         context['resources_form'] = VmResourcesForm(
             can_edit=can_edit, instance=instance)
             can_edit=can_edit, instance=instance)
 
 
@@ -269,7 +270,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
             return JsonResponse({'message': message})
             return JsonResponse({'message': message})
         else:
         else:
             return redirect(reverse_lazy("dashboard.views.detail",
             return redirect(reverse_lazy("dashboard.views.detail",
-                            kwargs={'pk': self.object.pk}))
+                                         kwargs={'pk': self.object.pk}))
 
 
     def __abort_operation(self, request):
     def __abort_operation(self, request):
         self.object = self.get_object()
         self.object = self.get_object()
@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
 
 
 
 
 class VmOperationView(AjaxOperationMixin, OperationView):
 class VmOperationView(AjaxOperationMixin, OperationView):
-
     model = Instance
     model = Instance
     context_object_name = 'instance'  # much simpler to mock object
     context_object_name = 'instance'  # much simpler to mock object
 
 
@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmAddInterfaceView(FormOperationMixin, VmOperationView):
 class VmAddInterfaceView(FormOperationMixin, VmOperationView):
-
     op = 'add_interface'
     op = 'add_interface'
     form_class = VmAddInterfaceForm
     form_class = VmAddInterfaceForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmCreateDiskView(FormOperationMixin, VmOperationView):
 class VmCreateDiskView(FormOperationMixin, VmOperationView):
-
     op = 'create_disk'
     op = 'create_disk'
     form_class = VmCreateDiskForm
     form_class = VmCreateDiskForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
         return val
         return val
 
 
 
 
-class VmDownloadDiskView(FormOperationMixin, VmOperationView):
+class VmImportDiskView(FormOperationMixin, VmOperationView):
+    op = 'import_disk'
+    form_class = VmImportDiskForm
+    show_in_toolbar = False
+    icon = 'upload'
+    effect = "success"
+    is_disk_operation = True
+    with_reload = True
+
+    def get_form_kwargs(self):
+        val = super(VmImportDiskView, self).get_form_kwargs()
+        val.update({'user': self.request.user})
+        return val
+
 
 
+class VmDownloadDiskView(FormOperationMixin, VmOperationView):
     op = 'download_disk'
     op = 'download_disk'
     form_class = VmDownloadDiskForm
     form_class = VmDownloadDiskForm
     show_in_toolbar = False
     show_in_toolbar = False
@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmMigrateView(FormOperationMixin, VmOperationView):
 class VmMigrateView(FormOperationMixin, VmOperationView):
-
     op = 'migrate'
     op = 'migrate'
     icon = 'truck'
     icon = 'truck'
     effect = 'info'
     effect = 'info'
@@ -449,8 +460,7 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
         if isinstance(inst, Instance):
         if isinstance(inst, Instance):
             nodes_w_traits = [
             nodes_w_traits = [
                 n.pk for n in Node.objects.filter(enabled=True)
                 n.pk for n in Node.objects.filter(enabled=True)
-                if n.online and
-                has_traits(inst.req_traits.all(), n)
+                if n.online and has_traits(inst.req_traits.all(), n)
             ]
             ]
             ctx['nodes_w_traits'] = nodes_w_traits
             ctx['nodes_w_traits'] = nodes_w_traits
 
 
@@ -458,7 +468,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmPortRemoveView(FormOperationMixin, VmOperationView):
 class VmPortRemoveView(FormOperationMixin, VmOperationView):
-
     template_name = 'dashboard/_vm-remove-port.html'
     template_name = 'dashboard/_vm-remove-port.html'
     op = 'remove_port'
     op = 'remove_port'
     show_in_toolbar = False
     show_in_toolbar = False
@@ -487,7 +496,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmPortAddView(FormOperationMixin, VmOperationView):
 class VmPortAddView(FormOperationMixin, VmOperationView):
-
     op = 'add_port'
     op = 'add_port'
     show_in_toolbar = False
     show_in_toolbar = False
     with_reload = True
     with_reload = True
@@ -514,7 +522,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
 
 
 
 
 class VmSaveView(FormOperationMixin, VmOperationView):
 class VmSaveView(FormOperationMixin, VmOperationView):
-
     op = 'save_as_template'
     op = 'save_as_template'
     icon = 'save'
     icon = 'save'
     effect = 'info'
     effect = 'info'
@@ -570,7 +577,7 @@ class TokenOperationView(OperationView):
     User can do the action with a valid token instead of logging in.
     User can do the action with a valid token instead of logging in.
     """
     """
     token_max_age = 3 * 24 * 3600
     token_max_age = 3 * 24 * 3600
-    redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
+    redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
 
 
     @classmethod
     @classmethod
     def get_salt(cls):
     def get_salt(cls):
@@ -642,7 +649,6 @@ class TokenOperationView(OperationView):
 
 
 
 
 class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
 class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
-
     op = 'renew'
     op = 'renew'
     icon = 'calendar'
     icon = 'calendar'
     effect = 'success'
     effect = 'success'
@@ -769,7 +775,11 @@ vm_ops = OrderedDict([
         extra_bases=[TokenOperationView],
         extra_bases=[TokenOperationView],
         op='destroy', icon='times', effect='danger')),
         op='destroy', icon='times', effect='danger')),
     ('create_disk', VmCreateDiskView),
     ('create_disk', VmCreateDiskView),
+    ('import_disk', VmImportDiskView),
     ('download_disk', VmDownloadDiskView),
     ('download_disk', VmDownloadDiskView),
+    ('export_disk', VmDiskModifyView.factory(
+        op='export_disk', form_class=VmDiskExportForm,
+        icon='download', effect='info')),
     ('resize_disk', VmDiskModifyView.factory(
     ('resize_disk', VmDiskModifyView.factory(
         op='resize_disk', form_class=VmDiskResizeForm,
         op='resize_disk', form_class=VmDiskResizeForm,
         icon='arrows-alt', effect="warning")),
         icon='arrows-alt', effect="warning")),
@@ -1015,7 +1025,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
         # remove "-" that means descending order
         # remove "-" that means descending order
         # also check if the column name is valid
         # also check if the column name is valid
         if (sort and
         if (sort and
-            (sort[1:] if sort[0] == "-" else sort)
+                (sort[1:] if sort[0] == "-" else sort)
                 in [i.name for i in Instance._meta.fields] + ["pk"]):
                 in [i.name for i in Instance._meta.fields] + ["pk"]):
             queryset = queryset.order_by(sort)
             queryset = queryset.order_by(sort)
 
 
@@ -1030,7 +1040,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
 
 
 
 
 class VmCreate(LoginRequiredMixin, TemplateView):
 class VmCreate(LoginRequiredMixin, TemplateView):
-
     form_class = VmCustomizeForm
     form_class = VmCustomizeForm
     form = None
     form = None
 
 
@@ -1139,7 +1148,7 @@ class VmCreate(LoginRequiredMixin, TemplateView):
             messages.success(request, ungettext_lazy(
             messages.success(request, ungettext_lazy(
                 "Successfully created %(count)d VM.",  # this should not happen
                 "Successfully created %(count)d VM.",  # this should not happen
                 "Successfully created %(count)d VMs.", len(instances)) % {
                 "Successfully created %(count)d VMs.", len(instances)) % {
-                'count': len(instances)})
+                                 'count': len(instances)})
             path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
             path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
         else:
         else:
             messages.success(request, _("VM successfully created."))
             messages.success(request, _("VM successfully created."))
@@ -1161,24 +1170,28 @@ class VmCreate(LoginRequiredMixin, TemplateView):
 
 
         # limit chekcs
         # limit chekcs
         try:
         try:
-            limit = user.profile.instance_limit
+            instance_limit = user.profile.instance_limit
+            template_instance_limit = user.profile.template_instance_limit
         except Exception as e:
         except Exception as e:
             logger.debug('No profile or instance limit: %s', e)
             logger.debug('No profile or instance limit: %s', e)
         else:
         else:
             try:
             try:
                 amount = int(request.POST.get("amount", 1))
                 amount = int(request.POST.get("amount", 1))
             except:
             except:
-                amount = limit  # TODO this should definitely use a Form
-            current = Instance.active.filter(owner=user).count()
-            logger.debug('current use: %d, limit: %d', current, limit)
-            if current + amount > limit:
-                messages.error(request,
-                               _('Instance limit (%d) exceeded.') % limit)
-                if request.is_ajax():
-                    return HttpResponse(json.dumps({'redirect': '/'}),
-                                        content_type="application/json")
-                else:
-                    return redirect('/')
+                # TODO this should definitely use a Form
+                amount = instance_limit
+            instances = Instance.active.filter(owner=user).count()
+            template_instances = template.get_user_instances(user).count()
+
+            logger.debug('current instance use: %d, limit: %d',
+                         instances, instance_limit)
+            logger.debug('current template instance use: %d, limit: %d',
+                         template_instances, template_instance_limit)
+
+            if instances + amount > instance_limit:
+                return self._limit_exceeded(instance_limit, request)
+            if template_instances + amount > template_instance_limit:
+                return self._limit_exceeded(template_instance_limit, request)
 
 
         create_func = (self.__create_normal if
         create_func = (self.__create_normal if
                        request.POST.get("customized") is None else
                        request.POST.get("customized") is None else
@@ -1186,6 +1199,16 @@ class VmCreate(LoginRequiredMixin, TemplateView):
 
 
         return create_func(request, template, *args, **kwargs)
         return create_func(request, template, *args, **kwargs)
 
 
+    def _limit_exceeded(self, limit, request):
+        messages.error(request,
+                       _('Instance limit (%d) exceeded.')
+                       % limit)
+        if request.is_ajax():
+            return HttpResponse(json.dumps({'redirect': '/'}),
+                                content_type="application/json")
+        else:
+            return redirect('/')
+
 
 
 @require_GET
 @require_GET
 def get_vm_screenshot(request, pk):
 def get_vm_screenshot(request, pk):
@@ -1331,7 +1354,7 @@ class TransferInstanceOwnershipConfirmView(TransferOwnershipConfirmView):
 
 
     def change_owner(self, instance, new_owner):
     def change_owner(self, instance, new_owner):
         with instance.activity(
         with instance.activity(
-            code_suffix='ownership-transferred',
+                code_suffix='ownership-transferred',
                 readable_name=ugettext_noop("transfer ownership"),
                 readable_name=ugettext_noop("transfer ownership"),
                 concurrency_check=False, user=new_owner):
                 concurrency_check=False, user=new_owner):
             super(TransferInstanceOwnershipConfirmView, self).change_owner(
             super(TransferInstanceOwnershipConfirmView, self).change_owner(

circle/manager/mancelery.py

@@ -17,13 +17,27 @@
 
 
 from celery import Celery
 from celery import Celery
 from celery.signals import worker_ready
 from celery.signals import worker_ready
+from celery.schedules import crontab
 from datetime import timedelta
 from datetime import timedelta
 from celery.schedules import crontab
 from celery.schedules import crontab
 from kombu import Queue, Exchange
 from kombu import Queue, Exchange
 from os import getenv
 from os import getenv
 
 
+
 HOSTNAME = "localhost"
 HOSTNAME = "localhost"
 QUEUE_NAME = HOSTNAME + '.man'
 QUEUE_NAME = HOSTNAME + '.man'
+AUTO_MIGRATION_CRONTAB = getenv('AUTO_MIGRATION_CRONTAB', '0 0 * * *')
+
+
+def crontab_parser(crontab):
+    fields = crontab.split(' ')
+    return dict(
+        minute=fields[0],
+        hour=fields[1],
+        day_of_month=fields[2],
+        month_of_year=fields[3],
+        day_of_week=fields[4],
+    )
 
 
 
 
 celery = Celery('manager',
 celery = Celery('manager',
@@ -56,6 +70,11 @@ celery.conf.update(
             'schedule': crontab(minute=10, hour=1),
             'schedule': crontab(minute=10, hour=1),
             'options': {'queue': 'localhost.man'}
             'options': {'queue': 'localhost.man'}
         },
         },
+#        'vm.local_periodic_tasks': {
+#            'task': 'vm.tasks.local_periodic_tasks.auto_migrate',
+#            'schedule': crontab(**crontab_parser(AUTO_MIGRATION_CRONTAB)),
+#            'options': {'queue': 'localhost.man.slow'},
+#        },
     }
     }
 
 
 )
 )

circle/manager/scheduler.py

@@ -15,15 +15,18 @@
 # You should have received a copy of the GNU General Public License along
 # You should have received a copy of the GNU General Public License along
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import datetime
+import json
+import random
 from logging import getLogger
 from logging import getLogger
 
 
+from django.conf import settings
+from django.core.cache import cache
+from django.utils import timezone
 from django.utils.translation import ugettext_noop
 from django.utils.translation import ugettext_noop
 
 
-from common.models import HumanReadableException
-
 from circle.settings.base import SCHEDULER_METHOD
 from circle.settings.base import SCHEDULER_METHOD
-
-import random
+from common.models import HumanReadableException
 
 
 logger = getLogger(__name__)
 logger = getLogger(__name__)
 
 
@@ -69,14 +72,14 @@ def common_select(instance, nodes):
         logger.warning('select_node: no enough RAM for %s', unicode(instance))
         logger.warning('select_node: no enough RAM for %s', unicode(instance))
         raise NotEnoughMemoryException()
         raise NotEnoughMemoryException()
 
 
-    # sort nodes first by processor usage, then priority
+    # sort nodes first by priority
     nodes.sort(key=lambda n: n.priority, reverse=True)
     nodes.sort(key=lambda n: n.priority, reverse=True)
-    nodes.sort(key=free_cpu_time, reverse=True)
     return nodes
     return nodes
 
 
 
 
 def common_evenly(instance, nodes):
 def common_evenly(instance, nodes):
     nodes = common_select(instance, nodes)
     nodes = common_select(instance, nodes)
+    nodes.sort(key=free_cpu_time, reverse=True)
     result = nodes[0]
     result = nodes[0]
     return result
     return result
 
 
@@ -87,6 +90,16 @@ def common_random(instance, nodes):
     return result
     return result
 
 
 
 
+def advanced_with_time_stamp(instance, nodes):
+    nodes = common_select(instance, nodes)
+    nodes.sort(key=sorting_key, reverse=True)
+    logger.info("SCHEDLOG: {}".format(json.dumps({
+        "event": "after_sort", 
+        "list": map(lambda node: unicode(node), nodes)})))
+    result = nodes[0]
+    return result
+
+
 def select_node(instance, nodes):
 def select_node(instance, nodes):
     ''' Select a node for hosting an instance based on its requirements.
     ''' Select a node for hosting an instance based on its requirements.
     '''
     '''
@@ -94,14 +107,72 @@ def select_node(instance, nodes):
         result = common_evenly(instance, nodes)
         result = common_evenly(instance, nodes)
     elif SCHEDULER_METHOD == 'random':
     elif SCHEDULER_METHOD == 'random':
         result = common_random(instance, nodes)
         result = common_random(instance, nodes)
+    elif SCHEDULER_METHOD == 'advanced':
+        result = advanced_with_time_stamp(instance, nodes)
     else:  # Default method is the random
     else:  # Default method is the random
         result = common_random(instance, nodes)
         result = common_random(instance, nodes)
 
 
-    logger.info('Scheduler method: %s selected', unicode(SCHEDULER_METHOD))
-    logger.info('select_node: %s for %s', unicode(result), unicode(instance))
+    logger.info("SCHEDLOG: {}".format(json.dumps(
+        {"event": "select",
+         "node": unicode(result),
+          "vm": unicode(instance)})))
+
+    set_time_stamp(result)
+
     return result
     return result
 
 
 
 
+def sorting_key(node):
+    """Determines how valuable a node is for scheduling.
+    """
+    key = 0
+    corr = last_scheduled_correction_factor(node)
+    if free_cpu_time(node) < free_ram(node):
+        key = free_cpu_time(node) * corr
+    else:
+        key = free_ram(node) * corr
+    logger.info("SCHEDLOG: {}".format(json.dumps({
+        "event": "sort",
+        "node": unicode(node),
+        "sorting_key": unicode(key),
+        "free_cpu_time": unicode(free_cpu_time(node)),
+        "free_ram": unicode(free_ram(node)),
+        "last_scheduled_correction_factor": unicode(last_scheduled_correction_factor(node))})))
+    return key
+
+
+def set_time_stamp(node):
+    cache.set('time_stamp{}'.format(node.id), timezone.now())
+
+
+def get_time_stamp(node):
+    time_stamp = cache.get('time_stamp{}'.format(node.id))
+    if time_stamp:
+        return time_stamp
+    return datetime.datetime(1970, 1, 1, tzinfo=timezone.get_current_timezone())
+
+
+def last_scheduled_correction_factor(node):
+    """Returns the time correction factor for a node.
+
+    The monitor data may be outdated, because of recent scheduling for a given node.
+    The return value is between 0 and 1, higher value indicates more time since the
+    last scheduling for the given node.
+    """
+    factor = 0
+    max_time_diff = settings.SCHEDULER_TIME_SENSITIVITY_IN_SECONDS
+    current_time = timezone.now()
+    time_difference_in_seconds = (
+        current_time - get_time_stamp(node)).total_seconds()
+    factor = time_difference_in_seconds/float(max_time_diff)
+    if factor > 1:
+        factor = 1
+    elif factor < 0:
+        factor = 1
+    logger.info('Scheduler set factor to %s', unicode(factor))
+    return factor
+
+
 def has_traits(traits, node):
 def has_traits(traits, node):
     """True, if the node has all specified traits; otherwise, false.
     """True, if the node has all specified traits; otherwise, false.
     """
     """
@@ -142,11 +213,27 @@ def free_cpu_time(node):
     Higher values indicate more idle time.
     Higher values indicate more idle time.
     """
     """
     try:
     try:
-        activity = node.cpu_usage / 100
-        inactivity = 1 - activity
-        cores = node.num_cores
-        return cores * inactivity
+        free_cpu_percent = 1 - node.cpu_usage
+        weight = node.cpu_weight
+        weighted_value = free_cpu_percent * weight
+        return weighted_value
+    except TypeError as e:
+        logger.exception('Got incorrect monitoring data for node %s. %s',
+                         unicode(node), unicode(e))
+        return 0  # will result lowest priority
+
+
+def free_ram(node):
+    """Get an indicator number for free RAM on the node.
+
+    Higher value indicates more RAM.
+    """
+    try:
+        free_ram_percent = 1 - node.ram_usage
+        weight = node.ram_weight
+        weighted_value = free_ram_percent * weight
+        return weighted_value
     except TypeError as e:
     except TypeError as e:
-        logger.warning('Got incorrect monitoring data for node %s. %s',
-                       unicode(node), unicode(e))
-        return False  # monitoring data is incorrect
+        logger.exception('Got incorrect monitoring data for node %s. %s',
+                         unicode(node), unicode(e))
+        return 0  # will result lowest priority

circle/request/forms.py

@@ -16,7 +16,7 @@
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 # with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
 from django.forms import (
 from django.forms import (
     ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
     ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
-    Textarea, ValidationError
+    Textarea, ValidationError, TextInput, IntegerField, EmailField
 )
 )
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 from django.template.loader import render_to_string
 from django.template.loader import render_to_string
@@ -27,11 +27,78 @@ from crispy_forms.helper import FormHelper
 from crispy_forms.layout import Submit
 from crispy_forms.layout import Submit
 
 
 from request.models import (
 from request.models import (
-    LeaseType, TemplateAccessType, TemplateAccessAction,
+    LeaseType, TemplateAccessType, TemplateAccessAction, RequestField
 )
 )
 from dashboard.forms import VmResourcesForm
 from dashboard.forms import VmResourcesForm
 
 
 
 
+class RequestFieldModelForm(ModelForm):
+    class Meta:
+        model = RequestField
+        fields = '__all__'
+        widgets = {
+            'choices': TextInput(attrs={'placeholder': 'Optional'}),
+            }
+
+    def __init__(self, *args, **kwargs):
+        super(RequestFieldModelForm, self).__init__(*args, **kwargs)
+        self.fields['choices'].required = False
+
+    @property
+    def helper(self):
+        helper = FormHelper()
+        return helper
+
+    def clean(self):
+        cleaned_data = super(RequestFieldModelForm, self).clean()
+        if cleaned_data['type'] == 'Email' and cleaned_data['choices']:
+            raise ValidationError(_("Email field can't have choices!"))
+        if cleaned_data['type'] == 'Integer':
+            for choice in cleaned_data['choices'].split(','):
+                try:
+                    int(choice)
+                except:
+                    raise ValidationError(_("IntegerField choices must be \
+                                            integers"))
+
+
+class EditableForm(Form):
+    def __init__(self, *args, **kwargs):
+        type = kwargs.pop('type', None)
+        kwargs.pop("request", None)
+        super(EditableForm, self).__init__(*args, **kwargs)
+        fields = RequestField.objects.filter(request_type=type)
+        n = 0
+        if fields:
+            for field in fields:
+                n = n+1
+                type = field.type
+                if(type == 'Char'):
+                    self.fields['field'+str(n)] = CharField(
+                                                   max_length=30,
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                elif (type == 'Integer'):
+                    self.fields['field'+str(n)] = IntegerField(
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                elif (type == 'Email'):
+                    self.fields['field'+str(n)] = EmailField(
+                                                   required=field.required,
+                                                   label=field.fieldname)
+                if(field.choices):
+                    choices = [(ch, ch)for ch in field.choices.split(',')]
+                    self.fields['field'+str(n)] = ChoiceField(
+                                                  choices=choices,
+                                                  required=field.required,
+                                                  label=field.fieldname)
+
+    def get_dynamic_fields(self):
+        for field_name in self.fields:
+            if field_name.startswith("field"):
+                yield self[field_name]
+
+
 class LeaseTypeForm(ModelForm):
 class LeaseTypeForm(ModelForm):
     @property
     @property
     def helper(self):
     def helper(self):
@@ -80,27 +147,18 @@ class InitialFromFileMixin(object):
         return message.strip()
         return message.strip()
 
 
 
 
-class TemplateRequestForm(InitialFromFileMixin, Form):
-    message = CharField(widget=Textarea, label=_("Message"))
+class TemplateRequestForm(EditableForm):
     template = ModelChoiceField(TemplateAccessType.objects.all(),
     template = ModelChoiceField(TemplateAccessType.objects.all(),
                                 label=_("Template share"))
                                 label=_("Template share"))
     level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
     level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
                         initial=TemplateAccessAction.LEVELS.user)
                         initial=TemplateAccessAction.LEVELS.user)
 
 
-    initial_template = "request/initials/template.html"
-
 
 
-class LeaseRequestForm(InitialFromFileMixin, Form):
+class LeaseRequestForm(EditableForm):
     lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
     lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
-    message = CharField(widget=Textarea, label=_("Message"))
-
-    initial_template = "request/initials/lease.html"
 
 
 
 
-class ResourceRequestForm(InitialFromFileMixin, VmResourcesForm):
-    message = CharField(widget=Textarea, label=_("Message"))
-
-    initial_template = "request/initials/resources.html"
+class ResourceRequestForm(EditableForm, VmResourcesForm):
 
 
     def clean(self):
     def clean(self):
         cleaned_data = super(ResourceRequestForm, self).clean()
         cleaned_data = super(ResourceRequestForm, self).clean()

circle/request/migrations/0005_requestfield.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-12 15:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0004_auto_20150629_1605'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RequestField',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('fieldname', models.CharField(max_length=50, unique=True)),
+                ('type', models.CharField(choices=[(b'Char', b'CharField'), (b'Integer', b'IntegerField'), (b'Email', b'EmailField')], default=b'Char', max_length=20)),
+                ('choices', models.CharField(max_length=100, null=True)),
+                ('required', models.BooleanField(default=True)),
+            ],
+        ),
+    ]

circle/request/migrations/0006_auto_20181115_1548.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-15 15:48
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0005_requestfield'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requestfield',
+            name='choices',
+            field=models.CharField(max_length=300, null=True),
+        ),
+    ]

circle/request/migrations/0007_requestfield_request_type.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-12-12 14:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('request', '0006_auto_20181115_1548'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='requestfield',
+            name='request_type',
+            field=models.CharField(choices=[(b'resource', 'resource request'), (b'lease', 'lease request'), (b'template', 'template access request'), (b'resize', 'disk resize request')], default=b'template', max_length=20),
+        ),
+    ]

circle/request/models.py

@@ -19,6 +19,7 @@ import logging
 
 
 from django.db.models import (
 from django.db.models import (
     Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
     Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
+    BooleanField,
 )
 )
 from django.db.models.signals import post_save
 from django.db.models.signals import post_save
 from django.conf import settings
 from django.conf import settings
@@ -157,6 +158,27 @@ class Request(TimeStampedModel):
         return self.action.is_acceptable()
         return self.action.is_acceptable()
 
 
 
 
+class RequestField(Model):
+    TYPES = (
+        ('Char', 'CharField'),
+        ('Integer', 'IntegerField'),
+        ('Email', 'EmailField')
+        )
+
+    fieldname = CharField(max_length=50, blank=False, unique=True)
+    type = CharField(choices=TYPES, default='Char', max_length=20)
+    request_type = CharField(choices=Request.TYPES, default='template',
+                             max_length=20)
+    choices = CharField(max_length=300, null=True)
+    required = BooleanField(default=True)
+
+    def __unicode__(self):
+        return self.fieldname
+
+    def get_absolute_url(self):
+        return reverse('fields_detail', kwargs={'pk': self.pk})
+
+
 class LeaseType(RequestType):
 class LeaseType(RequestType):
     lease = ForeignKey(Lease, verbose_name=_("Lease"))
     lease = ForeignKey(Lease, verbose_name=_("Lease"))
 
 

circle/request/templates/request/_request-field-form.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+{% load crispy_forms_tags %}
+
+{% block content %}
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading"></div>
+      <div class="panel-body">
+          <form action={% url 'request.views.request-field-add' %} method="post">
+            {% include "display-form-errors.html" %}
+            {% csrf_token %}
+            {% for field in form %}
+              {{ field|as_crispy_field}}
+            {% endfor %}
+            <button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
+          </form>
+      </div> <!-- .panel-body -->
+    </div>
+  </div>
+</div>
+{% endblock %}

circle/request/templates/request/_request-lease-form.html

@@ -6,5 +6,8 @@
   {% csrf_token %}
   {% csrf_token %}
   {{ form.lease|as_crispy_field }}
   {{ form.lease|as_crispy_field }}
   {{ form.message|as_crispy_field }}
   {{ form.message|as_crispy_field }}
+  {% for fields in form.get_dynamic_fields %}
+    {{ field|as_crispy_field }}
+  {% endfor %}
   <input type="submit" class="btn btn-primary"/>
   <input type="submit" class="btn btn-primary"/>
 </form>
 </form>

circle/request/templates/request/_request-template-form.html

@@ -21,6 +21,8 @@
     </label>
     </label>
   </div>
   </div>
   {% endfor %}
   {% endfor %}
-  {{ form.message|as_crispy_field }}
+  {% for field in form.get_dynamic_fields %}
+      {{ field|as_crispy_field }}
+  {% endfor %}
   <input type="submit" class="btn btn-primary"/>
   <input type="submit" class="btn btn-primary"/>
 </form>
 </form>

circle/request/templates/request/field-list.html

+{% extends "dashboard/base.html" %}
+{% load i18n %}
+{% load crispy_forms_field %}
+
+{% block content %}
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default">
+      <div class="panel-heading">
+          <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Request fields" %}</h3>
+         </div>
+            <div class="panel-body">
+              <div class="table-responsive">
+                <div class="table-container">
+                  <table class="text-center table table-bordered table-striped table-hover" >
+                    <thead class"align-center">
+                        <tr>
+                        <th class="text-center">{% trans "Fieldname" %}</th>
+                        <th class="text-center">{% trans "Type" %}</th>
+                        <th class="text-center">{% trans "RequestType" %}</th>
+                        <th class="text-center">{% trans "Choices" %}</th>
+                        <th class="text-center">{% trans "Required" %}</th>
+                        <th class="text-center">{% trans "Delete" %}</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for field in object_list %}
+                        <tr>
+                          <td>{{field.fieldname}}</td>
+                          <td>{{field.type}}</td>
+                          <td>{{field.request_type}}</td>
+                          <td>{{field.choices}}</td>
+                          <td>{{field.required}}</td>
+                          <td><a href={% url "request.views.field-delete" pk=field.pk %}>
+                            <i class="fa fa-times"></i></a>
+                          </td>
+                        </tr>
+                        {% endfor %}
+                        <tr>
+                        <form action={% url 'request.views.request-field-add' %} method="post">
+                          {% csrf_token %}
+                          {% for field in add_form %}
+                              <td> {% crispy_field field %}</td>
+                          {% endfor %}
+                        <td><button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
+                      </form>
+                      </tr>
+                      </tbody>
+                  </table>
+                </div>
+              </div> <!-- .table-responsive -->
+            </div> <!-- .panel-body -->
+          </div>
+        </div>
+      </div>
+
+{% endblock %}

circle/request/templates/request/list.html

@@ -11,9 +11,14 @@
   <div class="col-md-12">
   <div class="col-md-12">
     <div class="panel panel-default">
     <div class="panel panel-default">
       <div class="panel-heading">
       <div class="panel-heading">
-        <a class="btn btn-xs btn-primary pull-right "href="{% url "request.views.type-list" %}">
-          {% trans "Request types" %}
-        </a>
+        <div class="pull-right">
+            <a class="btn btn-xs btn-primary"href="{% url "request.views.type-list" %}">
+              {% trans "Request types" %}
+            </a>
+            <a class="btn btn-xs btn-success" href="{% url "request.views.field-list" %}">
+              {% trans "Request fields" %}
+            </a>
+          </div>
         <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Requests" %}</h3>
         <h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Requests" %}</h3>
       </div>
       </div>
       <div class="panel-body">
       <div class="panel-body">

circle/request/templates/request/request-resource.html

@@ -24,7 +24,9 @@
           </div>
           </div>
           {% include "display-form-errors.html" %}
           {% include "display-form-errors.html" %}
           {% include "dashboard/_resources-sliders.html" with field_priority=form.priority field_num_cores=form.num_cores field_ram_size=form.ram_size %}
           {% include "dashboard/_resources-sliders.html" with field_priority=form.priority field_num_cores=form.num_cores field_ram_size=form.ram_size %}
-          {{ form.message|as_crispy_field }}
+          {% for field in form.get_dynamic_fields %}
+              {{ field|as_crispy_field }}
+          {% endfor %}
           <button type="submit" class="btn btn-success">
           <button type="submit" class="btn btn-success">
             {% trans "Request new resources" %}
             {% trans "Request new resources" %}
           </button>
           </button>

circle/request/tests.py

@@ -25,7 +25,7 @@ from mock import Mock, patch
 from common.tests.celery_mock import MockCeleryMixin
 from common.tests.celery_mock import MockCeleryMixin
 from vm.models import Instance, InstanceTemplate, Lease
 from vm.models import Instance, InstanceTemplate, Lease
 from dashboard.models import Profile
 from dashboard.models import Profile
-from request.models import Request, LeaseType, TemplateAccessType
+from request.models import Request, LeaseType, TemplateAccessType, RequestField
 from dashboard.tests.test_views import LoginMixin
 from dashboard.tests.test_views import LoginMixin
 from vm.operations import ResourcesOperation
 from vm.operations import ResourcesOperation
 
 
@@ -67,12 +67,16 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
         inst = Instance.objects.get(pk=1)
         inst = Instance.objects.get(pk=1)
         inst.set_level(self.u1, 'owner')
         inst.set_level(self.u1, 'owner')
 
 
+        field = RequestField(fieldname="Oka", type="Char",
+                             request_type="resource", required=True)
+        field.save()
+
         req_count = Request.objects.count()
         req_count = Request.objects.count()
         resp = c.post("/request/resource/1/", {
         resp = c.post("/request/resource/1/", {
             'num_cores': 5,
             'num_cores': 5,
             'ram_size': 512,
             'ram_size': 512,
             'priority': 30,
             'priority': 30,
-            'message': "szia",
+            'field1': "szia",
         })
         })
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(req_count + 1, Request.objects.count())
         self.assertEqual(req_count + 1, Request.objects.count())
@@ -104,17 +108,25 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
         template = InstanceTemplate.objects.get(pk=1)
         template = InstanceTemplate.objects.get(pk=1)
         self.assertFalse(template.has_level(self.u1, "user"))
         self.assertFalse(template.has_level(self.u1, "user"))
 
 
+        field = RequestField(fieldname="Tanszek", type="Char",
+                             request_type="template", required=True)
+        field.save()
+        field = RequestField(fieldname="Szobaszam", type="Integer",
+                             request_type="template", required=False)
+        field.save()
+
         req_count = Request.objects.count()
         req_count = Request.objects.count()
         resp = c.post("/request/template/", {
         resp = c.post("/request/template/", {
             'template': 1,
             'template': 1,
             'level': "user",
             'level': "user",
-            'message': "szia",
+            'field1': "IIT",
+            'field2': 10,
         })
         })
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(req_count + 1, Request.objects.count())
         self.assertEqual(req_count + 1, Request.objects.count())
         new_request = Request.objects.latest("pk")
         new_request = Request.objects.latest("pk")
         self.assertEqual(new_request.status, "PENDING")
         self.assertEqual(new_request.status, "PENDING")
-
+        self.assertEqual(new_request.message, "Tanszek: IIT\nSzobaszam: 10\n")
         new_request.accept(self.us)
         new_request.accept(self.us)
 
 
         new_request = Request.objects.latest("pk")
         new_request = Request.objects.latest("pk")

circle/request/urls.py

@@ -24,6 +24,8 @@ from .views import (
     TemplateAccessTypeCreate, TemplateAccessTypeDetail,
     TemplateAccessTypeCreate, TemplateAccessTypeDetail,
     TemplateRequestView, LeaseRequestView, ResourceRequestView,
     TemplateRequestView, LeaseRequestView, ResourceRequestView,
     LeaseTypeDelete, TemplateAccessTypeDelete, ResizeRequestView,
     LeaseTypeDelete, TemplateAccessTypeDelete, ResizeRequestView,
+    RequestFieldFormView, RequestFieldListView, RequestFieldDetailView,
+    RequestFieldDeleteView,
 )
 )
 
 
 urlpatterns = [
 urlpatterns = [
@@ -35,6 +37,15 @@ urlpatterns = [
     url(r'^type/list/$', RequestTypeList.as_view(),
     url(r'^type/list/$', RequestTypeList.as_view(),
         name="request.views.type-list"),
         name="request.views.type-list"),
 
 
+    url(r'fields/add/$', RequestFieldFormView.as_view(),
+        name='request.views.request-field-add'),
+    url(r'fields/$', RequestFieldListView.as_view(),
+        name='request.views.field-list'),
+    url(r'fields/field/(?P<pk>[0-9]+)/$', RequestFieldDetailView.as_view(),
+        name='request.views.fields-detail'),
+    url(r'^field/delete/(?P<pk>\d+)/$', RequestFieldDeleteView.as_view(),
+        name='request.views.field-delete'),
+
     # request types
     # request types
     url(r'^type/lease/create/$', LeaseTypeCreate.as_view(),
     url(r'^type/lease/create/$', LeaseTypeCreate.as_view(),
         name="request.views.lease-type-create"),
         name="request.views.lease-type-create"),

circle/request/views.py

@@ -18,6 +18,7 @@ from __future__ import unicode_literals, absolute_import
 
 
 from django.views.generic import (
 from django.views.generic import (
     UpdateView, TemplateView, DetailView, CreateView, FormView, DeleteView,
     UpdateView, TemplateView, DetailView, CreateView, FormView, DeleteView,
+    ListView
 )
 )
 from django.contrib import messages
 from django.contrib import messages
 from django.contrib.messages.views import SuccessMessageMixin
 from django.contrib.messages.views import SuccessMessageMixin
@@ -32,7 +33,7 @@ from django_tables2 import SingleTableView
 
 
 from request.models import (
 from request.models import (
     Request, TemplateAccessType, LeaseType, TemplateAccessAction,
     Request, TemplateAccessType, LeaseType, TemplateAccessAction,
-    ExtendLeaseAction, ResourceChangeAction, DiskResizeAction
+    ExtendLeaseAction, ResourceChangeAction, DiskResizeAction, RequestField
 )
 )
 from storage.models import Disk
 from storage.models import Disk
 from vm.models import Instance
 from vm.models import Instance
@@ -42,7 +43,39 @@ from request.tables import (
 from request.forms import (
 from request.forms import (
     LeaseTypeForm, TemplateAccessTypeForm, TemplateRequestForm,
     LeaseTypeForm, TemplateAccessTypeForm, TemplateRequestForm,
     LeaseRequestForm, ResourceRequestForm, ResizeRequestForm,
     LeaseRequestForm, ResourceRequestForm, ResizeRequestForm,
+    RequestFieldModelForm
 )
 )
+from django.urls import reverse_lazy
+
+
+class RequestFieldFormView(LoginRequiredMixin, CreateView):
+    template_name = 'request/_request-field-form.html'
+    model = RequestField
+    form_class = RequestFieldModelForm
+    success_url = reverse_lazy('request.views.field-list')
+
+
+class RequestFieldListView(LoginRequiredMixin, ListView):
+    template_name = 'request/field-list.html'
+    model = RequestField
+
+    def get_context_data(self, *args, **kwargs):
+        ctx = super(RequestFieldListView, self).get_context_data(*args,
+                                                                 **kwargs)
+        ctx['add_form'] = RequestFieldModelForm()
+        ctx['add_form'].helper.form_show_labels = False
+
+        return ctx
+
+
+class RequestFieldDetailView(LoginRequiredMixin, DetailView):
+    model = RequestField
+
+
+class RequestFieldDeleteView(LoginRequiredMixin, DeleteView):
+    model = RequestField
+    template_name = "dashboard/confirm/base-delete.html"
+    success_url = reverse_lazy('request.views.field-list')
 
 
 
 
 class RequestList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
 class RequestList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
@@ -173,7 +206,7 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
 
 
     def get_form_kwargs(self):
     def get_form_kwargs(self):
         kwargs = super(TemplateRequestView, self).get_form_kwargs()
         kwargs = super(TemplateRequestView, self).get_form_kwargs()
-        kwargs['request'] = self.request
+        kwargs['type'] = 'template'
         return kwargs
         return kwargs
 
 
     def form_valid(self, form):
     def form_valid(self, form):
@@ -187,9 +220,13 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
         )
         )
         ta.save()
         ta.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.template,
             type=Request.TYPES.template,
             action=ta
             action=ta
         )
         )
@@ -236,6 +273,12 @@ class LeaseRequestView(VmRequestMixin, FormView):
     user_level = "operator"
     user_level = "operator"
     success_message = _("Request successfully sent.")
     success_message = _("Request successfully sent.")
 
 
+    def get_form_kwargs(self):
+        kwargs = super(LeaseRequestView, self).get_form_kwargs()
+        kwargs['type'] = 'resource'
+
+        return kwargs
+
     def form_valid(self, form):
     def form_valid(self, form):
         data = form.cleaned_data
         data = form.cleaned_data
         user = self.request.user
         user = self.request.user
@@ -247,9 +290,14 @@ class LeaseRequestView(VmRequestMixin, FormView):
         )
         )
         el.save()
         el.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
+
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.lease,
             type=Request.TYPES.lease,
             action=el
             action=el
         )
         )
@@ -267,6 +315,7 @@ class ResourceRequestView(VmRequestMixin, FormView):
 
 
     def get_form_kwargs(self):
     def get_form_kwargs(self):
         kwargs = super(ResourceRequestView, self).get_form_kwargs()
         kwargs = super(ResourceRequestView, self).get_form_kwargs()
+        kwargs['type'] = 'resource'
         kwargs['can_edit'] = True
         kwargs['can_edit'] = True
         kwargs['instance'] = self.get_vm()
         kwargs['instance'] = self.get_vm()
         return kwargs
         return kwargs
@@ -292,9 +341,14 @@ class ResourceRequestView(VmRequestMixin, FormView):
         )
         )
         rc.save()
         rc.save()
 
 
+        message = ''
+        for field in form.get_dynamic_fields():
+            message += "%s: %s\n" % (unicode(field.label),
+                                     unicode(data[field.name]))
+
         req = Request(
         req = Request(
             user=user,
             user=user,
-            message=data['message'],
+            message=message,
             type=Request.TYPES.resource,
             type=Request.TYPES.resource,
             action=rc
             action=rc
         )
         )

circle/storage/migrations/0003_auto_20200424_2000.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.25 on 2020-04-24 20:00
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storage', '0002_disk_bus'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='disk',
+            options={'ordering': ['name'], 'permissions': (('create_empty_disk', 'Can create an empty disk.'), ('download_disk', 'Can download a disk.'), ('resize_disk', 'Can resize a disk.'), ('import_disk', 'Can import a disk.'), ('export_disk', 'Can export a disk.')), 'verbose_name': 'disk', 'verbose_name_plural': 'disks'},
+        ),
+    ]

circle/storage/models.py

@@ -20,31 +20,30 @@
 from __future__ import unicode_literals
 from __future__ import unicode_literals
 
 
 import logging
 import logging
-from os.path import join
 import uuid
 import uuid
-import re
 
 
+import re
 from celery.contrib.abortable import AbortableAsyncResult
 from celery.contrib.abortable import AbortableAsyncResult
-from django.db.models import (Model, BooleanField, CharField, DateTimeField,
-                              ForeignKey)
+from celery.exceptions import TimeoutError
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.urlresolvers import reverse
 from django.core.urlresolvers import reverse
+from django.db.models import (Model, BooleanField, CharField, DateTimeField,
+                              ForeignKey)
 from django.utils import timezone
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from django.utils.translation import ugettext_lazy as _, ugettext_noop
 from model_utils.models import TimeStampedModel
 from model_utils.models import TimeStampedModel
+from os.path import join
 from sizefield.models import FileSizeField
 from sizefield.models import FileSizeField
 
 
-from .tasks import local_tasks, storage_tasks
-from celery.exceptions import TimeoutError
 from common.models import (
 from common.models import (
     WorkerNotFound, HumanReadableException, humanize_exception, method_cache
     WorkerNotFound, HumanReadableException, humanize_exception, method_cache
 )
 )
+from .tasks import local_tasks, storage_tasks
 
 
 logger = logging.getLogger(__name__)
 logger = logging.getLogger(__name__)
 
 
 
 
 class DataStore(Model):
 class DataStore(Model):
-
     """Collection of virtual disks.
     """Collection of virtual disks.
     """
     """
     name = CharField(max_length=100, unique=True, verbose_name=_('name'))
     name = CharField(max_length=100, unique=True, verbose_name=_('name'))
@@ -119,12 +118,15 @@ class DataStore(Model):
 
 
 
 
 class Disk(TimeStampedModel):
 class Disk(TimeStampedModel):
-
     """A virtual disk.
     """A virtual disk.
     """
     """
     TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
     TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
              ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
              ('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
     BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
     BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
+    EXPORT_FORMATS = (('qcow2', _('QEMU disk image')),
+                      ('vmdk', _('VMware disk image')),
+                      ('vdi', _('VirtualBox disk image')),
+                      ('vpc', _('HyperV disk image')))
     name = CharField(blank=True, max_length=100, verbose_name=_("name"))
     name = CharField(blank=True, max_length=100, verbose_name=_("name"))
     filename = CharField(max_length=256, unique=True,
     filename = CharField(max_length=256, unique=True,
                          verbose_name=_("filename"))
                          verbose_name=_("filename"))
@@ -149,7 +151,9 @@ class Disk(TimeStampedModel):
         permissions = (
         permissions = (
             ('create_empty_disk', _('Can create an empty disk.')),
             ('create_empty_disk', _('Can create an empty disk.')),
             ('download_disk', _('Can download a disk.')),
             ('download_disk', _('Can download a disk.')),
-            ('resize_disk', _('Can resize a disk.'))
+            ('resize_disk', _('Can resize a disk.')),
+            ('import_disk', _('Can import a disk.')),
+            ('export_disk', _('Can export a disk.'))
         )
         )
 
 
     class DiskError(HumanReadableException):
     class DiskError(HumanReadableException):
@@ -427,6 +431,19 @@ class Disk(TimeStampedModel):
         return disk
         return disk
 
 
     @classmethod
     @classmethod
+    def _run_abortable_task(cls, remote, task):
+        while True:
+            try:
+                result = remote.get(timeout=5)
+                break
+            except TimeoutError as e:
+                if task is not None and task.is_aborted():
+                    AbortableAsyncResult(remote.id).abort()
+                    raise humanize_exception(ugettext_noop(
+                        "Operation aborted by user."), e)
+        return result
+
+    @classmethod
     def download(cls, url, task, user=None, **params):
     def download(cls, url, task, user=None, **params):
         """Create disk object and download data from url synchronusly.
         """Create disk object and download data from url synchronusly.
 
 
@@ -451,15 +468,7 @@ class Disk(TimeStampedModel):
             kwargs={'url': url, 'parent_id': task.request.id,
             kwargs={'url': url, 'parent_id': task.request.id,
                     'disk': disk.get_disk_desc()},
                     'disk': disk.get_disk_desc()},
             queue=queue_name)
             queue=queue_name)
-        while True:
-            try:
-                result = remote.get(timeout=5)
-                break
-            except TimeoutError as e:
-                if task is not None and task.is_aborted():
-                    AbortableAsyncResult(remote.id).abort()
-                    raise humanize_exception(ugettext_noop(
-                        "Operation aborted by user."), e)
+        result = cls._run_abortable_task(remote, task)
         disk.size = result['size']
         disk.size = result['size']
         disk.type = result['type']
         disk.type = result['type']
         disk.checksum = result.get('checksum', None)
         disk.checksum = result.get('checksum', None)
@@ -467,6 +476,40 @@ class Disk(TimeStampedModel):
         disk.save()
         disk.save()
         return disk
         return disk
 
 
+    @classmethod
+    def import_disk(cls, user, name, download_link, task):
+        params = {'name': name,
+                  'type': 'qcow2-norm'}
+        disk = cls.__create(user=user, params=params)
+        queue_name = disk.get_remote_queue_name('storage', priority='slow')
+        remote = storage_tasks.import_disk.apply_async(
+            kwargs={
+                "disk_desc": disk.get_disk_desc(),
+                "url": download_link,
+                "task": task.request.id
+            },
+            queue=queue_name
+        )
+        result = cls._run_abortable_task(remote, task)
+        disk.size = result["size"]
+        disk.checksum = result["checksum"]
+        disk.is_ready = True
+        disk.save()
+        return disk
+
+    def export(self, exported_name, disk_format, upload_link, task):
+        queue_name = self.get_remote_queue_name('storage', priority='slow')
+        remote = storage_tasks.export_disk.apply_async(
+            kwargs={
+                "disk_desc": self.get_disk_desc(),
+                "disk_format": disk_format,
+                "exported_name": exported_name,
+                "upload_link": upload_link,
+                "task": task.request.id
+            },
+            queue=queue_name)
+        self._run_abortable_task(remote, task)
+
     def destroy(self, user=None, task_uuid=None):
     def destroy(self, user=None, task_uuid=None):
         if self.destroyed:
         if self.destroyed:
             return False
             return False
@@ -549,4 +592,8 @@ class Disk(TimeStampedModel):
 
 
     @property
     @property
     def is_resizable(self):
     def is_resizable(self):
-        return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap', )
+        return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap',)
+
+    @property
+    def is_exportable(self):
+        return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')

circle/storage/tasks/storage_tasks.py

@@ -38,6 +38,16 @@ def download(disk_desc, url):
     pass
     pass
 
 
 
 
+@celery.task(name='storagedriver.import_disk')
+def import_disk(disk_desc, url):
+    pass
+
+
+@celery.task(name='storagedriver.export_disk')
+def export_disk(disk_desc, format):
+    pass
+
+
 @celery.task(name='storagedriver.delete')
 @celery.task(name='storagedriver.delete')
 def delete(path):
 def delete(path):
     pass
     pass

circle/vm/management/commands/mass_create_vms.py

+# Copyright 2014 Budapest University of Technology and Economics (BME IK)
+#
+# This file is part of CIRCLE Cloud.
+#
+# CIRCLE is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along
+# with CIRCLE.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.management.base import BaseCommand
+from vm.models import Instance, InstanceTemplate
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('-t', '--template', type=int, required=True)
+        parser.add_argument('-u', '--users', required=True)
+        parser.add_argument('-a', '--admin')
+        parser.add_argument('-o', '--operator')
+
+    def handle(self, *args, **options):
+        template = InstanceTemplate.objects.get(id=options['template'])
+
+        with open(options['users']) as f:
+            users = f.read().splitlines()
+
+        missing_users = Instance.mass_create_for_users(
+            template, users, options['admin'],  options['operator']
+        )
+
+        if len(missing_users) > 0:
+            self.stdout.write('These users do not exist:')
+            for user in missing_users:
+                self.stdout.write(user)

circle/vm/migrations/0003_auto_20171213_2018.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2017-12-13 20:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vm', '0002_interface_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='node',
+            name='cpu_weight',
+            field=models.FloatField(default=1.0, help_text='Indicates the relative CPU power of this node.', verbose_name='CPU Weight'),
+        ),
+        migrations.AddField(
+            model_name='node',
+            name='ram_weight',
+            field=models.FloatField(default=1.0, help_text='Indicates the relative RAM quantity of this node.', verbose_name='RAM Weight'),
+        ),
+        migrations.AddField(
+            model_name='node',
+            name='time_stamp',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, help_text='A timestamp for the node, used by the scheduler.', verbose_name='Last Scheduled Time Stamp'),
+            preserve_default=False,
+        ),
+    ]

circle/vm/models/instance.py

@@ -200,6 +200,9 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
     def get_running_instances(self):
     def get_running_instances(self):
         return Instance.active.filter(template=self, status="RUNNING")
         return Instance.active.filter(template=self, status="RUNNING")
 
 
+    def get_user_instances(self, user):
+        return Instance.active.filter(template=self, owner=user)
+
     @property
     @property
     def metric_prefix(self):
     def metric_prefix(self):
         return 'template.%d' % self.pk
         return 'template.%d' % self.pk
@@ -439,6 +442,30 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
         return [cls.create(cps, disks, networks, req_traits, tags)
         return [cls.create(cps, disks, networks, req_traits, tags)
                 for cps in customized_params]
                 for cps in customized_params]
 
 
+    @classmethod
+    def mass_create_for_users(cls, template, users, admin=None, operator=None, **kwargs):
+        """
+        Create and deploy an instance of a template for each user
+        in a list of users. Returns the user IDs of missing users.
+        """
+        user_instances = []
+        missing_users = []
+        for user_id in users:
+            try:
+                user_instances.append(User.objects.get(profile__org_id=user_id))
+            except User.DoesNotExist:
+                missing_users.append(user_id)
+
+        for user in user_instances:
+            instance = cls.create_from_template(template, user, **kwargs)
+            if admin:
+                instance.set_level(User.objects.get(username=admin), 'owner')
+            if operator:
+                instance.set_level(User.objects.get(username=operator), 'operator')
+            instance.deploy(user=user)
+
+        return missing_users
+
     def clean(self, *args, **kwargs):
     def clean(self, *args, **kwargs):
         self.time_of_suspend, self.time_of_delete = self.get_renew_times()
         self.time_of_suspend, self.time_of_delete = self.get_renew_times()
         super(Instance, self).clean(*args, **kwargs)
         super(Instance, self).clean(*args, **kwargs)
@@ -576,7 +603,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
             host = self.get_connect_host(use_ipv6=use_ipv6)
             host = self.get_connect_host(use_ipv6=use_ipv6)
             proto = self.access_method
             proto = self.access_method
             if proto == 'rdp':
             if proto == 'rdp':
-                return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s' % {
+                return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s -f' % {
                     'port': port, 'proto': proto, 'pw': self.pw,
                     'port': port, 'proto': proto, 'pw': self.pw,
                     'host': host}
                     'host': host}
             elif proto == 'ssh':
             elif proto == 'ssh':
@@ -865,6 +892,53 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
     def metric_prefix(self):
     def metric_prefix(self):
         return 'vm.%s' % self.vm_name
         return 'vm.%s' % self.vm_name
 
 
+
+    class MonitorUnavailableException(Exception):
+        """Exception for monitor_info()
+
+            Indicates the unavailability of the monitoring server.
+        """
+        pass
+
+    def monitor_info(self):
+        metrics = ('cpu.percent', 'memory.usage')
+        prefix = self.metric_prefix
+        params = [('target', '%s.%s' % (prefix, metric))
+                  for metric in metrics]
+        params.append(('from', '-5min'))
+        params.append(('format', 'json'))
+
+        try:
+            logger.info('%s %s', settings.GRAPHITE_URL, params)
+            response = requests.get(settings.GRAPHITE_URL, params=params)
+
+            retval = {}
+            for target in response.json():
+                # Example:
+                # {"target": "circle.vm.{name}.cpu.usage",
+                #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
+                try:
+                    metric = target['target']
+                    if metric.startswith(prefix):
+                        metric = metric[len(prefix):]
+                    else:
+                        continue
+                    value = target['datapoints'][-2][0]
+                    retval[metric] = float(value)
+                except (KeyError, IndexError, ValueError):
+                    continue
+
+            return retval
+        except Exception:
+            logger.exception('Monitor server unavailable: ')
+            raise Instance.MonitorUnavailableException()
+
+    def cpu_usage(self):
+        return self.monitor_info().get('cpu.percent')
+
+    def ram_usage(self):
+        return self.monitor_info().get('memory.usage')
+
     @contextmanager
     @contextmanager
     def activity(self, code_suffix, readable_name, on_abort=None,
     def activity(self, code_suffix, readable_name, on_abort=None,
                  on_commit=None, task_uuid=None, user=None,
                  on_commit=None, task_uuid=None, user=None,

circle/vm/models/node.py

@@ -30,12 +30,12 @@ from time import time, sleep
 from django.conf import settings
 from django.conf import settings
 from django.db.models import (
 from django.db.models import (
     CharField, IntegerField, ForeignKey, BooleanField, ManyToManyField,
     CharField, IntegerField, ForeignKey, BooleanField, ManyToManyField,
-    FloatField, permalink, Sum
+    FloatField, DateTimeField, permalink, Sum
 )
 )
 from django.utils import timezone
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext_lazy as _
 
 
-from celery.exceptions import TimeoutError
+from celery.exceptions import TimeoutError, TaskRevokedError
 from model_utils.models import TimeStampedModel
 from model_utils.models import TimeStampedModel
 from taggit.managers import TaggableManager
 from taggit.managers import TaggableManager
 
 
@@ -128,11 +128,15 @@ class Node(OperatedMixin, TimeStampedModel):
     enabled = BooleanField(verbose_name=_('enabled'), default=False,
     enabled = BooleanField(verbose_name=_('enabled'), default=False,
                            help_text=_('Indicates whether the node can '
                            help_text=_('Indicates whether the node can '
                                        'be used for hosting.'))
                                        'be used for hosting.'))
-    schedule_enabled = BooleanField(verbose_name=_('schedule enabled'),
-                                    default=False, help_text=_(
-                                        'Indicates whether a vm can be '
-                                        'automatically scheduled to this '
-                                        'node.'))
+    schedule_enabled = BooleanField(
+        verbose_name=_('schedule enabled'),
+        default=False,
+        help_text=_(
+            'Indicates whether a vm can be '
+            'automatically scheduled to this '
+            'node.'
+        )
+    )
     traits = ManyToManyField(Trait, blank=True,
     traits = ManyToManyField(Trait, blank=True,
                              help_text=_("Declared traits."),
                              help_text=_("Declared traits."),
                              verbose_name=_('traits'))
                              verbose_name=_('traits'))
@@ -140,6 +144,21 @@ class Node(OperatedMixin, TimeStampedModel):
     overcommit = FloatField(default=1.0, verbose_name=_("overcommit ratio"),
     overcommit = FloatField(default=1.0, verbose_name=_("overcommit ratio"),
                             help_text=_("The ratio of total memory with "
                             help_text=_("The ratio of total memory with "
                                         "to without overcommit."))
                                         "to without overcommit."))
+    ram_weight = FloatField(
+        default=1.0,
+        help_text=_("Indicates the relative RAM quantity of this node."),
+        verbose_name=_("RAM Weight")
+    )
+    cpu_weight = FloatField(
+        default=1.0,
+        help_text=_("Indicates the relative CPU power of this node."),
+        verbose_name=_("CPU Weight")
+    )
+    time_stamp = DateTimeField(
+        auto_now_add=True,
+        help_text=_("A timestamp for the node, used by the scheduler."),
+        verbose_name=_("Last Scheduled Time Stamp")
+    )
 
 
     class Meta:
     class Meta:
         app_label = 'vm'
         app_label = 'vm'
@@ -162,7 +181,7 @@ class Node(OperatedMixin, TimeStampedModel):
             self.get_remote_queue_name("vm", "fast")
             self.get_remote_queue_name("vm", "fast")
             self.get_remote_queue_name("vm", "slow")
             self.get_remote_queue_name("vm", "slow")
             self.get_remote_queue_name("net", "fast")
             self.get_remote_queue_name("net", "fast")
-        except:
+        except Exception:
             return False
             return False
         else:
         else:
             return True
             return True
@@ -315,7 +334,7 @@ class Node(OperatedMixin, TimeStampedModel):
                 queue=self.get_remote_queue_name('vm', priority),
                 queue=self.get_remote_queue_name('vm', priority),
                 expires=timeout + 60)
                 expires=timeout + 60)
             return r.get(timeout=timeout)
             return r.get(timeout=timeout)
-        except (TimeoutError, WorkerNotFound):
+        except (TimeoutError, WorkerNotFound, TaskRevokedError):
             if raise_:
             if raise_:
                 raise
                 raise
             else:
             else:
@@ -341,19 +360,25 @@ class Node(OperatedMixin, TimeStampedModel):
                 # Example:
                 # Example:
                 # {"target": "circle.szianode.cpu.usage",
                 # {"target": "circle.szianode.cpu.usage",
                 #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
                 #  "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
+                logger.info('MONITOR_TARGET: %s', target)
                 try:
                 try:
                     metric = target['target']
                     metric = target['target']
                     if metric.startswith(prefix):
                     if metric.startswith(prefix):
                         metric = metric[len(prefix):]
                         metric = metric[len(prefix):]
                     else:
                     else:
+            	        logger.info('MONITOR_MET: %s %s', target, metric)
                         continue
                         continue
-                    value = target['datapoints'][-2][0]
+                    value = target['datapoints'][-1][0]
+                    if value is None: 
+                        value = target['datapoints'][-2][0]
                     retval[metric] = float(value)
                     retval[metric] = float(value)
+                    logger.info('MONITOR_RETVAL: %s %s, %s', target['target'], metric, retval[metric])
                 except (KeyError, IndexError, ValueError, TypeError):
                 except (KeyError, IndexError, ValueError, TypeError):
+                    logger.info('MONITOR_ERR: %s %s', metric, value)
                     continue
                     continue
 
 
             return retval
             return retval
-        except:
+        except Exception:
             logger.exception('Unhandled exception: ')
             logger.exception('Unhandled exception: ')
             return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
             return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
                                      priority="fast")
                                      priority="fast")
@@ -363,15 +388,27 @@ class Node(OperatedMixin, TimeStampedModel):
     def driver_version(self):
     def driver_version(self):
         return self.info.get('driver_version')
         return self.info.get('driver_version')
 
 
+    def get_monitor_info(self, metric):
+        # return with the metric value if the monitor info not none
+        # or return 0 if its None or the metric unreachable
+        if self.monitor_info is None:
+            logger.warning('Monitor info is None')
+            return 0
+        elif self.monitor_info.get(metric) is None:
+            logger.warning('Unreachable monitor info of: ' + metric)
+            return 0
+        else:
+            return self.monitor_info.get(metric)
+
     @property
     @property
     @node_available
     @node_available
     def cpu_usage(self):
     def cpu_usage(self):
-        return self.monitor_info.get('cpu.percent') / 100
+        return self.get_monitor_info('cpu.percent') / 100
 
 
     @property
     @property
     @node_available
     @node_available
     def ram_usage(self):
     def ram_usage(self):
-        return self.monitor_info.get('memory.usage') / 100
+        return self.get_monitor_info('memory.usage') / 100
 
 
     @property
     @property
     @node_available
     @node_available
@@ -404,7 +441,7 @@ class Node(OperatedMixin, TimeStampedModel):
         vm_state_changed hook.
         vm_state_changed hook.
         """
         """
         domains = {}
         domains = {}
-        domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5,
+        domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=10,
                                         priority="fast")
                                         priority="fast")
         if domain_list is None:
         if domain_list is None:
             logger.info("Monitoring failed at: %s", self.name)
             logger.info("Monitoring failed at: %s", self.name)
@@ -413,7 +450,7 @@ class Node(OperatedMixin, TimeStampedModel):
             # [{'name': 'cloud-1234', 'state': 'RUNNING', ...}, ...]
             # [{'name': 'cloud-1234', 'state': 'RUNNING', ...}, ...]
             try:
             try:
                 id = int(i['name'].split('-')[1])
                 id = int(i['name'].split('-')[1])
-            except:
+            except Exception:
                 pass  # name format doesn't match
                 pass  # name format doesn't match
             else:
             else:
                 domains[id] = i['state']
                 domains[id] = i['state']

circle/vm/tasks/local_periodic_tasks.py

@@ -17,7 +17,9 @@
 
 
 import logging
 import logging
 from django.utils import timezone
 from django.utils import timezone
+from datetime import timedelta
 from django.utils.translation import ugettext_noop
 from django.utils.translation import ugettext_noop
+from django.conf import settings
 
 
 from manager.mancelery import celery
 from manager.mancelery import celery
 from vm.models import Node, Instance
 from vm.models import Node, Instance
@@ -33,7 +35,7 @@ def update_domain_states():
 
 
 
 
 @celery.task(ignore_result=True)
 @celery.task(ignore_result=True)
-def garbage_collector(timeout=15):
+def garbage_collector(offset=timezone.timedelta(seconds=20)):
     """Garbage collector for instances.
     """Garbage collector for instances.
 
 
     Suspends and destroys expired instances.
     Suspends and destroys expired instances.
@@ -64,7 +66,7 @@ def garbage_collector(timeout=15):
             work_package -= 1
             work_package -= 1
             logger.info("Expired instance %d suspended." % i.pk)
             logger.info("Expired instance %d suspended." % i.pk)
             try:
             try:
-            	i.sleep.async(system=True)
+                i.sleep.async(system=True)
                 i.owner.profile.notify(
                 i.owner.profile.notify(
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop(
                     ugettext_noop(
@@ -72,8 +74,8 @@ def garbage_collector(timeout=15):
                         'has been suspended due to expiration. '
                         'has been suspended due to expiration. '
                         'You can resume or destroy it.'),
                         'You can resume or destroy it.'),
                     instance=i.name, url=i.get_absolute_url())
                     instance=i.name, url=i.get_absolute_url())
- 	    except ActivityInProgressError:
-            	logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
+            except ActivityInProgressError:
+                logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
             except Exception as e:
             except Exception as e:
                 logger.info('Could not notify owner of instance %d .%s',
                 logger.info('Could not notify owner of instance %d .%s',
                              i.pk, unicode(e))
                              i.pk, unicode(e))
@@ -81,4 +83,44 @@ def garbage_collector(timeout=15):
             logger.debug("Instance %d expires soon." % i.pk)
             logger.debug("Instance %d expires soon." % i.pk)
             i.notify_owners_about_expiration()
             i.notify_owners_about_expiration()
         else:
         else:
-            logger.debug("Instance %d didn't expire." % i.pk)
+            logger.debug("Instance %d didn't expire. bw:%d", i.pk, bw)
+
+
+@celery.task(ignore_result=True)
+def auto_migrate():
+    """Auto migration task for runtime scaling
+    """
+    time_limit = settings.AUTO_MIGRATION_TIME_LIMIT_IN_HOURS
+    available_time = timedelta(hours=int(time_limit))
+    deadline = timezone.now() + available_time
+    while timezone.now() < deadline:
+        migrate_one()
+
+
+def migrate_one():
+    """Migrate a VM syncronously.
+
+        The target node chosen by the scheduler.
+    """
+    nodes = [n for n in Node.objects.filter(enabled=True) if n.online]
+    node_max_cpu = max(nodes, key=lambda x: x.cpu_usage / x.cpu_weight)
+    node_max_ram = max(nodes, key=lambda x: x.ram_usage / x.ram_weight)
+    if node_max_cpu.cpu_usage > node_max_ram.ram_usage:
+        try:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=lambda x: x.cpu_usage())
+            instance_to_migrate.migrate(system=True)
+        except Instance.MonitorUnavailableException:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=(lambda x: x.get_vm_desc()["vcpu"] *
+                                           x.get_vm_desc()["cpu_share"]))
+            instance_to_migrate.migrate(system=True)
+    else:
+        try:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_ram.pk),
+                                      key=lambda x: x.ram_usage())
+            instance_to_migrate.migrate(system=True)
+        except Instance.MonitorUnavailableException:
+            instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
+                                      key=lambda x: x.get_vm_desc()["memory"])
+            instance_to_migrate.migrate(system=True)

requirements/base.txt

-cryptography==2.0
+cryptography==2.7
 amqp==1.4.7
 amqp==1.4.7
 anyjson==0.3.3
 anyjson==0.3.3
 arrow==0.7.0
 arrow==0.7.0
 billiard==3.3.0.20
 billiard==3.3.0.20
 bpython==0.14.1
 bpython==0.14.1
 celery==3.1.18
 celery==3.1.18
-Django==1.11.6
+Django==1.11.25
 django-appconf==1.0.2
 django-appconf==1.0.2
 django-autocomplete-light==3.2.9
 django-autocomplete-light==3.2.9
 django-braces==1.11.0
 django-braces==1.11.0
@@ -16,7 +16,7 @@ django-sizefield==0.9.1
 django-statici18n==1.4.0
 django-statici18n==1.4.0
 django-tables2==1.10.0
 django-tables2==1.10.0
 django-taggit==0.22.1
 django-taggit==0.22.1
-djangosaml2==0.16.10
+djangosaml2==0.17.1
 git+https://git.ik.bme.hu/circle/django-sshkey.git
 git+https://git.ik.bme.hu/circle/django-sshkey.git
 docutils==0.12
 docutils==0.12
 Jinja2==2.7.3
 Jinja2==2.7.3
@@ -26,9 +26,9 @@ logutils==0.3.3
 MarkupSafe==0.23
 MarkupSafe==0.23
 netaddr==0.7.14
 netaddr==0.7.14
 pip-tools==0.3.6
 pip-tools==0.3.6
-psycopg2==2.6
+psycopg2==2.8.3
 Pygments==2.0.2
 Pygments==2.0.2
-pylibmc==1.4.3
+pylibmc==1.6.0
 python-dateutil==2.4.2
 python-dateutil==2.4.2
 pyinotify==0.9.5
 pyinotify==0.9.5
 pyotp==2.1.1
 pyotp==2.1.1

requirements/production.txt

 # Pro-tip: Try not to put anything here. There should be no dependency in
 # Pro-tip: Try not to put anything here. There should be no dependency in
 # production that isn't in development.
 # production that isn't in development.
 -r base.txt
 -r base.txt
-uWSGI==2.0.13.1
+uWSGI==2.0.18