Skip to content

CIRCLE / cloud

Go to a project
Commit 1ab77c1f by Szeberényi Imre
Options

Merge branch 'master' into 'smallville_fix' 

# Conflicts:
#   circle/vm/tasks/local_periodic_tasks.py
parents fa743214 76a2a4a6
Pipeline #1391 failed with stage
in 0 seconds
Showing with 1271 additions and 242 deletions
.gitignore
......@@ -8,6 +8,8 @@
*.swp
*.swo
*~
.vscode
.idea
# Sphinx docs:
build
......
circle/circle/settings/base.py
......@@ -495,6 +495,7 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
},
'required_attributes': required_attrs,
'optional_attributes': optional_attrs,
'want_response_signed': False,
},
},
'metadata': {'local': [remote_metadata], },
......@@ -576,7 +577,7 @@ SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'random')
SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'advanced')
# Url to download the client: (e.g. http://circlecloud.org/client/download/)
CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
......@@ -590,3 +591,12 @@ REQUEST_HOOK_URL = get_env_variable("REQUEST_HOOK_URL", "")
SSHKEY_EMAIL_ADD_KEY = False
TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
# Default value is every day at midnight
AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *")
AUTO_MIGRATION_TIME_LIMIT_IN_HOURS = (
get_env_variable("AUTO_MIGRATION_TIME_LIMIT_IN_HOURS", "2"))
# Maximum time difference until the monitor's values get valid
SCHEDULER_TIME_SENSITIVITY_IN_SECONDS = (
get_env_variable("SCHEDULER_TIME_SENSITIVITY_IN_SECONDS", "60"))
circle/dashboard/fixtures/node.json
......@@ -65,7 +65,10 @@
"modified": "2014-02-19T21:11:34.671Z",
"priority": 1,
"traits": [],
"host": 1
"host": 1,
"ram_weight": 1.0,
"cpu_weight": 1.0,
"time_stamp": "2017-12-13T21:08:08.819Z"
}
}
]
circle/dashboard/forms.py
......@@ -17,56 +17,51 @@
from __future__ import absolute_import
from datetime import timedelta
from urlparse import urlparse
import os
import pyotp
from django.forms import ModelForm
from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm,
PasswordChangeForm,
)
from django.contrib.auth.models import User, Group
from django.core.validators import URLValidator
from django.core.exceptions import PermissionDenied, ValidationError
from dal import autocomplete
from crispy_forms.bootstrap import FormActions
from crispy_forms.helper import FormHelper
from crispy_forms.layout import (
Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
)
from crispy_forms.utils import render_field
from crispy_forms.bootstrap import FormActions
from dal import autocomplete
from datetime import timedelta
from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm,
PasswordChangeForm,
)
from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User, Group
from django.core.exceptions import PermissionDenied, ValidationError
from django.core.urlresolvers import reverse_lazy
from django.core.validators import URLValidator
from django.forms import ModelForm
from django.forms.widgets import TextInput, HiddenInput
from django.template.loader import render_to_string
from django.utils.html import escape, format_html
from django.utils.safestring import mark_safe
from django.utils.translation import string_concat
from django.utils.translation import ugettext_lazy as _
from django_sshkey.models import UserKey
from sizefield.widgets import FileSizeWidget
from django.core.urlresolvers import reverse_lazy
from django_sshkey.models import UserKey
from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
from dashboard.models import ConnectCommand, create_profile
from dashboard.store_api import Store
from firewall.models import Vlan, Host
from storage.models import DataStore, Disk
from vm.models import (
InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
)
from storage.models import DataStore, Disk
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.auth.models import Permission
from .models import Profile, GroupProfile, Message
from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
from django.utils.translation import string_concat
from .validators import domain_validator
from dashboard.models import ConnectCommand, create_profile
LANGUAGES_WITH_CODE = ((l[0], string_concat(l[1], " (", l[0], ")"))
for l in LANGUAGES)
......@@ -189,7 +184,7 @@ class VmCustomizeForm(forms.Form):
self.initial['ram_size'] = self.template.ram_size
else:
self.allowed_fields = ("name", "template", "customized", )
self.allowed_fields = ("name", "template", "customized",)
# initial name and template pk
self.initial['name'] = self.template.name
......@@ -214,7 +209,6 @@ class VmCustomizeForm(forms.Form):
class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
description = forms.CharField(label=_("Description"), required=False,
widget=forms.Textarea(attrs={'rows': 3}))
......@@ -258,7 +252,7 @@ class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
class Meta:
model = Group
fields = ('name', )
fields = ('name',)
class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
......@@ -276,6 +270,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
label=_('Directory identifier'))
if not new_groups:
self.fields['org_id'].widget = HiddenInput()
self.fields['disk_quota'].widget = HiddenInput()
self.fields['description'].widget = forms.Textarea(attrs={'rows': 3})
@property
......@@ -293,7 +288,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
class Meta:
model = GroupProfile
fields = ('description', 'org_id')
fields = ('description', 'org_id', 'disk_quota')
class HostForm(NoFormTagMixin, forms.ModelForm):
......@@ -513,7 +508,7 @@ class TemplateForm(forms.ModelForm):
self.allowed_fields += tuple(set(self.fields.keys()) -
set(['raw_data']))
if self.user.is_superuser:
self.allowed_fields += ('raw_data', )
self.allowed_fields += ('raw_data',)
for name, field in self.fields.items():
if name not in self.allowed_fields:
field.widget.attrs['disabled'] = 'disabled'
......@@ -602,7 +597,7 @@ class TemplateForm(forms.ModelForm):
class Meta:
model = InstanceTemplate
exclude = ('state', 'disks', )
exclude = ('state', 'disks',)
widgets = {
'system': forms.TextInput,
'max_ram_size': forms.HiddenInput,
......@@ -745,7 +740,6 @@ class LeaseForm(forms.ModelForm):
class VmRenewForm(OperationForm):
force = forms.BooleanField(required=False, label=_(
"Set expiration times even if they are shorter than "
"the current value."))
......@@ -785,7 +779,6 @@ class VmMigrateForm(forms.Form):
class VmStateChangeForm(OperationForm):
interrupt = forms.BooleanField(required=False, label=_(
"Forcibly interrupt all running activities."),
help_text=_("Set all activities to finished state, "
......@@ -830,6 +823,41 @@ class VmCreateDiskForm(OperationForm):
return size_in_bytes
class VmDiskExportForm(OperationForm):
exported_name = forms.CharField(max_length=100, label=_('Filename'))
disk_format = forms.ChoiceField(
choices=Disk.EXPORT_FORMATS,
label=_('Format'))
def __init__(self, *args, **kwargs):
choices = kwargs.pop('choices')
self.disk = kwargs.pop('default')
super(VmDiskExportForm, self).__init__(*args, **kwargs)
self.fields['disk'] = forms.ModelChoiceField(
queryset=choices, initial=self.disk, required=True,
empty_label=None, label=_('Disk'))
if self.disk:
self.fields['disk'].widget = HiddenInput()
@property
def helper(self):
helper = super(VmDiskExportForm, self).helper
if self.disk:
helper.layout = Layout(
AnyTag(
"div",
HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
css_class="form-group",
),
Field('disk'),
Field('exported_name'),
Field('disk_format')
)
return helper
class VmDiskResizeForm(OperationForm):
size = forms.CharField(
widget=FileSizeWidget, initial=(10 << 30), label=_('Size'),
......@@ -899,6 +927,20 @@ class VmDiskRemoveForm(OperationForm):
return helper
class VmImportDiskForm(OperationForm):
def __init__(self, *args, **kwargs):
self.user = kwargs.pop('user')
super(VmImportDiskForm, self).__init__(*args, **kwargs)
disk_paths = Store(self.user).get_disk_images()
disk_filenames = [os.path.basename(item) for item in disk_paths]
self.choices = zip(disk_paths, disk_filenames)
self.fields['name'] = forms.CharField(max_length=100, label=_('Name'))
self.fields['disk_path'] = forms.ChoiceField(label=_('Disk image'),
choices=self.choices)
class VmDownloadDiskForm(OperationForm):
name = forms.CharField(max_length=100, label=_("Name"), required=False)
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
......@@ -1138,7 +1180,6 @@ class CircleSetPasswordForm(SetPasswordForm):
class LinkButton(BaseInput):
"""
Used to create a link button descriptor for the {% crispy %} template tag::
......@@ -1224,7 +1265,7 @@ class MyProfileForm(forms.ModelForm):
class Meta:
fields = ('preferred_language', 'email_notifications',
'desktop_notifications', 'use_gravatar', )
'desktop_notifications', 'use_gravatar',)
model = Profile
@property
......@@ -1239,9 +1280,8 @@ class MyProfileForm(forms.ModelForm):
class UnsubscribeForm(forms.ModelForm):
class Meta:
fields = ('email_notifications', )
fields = ('email_notifications',)
model = Profile
@property
......@@ -1299,6 +1339,9 @@ class UserEditForm(forms.ModelForm):
instance_limit = forms.IntegerField(
label=_('Instance limit'),
min_value=0, widget=NumberInput)
template_instance_limit = forms.IntegerField(
label=_('Template instance limit'),
min_value=0, widget=NumberInput)
two_factor_secret = forms.CharField(
label=_('Two-factor authentication secret'),
help_text=_("Remove the secret key to disable two-factor "
......@@ -1308,18 +1351,23 @@ class UserEditForm(forms.ModelForm):
super(UserEditForm, self).__init__(*args, **kwargs)
self.fields["instance_limit"].initial = (
self.instance.profile.instance_limit)
self.fields["template_instance_limit"].initial = (
self.instance.profile.template_instance_limit)
self.fields["two_factor_secret"].initial = (
self.instance.profile.two_factor_secret)
class Meta:
model = User
fields = ('email', 'first_name', 'last_name', 'instance_limit',
'is_active', "two_factor_secret", )
fields = ('email', 'first_name', 'last_name',
'instance_limit', 'template_instance_limit',
'is_active', 'two_factor_secret',)
def save(self, commit=True):
user = super(UserEditForm, self).save()
user.profile.instance_limit = (
self.cleaned_data['instance_limit'] or None)
user.profile.template_instance_limit = (
self.cleaned_data['template_instance_limit'] or None)
user.profile.two_factor_secret = (
self.cleaned_data['two_factor_secret'] or None)
user.profile.save()
......@@ -1405,10 +1453,9 @@ class ConnectCommandForm(forms.ModelForm):
class TraitsForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('req_traits', )
fields = ('req_traits',)
@property
def helper(self):
......@@ -1428,7 +1475,7 @@ class RawDataForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('raw_data', )
fields = ('raw_data',)
@property
def helper(self):
......@@ -1490,7 +1537,7 @@ class GroupPermissionForm(forms.ModelForm):
class Meta:
model = Group
fields = ('permissions', )
fields = ('permissions',)
@property
def helper(self):
......@@ -1538,7 +1585,7 @@ class VmResourcesForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('num_cores', 'priority', 'ram_size', )
fields = ('num_cores', 'priority', 'ram_size',)
class VmRenameForm(forms.Form):
......@@ -1629,7 +1676,7 @@ class DataStoreForm(ModelForm):
class Meta:
model = DataStore
fields = ("name", "path", "hostname", )
fields = ("name", "path", "hostname",)
class DiskForm(ModelForm):
......@@ -1647,7 +1694,7 @@ class DiskForm(ModelForm):
class Meta:
model = Disk
fields = ("name", "filename", "datastore", "type", "bus", "size",
"base", "dev_num", "destroyed", "is_ready", )
"base", "dev_num", "destroyed", "is_ready",)
class MessageForm(ModelForm):
......@@ -1697,3 +1744,11 @@ class TwoFactorConfirmationForm(forms.Form):
totp = pyotp.TOTP(self.user.profile.two_factor_secret)
if not totp.verify(self.cleaned_data.get('confirmation_code')):
raise ValidationError(_("Invalid confirmation code."))
class AutoMigrationForm(forms.Form):
minute = forms.CharField()
hour = forms.CharField()
day_of_month = forms.CharField()
month_of_year = forms.CharField()
day_of_week = forms.CharField()
circle/dashboard/migrations/0007_groupprofile_disk_quota.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
import sizefield.models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0006_auto_20170707_1909'),
]
operations = [
migrations.AddField(
model_name='groupprofile',
name='disk_quota',
field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
),
]
circle/dashboard/migrations/0008_profile_template_instance_limit.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-11-06 13:33
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0007_groupprofile_disk_quota'),
]
operations = [
migrations.AddField(
model_name='profile',
name='template_instance_limit',
field=models.IntegerField(default=1),
),
]
circle/dashboard/models.py
......@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.models.instance import ACCESS_METHODS
from .store_api import Store, NoStoreException, NotOkException, Timeout
from .store_api import Store, NoStoreException, NotOkException
from .validators import connect_command_template_validator
logger = getLogger(__name__)
......@@ -162,7 +162,7 @@ class ConnectCommand(Model):
validators=[connect_command_template_validator])
class Meta:
ordering = ('id', )
ordering = ('id',)
def __unicode__(self):
return self.template
......@@ -178,6 +178,7 @@ class Profile(Model):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the person, e.g. a student number.'))
instance_limit = IntegerField(default=5)
template_instance_limit = IntegerField(default=1)
use_gravatar = BooleanField(
verbose_name=_("Use Gravatar"), default=True,
help_text=_("Whether to use email address as Gravatar profile image"))
......@@ -263,7 +264,7 @@ class Profile(Model):
super(Profile, self).save(*args, **kwargs)
class Meta:
ordering = ('id', )
ordering = ('id',)
permissions = (
('use_autocomplete', _('Can use autocomplete.')),
)
......@@ -275,7 +276,7 @@ class FutureMember(Model):
group = ForeignKey(Group)
class Meta:
ordering = ('id', )
ordering = ('id',)
unique_together = ('org_id', 'group')
def __unicode__(self):
......@@ -293,9 +294,13 @@ class GroupProfile(AclBase):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the group at the organization.'))
description = TextField(blank=True)
disk_quota = FileSizeField(
verbose_name=_('disk quota'),
default=2048 * 1024 * 1024,
help_text=_('Disk quota in mebibytes.'))
class Meta:
ordering = ('id', )
ordering = ('id',)
def __unicode__(self):
return self.group.name
......@@ -331,7 +336,11 @@ def create_profile(user):
profile, created = Profile.objects.get_or_create(user=user)
try:
Store(user).create_user(profile.smb_password, None, profile.disk_quota)
store = Store(user)
quotas = [profile.disk_quota]
quotas += [group.profile.disk_quota for group in user.groups.all()]
max_quota = max(quotas)
store.create_user(profile.smb_password, None, max_quota)
except:
logger.exception("Can't create user %s", unicode(user))
return created
......@@ -347,6 +356,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
logger.debug("Register save_org_id to djangosaml2 pre_user_save")
from djangosaml2.signals import pre_user_save
def save_org_id(sender, instance, attributes, **kwargs):
logger.debug("save_org_id called by %s", instance.username)
atr = settings.SAML_ORG_ID_ATTRIBUTE
......@@ -399,6 +409,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
return False # User did not change
pre_user_save.connect(save_org_id)
......@@ -411,7 +422,7 @@ def update_store_profile(sender, **kwargs):
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except (NotOkException, Timeout):
except NotOkException:
logger.critical("Store is not accepting connections.")
......
circle/dashboard/static/dashboard/activity.js
......@@ -41,12 +41,14 @@ $(function() {
$('#confirmation-modal').on('hidden.bs.modal', function() {
$('#confirmation-modal').remove();
});
$('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
}
});
e.preventDefault();
});
/* if the operation fails show the modal again */
$("body").on("click", "#confirmation-modal #op-form-send", function() {
var url = $(this).closest("form").prop("action");
......@@ -237,4 +239,3 @@ String.prototype.hashCode = function() {
}
return hash;
};
circle/dashboard/static/dashboard/dashboard.js
......@@ -558,3 +558,5 @@ $(function() {
inputs.prop("checked", !inputs.prop("checked"));
});
});
$.fn.modal.Constructor.prototype.enforceFocus = function() {};
circle/dashboard/static/dashboard/dashboard.less
......@@ -1079,6 +1079,10 @@ textarea[name="new_members"] {
max-width: 100%;
}
#node-list-auto-migration-body {
padding: 20px;
}
#vm-list-table td.state,
#vm-list-table td.memory {
white-space: nowrap;
......@@ -1088,7 +1092,7 @@ textarea[name="new_members"] {
vertical-align: middle;
}
.disk-resize-btn {
.disk-resize-btn, .disk-export-btn {
margin-right: 5px;
}
......
circle/dashboard/static/dashboard/node-list.js
......@@ -3,4 +3,11 @@ $(function() {
// find disabled nodes, set danger (red) on the rows
$('.node-disabled').closest("tr").addClass('danger');
});
$('#reschedule-now').click(function() {
$.get($(this).attr('href'), function(data){
highlight = data.result === 'ok' ? 'success' : 'danger';
addMessage(data.message, highlight);
});
return false;
});
});
circle/dashboard/store_api.py
......@@ -14,19 +14,20 @@
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from os.path import splitext
import json
import logging
from urlparse import urljoin
from datetime import datetime
from django.http import Http404
import os
from datetime import datetime
from django.conf import settings
from django.http import Http404
from os.path import splitext
from requests import get, post, codes
from requests.exceptions import Timeout # noqa
from sizefield.utils import filesizeformat
from storage.models import Disk
logger = logging.getLogger(__name__)
......@@ -47,6 +48,17 @@ class NoStoreException(StoreApiException):
class Store(object):
def __init__(self, user, default_timeout=0.5):
self.store_url = settings.STORE_URL
if not self.store_url:
raise NoStoreException
if user.is_superuser and not user.profile.org_id:
self.username = 'u-admin'
elif not user.profile.org_id:
raise NoStoreException
else:
self.username = 'u-%s' % user.profile.org_id
self.request_args = {'verify': settings.STORE_VERIFY_SSL}
if settings.STORE_SSL_AUTH:
self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
......@@ -54,18 +66,15 @@ class Store(object):
if settings.STORE_BASIC_AUTH:
self.request_args['auth'] = (settings.STORE_CLIENT_USER,
settings.STORE_CLIENT_PASSWORD)
self.username = "u-%d" % user.pk
self.default_timeout = default_timeout
self.store_url = settings.STORE_URL
if not self.store_url:
raise NoStoreException
def _request(self, url, method=get, timeout=None,
raise_status_code=True, **kwargs):
url = urljoin(self.store_url, url)
if timeout is None:
timeout = self.default_timeout
payload = json.dumps(kwargs) if kwargs else None
kwargs['USER'] = self.username
payload = json.dumps(kwargs)
try:
headers = {'content-type': 'application/json'}
response = method(url, data=payload, headers=headers,
......@@ -83,7 +92,7 @@ class Store(object):
return response
def _request_cmd(self, cmd, **kwargs):
return self._request(self.username, post, CMD=cmd, **kwargs)
return self._request("/user/", post, CMD=cmd, **kwargs)
def list(self, path, process=True):
r = self._request_cmd("LIST", PATH=path)
......@@ -101,6 +110,15 @@ class Store(object):
else:
return result
def get_disk_images(self, path='/'):
images = []
file_list = self.list(path, process=False)
export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
for item in file_list:
if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
images.append(os.path.join(path, item['NAME']))
return images
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
return r.json()['LINK']
......@@ -119,7 +137,7 @@ class Store(object):
self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
def get_quota(self): # no CMD? :o
r = self._request(self.username)
r = self._request("/user/")
quota = r.json()
quota.update({
'readable_used': filesizeformat(float(quota['used'])),
......@@ -129,17 +147,17 @@ class Store(object):
return quota
def set_quota(self, quota):
self._request("/quota/" + self.username, post, QUOTA=quota)
self._request("/quota/", post, QUOTA=quota)
def user_exist(self):
try:
self._request(self.username)
self._request("/user/")
return True
except NotOkException:
return False
def create_user(self, password, keys, quota):
self._request("/new/" + self.username, method=post,
self._request("/new/", method=post,
SMBPASSWD=password, KEYS=keys, QUOTA=quota)
@staticmethod
......
circle/dashboard/templates/dashboard/_disk-list-element.html
......@@ -6,15 +6,29 @@
<span class="operation-wrapper pull-right">
{% if d.is_exportable %}
{% if op.export_disk %}
<a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
{% if op.export_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
</a>
{% endif %}
{% else %}
<small class="btn-xs">
{% trans "Not exportable" %}
</small>
{% endif %}
{% if d.is_resizable %}
{% if op.resize_disk %}
<a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
<a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
{% if op.resize_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
</a>
{% else %}
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
class="btn btn-xs btn-primary operation">
<i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
</a>
{% endif %}
......@@ -24,8 +38,8 @@
</small>
{% endif %}
{% if op.remove_disk %}
<a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
<a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
{% if op.remove_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
</a>
......
circle/dashboard/templates/dashboard/node-list.html
......@@ -41,4 +41,23 @@
</div><!-- -col-md-12 -->
</div><!-- .row -->
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<a id="reschedule-now" class="btn btn-danger pull-right" href="{% url "dashboard.views.reschedule" %}">
<i class="fa fa-magic"></i> {% trans "Reschedule now" %}
</a>
<h3 class="no-margin"><i class="fa fa-truck"></i> {% trans "Virtual machine auto migration" %}</h3>
</div>
<div id="node-list-auto-migration-body">
<h1>Crontab</h1>
<form>
{{ auto_migration_form.as_p }}
</form>
</div>
</div>
</div><!-- -col-md-12 -->
</div><!-- .row -->
{% endblock %}
circle/dashboard/templates/dashboard/vm-detail/_disk-operations.html
{% load i18n %}
{% for op in ops %}
{% if op.is_disk_operation %}
<a href="{{op.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.op}}">
<i class="fa fa-{{op.icon}} fa-fw-12"></i>
{{op.name}} </a>
{% endif %}
{% if op.is_disk_operation %}
<a href="{{ op.get_url }}" class="btn btn-success btn-xs
operation operation-{{ op.op }}">
<i class="fa fa-{{ op.icon }} fa-fw-12"></i>
{{ op.name }} </a>
{% endif %}
{% endfor %}
circle/dashboard/templates/dashboard/vm-detail/network.html
......@@ -37,6 +37,9 @@
<dl>
<dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
<dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
{% if request.user.is_superuser %}
<dt>{% trans "MAC address" %}:</dt> <dd>{{ i.host.mac }}</dd>
{% endif %}
<dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
<dt>{% trans "Groups" %}:</dt>
<dd>
......@@ -114,7 +117,9 @@
{% if l.ipv6 %}
<tr>
<td>
{% autoescape off %}
{% display_portforward6 l %}
{% endautoescape %}
</td>
<td><i class="fa fa-long-arrow-right"></i></td>
<td>
......
circle/dashboard/tests/test_views.py
......@@ -534,7 +534,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
with patch.object(DeployOperation, 'async') as async:
response = c.post("/dashboard/vm/create/", {
'name': 'vm',
'amount': 2,
'amount': 1,
'customized': 1,
'template': 1,
'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
......@@ -543,7 +543,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
assert async.called
self.assertEqual(response.status_code, 302)
self.assertEqual(instance_count + 2, Instance.objects.all().count())
self.assertEqual(instance_count + 1, Instance.objects.all().count())
def test_unpermitted_description_update(self):
c = Client()
......
circle/dashboard/urls.py
......@@ -56,6 +56,7 @@ from .views import (
MessageList, MessageDetail, MessageCreate, MessageDelete,
EnableTwoFactorView, DisableTwoFactorView,
AclUserGroupAutocomplete, AclUserAutocomplete,
RescheduleView,
)
from .views.vm import vm_ops, vm_mass_ops
from .views.node import node_ops
......@@ -153,6 +154,8 @@ urlpatterns = [
r'(?P<time>[0-9]{1,2}[hdwy])$'),
NodeListGraphView.as_view(),
name='dashboard.views.node-list-graph'),
url(r'^node/reschedule/$', RescheduleView.as_view(),
name="dashboard.views.reschedule"),
url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
r'(?P<time>[0-9]{1,2}[hdwy])$'),
TemplateGraphView.as_view(),
......
circle/dashboard/views/node.py
......@@ -25,7 +25,7 @@ from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse_lazy
from django.db.models import Count
from django.forms.models import inlineformset_factory
from django.http import HttpResponse
from django.http import HttpResponse, JsonResponse
from django.shortcuts import redirect
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
......@@ -37,11 +37,14 @@ from django_tables2 import SingleTableView
from firewall.models import Host
from vm.models import Node, NodeActivity, Trait
from vm.tasks.vm_tasks import check_queue
from vm.tasks.local_periodic_tasks import auto_migrate
from ..forms import TraitForm, HostForm, NodeForm
from ..forms import TraitForm, HostForm, NodeForm, AutoMigrationForm
from ..tables import NodeListTable
from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
from manager.mancelery import crontab_parser
def get_operations(instance, user):
ops = []
......@@ -190,6 +193,14 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
table_class = NodeListTable
table_pagination = False
def get_crontab(self):
return crontab_parser(settings.AUTO_MIGRATION_CRONTAB)
def get_context_data(self):
context = super(NodeList, self).get_context_data()
context["auto_migration_form"] = AutoMigrationForm(self.get_crontab())
return context
def get(self, *args, **kwargs):
if not self.request.user.has_perm('vm.view_statistics'):
raise PermissionDenied()
......@@ -210,9 +221,20 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
return super(NodeList, self).get(*args, **kwargs)
def get_queryset(self):
self.wrong_nodes_message()
return Node.objects.annotate(
number_of_VMs=Count('instance_set')).select_related('host')
def wrong_nodes_message(self):
wrong_nodes = []
for node in Node.objects.all():
if node.monitor_info is None:
wrong_nodes.append(node.name)
message = ', '.join(wrong_nodes)
if wrong_nodes:
messages.error(self.request,
"Can't reach " + message + " monitor info")
class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
......@@ -356,3 +378,23 @@ class NodeActivityDetail(LoginRequiredMixin, SuperuserRequiredMixin,
).order_by('-started').select_related())
ctx['icon'] = _get_activity_icon(self.object)
return ctx
class RescheduleView(SuperuserRequiredMixin, View):
def get(self, *args, **kwargs):
try:
auto_migrate.apply_async(queue='localhost.man.slow')
except Exception as e:
msg = str(e)
result = 'error'
else:
result = 'ok'
msg = _('Reschedule has started.')
if self.request.is_ajax():
return JsonResponse({'result': result, 'message': msg})
else:
if result == 'ok':
messages.success(self.request, msg)
else:
messages.error(self.request, msg)
return redirect('dashboard.views.node-list')
circle/dashboard/views/store.py
......@@ -35,7 +35,8 @@ from django.views.generic import TemplateView
from braces.views import LoginRequiredMixin
from ..store_api import Store, NoStoreException, NotOkException
from ..store_api import (Store, NoStoreException,
NotOkException)
logger = logging.getLogger(__name__)
......
circle/dashboard/views/util.py
......@@ -682,7 +682,7 @@ class TransferOwnershipConfirmView(LoginRequiredMixin, View):
messages.error(request, _('This token is invalid or has expired.'))
raise PermissionDenied()
return render(request, self.template,
dictionary={'instance': instance, 'key': key})
{'instance': instance, 'key': key})
def change_owner(self, instance, new_owner):
instance.owner = new_owner
......
circle/dashboard/views/vm.py
......@@ -61,9 +61,10 @@ from .util import (
)
from ..forms import (
AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
VmImportDiskForm, VmSaveForm,
VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm,
......@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance
context_object_name = 'instance' # much simpler to mock object
......@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
class VmAddInterfaceView(FormOperationMixin, VmOperationView):
op = 'add_interface'
form_class = VmAddInterfaceForm
show_in_toolbar = False
......@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
class VmCreateDiskView(FormOperationMixin, VmOperationView):
op = 'create_disk'
form_class = VmCreateDiskForm
show_in_toolbar = False
......@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmImportDiskView(FormOperationMixin, VmOperationView):
op = 'import_disk'
form_class = VmImportDiskForm
show_in_toolbar = False
icon = 'upload'
effect = "success"
is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
val = super(VmImportDiskView, self).get_form_kwargs()
val.update({'user': self.request.user})
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
op = 'download_disk'
form_class = VmDownloadDiskForm
show_in_toolbar = False
......@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmMigrateView(FormOperationMixin, VmOperationView):
op = 'migrate'
icon = 'truck'
effect = 'info'
......@@ -449,8 +460,7 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
if isinstance(inst, Instance):
nodes_w_traits = [
n.pk for n in Node.objects.filter(enabled=True)
if n.online and
has_traits(inst.req_traits.all(), n)
if n.online and has_traits(inst.req_traits.all(), n)
]
ctx['nodes_w_traits'] = nodes_w_traits
......@@ -458,7 +468,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
class VmPortRemoveView(FormOperationMixin, VmOperationView):
template_name = 'dashboard/_vm-remove-port.html'
op = 'remove_port'
show_in_toolbar = False
......@@ -487,7 +496,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
class VmPortAddView(FormOperationMixin, VmOperationView):
op = 'add_port'
show_in_toolbar = False
with_reload = True
......@@ -514,7 +522,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
class VmSaveView(FormOperationMixin, VmOperationView):
op = 'save_as_template'
icon = 'save'
effect = 'info'
......@@ -570,7 +577,7 @@ class TokenOperationView(OperationView):
User can do the action with a valid token instead of logging in.
"""
token_max_age = 3 * 24 * 3600
redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
@classmethod
def get_salt(cls):
......@@ -642,7 +649,6 @@ class TokenOperationView(OperationView):
class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
op = 'renew'
icon = 'calendar'
effect = 'success'
......@@ -769,7 +775,11 @@ vm_ops = OrderedDict([
extra_bases=[TokenOperationView],
op='destroy', icon='times', effect='danger')),
('create_disk', VmCreateDiskView),
('import_disk', VmImportDiskView),
('download_disk', VmDownloadDiskView),
('export_disk', VmDiskModifyView.factory(
op='export_disk', form_class=VmDiskExportForm,
icon='download', effect='info')),
('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")),
......@@ -1030,7 +1040,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
class VmCreate(LoginRequiredMixin, TemplateView):
form_class = VmCustomizeForm
form = None
......@@ -1161,24 +1170,28 @@ class VmCreate(LoginRequiredMixin, TemplateView):
# limit chekcs
try:
limit = user.profile.instance_limit
instance_limit = user.profile.instance_limit
template_instance_limit = user.profile.template_instance_limit
except Exception as e:
logger.debug('No profile or instance limit: %s', e)
else:
try:
amount = int(request.POST.get("amount", 1))
except:
amount = limit # TODO this should definitely use a Form
current = Instance.active.filter(owner=user).count()
logger.debug('current use: %d, limit: %d', current, limit)
if current + amount > limit:
messages.error(request,
_('Instance limit (%d) exceeded.') % limit)
if request.is_ajax():
return HttpResponse(json.dumps({'redirect': '/'}),
content_type="application/json")
else:
return redirect('/')
# TODO this should definitely use a Form
amount = instance_limit
instances = Instance.active.filter(owner=user).count()
template_instances = template.get_user_instances(user).count()
logger.debug('current instance use: %d, limit: %d',
instances, instance_limit)
logger.debug('current template instance use: %d, limit: %d',
template_instances, template_instance_limit)
if instances + amount > instance_limit:
return self._limit_exceeded(instance_limit, request)
if template_instances + amount > template_instance_limit:
return self._limit_exceeded(template_instance_limit, request)
create_func = (self.__create_normal if
request.POST.get("customized") is None else
......@@ -1186,6 +1199,16 @@ class VmCreate(LoginRequiredMixin, TemplateView):
return create_func(request, template, *args, **kwargs)
def _limit_exceeded(self, limit, request):
messages.error(request,
_('Instance limit (%d) exceeded.')
% limit)
if request.is_ajax():
return HttpResponse(json.dumps({'redirect': '/'}),
content_type="application/json")
else:
return redirect('/')
@require_GET
def get_vm_screenshot(request, pk):
......
circle/manager/mancelery.py
......@@ -17,13 +17,27 @@
from celery import Celery
from celery.signals import worker_ready
from celery.schedules import crontab
from datetime import timedelta
from celery.schedules import crontab
from kombu import Queue, Exchange
from os import getenv
HOSTNAME = "localhost"
QUEUE_NAME = HOSTNAME + '.man'
AUTO_MIGRATION_CRONTAB = getenv('AUTO_MIGRATION_CRONTAB', '0 0 * * *')
def crontab_parser(crontab):
fields = crontab.split(' ')
return dict(
minute=fields[0],
hour=fields[1],
day_of_month=fields[2],
month_of_year=fields[3],
day_of_week=fields[4],
)
celery = Celery('manager',
......@@ -56,6 +70,11 @@ celery.conf.update(
'schedule': crontab(minute=10, hour=1),
'options': {'queue': 'localhost.man'}
},
# 'vm.local_periodic_tasks': {
# 'task': 'vm.tasks.local_periodic_tasks.auto_migrate',
# 'schedule': crontab(**crontab_parser(AUTO_MIGRATION_CRONTAB)),
# 'options': {'queue': 'localhost.man.slow'},
# },
}
)
......
circle/manager/scheduler.py
......@@ -15,15 +15,18 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
import datetime
import json
import random
from logging import getLogger
from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
from django.utils.translation import ugettext_noop
from common.models import HumanReadableException
from circle.settings.base import SCHEDULER_METHOD
import random
from common.models import HumanReadableException
logger = getLogger(__name__)
......@@ -69,14 +72,14 @@ def common_select(instance, nodes):
logger.warning('select_node: no enough RAM for %s', unicode(instance))
raise NotEnoughMemoryException()
# sort nodes first by processor usage, then priority
# sort nodes first by priority
nodes.sort(key=lambda n: n.priority, reverse=True)
nodes.sort(key=free_cpu_time, reverse=True)
return nodes
def common_evenly(instance, nodes):
nodes = common_select(instance, nodes)
nodes.sort(key=free_cpu_time, reverse=True)
result = nodes[0]
return result
......@@ -87,6 +90,16 @@ def common_random(instance, nodes):
return result
def advanced_with_time_stamp(instance, nodes):
nodes = common_select(instance, nodes)
nodes.sort(key=sorting_key, reverse=True)
logger.info("SCHEDLOG: {}".format(json.dumps({
"event": "after_sort",
"list": map(lambda node: unicode(node), nodes)})))
result = nodes[0]
return result
def select_node(instance, nodes):
''' Select a node for hosting an instance based on its requirements.
'''
......@@ -94,14 +107,72 @@ def select_node(instance, nodes):
result = common_evenly(instance, nodes)
elif SCHEDULER_METHOD == 'random':
result = common_random(instance, nodes)
elif SCHEDULER_METHOD == 'advanced':
result = advanced_with_time_stamp(instance, nodes)
else: # Default method is the random
result = common_random(instance, nodes)
logger.info('Scheduler method: %s selected', unicode(SCHEDULER_METHOD))
logger.info('select_node: %s for %s', unicode(result), unicode(instance))
logger.info("SCHEDLOG: {}".format(json.dumps(
{"event": "select",
"node": unicode(result),
"vm": unicode(instance)})))
set_time_stamp(result)
return result
def sorting_key(node):
"""Determines how valuable a node is for scheduling.
"""
key = 0
corr = last_scheduled_correction_factor(node)
if free_cpu_time(node) < free_ram(node):
key = free_cpu_time(node) * corr
else:
key = free_ram(node) * corr
logger.info("SCHEDLOG: {}".format(json.dumps({
"event": "sort",
"node": unicode(node),
"sorting_key": unicode(key),
"free_cpu_time": unicode(free_cpu_time(node)),
"free_ram": unicode(free_ram(node)),
"last_scheduled_correction_factor": unicode(last_scheduled_correction_factor(node))})))
return key
def set_time_stamp(node):
cache.set('time_stamp{}'.format(node.id), timezone.now())
def get_time_stamp(node):
time_stamp = cache.get('time_stamp{}'.format(node.id))
if time_stamp:
return time_stamp
return datetime.datetime(1970, 1, 1, tzinfo=timezone.get_current_timezone())
def last_scheduled_correction_factor(node):
"""Returns the time correction factor for a node.
The monitor data may be outdated, because of recent scheduling for a given node.
The return value is between 0 and 1, higher value indicates more time since the
last scheduling for the given node.
"""
factor = 0
max_time_diff = settings.SCHEDULER_TIME_SENSITIVITY_IN_SECONDS
current_time = timezone.now()
time_difference_in_seconds = (
current_time - get_time_stamp(node)).total_seconds()
factor = time_difference_in_seconds/float(max_time_diff)
if factor > 1:
factor = 1
elif factor < 0:
factor = 1
logger.info('Scheduler set factor to %s', unicode(factor))
return factor
def has_traits(traits, node):
"""True, if the node has all specified traits; otherwise, false.
"""
......@@ -142,11 +213,27 @@ def free_cpu_time(node):
Higher values indicate more idle time.
"""
try:
activity = node.cpu_usage / 100
inactivity = 1 - activity
cores = node.num_cores
return cores * inactivity
free_cpu_percent = 1 - node.cpu_usage
weight = node.cpu_weight
weighted_value = free_cpu_percent * weight
return weighted_value
except TypeError as e:
logger.exception('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return 0 # will result lowest priority
def free_ram(node):
"""Get an indicator number for free RAM on the node.
Higher value indicates more RAM.
"""
try:
free_ram_percent = 1 - node.ram_usage
weight = node.ram_weight
weighted_value = free_ram_percent * weight
return weighted_value
except TypeError as e:
logger.warning('Got incorrect monitoring data for node %s. %s',
logger.exception('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return False # monitoring data is incorrect
return 0 # will result lowest priority
circle/request/forms.py
......@@ -16,7 +16,7 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from django.forms import (
ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
Textarea, ValidationError
Textarea, ValidationError, TextInput, IntegerField, EmailField
)
from django.utils.translation import ugettext_lazy as _
from django.template.loader import render_to_string
......@@ -27,11 +27,78 @@ from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit
from request.models import (
LeaseType, TemplateAccessType, TemplateAccessAction,
LeaseType, TemplateAccessType, TemplateAccessAction, RequestField
)
from dashboard.forms import VmResourcesForm
class RequestFieldModelForm(ModelForm):
class Meta:
model = RequestField
fields = '__all__'
widgets = {
'choices': TextInput(attrs={'placeholder': 'Optional'}),
}
def __init__(self, *args, **kwargs):
super(RequestFieldModelForm, self).__init__(*args, **kwargs)
self.fields['choices'].required = False
@property
def helper(self):
helper = FormHelper()
return helper
def clean(self):
cleaned_data = super(RequestFieldModelForm, self).clean()
if cleaned_data['type'] == 'Email' and cleaned_data['choices']:
raise ValidationError(_("Email field can't have choices!"))
if cleaned_data['type'] == 'Integer':
for choice in cleaned_data['choices'].split(','):
try:
int(choice)
except:
raise ValidationError(_("IntegerField choices must be \
integers"))
class EditableForm(Form):
def __init__(self, *args, **kwargs):
type = kwargs.pop('type', None)
kwargs.pop("request", None)
super(EditableForm, self).__init__(*args, **kwargs)
fields = RequestField.objects.filter(request_type=type)
n = 0
if fields:
for field in fields:
n = n+1
type = field.type
if(type == 'Char'):
self.fields['field'+str(n)] = CharField(
max_length=30,
required=field.required,
label=field.fieldname)
elif (type == 'Integer'):
self.fields['field'+str(n)] = IntegerField(
required=field.required,
label=field.fieldname)
elif (type == 'Email'):
self.fields['field'+str(n)] = EmailField(
required=field.required,
label=field.fieldname)
if(field.choices):
choices = [(ch, ch)for ch in field.choices.split(',')]
self.fields['field'+str(n)] = ChoiceField(
choices=choices,
required=field.required,
label=field.fieldname)
def get_dynamic_fields(self):
for field_name in self.fields:
if field_name.startswith("field"):
yield self[field_name]
class LeaseTypeForm(ModelForm):
@property
def helper(self):
......@@ -80,27 +147,18 @@ class InitialFromFileMixin(object):
return message.strip()
class TemplateRequestForm(InitialFromFileMixin, Form):
message = CharField(widget=Textarea, label=_("Message"))
class TemplateRequestForm(EditableForm):
template = ModelChoiceField(TemplateAccessType.objects.all(),
label=_("Template share"))
level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
initial=TemplateAccessAction.LEVELS.user)
initial_template = "request/initials/template.html"
class LeaseRequestForm(InitialFromFileMixin, Form):
class LeaseRequestForm(EditableForm):
lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
message = CharField(widget=Textarea, label=_("Message"))
initial_template = "request/initials/lease.html"
class ResourceRequestForm(InitialFromFileMixin, VmResourcesForm):
message = CharField(widget=Textarea, label=_("Message"))
initial_template = "request/initials/resources.html"
class ResourceRequestForm(EditableForm, VmResourcesForm):
def clean(self):
cleaned_data = super(ResourceRequestForm, self).clean()
......
circle/request/migrations/0005_requestfield.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-11-12 15:20
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0004_auto_20150629_1605'),
]
operations = [
migrations.CreateModel(
name='RequestField',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('fieldname', models.CharField(max_length=50, unique=True)),
('type', models.CharField(choices=[(b'Char', b'CharField'), (b'Integer', b'IntegerField'), (b'Email', b'EmailField')], default=b'Char', max_length=20)),
('choices', models.CharField(max_length=100, null=True)),
('required', models.BooleanField(default=True)),
],
),
]
circle/request/migrations/0006_auto_20181115_1548.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-11-15 15:48
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0005_requestfield'),
]
operations = [
migrations.AlterField(
model_name='requestfield',
name='choices',
field=models.CharField(max_length=300, null=True),
),
]
circle/request/migrations/0007_requestfield_request_type.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-12-12 14:12
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0006_auto_20181115_1548'),
]
operations = [
migrations.AddField(
model_name='requestfield',
name='request_type',
field=models.CharField(choices=[(b'resource', 'resource request'), (b'lease', 'lease request'), (b'template', 'template access request'), (b'resize', 'disk resize request')], default=b'template', max_length=20),
),
]
circle/request/models.py
......@@ -19,6 +19,7 @@ import logging
from django.db.models import (
Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
BooleanField,
)
from django.db.models.signals import post_save
from django.conf import settings
......@@ -157,6 +158,27 @@ class Request(TimeStampedModel):
return self.action.is_acceptable()
class RequestField(Model):
TYPES = (
('Char', 'CharField'),
('Integer', 'IntegerField'),
('Email', 'EmailField')
)
fieldname = CharField(max_length=50, blank=False, unique=True)
type = CharField(choices=TYPES, default='Char', max_length=20)
request_type = CharField(choices=Request.TYPES, default='template',
max_length=20)
choices = CharField(max_length=300, null=True)
required = BooleanField(default=True)
def __unicode__(self):
return self.fieldname
def get_absolute_url(self):
return reverse('fields_detail', kwargs={'pk': self.pk})
class LeaseType(RequestType):
lease = ForeignKey(Lease, verbose_name=_("Lease"))
......
circle/request/templates/request/_request-field-form.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load i18n %}
{% load crispy_forms_tags %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading"></div>
<div class="panel-body">
<form action={% url 'request.views.request-field-add' %} method="post">
{% include "display-form-errors.html" %}
{% csrf_token %}
{% for field in form %}
{{ field|as_crispy_field}}
{% endfor %}
<button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
</form>
</div> <!-- .panel-body -->
</div>
</div>
</div>
{% endblock %}
circle/request/templates/request/_request-lease-form.html
......@@ -6,5 +6,8 @@
{% csrf_token %}
{{ form.lease|as_crispy_field }}
{{ form.message|as_crispy_field }}
{% for fields in form.get_dynamic_fields %}
{{ field|as_crispy_field }}
{% endfor %}
<input type="submit" class="btn btn-primary"/>
</form>
circle/request/templates/request/_request-template-form.html
......@@ -21,6 +21,8 @@
</label>
</div>
{% endfor %}
{{ form.message|as_crispy_field }}
{% for field in form.get_dynamic_fields %}
{{ field|as_crispy_field }}
{% endfor %}
<input type="submit" class="btn btn-primary"/>
</form>
circle/request/templates/request/field-list.html 0 → 100644
{% extends "dashboard/base.html" %}
{% load i18n %}
{% load crispy_forms_field %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Request fields" %}</h3>
</div>
<div class="panel-body">
<div class="table-responsive">
<div class="table-container">
<table class="text-center table table-bordered table-striped table-hover" >
<thead class"align-center">
<tr>
<th class="text-center">{% trans "Fieldname" %}</th>
<th class="text-center">{% trans "Type" %}</th>
<th class="text-center">{% trans "RequestType" %}</th>
<th class="text-center">{% trans "Choices" %}</th>
<th class="text-center">{% trans "Required" %}</th>
<th class="text-center">{% trans "Delete" %}</th>
</tr>
</thead>
<tbody>
{% for field in object_list %}
<tr>
<td>{{field.fieldname}}</td>
<td>{{field.type}}</td>
<td>{{field.request_type}}</td>
<td>{{field.choices}}</td>
<td>{{field.required}}</td>
<td><a href={% url "request.views.field-delete" pk=field.pk %}>
<i class="fa fa-times"></i></a>
</td>
</tr>
{% endfor %}
<tr>
<form action={% url 'request.views.request-field-add' %} method="post">
{% csrf_token %}
{% for field in add_form %}
<td> {% crispy_field field %}</td>
{% endfor %}
<td><button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
</form>
</tr>
</tbody>
</table>
</div>
</div> <!-- .table-responsive -->
</div> <!-- .panel-body -->
</div>
</div>
</div>
{% endblock %}
circle/request/templates/request/list.html
......@@ -11,9 +11,14 @@
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<a class="btn btn-xs btn-primary pull-right "href="{% url "request.views.type-list" %}">
<div class="pull-right">
<a class="btn btn-xs btn-primary"href="{% url "request.views.type-list" %}">
{% trans "Request types" %}
</a>
<a class="btn btn-xs btn-success" href="{% url "request.views.field-list" %}">
{% trans "Request fields" %}
</a>
</div>
<h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Requests" %}</h3>
</div>
<div class="panel-body">
......
circle/request/templates/request/request-resource.html
......@@ -24,7 +24,9 @@
</div>
{% include "display-form-errors.html" %}
{% include "dashboard/_resources-sliders.html" with field_priority=form.priority field_num_cores=form.num_cores field_ram_size=form.ram_size %}
{{ form.message|as_crispy_field }}
{% for field in form.get_dynamic_fields %}
{{ field|as_crispy_field }}
{% endfor %}
<button type="submit" class="btn btn-success">
{% trans "Request new resources" %}
</button>
......
circle/request/tests.py
......@@ -25,7 +25,7 @@ from mock import Mock, patch
from common.tests.celery_mock import MockCeleryMixin
from vm.models import Instance, InstanceTemplate, Lease
from dashboard.models import Profile
from request.models import Request, LeaseType, TemplateAccessType
from request.models import Request, LeaseType, TemplateAccessType, RequestField
from dashboard.tests.test_views import LoginMixin
from vm.operations import ResourcesOperation
......@@ -67,12 +67,16 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
inst = Instance.objects.get(pk=1)
inst.set_level(self.u1, 'owner')
field = RequestField(fieldname="Oka", type="Char",
request_type="resource", required=True)
field.save()
req_count = Request.objects.count()
resp = c.post("/request/resource/1/", {
'num_cores': 5,
'ram_size': 512,
'priority': 30,
'message': "szia",
'field1': "szia",
})
self.assertEqual(resp.status_code, 302)
self.assertEqual(req_count + 1, Request.objects.count())
......@@ -104,17 +108,25 @@ class RequestTest(LoginMixin, MockCeleryMixin, TestCase):
template = InstanceTemplate.objects.get(pk=1)
self.assertFalse(template.has_level(self.u1, "user"))
field = RequestField(fieldname="Tanszek", type="Char",
request_type="template", required=True)
field.save()
field = RequestField(fieldname="Szobaszam", type="Integer",
request_type="template", required=False)
field.save()
req_count = Request.objects.count()
resp = c.post("/request/template/", {
'template': 1,
'level': "user",
'message': "szia",
'field1': "IIT",
'field2': 10,
})
self.assertEqual(resp.status_code, 302)
self.assertEqual(req_count + 1, Request.objects.count())
new_request = Request.objects.latest("pk")
self.assertEqual(new_request.status, "PENDING")
self.assertEqual(new_request.message, "Tanszek: IIT\nSzobaszam: 10\n")
new_request.accept(self.us)
new_request = Request.objects.latest("pk")
......
circle/request/urls.py
......@@ -24,6 +24,8 @@ from .views import (
TemplateAccessTypeCreate, TemplateAccessTypeDetail,
TemplateRequestView, LeaseRequestView, ResourceRequestView,
LeaseTypeDelete, TemplateAccessTypeDelete, ResizeRequestView,
RequestFieldFormView, RequestFieldListView, RequestFieldDetailView,
RequestFieldDeleteView,
)
urlpatterns = [
......@@ -35,6 +37,15 @@ urlpatterns = [
url(r'^type/list/$', RequestTypeList.as_view(),
name="request.views.type-list"),
url(r'fields/add/$', RequestFieldFormView.as_view(),
name='request.views.request-field-add'),
url(r'fields/$', RequestFieldListView.as_view(),
name='request.views.field-list'),
url(r'fields/field/(?P<pk>[0-9]+)/$', RequestFieldDetailView.as_view(),
name='request.views.fields-detail'),
url(r'^field/delete/(?P<pk>\d+)/$', RequestFieldDeleteView.as_view(),
name='request.views.field-delete'),
# request types
url(r'^type/lease/create/$', LeaseTypeCreate.as_view(),
name="request.views.lease-type-create"),
......
circle/request/views.py
......@@ -18,6 +18,7 @@ from __future__ import unicode_literals, absolute_import
from django.views.generic import (
UpdateView, TemplateView, DetailView, CreateView, FormView, DeleteView,
ListView
)
from django.contrib import messages
from django.contrib.messages.views import SuccessMessageMixin
......@@ -32,7 +33,7 @@ from django_tables2 import SingleTableView
from request.models import (
Request, TemplateAccessType, LeaseType, TemplateAccessAction,
ExtendLeaseAction, ResourceChangeAction, DiskResizeAction
ExtendLeaseAction, ResourceChangeAction, DiskResizeAction, RequestField
)
from storage.models import Disk
from vm.models import Instance
......@@ -42,7 +43,39 @@ from request.tables import (
from request.forms import (
LeaseTypeForm, TemplateAccessTypeForm, TemplateRequestForm,
LeaseRequestForm, ResourceRequestForm, ResizeRequestForm,
RequestFieldModelForm
)
from django.urls import reverse_lazy
class RequestFieldFormView(LoginRequiredMixin, CreateView):
template_name = 'request/_request-field-form.html'
model = RequestField
form_class = RequestFieldModelForm
success_url = reverse_lazy('request.views.field-list')
class RequestFieldListView(LoginRequiredMixin, ListView):
template_name = 'request/field-list.html'
model = RequestField
def get_context_data(self, *args, **kwargs):
ctx = super(RequestFieldListView, self).get_context_data(*args,
**kwargs)
ctx['add_form'] = RequestFieldModelForm()
ctx['add_form'].helper.form_show_labels = False
return ctx
class RequestFieldDetailView(LoginRequiredMixin, DetailView):
model = RequestField
class RequestFieldDeleteView(LoginRequiredMixin, DeleteView):
model = RequestField
template_name = "dashboard/confirm/base-delete.html"
success_url = reverse_lazy('request.views.field-list')
class RequestList(LoginRequiredMixin, SuperuserRequiredMixin, SingleTableView):
......@@ -173,7 +206,7 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
def get_form_kwargs(self):
kwargs = super(TemplateRequestView, self).get_form_kwargs()
kwargs['request'] = self.request
kwargs['type'] = 'template'
return kwargs
def form_valid(self, form):
......@@ -187,9 +220,13 @@ class TemplateRequestView(LoginRequiredMixin, FormView):
)
ta.save()
message = ''
for field in form.get_dynamic_fields():
message += "%s: %s\n" % (unicode(field.label),
unicode(data[field.name]))
req = Request(
user=user,
message=data['message'],
message=message,
type=Request.TYPES.template,
action=ta
)
......@@ -236,6 +273,12 @@ class LeaseRequestView(VmRequestMixin, FormView):
user_level = "operator"
success_message = _("Request successfully sent.")
def get_form_kwargs(self):
kwargs = super(LeaseRequestView, self).get_form_kwargs()
kwargs['type'] = 'resource'
return kwargs
def form_valid(self, form):
data = form.cleaned_data
user = self.request.user
......@@ -247,9 +290,14 @@ class LeaseRequestView(VmRequestMixin, FormView):
)
el.save()
message = ''
for field in form.get_dynamic_fields():
message += "%s: %s\n" % (unicode(field.label),
unicode(data[field.name]))
req = Request(
user=user,
message=data['message'],
message=message,
type=Request.TYPES.lease,
action=el
)
......@@ -267,6 +315,7 @@ class ResourceRequestView(VmRequestMixin, FormView):
def get_form_kwargs(self):
kwargs = super(ResourceRequestView, self).get_form_kwargs()
kwargs['type'] = 'resource'
kwargs['can_edit'] = True
kwargs['instance'] = self.get_vm()
return kwargs
......@@ -292,9 +341,14 @@ class ResourceRequestView(VmRequestMixin, FormView):
)
rc.save()
message = ''
for field in form.get_dynamic_fields():
message += "%s: %s\n" % (unicode(field.label),
unicode(data[field.name]))
req = Request(
user=user,
message=data['message'],
message=message,
type=Request.TYPES.resource,
action=rc
)
......
circle/storage/migrations/0003_auto_20200424_2000.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0002_disk_bus'),
]
operations = [
migrations.AlterModelOptions(
name='disk',
options={'ordering': ['name'], 'permissions': (('create_empty_disk', 'Can create an empty disk.'), ('download_disk', 'Can download a disk.'), ('resize_disk', 'Can resize a disk.'), ('import_disk', 'Can import a disk.'), ('export_disk', 'Can export a disk.')), 'verbose_name': 'disk', 'verbose_name_plural': 'disks'},
),
]
circle/storage/models.py
......@@ -20,31 +20,30 @@
from __future__ import unicode_literals
import logging
from os.path import join
import uuid
import re
import re
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from celery.exceptions import TimeoutError
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel
from os.path import join
from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks
from celery.exceptions import TimeoutError
from common.models import (
WorkerNotFound, HumanReadableException, humanize_exception, method_cache
)
from .tasks import local_tasks, storage_tasks
logger = logging.getLogger(__name__)
class DataStore(Model):
"""Collection of virtual disks.
"""
name = CharField(max_length=100, unique=True, verbose_name=_('name'))
......@@ -119,12 +118,15 @@ class DataStore(Model):
class Disk(TimeStampedModel):
"""A virtual disk.
"""
TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
EXPORT_FORMATS = (('qcow2', _('QEMU disk image')),
('vmdk', _('VMware disk image')),
('vdi', _('VirtualBox disk image')),
('vpc', _('HyperV disk image')))
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True,
verbose_name=_("filename"))
......@@ -149,7 +151,9 @@ class Disk(TimeStampedModel):
permissions = (
('create_empty_disk', _('Can create an empty disk.')),
('download_disk', _('Can download a disk.')),
('resize_disk', _('Can resize a disk.'))
('resize_disk', _('Can resize a disk.')),
('import_disk', _('Can import a disk.')),
('export_disk', _('Can export a disk.'))
)
class DiskError(HumanReadableException):
......@@ -427,6 +431,19 @@ class Disk(TimeStampedModel):
return disk
@classmethod
def _run_abortable_task(cls, remote, task):
while True:
try:
result = remote.get(timeout=5)
break
except TimeoutError as e:
if task is not None and task.is_aborted():
AbortableAsyncResult(remote.id).abort()
raise humanize_exception(ugettext_noop(
"Operation aborted by user."), e)
return result
@classmethod
def download(cls, url, task, user=None, **params):
"""Create disk object and download data from url synchronusly.
......@@ -451,15 +468,7 @@ class Disk(TimeStampedModel):
kwargs={'url': url, 'parent_id': task.request.id,
'disk': disk.get_disk_desc()},
queue=queue_name)
while True:
try:
result = remote.get(timeout=5)
break
except TimeoutError as e:
if task is not None and task.is_aborted():
AbortableAsyncResult(remote.id).abort()
raise humanize_exception(ugettext_noop(
"Operation aborted by user."), e)
result = cls._run_abortable_task(remote, task)
disk.size = result['size']
disk.type = result['type']
disk.checksum = result.get('checksum', None)
......@@ -467,6 +476,40 @@ class Disk(TimeStampedModel):
disk.save()
return disk
@classmethod
def import_disk(cls, user, name, download_link, task):
params = {'name': name,
'type': 'qcow2-norm'}
disk = cls.__create(user=user, params=params)
queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.import_disk.apply_async(
kwargs={
"disk_desc": disk.get_disk_desc(),
"url": download_link,
"task": task.request.id
},
queue=queue_name
)
result = cls._run_abortable_task(remote, task)
disk.size = result["size"]
disk.checksum = result["checksum"]
disk.is_ready = True
disk.save()
return disk
def export(self, exported_name, disk_format, upload_link, task):
queue_name = self.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.export_disk.apply_async(
kwargs={
"disk_desc": self.get_disk_desc(),
"disk_format": disk_format,
"exported_name": exported_name,
"upload_link": upload_link,
"task": task.request.id
},
queue=queue_name)
self._run_abortable_task(remote, task)
def destroy(self, user=None, task_uuid=None):
if self.destroyed:
return False
......@@ -549,4 +592,8 @@ class Disk(TimeStampedModel):
@property
def is_resizable(self):
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap', )
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap',)
@property
def is_exportable(self):
return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')
circle/storage/tasks/storage_tasks.py
......@@ -38,6 +38,16 @@ def download(disk_desc, url):
pass
@celery.task(name='storagedriver.import_disk')
def import_disk(disk_desc, url):
pass
@celery.task(name='storagedriver.export_disk')
def export_disk(disk_desc, format):
pass
@celery.task(name='storagedriver.delete')
def delete(path):
pass
......
circle/vm/management/commands/mass_create_vms.py 0 → 100644
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import BaseCommand
from vm.models import Instance, InstanceTemplate
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('-t', '--template', type=int, required=True)
parser.add_argument('-u', '--users', required=True)
parser.add_argument('-a', '--admin')
parser.add_argument('-o', '--operator')
def handle(self, *args, **options):
template = InstanceTemplate.objects.get(id=options['template'])
with open(options['users']) as f:
users = f.read().splitlines()
missing_users = Instance.mass_create_for_users(
template, users, options['admin'], options['operator']
)
if len(missing_users) > 0:
self.stdout.write('These users do not exist:')
for user in missing_users:
self.stdout.write(user)
circle/vm/migrations/0003_auto_20171213_2018.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2017-12-13 20:18
from __future__ import unicode_literals
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('vm', '0002_interface_model'),
]
operations = [
migrations.AddField(
model_name='node',
name='cpu_weight',
field=models.FloatField(default=1.0, help_text='Indicates the relative CPU power of this node.', verbose_name='CPU Weight'),
),
migrations.AddField(
model_name='node',
name='ram_weight',
field=models.FloatField(default=1.0, help_text='Indicates the relative RAM quantity of this node.', verbose_name='RAM Weight'),
),
migrations.AddField(
model_name='node',
name='time_stamp',
field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, help_text='A timestamp for the node, used by the scheduler.', verbose_name='Last Scheduled Time Stamp'),
preserve_default=False,
),
]
circle/vm/models/instance.py
......@@ -200,6 +200,9 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
def get_running_instances(self):
return Instance.active.filter(template=self, status="RUNNING")
def get_user_instances(self, user):
return Instance.active.filter(template=self, owner=user)
@property
def metric_prefix(self):
return 'template.%d' % self.pk
......@@ -439,6 +442,30 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
return [cls.create(cps, disks, networks, req_traits, tags)
for cps in customized_params]
@classmethod
def mass_create_for_users(cls, template, users, admin=None, operator=None, **kwargs):
"""
Create and deploy an instance of a template for each user
in a list of users. Returns the user IDs of missing users.
"""
user_instances = []
missing_users = []
for user_id in users:
try:
user_instances.append(User.objects.get(profile__org_id=user_id))
except User.DoesNotExist:
missing_users.append(user_id)
for user in user_instances:
instance = cls.create_from_template(template, user, **kwargs)
if admin:
instance.set_level(User.objects.get(username=admin), 'owner')
if operator:
instance.set_level(User.objects.get(username=operator), 'operator')
instance.deploy(user=user)
return missing_users
def clean(self, *args, **kwargs):
self.time_of_suspend, self.time_of_delete = self.get_renew_times()
super(Instance, self).clean(*args, **kwargs)
......@@ -576,7 +603,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
host = self.get_connect_host(use_ipv6=use_ipv6)
proto = self.access_method
if proto == 'rdp':
return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s' % {
return 'rdesktop %(host)s:%(port)d -u cloud -p %(pw)s -f' % {
'port': port, 'proto': proto, 'pw': self.pw,
'host': host}
elif proto == 'ssh':
......@@ -865,6 +892,53 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
def metric_prefix(self):
return 'vm.%s' % self.vm_name
class MonitorUnavailableException(Exception):
"""Exception for monitor_info()
Indicates the unavailability of the monitoring server.
"""
pass
def monitor_info(self):
metrics = ('cpu.percent', 'memory.usage')
prefix = self.metric_prefix
params = [('target', '%s.%s' % (prefix, metric))
for metric in metrics]
params.append(('from', '-5min'))
params.append(('format', 'json'))
try:
logger.info('%s %s', settings.GRAPHITE_URL, params)
response = requests.get(settings.GRAPHITE_URL, params=params)
retval = {}
for target in response.json():
# Example:
# {"target": "circle.vm.{name}.cpu.usage",
# "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
try:
metric = target['target']
if metric.startswith(prefix):
metric = metric[len(prefix):]
else:
continue
value = target['datapoints'][-2][0]
retval[metric] = float(value)
except (KeyError, IndexError, ValueError):
continue
return retval
except Exception:
logger.exception('Monitor server unavailable: ')
raise Instance.MonitorUnavailableException()
def cpu_usage(self):
return self.monitor_info().get('cpu.percent')
def ram_usage(self):
return self.monitor_info().get('memory.usage')
@contextmanager
def activity(self, code_suffix, readable_name, on_abort=None,
on_commit=None, task_uuid=None, user=None,
......
circle/vm/models/node.py
......@@ -30,12 +30,12 @@ from time import time, sleep
from django.conf import settings
from django.db.models import (
CharField, IntegerField, ForeignKey, BooleanField, ManyToManyField,
FloatField, permalink, Sum
FloatField, DateTimeField, permalink, Sum
)
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from celery.exceptions import TimeoutError
from celery.exceptions import TimeoutError, TaskRevokedError
from model_utils.models import TimeStampedModel
from taggit.managers import TaggableManager
......@@ -128,11 +128,15 @@ class Node(OperatedMixin, TimeStampedModel):
enabled = BooleanField(verbose_name=_('enabled'), default=False,
help_text=_('Indicates whether the node can '
'be used for hosting.'))
schedule_enabled = BooleanField(verbose_name=_('schedule enabled'),
default=False, help_text=_(
schedule_enabled = BooleanField(
verbose_name=_('schedule enabled'),
default=False,
help_text=_(
'Indicates whether a vm can be '
'automatically scheduled to this '
'node.'))
'node.'
)
)
traits = ManyToManyField(Trait, blank=True,
help_text=_("Declared traits."),
verbose_name=_('traits'))
......@@ -140,6 +144,21 @@ class Node(OperatedMixin, TimeStampedModel):
overcommit = FloatField(default=1.0, verbose_name=_("overcommit ratio"),
help_text=_("The ratio of total memory with "
"to without overcommit."))
ram_weight = FloatField(
default=1.0,
help_text=_("Indicates the relative RAM quantity of this node."),
verbose_name=_("RAM Weight")
)
cpu_weight = FloatField(
default=1.0,
help_text=_("Indicates the relative CPU power of this node."),
verbose_name=_("CPU Weight")
)
time_stamp = DateTimeField(
auto_now_add=True,
help_text=_("A timestamp for the node, used by the scheduler."),
verbose_name=_("Last Scheduled Time Stamp")
)
class Meta:
app_label = 'vm'
......@@ -162,7 +181,7 @@ class Node(OperatedMixin, TimeStampedModel):
self.get_remote_queue_name("vm", "fast")
self.get_remote_queue_name("vm", "slow")
self.get_remote_queue_name("net", "fast")
except:
except Exception:
return False
else:
return True
......@@ -315,7 +334,7 @@ class Node(OperatedMixin, TimeStampedModel):
queue=self.get_remote_queue_name('vm', priority),
expires=timeout + 60)
return r.get(timeout=timeout)
except (TimeoutError, WorkerNotFound):
except (TimeoutError, WorkerNotFound, TaskRevokedError):
if raise_:
raise
else:
......@@ -341,19 +360,25 @@ class Node(OperatedMixin, TimeStampedModel):
# Example:
# {"target": "circle.szianode.cpu.usage",
# "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
logger.info('MONITOR_TARGET: %s', target)
try:
metric = target['target']
if metric.startswith(prefix):
metric = metric[len(prefix):]
else:
logger.info('MONITOR_MET: %s %s', target, metric)
continue
value = target['datapoints'][-1][0]
if value is None:
value = target['datapoints'][-2][0]
retval[metric] = float(value)
logger.info('MONITOR_RETVAL: %s %s, %s', target['target'], metric, retval[metric])
except (KeyError, IndexError, ValueError, TypeError):
logger.info('MONITOR_ERR: %s %s', metric, value)
continue
return retval
except:
except Exception:
logger.exception('Unhandled exception: ')
return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
priority="fast")
......@@ -363,15 +388,27 @@ class Node(OperatedMixin, TimeStampedModel):
def driver_version(self):
return self.info.get('driver_version')
def get_monitor_info(self, metric):
# return with the metric value if the monitor info not none
# or return 0 if its None or the metric unreachable
if self.monitor_info is None:
logger.warning('Monitor info is None')
return 0
elif self.monitor_info.get(metric) is None:
logger.warning('Unreachable monitor info of: ' + metric)
return 0
else:
return self.monitor_info.get(metric)
@property
@node_available
def cpu_usage(self):
return self.monitor_info.get('cpu.percent') / 100
return self.get_monitor_info('cpu.percent') / 100
@property
@node_available
def ram_usage(self):
return self.monitor_info.get('memory.usage') / 100
return self.get_monitor_info('memory.usage') / 100
@property
@node_available
......@@ -404,7 +441,7 @@ class Node(OperatedMixin, TimeStampedModel):
vm_state_changed hook.
"""
domains = {}
domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5,
domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=10,
priority="fast")
if domain_list is None:
logger.info("Monitoring failed at: %s", self.name)
......@@ -413,7 +450,7 @@ class Node(OperatedMixin, TimeStampedModel):
# [{'name': 'cloud-1234', 'state': 'RUNNING', ...}, ...]
try:
id = int(i['name'].split('-')[1])
except:
except Exception:
pass # name format doesn't match
else:
domains[id] = i['state']
......
circle/vm/operations.py
......@@ -16,53 +16,51 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import, unicode_literals
from StringIO import StringIO
from base64 import encodestring
from hashlib import md5
from logging import getLogger
import os
from re import search
from string import ascii_lowercase
from StringIO import StringIO
from tarfile import TarFile, TarInfo
import time
from urlparse import urlsplit
import os
import time
from celery.contrib.abortable import AbortableAsyncResult
from celery.exceptions import TimeLimitExceeded, TimeoutError
from django.conf import settings
from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from django.conf import settings
from django.db.models import Q
from re import search
from sizefield.utils import filesizeformat
from celery.contrib.abortable import AbortableAsyncResult
from celery.exceptions import TimeLimitExceeded, TimeoutError
from common.models import (
create_readable, humanize_exception, HumanReadableException
)
from common.operations import Operation, register_operation, SubOperationMixin
from dashboard.store_api import Store, NoStoreException
from firewall.models import Host
from manager.scheduler import SchedulerError
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
from monitor.client import Client
from storage.models import Disk
from storage.tasks import storage_tasks
from .models import (
Instance, InstanceActivity, InstanceTemplate, Interface, Node,
NodeActivity, pwgen
)
from .tasks import agent_tasks, vm_tasks
from dashboard.store_api import Store, NoStoreException
from firewall.models import Host
from monitor.client import Client
from storage.tasks import storage_tasks
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
logger = getLogger(__name__)
class RemoteOperationMixin(object):
remote_timeout = 30
def _operation(self, **kwargs):
......@@ -167,7 +165,6 @@ class InstanceOperation(Operation):
class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
remote_queue = ('vm', 'fast')
def _get_remote_queue(self):
......@@ -178,7 +175,7 @@ class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
class EnsureAgentMixin(object):
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
def check_precond(self):
super(EnsureAgentMixin, self).check_precond()
......@@ -198,7 +195,7 @@ class EnsureAgentMixin(object):
class RemoteAgentOperation(EnsureAgentMixin, RemoteInstanceOperation):
remote_queue = ('agent', )
remote_queue = ('agent',)
concurrency_check = False
......@@ -248,11 +245,10 @@ class AddInterfaceOperation(InstanceOperation):
@register_operation
class CreateDiskOperation(InstanceOperation):
id = 'create_disk'
name = _("create disk")
description = _("Create and attach empty disk to the virtual machine.")
required_perms = ('storage.create_empty_disk', )
required_perms = ('storage.create_empty_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, size, activity, name=None):
......@@ -285,13 +281,12 @@ class CreateDiskOperation(InstanceOperation):
@register_operation
class ResizeDiskOperation(RemoteInstanceOperation):
id = 'resize_disk'
name = _("resize disk")
description = _("Resize the virtual disk image. "
"Size must be greater value than the actual size.")
required_perms = ('storage.resize_disk', )
accept_states = ('RUNNING', )
required_perms = ('storage.resize_disk',)
accept_states = ('RUNNING',)
async_queue = "localhost.man.slow"
remote_queue = ('vm', 'slow')
task = vm_tasks.resize_disk
......@@ -324,13 +319,11 @@ class DownloadDiskOperation(InstanceOperation):
"machine.")
abortable = True
has_percentage = True
required_perms = ('storage.download_disk', )
required_perms = ('storage.download_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = "localhost.man.slow"
def _operation(self, user, url, task, activity, name=None):
from storage.models import Disk
disk = Disk.download(url=url, name=name, task=task)
devnums = list(ascii_lowercase)
for d in self.instance.disks.all():
......@@ -352,6 +345,57 @@ class DownloadDiskOperation(InstanceOperation):
@register_operation
class ImportDiskOperation(InstanceOperation):
id = 'import_disk'
name = _('import disk')
description = _('Import and attach a disk image to the virtual machine '
'from the user store. The disk image has to be in the '
'root directory of the store.')
abortable = True
has_percentage = True
required_perms = ('storage.import_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = 'localhost.man.slow'
def check_auth(self, user):
super(ImportDiskOperation, self).check_auth(user)
try:
Store(user)
except NoStoreException:
raise PermissionDenied
def _operation(self, user, name, disk_path, task):
store = Store(user)
download_link = store.request_download(disk_path)
disk = Disk.import_disk(user, name, download_link, task)
self.instance.disks.add(disk)
@register_operation
class ExportDiskOperation(InstanceOperation):
id = 'export_disk'
name = _('export disk')
description = _('Export disk to the selected format.')
abortable = True
has_percentage = True
required_perms = ('storage.export_disk',)
accept_states = ('STOPPED',)
async_queue = 'localhost.man.slow'
def check_auth(self, user):
super(ExportDiskOperation, self).check_auth(user)
try:
Store(user)
except NoStoreException:
raise PermissionDenied
def _operation(self, user, disk, exported_name, disk_format, task):
store = Store(user)
upload_link = store.request_upload('/')
disk.export(exported_name, disk_format, upload_link, task)
@register_operation
class DeployOperation(InstanceOperation):
id = 'deploy'
name = _("deploy")
......@@ -422,6 +466,7 @@ class DeployOperation(InstanceOperation):
remote_queue = ("vm", "slow")
task = vm_tasks.deploy
remote_timeout = 120
def _get_remote_args(self, **kwargs):
return [self.instance.get_vm_desc()]
# intentionally not calling super
......@@ -530,7 +575,7 @@ class MigrateOperation(RemoteInstanceOperation):
"keeping its full state.")
required_perms = ()
superuser_required = True
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
async_queue = "localhost.man.slow"
task = vm_tasks.migrate
remote_queue = ("vm", "slow")
......@@ -589,7 +634,7 @@ class RebootOperation(RemoteInstanceOperation):
description = _("Warm reboot virtual machine by sending Ctrl+Alt+Del "
"signal to its console.")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.reboot
def _operation(self, activity):
......@@ -630,7 +675,7 @@ class RemovePortOperation(InstanceOperation):
description = _("Close the specified port.")
concurrency_check = False
acl_level = "operator"
required_perms = ('vm.config_ports', )
required_perms = ('vm.config_ports',)
def _operation(self, activity, rule):
interface = rule.host.interface_set.get()
......@@ -649,7 +694,7 @@ class AddPortOperation(InstanceOperation):
description = _("Open the specified port.")
concurrency_check = False
acl_level = "operator"
required_perms = ('vm.config_ports', )
required_perms = ('vm.config_ports',)
def _operation(self, activity, host, proto, port):
if host.interface_set.get().instance != self.instance:
......@@ -690,7 +735,7 @@ class ResetOperation(RemoteInstanceOperation):
name = _("reset")
description = _("Cold reboot virtual machine (power cycle).")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.reset
def _operation(self, activity):
......@@ -710,7 +755,7 @@ class SaveAsTemplateOperation(InstanceOperation):
"start an instance of it.")
has_percentage = True
abortable = True
required_perms = ('vm.create_template', )
required_perms = ('vm.create_template',)
accept_states = ('RUNNING', 'STOPPED')
async_queue = "localhost.man.slow"
......@@ -822,7 +867,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin,
"turn itself off in a period.")
abortable = True
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
resultant_state = 'STOPPED'
task = vm_tasks.shutdown
remote_queue = ("vm", "slow")
......@@ -883,7 +928,7 @@ class SleepOperation(InstanceOperation):
"resumed. In the meantime, the machine will only use "
"storage resources, and keep network resources allocated.")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
resultant_state = 'SUSPENDED'
async_queue = "localhost.man.slow"
......@@ -927,7 +972,7 @@ class WakeUpOperation(InstanceOperation):
"load the saved memory of the system and start the "
"virtual machine from this state.")
required_perms = ()
accept_states = ('SUSPENDED', )
accept_states = ('SUSPENDED',)
resultant_state = 'RUNNING'
async_queue = "localhost.man.slow"
......@@ -1039,7 +1084,7 @@ class ChangeStateOperation(InstanceOperation):
"redeployed without losing its storage and network "
"resources.")
acl_level = "owner"
required_perms = ('vm.emergency_change_state', )
required_perms = ('vm.emergency_change_state',)
concurrency_check = False
def _operation(self, user, activity, new_state="NOSTATE", interrupt=False,
......@@ -1066,7 +1111,7 @@ class RedeployOperation(InstanceOperation):
"and redeploy the VM. This operation allows starting "
"machines formerly running on a failed node.")
acl_level = "owner"
required_perms = ('vm.redeploy', )
required_perms = ('vm.redeploy',)
concurrency_check = False
def _operation(self, user, activity, with_emergency_change_state=True):
......@@ -1320,7 +1365,7 @@ class ScreenshotOperation(RemoteInstanceOperation):
"screensaver.")
acl_level = "owner"
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.screenshot
......@@ -1332,8 +1377,8 @@ class RecoverOperation(InstanceOperation):
"state. Network resources (allocations) are already lost, "
"so you will have to manually add interfaces afterwards.")
acl_level = "owner"
required_perms = ('vm.recover', )
accept_states = ('DESTROYED', )
required_perms = ('vm.recover',)
accept_states = ('DESTROYED',)
resultant_state = 'PENDING'
def check_precond(self):
......@@ -1371,7 +1416,7 @@ class ResourcesOperation(InstanceOperation):
name = _("resources change")
description = _("Change resources of a stopped virtual machine.")
acl_level = "owner"
required_perms = ('vm.change_resources', )
required_perms = ('vm.change_resources',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, activity,
......@@ -1677,7 +1722,7 @@ class UpdateAgentOperation(RemoteAgentOperation):
index = 0
filename = settings.AGENT_VERSION + ".tar"
while True:
chunk = data[index:index+chunk_size]
chunk = data[index:index + chunk_size]
if chunk:
agent_tasks.append.apply_async(
queue=queue,
......
circle/vm/tasks/local_periodic_tasks.py
......@@ -17,7 +17,9 @@
import logging
from django.utils import timezone
from datetime import timedelta
from django.utils.translation import ugettext_noop
from django.conf import settings
from manager.mancelery import celery
from vm.models import Node, Instance
......@@ -33,7 +35,7 @@ def update_domain_states():
@celery.task(ignore_result=True)
def garbage_collector(timeout=15):
def garbage_collector(offset=timezone.timedelta(seconds=20)):
"""Garbage collector for instances.
Suspends and destroys expired instances.
......@@ -81,4 +83,44 @@ def garbage_collector(timeout=15):
logger.debug("Instance %d expires soon." % i.pk)
i.notify_owners_about_expiration()
else:
logger.debug("Instance %d didn't expire." % i.pk)
logger.debug("Instance %d didn't expire. bw:%d", i.pk, bw)
@celery.task(ignore_result=True)
def auto_migrate():
"""Auto migration task for runtime scaling
"""
time_limit = settings.AUTO_MIGRATION_TIME_LIMIT_IN_HOURS
available_time = timedelta(hours=int(time_limit))
deadline = timezone.now() + available_time
while timezone.now() < deadline:
migrate_one()
def migrate_one():
"""Migrate a VM syncronously.
The target node chosen by the scheduler.
"""
nodes = [n for n in Node.objects.filter(enabled=True) if n.online]
node_max_cpu = max(nodes, key=lambda x: x.cpu_usage / x.cpu_weight)
node_max_ram = max(nodes, key=lambda x: x.ram_usage / x.ram_weight)
if node_max_cpu.cpu_usage > node_max_ram.ram_usage:
try:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=lambda x: x.cpu_usage())
instance_to_migrate.migrate(system=True)
except Instance.MonitorUnavailableException:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=(lambda x: x.get_vm_desc()["vcpu"] *
x.get_vm_desc()["cpu_share"]))
instance_to_migrate.migrate(system=True)
else:
try:
instance_to_migrate = max(Instance.objects.filter(node=node_max_ram.pk),
key=lambda x: x.ram_usage())
instance_to_migrate.migrate(system=True)
except Instance.MonitorUnavailableException:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=lambda x: x.get_vm_desc()["memory"])
instance_to_migrate.migrate(system=True)
requirements/base.txt
cryptography==2.0
cryptography==2.7
amqp==1.4.7
anyjson==0.3.3
arrow==0.7.0
billiard==3.3.0.20
bpython==0.14.1
celery==3.1.18
Django==1.11.6
Django==1.11.25
django-appconf==1.0.2
django-autocomplete-light==3.2.9
django-braces==1.11.0
......@@ -16,7 +16,7 @@ django-sizefield==0.9.1
django-statici18n==1.4.0
django-tables2==1.10.0
django-taggit==0.22.1
djangosaml2==0.16.10
djangosaml2==0.17.1
git+https://git.ik.bme.hu/circle/django-sshkey.git
docutils==0.12
Jinja2==2.7.3
......@@ -26,9 +26,9 @@ logutils==0.3.3
MarkupSafe==0.23
netaddr==0.7.14
pip-tools==0.3.6
psycopg2==2.6
psycopg2==2.8.3
Pygments==2.0.2
pylibmc==1.4.3
pylibmc==1.6.0
python-dateutil==2.4.2
pyinotify==0.9.5
pyotp==2.1.1
......
requirements/production.txt
# Pro-tip: Try not to put anything here. There should be no dependency in
# production that isn't in development.
-r base.txt
uWSGI==2.0.13.1
uWSGI==2.0.18
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment