...
 
Commits (108)
......@@ -8,6 +8,8 @@
*.swp
*.swo
*~
.vscode
.idea
# Sphinx docs:
build
......
......@@ -495,6 +495,7 @@ if get_env_variable('DJANGO_SAML', 'FALSE') == 'TRUE':
},
'required_attributes': required_attrs,
'optional_attributes': optional_attrs,
'want_response_signed': False,
},
},
'metadata': {'local': [remote_metadata], },
......@@ -576,7 +577,7 @@ SESSION_COOKIE_NAME = "csessid%x" % (((getnode() // 139) ^
MAX_NODE_RAM = get_env_variable("MAX_NODE_RAM", 1024)
MAX_NODE_CPU_CORE = get_env_variable("MAX_NODE_CPU_CORE", 10)
SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'random')
SCHEDULER_METHOD = get_env_variable("SCHEDULER_METHOD", 'advanced')
# Url to download the client: (e.g. http://circlecloud.org/client/download/)
CLIENT_DOWNLOAD_URL = get_env_variable('CLIENT_DOWNLOAD_URL', 'http://circlecloud.org/client/download/')
......@@ -590,3 +591,12 @@ REQUEST_HOOK_URL = get_env_variable("REQUEST_HOOK_URL", "")
SSHKEY_EMAIL_ADD_KEY = False
TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
# Default value is every day at midnight
AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *")
AUTO_MIGRATION_TIME_LIMIT_IN_HOURS = (
get_env_variable("AUTO_MIGRATION_TIME_LIMIT_IN_HOURS", "2"))
# Maximum time difference until the monitor's values get valid
SCHEDULER_TIME_SENSITIVITY_IN_SECONDS = (
get_env_variable("SCHEDULER_TIME_SENSITIVITY_IN_SECONDS", "60"))
......@@ -65,7 +65,10 @@
"modified": "2014-02-19T21:11:34.671Z",
"priority": 1,
"traits": [],
"host": 1
"host": 1,
"ram_weight": 1.0,
"cpu_weight": 1.0,
"time_stamp": "2017-12-13T21:08:08.819Z"
}
}
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
import sizefield.models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0006_auto_20170707_1909'),
]
operations = [
migrations.AddField(
model_name='groupprofile',
name='disk_quota',
field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-11-06 13:33
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0007_groupprofile_disk_quota'),
]
operations = [
migrations.AddField(
model_name='profile',
name='template_instance_limit',
field=models.IntegerField(default=1),
),
]
......@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.models.instance import ACCESS_METHODS
from .store_api import Store, NoStoreException, NotOkException, Timeout
from .store_api import Store, NoStoreException, NotOkException
from .validators import connect_command_template_validator
logger = getLogger(__name__)
......@@ -162,7 +162,7 @@ class ConnectCommand(Model):
validators=[connect_command_template_validator])
class Meta:
ordering = ('id', )
ordering = ('id',)
def __unicode__(self):
return self.template
......@@ -178,6 +178,7 @@ class Profile(Model):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the person, e.g. a student number.'))
instance_limit = IntegerField(default=5)
template_instance_limit = IntegerField(default=1)
use_gravatar = BooleanField(
verbose_name=_("Use Gravatar"), default=True,
help_text=_("Whether to use email address as Gravatar profile image"))
......@@ -218,7 +219,7 @@ class Profile(Model):
'id': command.id,
'cmd': command.template % {
'port': instance.get_connect_port(use_ipv6=use_ipv6),
'host': instance.get_connect_host(use_ipv6=use_ipv6),
'host': instance.get_connect_host(use_ipv6=use_ipv6),
'password': instance.pw,
'username': 'cloud',
}} for command in commands]
......@@ -263,7 +264,7 @@ class Profile(Model):
super(Profile, self).save(*args, **kwargs)
class Meta:
ordering = ('id', )
ordering = ('id',)
permissions = (
('use_autocomplete', _('Can use autocomplete.')),
)
......@@ -275,7 +276,7 @@ class FutureMember(Model):
group = ForeignKey(Group)
class Meta:
ordering = ('id', )
ordering = ('id',)
unique_together = ('org_id', 'group')
def __unicode__(self):
......@@ -293,9 +294,13 @@ class GroupProfile(AclBase):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the group at the organization.'))
description = TextField(blank=True)
disk_quota = FileSizeField(
verbose_name=_('disk quota'),
default=2048 * 1024 * 1024,
help_text=_('Disk quota in mebibytes.'))
class Meta:
ordering = ('id', )
ordering = ('id',)
def __unicode__(self):
return self.group.name
......@@ -331,7 +336,11 @@ def create_profile(user):
profile, created = Profile.objects.get_or_create(user=user)
try:
Store(user).create_user(profile.smb_password, None, profile.disk_quota)
store = Store(user)
quotas = [profile.disk_quota]
quotas += [group.profile.disk_quota for group in user.groups.all()]
max_quota = max(quotas)
store.create_user(profile.smb_password, None, max_quota)
except:
logger.exception("Can't create user %s", unicode(user))
return created
......@@ -347,6 +356,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
logger.debug("Register save_org_id to djangosaml2 pre_user_save")
from djangosaml2.signals import pre_user_save
def save_org_id(sender, instance, attributes, **kwargs):
logger.debug("save_org_id called by %s", instance.username)
atr = settings.SAML_ORG_ID_ATTRIBUTE
......@@ -399,6 +409,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
return False # User did not change
pre_user_save.connect(save_org_id)
......@@ -411,7 +422,7 @@ def update_store_profile(sender, **kwargs):
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except (NotOkException, Timeout):
except NotOkException:
logger.critical("Store is not accepting connections.")
......
......@@ -41,12 +41,14 @@ $(function() {
$('#confirmation-modal').on('hidden.bs.modal', function() {
$('#confirmation-modal').remove();
});
$('#vm-migrate-node-list li input:checked').closest('li').addClass('panel-primary');
}
});
e.preventDefault();
});
/* if the operation fails show the modal again */
$("body").on("click", "#confirmation-modal #op-form-send", function() {
var url = $(this).closest("form").prop("action");
......@@ -237,4 +239,3 @@ String.prototype.hashCode = function() {
}
return hash;
};
......@@ -558,3 +558,5 @@ $(function() {
inputs.prop("checked", !inputs.prop("checked"));
});
});
$.fn.modal.Constructor.prototype.enforceFocus = function() {};
......@@ -1079,6 +1079,10 @@ textarea[name="new_members"] {
max-width: 100%;
}
#node-list-auto-migration-body {
padding: 20px;
}
#vm-list-table td.state,
#vm-list-table td.memory {
white-space: nowrap;
......@@ -1088,7 +1092,7 @@ textarea[name="new_members"] {
vertical-align: middle;
}
.disk-resize-btn {
.disk-resize-btn, .disk-export-btn {
margin-right: 5px;
}
......
......@@ -3,4 +3,11 @@ $(function() {
// find disabled nodes, set danger (red) on the rows
$('.node-disabled').closest("tr").addClass('danger');
});
$('#reschedule-now').click(function() {
$.get($(this).attr('href'), function(data){
highlight = data.result === 'ok' ? 'success' : 'danger';
addMessage(data.message, highlight);
});
return false;
});
});
......@@ -14,19 +14,20 @@
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from os.path import splitext
import json
import logging
from urlparse import urljoin
from datetime import datetime
from django.http import Http404
import os
from datetime import datetime
from django.conf import settings
from django.http import Http404
from os.path import splitext
from requests import get, post, codes
from requests.exceptions import Timeout # noqa
from sizefield.utils import filesizeformat
from storage.models import Disk
logger = logging.getLogger(__name__)
......@@ -47,6 +48,17 @@ class NoStoreException(StoreApiException):
class Store(object):
def __init__(self, user, default_timeout=0.5):
self.store_url = settings.STORE_URL
if not self.store_url:
raise NoStoreException
if user.is_superuser and not user.profile.org_id:
self.username = 'u-admin'
elif not user.profile.org_id:
raise NoStoreException
else:
self.username = 'u-%s' % user.profile.org_id
self.request_args = {'verify': settings.STORE_VERIFY_SSL}
if settings.STORE_SSL_AUTH:
self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
......@@ -54,18 +66,15 @@ class Store(object):
if settings.STORE_BASIC_AUTH:
self.request_args['auth'] = (settings.STORE_CLIENT_USER,
settings.STORE_CLIENT_PASSWORD)
self.username = "u-%d" % user.pk
self.default_timeout = default_timeout
self.store_url = settings.STORE_URL
if not self.store_url:
raise NoStoreException
def _request(self, url, method=get, timeout=None,
raise_status_code=True, **kwargs):
url = urljoin(self.store_url, url)
if timeout is None:
timeout = self.default_timeout
payload = json.dumps(kwargs) if kwargs else None
kwargs['USER'] = self.username
payload = json.dumps(kwargs)
try:
headers = {'content-type': 'application/json'}
response = method(url, data=payload, headers=headers,
......@@ -83,7 +92,7 @@ class Store(object):
return response
def _request_cmd(self, cmd, **kwargs):
return self._request(self.username, post, CMD=cmd, **kwargs)
return self._request("/user/", post, CMD=cmd, **kwargs)
def list(self, path, process=True):
r = self._request_cmd("LIST", PATH=path)
......@@ -101,13 +110,22 @@ class Store(object):
else:
return result
def get_disk_images(self, path='/'):
images = []
file_list = self.list(path, process=False)
export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
for item in file_list:
if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
images.append(os.path.join(path, item['NAME']))
return images
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
return r.json()['LINK']
def request_upload(self, path):
r = self._request_cmd("UPLOAD", PATH=path)
return r.json()['LINK']
r = self._request_cmd("UPLOAD", PATH=path)
return r.json()['LINK']
def remove(self, path):
self._request_cmd("REMOVE", PATH=path)
......@@ -119,7 +137,7 @@ class Store(object):
self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
def get_quota(self): # no CMD? :o
r = self._request(self.username)
r = self._request("/user/")
quota = r.json()
quota.update({
'readable_used': filesizeformat(float(quota['used'])),
......@@ -129,17 +147,17 @@ class Store(object):
return quota
def set_quota(self, quota):
self._request("/quota/" + self.username, post, QUOTA=quota)
self._request("/quota/", post, QUOTA=quota)
def user_exist(self):
try:
self._request(self.username)
self._request("/user/")
return True
except NotOkException:
return False
def create_user(self, password, keys, quota):
self._request("/new/" + self.username, method=post,
self._request("/new/", method=post,
SMBPASSWD=password, KEYS=keys, QUOTA=quota)
@staticmethod
......
......@@ -6,15 +6,29 @@
<span class="operation-wrapper pull-right">
{% if d.is_exportable %}
{% if op.export_disk %}
<a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
{% if op.export_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
</a>
{% endif %}
{% else %}
<small class="btn-xs">
{% trans "Not exportable" %}
</small>
{% endif %}
{% if d.is_resizable %}
{% if op.resize_disk %}
<a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
<a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
{% if op.resize_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
</a>
{% else %}
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
class="btn btn-xs btn-primary operation">
<i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
</a>
{% endif %}
......@@ -24,8 +38,8 @@
</small>
{% endif %}
{% if op.remove_disk %}
<a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
<a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
{% if op.remove_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
</a>
......
......@@ -41,4 +41,23 @@
</div><!-- -col-md-12 -->
</div><!-- .row -->
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<a id="reschedule-now" class="btn btn-danger pull-right" href="{% url "dashboard.views.reschedule" %}">
<i class="fa fa-magic"></i> {% trans "Reschedule now" %}
</a>
<h3 class="no-margin"><i class="fa fa-truck"></i> {% trans "Virtual machine auto migration" %}</h3>
</div>
<div id="node-list-auto-migration-body">
<h1>Crontab</h1>
<form>
{{ auto_migration_form.as_p }}
</form>
</div>
</div>
</div><!-- -col-md-12 -->
</div><!-- .row -->
{% endblock %}
......@@ -155,7 +155,7 @@
{% for d in disks %}
<li>
<i class="fa fa-file"></i>
{{ d.name }} (#{{ d.id }})
{{ d.filename }} (#{{ d.id }})
<a href="{% url "dashboard.views.disk-remove" pk=d.pk %}?next={{ request.path }}"
data-disk-pk="{{ d.pk }}" class="btn btn-xs btn-danger pull-right disk-remove"
{% if not long_remove %}title="{% trans "Remove" %}"{% endif %}>
......
{% load i18n %}
{% for op in ops %}
{% if op.is_disk_operation %}
<a href="{{op.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.op}}">
<i class="fa fa-{{op.icon}} fa-fw-12"></i>
{{op.name}} </a>
{% endif %}
{% if op.is_disk_operation %}
<a href="{{ op.get_url }}" class="btn btn-success btn-xs
operation operation-{{ op.op }}">
<i class="fa fa-{{ op.icon }} fa-fw-12"></i>
{{ op.name }} </a>
{% endif %}
{% endfor %}
......@@ -37,6 +37,9 @@
<dl>
<dt>{% trans "IPv4 address" %}:</dt> <dd>{{ i.host.ipv4 }}</dd>
<dt>{% trans "IPv6 address" %}:</dt> <dd>{{ i.host.ipv6 }}</dd>
{% if request.user.is_superuser %}
<dt>{% trans "MAC address" %}:</dt> <dd>{{ i.host.mac }}</dd>
{% endif %}
<dt>{% trans "DNS name" %}:</dt> <dd>{{ i.host.get_fqdn }}</dd>
<dt>{% trans "Groups" %}:</dt>
<dd>
......@@ -114,7 +117,9 @@
{% if l.ipv6 %}
<tr>
<td>
{% display_portforward6 l %}
{% autoescape off %}
{% display_portforward6 l %}
{% endautoescape %}
</td>
<td><i class="fa fa-long-arrow-right"></i></td>
<td>
......
......@@ -534,7 +534,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
with patch.object(DeployOperation, 'async') as async:
response = c.post("/dashboard/vm/create/", {
'name': 'vm',
'amount': 2,
'amount': 1,
'customized': 1,
'template': 1,
'cpu_priority': 10, 'cpu_count': 1, 'ram_size': 128,
......@@ -543,7 +543,7 @@ class VmDetailTest(LoginMixin, MockCeleryMixin, TestCase):
assert async.called
self.assertEqual(response.status_code, 302)
self.assertEqual(instance_count + 2, Instance.objects.all().count())
self.assertEqual(instance_count + 1, Instance.objects.all().count())
def test_unpermitted_description_update(self):
c = Client()
......
......@@ -56,6 +56,7 @@ from .views import (
MessageList, MessageDetail, MessageCreate, MessageDelete,
EnableTwoFactorView, DisableTwoFactorView,
AclUserGroupAutocomplete, AclUserAutocomplete,
RescheduleView,
)
from .views.vm import vm_ops, vm_mass_ops
from .views.node import node_ops
......@@ -153,6 +154,8 @@ urlpatterns = [
r'(?P<time>[0-9]{1,2}[hdwy])$'),
NodeListGraphView.as_view(),
name='dashboard.views.node-list-graph'),
url(r'^node/reschedule/$', RescheduleView.as_view(),
name="dashboard.views.reschedule"),
url((r'^template/(?P<pk>\d+)/graph/(?P<metric>[a-z]+)/'
r'(?P<time>[0-9]{1,2}[hdwy])$'),
TemplateGraphView.as_view(),
......
......@@ -25,7 +25,7 @@ from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse_lazy
from django.db.models import Count
from django.forms.models import inlineformset_factory
from django.http import HttpResponse
from django.http import HttpResponse, JsonResponse
from django.shortcuts import redirect
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
......@@ -37,11 +37,14 @@ from django_tables2 import SingleTableView
from firewall.models import Host
from vm.models import Node, NodeActivity, Trait
from vm.tasks.vm_tasks import check_queue
from vm.tasks.local_periodic_tasks import auto_migrate
from ..forms import TraitForm, HostForm, NodeForm
from ..forms import TraitForm, HostForm, NodeForm, AutoMigrationForm
from ..tables import NodeListTable
from .util import AjaxOperationMixin, OperationView, GraphMixin, DeleteViewBase
from manager.mancelery import crontab_parser
def get_operations(instance, user):
ops = []
......@@ -190,6 +193,14 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
table_class = NodeListTable
table_pagination = False
def get_crontab(self):
return crontab_parser(settings.AUTO_MIGRATION_CRONTAB)
def get_context_data(self):
context = super(NodeList, self).get_context_data()
context["auto_migration_form"] = AutoMigrationForm(self.get_crontab())
return context
def get(self, *args, **kwargs):
if not self.request.user.has_perm('vm.view_statistics'):
raise PermissionDenied()
......@@ -210,9 +221,20 @@ class NodeList(LoginRequiredMixin, GraphMixin, SingleTableView):
return super(NodeList, self).get(*args, **kwargs)
def get_queryset(self):
self.wrong_nodes_message()
return Node.objects.annotate(
number_of_VMs=Count('instance_set')).select_related('host')
def wrong_nodes_message(self):
wrong_nodes = []
for node in Node.objects.all():
if node.monitor_info is None:
wrong_nodes.append(node.name)
message = ', '.join(wrong_nodes)
if wrong_nodes:
messages.error(self.request,
"Can't reach " + message + " monitor info")
class NodeCreate(LoginRequiredMixin, SuperuserRequiredMixin, TemplateView):
......@@ -356,3 +378,23 @@ class NodeActivityDetail(LoginRequiredMixin, SuperuserRequiredMixin,
).order_by('-started').select_related())
ctx['icon'] = _get_activity_icon(self.object)
return ctx
class RescheduleView(SuperuserRequiredMixin, View):
def get(self, *args, **kwargs):
try:
auto_migrate.apply_async(queue='localhost.man.slow')
except Exception as e:
msg = str(e)
result = 'error'
else:
result = 'ok'
msg = _('Reschedule has started.')
if self.request.is_ajax():
return JsonResponse({'result': result, 'message': msg})
else:
if result == 'ok':
messages.success(self.request, msg)
else:
messages.error(self.request, msg)
return redirect('dashboard.views.node-list')
......@@ -35,7 +35,8 @@ from django.views.generic import TemplateView
from braces.views import LoginRequiredMixin
from ..store_api import Store, NoStoreException, NotOkException
from ..store_api import (Store, NoStoreException,
NotOkException)
logger = logging.getLogger(__name__)
......
......@@ -682,7 +682,7 @@ class TransferOwnershipConfirmView(LoginRequiredMixin, View):
messages.error(request, _('This token is invalid or has expired.'))
raise PermissionDenied()
return render(request, self.template,
dictionary={'instance': instance, 'key': key})
{'instance': instance, 'key': key})
def change_owner(self, instance, new_owner):
instance.owner = new_owner
......
......@@ -61,9 +61,10 @@ from .util import (
)
from ..forms import (
AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
VmImportDiskForm, VmSaveForm,
VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm,
......@@ -166,8 +167,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
# resources forms
can_edit = (
instance.has_level(user, "owner") and
self.request.user.has_perm("vm.change_resources"))
instance.has_level(user, "owner") and
self.request.user.has_perm("vm.change_resources"))
context['resources_form'] = VmResourcesForm(
can_edit=can_edit, instance=instance)
......@@ -269,7 +270,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
return JsonResponse({'message': message})
else:
return redirect(reverse_lazy("dashboard.views.detail",
kwargs={'pk': self.object.pk}))
kwargs={'pk': self.object.pk}))
def __abort_operation(self, request):
self.object = self.get_object()
......@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance
context_object_name = 'instance' # much simpler to mock object
......@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
class VmAddInterfaceView(FormOperationMixin, VmOperationView):
op = 'add_interface'
form_class = VmAddInterfaceForm
show_in_toolbar = False
......@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
class VmCreateDiskView(FormOperationMixin, VmOperationView):
op = 'create_disk'
form_class = VmCreateDiskForm
show_in_toolbar = False
......@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmImportDiskView(FormOperationMixin, VmOperationView):
op = 'import_disk'
form_class = VmImportDiskForm
show_in_toolbar = False
icon = 'upload'
effect = "success"
is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
val = super(VmImportDiskView, self).get_form_kwargs()
val.update({'user': self.request.user})
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
op = 'download_disk'
form_class = VmDownloadDiskForm
show_in_toolbar = False
......@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmMigrateView(FormOperationMixin, VmOperationView):
op = 'migrate'
icon = 'truck'
effect = 'info'
......@@ -449,8 +460,7 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
if isinstance(inst, Instance):
nodes_w_traits = [
n.pk for n in Node.objects.filter(enabled=True)
if n.online and
has_traits(inst.req_traits.all(), n)
if n.online and has_traits(inst.req_traits.all(), n)
]
ctx['nodes_w_traits'] = nodes_w_traits
......@@ -458,7 +468,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
class VmPortRemoveView(FormOperationMixin, VmOperationView):
template_name = 'dashboard/_vm-remove-port.html'
op = 'remove_port'
show_in_toolbar = False
......@@ -487,7 +496,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
class VmPortAddView(FormOperationMixin, VmOperationView):
op = 'add_port'
show_in_toolbar = False
with_reload = True
......@@ -514,7 +522,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
class VmSaveView(FormOperationMixin, VmOperationView):
op = 'save_as_template'
icon = 'save'
effect = 'info'
......@@ -570,7 +577,7 @@ class TokenOperationView(OperationView):
User can do the action with a valid token instead of logging in.
"""
token_max_age = 3 * 24 * 3600
redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
@classmethod
def get_salt(cls):
......@@ -642,7 +649,6 @@ class TokenOperationView(OperationView):
class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
op = 'renew'
icon = 'calendar'
effect = 'success'
......@@ -769,7 +775,11 @@ vm_ops = OrderedDict([
extra_bases=[TokenOperationView],
op='destroy', icon='times', effect='danger')),
('create_disk', VmCreateDiskView),
('import_disk', VmImportDiskView),
('download_disk', VmDownloadDiskView),
('export_disk', VmDiskModifyView.factory(
op='export_disk', form_class=VmDiskExportForm,
icon='download', effect='info')),
('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")),
......@@ -1014,7 +1024,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
# remove "-" that means descending order
# also check if the column name is valid
if (sort and
(sort[1:] if sort[0] == "-" else sort)
(sort[1:] if sort[0] == "-" else sort)
in [i.name for i in Instance._meta.fields] + ["pk"]):
queryset = queryset.order_by(sort)
......@@ -1025,7 +1035,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
class VmCreate(LoginRequiredMixin, TemplateView):
form_class = VmCustomizeForm
form = None
......@@ -1134,7 +1143,7 @@ class VmCreate(LoginRequiredMixin, TemplateView):
messages.success(request, ungettext_lazy(
"Successfully created %(count)d VM.", # this should not happen
"Successfully created %(count)d VMs.", len(instances)) % {
'count': len(instances)})
'count': len(instances)})
path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
else:
messages.success(request, _("VM successfully created."))
......@@ -1156,24 +1165,28 @@ class VmCreate(LoginRequiredMixin, TemplateView):
# limit chekcs
try:
limit = user.profile.instance_limit
instance_limit = user.profile.instance_limit
template_instance_limit = user.profile.template_instance_limit
except Exception as e:
logger.debug('No profile or instance limit: %s', e)
else:
try:
amount = int(request.POST.get("amount", 1))
except:
amount = limit # TODO this should definitely use a Form
current = Instance.active.filter(owner=user).count()
logger.debug('current use: %d, limit: %d', current, limit)
if current + amount > limit:
messages.error(request,
_('Instance limit (%d) exceeded.') % limit)
if request.is_ajax():
return HttpResponse(json.dumps({'redirect': '/'}),
content_type="application/json")
else:
return redirect('/')
# TODO this should definitely use a Form
amount = instance_limit
instances = Instance.active.filter(owner=user).count()
template_instances = template.get_user_instances(user).count()
logger.debug('current instance use: %d, limit: %d',
instances, instance_limit)
logger.debug('current template instance use: %d, limit: %d',
template_instances, template_instance_limit)
if instances + amount > instance_limit:
return self._limit_exceeded(instance_limit, request)
if template_instances + amount > template_instance_limit:
return self._limit_exceeded(template_instance_limit, request)
create_func = (self.__create_normal if
request.POST.get("customized") is None else
......@@ -1181,6 +1194,16 @@ class VmCreate(LoginRequiredMixin, TemplateView):
return create_func(request, template, *args, **kwargs)
def _limit_exceeded(self, limit, request):
messages.error(request,
_('Instance limit (%d) exceeded.')
% limit)
if request.is_ajax():
return HttpResponse(json.dumps({'redirect': '/'}),
content_type="application/json")
else:
return redirect('/')
@require_GET
def get_vm_screenshot(request, pk):
......@@ -1326,7 +1349,7 @@ class TransferInstanceOwnershipConfirmView(TransferOwnershipConfirmView):
def change_owner(self, instance, new_owner):
with instance.activity(
code_suffix='ownership-transferred',
code_suffix='ownership-transferred',
readable_name=ugettext_noop("transfer ownership"),
concurrency_check=False, user=new_owner):
super(TransferInstanceOwnershipConfirmView, self).change_owner(
......
......@@ -17,13 +17,27 @@
from celery import Celery
from celery.signals import worker_ready
from celery.schedules import crontab
from datetime import timedelta
from celery.schedules import crontab
from kombu import Queue, Exchange
from os import getenv
HOSTNAME = "localhost"
QUEUE_NAME = HOSTNAME + '.man'
AUTO_MIGRATION_CRONTAB = getenv('AUTO_MIGRATION_CRONTAB', '0 0 * * *')
def crontab_parser(crontab):
fields = crontab.split(' ')
return dict(
minute=fields[0],
hour=fields[1],
day_of_month=fields[2],
month_of_year=fields[3],
day_of_week=fields[4],
)
celery = Celery('manager',
......@@ -56,6 +70,11 @@ celery.conf.update(
'schedule': crontab(minute=10, hour=1),
'options': {'queue': 'localhost.man'}
},
# 'vm.local_periodic_tasks': {
# 'task': 'vm.tasks.local_periodic_tasks.auto_migrate',
# 'schedule': crontab(**crontab_parser(AUTO_MIGRATION_CRONTAB)),
# 'options': {'queue': 'localhost.man.slow'},
# },
}
)
......
......@@ -15,15 +15,18 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
import datetime
import json
import random
from logging import getLogger
from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
from django.utils.translation import ugettext_noop
from common.models import HumanReadableException
from circle.settings.base import SCHEDULER_METHOD
import random
from common.models import HumanReadableException
logger = getLogger(__name__)
......@@ -69,14 +72,14 @@ def common_select(instance, nodes):
logger.warning('select_node: no enough RAM for %s', unicode(instance))
raise NotEnoughMemoryException()
# sort nodes first by processor usage, then priority
# sort nodes first by priority
nodes.sort(key=lambda n: n.priority, reverse=True)
nodes.sort(key=free_cpu_time, reverse=True)
return nodes
def common_evenly(instance, nodes):
nodes = common_select(instance, nodes)
nodes.sort(key=free_cpu_time, reverse=True)
result = nodes[0]
return result
......@@ -87,6 +90,16 @@ def common_random(instance, nodes):
return result
def advanced_with_time_stamp(instance, nodes):
nodes = common_select(instance, nodes)
nodes.sort(key=sorting_key, reverse=True)
logger.info("SCHEDLOG: {}".format(json.dumps({
"event": "after_sort",
"list": map(lambda node: unicode(node), nodes)})))
result = nodes[0]
return result
def select_node(instance, nodes):
''' Select a node for hosting an instance based on its requirements.
'''
......@@ -94,14 +107,72 @@ def select_node(instance, nodes):
result = common_evenly(instance, nodes)
elif SCHEDULER_METHOD == 'random':
result = common_random(instance, nodes)
elif SCHEDULER_METHOD == 'advanced':
result = advanced_with_time_stamp(instance, nodes)
else: # Default method is the random
result = common_random(instance, nodes)
logger.info('Scheduler method: %s selected', unicode(SCHEDULER_METHOD))
logger.info('select_node: %s for %s', unicode(result), unicode(instance))
logger.info("SCHEDLOG: {}".format(json.dumps(
{"event": "select",
"node": unicode(result),
"vm": unicode(instance)})))
set_time_stamp(result)
return result
def sorting_key(node):
"""Determines how valuable a node is for scheduling.
"""
key = 0
corr = last_scheduled_correction_factor(node)
if free_cpu_time(node) < free_ram(node):
key = free_cpu_time(node) * corr
else:
key = free_ram(node) * corr
logger.info("SCHEDLOG: {}".format(json.dumps({
"event": "sort",
"node": unicode(node),
"sorting_key": unicode(key),
"free_cpu_time": unicode(free_cpu_time(node)),
"free_ram": unicode(free_ram(node)),
"last_scheduled_correction_factor": unicode(last_scheduled_correction_factor(node))})))
return key
def set_time_stamp(node):
cache.set('time_stamp{}'.format(node.id), timezone.now())
def get_time_stamp(node):
time_stamp = cache.get('time_stamp{}'.format(node.id))
if time_stamp:
return time_stamp
return datetime.datetime(1970, 1, 1, tzinfo=timezone.get_current_timezone())
def last_scheduled_correction_factor(node):
"""Returns the time correction factor for a node.
The monitor data may be outdated, because of recent scheduling for a given node.
The return value is between 0 and 1, higher value indicates more time since the
last scheduling for the given node.
"""
factor = 0
max_time_diff = settings.SCHEDULER_TIME_SENSITIVITY_IN_SECONDS
current_time = timezone.now()
time_difference_in_seconds = (
current_time - get_time_stamp(node)).total_seconds()
factor = time_difference_in_seconds/float(max_time_diff)
if factor > 1:
factor = 1
elif factor < 0:
factor = 1
logger.info('Scheduler set factor to %s', unicode(factor))
return factor
def has_traits(traits, node):
"""True, if the node has all specified traits; otherwise, false.
"""
......@@ -142,11 +213,27 @@ def free_cpu_time(node):
Higher values indicate more idle time.
"""
try:
activity = node.cpu_usage / 100
inactivity = 1 - activity
cores = node.num_cores
return cores * inactivity
free_cpu_percent = 1 - node.cpu_usage
weight = node.cpu_weight
weighted_value = free_cpu_percent * weight
return weighted_value
except TypeError as e:
logger.exception('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return 0 # will result lowest priority
def free_ram(node):
"""Get an indicator number for free RAM on the node.
Higher value indicates more RAM.
"""
try:
free_ram_percent = 1 - node.ram_usage
weight = node.ram_weight
weighted_value = free_ram_percent * weight
return weighted_value
except TypeError as e:
logger.warning('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return False # monitoring data is incorrect
logger.exception('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return 0 # will result lowest priority
......@@ -16,7 +16,7 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from django.forms import (
ModelForm, ModelChoiceField, ChoiceField, Form, CharField, RadioSelect,
Textarea, ValidationError
Textarea, ValidationError, TextInput, IntegerField, EmailField
)
from django.utils.translation import ugettext_lazy as _
from django.template.loader import render_to_string
......@@ -27,11 +27,78 @@ from crispy_forms.helper import FormHelper
from crispy_forms.layout import Submit
from request.models import (
LeaseType, TemplateAccessType, TemplateAccessAction,
LeaseType, TemplateAccessType, TemplateAccessAction, RequestField
)
from dashboard.forms import VmResourcesForm
class RequestFieldModelForm(ModelForm):
class Meta:
model = RequestField
fields = '__all__'
widgets = {
'choices': TextInput(attrs={'placeholder': 'Optional'}),
}
def __init__(self, *args, **kwargs):
super(RequestFieldModelForm, self).__init__(*args, **kwargs)
self.fields['choices'].required = False
@property
def helper(self):
helper = FormHelper()
return helper
def clean(self):
cleaned_data = super(RequestFieldModelForm, self).clean()
if cleaned_data['type'] == 'Email' and cleaned_data['choices']:
raise ValidationError(_("Email field can't have choices!"))
if cleaned_data['type'] == 'Integer':
for choice in cleaned_data['choices'].split(','):
try:
int(choice)
except:
raise ValidationError(_("IntegerField choices must be \
integers"))
class EditableForm(Form):
def __init__(self, *args, **kwargs):
type = kwargs.pop('type', None)
kwargs.pop("request", None)
super(EditableForm, self).__init__(*args, **kwargs)
fields = RequestField.objects.filter(request_type=type)
n = 0
if fields:
for field in fields:
n = n+1
type = field.type
if(type == 'Char'):
self.fields['field'+str(n)] = CharField(
max_length=30,
required=field.required,
label=field.fieldname)
elif (type == 'Integer'):
self.fields['field'+str(n)] = IntegerField(
required=field.required,
label=field.fieldname)
elif (type == 'Email'):
self.fields['field'+str(n)] = EmailField(
required=field.required,
label=field.fieldname)
if(field.choices):
choices = [(ch, ch)for ch in field.choices.split(',')]
self.fields['field'+str(n)] = ChoiceField(
choices=choices,
required=field.required,
label=field.fieldname)
def get_dynamic_fields(self):
for field_name in self.fields:
if field_name.startswith("field"):
yield self[field_name]
class LeaseTypeForm(ModelForm):
@property
def helper(self):
......@@ -80,27 +147,18 @@ class InitialFromFileMixin(object):
return message.strip()
class TemplateRequestForm(InitialFromFileMixin, Form):
message = CharField(widget=Textarea, label=_("Message"))
class TemplateRequestForm(EditableForm):
template = ModelChoiceField(TemplateAccessType.objects.all(),
label=_("Template share"))
level = ChoiceField(TemplateAccessAction.LEVELS, widget=RadioSelect,
initial=TemplateAccessAction.LEVELS.user)
initial_template = "request/initials/template.html"
class LeaseRequestForm(InitialFromFileMixin, Form):
class LeaseRequestForm(EditableForm):
lease = ModelChoiceField(LeaseType.objects.all(), label=_("Lease"))
message = CharField(widget=Textarea, label=_("Message"))
initial_template = "request/initials/lease.html"
class ResourceRequestForm(InitialFromFileMixin, VmResourcesForm):
message = CharField(widget=Textarea, label=_("Message"))
initial_template = "request/initials/resources.html"
class ResourceRequestForm(EditableForm, VmResourcesForm):
def clean(self):
cleaned_data = super(ResourceRequestForm, self).clean()
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-11-12 15:20
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0004_auto_20150629_1605'),
]
operations = [
migrations.CreateModel(
name='RequestField',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('fieldname', models.CharField(max_length=50, unique=True)),
('type', models.CharField(choices=[(b'Char', b'CharField'), (b'Integer', b'IntegerField'), (b'Email', b'EmailField')], default=b'Char', max_length=20)),
('choices', models.CharField(max_length=100, null=True)),
('required', models.BooleanField(default=True)),
],
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-11-15 15:48
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0005_requestfield'),
]
operations = [
migrations.AlterField(
model_name='requestfield',
name='choices',
field=models.CharField(max_length=300, null=True),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.6 on 2018-12-12 14:12
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('request', '0006_auto_20181115_1548'),
]
operations = [
migrations.AddField(
model_name='requestfield',
name='request_type',
field=models.CharField(choices=[(b'resource', 'resource request'), (b'lease', 'lease request'), (b'template', 'template access request'), (b'resize', 'disk resize request')], default=b'template', max_length=20),
),
]
......@@ -19,6 +19,7 @@ import logging
from django.db.models import (
Model, CharField, IntegerField, TextField, ForeignKey, ManyToManyField,
BooleanField,
)
from django.db.models.signals import post_save
from django.conf import settings
......@@ -157,6 +158,27 @@ class Request(TimeStampedModel):
return self.action.is_acceptable()
class RequestField(Model):
TYPES = (
('Char', 'CharField'),
('Integer', 'IntegerField'),
('Email', 'EmailField')
)
fieldname = CharField(max_length=50, blank=False, unique=True)
type = CharField(choices=TYPES, default='Char', max_length=20)
request_type = CharField(choices=Request.TYPES, default='template',
max_length=20)
choices = CharField(max_length=300, null=True)
required = BooleanField(default=True)
def __unicode__(self):
return self.fieldname
def get_absolute_url(self):
return reverse('fields_detail', kwargs={'pk': self.pk})
class LeaseType(RequestType):
lease = ForeignKey(Lease, verbose_name=_("Lease"))
......
{% extends "dashboard/base.html" %}
{% load i18n %}
{% load crispy_forms_tags %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading"></div>
<div class="panel-body">
<form action={% url 'request.views.request-field-add' %} method="post">
{% include "display-form-errors.html" %}
{% csrf_token %}
{% for field in form %}
{{ field|as_crispy_field}}
{% endfor %}
<button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
</form>
</div> <!-- .panel-body -->
</div>
</div>
</div>
{% endblock %}
......@@ -6,5 +6,8 @@
{% csrf_token %}
{{ form.lease|as_crispy_field }}
{{ form.message|as_crispy_field }}
{% for fields in form.get_dynamic_fields %}
{{ field|as_crispy_field }}
{% endfor %}
<input type="submit" class="btn btn-primary"/>
</form>
......@@ -21,6 +21,8 @@
</label>
</div>
{% endfor %}
{{ form.message|as_crispy_field }}
{% for field in form.get_dynamic_fields %}
{{ field|as_crispy_field }}
{% endfor %}
<input type="submit" class="btn btn-primary"/>
</form>
{% extends "dashboard/base.html" %}
{% load i18n %}
{% load crispy_forms_field %}
{% block content %}
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="no-margin"><i class="fa fa-phone"></i> {% trans "Request fields" %}</h3>
</div>
<div class="panel-body">
<div class="table-responsive">
<div class="table-container">
<table class="text-center table table-bordered table-striped table-hover" >
<thead class"align-center">
<tr>
<th class="text-center">{% trans "Fieldname" %}</th>
<th class="text-center">{% trans "Type" %}</th>
<th class="text-center">{% trans "RequestType" %}</th>
<th class="text-center">{% trans "Choices" %}</th>
<th class="text-center">{% trans "Required" %}</th>
<th class="text-center">{% trans "Delete" %}</th>
</tr>
</thead>
<tbody>
{% for field in object_list %}
<tr>
<td>{{field.fieldname}}</td>
<td>{{field.type}}</td>
<td>{{field.request_type}}</td>
<td>{{field.choices}}</td>
<td>{{field.required}}</td>
<td><a href={% url "request.views.field-delete" pk=field.pk %}>
<i class="fa fa-times"></i></a>
</td>
</tr>
{% endfor %}
<tr>
<form action={% url 'request.views.request-field-add' %} method="post">
{% csrf_token %}
{% for field in add_form %}
<td> {% crispy_field field %}</td>
{% endfor %}
<td><button type="submit" class="btn btn-sm btn-success">{% trans "Add" %}</button></td>
</form>
</tr>
</tbody>
</table>
</div>
</div> <!-- .table-responsive -->
</div> <!-- .panel-body -->
</div>
</div>