Skip to content

CIRCLE / cloud

Go to a project
Commit 48308ff9 by Belákovics Ádám
Options

Merge branch 'export_import_disk' into 'master' 

Export and import disk images to store

See merge request !414
parents 9cb97bdb 6085e2eb
Pipeline #1140 passed with stage
in 0 seconds
Showing with 385 additions and 195 deletions
.gitignore
......@@ -8,6 +8,8 @@
*.swp
*.swo
*~
.vscode
.idea
# Sphinx docs:
build
......
circle/dashboard/forms.py
......@@ -17,56 +17,51 @@
from __future__ import absolute_import
from datetime import timedelta
from urlparse import urlparse
import os
import pyotp
from django.forms import ModelForm
from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm,
PasswordChangeForm,
)
from django.contrib.auth.models import User, Group
from django.core.validators import URLValidator
from django.core.exceptions import PermissionDenied, ValidationError
from dal import autocomplete
from crispy_forms.bootstrap import FormActions
from crispy_forms.helper import FormHelper
from crispy_forms.layout import (
Layout, Div, BaseInput, Field, HTML, Submit, TEMPLATE_PACK, Fieldset
)
from crispy_forms.utils import render_field
from crispy_forms.bootstrap import FormActions
from dal import autocomplete
from datetime import timedelta
from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.auth.forms import (
AuthenticationForm, PasswordResetForm, SetPasswordForm,
PasswordChangeForm,
)
from django.contrib.auth.forms import UserCreationForm as OrgUserCreationForm
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User, Group
from django.core.exceptions import PermissionDenied, ValidationError
from django.core.urlresolvers import reverse_lazy
from django.core.validators import URLValidator
from django.forms import ModelForm
from django.forms.widgets import TextInput, HiddenInput
from django.template.loader import render_to_string
from django.utils.html import escape, format_html
from django.utils.safestring import mark_safe
from django.utils.translation import string_concat
from django.utils.translation import ugettext_lazy as _
from django_sshkey.models import UserKey
from sizefield.widgets import FileSizeWidget
from django.core.urlresolvers import reverse_lazy
from django_sshkey.models import UserKey
from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
from dashboard.models import ConnectCommand, create_profile
from dashboard.store_api import Store
from firewall.models import Vlan, Host
from storage.models import DataStore, Disk
from vm.models import (
InstanceTemplate, Lease, InterfaceTemplate, Node, Trait, Instance
)
from storage.models import DataStore, Disk
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.auth.models import Permission
from .models import Profile, GroupProfile, Message
from circle.settings.base import LANGUAGES, MAX_NODE_RAM, MAX_NODE_CPU_CORE
from django.utils.translation import string_concat
from .validators import domain_validator
from dashboard.models import ConnectCommand, create_profile
LANGUAGES_WITH_CODE = ((l[0], string_concat(l[1], " (", l[0], ")"))
for l in LANGUAGES)
......@@ -189,7 +184,7 @@ class VmCustomizeForm(forms.Form):
self.initial['ram_size'] = self.template.ram_size
else:
self.allowed_fields = ("name", "template", "customized", )
self.allowed_fields = ("name", "template", "customized",)
# initial name and template pk
self.initial['name'] = self.template.name
......@@ -214,7 +209,6 @@ class VmCustomizeForm(forms.Form):
class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
description = forms.CharField(label=_("Description"), required=False,
widget=forms.Textarea(attrs={'rows': 3}))
......@@ -258,7 +252,7 @@ class GroupCreateForm(NoFormTagMixin, forms.ModelForm):
class Meta:
model = Group
fields = ('name', )
fields = ('name',)
class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
......@@ -276,6 +270,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
label=_('Directory identifier'))
if not new_groups:
self.fields['org_id'].widget = HiddenInput()
self.fields['disk_quota'].widget = HiddenInput()
self.fields['description'].widget = forms.Textarea(attrs={'rows': 3})
@property
......@@ -293,7 +288,7 @@ class GroupProfileUpdateForm(NoFormTagMixin, forms.ModelForm):
class Meta:
model = GroupProfile
fields = ('description', 'org_id')
fields = ('description', 'org_id', 'disk_quota')
class HostForm(NoFormTagMixin, forms.ModelForm):
......@@ -513,7 +508,7 @@ class TemplateForm(forms.ModelForm):
self.allowed_fields += tuple(set(self.fields.keys()) -
set(['raw_data']))
if self.user.is_superuser:
self.allowed_fields += ('raw_data', )
self.allowed_fields += ('raw_data',)
for name, field in self.fields.items():
if name not in self.allowed_fields:
field.widget.attrs['disabled'] = 'disabled'
......@@ -525,8 +520,8 @@ class TemplateForm(forms.ModelForm):
self.initial['max_ram_size'] = 512
lease_queryset = (
Lease.get_objects_with_level("operator", self.user).distinct() |
Lease.objects.filter(pk=self.instance.lease_id).distinct())
Lease.get_objects_with_level("operator", self.user).distinct() |
Lease.objects.filter(pk=self.instance.lease_id).distinct())
self.fields["lease"].queryset = lease_queryset
......@@ -602,7 +597,7 @@ class TemplateForm(forms.ModelForm):
class Meta:
model = InstanceTemplate
exclude = ('state', 'disks', )
exclude = ('state', 'disks',)
widgets = {
'system': forms.TextInput,
'max_ram_size': forms.HiddenInput,
......@@ -745,7 +740,6 @@ class LeaseForm(forms.ModelForm):
class VmRenewForm(OperationForm):
force = forms.BooleanField(required=False, label=_(
"Set expiration times even if they are shorter than "
"the current value."))
......@@ -785,11 +779,10 @@ class VmMigrateForm(forms.Form):
class VmStateChangeForm(OperationForm):
interrupt = forms.BooleanField(required=False, label=_(
"Forcibly interrupt all running activities."),
help_text=_("Set all activities to finished state, "
"but don't interrupt any tasks."))
help_text=_("Set all activities to finished state, "
"but don't interrupt any tasks."))
new_state = forms.ChoiceField(Instance.STATUS, label=_(
"New status"))
reset_node = forms.BooleanField(required=False, label=_("Reset node"))
......@@ -830,6 +823,38 @@ class VmCreateDiskForm(OperationForm):
return size_in_bytes
class VmDiskExportForm(OperationForm):
format = forms.ChoiceField(
choices=Disk.EXPORT_FORMATS,
label=_('Format'))
def __init__(self, *args, **kwargs):
choices = kwargs.pop('choices')
self.disk = kwargs.pop('default')
super(VmDiskExportForm, self).__init__(*args, **kwargs)
self.fields['disk'] = forms.ModelChoiceField(
queryset=choices, initial=self.disk, required=True,
empty_label=None, label=_('Disk'))
if self.disk:
self.fields['disk'].widget = HiddenInput()
@property
def helper(self):
helper = super(VmDiskExportForm, self).helper
if self.disk:
helper.layout = Layout(
AnyTag(
"div",
HTML(_("<label>Disk:</label> %s") % escape(self.disk)),
css_class="form-group",
),
Field('disk'), Field('format')
)
return helper
class VmDiskResizeForm(OperationForm):
size = forms.CharField(
widget=FileSizeWidget, initial=(10 << 30), label=_('Size'),
......@@ -858,7 +883,7 @@ class VmDiskResizeForm(OperationForm):
" GB or MB!"))
if int(size_in_bytes) < int(disk.size):
raise forms.ValidationError(_("Disk size must be greater than the "
"actual size."))
"actual size."))
return cleaned_data
@property
......@@ -899,6 +924,20 @@ class VmDiskRemoveForm(OperationForm):
return helper
class VmImportDiskForm(OperationForm):
def __init__(self, *args, **kwargs):
self.user = kwargs.pop('user')
super(VmImportDiskForm, self).__init__(*args, **kwargs)
disk_paths = Store(self.user).get_disk_images()
disk_filenames = [os.path.basename(item) for item in disk_paths]
self.choices = zip(disk_paths, disk_filenames)
self.fields['name'] = forms.CharField(max_length=100, label=_('Name'))
self.fields['disk_path'] = forms.ChoiceField(label=_('Disk image'),
choices=self.choices)
class VmDownloadDiskForm(OperationForm):
name = forms.CharField(max_length=100, label=_("Name"), required=False)
url = forms.CharField(label=_('URL'), validators=[URLValidator(), ])
......@@ -1138,7 +1177,6 @@ class CircleSetPasswordForm(SetPasswordForm):
class LinkButton(BaseInput):
"""
Used to create a link button descriptor for the {% crispy %} template tag::
......@@ -1224,7 +1262,7 @@ class MyProfileForm(forms.ModelForm):
class Meta:
fields = ('preferred_language', 'email_notifications',
'desktop_notifications', 'use_gravatar', )
'desktop_notifications', 'use_gravatar',)
model = Profile
@property
......@@ -1239,9 +1277,8 @@ class MyProfileForm(forms.ModelForm):
class UnsubscribeForm(forms.ModelForm):
class Meta:
fields = ('email_notifications', )
fields = ('email_notifications',)
model = Profile
@property
......@@ -1314,14 +1351,14 @@ class UserEditForm(forms.ModelForm):
class Meta:
model = User
fields = ('email', 'first_name', 'last_name', 'instance_limit',
'is_active', "two_factor_secret", )
'is_active', "two_factor_secret",)
def save(self, commit=True):
user = super(UserEditForm, self).save()
user.profile.instance_limit = (
self.cleaned_data['instance_limit'] or None)
self.cleaned_data['instance_limit'] or None)
user.profile.two_factor_secret = (
self.cleaned_data['two_factor_secret'] or None)
self.cleaned_data['two_factor_secret'] or None)
user.profile.save()
return user
......@@ -1405,10 +1442,9 @@ class ConnectCommandForm(forms.ModelForm):
class TraitsForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('req_traits', )
fields = ('req_traits',)
@property
def helper(self):
......@@ -1428,7 +1464,7 @@ class RawDataForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('raw_data', )
fields = ('raw_data',)
@property
def helper(self):
......@@ -1490,7 +1526,7 @@ class GroupPermissionForm(forms.ModelForm):
class Meta:
model = Group
fields = ('permissions', )
fields = ('permissions',)
@property
def helper(self):
......@@ -1538,7 +1574,7 @@ class VmResourcesForm(forms.ModelForm):
class Meta:
model = Instance
fields = ('num_cores', 'priority', 'ram_size', )
fields = ('num_cores', 'priority', 'ram_size',)
class VmRenameForm(forms.Form):
......@@ -1629,7 +1665,7 @@ class DataStoreForm(ModelForm):
class Meta:
model = DataStore
fields = ("name", "path", "hostname", )
fields = ("name", "path", "hostname",)
class DiskForm(ModelForm):
......@@ -1647,7 +1683,7 @@ class DiskForm(ModelForm):
class Meta:
model = Disk
fields = ("name", "filename", "datastore", "type", "bus", "size",
"base", "dev_num", "destroyed", "is_ready", )
"base", "dev_num", "destroyed", "is_ready",)
class MessageForm(ModelForm):
......
circle/dashboard/migrations/0007_groupprofile_disk_quota.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
import sizefield.models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0006_auto_20170707_1909'),
]
operations = [
migrations.AddField(
model_name='groupprofile',
name='disk_quota',
field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
),
]
circle/dashboard/models.py
......@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.models.instance import ACCESS_METHODS
from .store_api import Store, NoStoreException, NotOkException, Timeout
from .store_api import Store, NoStoreException, NotOkException
from .validators import connect_command_template_validator
logger = getLogger(__name__)
......@@ -293,6 +293,10 @@ class GroupProfile(AclBase):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the group at the organization.'))
description = TextField(blank=True)
disk_quota = FileSizeField(
verbose_name=_('disk quota'),
default=2048 * 1024 * 1024,
help_text=_('Disk quota in mebibytes.'))
class Meta:
ordering = ('id',)
......@@ -332,10 +336,10 @@ def create_profile(user):
try:
store = Store(user)
if store.user_exist():
profile.disk_quota = store.get_quota()['soft']
profile.save()
store.create_user(profile.smb_password, None, profile.disk_quota)
quotas = [profile.disk_quota]
quotas += [group.profile.disk_quota for group in user.groups.all()]
max_quota = max(quotas)
store.create_user(profile.smb_password, None, max_quota)
except:
logger.exception("Can't create user %s", unicode(user))
return created
......@@ -351,6 +355,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
logger.debug("Register save_org_id to djangosaml2 pre_user_save")
from djangosaml2.signals import pre_user_save
def save_org_id(sender, instance, attributes, **kwargs):
logger.debug("save_org_id called by %s", instance.username)
atr = settings.SAML_ORG_ID_ATTRIBUTE
......@@ -403,6 +408,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
return False # User did not change
pre_user_save.connect(save_org_id)
......@@ -415,7 +421,7 @@ def update_store_profile(sender, **kwargs):
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except (NotOkException, Timeout):
except NotOkException:
logger.critical("Store is not accepting connections.")
......
circle/dashboard/static/dashboard/dashboard.less
......@@ -1092,7 +1092,7 @@ textarea[name="new_members"] {
vertical-align: middle;
}
.disk-resize-btn {
.disk-resize-btn, .disk-export-btn {
margin-right: 5px;
}
......
circle/dashboard/store_api.py
......@@ -14,19 +14,20 @@
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from os.path import splitext
import json
import logging
from urlparse import urljoin
from datetime import datetime
from django.http import Http404
import os
from datetime import datetime
from django.conf import settings
from django.http import Http404
from os.path import splitext
from requests import get, post, codes
from requests.exceptions import Timeout # noqa
from sizefield.utils import filesizeformat
from storage.models import Disk
logger = logging.getLogger(__name__)
......@@ -44,18 +45,12 @@ class NoStoreException(StoreApiException):
pass
class NoOrgIdException(StoreApiException):
pass
class Store(object):
def __init__(self, user, default_timeout=0.5):
self.store_url = settings.STORE_URL
if not self.store_url:
if not self.store_url or not user.profile.org_id:
raise NoStoreException
if not user.profile.org_id:
raise NoOrgIdException
self.username = 'u-%s' % user.profile.org_id
self.request_args = {'verify': settings.STORE_VERIFY_SSL}
if settings.STORE_SSL_AUTH:
......@@ -108,6 +103,15 @@ class Store(object):
else:
return result
def get_disk_images(self, path='/'):
images = []
file_list = self.list(path, process=False)
export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
for item in file_list:
if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
images.append(os.path.join(path, item['NAME']))
return images
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
return r.json()['LINK']
......
circle/dashboard/templates/dashboard/_disk-list-element.html
......@@ -6,15 +6,29 @@
<span class="operation-wrapper pull-right">
{% if d.is_exportable %}
{% if op.export_disk %}
<a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
{% if op.export_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
</a>
{% endif %}
{% else %}
<small class="btn-xs">
{% trans "Not exportable" %}
</small>
{% endif %}
{% if d.is_resizable %}
{% if op.resize_disk %}
<a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
<a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
{% if op.resize_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
</a>
{% else %}
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
class="btn btn-xs btn-primary operation">
<i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
</a>
{% endif %}
......@@ -24,8 +38,8 @@
</small>
{% endif %}
{% if op.remove_disk %}
<a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
<a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
{% if op.remove_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
</a>
......
circle/dashboard/templates/dashboard/vm-detail/_disk-operations.html
{% load i18n %}
{% for op in ops %}
{% if op.is_disk_operation %}
<a href="{{op.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.op}}">
<i class="fa fa-{{op.icon}} fa-fw-12"></i>
{{op.name}} </a>
{% endif %}
{% if op.is_disk_operation %}
<a href="{{ op.get_url }}" class="btn btn-success btn-xs
operation operation-{{ op.op }}">
<i class="fa fa-{{ op.icon }} fa-fw-12"></i>
{{ op.name }} </a>
{% endif %}
{% endfor %}
circle/dashboard/views/store.py
......@@ -36,7 +36,7 @@ from django.views.generic import TemplateView
from braces.views import LoginRequiredMixin
from ..store_api import (Store, NoStoreException,
NotOkException, NoOrgIdException)
NotOkException)
logger = logging.getLogger(__name__)
......@@ -71,11 +71,6 @@ class StoreList(LoginRequiredMixin, TemplateView):
return super(StoreList, self).get(*args, **kwargs)
except NoStoreException:
messages.warning(self.request, _("No store."))
except NoOrgIdException:
messages.warning(self.request,
_("Your organization ID is not set."
" To use the store, you need a"
" unique organization ID."))
except NotOkException:
messages.warning(self.request, _("Store has some problems now."
" Try again later."))
......
circle/dashboard/views/vm.py
......@@ -61,9 +61,10 @@ from .util import (
)
from ..forms import (
AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
VmImportDiskForm, VmSaveForm,
VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm,
......@@ -166,8 +167,8 @@ class VmDetailView(GraphMixin, CheckedDetailView):
# resources forms
can_edit = (
instance.has_level(user, "owner") and
self.request.user.has_perm("vm.change_resources"))
instance.has_level(user, "owner") and
self.request.user.has_perm("vm.change_resources"))
context['resources_form'] = VmResourcesForm(
can_edit=can_edit, instance=instance)
......@@ -269,7 +270,7 @@ class VmDetailView(GraphMixin, CheckedDetailView):
return JsonResponse({'message': message})
else:
return redirect(reverse_lazy("dashboard.views.detail",
kwargs={'pk': self.object.pk}))
kwargs={'pk': self.object.pk}))
def __abort_operation(self, request):
self.object = self.get_object()
......@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance
context_object_name = 'instance' # much simpler to mock object
......@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
class VmAddInterfaceView(FormOperationMixin, VmOperationView):
op = 'add_interface'
form_class = VmAddInterfaceForm
show_in_toolbar = False
......@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
class VmCreateDiskView(FormOperationMixin, VmOperationView):
op = 'create_disk'
form_class = VmCreateDiskForm
show_in_toolbar = False
......@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmImportDiskView(FormOperationMixin, VmOperationView):
op = 'import_disk'
form_class = VmImportDiskForm
show_in_toolbar = False
icon = 'upload'
effect = "success"
is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
val = super(VmImportDiskView, self).get_form_kwargs()
val.update({'user': self.request.user})
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
op = 'download_disk'
form_class = VmDownloadDiskForm
show_in_toolbar = False
......@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmMigrateView(FormOperationMixin, VmOperationView):
op = 'migrate'
icon = 'truck'
effect = 'info'
......@@ -450,7 +461,7 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
nodes_w_traits = [
n.pk for n in Node.objects.filter(enabled=True)
if n.online and
has_traits(inst.req_traits.all(), n)
has_traits(inst.req_traits.all(), n)
]
ctx['nodes_w_traits'] = nodes_w_traits
......@@ -458,7 +469,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
class VmPortRemoveView(FormOperationMixin, VmOperationView):
template_name = 'dashboard/_vm-remove-port.html'
op = 'remove_port'
show_in_toolbar = False
......@@ -487,7 +497,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
class VmPortAddView(FormOperationMixin, VmOperationView):
op = 'add_port'
show_in_toolbar = False
with_reload = True
......@@ -514,7 +523,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
class VmSaveView(FormOperationMixin, VmOperationView):
op = 'save_as_template'
icon = 'save'
effect = 'info'
......@@ -570,7 +578,7 @@ class TokenOperationView(OperationView):
User can do the action with a valid token instead of logging in.
"""
token_max_age = 3 * 24 * 3600
redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
@classmethod
def get_salt(cls):
......@@ -642,7 +650,6 @@ class TokenOperationView(OperationView):
class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
op = 'renew'
icon = 'calendar'
effect = 'success'
......@@ -769,7 +776,11 @@ vm_ops = OrderedDict([
extra_bases=[TokenOperationView],
op='destroy', icon='times', effect='danger')),
('create_disk', VmCreateDiskView),
('import_disk', VmImportDiskView),
('download_disk', VmDownloadDiskView),
('export_disk', VmDiskModifyView.factory(
op='export_disk', form_class=VmDiskExportForm,
icon='download', effect='info')),
('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")),
......@@ -1014,7 +1025,7 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
# remove "-" that means descending order
# also check if the column name is valid
if (sort and
(sort[1:] if sort[0] == "-" else sort)
(sort[1:] if sort[0] == "-" else sort)
in [i.name for i in Instance._meta.fields] + ["pk"]):
queryset = queryset.order_by(sort)
......@@ -1025,7 +1036,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
class VmCreate(LoginRequiredMixin, TemplateView):
form_class = VmCustomizeForm
form = None
......@@ -1134,7 +1144,7 @@ class VmCreate(LoginRequiredMixin, TemplateView):
messages.success(request, ungettext_lazy(
"Successfully created %(count)d VM.", # this should not happen
"Successfully created %(count)d VMs.", len(instances)) % {
'count': len(instances)})
'count': len(instances)})
path = "%s?stype=owned" % reverse("dashboard.views.vm-list")
else:
messages.success(request, _("VM successfully created."))
......@@ -1326,7 +1336,7 @@ class TransferInstanceOwnershipConfirmView(TransferOwnershipConfirmView):
def change_owner(self, instance, new_owner):
with instance.activity(
code_suffix='ownership-transferred',
code_suffix='ownership-transferred',
readable_name=ugettext_noop("transfer ownership"),
concurrency_check=False, user=new_owner):
super(TransferInstanceOwnershipConfirmView, self).change_owner(
......
circle/storage/migrations/0003_auto_20200424_2000.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0002_disk_bus'),
]
operations = [
migrations.AlterModelOptions(
name='disk',
options={'ordering': ['name'], 'permissions': (('create_empty_disk', 'Can create an empty disk.'), ('download_disk', 'Can download a disk.'), ('resize_disk', 'Can resize a disk.'), ('import_disk', 'Can import a disk.'), ('export_disk', 'Can export a disk.')), 'verbose_name': 'disk', 'verbose_name_plural': 'disks'},
),
]
circle/storage/models.py
......@@ -20,31 +20,30 @@
from __future__ import unicode_literals
import logging
from os.path import join
import uuid
import re
import re
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from celery.exceptions import TimeoutError
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel
from os.path import join
from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks
from celery.exceptions import TimeoutError
from common.models import (
WorkerNotFound, HumanReadableException, humanize_exception, method_cache
)
from .tasks import local_tasks, storage_tasks
logger = logging.getLogger(__name__)
class DataStore(Model):
"""Collection of virtual disks.
"""
name = CharField(max_length=100, unique=True, verbose_name=_('name'))
......@@ -119,12 +118,15 @@ class DataStore(Model):
class Disk(TimeStampedModel):
"""A virtual disk.
"""
TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
EXPORT_FORMATS = (('vmdk', _('VMware disk image')),
('qcow2', _('QEMU disk image')),
('vdi', _('VirtualBox disk image')),
('vpc', _('HyperV disk image')))
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True,
verbose_name=_("filename"))
......@@ -149,7 +151,9 @@ class Disk(TimeStampedModel):
permissions = (
('create_empty_disk', _('Can create an empty disk.')),
('download_disk', _('Can download a disk.')),
('resize_disk', _('Can resize a disk.'))
('resize_disk', _('Can resize a disk.')),
('import_disk', _('Can import a disk.')),
('export_disk', _('Can export a disk.'))
)
class DiskError(HumanReadableException):
......@@ -467,6 +471,30 @@ class Disk(TimeStampedModel):
disk.save()
return disk
@classmethod
def import_disk(cls, user, name, download_link, timeout=3600):
params = {'name': name,
'type': 'qcow2-norm'}
disk = cls.create(user, **params)
queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.import_disk.apply_async(
args=[disk.get_disk_desc(), download_link],
queue=queue_name
)
disk_size = remote.get(timeout=timeout)
disk.size = disk_size
disk.is_ready = True
disk.save()
return disk
def export(self, format, upload_link, timeout=3600):
exported_name = self.name if self.name != '' else self.filename
queue_name = self.get_remote_queue_name('storage', priority='slow')
storage_tasks.export.apply_async(
args=[self.get_disk_desc(), format, exported_name, upload_link],
queue=queue_name).get(timeout=timeout)
def destroy(self, user=None, task_uuid=None):
if self.destroyed:
return False
......@@ -549,4 +577,8 @@ class Disk(TimeStampedModel):
@property
def is_resizable(self):
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap', )
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap',)
@property
def is_exportable(self):
return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')
circle/storage/tasks/storage_tasks.py
......@@ -38,6 +38,16 @@ def download(disk_desc, url):
pass
@celery.task(name='storagedriver.import_disk')
def import_disk(disk_desc, url):
pass
@celery.task(name='storagedriver.export')
def export(disk_desc, format):
pass
@celery.task(name='storagedriver.delete')
def delete(path):
pass
......
circle/vm/operations.py
......@@ -16,53 +16,51 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import, unicode_literals
from StringIO import StringIO
from base64 import encodestring
from hashlib import md5
from logging import getLogger
import os
from re import search
from string import ascii_lowercase
from StringIO import StringIO
from tarfile import TarFile, TarInfo
import time
from urlparse import urlsplit
import os
import time
from celery.contrib.abortable import AbortableAsyncResult
from celery.exceptions import TimeLimitExceeded, TimeoutError
from django.conf import settings
from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from django.conf import settings
from django.db.models import Q
from re import search
from sizefield.utils import filesizeformat
from celery.contrib.abortable import AbortableAsyncResult
from celery.exceptions import TimeLimitExceeded, TimeoutError
from common.models import (
create_readable, humanize_exception, HumanReadableException
)
from common.operations import Operation, register_operation, SubOperationMixin
from dashboard.store_api import Store, NoStoreException
from firewall.models import Host
from manager.scheduler import SchedulerError
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
from monitor.client import Client
from storage.models import Disk
from storage.tasks import storage_tasks
from .models import (
Instance, InstanceActivity, InstanceTemplate, Interface, Node,
NodeActivity, pwgen
)
from .tasks import agent_tasks, vm_tasks
from dashboard.store_api import Store, NoStoreException
from firewall.models import Host
from monitor.client import Client
from storage.tasks import storage_tasks
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
logger = getLogger(__name__)
class RemoteOperationMixin(object):
remote_timeout = 30
def _operation(self, **kwargs):
......@@ -167,7 +165,6 @@ class InstanceOperation(Operation):
class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
remote_queue = ('vm', 'fast')
def _get_remote_queue(self):
......@@ -178,7 +175,7 @@ class RemoteInstanceOperation(RemoteOperationMixin, InstanceOperation):
class EnsureAgentMixin(object):
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
def check_precond(self):
super(EnsureAgentMixin, self).check_precond()
......@@ -198,7 +195,7 @@ class EnsureAgentMixin(object):
class RemoteAgentOperation(EnsureAgentMixin, RemoteInstanceOperation):
remote_queue = ('agent', )
remote_queue = ('agent',)
concurrency_check = False
......@@ -213,7 +210,7 @@ class AddInterfaceOperation(InstanceOperation):
def rollback(self, net, activity):
with activity.sub_activity(
'destroying_net',
'destroying_net',
readable_name=ugettext_noop("destroy network (rollback)")):
net.destroy()
net.delete()
......@@ -248,11 +245,10 @@ class AddInterfaceOperation(InstanceOperation):
@register_operation
class CreateDiskOperation(InstanceOperation):
id = 'create_disk'
name = _("create disk")
description = _("Create and attach empty disk to the virtual machine.")
required_perms = ('storage.create_empty_disk', )
required_perms = ('storage.create_empty_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, size, activity, name=None):
......@@ -271,8 +267,8 @@ class CreateDiskOperation(InstanceOperation):
if self.instance.is_running:
with activity.sub_activity(
'deploying_disk',
readable_name=ugettext_noop("deploying disk")
'deploying_disk',
readable_name=ugettext_noop("deploying disk")
):
disk.deploy()
self.instance._attach_disk(parent_activity=activity, disk=disk)
......@@ -285,13 +281,12 @@ class CreateDiskOperation(InstanceOperation):
@register_operation
class ResizeDiskOperation(RemoteInstanceOperation):
id = 'resize_disk'
name = _("resize disk")
description = _("Resize the virtual disk image. "
"Size must be greater value than the actual size.")
required_perms = ('storage.resize_disk', )
accept_states = ('RUNNING', )
required_perms = ('storage.resize_disk',)
accept_states = ('RUNNING',)
async_queue = "localhost.man.slow"
remote_queue = ('vm', 'slow')
task = vm_tasks.resize_disk
......@@ -324,13 +319,11 @@ class DownloadDiskOperation(InstanceOperation):
"machine.")
abortable = True
has_percentage = True
required_perms = ('storage.download_disk', )
required_perms = ('storage.download_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = "localhost.man.slow"
def _operation(self, user, url, task, activity, name=None):
from storage.models import Disk
disk = Disk.download(url=url, name=name, task=task)
devnums = list(ascii_lowercase)
for d in self.instance.disks.all():
......@@ -352,6 +345,53 @@ class DownloadDiskOperation(InstanceOperation):
@register_operation
class ImportDiskOperation(InstanceOperation):
id = 'import_disk'
name = _('import disk')
description = _('Import and attach a disk image to the virtual machine '
'from the user store. The disk image has to be in the '
'root directory of the store.')
required_perms = ('storage.import_disk',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
async_queue = 'localhost.man.slow'
def check_auth(self, user):
super(ImportDiskOperation, self).check_auth(user)
try:
Store(user)
except NoStoreException:
raise PermissionDenied
def _operation(self, user, name, disk_path):
store = Store(user)
download_link = store.request_download(disk_path)
disk = Disk.import_disk(user, name, download_link)
self.instance.disks.add(disk)
@register_operation
class ExportDiskOperation(InstanceOperation):
id = 'export_disk'
name = _('export disk')
description = _('Export disk to the selected format.')
required_perms = ('storage.export_disk',)
accept_states = ('STOPPED',)
async_queue = 'localhost.man.slow'
def check_auth(self, user):
super(ExportDiskOperation, self).check_auth(user)
try:
Store(user)
except NoStoreException:
raise PermissionDenied
def _operation(self, user, disk, format):
store = Store(user)
upload_link = store.request_upload('/')
disk.export(format, upload_link)
@register_operation
class DeployOperation(InstanceOperation):
id = 'deploy'
name = _("deploy")
......@@ -399,8 +439,8 @@ class DeployOperation(InstanceOperation):
# Establish network connection (vmdriver)
with activity.sub_activity(
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
self.instance.deploy_net()
try:
......@@ -421,7 +461,8 @@ class DeployOperation(InstanceOperation):
description = _("Deploy virtual machine.")
remote_queue = ("vm", "slow")
task = vm_tasks.deploy
remote_timeout = 120
remote_timeout = 120
def _get_remote_args(self, **kwargs):
return [self.instance.get_vm_desc()]
# intentionally not calling super
......@@ -530,7 +571,7 @@ class MigrateOperation(RemoteInstanceOperation):
"keeping its full state.")
required_perms = ()
superuser_required = True
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
async_queue = "localhost.man.slow"
task = vm_tasks.migrate
remote_queue = ("vm", "slow")
......@@ -542,8 +583,8 @@ class MigrateOperation(RemoteInstanceOperation):
def rollback(self, activity):
with activity.sub_activity(
'rollback_net', readable_name=ugettext_noop(
"redeploy network (rollback)")):
'rollback_net', readable_name=ugettext_noop(
"redeploy network (rollback)")):
self.instance.deploy_net()
def _operation(self, activity, to_node=None, live_migration=True):
......@@ -556,8 +597,8 @@ class MigrateOperation(RemoteInstanceOperation):
try:
with activity.sub_activity(
'migrate_vm', readable_name=create_readable(
ugettext_noop("migrate to %(node)s"), node=to_node)):
'migrate_vm', readable_name=create_readable(
ugettext_noop("migrate to %(node)s"), node=to_node)):
super(MigrateOperation, self)._operation(
to_node=to_node, live_migration=live_migration)
except Exception as e:
......@@ -567,8 +608,8 @@ class MigrateOperation(RemoteInstanceOperation):
# Shutdown networks
with activity.sub_activity(
'shutdown_net', readable_name=ugettext_noop(
"shutdown network")):
'shutdown_net', readable_name=ugettext_noop(
"shutdown network")):
self.instance.shutdown_net()
# Refresh node information
......@@ -577,8 +618,8 @@ class MigrateOperation(RemoteInstanceOperation):
# Estabilish network connection (vmdriver)
with activity.sub_activity(
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
self.instance.deploy_net()
......@@ -589,7 +630,7 @@ class RebootOperation(RemoteInstanceOperation):
description = _("Warm reboot virtual machine by sending Ctrl+Alt+Del "
"signal to its console.")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.reboot
def _operation(self, activity):
......@@ -630,7 +671,7 @@ class RemovePortOperation(InstanceOperation):
description = _("Close the specified port.")
concurrency_check = False
acl_level = "operator"
required_perms = ('vm.config_ports', )
required_perms = ('vm.config_ports',)
def _operation(self, activity, rule):
interface = rule.host.interface_set.get()
......@@ -649,7 +690,7 @@ class AddPortOperation(InstanceOperation):
description = _("Open the specified port.")
concurrency_check = False
acl_level = "operator"
required_perms = ('vm.config_ports', )
required_perms = ('vm.config_ports',)
def _operation(self, activity, host, proto, port):
if host.interface_set.get().instance != self.instance:
......@@ -673,8 +714,8 @@ class RemoveDiskOperation(InstanceOperation):
if self.instance.is_running and disk.type not in ["iso"]:
self.instance._detach_disk(disk=disk, parent_activity=activity)
with activity.sub_activity(
'destroy_disk',
readable_name=ugettext_noop('destroy disk')
'destroy_disk',
readable_name=ugettext_noop('destroy disk')
):
disk.destroy()
return self.instance.disks.remove(disk)
......@@ -690,7 +731,7 @@ class ResetOperation(RemoteInstanceOperation):
name = _("reset")
description = _("Cold reboot virtual machine (power cycle).")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.reset
def _operation(self, activity):
......@@ -710,7 +751,7 @@ class SaveAsTemplateOperation(InstanceOperation):
"start an instance of it.")
has_percentage = True
abortable = True
required_perms = ('vm.create_template', )
required_perms = ('vm.create_template',)
accept_states = ('RUNNING', 'STOPPED')
async_queue = "localhost.man.slow"
......@@ -778,10 +819,10 @@ class SaveAsTemplateOperation(InstanceOperation):
self.disks = []
for disk in self.instance.disks.all():
with activity.sub_activity(
'saving_disk',
readable_name=create_readable(
ugettext_noop("saving disk %(name)s"),
name=disk.name)
'saving_disk',
readable_name=create_readable(
ugettext_noop("saving disk %(name)s"),
name=disk.name)
):
self.disks.append(__try_save_disk(disk))
......@@ -822,7 +863,7 @@ class ShutdownOperation(AbortableRemoteOperationMixin,
"turn itself off in a period.")
abortable = True
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
resultant_state = 'STOPPED'
task = vm_tasks.shutdown
remote_queue = ("vm", "slow")
......@@ -883,7 +924,7 @@ class SleepOperation(InstanceOperation):
"resumed. In the meantime, the machine will only use "
"storage resources, and keep network resources allocated.")
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
resultant_state = 'SUSPENDED'
async_queue = "localhost.man.slow"
......@@ -927,7 +968,7 @@ class WakeUpOperation(InstanceOperation):
"load the saved memory of the system and start the "
"virtual machine from this state.")
required_perms = ()
accept_states = ('SUSPENDED', )
accept_states = ('SUSPENDED',)
resultant_state = 'RUNNING'
async_queue = "localhost.man.slow"
......@@ -950,8 +991,8 @@ class WakeUpOperation(InstanceOperation):
# Estabilish network connection (vmdriver)
with activity.sub_activity(
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
'deploying_net', readable_name=ugettext_noop(
"deploy network")):
self.instance.deploy_net()
try:
......@@ -987,7 +1028,7 @@ class RenewOperation(InstanceOperation):
def set_time_of_suspend(self, activity, suspend, force):
with activity.sub_activity(
'renew_suspend', concurrency_check=False,
'renew_suspend', concurrency_check=False,
readable_name=ugettext_noop('set time of suspend')):
if (not force and suspend and self.instance.time_of_suspend and
suspend < self.instance.time_of_suspend):
......@@ -998,7 +1039,7 @@ class RenewOperation(InstanceOperation):
def set_time_of_delete(self, activity, delete, force):
with activity.sub_activity(
'renew_delete', concurrency_check=False,
'renew_delete', concurrency_check=False,
readable_name=ugettext_noop('set time of delete')):
if (not force and delete and self.instance.time_of_delete and
delete < self.instance.time_of_delete):
......@@ -1039,7 +1080,7 @@ class ChangeStateOperation(InstanceOperation):
"redeployed without losing its storage and network "
"resources.")
acl_level = "owner"
required_perms = ('vm.emergency_change_state', )
required_perms = ('vm.emergency_change_state',)
concurrency_check = False
def _operation(self, user, activity, new_state="NOSTATE", interrupt=False,
......@@ -1066,7 +1107,7 @@ class RedeployOperation(InstanceOperation):
"and redeploy the VM. This operation allows starting "
"machines formerly running on a failed node.")
acl_level = "owner"
required_perms = ('vm.redeploy', )
required_perms = ('vm.redeploy',)
concurrency_check = False
def _operation(self, user, activity, with_emergency_change_state=True):
......@@ -1320,7 +1361,7 @@ class ScreenshotOperation(RemoteInstanceOperation):
"screensaver.")
acl_level = "owner"
required_perms = ()
accept_states = ('RUNNING', )
accept_states = ('RUNNING',)
task = vm_tasks.screenshot
......@@ -1332,8 +1373,8 @@ class RecoverOperation(InstanceOperation):
"state. Network resources (allocations) are already lost, "
"so you will have to manually add interfaces afterwards.")
acl_level = "owner"
required_perms = ('vm.recover', )
accept_states = ('DESTROYED', )
required_perms = ('vm.recover',)
accept_states = ('DESTROYED',)
resultant_state = 'PENDING'
def check_precond(self):
......@@ -1344,7 +1385,7 @@ class RecoverOperation(InstanceOperation):
def _operation(self, user, activity):
with activity.sub_activity(
'recover_instance',
'recover_instance',
readable_name=ugettext_noop("recover instance")):
self.instance.destroyed_at = None
for disk in self.instance.disks.all():
......@@ -1371,7 +1412,7 @@ class ResourcesOperation(InstanceOperation):
name = _("resources change")
description = _("Change resources of a stopped virtual machine.")
acl_level = "owner"
required_perms = ('vm.change_resources', )
required_perms = ('vm.change_resources',)
accept_states = ('STOPPED', 'PENDING', 'RUNNING')
def _operation(self, user, activity,
......@@ -1677,7 +1718,7 @@ class UpdateAgentOperation(RemoteAgentOperation):
index = 0
filename = settings.AGENT_VERSION + ".tar"
while True:
chunk = data[index:index+chunk_size]
chunk = data[index:index + chunk_size]
if chunk:
agent_tasks.append.apply_async(
queue=queue,
......
requirements/base.txt
cryptography==2.0
cryptography==2.7
amqp==1.4.7
anyjson==0.3.3
arrow==0.7.0
billiard==3.3.0.20
bpython==0.14.1
celery==3.1.18
Django==1.11.6
Django==1.11.25
django-appconf==1.0.2
django-autocomplete-light==3.2.9
django-braces==1.11.0
......@@ -26,9 +26,9 @@ logutils==0.3.3
MarkupSafe==0.23
netaddr==0.7.14
pip-tools==0.3.6
psycopg2==2.6
psycopg2==2.8.3
Pygments==2.0.2
pylibmc==1.4.3
pylibmc==1.6.0
python-dateutil==2.4.2
pyinotify==0.9.5
pyotp==2.1.1
......
requirements/production.txt
# Pro-tip: Try not to put anything here. There should be no dependency in
# production that isn't in development.
-r base.txt
uWSGI==2.0.13.1
uWSGI==2.0.18
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment