Skip to content

CIRCLE / cloud

Go to a project
Commit 48308ff9 by Belákovics Ádám
Options

Merge branch 'export_import_disk' into 'master' 

Export and import disk images to store

See merge request !414
parents 9cb97bdb 6085e2eb
Pipeline #1140 passed with stage
in 0 seconds
Showing with 258 additions and 109 deletions
.gitignore
......@@ -8,6 +8,8 @@
*.swp
*.swo
*~
.vscode
.idea
# Sphinx docs:
build
......
circle/dashboard/migrations/0007_groupprofile_disk_quota.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
import sizefield.models
class Migration(migrations.Migration):
dependencies = [
('dashboard', '0006_auto_20170707_1909'),
]
operations = [
migrations.AddField(
model_name='groupprofile',
name='disk_quota',
field=sizefield.models.FileSizeField(default=2147483648, help_text='Disk quota in mebibytes.', verbose_name='disk quota'),
),
]
circle/dashboard/models.py
......@@ -50,7 +50,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.models.instance import ACCESS_METHODS
from .store_api import Store, NoStoreException, NotOkException, Timeout
from .store_api import Store, NoStoreException, NotOkException
from .validators import connect_command_template_validator
logger = getLogger(__name__)
......@@ -293,6 +293,10 @@ class GroupProfile(AclBase):
unique=True, blank=True, null=True, max_length=64,
help_text=_('Unique identifier of the group at the organization.'))
description = TextField(blank=True)
disk_quota = FileSizeField(
verbose_name=_('disk quota'),
default=2048 * 1024 * 1024,
help_text=_('Disk quota in mebibytes.'))
class Meta:
ordering = ('id',)
......@@ -332,10 +336,10 @@ def create_profile(user):
try:
store = Store(user)
if store.user_exist():
profile.disk_quota = store.get_quota()['soft']
profile.save()
store.create_user(profile.smb_password, None, profile.disk_quota)
quotas = [profile.disk_quota]
quotas += [group.profile.disk_quota for group in user.groups.all()]
max_quota = max(quotas)
store.create_user(profile.smb_password, None, max_quota)
except:
logger.exception("Can't create user %s", unicode(user))
return created
......@@ -351,6 +355,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
logger.debug("Register save_org_id to djangosaml2 pre_user_save")
from djangosaml2.signals import pre_user_save
def save_org_id(sender, instance, attributes, **kwargs):
logger.debug("save_org_id called by %s", instance.username)
atr = settings.SAML_ORG_ID_ATTRIBUTE
......@@ -403,6 +408,7 @@ if hasattr(settings, 'SAML_ORG_ID_ATTRIBUTE'):
return False # User did not change
pre_user_save.connect(save_org_id)
......@@ -415,7 +421,7 @@ def update_store_profile(sender, **kwargs):
profile.disk_quota)
except NoStoreException:
logger.debug("Store is not available.")
except (NotOkException, Timeout):
except NotOkException:
logger.critical("Store is not accepting connections.")
......
circle/dashboard/static/dashboard/dashboard.less
......@@ -1092,7 +1092,7 @@ textarea[name="new_members"] {
vertical-align: middle;
}
.disk-resize-btn {
.disk-resize-btn, .disk-export-btn {
margin-right: 5px;
}
......
circle/dashboard/store_api.py
......@@ -14,19 +14,20 @@
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from os.path import splitext
import json
import logging
from urlparse import urljoin
from datetime import datetime
from django.http import Http404
import os
from datetime import datetime
from django.conf import settings
from django.http import Http404
from os.path import splitext
from requests import get, post, codes
from requests.exceptions import Timeout # noqa
from sizefield.utils import filesizeformat
from storage.models import Disk
logger = logging.getLogger(__name__)
......@@ -44,18 +45,12 @@ class NoStoreException(StoreApiException):
pass
class NoOrgIdException(StoreApiException):
pass
class Store(object):
def __init__(self, user, default_timeout=0.5):
self.store_url = settings.STORE_URL
if not self.store_url:
if not self.store_url or not user.profile.org_id:
raise NoStoreException
if not user.profile.org_id:
raise NoOrgIdException
self.username = 'u-%s' % user.profile.org_id
self.request_args = {'verify': settings.STORE_VERIFY_SSL}
if settings.STORE_SSL_AUTH:
......@@ -108,6 +103,15 @@ class Store(object):
else:
return result
def get_disk_images(self, path='/'):
images = []
file_list = self.list(path, process=False)
export_formats = [item[0] for item in Disk.EXPORT_FORMATS]
for item in file_list:
if os.path.splitext(item['NAME'])[1].strip('.') in export_formats:
images.append(os.path.join(path, item['NAME']))
return images
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path, timeout=10)
return r.json()['LINK']
......
circle/dashboard/templates/dashboard/_disk-list-element.html
......@@ -6,15 +6,29 @@
<span class="operation-wrapper pull-right">
{% if d.is_exportable %}
{% if op.export_disk %}
<a href="{{ op.export_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn
{% if op.export_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
</a>
{% endif %}
{% else %}
<small class="btn-xs">
{% trans "Not exportable" %}
</small>
{% endif %}
{% if d.is_resizable %}
{% if op.resize_disk %}
<a href="{{ op.resize_disk.get_url }}?disk={{d.pk}}"
<a href="{{ op.resize_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.resize_disk.effect }} operation disk-resize-btn
{% if op.resize_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.resize_disk.icon }} fa-fw-12"></i> {% trans "Resize" %}
</a>
{% else %}
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}" class="btn btn-xs btn-primary operation">
<a href="{% url "request.views.request-resize" vm_pk=instance.pk disk_pk=d.pk %}"
class="btn btn-xs btn-primary operation">
<i class="fa fa-arrows-alt fa-fw-12"></i> {% trans "Request resize" %}
</a>
{% endif %}
......@@ -24,8 +38,8 @@
</small>
{% endif %}
{% if op.remove_disk %}
<a href="{{ op.remove_disk.get_url }}?disk={{d.pk}}"
class="btn btn-xs btn-{{ op.remove_disk.effect}} operation disk-remove-btn
<a href="{{ op.remove_disk.get_url }}?disk={{ d.pk }}"
class="btn btn-xs btn-{{ op.remove_disk.effect }} operation disk-remove-btn
{% if op.remove_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.remove_disk.icon }} fa-fw-12"></i> {% trans "Remove" %}
</a>
......
circle/dashboard/templates/dashboard/vm-detail/_disk-operations.html
{% load i18n %}
{% for op in ops %}
{% if op.is_disk_operation %}
<a href="{{op.get_url}}" class="btn btn-success btn-xs
operation operation-{{op.op}}">
<i class="fa fa-{{op.icon}} fa-fw-12"></i>
{{op.name}} </a>
{% endif %}
{% if op.is_disk_operation %}
<a href="{{ op.get_url }}" class="btn btn-success btn-xs
operation operation-{{ op.op }}">
<i class="fa fa-{{ op.icon }} fa-fw-12"></i>
{{ op.name }} </a>
{% endif %}
{% endfor %}
circle/dashboard/views/store.py
......@@ -36,7 +36,7 @@ from django.views.generic import TemplateView
from braces.views import LoginRequiredMixin
from ..store_api import (Store, NoStoreException,
NotOkException, NoOrgIdException)
NotOkException)
logger = logging.getLogger(__name__)
......@@ -71,11 +71,6 @@ class StoreList(LoginRequiredMixin, TemplateView):
return super(StoreList, self).get(*args, **kwargs)
except NoStoreException:
messages.warning(self.request, _("No store."))
except NoOrgIdException:
messages.warning(self.request,
_("Your organization ID is not set."
" To use the store, you need a"
" unique organization ID."))
except NotOkException:
messages.warning(self.request, _("Store has some problems now."
" Try again later."))
......
circle/dashboard/views/vm.py
......@@ -61,9 +61,10 @@ from .util import (
)
from ..forms import (
AclUserOrGroupAddForm, VmResourcesForm, TraitsForm, RawDataForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm, VmSaveForm,
VmAddInterfaceForm, VmCreateDiskForm, VmDownloadDiskForm,
VmImportDiskForm, VmSaveForm,
VmRenewForm, VmStateChangeForm, VmListSearchForm, VmCustomizeForm,
VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmDiskExportForm, VmDiskResizeForm, RedeployForm, VmDiskRemoveForm,
VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm,
......@@ -301,7 +302,6 @@ class VmRawDataUpdate(SuperuserRequiredMixin, UpdateView):
class VmOperationView(AjaxOperationMixin, OperationView):
model = Instance
context_object_name = 'instance' # much simpler to mock object
......@@ -350,7 +350,6 @@ class VmRemoveInterfaceView(FormOperationMixin, VmOperationView):
class VmAddInterfaceView(FormOperationMixin, VmOperationView):
op = 'add_interface'
form_class = VmAddInterfaceForm
show_in_toolbar = False
......@@ -391,7 +390,6 @@ class VmDiskModifyView(FormOperationMixin, VmOperationView):
class VmCreateDiskView(FormOperationMixin, VmOperationView):
op = 'create_disk'
form_class = VmCreateDiskForm
show_in_toolbar = False
......@@ -408,8 +406,22 @@ class VmCreateDiskView(FormOperationMixin, VmOperationView):
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmImportDiskView(FormOperationMixin, VmOperationView):
op = 'import_disk'
form_class = VmImportDiskForm
show_in_toolbar = False
icon = 'upload'
effect = "success"
is_disk_operation = True
with_reload = True
def get_form_kwargs(self):
val = super(VmImportDiskView, self).get_form_kwargs()
val.update({'user': self.request.user})
return val
class VmDownloadDiskView(FormOperationMixin, VmOperationView):
op = 'download_disk'
form_class = VmDownloadDiskForm
show_in_toolbar = False
......@@ -420,7 +432,6 @@ class VmDownloadDiskView(FormOperationMixin, VmOperationView):
class VmMigrateView(FormOperationMixin, VmOperationView):
op = 'migrate'
icon = 'truck'
effect = 'info'
......@@ -458,7 +469,6 @@ class VmMigrateView(FormOperationMixin, VmOperationView):
class VmPortRemoveView(FormOperationMixin, VmOperationView):
template_name = 'dashboard/_vm-remove-port.html'
op = 'remove_port'
show_in_toolbar = False
......@@ -487,7 +497,6 @@ class VmPortRemoveView(FormOperationMixin, VmOperationView):
class VmPortAddView(FormOperationMixin, VmOperationView):
op = 'add_port'
show_in_toolbar = False
with_reload = True
......@@ -514,7 +523,6 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
class VmSaveView(FormOperationMixin, VmOperationView):
op = 'save_as_template'
icon = 'save'
effect = 'info'
......@@ -570,7 +578,7 @@ class TokenOperationView(OperationView):
User can do the action with a valid token instead of logging in.
"""
token_max_age = 3 * 24 * 3600
redirect_exception_classes = (PermissionDenied, SuspiciousOperation, )
redirect_exception_classes = (PermissionDenied, SuspiciousOperation,)
@classmethod
def get_salt(cls):
......@@ -642,7 +650,6 @@ class TokenOperationView(OperationView):
class VmRenewView(FormOperationMixin, TokenOperationView, VmOperationView):
op = 'renew'
icon = 'calendar'
effect = 'success'
......@@ -769,7 +776,11 @@ vm_ops = OrderedDict([
extra_bases=[TokenOperationView],
op='destroy', icon='times', effect='danger')),
('create_disk', VmCreateDiskView),
('import_disk', VmImportDiskView),
('download_disk', VmDownloadDiskView),
('export_disk', VmDiskModifyView.factory(
op='export_disk', form_class=VmDiskExportForm,
icon='download', effect='info')),
('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")),
......@@ -1025,7 +1036,6 @@ class VmList(LoginRequiredMixin, FilterMixin, ListView):
class VmCreate(LoginRequiredMixin, TemplateView):
form_class = VmCustomizeForm
form = None
......
circle/storage/migrations/0003_auto_20200424_2000.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2020-04-24 20:00
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('storage', '0002_disk_bus'),
]
operations = [
migrations.AlterModelOptions(
name='disk',
options={'ordering': ['name'], 'permissions': (('create_empty_disk', 'Can create an empty disk.'), ('download_disk', 'Can download a disk.'), ('resize_disk', 'Can resize a disk.'), ('import_disk', 'Can import a disk.'), ('export_disk', 'Can export a disk.')), 'verbose_name': 'disk', 'verbose_name_plural': 'disks'},
),
]
circle/storage/models.py
......@@ -20,31 +20,30 @@
from __future__ import unicode_literals
import logging
from os.path import join
import uuid
import re
import re
from celery.contrib.abortable import AbortableAsyncResult
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from celery.exceptions import TimeoutError
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db.models import (Model, BooleanField, CharField, DateTimeField,
ForeignKey)
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel
from os.path import join
from sizefield.models import FileSizeField
from .tasks import local_tasks, storage_tasks
from celery.exceptions import TimeoutError
from common.models import (
WorkerNotFound, HumanReadableException, humanize_exception, method_cache
)
from .tasks import local_tasks, storage_tasks
logger = logging.getLogger(__name__)
class DataStore(Model):
"""Collection of virtual disks.
"""
name = CharField(max_length=100, unique=True, verbose_name=_('name'))
......@@ -119,12 +118,15 @@ class DataStore(Model):
class Disk(TimeStampedModel):
"""A virtual disk.
"""
TYPES = [('qcow2-norm', 'qcow2 normal'), ('qcow2-snap', 'qcow2 snapshot'),
('iso', 'iso'), ('raw-ro', 'raw read-only'), ('raw-rw', 'raw')]
BUS_TYPES = (('virtio', 'virtio'), ('ide', 'ide'), ('scsi', 'scsi'))
EXPORT_FORMATS = (('vmdk', _('VMware disk image')),
('qcow2', _('QEMU disk image')),
('vdi', _('VirtualBox disk image')),
('vpc', _('HyperV disk image')))
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True,
verbose_name=_("filename"))
......@@ -149,7 +151,9 @@ class Disk(TimeStampedModel):
permissions = (
('create_empty_disk', _('Can create an empty disk.')),
('download_disk', _('Can download a disk.')),
('resize_disk', _('Can resize a disk.'))
('resize_disk', _('Can resize a disk.')),
('import_disk', _('Can import a disk.')),
('export_disk', _('Can export a disk.'))
)
class DiskError(HumanReadableException):
......@@ -467,6 +471,30 @@ class Disk(TimeStampedModel):
disk.save()
return disk
@classmethod
def import_disk(cls, user, name, download_link, timeout=3600):
params = {'name': name,
'type': 'qcow2-norm'}
disk = cls.create(user, **params)
queue_name = disk.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.import_disk.apply_async(
args=[disk.get_disk_desc(), download_link],
queue=queue_name
)
disk_size = remote.get(timeout=timeout)
disk.size = disk_size
disk.is_ready = True
disk.save()
return disk
def export(self, format, upload_link, timeout=3600):
exported_name = self.name if self.name != '' else self.filename
queue_name = self.get_remote_queue_name('storage', priority='slow')
storage_tasks.export.apply_async(
args=[self.get_disk_desc(), format, exported_name, upload_link],
queue=queue_name).get(timeout=timeout)
def destroy(self, user=None, task_uuid=None):
if self.destroyed:
return False
......@@ -549,4 +577,8 @@ class Disk(TimeStampedModel):
@property
def is_resizable(self):
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap', )
return self.type in ('qcow2-norm', 'raw-rw', 'qcow2-snap',)
@property
def is_exportable(self):
return self.type in ('qcow2-norm', 'qcow2-snap', 'raw-rw', 'raw-ro')
circle/storage/tasks/storage_tasks.py
......@@ -38,6 +38,16 @@ def download(disk_desc, url):
pass
@celery.task(name='storagedriver.import_disk')
def import_disk(disk_desc, url):
pass
@celery.task(name='storagedriver.export')
def export(disk_desc, format):
pass
@celery.task(name='storagedriver.delete')
def delete(path):
pass
......
requirements/base.txt
cryptography==2.0
cryptography==2.7
amqp==1.4.7
anyjson==0.3.3
arrow==0.7.0
billiard==3.3.0.20
bpython==0.14.1
celery==3.1.18
Django==1.11.6
Django==1.11.25
django-appconf==1.0.2
django-autocomplete-light==3.2.9
django-braces==1.11.0
......@@ -26,9 +26,9 @@ logutils==0.3.3
MarkupSafe==0.23
netaddr==0.7.14
pip-tools==0.3.6
psycopg2==2.6
psycopg2==2.8.3
Pygments==2.0.2
pylibmc==1.4.3
pylibmc==1.6.0
python-dateutil==2.4.2
pyinotify==0.9.5
pyotp==2.1.1
......
requirements/production.txt
# Pro-tip: Try not to put anything here. There should be no dependency in
# production that isn't in development.
-r base.txt
uWSGI==2.0.13.1
uWSGI==2.0.18
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment