Skip to content

CIRCLE3 / cloud

Go to a project
Commit d7c3c84c by Carpoon
Options

VM import export funtionailty 

VM import export funtionailty
Extend Export disk functions
parent 34920b70
Showing with 871 additions and 29 deletions
circle/dashboard/templates/_import_vm_choose.html 0 → 100644
{% load i18n %}
<form action="{% url "dashboard.views.template-import" %}" method="POST"
id="template-choose-form">
{% csrf_token %}
<div class="template-choose-list">
{% for t in templates %}
<div class="panel panel-default template-choose-list-element">
<input type="radio" name="parent" value="{{ t.pk }}"/>
{{ t.name }} - {{ t.system }}
<small>Cores: {{ t.num_cores }} RAM: {{ t.ram_size }}</small>
<div class="clearfix"></div>
</div>
{% endfor %}
<button type="submit" id="template-choose-next-button" class="btn btn-success pull-right">{% trans "Next" %}</button>
<div class="clearfix"></div>
</div>
</form>
<script>
$(function() {
$(".template-choose-list-element").click(function() {
$("input", $(this)).prop("checked", true);
});
$(".template-choose-list-element").hover(
function() {
$("small", $(this)).stop().fadeIn(200);
},
function() {
$("small", $(this)).stop().fadeOut(200);
}
);
});
</script>
circle/dashboard/templates/dashboard/_disk-list-element.html
...@@ -15,18 +15,13 @@ ...@@ -15,18 +15,13 @@
{% if op.export_disk.disabled %}disabled{% endif %}"> {% if op.export_disk.disabled %}disabled{% endif %}">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %} <i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Export" %}
</a> </a>
<a href="/image-dl/{{ d.filename }}.vmdk"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn"> {% for export in d.exporteddisk_set.all %}
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "VMDK" %} <a href="/image-dl/{{ export.filename }}"
</a>
<a href="/image-dl/{{ d.filename }}.vdi"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "VDI" %}
</a>
<a href="/image-dl/{{ d.filename }}.vdi"
class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn"> class="btn btn-xs btn-{{ op.export_disk.effect }} operation disk-export-btn">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "VPC" %} <i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans export.format %}
</a> </a>
{% endfor %}
{% endif %} {% endif %}
{% else %} {% else %}
<small class="btn-xs"> <small class="btn-xs">
......
circle/dashboard/templates/dashboard/_template-choose.html
...@@ -26,6 +26,12 @@ ...@@ -26,6 +26,12 @@
{% trans "Create a new base VM without disk" %} {% trans "Create a new base VM without disk" %}
</div> </div>
{% endif %} {% endif %}
{% if perms.vm.import_template %}
<div class="panel panel-default template-choose-list-element">
<input type="radio" name="parent" value="import_vm"/>
{% trans "Import a VM" %}
</div>
{% endif %}
<button type="submit" id="template-choose-next-button" class="btn btn-success pull-right">{% trans "Next" %}</button> <button type="submit" id="template-choose-next-button" class="btn btn-success pull-right">{% trans "Next" %}</button>
<div class="clearfix"></div> <div class="clearfix"></div>
</div> </div>
......
circle/dashboard/templates/dashboard/template-import.html 0 → 100644
{% load crispy_forms_tags %}
{% load i18n %}
<p class="text-muted">
{% trans "Import a previously exported VM from the user store." %}
</p>
<p class="alert alert-info">
{% trans "Please don't forget to add network interfaces, as those won't be imported!" %}
</p>
<form method="POST" action="{% url "dashboard.views.template-import" %}">
{% csrf_token %}
{% crispy form %}
</form>
circle/dashboard/templates/dashboard/vm-detail.html
...@@ -231,6 +231,13 @@ ...@@ -231,6 +231,13 @@
<i class="fa fa-cloud-upload fa-2x"></i><br> <i class="fa fa-cloud-upload fa-2x"></i><br>
{% trans "Cloud-init" %}</a> {% trans "Cloud-init" %}</a>
</li> </li>
{% if perms.vm.export_vm %}
<li>
<a href="#exports" data-toggle="pill" data-target="#_exports" class="text-center">
<i class="fa fa fa-cloud-download fa-2x"></i><br>
{% trans "Exports" %}</a>
</li>
{% endif %}
<li> <li>
<a href="#activity" data-toggle="pill" data-target="#_activity" class="text-center" <a href="#activity" data-toggle="pill" data-target="#_activity" class="text-center"
data-activity-url="{% url "dashboard.views.vm-activity-list" instance.pk %}"> data-activity-url="{% url "dashboard.views.vm-activity-list" instance.pk %}">
...@@ -253,6 +260,10 @@ ...@@ -253,6 +260,10 @@
<hr class="js-hidden"/> <hr class="js-hidden"/>
<div class="not-tab-pane" id="_activity">{% include "dashboard/vm-detail/activity.html" %}</div> <div class="not-tab-pane" id="_activity">{% include "dashboard/vm-detail/activity.html" %}</div>
<hr class="js-hidden"/> <hr class="js-hidden"/>
{% if perms.vm.export_vm %}
<div class="not-tab-pane" id="_exports">{% include "dashboard/vm-detail/exports.html" %}</div>
<hr class="js-hidden"/>
{% endif %}
</div> </div>
</div> </div>
</div> </div>
......
circle/dashboard/templates/dashboard/vm-detail/exports.html 0 → 100644
{% load i18n %}
{% load sizefieldtags %}
{% load crispy_forms_tags %}
{% load static %}
<div>
<h3>
{% trans "Exports" %}
</h3>
<div class="clearfix"></div>
{% if not instance.exportedvm_set.all %}
{% trans "No exports are created." %}
{% endif %}
{% for export in instance.exportedvm_set.all %}
<h4 class="list-group-item-heading dashboard-vm-details-network-h3">
<i class="fa fa-file"></i> {{ export.name }} exportend on {{ export.created }}
<a href="/image-dl/{{ export.filename }}" class="btn btn-info operation disk-export-btn">
<i class="fa fa-{{ op.export_disk.icon }} fa-fw-12"></i> {% trans "Download" %}
</a>
<a href="{% url "dashboard.views.exportedvm-delete" pk=export.pk %}" class="btn btn-danger operation disk-export-btn">
<i class="fa fa-times fa-fw-12"></i> {% trans "Delete" %}
</a>
</h4>
{% endfor %}
</div>
circle/dashboard/views/__init__.py
...@@ -16,3 +16,4 @@ from .storage import * ...@@ -16,3 +16,4 @@ from .storage import *
from request import * from request import *
from .message import * from .message import *
from .autocomplete import * from .autocomplete import *
from .exam import *
circle/dashboard/views/template.py
...@@ -14,14 +14,23 @@ ...@@ -14,14 +14,23 @@
# #
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import subprocess
import sys
from datetime import timedelta from datetime import timedelta
import json import json
import logging import logging
from glob import glob
from hashlib import sha256, sha1
from shutil import move, rmtree
from string import Template from string import Template
from tarfile import TarFile
from time import sleep, time
from xml.etree.cElementTree import parse as XMLparse
from xml.dom import NotFoundErr from xml.dom import NotFoundErr
from django.conf import settings
from django.contrib import messages from django.contrib import messages
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.contrib.messages.views import SuccessMessageMixin from django.contrib.messages.views import SuccessMessageMixin
...@@ -35,19 +44,21 @@ from django.utils.translation import ugettext as _, ugettext_noop ...@@ -35,19 +44,21 @@ from django.utils.translation import ugettext as _, ugettext_noop
from django.views.generic import ( from django.views.generic import (
TemplateView, CreateView, UpdateView, TemplateView, CreateView, UpdateView,
) )
#import magic
from ..store_api import Store, NoStoreException
from braces.views import ( from braces.views import (
LoginRequiredMixin, PermissionRequiredMixin, LoginRequiredMixin, PermissionRequiredMixin,
) )
from django_tables2 import SingleTableView from django_tables2 import SingleTableView
from vm.models import ( from vm.models import (
InstanceTemplate, InterfaceTemplate, Instance, Lease, InstanceActivity InstanceTemplate, InterfaceTemplate, Instance, Lease, InstanceActivity, ExportedVM, OS_TYPES
) )
from storage.models import Disk from storage.models import Disk, DataStore
from ..forms import ( from ..forms import (
TemplateForm, TemplateListSearchForm, AclUserOrGroupAddForm, LeaseForm, TemplateForm, TemplateListSearchForm, AclUserOrGroupAddForm, LeaseForm, TemplateImportForm,
) )
from ..tables import TemplateListTable, LeaseListTable from ..tables import TemplateListTable, LeaseListTable
...@@ -107,6 +118,8 @@ class TemplateChoose(LoginRequiredMixin, TemplateView): ...@@ -107,6 +118,8 @@ class TemplateChoose(LoginRequiredMixin, TemplateView):
template = request.POST.get("parent") template = request.POST.get("parent")
if template == "base_vm": if template == "base_vm":
return redirect(reverse("dashboard.views.template-create")) return redirect(reverse("dashboard.views.template-create"))
elif template == "import_vm":
return redirect(reverse("dashboard.views.template-import"))
elif template is None: elif template is None:
messages.warning(request, _("Select an option to proceed.")) messages.warning(request, _("Select an option to proceed."))
return redirect(reverse("dashboard.views.template-choose")) return redirect(reverse("dashboard.views.template-choose"))
...@@ -193,6 +206,281 @@ class TemplateCreate(SuccessMessageMixin, CreateView): ...@@ -193,6 +206,281 @@ class TemplateCreate(SuccessMessageMixin, CreateView):
return reverse_lazy("dashboard.views.template-list") return reverse_lazy("dashboard.views.template-list")
class TemplateImport(LoginRequiredMixin, TemplateView):
form_class = TemplateImportForm
def get_template_names(self):
if self.request.is_ajax():
return ['dashboard/_modal.html']
else:
return ['dashboard/nojs-wrapper.html']
def get(self, request, form=None, *args, **kwargs):
if not request.user.has_perm('vm.import_template'):
raise PermissionDenied()
try:
Store(request.user)
except NoStoreException:
raise PermissionDenied
if form is None:
form = self.form_class(user=request.user)
context = self.get_context_data(**kwargs)
context.update({
'template': 'dashboard/template-import.html',
'box_title': _('Import a VM to a template'),
'form': form,
'ajax_title': True,
})
return self.render_to_response(context)
def post(self, request, *args, **kwargs):
if not request.user.has_module_perms('vm.import_template'):
raise PermissionDenied()
try:
store = Store(request.user)
except NoStoreException:
raise PermissionDenied()
form = self.form_class(request.POST, user=request.user)
if form.is_valid():
datastore = DataStore.objects.filter(name=settings.EXPORT_DATASTORE).get()
ova_path = form.cleaned_data["ova_path"]
template_name = form.cleaned_data["template_name"]
url, port = store.request_ssh_download(ova_path)
ova_filename = re.split('[:/]', url)[-1]
tmp_path = os.path.join(datastore.path, "tmp", ova_filename + str(time()))
ova_file = os.path.join(tmp_path, ova_filename)
try:
os.mkdir(os.path.join(datastore.path, "tmp"))
except FileExistsError:
pass
try:
os.mkdir(tmp_path)
except FileExistsError:
pass
if settings.STORE_SSH_MODE == "scp":
cmdline = ['scp', '-B', '-P', str(port), url, ova_file]
if settings.STORE_IDENTITY_FILE is not None and settings.STORE_IDENTITY_FILE != "":
cmdline.append("-i")
cmdline.append(settings.STORE_IDENTITY_FILE)
elif settings.STORE_SSH_MODE == "rsync":
cmdline = ["rsync", "-qLS", url, ova_file]
cmdline.append("-e")
if settings.STORE_IDENTITY_FILE is not None:
cmdline.append("ssh -i %s -p %s" % (settings.STORE_IDENTITY_FILE, str(port)))
else:
cmdline.append("ssh -p %s" % str(port))
else:
logger.error("Invalid mode for disk export: %s" % settings.STORE_SSH_MODE)
raise Exception("Invalid mode for disk export: %s" % settings.STORE_SSH_MODE)
logger.debug("Calling file transfer with command line: %s" % str(cmdline))
try:
# let's try the file transfer 5 times, it may be an intermittent network issue
for i in range(4, -1, -1):
proc = subprocess.Popen(cmdline)
while proc.poll() is None:
sleep(2)
if proc.returncode == 0:
break
else:
logger.error("Copy over ssh failed with return code: %s, will try %s more time(s)..." % (str(proc.returncode), str(i)))
if proc.stdout is not None:
logger.info(proc.stdout.read())
if proc.stdout is not None:
logger.error(proc.stderr.read())
with TarFile.open(name=ova_file, mode="r") as tar:
if sys.version_info >= (3, 12):
tar.extractall(path=tmp_path, filter='data')
else:
for tarinfo in tar:
if tarinfo.name.startswith("..") or tarinfo.name.startswith("/") or tarinfo.name.find():
raise Exception("import template: invalid path in tar file")
if tarinfo.isreg():
tar.extract(tarinfo, path=tmp_path, set_attrs=False)
elif tarinfo.isdir():
tar.extract(tarinfo, path=tmp_path, set_attrs=False)
else:
raise Exception("import template: invalid file type in tar file")
os.unlink(ova_file)
mf = glob(os.path.join(tmp_path, "*.mf"))
if len(mf) > 1:
logger.error("import template: Invalid ova: multiple mf files!")
messages.error("import template: Invalid ova: multiple mf files!")
raise Exception("Invalid ova: multiple mf files")
elif len(mf) == 1:
with open(os.path.join(tmp_path, mf[0]), 'r') as mf_file:
def compute_sha256(file_name):
hash_sha256 = sha256()
with open(file_name, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_sha256.update(chunk)
return hash_sha256.hexdigest()
def compute_sha1(file_name):
hash_sha1 = sha1()
with open(file_name, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_sha1.update(chunk)
return hash_sha1.hexdigest()
for line in mf_file:
if line.startswith("SHA1"):
line_split = line.split("=")
filename = line_split[0][4:].strip()[1:-1]
hash_value = line_split[1].strip()
if compute_sha1(os.path.join(tmp_path, filename)) != hash_value:
logger.error("import template: mf: hash check failed!")
messages.error("import template: mf: hash check failed!")
raise Exception("import template: mf: hash check failed!")
else:
logger.info("%s passed hash test" % filename)
elif line.startswith("SHA256"):
line_split = line.split("=")
filename = line_split[0][6:].strip()[1:-1]
hash_value = line_split[1].strip()
if compute_sha256(os.path.join(tmp_path, filename)) != hash_value:
logger.error("import template: mf: hash check failed!")
messages.error("import template: mf: hash check failed!")
else:
logger.info("%s passed hash test" % filename)
else:
logger.error("import template: mf: Invalid hash algorythm!")
messages.error("import template: mf: Invalid hash algorythm!")
os.unlink(os.path.join(tmp_path, mf[0]))
ovf = glob(os.path.join(tmp_path, "*.ovf"))
if len(ovf) != 1:
logger.error("import template: Invalid ova: multiple ovf files!")
messages.error("import template: Invalid ova: multiple ovf files!")
raise Exception("Invalid ova: multiple ovf files")
xml = XMLparse(ovf[0])
xml_root = xml.getroot()
files = {}
disks = {}
disks_circle = []
xml_references = xml_root.findall("{http://schemas.dmtf.org/ovf/envelope/2}References")
if len(xml_references) == 1:
logger.error(xml_references)
for xml_reference in xml_references[0].findall("{http://schemas.dmtf.org/ovf/envelope/2}File"):
files[xml_reference.get("{http://schemas.dmtf.org/ovf/envelope/2}id")] = xml_reference.get("{http://schemas.dmtf.org/ovf/envelope/2}href")
logger.error(files)
xml_disk_section = xml_root.findall("{http://schemas.dmtf.org/ovf/envelope/2}DiskSection")
if len(xml_disk_section) == 1:
for disk in xml_disk_section[0].findall("{http://schemas.dmtf.org/ovf/envelope/2}Disk"):
disks[disk.get("{http://schemas.dmtf.org/ovf/envelope/2}diskId")] = {"name": files[disk.get("{http://schemas.dmtf.org/ovf/envelope/2}fileRef")], "type": disk.get("{http://schemas.dmtf.org/ovf/envelope/2}format") }
logger.error(disks)
xml_VirtualSystem = xml_root.findall("{http://schemas.dmtf.org/ovf/envelope/2}VirtualSystem")[0]
ovf_os = xml_VirtualSystem.findall("{http://schemas.dmtf.org/ovf/envelope/2}OperatingSystemSection")[0].findall("{http://schemas.dmtf.org/ovf/envelope/2}Description")[0].text
logger.error(ovf_os)
arch_id = xml_VirtualSystem.findall("{http://schemas.dmtf.org/ovf/envelope/2}OperatingSystemSection")[0].get("{http://schemas.dmtf.org/ovf/envelope/2}id")
if arch_id == "102":
arch = "x86_64"
elif arch_id == 0 or arch_id == 1:
arch = "i686"
else:
os_type: str = OS_TYPES[int(arch_id)]
if os_type.endswith("64-Bit"):
arch = "x86_64"
else:
arch = "i686"
try:
ovf_description = xml_VirtualSystem.findall("{http://schemas.dmtf.org/ovf/envelope/2}Description")[0].text
except Exception as e:
logger.error("Couldn't load description from ovf: %s" % e)
ovf_description = ""
xml_hardware = xml_VirtualSystem.findall("{http://schemas.dmtf.org/ovf/envelope/2}VirtualHardwareSection")[0]
for item in xml_hardware.iter():
if item.tag == "{http://schemas.dmtf.org/ovf/envelope/2}Item":
resource_type = item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData}ResourceType")[0].text
# CPU
if resource_type == "3":
cores = int(item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData}VirtualQuantity")[0].text)
logger.info("import ovf: cores: %s" % cores)
# memory
if resource_type == "4":
memory = int(item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData}VirtualQuantity")[0].text)
try:
unit = item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData}VirtualQuantityUnits")[0].text.lower()
except:
try:
unit = item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData}AllocationUnits")[0].text.lower()
except:
raise Exception("No unit for memory.")
unit_split = unit.split("*")
unit_base = unit_split[0].strip()
if unit_base in ["bytes", "kilobytes", "megabytes", "gigayytes"]:
if unit_base == "kilobytes":
memory = memory * 1000
elif unit_base == "megabytes":
memory = memory * 1000 * 1000
elif unit_base == "gigabytes":
memory = memory * 1000 * 1000 * 1000
else:
raise Exception("Invalid unit for memory.")
if len(unit_split) == 2:
unit_numbers = unit_split[1].strip().split("^")
memory = memory * (int(unit_numbers[0].strip()) ** int(unit_numbers[1].strip()))
memory = int(memory / 1024 / 1024)
logger.info("import ovf: memory: %s MiB" % memory)
elif item.tag == "{http://schemas.dmtf.org/ovf/envelope/2}StorageItem":
if item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_StorageAllocationSettingData.xsd}ResourceType")[0].text.lower() == str(17):
disk_no = item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_StorageAllocationSettingData.xsd}HostResource")[0].text.split("/")[-1]
disk_name = item.findall("{http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_StorageAllocationSettingData.xsd}Caption")[0].text
datastore = DataStore.objects.filter(name='default').get()
circle_disk = Disk.create(datastore=datastore, type="qcow2-norm", name=disk_name)
imported_path = os.path.join(tmp_path, disks[disk_no]["name"])
#with magic.Magic() as m:
# ftype = m.id_filename(imported_path)
#if 'qcow' in ftype.lower():
# move(imported_path, circle_disk.filename)
#else:
logger.debug("Calling qemu-img with command line: %s" % str(cmdline))
cmdline = ['ionice', '-c', 'idle',
'qemu-img', 'convert',
'-m', '4', '-O', 'qcow2',
imported_path,
os.path.join(datastore.path, circle_disk.filename)]
circle_disk.is_ready = True
circle_disk.save()
logger.debug("Calling qemu-img with command line: %s" % str(cmdline))
subprocess.check_output(cmdline)
disks_circle.append(circle_disk)
template = InstanceTemplate(name=template_name,
access_method='ssh',
description=ovf_description,
system=ovf_os,
num_cores=cores,
num_cores_max=cores,
ram_size=memory,
max_ram_size=memory,
arch=arch,
priority=0,
owner=self.request.user,
lease=form.cleaned_data["lease"],
)
template.save()
for disk in disks_circle:
template.disks.add(disk)
template.save()
return redirect(template.get_absolute_url())
except Exception as e:
logger.error(e)
raise
finally:
if os.path.exists(ova_path):
os.unlink(ova_path)
if os.path.exists(tmp_path):
rmtree(tmp_path)
else:
return self.get(request, form, *args, **kwargs)
class TemplateREST(APIView): class TemplateREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
...@@ -684,3 +972,19 @@ class TransferTemplateOwnershipView(TransferOwnershipView): ...@@ -684,3 +972,19 @@ class TransferTemplateOwnershipView(TransferOwnershipView):
'class="btn btn-success btn-small">Accept</a>') 'class="btn btn-success btn-small">Accept</a>')
token_url = 'dashboard.views.template-transfer-ownership-confirm' token_url = 'dashboard.views.template-transfer-ownership-confirm'
template = "dashboard/template-tx-owner.html" template = "dashboard/template-tx-owner.html"
class ExportedVMDelete(DeleteViewBase):
model = ExportedVM
success_message = _("Exported VM successfully deleted.")
def get_success_url(self):
#return reverse("dashboard.views.vm-list")
return reverse_lazy("dashboard.views.detail", kwargs={'pk': self.vm_pk})
def check_auth(self):
if not self.get_object().vm.has_level(self.request.user, self.level):
raise PermissionDenied()
def delete_obj(self, request, *args, **kwargs):
self.get_object().delete()
circle/dashboard/views/vm.py
...@@ -49,7 +49,7 @@ from braces.views import SuperuserRequiredMixin, LoginRequiredMixin ...@@ -49,7 +49,7 @@ from braces.views import SuperuserRequiredMixin, LoginRequiredMixin
from storage.tasks import storage_tasks from storage.tasks import storage_tasks
from vm.tasks.local_tasks import abortable_async_downloaddisk_operation from vm.tasks.local_tasks import abortable_async_downloaddisk_operation
from vm.operations import (DeployOperation, DestroyOperation, DownloadDiskOperation, RemovePortOperation, ShutdownOperation, RenewOperation, from vm.operations import (DeployOperation, DestroyOperation, DownloadDiskOperation, RemovePortOperation, ShutdownOperation, RenewOperation,
ResizeDiskOperation, RemoveDiskOperation, SleepOperation, WakeUpOperation, AddPortOperation, SaveAsTemplateOperation, ResizeDiskOperation, RemoveDiskOperation, SleepOperation, WakeUpOperation, AddPortOperation, SaveAsTemplateOperation, ExportVmOperation,
) )
from common.models import ( from common.models import (
...@@ -77,7 +77,7 @@ from ..forms import ( ...@@ -77,7 +77,7 @@ from ..forms import (
VmMigrateForm, VmDeployForm, VmMigrateForm, VmDeployForm,
VmPortRemoveForm, VmPortAddForm, VmPortRemoveForm, VmPortAddForm,
VmRemoveInterfaceForm, VmRemoveInterfaceForm,
VmRenameForm, VmRenameForm, VmExportViewForm,
) )
from django.views.generic.edit import FormMixin from django.views.generic.edit import FormMixin
from request.models import TemplateAccessType, LeaseType from request.models import TemplateAccessType, LeaseType
...@@ -168,6 +168,7 @@ class DownloadPersistentDiskREST(APIView): ...@@ -168,6 +168,7 @@ class DownloadPersistentDiskREST(APIView):
return JsonResponse(serializer.data, status=201) return JsonResponse(serializer.data, status=201)
return JsonResponse(serializer.errors, status=400) return JsonResponse(serializer.errors, status=400)
class VlanREST(APIView): class VlanREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
...@@ -209,6 +210,7 @@ class HotplugMemSetREST(APIView): ...@@ -209,6 +210,7 @@ class HotplugMemSetREST(APIView):
serializer = InstanceSerializer(instance) serializer = InstanceSerializer(instance)
return JsonResponse(serializer.data, status=201) return JsonResponse(serializer.data, status=201)
class HotplugVCPUSetREST(APIView): class HotplugVCPUSetREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
...@@ -223,6 +225,7 @@ class HotplugVCPUSetREST(APIView): ...@@ -223,6 +225,7 @@ class HotplugVCPUSetREST(APIView):
serializer = InstanceSerializer(instance) serializer = InstanceSerializer(instance)
return JsonResponse(serializer.data, status=201) return JsonResponse(serializer.data, status=201)
class InterfaceREST(APIView): class InterfaceREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
...@@ -466,6 +469,7 @@ class DownloadDiskREST(APIView): ...@@ -466,6 +469,7 @@ class DownloadDiskREST(APIView):
return JsonResponse(serializer.data, status=201) return JsonResponse(serializer.data, status=201)
return JsonResponse(serializer.errors, status=400) return JsonResponse(serializer.errors, status=400)
class CreateTemplateREST(APIView): class CreateTemplateREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
...@@ -483,6 +487,7 @@ class CreateTemplateREST(APIView): ...@@ -483,6 +487,7 @@ class CreateTemplateREST(APIView):
return JsonResponse(serializer.data, status=201) return JsonResponse(serializer.data, status=201)
return JsonResponse(serializer.errors, status=400) return JsonResponse(serializer.errors, status=400)
class CreateDiskREST(APIView): class CreateDiskREST(APIView):
authentication_classes = [TokenAuthentication,BasicAuthentication] authentication_classes = [TokenAuthentication,BasicAuthentication]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
...@@ -988,6 +993,16 @@ class VmPortAddView(FormOperationMixin, VmOperationView): ...@@ -988,6 +993,16 @@ class VmPortAddView(FormOperationMixin, VmOperationView):
return val return val
class VmExportView(FormOperationMixin, VmOperationView):
op = 'export_vm'
form_class = VmExportViewForm
def get_form_kwargs(self):
op = self.get_op()
val = super(VmExportView, self).get_form_kwargs()
return val
class VmSaveView(FormOperationMixin, VmOperationView): class VmSaveView(FormOperationMixin, VmOperationView):
op = 'save_as_template' op = 'save_as_template'
icon = 'save' icon = 'save'
...@@ -1004,6 +1019,7 @@ class VmSaveView(FormOperationMixin, VmOperationView): ...@@ -1004,6 +1019,7 @@ class VmSaveView(FormOperationMixin, VmOperationView):
val['clone'] = True val['clone'] = True
return val return val
class CIDataUpdate(VmOperationView): class CIDataUpdate(VmOperationView):
op = 'cloudinit_change' op = 'cloudinit_change'
icon = "cloud-upload" icon = "cloud-upload"
...@@ -1278,6 +1294,9 @@ vm_ops = OrderedDict([ ...@@ -1278,6 +1294,9 @@ vm_ops = OrderedDict([
('export_disk', VmDiskModifyView.factory( ('export_disk', VmDiskModifyView.factory(
op='export_disk', form_class=VmDiskExportForm, op='export_disk', form_class=VmDiskExportForm,
icon='download', effect='info')), icon='download', effect='info')),
('export_vm', VmExportView.factory(
op='export_vm', form_class=VmExportViewForm,
icon='download', effect="info")),
('resize_disk', VmDiskModifyView.factory( ('resize_disk', VmDiskModifyView.factory(
op='resize_disk', form_class=VmDiskResizeForm, op='resize_disk', form_class=VmDiskResizeForm,
icon='arrows-alt', effect="warning")), icon='arrows-alt', effect="warning")),
......
circle/storage/models.py
...@@ -18,8 +18,8 @@ ...@@ -18,8 +18,8 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
import logging import logging
import os
import uuid import uuid
import time import time
...@@ -29,6 +29,7 @@ from celery.result import allow_join_result ...@@ -29,6 +29,7 @@ from celery.result import allow_join_result
from celery.exceptions import TimeoutError from celery.exceptions import TimeoutError
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.urls import reverse from django.urls import reverse
from django.utils import timezone
from django.db.models import (Model, BooleanField, CharField, DateTimeField, IntegerField, from django.db.models import (Model, BooleanField, CharField, DateTimeField, IntegerField,
ForeignKey) ForeignKey)
from django.db import models from django.db import models
...@@ -37,6 +38,8 @@ from django.utils.translation import ugettext_lazy as _, ugettext_noop ...@@ -37,6 +38,8 @@ from django.utils.translation import ugettext_lazy as _, ugettext_noop
from model_utils.models import TimeStampedModel from model_utils.models import TimeStampedModel
from os.path import join from os.path import join
from sizefield.models import FileSizeField from sizefield.models import FileSizeField
from django.db.models.signals import pre_delete
from django.dispatch import receiver
from common.models import ( from common.models import (
WorkerNotFound, HumanReadableException, humanize_exception, join_activity_code, method_cache WorkerNotFound, HumanReadableException, humanize_exception, join_activity_code, method_cache
...@@ -160,7 +163,7 @@ class Disk(TimeStampedModel): ...@@ -160,7 +163,7 @@ class Disk(TimeStampedModel):
('download_disk', _('Can download a disk.')), ('download_disk', _('Can download a disk.')),
('resize_disk', _('Can resize a disk.')), ('resize_disk', _('Can resize a disk.')),
('import_disk', _('Can import a disk.')), ('import_disk', _('Can import a disk.')),
('export_disk', _('Can export a disk.')) ('export_disk', _('Can export a disk.')),
) )
class DiskError(HumanReadableException): class DiskError(HumanReadableException):
...@@ -550,13 +553,14 @@ class Disk(TimeStampedModel): ...@@ -550,13 +553,14 @@ class Disk(TimeStampedModel):
queue=queue_name) queue=queue_name)
return self._run_abortable_task(remote, task) return self._run_abortable_task(remote, task)
def export_disk_to_datastore(self, task, disk_format, datastore): def export_disk_to_datastore(self, task, disk_format, datastore, folder="exports"):
queue_name = self.get_remote_queue_name('storage', priority='slow') queue_name = self.get_remote_queue_name('storage', priority='slow')
remote = storage_tasks.export_disk_to_datastore.apply_async( remote = storage_tasks.export_disk_to_datastore.apply_async(
kwargs={ kwargs={
"disk_desc": self.get_disk_desc(), "disk_desc": self.get_disk_desc(),
"disk_format": disk_format, "disk_format": disk_format,
"datastore": datastore, "datastore": datastore,
"folder": folder,
}, },
queue=queue_name) queue=queue_name)
return self._run_abortable_task(remote, task) return self._run_abortable_task(remote, task)
...@@ -670,3 +674,19 @@ class StorageActivity(ActivityModel): ...@@ -670,3 +674,19 @@ class StorageActivity(ActivityModel):
started=timezone.now(), task_uuid=task_uuid, user=user) started=timezone.now(), task_uuid=task_uuid, user=user)
act.save() act.save()
return act return act
class ExportedDisk(Model):
FORMAT = Disk.EXPORT_FORMATS
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
format = models.CharField(max_length=5, choices=FORMAT)
filename = CharField(max_length=256, unique=True, verbose_name=_("filename"))
datastore = ForeignKey(DataStore, verbose_name=_("datastore"), help_text=_("The datastore that holds the exported disk."), on_delete=models.CASCADE)
disk = ForeignKey(Disk, verbose_name=_("disk"), help_text=_("The disk that the export was made from."), on_delete=models.CASCADE)
created = DateTimeField(blank=True, default=timezone.now, null=False, editable=False)
@receiver(pre_delete, sender=ExportedDisk)
def delete_repo(sender, instance, **kwargs):
if os.path.exists(os.path.join(instance.datastore.path, "exports", instance.filename)):
os.unlink(os.path.join(instance.datastore.path, "exports", instance.filename))
circle/vm/fixtures/ova.xml.j2 0 → 100644
<?xml version="1.0"?>
<Envelope ovf:version="2.0" xml:lang="en-US" xmlns="http://schemas.dmtf.org/ovf/envelope/2" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/2" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:epasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_EthernetPortAllocationSettingData.xsd" xmlns:sasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_StorageAllocationSettingData.xsd">
<References>
{%- for disk in disks %}
<File ovf:id="file{{ loop.index }}" ovf:href="{{ disk.filename }}.vmdk"/>
{%- endfor %}
</References>
<DiskSection>
{%- for disk in disks %}
<Disk ovf:capacity="{{ disk.size }}" ovf:diskId="disk{{ loop.index }}" ovf:fileRef="file{{ loop.index }}" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized"/>
{%- endfor %}
</DiskSection>
<NetworkSection>
<Info>Logical networks used in the package</Info>
<Network ovf:name="NAT">
<Description>Logical network used by this appliance.</Description>
</Network>
</NetworkSection>
<VirtualSystem ovf:id="{{ name }}">
<Description>{{ vm.description }}</Description>
<name>{{ name }}</name>
<OperatingSystemSection ovf:id="{{ os_id }}">
<Info>Specifies the operating system installed</Info>
<Description>{{ vm.system }}</Description>
</OperatingSystemSection>
<VirtualHardwareSection>
<Info>Virtual hardware requirements for a virtual machine</Info>
<Item>
<rasd:Description>Virtual CPU</rasd:Description>
<rasd:InstanceID>1</rasd:InstanceID>
<rasd:ResourceType>3</rasd:ResourceType>
<rasd:VirtualQuantity>{{ vm.num_cores }}</rasd:VirtualQuantity>
<rasd:VirtualQuantityUnit>Count</rasd:VirtualQuantityUnit>
</Item>
<Item>
<rasd:AllocationUnits>MegaBytes</rasd:AllocationUnits>
<rasd:Description>Memory Size</rasd:Description>
<rasd:InstanceID>2</rasd:InstanceID>
<rasd:ResourceType>4</rasd:ResourceType>
<rasd:VirtualQuantity>{{ vm.ram_size }}</rasd:VirtualQuantity>
</Item>
<Item>
<rasd:Address>0</rasd:Address>
<rasd:InstanceID>3</rasd:InstanceID>
<rasd:ResourceSubType>PIIX4</rasd:ResourceSubType>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
<Item>
<rasd:Address>1</rasd:Address>
<rasd:InstanceID>4</rasd:InstanceID>
<rasd:ResourceSubType>PIIX4</rasd:ResourceSubType>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
{%- set ns = namespace(InstanceID = 4) %}{%- for disk in disks %}{%- set ns.InstanceID = ns.InstanceID + 1 %}
<StorageItem>
<sasd:AddressOnParent>{{ (loop.index - 1) % 2}}</sasd:AddressOnParent>
<sasd:Caption>{{ disk.name }}</sasd:Caption>
<sasd:Description>Disk Image</sasd:Description>
<sasd:Parent>{% if loop.index < 3 %}3{% else %}4{% endif %}</sasd:Parent>
<sasd:HostResource>/disk/disk{{ loop.index }}</sasd:HostResource>
<sasd:InstanceID>{{ ns.InstanceID }}</sasd:InstanceID>
<sasd:ResourceType>17</sasd:ResourceType>
</StorageItem>
{%- endfor %}
{%- for interface in interfaces %}{%- set ns.InstanceID = ns.InstanceID + 1 %}
<EthernetPortItem>
<epasd:AutomaticAllocation>true</epasd:AutomaticAllocation>
<epasd:Caption>{{ interface.vlan.name }}</epasd:Caption>
<epasd:Connection>NAT</epasd:Connection>
<epasd:InstanceID>{{ ns.InstanceID }}</epasd:InstanceID>
<epasd:ResourceType>10</epasd:ResourceType>
</EthernetPortItem>
{%- endfor %}
</VirtualHardwareSection>
</VirtualSystem>
</Envelope>
circle/vm/models/__init__.py
...@@ -12,6 +12,8 @@ from .instance import Instance ...@@ -12,6 +12,8 @@ from .instance import Instance
from .instance import post_state_changed from .instance import post_state_changed
from .instance import pre_state_changed from .instance import pre_state_changed
from .instance import pwgen from .instance import pwgen
from .instance import ExportedVM
from .instance import OS_TYPES
from .network import InterfaceTemplate from .network import InterfaceTemplate
from .network import Interface from .network import Interface
from .node import Node from .node import Node
...@@ -21,5 +23,5 @@ __all__ = [ ...@@ -21,5 +23,5 @@ __all__ = [
'NamedBaseResourceConfig', 'VirtualMachineDescModel', 'InstanceTemplate', 'NamedBaseResourceConfig', 'VirtualMachineDescModel', 'InstanceTemplate',
'Instance', 'post_state_changed', 'pre_state_changed', 'InterfaceTemplate', 'Instance', 'post_state_changed', 'pre_state_changed', 'InterfaceTemplate',
'Interface', 'Trait', 'Node', 'NodeActivity', 'Lease', 'node_activity', 'Interface', 'Trait', 'Node', 'NodeActivity', 'Lease', 'node_activity',
'pwgen' 'pwgen', 'ExportedVM', 'OS_TYPES',
] ]
circle/vm/models/instance.py
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# #
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
import os.path
import random, string import random, string
from contextlib import contextmanager from contextlib import contextmanager
from datetime import timedelta from datetime import timedelta
...@@ -27,13 +27,15 @@ from urllib import request ...@@ -27,13 +27,15 @@ from urllib import request
from warnings import warn from warnings import warn
from xml.dom.minidom import Text from xml.dom.minidom import Text
from django.db.models.signals import pre_delete
from django.dispatch import receiver
import django.conf import django.conf
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core import signing from django.core import signing
from django.core.exceptions import PermissionDenied from django.core.exceptions import PermissionDenied
from django.db.models import (BooleanField, CharField, DateTimeField, from django.db.models import (BooleanField, CharField, DateTimeField,
IntegerField, ForeignKey, Manager, IntegerField, ForeignKey, Manager,
ManyToManyField, SET_NULL, TextField) ManyToManyField, SET_NULL, TextField, Model)
from django.db import IntegrityError from django.db import IntegrityError
from django.dispatch import Signal from django.dispatch import Signal
from django.urls import reverse from django.urls import reverse
...@@ -64,6 +66,8 @@ from .activity import (ActivityInProgressError, InstanceActivity) ...@@ -64,6 +66,8 @@ from .activity import (ActivityInProgressError, InstanceActivity)
from .common import BaseResourceConfigModel, Lease, Variable from .common import BaseResourceConfigModel, Lease, Variable
from .network import Interface from .network import Interface
from .node import Node, Trait from .node import Node, Trait
from storage.models import DataStore
import subprocess
logger = getLogger(__name__) logger = getLogger(__name__)
...@@ -78,6 +82,131 @@ ACCESS_PROTOCOLS = django.conf.settings.VM_ACCESS_PROTOCOLS ...@@ -78,6 +82,131 @@ ACCESS_PROTOCOLS = django.conf.settings.VM_ACCESS_PROTOCOLS
ACCESS_METHODS = [(key, name) for key, (name, port, transport) ACCESS_METHODS = [(key, name) for key, (name, port, transport)
in list(ACCESS_PROTOCOLS.items())] in list(ACCESS_PROTOCOLS.items())]
# CIM operationg system types 2.54.0
OS_TYPES = {
0: "Unknown",
1: "Other",
2: "MACOS",
3: "ATTUNIX",
4: "DGUX",
5: "DECNT",
6: "Tru64 UNIX",
7: "OpenVMS",
8: "HPUX",
9: "AIX",
10: "MVS",
11: "OS400",
12: "OS/2",
13: "JavaVM",
14: "MSDOS",
15: "WIN3x",
16: "WIN95",
17: "WIN98",
18: "WINNT",
19: "WINCE",
20: "NCR3000",
21: "NetWare",
22: "OSF",
23: "DC/OS",
24: "Reliant UNIX",
25: "SCO UnixWare",
26: "SCO OpenServer",
27: "Sequent",
28: "IRIX",
29: "Solaris",
30: "SunOS",
31: "U6000",
32: "ASERIES",
33: "HP NonStop OS",
34: "HP NonStop OSS",
35: "BS2000",
36: "LINUX",
37: "Lynx",
38: "XENIX",
39: "VM",
40: "Interactive UNIX",
41: "BSDUNIX",
42: "FreeBSD",
43: "NetBSD",
44: "GNU Hurd",
45: "OS9",
46: "MACH Kernel",
47: "Inferno",
48: "QNX",
49: "EPOC",
50: "IxWorks",
51: "VxWorks",
52: "MiNT",
53: "BeOS",
54: "HP MPE",
55: "NextStep",
56: "PalmPilot",
57: "Rhapsody",
58: "Windows 2000",
59: "Dedicated",
60: "OS/390",
61: "VSE",
62: "TPF",
63: "Windows (R) Me",
64: "Caldera Open UNIX",
65: "OpenBSD",
66: "Not Applicable",
67: "Windows XP",
68: "z/OS",
69: "Microsoft Windows Server 2003",
70: "Microsoft Windows Server 2003 64-Bit",
71: "Windows XP 64-Bit",
72: "Windows XP Embedded",
73: "Windows Vista",
74: "Windows Vista 64-Bit",
75: "Windows Embedded for Point of Service",
76: "Microsoft Windows Server 2008",
77: "Microsoft Windows Server 2008 64-Bit",
78: "FreeBSD 64-Bit",
79: "RedHat Enterprise Linux",
80: "RedHat Enterprise Linux 64-Bit",
81: "Solaris 64-Bit",
82: "SUSE",
83: "SUSE 64-Bit",
84: "SLES",
85: "SLES 64-Bit",
86: "Novell OES",
87: "Novell Linux Desktop",
88: "Sun Java Desktop System",
89: "Mandriva",
90: "Mandriva 64-Bit",
91: "TurboLinux",
92: "TurboLinux 64-Bit",
93: "Ubuntu",
94: "Ubuntu 64-Bit",
95: "Debian",
96: "Debian 64-Bit",
97: "Linux 2.4.x",
98: "Linux 2.4.x 64-Bit",
99: "Linux 2.6.x",
100: "Linux 2.6.x 64-Bit",
101: "Linux 64-Bit",
102: "Other 64-Bit",
103: "Microsoft Windows Server 2008 R2",
104: "VMware ESXi",
105: "Microsoft Windows 7",
106: "CentOS 32-bit",
107: "CentOS 64-bit",
108: "Oracle Linux 32-bit",
109: "Oracle Linux 64-bit",
110: "eComStation 32-bitx",
111: "Microsoft Windows Server 2011",
113: "Microsoft Windows Server 2012",
114: "Microsoft Windows 8",
115: "Microsoft Windows 8 64-bit",
116: "Microsoft Windows Server 2012 R2",
117: "Microsoft Windows Server 2016",
118: "Microsoft Windows 8.1",
119: "Microsoft Windows 8.1 64-bit",
120: "Microsoft Windows 10",
121: "Microsoft Windows 10 64-bit",
}
CI_META_DATA_DEF = """ CI_META_DATA_DEF = """
instance-id: {{ hostname }} instance-id: {{ hostname }}
local-hostname: {{ hostname }} local-hostname: {{ hostname }}
...@@ -216,6 +345,7 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -216,6 +345,7 @@ class InstanceTemplate(AclBase, VirtualMachineDescModel, TimeStampedModel):
ordering = ('name', ) ordering = ('name', )
permissions = ( permissions = (
('create_template', _('Can create an instance template.')), ('create_template', _('Can create an instance template.')),
('import_template', _('Can import an instance template.')),
('create_base_template', ('create_base_template',
_('Can create an instance template (base).')), _('Can create an instance template (base).')),
('change_template_resources', ('change_template_resources',
...@@ -397,6 +527,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -397,6 +527,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
('set_resources', _('Can change resources of a new VM.')), ('set_resources', _('Can change resources of a new VM.')),
('create_vm', _('Can create a new VM.')), ('create_vm', _('Can create a new VM.')),
('redeploy', _('Can redeploy a VM.')), ('redeploy', _('Can redeploy a VM.')),
('export_vm', _('Can export a vm.')),
('config_ports', _('Can configure port forwards.')), ('config_ports', _('Can configure port forwards.')),
('recover', _('Can recover a destroyed VM.')), ('recover', _('Can recover a destroyed VM.')),
('emergency_change_state', _('Can change VM state to NOSTATE.')), ('emergency_change_state', _('Can change VM state to NOSTATE.')),
...@@ -634,6 +765,9 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -634,6 +765,9 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
missing_users = [] missing_users = []
instances = [] instances = []
for user_id in users: for user_id in users:
if isinstance(user_id, User):
user_instances.append(user_id)
else:
try: try:
user_instances.append(User.objects.get(profile__org_id=user_id)) user_instances.append(User.objects.get(profile__org_id=user_id))
except User.DoesNotExist: except User.DoesNotExist:
...@@ -645,8 +779,22 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -645,8 +779,22 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
for user in user_instances: for user in user_instances:
instance = cls.create_from_template(template, user, **kwargs) instance = cls.create_from_template(template, user, **kwargs)
if admin: if admin:
if hasattr(admin, '__iter__') and not isinstance(admin, str):
for admin_user in admin:
if isinstance(admin_user, User):
instance.set_level(admin_user, 'owner')
else:
instance.set_level(User.objects.get(username=admin_user), 'owner')
else:
instance.set_level(User.objects.get(username=admin), 'owner') instance.set_level(User.objects.get(username=admin), 'owner')
if operator: if operator:
if hasattr(operator, '__iter__') and not isinstance(operator, str):
for operator_user in operator:
if isinstance(operator_user, User):
instance.set_level(operator_user, 'operator')
else:
instance.set_level(User.objects.get(username=operator_user), 'operator')
else:
instance.set_level(User.objects.get(username=operator), 'operator') instance.set_level(User.objects.get(username=operator), 'operator')
instance.deploy._async(user=user) instance.deploy._async(user=user)
instances.append(instance) instances.append(instance)
...@@ -1165,3 +1313,23 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -1165,3 +1313,23 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
user=user, concurrency_check=concurrency_check, user=user, concurrency_check=concurrency_check,
readable_name=readable_name, resultant_state=resultant_state) readable_name=readable_name, resultant_state=resultant_state)
return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit) return activitycontextimpl(act, on_abort=on_abort, on_commit=on_commit)
class ExportedVM(Model):
name = CharField(blank=True, max_length=100, verbose_name=_("name"))
filename = CharField(max_length=256, unique=True, verbose_name=_("filename"))
datastore = ForeignKey(DataStore, verbose_name=_("datastore"), help_text=_("The datastore that holds the exported VM."), on_delete=models.CASCADE)
vm = ForeignKey(Instance, verbose_name=_("disk"), help_text=_("The vm that the export was made from."), on_delete=models.CASCADE)
created = DateTimeField(blank=True, default=timezone.now, null=False, editable=False)
def has_level(self, user, level):
self.vm.has_level(user, level)
def __str__(self):
return self.name
@receiver(pre_delete, sender=ExportedVM)
def delete_repo(sender, instance, **kwargs):
if os.path.exists(os.path.join(instance.datastore.path, "exports", instance.filename)):
os.unlink(os.path.join(instance.datastore.path, "exports", instance.filename))
circle/vm/operations.py
...@@ -14,9 +14,10 @@ ...@@ -14,9 +14,10 @@
# #
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>. # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from hashlib import sha256
from shutil import rmtree, move
import subprocess
import tarfile
from io import StringIO from io import StringIO
from base64 import encodestring from base64 import encodestring
from hashlib import md5 from hashlib import md5
...@@ -28,6 +29,9 @@ from urllib.parse import urlsplit ...@@ -28,6 +29,9 @@ from urllib.parse import urlsplit
import os import os
import time import time
from uuid import uuid4
from jinja2 import Environment, FileSystemLoader
from celery.contrib.abortable import AbortableAsyncResult from celery.contrib.abortable import AbortableAsyncResult
from celery.exceptions import TimeLimitExceeded, TimeoutError from celery.exceptions import TimeLimitExceeded, TimeoutError
from django.conf import settings from django.conf import settings
...@@ -47,18 +51,19 @@ from dashboard.store_api import Store, NoStoreException ...@@ -47,18 +51,19 @@ from dashboard.store_api import Store, NoStoreException
from firewall.models import Host from firewall.models import Host
from manager.scheduler import SchedulerError from manager.scheduler import SchedulerError
from monitor.client import Client from monitor.client import Client
from storage.models import Disk from storage.models import Disk, ExportedDisk
from storage.tasks import storage_tasks from storage.tasks import storage_tasks
from storage.models import DataStore from storage.models import DataStore
from .models import ( from .models import (
Instance, InstanceActivity, InstanceTemplate, Interface, Node, Instance, InstanceActivity, InstanceTemplate, Interface, Node,
NodeActivity, pwgen NodeActivity, pwgen, ExportedVM
) )
from .tasks import agent_tasks, vm_tasks from .tasks import agent_tasks, vm_tasks
from .tasks.local_tasks import ( from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation, abortable_async_instance_operation, abortable_async_node_operation,
) )
from django.conf import settings from django.conf import settings
from time import sleep
try: try:
# Python 2: "unicode" is built-in # Python 2: "unicode" is built-in
...@@ -411,7 +416,6 @@ class ImportDiskOperation(InstanceOperation): ...@@ -411,7 +416,6 @@ class ImportDiskOperation(InstanceOperation):
def _operation(self, user, name, disk_path, task): def _operation(self, user, name, disk_path, task):
store = Store(user) store = Store(user)
download_link, port = store.request_ssh_download(disk_path) download_link, port = store.request_ssh_download(disk_path)
ogging.debug(settings)
disk = Disk.import_disk(task, user, name, download_link, port, settings.STORE_IDENTITY_FILE, settings.STORE_SSH_MODE) disk = Disk.import_disk(task, user, name, download_link, port, settings.STORE_IDENTITY_FILE, settings.STORE_SSH_MODE)
self.instance.disks.add(disk) self.instance.disks.add(disk)
...@@ -441,7 +445,10 @@ class ExportDiskOperation(InstanceOperation): ...@@ -441,7 +445,10 @@ class ExportDiskOperation(InstanceOperation):
store.ssh_upload_finished(file_name, exported_name + '.' + disk_format) store.ssh_upload_finished(file_name, exported_name + '.' + disk_format)
elif export_target == "datastore": elif export_target == "datastore":
datastore = DataStore.objects.filter(name=settings.EXPORT_DATASTORE).get() datastore = DataStore.objects.filter(name=settings.EXPORT_DATASTORE).get()
export_filename = os.path.join(disk.filename + '.' + disk_format)
disk.export_disk_to_datastore(task, disk_format, datastore.path) disk.export_disk_to_datastore(task, disk_format, datastore.path)
export = ExportedDisk(name=exported_name, format=disk_format, filename=export_filename, datastore=datastore, disk=disk)
export.save()
else: else:
raise Exception("Invalid export target") raise Exception("Invalid export target")
...@@ -1991,3 +1998,159 @@ class DetachNetwork(DetachMixin, AbstractNetworkOperation): ...@@ -1991,3 +1998,159 @@ class DetachNetwork(DetachMixin, AbstractNetworkOperation):
id = "_detach_network" id = "_detach_network"
name = _("detach network") name = _("detach network")
task = vm_tasks.detach_network task = vm_tasks.detach_network
@register_operation
class ExportVmOperation(InstanceOperation):
id = 'export_vm'
name = _("export vm")
description = _("Export the virtual machine.")
required_perms = ('vm.export_vm',)
accept_states = ('STOPPED')
acl_level = "operator"
concurrency_check = True
superuser_required = False
def _operation(self, user, task, activity, exported_name, export_target):
if export_target == "user_store":
filename = exported_name
elif export_target == "datastore":
filename = str(uuid4())
else:
raise Exception("Invalid export target")
datastore = DataStore.objects.filter(name=settings.EXPORT_DATASTORE).get()
tmp_folder = str(uuid4())
tmp_path = os.path.join(datastore.path, "tmp", tmp_folder)
try:
os.mkdir(os.path.join(datastore.path, "tmp"))
except FileExistsError:
pass
try:
try:
os.mkdir(tmp_path)
except FileExistsError:
pass
for disk in self.instance.disks.all():
if not disk.ci_disk:
with activity.sub_activity(
'exporting_disk',
readable_name=create_readable(ugettext_noop("exporting disk %(name)s"), name=disk.name)
):
disk.export_disk_to_datastore(task, "vmdk", datastore.path, folder=os.path.join("tmp", tmp_folder))
with (activity.sub_activity(
'generate_ovf',
readable_name=create_readable(ugettext_noop("generate ovf"))
)):
loader = FileSystemLoader(searchpath="./vm/fixtures/")
env = Environment(loader=loader)
j2template = env.get_template("ova.xml.j2")
# see CIM os types, instance.OS_TYPES
if self.instance.arch == "x86_64":
os_id = "102"
elif self.instance.arch == "i686":
os_id = "1"
else:
os_id = "0"
output_from_parsed_template = j2template.render(
name=exported_name,
disks=self.instance.disks.all(),
vm=self.instance,
interfaces=self.instance.interface_set.all(),
os_id=os_id
)
# to save the results
with open(os.path.join(tmp_path, exported_name + ".ovf"), "w") as fh:
fh.write(output_from_parsed_template)
def compute_sha256(file_name):
hash_sha256 = sha256()
with open(file_name, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_sha256.update(chunk)
return hash_sha256.hexdigest()
with activity.sub_activity(
'generate_mf',
readable_name=create_readable(ugettext_noop("create %s.mf checksums" % exported_name))
):
with open(os.path.join(tmp_path, exported_name + ".mf"), "w") as mf:
for disk in self.instance.disks.all():
if not disk.ci_disk:
mf.write("SHA256 (%s) = %s\n" % (os.path.basename(disk.path) + ".vmdk", compute_sha256(os.path.join(tmp_path, os.path.basename(disk.path) + ".vmdk"))))
mf.write("SHA256 (%s) = %s\n" % (exported_name + ".ovf", compute_sha256(os.path.join(tmp_path, exported_name + ".ovf"))))
with activity.sub_activity(
'generate_ova',
readable_name=create_readable(ugettext_noop("create %s.ova archive" % exported_name))
):
with tarfile.open(os.path.join(tmp_path, filename + ".ova"), "w") as tar:
for disk in self.instance.disks.all():
if not disk.ci_disk:
tar.add(os.path.join(tmp_path, os.path.basename(disk.path) + ".vmdk"), arcname=os.path.basename(disk.path) + ".vmdk")
tar.add(os.path.join(tmp_path, exported_name + ".mf"), arcname=exported_name + ".mf")
tar.add(os.path.join(tmp_path, exported_name + ".ovf"), arcname=exported_name + ".ovf")
if export_target == "user_store":
with activity.sub_activity(
'upload to store',
readable_name=create_readable(ugettext_noop("upload %s.ova to store" % exported_name))
):
store = Store(user)
upload_link, port = store.request_ssh_upload()
file_path = os.path.join(tmp_path, filename + ".ova")
exported_path = filename + ".ova"
mode = settings.STORE_SSH_MODE
identity = settings.STORE_IDENTITY_FILE
if mode == "scp":
cmdline = ['scp', '-B', '-P', str(port), file_path, upload_link]
if identity is not None and identity != "":
cmdline.append("-i")
cmdline.append(identity)
elif mode == "rsync":
cmdline = ["rsync", "-qSL", file_path, upload_link]
cmdline.append("-e")
if identity is not None:
cmdline.append("ssh -i %s -p %s" % (identity, str(port)))
else:
cmdline.append("ssh -p %s" % str(port))
else:
logger.error("Invalid mode for disk export: %s" % mode)
raise Exception()
logger.debug("Calling file transfer with command line. %s" % str(cmdline))
try:
# let's try the file transfer 5 times, it may be an intermittent network issue
for i in range(4, -1, -1):
proc = subprocess.Popen(cmdline)
while proc.poll() is None:
if task.is_aborted():
raise Exception()
sleep(2)
if proc.returncode == 0:
break
else:
logger.error("Copy over ssh failed with return code: %s, will try %s more time(s)..." % (str(proc.returncode), str(i)))
if proc.stdout is not None:
logger.info(proc.stdout.read())
if proc.stdout is not None:
logger.error(proc.stderr.read())
except Exception:
proc.terminate()
logger.info("Export of disk %s aborted" % self.name)
store.ssh_upload_finished(exported_path, exported_path)
elif export_target == "datastore":
with activity.sub_activity(
'save to exports',
readable_name=create_readable(ugettext_noop("save %s.ova to exports" % exported_name))
):
exported_path = os.path.join(datastore.path, "exports", filename + ".ova")
move(os.path.join(tmp_path, filename + ".ova"), exported_path)
export = ExportedVM(name=exported_name, filename=filename + ".ova", datastore=datastore, vm=self.instance)
export.save()
else:
raise Exception("Invalid export target")
finally:
rmtree(tmp_path)
circle/vm/tasks/vm_tasks.py
...@@ -194,3 +194,7 @@ def hotplug_memset(params): ...@@ -194,3 +194,7 @@ def hotplug_memset(params):
@celery.task(name='vmdriver.hotplug_vcpuset') @celery.task(name='vmdriver.hotplug_vcpuset')
def hotplug_vcpuset(params): def hotplug_vcpuset(params):
pass pass
@celery.task(name='vmdriver.export_vm')
def export_vm(params):
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment