Skip to content

Gelencsér Szabolcs / cloud

Go to a project
Commit 3f526ee4 by Bach Dániel
Options

one: remove occi code from models.py 

fixes #122
parent 57a3fb8a
Showing with 319 additions and 190 deletions
cloud/settings.py
......@@ -194,6 +194,11 @@ AUTH_PROFILE_MODULE = 'school.Person'
import djcelery
djcelery.setup_loader()
CELERY_CACHE_BACKEND = "default"
CELERY_RESULT_BACKEND = "amqp"
CELERY_TASK_RESULT_EXPIRES = 3600
BROKER_URL = 'amqp://nyuszi:teszt@localhost:5672/django'
CELERY_ROUTES = {
'firewall.tasks.ReloadTask': {'queue': 'local'},
......@@ -203,7 +208,14 @@ CELERY_ROUTES = {
'firewall.tasks.reload_blacklist_task': {'queue': 'firewall'},
'firewall.tasks.Periodic': {'queue': 'local'},
'one.tasks.SendMailTask': {'queue': 'local'},
'one.tasks.UpdateInstanceStateTask': {'queue': 'local'}
'one.tasks.UpdateInstanceStateTask': {'queue': 'local'},
'one.tasks.UpdateDiskTask': {'queue': 'opennebula'},
'one.tasks.UpdateNetworkTask': {'queue': 'opennebula'},
'one.tasks.ChangeInstanceStateTask': {'queue': 'opennebula'},
'one.tasks.SaveAsTask': {'queue': 'opennebula'},
'one.tasks.CreateInstanceTask': {'queue': 'opennebula'},
'one.tasks.DeleteInstanceTask': {'queue': 'opennebula'},
}
CACHES = {
......
miscellaneous/celery/opennebula_celery.py 0 → 100755
#!/usr/bin/env python
import subprocess
from celery import Celery, task
import time, re
import socket
import sys
import tempfile, os, stat, re, base64, struct, logging
from celery.contrib import rdb
BROKER_URL = 'amqp://nyuszi:teszt@localhost:5672/django'
try:
from local_settings import *
except:
pass
celery = Celery('tasks', broker=BROKER_URL, backend=BROKER_URL)
celery.conf.update(
CELERY_TASK_RESULT_EXPIRES=3600,
)
def update_vm(one_id, template):
out = ""
with tempfile.NamedTemporaryFile(delete=False) as f:
os.chmod(f.name, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
tpl = u'''
<COMPUTE>
<ID>%(id)d</ID>
%(template)s
</COMPUTE>''' % {
"id": one_id,
"template": template
}
f.write(tpl)
f.close()
proc = subprocess.Popen(["/opt/occi.sh", "compute", "update",
f.name], stdout=subprocess.PIPE)
try:
(out, err) = proc.communicate()
except:
pass
os.unlink(f.name)
@task(name="one.tasks.CreateInstanceTask")
def t(name, instance_type, disk_id, network_id, ctx):
out = ''
f2 = tempfile.NamedTemporaryFile(delete=False)
f2.close()
with tempfile.NamedTemporaryFile(delete=False) as f:
os.chmod(f.name, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
tpl = u"""
<COMPUTE>
<NAME>%(name)s</NAME>
<INSTANCE_TYPE href="http://www.opennebula.org/instance_type/%(instance)s"/>
<DISK>
<STORAGE href="http://www.opennebula.org/storage/%(disk)d"/>
</DISK>
<NIC>
<NETWORK href="http://www.opennebula.org/network/%(net)d"/>
</NIC>
<CONTEXT>
%(context)s
</CONTEXT>
</COMPUTE>""" % {
"name": name,
"instance": instance_type,
"disk": disk_id,
"net": network_id,
"context": ctx,
}
f.write(tpl)
f.close()
proc = subprocess.Popen(["/opt/occi.sh compute creatE %s > %s" %
( f.name, f2.name )], shell=True)
try:
proc.communicate()
except:
pass
with open(f2.name, 'r') as f3:
out = f3.read()
os.unlink(f.name)
os.unlink(f2.name)
from xml.dom.minidom import parse, parseString
try:
x = parseString(out)
return {
'one_id': int(x.getElementsByTagName("ID")[0].childNodes[0].nodeValue),
'interfaces': [
{
'ip': x.getElementsByTagName("IP")[0].childNodes[0].nodeValue,
'mac': x.getElementsByTagName("MAC")[0].childNodes[0].nodeValue,
},
],
}
except:
pass
@task(name="one.tasks.ChangeInstanceStateTask")
def t(one_id, new_state):
update_vm(one_id, '<STATE>%s</STATE>' % (new_state, ))
@task(name="one.tasks.SaveAsTask")
def t(one_id, new_img):
update_vm(one_id, '<DISK id="0"><SAVE_AS name="%s"/></DISK>' % new_img)
@task(name="one.tasks.UpdateDiskTask")
def t():
f = tempfile.NamedTemporaryFile(delete=False)
f.close()
out=''
proc = subprocess.Popen(["/opt/occi.sh storage list > %s" % f.name],
shell=True)
try:
(out, err) = proc.communicate()
except:
pass
from xml.dom.minidom import parse, parseString
try:
with open(f.name, 'r') as f2:
out = f2.read()
x = parseString(out)
return [ {
'id': int(d.getAttributeNode('href').nodeValue.split('/')[-1]),
'name': d.getAttributeNode('name').nodeValue,
} for d in x.getElementsByTagName("STORAGE")
]
except:
pass
os.unlink(f)
@task(name="one.tasks.UpdateNetworkTask")
def t():
f = tempfile.NamedTemporaryFile(delete=False)
f.close()
out=''
proc = subprocess.Popen(["/opt/occi.sh network list > %s" % f.name],
shell=True)
try:
(out, err) = proc.communicate()
except:
pass
from xml.dom.minidom import parse, parseString
try:
with open(f.name, 'r') as f2:
out = f2.read()
x = parseString(out)
return [ {
'id': int(d.getAttributeNode('href').nodeValue.split('/')[-1]),
'name': d.getAttributeNode('name').nodeValue,
} for d in x.getElementsByTagName("NETWORK")
]
except:
pass
os.unlink(f)
@task(name="one.tasks.DeleteInstanceTask")
def t(one_id):
proc = subprocess.Popen(["/opt/occi.sh", "compute", "delete",
"%d" % one_id], stdout=subprocess.PIPE)
try:
(out, err) = proc.communicate()
except:
pass
def update_state(one_id):
"""Get and update VM state from OpenNebula."""
proc = subprocess.Popen(["/opt/occi.sh", "compute", "show",
"%d" % one_id], stdout=subprocess.PIPE)
(out, err) = proc.communicate()
state = 'UNKNOWN'
try:
from xml.dom.minidom import parse, parseString
x = parseString(out)
state = x.getElementsByTagName("STATE")[0].childNodes[0].nodeValue
except:
state = 'UNKNOWN'
print state
celery.send_task('one.tasks.UpdateInstanceStateTask', [one_id, state],
queue='local')
if __name__ == "__main__":
update_state(int(sys.argv[1]))
miscellaneous/celery/update_state.py deleted 100755 → 0
#!/usr/bin/env python
from celery import Celery, task
import subprocess
import time, re
import socket
import sys
BROKER_URL = 'amqp://nyuszi:teszt@localhost:5672/django'
try:
from local_settings import *
except:
pass
CELERY_CREATE_MISSING_QUEUES=True
celery = Celery('tasks', broker=BROKER_URL)
def main(argv):
celery.send_task('one.tasks.UpdateInstanceStateTask', [ int(sys.argv[1]),
], queue='local')
if __name__ == "__main__":
main(sys.argv)
one/models.py
......@@ -244,23 +244,23 @@ class Disk(models.Model):
@staticmethod
def update(delete=True):
"""Get and register virtual disks from OpenNebula."""
import subprocess
proc = subprocess.Popen(["/opt/occi.sh", "storage", "list"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = proc.communicate()
from xml.dom.minidom import parse, parseString
x = parseString(out)
try:
from .tasks import UpdateDiskTask
x = UpdateDiskTask.delay().get(timeout=10)
x[0]
except:
return
with transaction.commit_on_success():
l = []
for d in x.getElementsByTagName("STORAGE"):
id = int(d.getAttributeNode('href').nodeValue.split('/')[-1])
name=d.getAttributeNode('name').nodeValue
for d in x:
id = int(d['id'])
name = d['name']
try:
d = Disk.objects.get(id=id)
d.name=name
d, created = Disk.objects.get_or_create(id=id)
d.name = name
d.save()
except:
Disk(id=id, name=name).save()
pass
l.append(id)
if delete:
Disk.objects.exclude(id__in=l).delete()
......@@ -286,23 +286,23 @@ class Network(models.Model):
@staticmethod
def update():
"""Get and register virtual networks from OpenNebula."""
import subprocess
proc = subprocess.Popen(["/opt/occi.sh", "network", "list"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = proc.communicate()
from xml.dom.minidom import parse, parseString
x = parseString(out)
try:
from .tasks import UpdateNetworkTask
x = UpdateNetworkTask.delay().get(timeout=10)
x[0]
except:
return
with transaction.commit_on_success():
l = []
for d in x.getElementsByTagName("NETWORK"):
id = int(d.getAttributeNode('href').nodeValue.split('/')[-1])
name=d.getAttributeNode('name').nodeValue
for n in x:
id = int(n['id'])
name = n['name']
try:
n = Network.objects.get(id=id)
n, created = Network.objects.get_or_create(id=id)
n.name = name
n.save()
except:
Network(id=id, name=name).save()
pass
l.append(id)
Network.objects.exclude(id__in=l).delete()
def get_vlan(self):
......@@ -461,33 +461,6 @@ class Instance(models.Model):
except:
return
def update_state(self):
"""Get and update VM state from OpenNebula."""
import subprocess
if not self.one_id:
return
proc = subprocess.Popen(["/opt/occi.sh", "compute", "show",
"%d" % self.one_id], stdout=subprocess.PIPE)
(out, err) = proc.communicate()
x = None
old_state = self.state
try:
from xml.dom.minidom import parse, parseString
x = parseString(out)
self.vnet_ip = (x.getElementsByTagName("IP")[0].childNodes[0]
.nodeValue.split('.')[3])
state = x.getElementsByTagName("STATE")[0].childNodes[0].nodeValue
self.state = state
except:
self.state = 'UNKNOWN'
if self.state != old_state:
self.waiting = False
self.save()
if self.template.state == 'SAVING':
self.check_if_is_save_as_done()
return x
@property
def nat(self):
if self.firewall_host is not None:
......@@ -512,81 +485,69 @@ class Instance(models.Model):
def submit(cls, template, owner, extra="", share=None):
"""Submit a new instance to OpenNebula."""
from django.template.defaultfilters import escape
out = ""
inst = Instance(pw=pwgen(), template=template, owner=owner,
share=share, state='PENDING')
inst.save()
hostname = u"%d" % (inst.id, )
with tempfile.NamedTemporaryFile(delete=False) as f:
os.chmod(f.name, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
token = signing.dumps(inst.id, salt='activate')
try:
details = owner.cloud_details
except:
details = UserCloudDetails(user=owner)
details.save()
tpl = u"""
<COMPUTE>
<NAME>%(name)s</NAME>
<INSTANCE_TYPE href="http://www.opennebula.org/instance_type/%(instance)s"/>
<DISK>
<STORAGE href="http://www.opennebula.org/storage/%(disk)d"/>
</DISK>
<NIC>
<NETWORK href="http://www.opennebula.org/network/%(net)d"/>
</NIC>
<CONTEXT>
<SOURCE>web</SOURCE>
<HOSTNAME>%(hostname)s</HOSTNAME>
<NEPTUN>%(neptun)s</NEPTUN>
<USERPW>%(pw)s</USERPW>
<SMBPW>%(smbpw)s</SMBPW>
<SSHPRIV>%(sshkey)s</SSHPRIV>
<BOOTURL>%(booturl)s</BOOTURL>
<SERVER>store.cloud.ik.bme.hu</SERVER>
%(extra)s
</CONTEXT>
</COMPUTE>""" % {"name": u"%s %d" % (owner.username, inst.id),
"instance": template.instance_type,
"disk": template.disk.id,
"net": template.network.id,
"pw": escape(inst.pw),
"hostname": escape(hostname),
"smbpw": escape(details.smb_password),
"sshkey": escape(details.ssh_private_key),
"neptun": escape(owner.username),
"booturl": "%sb/%s/" % ( CLOUD_URL, token ),
"extra": extra}
f.write(tpl)
f.close()
import subprocess
proc = subprocess.Popen(["/opt/occi.sh", "compute", "create",
f.name], stdout=subprocess.PIPE)
(out, err) = proc.communicate()
os.unlink(f.name)
from xml.dom.minidom import parse, parseString
token = signing.dumps(inst.id, salt='activate')
try:
details = owner.cloud_details
except:
details = UserCloudDetails(user=owner)
details.save()
ctx = u'''
<SOURCE>web</SOURCE>
<HOSTNAME>%(hostname)s</HOSTNAME>
<NEPTUN>%(neptun)s</NEPTUN>
<USERPW>%(pw)s</USERPW>
<SMBPW>%(smbpw)s</SMBPW>
<SSHPRIV>%(sshkey)s</SSHPRIV>
<BOOTURL>%(booturl)s</BOOTURL>
<SERVER>store.cloud.ik.bme.hu</SERVER>
%(extra)s
''' % {
"pw": escape(inst.pw),
"hostname": escape(hostname),
"smbpw": escape(details.smb_password),
"sshkey": escape(details.ssh_private_key),
"neptun": escape(owner.username),
"booturl": "%sb/%s/" % ( CLOUD_URL, token ),
"extra": extra
}
try:
x = parseString(out)
from .tasks import CreateInstanceTask
x = CreateInstanceTask.delay(
name=u"%s %d" % (owner.username, inst.id),
instance_type=template.instance_type.name,
disk_id=int(template.disk.id),
network_id=int(template.network.id),
ctx=ctx,
)
res = x.get(timeout=10)
res['one_id']
except:
inst.delete()
raise Exception("Unable to create VM instance.")
inst.one_id = int(x.getElementsByTagName("ID")[0].childNodes[0]
.nodeValue)
inst.ip = x.getElementsByTagName("IP")[0].childNodes[0].nodeValue
inst.one_id = res['one_id']
inst.ip = res['interfaces'][0]['ip']
inst.name = ("%(neptun)s %(template)s (%(id)d)" %
{'neptun': owner.username, 'template': template.name,
'id': inst.one_id})
inst.save()
host = Host(vlan=Vlan.objects.get(name=template.network.name),
owner=owner)
host.hostname = hostname
host.mac = x.getElementsByTagName("MAC")[0].childNodes[0].nodeValue
host.ipv4 = inst.ip
host = Host(
vlan=Vlan.objects.get(name=template.network.name),
owner=owner, hostname=hostname,
mac=res['interfaces'][0]['mac'],
ipv4=res['interfaces'][0]['ip'], ipv6='auto',
)
if inst.template.network.nat:
host.pub_ipv4 = Vlan.objects.get(name=template.network.name).snat_ip
host.shared_ip = True
host.ipv6 = "auto"
try:
host.save()
except:
......@@ -597,6 +558,7 @@ class Instance(models.Model):
logger.warning('Delete orphan fw host (%s) of %s.' % (i, inst))
i.delete()
host.save()
host.enable_net()
host.add_port("tcp", inst.get_port(), {"rdp": 3389, "nx": 22,
"ssh": 22}[inst.template.access_type])
......@@ -606,10 +568,11 @@ class Instance(models.Model):
def one_delete(self):
"""Delete host in OpenNebula."""
if self.template.state != "DONE":
self.check_if_is_save_as_done()
if self.one_id and self.state != 'DONE':
proc = subprocess.Popen(["/opt/occi.sh", "compute", "delete",
"%d" % self.one_id], stdout=subprocess.PIPE)
(out, err) = proc.communicate()
from .tasks import DeleteInstanceTask
DeleteInstanceTask.delay(one_id=self.one_id)
self.firewall_host_delete()
def firewall_host_delete(self):
......@@ -622,28 +585,10 @@ class Instance(models.Model):
pass
h.delete()
def _update_vm(self, template):
out = ""
with tempfile.NamedTemporaryFile(delete=False) as f:
os.chmod(f.name, stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH)
tpl = u"""
<COMPUTE>
<ID>%(id)d</ID>
%(template)s
</COMPUTE>""" % {"id": self.one_id,
"template": template}
f.write(tpl)
f.close()
import subprocess
proc = subprocess.Popen(["/opt/occi.sh", "compute", "update",
f.name], stdout=subprocess.PIPE)
(out, err) = proc.communicate()
os.unlink(f.name)
print "out: " + out
def _change_state(self, new_state):
"""Change host state in OpenNebula."""
self._update_vm("<STATE>" + new_state + "</STATE>")
from .tasks import ChangeInstanceStateTask
ChangeInstanceStateTask.delay(one_id=self.one_id, new_state=new_state)
self.waiting = True
self.save()
......@@ -680,7 +625,8 @@ class Instance(models.Model):
def save_as(self):
"""Save image and shut down."""
imgname = "template-%d-%d" % (self.template.id, self.id)
self._update_vm('<DISK id="0"><SAVE_AS name="%s"/></DISK>' % imgname)
from .tasks import SaveAsTask
SaveAsTask.delay(one_id=self.one_id, new_img=imgname)
self._change_state("SHUTDOWN")
self.save()
t = self.template
......@@ -701,22 +647,9 @@ class Instance(models.Model):
self.firewall_host_delete()
return True
def delete_instance(sender, instance, using, **kwargs):
if instance.state != "DONE":
def delete_instance_pre(sender, instance, using, **kwargs):
if instance.state != 'DONE':
instance.one_delete()
try:
instance.firewall_host_delete()
except:
pass
post_delete.connect(delete_instance, sender=Instance,
dispatch_uid="delete_instance")
def delete_instance_pre(sender, instance, using, **kwargs):
try:
if instance.template.state != "DONE":
instance.check_if_is_save_as_done()
except:
pass
pre_delete.connect(delete_instance_pre, sender=Instance,
dispatch_uid="delete_instance_pre")
one/tasks.py
......@@ -16,14 +16,44 @@ class SendMailTask(Task):
class UpdateInstanceStateTask(Task):
def run(self, one_id):
def run(self, one_id, new_state):
print one_id
try:
inst = Instance.objects.get(one_id=one_id)
except:
print 'nincs ilyen'
return
inst.update_state()
inst.state = new_state
inst.waiting = False
inst.save()
if inst.template.state == 'SAVING':
inst.check_if_is_save_as_done()
print inst.state
# ezek csak azert vannak felveve, hogy szepen meg lehessen hivni oket
# ezeket a fejgepen futo celery futtatja
class CreateInstanceTask(Task):
def run(self, name, instance_type, disk_id, network_id, ctx):
pass
class DeleteInstanceTask(Task):
def run(self, one_id):
pass
class ChangeInstanceStateTask(Task):
def run(self, one_id, new_state):
pass
class SaveAsTask(Task):
def run(self, one_id, new_img):
pass
class UpdateDiskTask(Task):
def run(self):
pass
class UpdateNetworkTask(Task):
def run(self):
pass
one/views.py
......@@ -16,7 +16,7 @@ from django.template import RequestContext
from django.template.loader import render_to_string
from django.utils.decorators import method_decorator
from django.utils.translation import get_language as lang
from django.utils.translation import ugettext_lazy as _, ungettext_lazy
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.http import *
from django.views.generic import *
from firewall.tasks import *
......@@ -300,7 +300,7 @@ def _check_quota(request, template, share):
@login_required
def vm_new(request, template=None, share=None, redir=True):
base = None
extra = None
extra = ''
if template:
base = get_object_or_404(Template, pk=template)
else:
......@@ -460,7 +460,7 @@ class VmDeleteView(View):
inst = get_object_or_404(Instance, id=iid, owner=request.user)
if inst.template.state != 'READY' and inst.template.owner == request.user:
inst.template.delete()
inst.delete()
inst.one_delete()
messages.success(request, _('Virtual machine is successfully deleted.'))
except:
messages.error(request, _('Failed to delete virtual machine.'))
......@@ -484,21 +484,12 @@ def vm_unshare(request, id, *args, **kwargs):
if not g.owners.filter(user=request.user).exists():
raise PermissionDenied()
try:
n = s.get_running()
m = s.get_running_or_stopped()
if n > 0:
messages.error(request, ungettext_lazy('There is a machine running of this share.',
'There are %(n)d machines running of this share.', n) %
{'n' : n})
elif m > 0:
messages.error(request, ungettext_lazy('There is a suspended machine of this share.',
'There are %(m)d suspended machines of this share.', m) %
{'m' : m})
if s.get_running_or_stopped() > 0:
messages.error(request, _('There are machines running of this share.'))
else:
s.delete()
messages.success(request, _('Share is successfully removed.'))
except Exception as e:
print e
except:
messages.error(request, _('Failed to remove share.'))
return redirect(g)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment