Skip to content

CIRCLE / cloud

Go to a project
Commit 0bfec6c9 by Guba Sándor
Options

instance/storage: added default timeout values fixes #49

parent e9f05037
Showing with 40 additions and 27 deletions
circle/storage/models.py
...@@ -179,7 +179,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -179,7 +179,7 @@ class Disk(AclBase, TimeStampedModel):
self.size = self.base.size self.size = self.base.size
super(Disk, self).clean(*args, **kwargs) super(Disk, self).clean(*args, **kwargs)
def deploy(self, user=None, task_uuid=None): def deploy(self, user=None, task_uuid=None, timeout=15):
"""Reify the disk model on the associated data store. """Reify the disk model on the associated data store.
:param self: the disk model to reify :param self: the disk model to reify
...@@ -212,11 +212,13 @@ class Disk(AclBase, TimeStampedModel): ...@@ -212,11 +212,13 @@ class Disk(AclBase, TimeStampedModel):
if self.type == 'qcow2-snap': if self.type == 'qcow2-snap':
with act.sub_activity('creating_snapshot'): with act.sub_activity('creating_snapshot'):
remote_tasks.snapshot.apply_async(args=[disk_desc], remote_tasks.snapshot.apply_async(args=[disk_desc],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
else: else:
with act.sub_activity('creating_disk'): with act.sub_activity('creating_disk'):
remote_tasks.create.apply_async(args=[disk_desc], remote_tasks.create.apply_async(args=[disk_desc],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
self.ready = True self.ready = True
self.save() self.save()
...@@ -310,7 +312,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -310,7 +312,7 @@ class Disk(AclBase, TimeStampedModel):
local_tasks.restore.apply_async(args=[self, user], local_tasks.restore.apply_async(args=[self, user],
queue='localhost.man') queue='localhost.man')
def save_as(self, user=None, task_uuid=None): def save_as(self, user=None, task_uuid=None, timeout=120):
mapping = { mapping = {
'qcow2-snap': ('qcow2-norm', self.base), 'qcow2-snap': ('qcow2-norm', self.base),
} }
...@@ -324,7 +326,7 @@ class Disk(AclBase, TimeStampedModel): ...@@ -324,7 +326,7 @@ class Disk(AclBase, TimeStampedModel):
# going to be used until the operation is complete # going to be used until the operation is complete
with disk_activity(code_suffix='save_as', disk=self, with disk_activity(code_suffix='save_as', disk=self,
task_uuid=task_uuid, user=user): task_uuid=task_uuid, user=user, timeout=300):
filename = str(uuid.uuid4()) filename = str(uuid.uuid4())
new_type, new_base = mapping[self.type] new_type, new_base = mapping[self.type]
...@@ -336,7 +338,8 @@ class Disk(AclBase, TimeStampedModel): ...@@ -336,7 +338,8 @@ class Disk(AclBase, TimeStampedModel):
queue_name = self.get_remote_queue_name('storage') queue_name = self.get_remote_queue_name('storage')
remote_tasks.merge.apply_async(args=[self.get_disk_desc(), remote_tasks.merge.apply_async(args=[self.get_disk_desc(),
disk.get_disk_desc()], disk.get_disk_desc()],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
disk.ready = True disk.ready = True
disk.save() disk.save()
......
circle/vm/models/instance.py
...@@ -597,7 +597,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -597,7 +597,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
self.save() self.save()
def __deploy_vm(self, act): def __deploy_vm(self, act, timeout=15):
"""Deploy the virtual machine. """Deploy the virtual machine.
:param self: The virtual machine. :param self: The virtual machine.
...@@ -609,7 +609,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -609,7 +609,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
# Deploy VM on remote machine # Deploy VM on remote machine
with act.sub_activity('deploying_vm'): with act.sub_activity('deploying_vm'):
vm_tasks.deploy.apply_async(args=[self.get_vm_desc()], vm_tasks.deploy.apply_async(args=[self.get_vm_desc()],
queue=queue_name).get() queue=queue_name).get(timeout=timeout)
# Estabilish network connection (vmdriver) # Estabilish network connection (vmdriver)
with act.sub_activity('deploying_net'): with act.sub_activity('deploying_net'):
...@@ -619,7 +619,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -619,7 +619,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
# Resume vm # Resume vm
with act.sub_activity('booting'): with act.sub_activity('booting'):
vm_tasks.resume.apply_async(args=[self.vm_name], vm_tasks.resume.apply_async(args=[self.vm_name],
queue=queue_name).get() queue=queue_name).get(timeout=timeout)
self.renew('suspend') self.renew('suspend')
...@@ -671,7 +671,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -671,7 +671,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.deploy.apply_async(args=[self, user], return local_tasks.deploy.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def __destroy_vm(self, act): def __destroy_vm(self, act, timeout):
"""Destroy the virtual machine and its associated networks. """Destroy the virtual machine and its associated networks.
:param self: The virtual machine. :param self: The virtual machine.
...@@ -688,13 +688,17 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -688,13 +688,17 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
queue_name = self.get_remote_queue_name('vm') queue_name = self.get_remote_queue_name('vm')
try: try:
vm_tasks.destroy.apply_async(args=[self.vm_name], vm_tasks.destroy.apply_async(args=[self.vm_name],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
except Exception as e: except Exception as e:
if e.libvirtError is True: if e.libvirtError is True and "Domain not found" in str(e):
if "Domain not found" in str(e): logger.debug("Domain %s was not found at %s"
% (self.vm_name, queue_name))
pass pass
else:
raise
def __cleanup_after_destroy_vm(self, act): def __cleanup_after_destroy_vm(self, act, timeout=15):
"""Clean up the virtual machine's data after destroy. """Clean up the virtual machine's data after destroy.
:param self: The virtual machine. :param self: The virtual machine.
...@@ -707,7 +711,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -707,7 +711,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
try: try:
from storage.tasks.remote_tasks import delete_dump from storage.tasks.remote_tasks import delete_dump
delete_dump.apply_async(args=[self.mem_dump['path']], delete_dump.apply_async(args=[self.mem_dump['path']],
queue=queue_name).get() queue=queue_name).get(timeout=timeout)
except: except:
pass pass
...@@ -788,7 +792,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -788,7 +792,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.destroy.apply_async(args=[self, user], return local_tasks.destroy.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def sleep(self, user=None, task_uuid=None): def sleep(self, user=None, task_uuid=None, timeout=60):
"""Suspend virtual machine with memory dump. """Suspend virtual machine with memory dump.
""" """
if self.state not in ['RUNNING']: if self.state not in ['RUNNING']:
...@@ -817,7 +821,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -817,7 +821,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
queue_name = self.get_remote_queue_name('vm') queue_name = self.get_remote_queue_name('vm')
vm_tasks.sleep.apply_async(args=[self.vm_name, vm_tasks.sleep.apply_async(args=[self.vm_name,
self.mem_dump['path']], self.mem_dump['path']],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
self.node = None self.node = None
self.vnc_port = None self.vnc_port = None
self.save() self.save()
...@@ -828,7 +833,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -828,7 +833,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.sleep.apply_async(args=[self, user], return local_tasks.sleep.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def wake_up(self, user=None, task_uuid=None): def wake_up(self, user=None, task_uuid=None, timeout=60):
if self.state not in ['SUSPENDED']: if self.state not in ['SUSPENDED']:
raise self.WrongStateError(self) raise self.WrongStateError(self)
...@@ -850,7 +855,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -850,7 +855,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
with act.sub_activity('resuming'): with act.sub_activity('resuming'):
vm_tasks.wake_up.apply_async(args=[self.vm_name, vm_tasks.wake_up.apply_async(args=[self.vm_name,
self.mem_dump['path']], self.mem_dump['path']],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
# Estabilish network connection (vmdriver) # Estabilish network connection (vmdriver)
with act.sub_activity('deploying_net'): with act.sub_activity('deploying_net'):
...@@ -863,7 +869,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -863,7 +869,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.wake_up.apply_async(args=[self, user], return local_tasks.wake_up.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def shutdown(self, user=None, task_uuid=None): def shutdown(self, user=None, task_uuid=None, timeout=120):
"""Shutdown virtual machine with ACPI signal. """Shutdown virtual machine with ACPI signal.
""" """
def __on_abort(activity, error): def __on_abort(activity, error):
...@@ -882,7 +888,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -882,7 +888,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
logger.debug("RPC Shutdown at queue: %s, for vm: %s.", logger.debug("RPC Shutdown at queue: %s, for vm: %s.",
self.vm_name, queue_name) self.vm_name, queue_name)
vm_tasks.shutdown.apply_async(kwargs={'name': self.vm_name}, vm_tasks.shutdown.apply_async(kwargs={'name': self.vm_name},
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
self.node = None self.node = None
self.vnc_port = None self.vnc_port = None
self.save() self.save()
...@@ -893,7 +900,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -893,7 +900,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.shutdown.apply_async(args=[self, user], return local_tasks.shutdown.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def reset(self, user=None, task_uuid=None): def reset(self, user=None, task_uuid=None, timeout=5):
"""Reset virtual machine (reset button) """Reset virtual machine (reset button)
""" """
with instance_activity(code_suffix='reset', instance=self, with instance_activity(code_suffix='reset', instance=self,
...@@ -901,7 +908,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -901,7 +908,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
queue_name = self.get_remote_queue_name('vm') queue_name = self.get_remote_queue_name('vm')
vm_tasks.restart.apply_async(args=[self.vm_name], vm_tasks.restart.apply_async(args=[self.vm_name],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
def reset_async(self, user=None): def reset_async(self, user=None):
"""Execute reset asynchronously. """Execute reset asynchronously.
...@@ -909,7 +917,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -909,7 +917,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.restart.apply_async(args=[self, user], return local_tasks.restart.apply_async(args=[self, user],
queue="localhost.man") queue="localhost.man")
def reboot(self, user=None, task_uuid=None): def reboot(self, user=None, task_uuid=Nonei, timeout=5):
"""Reboot virtual machine with Ctrl+Alt+Del signal. """Reboot virtual machine with Ctrl+Alt+Del signal.
""" """
with instance_activity(code_suffix='reboot', instance=self, with instance_activity(code_suffix='reboot', instance=self,
...@@ -917,7 +925,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -917,7 +925,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
queue_name = self.get_remote_queue_name('vm') queue_name = self.get_remote_queue_name('vm')
vm_tasks.reboot.apply_async(args=[self.vm_name], vm_tasks.reboot.apply_async(args=[self.vm_name],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
def reboot_async(self, user=None): def reboot_async(self, user=None):
"""Execute reboot asynchronously. """ """Execute reboot asynchronously. """
...@@ -929,7 +938,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -929,7 +938,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
return local_tasks.migrate.apply_async(args=[self, to_node, user], return local_tasks.migrate.apply_async(args=[self, to_node, user],
queue="localhost.man") queue="localhost.man")
def migrate(self, to_node, user=None, task_uuid=None): def migrate(self, to_node, user=None, task_uuid=None, timeout=120):
"""Live migrate running vm to another node. """ """Live migrate running vm to another node. """
with instance_activity(code_suffix='migrate', instance=self, with instance_activity(code_suffix='migrate', instance=self,
task_uuid=task_uuid, user=user) as act: task_uuid=task_uuid, user=user) as act:
...@@ -942,7 +951,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel): ...@@ -942,7 +951,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
queue_name = self.get_remote_queue_name('vm') queue_name = self.get_remote_queue_name('vm')
vm_tasks.migrate.apply_async(args=[self.vm_name, vm_tasks.migrate.apply_async(args=[self.vm_name,
to_node.host.hostname], to_node.host.hostname],
queue=queue_name).get() queue=queue_name
).get(timeout=timeout)
# Refresh node information # Refresh node information
self.node = to_node self.node = to_node
self.save() self.save()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment