instance/storage: added default timeout values fixes #49

0bfec6c9 · Guba Sándor · e9f05037 · 0bfec6c9 · 0bfec6c9
Commit 0bfec6c9 authored Feb 24, 2014 by Guba Sándor
Hide whitespace changes
Inline Side-by-side

Showing with 42 additions and 29 deletions

circle/storage/models.py
+9 -6

circle/vm/models/instance.py
+33 -23

No files found.
--- a/circle/storage/models.py
+++ b/circle/storage/models.py
@@ -179,7 +179,7 @@ class Disk(AclBase, TimeStampedModel):
            self.size = self.base.size
        super(Disk, self).clean(*args, **kwargs)
-    def deploy(self, user=None, task_uuid=None):
+    def deploy(self, user=None, task_uuid=None, timeout=15):
        """Reify the disk model on the associated data store.
        :param self: the disk model to reify
@@ -212,11 +212,13 @@ class Disk(AclBase, TimeStampedModel):
            if self.type == 'qcow2-snap':
                with act.sub_activity('creating_snapshot'):
                    remote_tasks.snapshot.apply_async(args=[disk_desc],
-                                                      queue=queue_name).get()
+                                                      queue=queue_name
+                                                      ).get(timeout=timeout)
            else:
                with act.sub_activity('creating_disk'):
                    remote_tasks.create.apply_async(args=[disk_desc],
-                                                    queue=queue_name).get()
+                                                    queue=queue_name
+                                                    ).get(timeout=timeout)
            self.ready = True
            self.save()
@@ -310,7 +312,7 @@ class Disk(AclBase, TimeStampedModel):
        local_tasks.restore.apply_async(args=[self, user],
                                        queue='localhost.man')
-    def save_as(self, user=None, task_uuid=None):
+    def save_as(self, user=None, task_uuid=None, timeout=120):
        mapping = {
            'qcow2-snap': ('qcow2-norm', self.base),
        }
@@ -324,7 +326,7 @@ class Disk(AclBase, TimeStampedModel):
        # going to be used until the operation is complete
        with disk_activity(code_suffix='save_as', disk=self,
-                           task_uuid=task_uuid, user=user):
+                           task_uuid=task_uuid, user=user, timeout=300):
            filename = str(uuid.uuid4())
            new_type, new_base = mapping[self.type]
@@ -336,7 +338,8 @@ class Disk(AclBase, TimeStampedModel):
            queue_name = self.get_remote_queue_name('storage')
            remote_tasks.merge.apply_async(args=[self.get_disk_desc(),
                                                 disk.get_disk_desc()],
-                                           queue=queue_name).get()
+                                           queue=queue_name
+                                           ).get(timeout=timeout)
            disk.ready = True
            disk.save()

--- a/circle/vm/models/instance.py
+++ b/circle/vm/models/instance.py
@@ -597,7 +597,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        self.save()
-    def __deploy_vm(self, act):
+    def __deploy_vm(self, act, timeout=15):
        """Deploy the virtual machine.
        :param self: The virtual machine.
@@ -609,7 +609,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        # Deploy VM on remote machine
        with act.sub_activity('deploying_vm'):
            vm_tasks.deploy.apply_async(args=[self.get_vm_desc()],
-                                        queue=queue_name).get()
+                                        queue=queue_name).get(timeout=timeout)
        # Estabilish network connection (vmdriver)
        with act.sub_activity('deploying_net'):
@@ -619,7 +619,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        # Resume vm
        with act.sub_activity('booting'):
            vm_tasks.resume.apply_async(args=[self.vm_name],
-                                        queue=queue_name).get()
+                                        queue=queue_name).get(timeout=timeout)
        self.renew('suspend')
@@ -671,7 +671,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.deploy.apply_async(args=[self, user],
                                              queue="localhost.man")
-    def __destroy_vm(self, act):
+    def __destroy_vm(self, act, timeout):
        """Destroy the virtual machine and its associated networks.
        :param self: The virtual machine.
@@ -688,13 +688,17 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
            queue_name = self.get_remote_queue_name('vm')
            try:
                vm_tasks.destroy.apply_async(args=[self.vm_name],
-                                             queue=queue_name).get()
+                                             queue=queue_name
+                                             ).get(timeout=timeout)
            except Exception as e:
-                if e.libvirtError is True:
+                if e.libvirtError is True and "Domain not found" in str(e):
-                    if "Domain not found" in str(e):
+                    logger.debug("Domain %s was not found at %s"
-                        pass
+                                 % (self.vm_name, queue_name))
+                    pass
-    def __cleanup_after_destroy_vm(self, act):
+                else:
+                    raise
+    def __cleanup_after_destroy_vm(self, act, timeout=15):
        """Clean up the virtual machine's data after destroy.
        :param self: The virtual machine.
@@ -707,7 +711,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        try:
            from storage.tasks.remote_tasks import delete_dump
            delete_dump.apply_async(args=[self.mem_dump['path']],
-                                    queue=queue_name).get()
+                                    queue=queue_name).get(timeout=timeout)
        except:
            pass
@@ -788,7 +792,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.destroy.apply_async(args=[self, user],
                                               queue="localhost.man")
-    def sleep(self, user=None, task_uuid=None):
+    def sleep(self, user=None, task_uuid=None, timeout=60):
        """Suspend virtual machine with memory dump.
        """
        if self.state not in ['RUNNING']:
@@ -817,7 +821,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
                queue_name = self.get_remote_queue_name('vm')
                vm_tasks.sleep.apply_async(args=[self.vm_name,
                                                 self.mem_dump['path']],
-                                           queue=queue_name).get()
+                                           queue=queue_name
+                                           ).get(timeout=timeout)
                self.node = None
                self.vnc_port = None
                self.save()
@@ -828,7 +833,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.sleep.apply_async(args=[self, user],
                                             queue="localhost.man")
-    def wake_up(self, user=None, task_uuid=None):
+    def wake_up(self, user=None, task_uuid=None, timeout=60):
        if self.state not in ['SUSPENDED']:
            raise self.WrongStateError(self)
@@ -850,7 +855,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
            with act.sub_activity('resuming'):
                vm_tasks.wake_up.apply_async(args=[self.vm_name,
                                                   self.mem_dump['path']],
-                                             queue=queue_name).get()
+                                             queue=queue_name
+                                             ).get(timeout=timeout)
            # Estabilish network connection (vmdriver)
            with act.sub_activity('deploying_net'):
@@ -863,7 +869,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.wake_up.apply_async(args=[self, user],
                                               queue="localhost.man")
-    def shutdown(self, user=None, task_uuid=None):
+    def shutdown(self, user=None, task_uuid=None, timeout=120):
        """Shutdown virtual machine with ACPI signal.
        """
        def __on_abort(activity, error):
@@ -882,7 +888,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
            logger.debug("RPC Shutdown at queue: %s, for vm: %s.",
                         self.vm_name, queue_name)
            vm_tasks.shutdown.apply_async(kwargs={'name': self.vm_name},
-                                          queue=queue_name).get()
+                                          queue=queue_name
+                                          ).get(timeout=timeout)
            self.node = None
            self.vnc_port = None
            self.save()
@@ -893,7 +900,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.shutdown.apply_async(args=[self, user],
                                                queue="localhost.man")
-    def reset(self, user=None, task_uuid=None):
+    def reset(self, user=None, task_uuid=None, timeout=5):
        """Reset virtual machine (reset button)
        """
        with instance_activity(code_suffix='reset', instance=self,
@@ -901,7 +908,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
            queue_name = self.get_remote_queue_name('vm')
            vm_tasks.restart.apply_async(args=[self.vm_name],
-                                         queue=queue_name).get()
+                                         queue=queue_name
+                                         ).get(timeout=timeout)
    def reset_async(self, user=None):
        """Execute reset asynchronously.
@@ -909,7 +917,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.restart.apply_async(args=[self, user],
                                               queue="localhost.man")
-    def reboot(self, user=None, task_uuid=None):
+    def reboot(self, user=None, task_uuid=Nonei, timeout=5):
        """Reboot virtual machine with Ctrl+Alt+Del signal.
        """
        with instance_activity(code_suffix='reboot', instance=self,
@@ -917,7 +925,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
            queue_name = self.get_remote_queue_name('vm')
            vm_tasks.reboot.apply_async(args=[self.vm_name],
-                                        queue=queue_name).get()
+                                        queue=queue_name
+                                        ).get(timeout=timeout)
    def reboot_async(self, user=None):
        """Execute reboot asynchronously. """
@@ -929,7 +938,7 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
        return local_tasks.migrate.apply_async(args=[self, to_node, user],
                                               queue="localhost.man")
-    def migrate(self, to_node, user=None, task_uuid=None):
+    def migrate(self, to_node, user=None, task_uuid=None, timeout=120):
        """Live migrate running vm to another node. """
        with instance_activity(code_suffix='migrate', instance=self,
                               task_uuid=task_uuid, user=user) as act:
@@ -942,7 +951,8 @@ class Instance(AclBase, VirtualMachineDescModel, TimeStampedModel):
                queue_name = self.get_remote_queue_name('vm')
                vm_tasks.migrate.apply_async(args=[self.vm_name,
                                             to_node.host.hostname],
-                                             queue=queue_name).get()
+                                             queue=queue_name
+                                             ).get(timeout=timeout)
            # Refresh node information
            self.node = to_node
            self.save()