Manager: increase timeout for deploy and destroy

Manager: email notification fix
Storage: timeout + file type fix
VM: ActionInProgressError fix
VM: increased timeout

circle/firewall/tasks/local_tasks.py

@@ -39,7 +39,7 @@ def _apply_once(name, tasks, queues, task, data):
     data = data()
     data = data()
     for queue in queues:
     for queue in queues:
         try:
         try:
-            task.apply_async(args=data, queue=queue, expires=60).get(timeout=2)
+            task.apply_async(args=data, queue=queue, expires=60).get(timeout=5)
             logger.info("%s configuration is reloaded. (queue: %s)",
             logger.info("%s configuration is reloaded. (queue: %s)",
                         name, queue)
                         name, queue)
         except TimeoutError as e:
         except TimeoutError as e:
@@ -76,8 +76,14 @@ def reloadtask_worker():
     logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
     logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
 
 
     firewall_queues = get_firewall_queues()
     firewall_queues = get_firewall_queues()
-    dns_queues = [("%s.dns" % i) for i in
-                  settings.get('dns_queues', [gethostname()])]
+
+    dns_queues = settings.get('dns_queues', [gethostname()])
+    if isinstance(dns_queues, (str, unicode)):
+        dns_queues = [dns_queues]
+    dns_queues = [("%s.dns" % i) for i in dns_queues]
+
+#    dns_queues = [("%s.dns" % i) for i in
+#                  settings.get('dns_queues', [gethostname()])]
 
 
     _apply_once('dns', tasks, dns_queues, reload_dns,
     _apply_once('dns', tasks, dns_queues, reload_dns,
                 lambda: (dns(), ))
                 lambda: (dns(), ))

circle/manager/mancelery.py

@@ -18,6 +18,7 @@
 from celery import Celery
 from celery import Celery
 from celery.signals import worker_ready
 from celery.signals import worker_ready
 from datetime import timedelta
 from datetime import timedelta
+from celery.schedules import crontab
 from kombu import Queue, Exchange
 from kombu import Queue, Exchange
 from os import getenv
 from os import getenv
 
 
@@ -52,7 +53,7 @@ celery.conf.update(
         'dashboard.send_email_notifications': {
         'dashboard.send_email_notifications': {
             'task': 'dashboard.tasks.local_periodic_tasks.'
             'task': 'dashboard.tasks.local_periodic_tasks.'
             'send_email_notifications',
             'send_email_notifications',
-            'schedule': timedelta(hours=24),
+            'schedule': crontab(minute=10, hour=1),
             'options': {'queue': 'localhost.man'}
             'options': {'queue': 'localhost.man'}
         },
         },
     }
     }

circle/storage/models.py

@@ -86,7 +86,7 @@ class DataStore(Model):
             args=[self.path], queue=q).get(timeout=timeout)
             args=[self.path], queue=q).get(timeout=timeout)
 
 
     @method_cache(30)
     @method_cache(30)
-    def get_orphan_disks(self, timeout=15):
+    def get_orphan_disks(self, timeout=25):
         """Disk image files without Disk object in the database.
         """Disk image files without Disk object in the database.
         """
         """
         queue_name = self.get_remote_queue_name('storage', "slow")
         queue_name = self.get_remote_queue_name('storage', "slow")
@@ -101,7 +101,7 @@ class DataStore(Model):
         return orphans
         return orphans
 
 
     @method_cache(30)
     @method_cache(30)
-    def get_missing_disks(self, timeout=15):
+    def get_missing_disks(self, timeout=25):
         """Disk objects without disk image files.
         """Disk objects without disk image files.
         """
         """
         queue_name = self.get_remote_queue_name('storage', "slow")
         queue_name = self.get_remote_queue_name('storage', "slow")
@@ -111,7 +111,7 @@ class DataStore(Model):
         return disks.exclude(filename__in=files)
         return disks.exclude(filename__in=files)
 
 
     @method_cache(120)
     @method_cache(120)
-    def get_file_statistics(self, timeout=30):
+    def get_file_statistics(self, timeout=90):
         queue_name = self.get_remote_queue_name('storage', "slow")
         queue_name = self.get_remote_queue_name('storage', "slow")
         data = storage_tasks.get_file_statistics.apply_async(
         data = storage_tasks.get_file_statistics.apply_async(
             args=[self.path], queue=queue_name).get(timeout=timeout)
             args=[self.path], queue=queue_name).get(timeout=timeout)

circle/vm/models/node.py

@@ -349,7 +349,7 @@ class Node(OperatedMixin, TimeStampedModel):
                         continue
                         continue
                     value = target['datapoints'][-2][0]
                     value = target['datapoints'][-2][0]
                     retval[metric] = float(value)
                     retval[metric] = float(value)
-                except (KeyError, IndexError, ValueError):
+                except (KeyError, IndexError, ValueError, TypeError):
                     continue
                     continue
 
 
             return retval
             return retval

circle/vm/operations.py

@@ -421,7 +421,7 @@ class DeployOperation(InstanceOperation):
         description = _("Deploy virtual machine.")
         description = _("Deploy virtual machine.")
         remote_queue = ("vm", "slow")
         remote_queue = ("vm", "slow")
         task = vm_tasks.deploy
         task = vm_tasks.deploy
-
+	remote_timeout = 120
         def _get_remote_args(self, **kwargs):
         def _get_remote_args(self, **kwargs):
             return [self.instance.get_vm_desc()]
             return [self.instance.get_vm_desc()]
             # intentionally not calling super
             # intentionally not calling super

circle/vm/tasks/local_periodic_tasks.py

@@ -58,9 +58,9 @@ def garbage_collector(timeout=15):
                              i.pk, unicode(e))
                              i.pk, unicode(e))
         elif (i.time_of_suspend and now > i.time_of_suspend and
         elif (i.time_of_suspend and now > i.time_of_suspend and
               i.state == 'RUNNING'):
               i.state == 'RUNNING'):
-            i.sleep.async(system=True)
             logger.info("Expired instance %d suspended." % i.pk)
             logger.info("Expired instance %d suspended." % i.pk)
             try:
             try:
+            	i.sleep.async(system=True)
                 i.owner.profile.notify(
                 i.owner.profile.notify(
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop('%(instance)s suspended'),
                     ugettext_noop(
                     ugettext_noop(
@@ -68,8 +68,10 @@ def garbage_collector(timeout=15):
                         'has been suspended due to expiration. '
                         'has been suspended due to expiration. '
                         'You can resume or destroy it.'),
                         'You can resume or destroy it.'),
                     instance=i.name, url=i.get_absolute_url())
                     instance=i.name, url=i.get_absolute_url())
+ 	    except ActivityInProgressError:
+            	logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
             except Exception as e:
             except Exception as e:
-                logger.debug('Could not notify owner of instance %d .%s',
+                logger.info('Could not notify owner of instance %d .%s',
                              i.pk, unicode(e))
                              i.pk, unicode(e))
         elif i.is_expiring():
         elif i.is_expiring():
             logger.debug("Instance %d expires soon." % i.pk)
             logger.debug("Instance %d expires soon." % i.pk)