Commit a58577ec by szebi

Manager: increase timeout for deploy and destroy

Manager: email notification fix
Storage: timeout + file type fix
VM: ActionInProgressError fix
VM: increased timeout
parent a001ad0d
......@@ -39,7 +39,7 @@ def _apply_once(name, tasks, queues, task, data):
data = data()
for queue in queues:
try:
task.apply_async(args=data, queue=queue, expires=60).get(timeout=2)
task.apply_async(args=data, queue=queue, expires=60).get(timeout=5)
logger.info("%s configuration is reloaded. (queue: %s)",
name, queue)
except TimeoutError as e:
......@@ -76,8 +76,14 @@ def reloadtask_worker():
logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
firewall_queues = get_firewall_queues()
dns_queues = [("%s.dns" % i) for i in
settings.get('dns_queues', [gethostname()])]
dns_queues = settings.get('dns_queues', [gethostname()])
if isinstance(dns_queues, (str, unicode)):
dns_queues = [dns_queues]
dns_queues = [("%s.dns" % i) for i in dns_queues]
# dns_queues = [("%s.dns" % i) for i in
# settings.get('dns_queues', [gethostname()])]
_apply_once('dns', tasks, dns_queues, reload_dns,
lambda: (dns(), ))
......
......@@ -18,6 +18,7 @@
from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta
from celery.schedules import crontab
from kombu import Queue, Exchange
from os import getenv
......@@ -52,7 +53,7 @@ celery.conf.update(
'dashboard.send_email_notifications': {
'task': 'dashboard.tasks.local_periodic_tasks.'
'send_email_notifications',
'schedule': timedelta(hours=24),
'schedule': crontab(minute=10, hour=1),
'options': {'queue': 'localhost.man'}
},
}
......
......@@ -86,7 +86,7 @@ class DataStore(Model):
args=[self.path], queue=q).get(timeout=timeout)
@method_cache(30)
def get_orphan_disks(self, timeout=15):
def get_orphan_disks(self, timeout=25):
"""Disk image files without Disk object in the database.
"""
queue_name = self.get_remote_queue_name('storage', "slow")
......@@ -101,7 +101,7 @@ class DataStore(Model):
return orphans
@method_cache(30)
def get_missing_disks(self, timeout=15):
def get_missing_disks(self, timeout=25):
"""Disk objects without disk image files.
"""
queue_name = self.get_remote_queue_name('storage', "slow")
......@@ -111,7 +111,7 @@ class DataStore(Model):
return disks.exclude(filename__in=files)
@method_cache(120)
def get_file_statistics(self, timeout=30):
def get_file_statistics(self, timeout=90):
queue_name = self.get_remote_queue_name('storage', "slow")
data = storage_tasks.get_file_statistics.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout)
......
......@@ -349,7 +349,7 @@ class Node(OperatedMixin, TimeStampedModel):
continue
value = target['datapoints'][-2][0]
retval[metric] = float(value)
except (KeyError, IndexError, ValueError):
except (KeyError, IndexError, ValueError, TypeError):
continue
return retval
......
......@@ -421,7 +421,7 @@ class DeployOperation(InstanceOperation):
description = _("Deploy virtual machine.")
remote_queue = ("vm", "slow")
task = vm_tasks.deploy
remote_timeout = 120
def _get_remote_args(self, **kwargs):
return [self.instance.get_vm_desc()]
# intentionally not calling super
......
......@@ -58,9 +58,9 @@ def garbage_collector(timeout=15):
i.pk, unicode(e))
elif (i.time_of_suspend and now > i.time_of_suspend and
i.state == 'RUNNING'):
i.sleep.async(system=True)
logger.info("Expired instance %d suspended." % i.pk)
try:
i.sleep.async(system=True)
i.owner.profile.notify(
ugettext_noop('%(instance)s suspended'),
ugettext_noop(
......@@ -68,8 +68,10 @@ def garbage_collector(timeout=15):
'has been suspended due to expiration. '
'You can resume or destroy it.'),
instance=i.name, url=i.get_absolute_url())
except ActivityInProgressError:
logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
except Exception as e:
logger.debug('Could not notify owner of instance %d .%s',
logger.info('Could not notify owner of instance %d .%s',
i.pk, unicode(e))
elif i.is_expiring():
logger.debug("Instance %d expires soon." % i.pk)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment