Commit a58577ec by szebi

Manager: increase timeout for deploy and destroy

Manager: email notification fix
Storage: timeout + file type fix
VM: ActionInProgressError fix
VM: increased timeout
parent a001ad0d
Pipeline #879 passed with stage
in 0 seconds
...@@ -39,7 +39,7 @@ def _apply_once(name, tasks, queues, task, data): ...@@ -39,7 +39,7 @@ def _apply_once(name, tasks, queues, task, data):
data = data() data = data()
for queue in queues: for queue in queues:
try: try:
task.apply_async(args=data, queue=queue, expires=60).get(timeout=2) task.apply_async(args=data, queue=queue, expires=60).get(timeout=5)
logger.info("%s configuration is reloaded. (queue: %s)", logger.info("%s configuration is reloaded. (queue: %s)",
name, queue) name, queue)
except TimeoutError as e: except TimeoutError as e:
...@@ -76,8 +76,14 @@ def reloadtask_worker(): ...@@ -76,8 +76,14 @@ def reloadtask_worker():
logger.info("reloadtask_worker: Reload %s", ", ".join(tasks)) logger.info("reloadtask_worker: Reload %s", ", ".join(tasks))
firewall_queues = get_firewall_queues() firewall_queues = get_firewall_queues()
dns_queues = [("%s.dns" % i) for i in
settings.get('dns_queues', [gethostname()])] dns_queues = settings.get('dns_queues', [gethostname()])
if isinstance(dns_queues, (str, unicode)):
dns_queues = [dns_queues]
dns_queues = [("%s.dns" % i) for i in dns_queues]
# dns_queues = [("%s.dns" % i) for i in
# settings.get('dns_queues', [gethostname()])]
_apply_once('dns', tasks, dns_queues, reload_dns, _apply_once('dns', tasks, dns_queues, reload_dns,
lambda: (dns(), )) lambda: (dns(), ))
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
from celery import Celery from celery import Celery
from celery.signals import worker_ready from celery.signals import worker_ready
from datetime import timedelta from datetime import timedelta
from celery.schedules import crontab
from kombu import Queue, Exchange from kombu import Queue, Exchange
from os import getenv from os import getenv
...@@ -52,7 +53,7 @@ celery.conf.update( ...@@ -52,7 +53,7 @@ celery.conf.update(
'dashboard.send_email_notifications': { 'dashboard.send_email_notifications': {
'task': 'dashboard.tasks.local_periodic_tasks.' 'task': 'dashboard.tasks.local_periodic_tasks.'
'send_email_notifications', 'send_email_notifications',
'schedule': timedelta(hours=24), 'schedule': crontab(minute=10, hour=1),
'options': {'queue': 'localhost.man'} 'options': {'queue': 'localhost.man'}
}, },
} }
......
...@@ -86,7 +86,7 @@ class DataStore(Model): ...@@ -86,7 +86,7 @@ class DataStore(Model):
args=[self.path], queue=q).get(timeout=timeout) args=[self.path], queue=q).get(timeout=timeout)
@method_cache(30) @method_cache(30)
def get_orphan_disks(self, timeout=15): def get_orphan_disks(self, timeout=25):
"""Disk image files without Disk object in the database. """Disk image files without Disk object in the database.
""" """
queue_name = self.get_remote_queue_name('storage', "slow") queue_name = self.get_remote_queue_name('storage', "slow")
...@@ -101,7 +101,7 @@ class DataStore(Model): ...@@ -101,7 +101,7 @@ class DataStore(Model):
return orphans return orphans
@method_cache(30) @method_cache(30)
def get_missing_disks(self, timeout=15): def get_missing_disks(self, timeout=25):
"""Disk objects without disk image files. """Disk objects without disk image files.
""" """
queue_name = self.get_remote_queue_name('storage', "slow") queue_name = self.get_remote_queue_name('storage', "slow")
...@@ -111,7 +111,7 @@ class DataStore(Model): ...@@ -111,7 +111,7 @@ class DataStore(Model):
return disks.exclude(filename__in=files) return disks.exclude(filename__in=files)
@method_cache(120) @method_cache(120)
def get_file_statistics(self, timeout=30): def get_file_statistics(self, timeout=90):
queue_name = self.get_remote_queue_name('storage', "slow") queue_name = self.get_remote_queue_name('storage', "slow")
data = storage_tasks.get_file_statistics.apply_async( data = storage_tasks.get_file_statistics.apply_async(
args=[self.path], queue=queue_name).get(timeout=timeout) args=[self.path], queue=queue_name).get(timeout=timeout)
......
...@@ -349,7 +349,7 @@ class Node(OperatedMixin, TimeStampedModel): ...@@ -349,7 +349,7 @@ class Node(OperatedMixin, TimeStampedModel):
continue continue
value = target['datapoints'][-2][0] value = target['datapoints'][-2][0]
retval[metric] = float(value) retval[metric] = float(value)
except (KeyError, IndexError, ValueError): except (KeyError, IndexError, ValueError, TypeError):
continue continue
return retval return retval
......
...@@ -421,7 +421,7 @@ class DeployOperation(InstanceOperation): ...@@ -421,7 +421,7 @@ class DeployOperation(InstanceOperation):
description = _("Deploy virtual machine.") description = _("Deploy virtual machine.")
remote_queue = ("vm", "slow") remote_queue = ("vm", "slow")
task = vm_tasks.deploy task = vm_tasks.deploy
remote_timeout = 120
def _get_remote_args(self, **kwargs): def _get_remote_args(self, **kwargs):
return [self.instance.get_vm_desc()] return [self.instance.get_vm_desc()]
# intentionally not calling super # intentionally not calling super
......
...@@ -58,9 +58,9 @@ def garbage_collector(timeout=15): ...@@ -58,9 +58,9 @@ def garbage_collector(timeout=15):
i.pk, unicode(e)) i.pk, unicode(e))
elif (i.time_of_suspend and now > i.time_of_suspend and elif (i.time_of_suspend and now > i.time_of_suspend and
i.state == 'RUNNING'): i.state == 'RUNNING'):
i.sleep.async(system=True)
logger.info("Expired instance %d suspended." % i.pk) logger.info("Expired instance %d suspended." % i.pk)
try: try:
i.sleep.async(system=True)
i.owner.profile.notify( i.owner.profile.notify(
ugettext_noop('%(instance)s suspended'), ugettext_noop('%(instance)s suspended'),
ugettext_noop( ugettext_noop(
...@@ -68,8 +68,10 @@ def garbage_collector(timeout=15): ...@@ -68,8 +68,10 @@ def garbage_collector(timeout=15):
'has been suspended due to expiration. ' 'has been suspended due to expiration. '
'You can resume or destroy it.'), 'You can resume or destroy it.'),
instance=i.name, url=i.get_absolute_url()) instance=i.name, url=i.get_absolute_url())
except ActivityInProgressError:
logger.error("Expired instance %d can't be destroyed due the AtctivityInPorgressError.", i.pk)
except Exception as e: except Exception as e:
logger.debug('Could not notify owner of instance %d .%s', logger.info('Could not notify owner of instance %d .%s',
i.pk, unicode(e)) i.pk, unicode(e))
elif i.is_expiring(): elif i.is_expiring():
logger.debug("Instance %d expires soon." % i.pk) logger.debug("Instance %d expires soon." % i.pk)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment