Commit 2d99cd14 by Guba Sándor

Merge branch 'feature-priority-queues'

Conflicts:
	circle/vm/models/instance.py
parents 5531552b 51461497
......@@ -59,11 +59,12 @@ class DataStore(Model):
def __unicode__(self):
return u'%s (%s)' % (self.name, self.path)
def get_remote_queue_name(self, queue_id, check_worker=True):
def get_remote_queue_name(self, queue_id, priority=None,
check_worker=True):
logger.debug("Checking for storage queue %s.%s",
self.hostname, queue_id)
if not check_worker or local_tasks.check_queue(self.hostname,
queue_id):
queue_id, priority):
return self.hostname + '.' + queue_id
else:
raise WorkerNotFound()
......@@ -292,11 +293,13 @@ class Disk(AclBase, TimeStampedModel):
'type': 'snapshot' if self.base else 'normal'
}
def get_remote_queue_name(self, queue_id='storage', check_worker=True):
def get_remote_queue_name(self, queue_id='storage', priority=None,
check_worker=True):
"""Returns the proper queue name based on the datastore.
"""
if self.datastore:
return self.datastore.get_remote_queue_name(queue_id, check_worker)
return self.datastore.get_remote_queue_name(queue_id, priority,
check_worker)
else:
return None
......
......@@ -20,14 +20,16 @@ from celery.contrib.abortable import AbortableTask
@celery.task
def check_queue(storage, queue_id):
def check_queue(storage, queue_id, priority):
''' Celery inspect job to check for active workers at queue_id
return True/False
'''
drivers = ['storage', 'download']
worker_list = [storage + "." + d for d in drivers]
queue_name = storage + "." + queue_id
active_queues = celery.control.inspect(worker_list).active_queues()
if priority is not None:
queue_name = queue_name + "." + priority
inspect = celery.control.inspect()
inspect.timeout = 0.1
active_queues = inspect.active_queues()
if active_queues is None:
return False
......
......@@ -630,12 +630,12 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
'raw_data': "" if not self.raw_data else self.raw_data
}
def get_remote_queue_name(self, queue_id):
def get_remote_queue_name(self, queue_id, priority=None):
"""Get the remote worker queue name of this instance with the specified
queue ID.
"""
if self.node:
return self.node.get_remote_queue_name(queue_id)
return self.node.get_remote_queue_name(queue_id, priority)
else:
raise Node.DoesNotExist()
......@@ -796,7 +796,7 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
net.shutdown()
def delete_vm(self, timeout=15):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'fast')
try:
return vm_tasks.destroy.apply_async(args=[self.vm_name],
queue=queue_name
......@@ -809,38 +809,38 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
raise
def deploy_vm(self, timeout=15):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
return vm_tasks.deploy.apply_async(args=[self.get_vm_desc()],
queue=queue_name
).get(timeout=timeout)
def migrate_vm(self, to_node, timeout=120):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
return vm_tasks.migrate.apply_async(args=[self.vm_name,
to_node.host.hostname],
queue=queue_name
).get(timeout=timeout)
def reboot_vm(self, timeout=5):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'fast')
return vm_tasks.reboot.apply_async(args=[self.vm_name],
queue=queue_name
).get(timeout=timeout)
def reset_vm(self, timeout=5):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'fast')
return vm_tasks.reset.apply_async(args=[self.vm_name],
queue=queue_name
).get(timeout=timeout)
def resume_vm(self, timeout=15):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
return vm_tasks.resume.apply_async(args=[self.vm_name],
queue=queue_name
).get(timeout=timeout)
def shutdown_vm(self, task=None, step=5):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
logger.debug("RPC Shutdown at queue: %s, for vm: %s.", queue_name,
self.vm_name)
remote = vm_tasks.shutdown.apply_async(kwargs={'name': self.vm_name},
......@@ -855,14 +855,14 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
raise Exception("Shutdown aborted by user.")
def suspend_vm(self, timeout=60):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
return vm_tasks.sleep.apply_async(args=[self.vm_name,
self.mem_dump['path']],
queue=queue_name
).get(timeout=timeout)
def wake_up_vm(self, timeout=60):
queue_name = self.get_remote_queue_name('vm')
queue_name = self.get_remote_queue_name('vm', 'slow')
return vm_tasks.wake_up.apply_async(args=[self.vm_name,
self.mem_dump['path']],
queue=queue_name
......
......@@ -93,7 +93,7 @@ class Node(OperatedMixin, TimeStampedModel):
Check if node is online by queue is available.
"""
try:
self.get_remote_queue_name("vm")
self.get_remote_queue_name("vm", "fast")
except:
return False
else:
......@@ -105,6 +105,7 @@ class Node(OperatedMixin, TimeStampedModel):
@method_cache(300)
def get_info(self):
return self.remote_query(vm_tasks.get_info,
priority='fast',
default={'core_num': '',
'ram_size': '0',
'architecture': ''})
......@@ -163,16 +164,19 @@ class Node(OperatedMixin, TimeStampedModel):
return self.ram_size * self.overcommit
@method_cache(30)
def get_remote_queue_name(self, queue_id):
def get_remote_queue_name(self, queue_id, priority=None):
"""Returns the name of the remote celery queue for this node.
Throws Exception if there is no worker on the queue.
The result may include dead queues because of caching.
"""
if vm_tasks.check_queue(self.host.hostname, queue_id):
if vm_tasks.check_queue(self.host.hostname, queue_id, priority):
queue_name = self.host.hostname + "." + queue_id
if priority is not None:
queue_name = queue_name + "." + priority
self.node_online()
return self.host.hostname + "." + queue_id
return queue_name
else:
if self.enabled:
self.node_offline()
......@@ -222,7 +226,8 @@ class Node(OperatedMixin, TimeStampedModel):
", but enabled" if self.enabled else "")
# TODO: check if we should reschedule any VMs?
def remote_query(self, task, timeout=30, raise_=False, default=None):
def remote_query(self, task, timeout=30, priority=None, raise_=False,
default=None):
"""Query the given task, and get the result.
If the result is not ready or worker not reachable
......@@ -231,7 +236,8 @@ class Node(OperatedMixin, TimeStampedModel):
"""
try:
r = task.apply_async(
queue=self.get_remote_queue_name('vm'), expires=timeout + 60)
queue=self.get_remote_queue_name('vm', priority),
expires=timeout + 60)
return r.get(timeout=timeout)
except (TimeoutError, WorkerNotFound):
if raise_:
......@@ -244,7 +250,8 @@ class Node(OperatedMixin, TimeStampedModel):
try:
handler = GraphiteHandler()
except RuntimeError:
return self.remote_query(vm_tasks.get_node_metrics, 30)
return self.remote_query(vm_tasks.get_node_metrics, timeout=30,
priority="fast")
query = Query()
query.set_target(self.host.hostname + ".circle")
......@@ -309,7 +316,8 @@ class Node(OperatedMixin, TimeStampedModel):
vm_state_changed hook.
"""
domains = {}
domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5)
domain_list = self.remote_query(vm_tasks.list_domains_info, timeout=5,
priority="fast")
if domain_list is None:
logger.info("Monitoring failed at: %s", self.name)
return
......
......@@ -23,16 +23,19 @@ from manager.mancelery import celery
logger = getLogger(__name__)
def check_queue(node_hostname, queue_id):
def check_queue(node_hostname, queue_id, priority=None):
"""True if the queue is alive.
Example: check_queue('node01', 'vm'):
Example: check_queue('node01', 'vm', 'slow'):
:param node_hostname: Short hostname of the node.
:param queue_id: Queue identifier (eg. vm).
:param priority: can be 'slow', 'fast' or None
"""
# drivers = ['vmdriver', 'netdriver', 'agentdriver']
# worker_list = [node_hostname + "." + d for d in drivers]
queue_name = node_hostname + "." + queue_id
if priority is not None:
queue_name = queue_name + "." + priority
active_queues = get_queues()
if active_queues is None:
return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment