Skip to content

Gutyán Gábor / circlestack

Go to a project
Commit feeea6da by Szabolcs Gelencser
Options

Comment unused celery tasks, configure local celery transport

parent 1d8b8edb
Showing with 416 additions and 404 deletions
circle/celery 0 → 100644
File added
circle/circle/db.sqlite3
No preview for this file type
circle/circle/settings/base.py
......@@ -368,6 +368,8 @@ DJANGO_APPS = (
# Admin panel and documentation:
'django.contrib.admin',
# 'django.contrib.admindocs',
'kombu.transport.django',
'djcelery',
)
THIRD_PARTY_APPS = (
......@@ -399,6 +401,13 @@ LOCAL_APPS = (
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
########## END APP CONFIGURATION
BROKER_URL = "django://" # tell kombu to use the Django database as the message queue
import djcelery
djcelery.setup_loader()
CELERY_IMPORTS = ('vm.tasks',)
AUTHENTICATION_BACKENDS = ('openstack_auth.backend.KeystoneBackend',)
AUTHENTICATION_URLS = ['openstack_auth.urls']
AUTH_USER_MODEL = 'openstack_auth.User'
......
circle/common/models.py
......@@ -38,7 +38,7 @@ from django.utils.functional import Promise
from django.utils.translation import ugettext_lazy as _, ugettext_noop
from jsonfield import JSONField
from manager.mancelery import celery
# from manager.mancelery import celery
from model_utils.models import TimeStampedModel
from openstack_auth.user import User
......@@ -242,7 +242,7 @@ class ActivityModel(TimeStampedModel):
return 'failed'
@celery.task()
# @celery.task()
def compute_cached(method, instance, memcached_seconds,
key, start, *args, **kwargs):
"""Compute and store actual value of cached method."""
......
circle/dashboard/views/vm.py
......@@ -47,7 +47,7 @@ from common.models import (
split_activity_code,
)
from firewall.models import Vlan, Host, Rule
from manager.scheduler import SchedulerError
# from manager.scheduler import SchedulerError
from network.models import DefaultPublicRouter, DefaultPublicRoutedNet
from openstack_api.nova import Server
from request.forms import TemplateRequestForm
......
circle/firewall/tasks/local_tasks.py
......@@ -22,7 +22,7 @@ import django.conf
from django.core.cache import cache
from celery.exceptions import TimeoutError
from manager.mancelery import celery
# from manager.mancelery import celery
from common.models import WorkerNotFound
settings = django.conf.settings.FIREWALL_SETTINGS
......@@ -60,7 +60,7 @@ def get_firewall_queues():
return list(retval)
@celery.task
#@celery.task
def reloadtask_worker():
from firewall.fw import BuildFirewall, dhcp, dns, ipset, vlan
from remote_tasks import (reload_dns, reload_dhcp, reload_firewall,
......@@ -91,7 +91,7 @@ def reloadtask_worker():
lambda: (list(ipset()), ))
@celery.task
#@celery.task
def reloadtask(type='Host', timeout=15, sync=False):
reload = {
'Host': ['dns', 'dhcp', 'firewall'],
......
circle/firewall/tasks/remote_tasks.py
......@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from manager.mancelery import celery
# from manager.mancelery import celery
def check_queue(firewall, queue_id, priority):
......@@ -36,32 +36,32 @@ def check_queue(firewall, queue_id, priority):
return queue_name in queue_names
@celery.task(name='firewall.reload_dns')
#@celery.task(name='firewall.reload_dns')
def reload_dns(data):
pass
@celery.task(name='firewall.reload_firewall')
#@celery.task(name='firewall.reload_firewall')
def reload_firewall(data4, data6):
pass
@celery.task(name='firewall.reload_firewall_vlan')
#@celery.task(name='firewall.reload_firewall_vlan')
def reload_firewall_vlan(data):
pass
@celery.task(name='firewall.reload_dhcp')
#@celery.task(name='firewall.reload_dhcp')
def reload_dhcp(data):
pass
@celery.task(name='firewall.reload_blacklist')
#@celery.task(name='firewall.reload_blacklist')
def reload_blacklist(data):
pass
@celery.task(name='firewall.get_dhcp_clients')
#@celery.task(name='firewall.get_dhcp_clients')
def get_dhcp_clients():
# {'00:21:5a:73:72:cd': {'interface': 'OFF', 'ip': None, 'hostname': None}}
pass
circle/manager/mancelery.py
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
# # Copyright 2014 Budapest University of Technology and Economics (BME IK)
# #
# # This file is part of CIRCLE Cloud.
# #
# # CIRCLE is free software: you can redistribute it and/or modify it under
# # the terms of the GNU General Public License as published by the Free
# # Software Foundation, either version 3 of the License, or (at your option)
# # any later version.
# #
# # CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# # details.
# #
# # You should have received a copy of the GNU General Public License along
# # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
#
# This file is part of CIRCLE Cloud.
# from celery import Celery
# from celery.signals import worker_ready
# from datetime import timedelta
# from kombu import Queue, Exchange
# from os import getenv
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
# HOSTNAME = "localhost"
# QUEUE_NAME = HOSTNAME + '.man'
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta
from kombu import Queue, Exchange
from os import getenv
HOSTNAME = "localhost"
QUEUE_NAME = HOSTNAME + '.man'
celery = Celery('manager',
broker=getenv("AMQP_URI"),
include=['vm.tasks.local_tasks',
'vm.tasks.local_periodic_tasks',
'vm.tasks.local_agent_tasks',
'storage.tasks.local_tasks',
'storage.tasks.periodic_tasks',
'firewall.tasks.local_tasks',
'dashboard.tasks.local_periodic_tasks',
])
celery.conf.update(
CELERY_RESULT_BACKEND='amqp',
CELERY_TASK_RESULT_EXPIRES=300,
CELERY_QUEUES=(
Queue(HOSTNAME + '.man', Exchange('manager', type='direct'),
routing_key="manager"),
),
CELERYBEAT_SCHEDULE={
'storage.periodic_tasks': {
'task': 'storage.tasks.periodic_tasks.garbage_collector',
'schedule': timedelta(hours=1),
'options': {'queue': 'localhost.man'}
},
'dashboard.send_email_notifications': {
'task': 'dashboard.tasks.local_periodic_tasks.'
'send_email_notifications',
'schedule': timedelta(hours=24),
'options': {'queue': 'localhost.man'}
},
}
)
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
# celery = Celery('manager',
# broker=getenv("AMQP_URI"),
# include=['vm.tasks.local_tasks',
# 'vm.tasks.local_periodic_tasks',
# 'vm.tasks.local_agent_tasks',
# 'storage.tasks.local_tasks',
# 'storage.tasks.periodic_tasks',
# 'firewall.tasks.local_tasks',
# 'dashboard.tasks.local_periodic_tasks',
# ])
#
# celery.conf.update(
# CELERY_RESULT_BACKEND='amqp',
# CELERY_TASK_RESULT_EXPIRES=300,
# CELERY_QUEUES=(
# Queue(HOSTNAME + '.man', Exchange('manager', type='direct'),
# routing_key="manager"),
# ),
# CELERYBEAT_SCHEDULE={
# 'storage.periodic_tasks': {
# 'task': 'storage.tasks.periodic_tasks.garbage_collector',
# 'schedule': timedelta(hours=1),
# 'options': {'queue': 'localhost.man'}
# },
# 'dashboard.send_email_notifications': {
# 'task': 'dashboard.tasks.local_periodic_tasks.'
# 'send_email_notifications',
# 'schedule': timedelta(hours=24),
# 'options': {'queue': 'localhost.man'}
# },
# }
#
# )
#
#
# @worker_ready.connect()
# def cleanup_tasks(conf=None, **kwargs):
# '''Discard all task and clean up activity.'''
# from vm.models.activity import cleanup
# cleanup(queue_name=QUEUE_NAME)
circle/manager/moncelery.py
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
# # Copyright 2014 Budapest University of Technology and Economics (BME IK)
# #
# # This file is part of CIRCLE Cloud.
# #
# # CIRCLE is free software: you can redistribute it and/or modify it under
# # the terms of the GNU General Public License as published by the Free
# # Software Foundation, either version 3 of the License, or (at your option)
# # any later version.
# #
# # CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# # details.
# #
# # You should have received a copy of the GNU General Public License along
# # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
#
# This file is part of CIRCLE Cloud.
# from celery import Celery
# from celery.signals import worker_ready
# from datetime import timedelta
# from kombu import Queue, Exchange
# from os import getenv
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
# HOSTNAME = "localhost"
# QUEUE_NAME = HOSTNAME + '.monitor'
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
# celery = Celery('monitor',
# broker=getenv("AMQP_URI"),
# include=['vm.tasks.local_periodic_tasks',
# 'monitor.tasks.local_periodic_tasks',
# ])
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta
from kombu import Queue, Exchange
from os import getenv
HOSTNAME = "localhost"
QUEUE_NAME = HOSTNAME + '.monitor'
celery = Celery('monitor',
broker=getenv("AMQP_URI"),
include=['vm.tasks.local_periodic_tasks',
'monitor.tasks.local_periodic_tasks',
])
celery.conf.update(
CELERY_RESULT_BACKEND='amqp',
CELERY_TASK_RESULT_EXPIRES=300,
CELERY_QUEUES=(
Queue(QUEUE_NAME, Exchange('monitor', type='direct'),
routing_key="monitor"),
),
CELERYBEAT_SCHEDULE={
'vm.update_domain_states': {
'task': 'vm.tasks.local_periodic_tasks.update_domain_states',
'schedule': timedelta(seconds=10),
'options': {'queue': 'localhost.monitor'}
},
'monitor.measure_response_time': {
'task': 'monitor.tasks.local_periodic_tasks.'
'measure_response_time',
'schedule': timedelta(seconds=30),
'options': {'queue': 'localhost.monitor'}
},
'monitor.check_celery_queues': {
'task': 'monitor.tasks.local_periodic_tasks.'
'check_celery_queues',
'schedule': timedelta(seconds=60),
'options': {'queue': 'localhost.monitor'}
},
'monitor.instance_per_template': {
'task': 'monitor.tasks.local_periodic_tasks.'
'instance_per_template',
'schedule': timedelta(seconds=30),
'options': {'queue': 'localhost.monitor'}
},
'monitor.allocated_memory': {
'task': 'monitor.tasks.local_periodic_tasks.'
'allocated_memory',
'schedule': timedelta(seconds=30),
'options': {'queue': 'localhost.monitor'}
},
}
)
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
# celery.conf.update(
# CELERY_RESULT_BACKEND='amqp',
# CELERY_TASK_RESULT_EXPIRES=300,
# CELERY_QUEUES=(
# Queue(QUEUE_NAME, Exchange('monitor', type='direct'),
# routing_key="monitor"),
# ),
# CELERYBEAT_SCHEDULE={
# 'vm.update_domain_states': {
# 'task': 'vm.tasks.local_periodic_tasks.update_domain_states',
# 'schedule': timedelta(seconds=10),
# 'options': {'queue': 'localhost.monitor'}
# },
# 'monitor.measure_response_time': {
# 'task': 'monitor.tasks.local_periodic_tasks.'
# 'measure_response_time',
# 'schedule': timedelta(seconds=30),
# 'options': {'queue': 'localhost.monitor'}
# },
# 'monitor.check_celery_queues': {
# 'task': 'monitor.tasks.local_periodic_tasks.'
# 'check_celery_queues',
# 'schedule': timedelta(seconds=60),
# 'options': {'queue': 'localhost.monitor'}
# },
# 'monitor.instance_per_template': {
# 'task': 'monitor.tasks.local_periodic_tasks.'
# 'instance_per_template',
# 'schedule': timedelta(seconds=30),
# 'options': {'queue': 'localhost.monitor'}
# },
# 'monitor.allocated_memory': {
# 'task': 'monitor.tasks.local_periodic_tasks.'
# 'allocated_memory',
# 'schedule': timedelta(seconds=30),
# 'options': {'queue': 'localhost.monitor'}
# },
# }
#
# )
#
#
# @worker_ready.connect()
# def cleanup_tasks(conf=None, **kwargs):
# '''Discard all task and clean up activity.'''
# from vm.models.activity import cleanup
# cleanup(queue_name=QUEUE_NAME)
circle/manager/scheduler.py
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
#
# This file is part of CIRCLE Cloud.
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from logging import getLogger
from django.utils.translation import ugettext_noop
from common.models import HumanReadableException
logger = getLogger(__name__)
class SchedulerError(HumanReadableException):
admin_message = None
def __init__(self, params=None, level=None, **kwargs):
kwargs.update(params or {})
super(SchedulerError, self).__init__(
level, self.message, self.admin_message or self.message,
kwargs)
class NotEnoughMemoryException(SchedulerError):
message = ugettext_noop(
"The resources required for launching the virtual machine are not "
"available currently. Please try again later.")
admin_message = ugettext_noop(
"The required free memory for launching the virtual machine is not "
"available on any usable node currently. Please try again later.")
class TraitsUnsatisfiableException(SchedulerError):
message = ugettext_noop(
"No node can satisfy the required traits of the "
"new virtual machine currently.")
def select_node(instance, nodes):
''' Select a node for hosting an instance based on its requirements.
'''
# check required traits
nodes = [n for n in nodes
if n.schedule_enabled and n.online and
has_traits(instance.req_traits.all(), n)]
if not nodes:
logger.warning('select_node: no usable node for %s', unicode(instance))
raise TraitsUnsatisfiableException()
# check required RAM
nodes = [n for n in nodes if has_enough_ram(instance.ram_size, n)]
if not nodes:
logger.warning('select_node: no enough RAM for %s', unicode(instance))
raise NotEnoughMemoryException()
# sort nodes first by processor usage, then priority
nodes.sort(key=lambda n: n.priority, reverse=True)
nodes.sort(key=free_cpu_time, reverse=True)
result = nodes[0]
logger.info('select_node: %s for %s', unicode(result), unicode(instance))
return result
def has_traits(traits, node):
"""True, if the node has all specified traits; otherwise, false.
"""
traits = set(traits)
return traits.issubset(node.traits.all())
def has_enough_ram(ram_size, node):
"""True, if the node has enough memory to accomodate a guest requiring
ram_size mebibytes of memory; otherwise, false.
"""
ram_size = ram_size * 1024 * 1024
try:
total = node.ram_size
used = node.byte_ram_usage
unused = total - used
overcommit = node.ram_size_with_overcommit
reserved = node.allocated_ram
free = overcommit - reserved
retval = ram_size < unused and ram_size < free
logger.debug('has_enough_ram(%d, %s)=%s (total=%s unused=%s'
' overcommit=%s free=%s free_ok=%s overcommit_ok=%s)',
ram_size, node, retval, total, unused, overcommit, free,
ram_size < unused, ram_size < free)
return retval
except TypeError as e:
logger.exception('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return False
def free_cpu_time(node):
"""Get an indicator number for idle processor time on the node.
Higher values indicate more idle time.
"""
try:
activity = node.cpu_usage / 100
inactivity = 1 - activity
cores = node.num_cores
return cores * inactivity
except TypeError as e:
logger.warning('Got incorrect monitoring data for node %s. %s',
unicode(node), unicode(e))
return False # monitoring data is incorrect
# # Copyright 2014 Budapest University of Technology and Economics (BME IK)
# #
# # This file is part of CIRCLE Cloud.
# #
# # CIRCLE is free software: you can redistribute it and/or modify it under
# # the terms of the GNU General Public License as published by the Free
# # Software Foundation, either version 3 of the License, or (at your option)
# # any later version.
# #
# # CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# # details.
# #
# # You should have received a copy of the GNU General Public License along
# # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
#
# from logging import getLogger
#
# from django.utils.translation import ugettext_noop
#
# from common.models import HumanReadableException
#
# logger = getLogger(__name__)
#
#
# class SchedulerError(HumanReadableException):
# admin_message = None
#
# def __init__(self, params=None, level=None, **kwargs):
# kwargs.update(params or {})
# super(SchedulerError, self).__init__(
# level, self.message, self.admin_message or self.message,
# kwargs)
#
#
# class NotEnoughMemoryException(SchedulerError):
# message = ugettext_noop(
# "The resources required for launching the virtual machine are not "
# "available currently. Please try again later.")
#
# admin_message = ugettext_noop(
# "The required free memory for launching the virtual machine is not "
# "available on any usable node currently. Please try again later.")
#
#
# class TraitsUnsatisfiableException(SchedulerError):
# message = ugettext_noop(
# "No node can satisfy the required traits of the "
# "new virtual machine currently.")
#
#
# def select_node(instance, nodes):
# ''' Select a node for hosting an instance based on its requirements.
# '''
# # check required traits
# nodes = [n for n in nodes
# if n.schedule_enabled and n.online and
# has_traits(instance.req_traits.all(), n)]
# if not nodes:
# logger.warning('select_node: no usable node for %s', unicode(instance))
# raise TraitsUnsatisfiableException()
#
# # check required RAM
# nodes = [n for n in nodes if has_enough_ram(instance.ram_size, n)]
# if not nodes:
# logger.warning('select_node: no enough RAM for %s', unicode(instance))
# raise NotEnoughMemoryException()
#
# # sort nodes first by processor usage, then priority
# nodes.sort(key=lambda n: n.priority, reverse=True)
# nodes.sort(key=free_cpu_time, reverse=True)
# result = nodes[0]
#
# logger.info('select_node: %s for %s', unicode(result), unicode(instance))
# return result
#
#
# def has_traits(traits, node):
# """True, if the node has all specified traits; otherwise, false.
# """
# traits = set(traits)
# return traits.issubset(node.traits.all())
#
#
# def has_enough_ram(ram_size, node):
# """True, if the node has enough memory to accomodate a guest requiring
# ram_size mebibytes of memory; otherwise, false.
# """
# ram_size = ram_size * 1024 * 1024
# try:
# total = node.ram_size
# used = node.byte_ram_usage
# unused = total - used
#
# overcommit = node.ram_size_with_overcommit
# reserved = node.allocated_ram
# free = overcommit - reserved
#
# retval = ram_size < unused and ram_size < free
#
# logger.debug('has_enough_ram(%d, %s)=%s (total=%s unused=%s'
# ' overcommit=%s free=%s free_ok=%s overcommit_ok=%s)',
# ram_size, node, retval, total, unused, overcommit, free,
# ram_size < unused, ram_size < free)
# return retval
# except TypeError as e:
# logger.exception('Got incorrect monitoring data for node %s. %s',
# unicode(node), unicode(e))
# return False
#
#
# def free_cpu_time(node):
# """Get an indicator number for idle processor time on the node.
#
# Higher values indicate more idle time.
# """
# try:
# activity = node.cpu_usage / 100
# inactivity = 1 - activity
# cores = node.num_cores
# return cores * inactivity
# except TypeError as e:
# logger.warning('Got incorrect monitoring data for node %s. %s',
# unicode(node), unicode(e))
# return False # monitoring data is incorrect
circle/manager/slowcelery.py
# Copyright 2014 Budapest University of Technology and Economics (BME IK)
# # Copyright 2014 Budapest University of Technology and Economics (BME IK)
# #
# # This file is part of CIRCLE Cloud.
# #
# # CIRCLE is free software: you can redistribute it and/or modify it under
# # the terms of the GNU General Public License as published by the Free
# # Software Foundation, either version 3 of the License, or (at your option)
# # any later version.
# #
# # CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# # details.
# #
# # You should have received a copy of the GNU General Public License along
# # with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
#
# This file is part of CIRCLE Cloud.
# from celery import Celery
# from celery.signals import worker_ready
# from datetime import timedelta
# from kombu import Queue, Exchange
# from os import getenv
#
# CIRCLE is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
# HOSTNAME = "localhost"
# QUEUE_NAME = HOSTNAME + '.man.slow'
#
# CIRCLE is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
# celery = Celery('manager.slow',
# broker=getenv("AMQP_URI"),
# include=['vm.tasks.local_tasks',
# 'vm.tasks.local_periodic_tasks',
# 'storage.tasks.local_tasks',
# 'storage.tasks.periodic_tasks',
# ])
#
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery import Celery
from celery.signals import worker_ready
from datetime import timedelta
from kombu import Queue, Exchange
from os import getenv
HOSTNAME = "localhost"
QUEUE_NAME = HOSTNAME + '.man.slow'
celery = Celery('manager.slow',
broker=getenv("AMQP_URI"),
include=['vm.tasks.local_tasks',
'vm.tasks.local_periodic_tasks',
'storage.tasks.local_tasks',
'storage.tasks.periodic_tasks',
])
celery.conf.update(
CELERY_RESULT_BACKEND='amqp',
CELERY_TASK_RESULT_EXPIRES=300,
CELERY_QUEUES=(
Queue(QUEUE_NAME, Exchange('manager.slow', type='direct'),
routing_key="manager.slow"),
),
CELERYBEAT_SCHEDULE={
'vm.garbage_collector': {
'task': 'vm.tasks.local_periodic_tasks.garbage_collector',
'schedule': timedelta(minutes=10),
'options': {'queue': 'localhost.man.slow'}
},
}
)
@worker_ready.connect()
def cleanup_tasks(conf=None, **kwargs):
'''Discard all task and clean up activity.'''
from vm.models.activity import cleanup
cleanup(queue_name=QUEUE_NAME)
# celery.conf.update(
# CELERY_RESULT_BACKEND='amqp',
# CELERY_TASK_RESULT_EXPIRES=300,
# CELERY_QUEUES=(
# Queue(QUEUE_NAME, Exchange('manager.slow', type='direct'),
# routing_key="manager.slow"),
# ),
# CELERYBEAT_SCHEDULE={
# 'vm.garbage_collector': {
# 'task': 'vm.tasks.local_periodic_tasks.garbage_collector',
# 'schedule': timedelta(minutes=10),
# 'options': {'queue': 'localhost.man.slow'}
# },
# }
#
# )
#
#
# @worker_ready.connect()
# def cleanup_tasks(conf=None, **kwargs):
# '''Discard all task and clean up activity.'''
# from vm.models.activity import cleanup
# cleanup(queue_name=QUEUE_NAME)
circle/storage/tasks/local_tasks.py
......@@ -15,11 +15,11 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from manager.mancelery import celery
# from manager.mancelery import celery
from celery.contrib.abortable import AbortableTask
@celery.task
#@celery.task
def check_queue(storage, queue_id, priority):
''' Celery inspect job to check for active workers at queue_id
return True/False
......@@ -38,34 +38,34 @@ def check_queue(storage, queue_id, priority):
return queue_name in queue_names
@celery.task
#@celery.task
def save_as(disk, timeout, user):
disk.save_disk_as(task_uuid=save_as.request.id, user=user,
disk=disk, timeout=timeout)
@celery.task
#@celery.task
def clone(disk, new_disk, timeout, user):
disk.clone(task_uuid=save_as.request.id, user=user,
disk=new_disk, timeout=timeout)
@celery.task
#@celery.task
def deploy(disk, user):
disk.deploy(task_uuid=deploy.request.id, user=user)
@celery.task
#@celery.task
def destroy(disk, user):
disk.destroy(task_uuid=destroy.request.id, user=user)
@celery.task
#@celery.task
def restore(disk, user):
disk.restore(task_uuid=restore.request.id, user=user)
@celery.task(base=AbortableTask, bind=True)
#@celery.task(base=AbortableTask, bind=True)
def create_from_url(self, **kwargs):
Disk = kwargs.pop('cls')
Disk.create_from_url(url=kwargs.pop('url'),
......@@ -74,7 +74,7 @@ def create_from_url(self, **kwargs):
**kwargs)
@celery.task
#@celery.task
def create_empty(Disk, instance, user, params):
Disk.create_empty(instance, user,
task_uuid=create_empty.request.id,
......
circle/storage/tasks/storage_tasks.py
......@@ -15,74 +15,74 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from manager.mancelery import celery
# from manager.mancelery import celery
@celery.task(name='storagedriver.list')
#@celery.task(name='storagedriver.list')
def list(dir):
pass
@celery.task(name='storagedriver.list_files')
#@celery.task(name='storagedriver.list_files')
def list_files(dir):
pass
@celery.task(name='storagedriver.create')
#@celery.task(name='storagedriver.create')
def create(disk_desc):
pass
@celery.task(name='storagedriver.download')
#@celery.task(name='storagedriver.download')
def download(disk_desc, url):
pass
@celery.task(name='storagedriver.delete')
#@celery.task(name='storagedriver.delete')
def delete(path):
pass
@celery.task(name='storagedriver.delete_dump')
#@celery.task(name='storagedriver.delete_dump')
def delete_dump(path):
pass
@celery.task(name='storagedriver.snapshot')
#@celery.task(name='storagedriver.snapshot')
def snapshot(disk_desc):
pass
@celery.task(name='storagedriver.get')
#@celery.task(name='storagedriver.get')
def get(path):
pass
@celery.task(name='storagedriver.merge')
#@celery.task(name='storagedriver.merge')
def merge(src_disk_desc, dst_disk_desc):
pass
@celery.task(name='storagedriver.make_free_space')
#@celery.task(name='storagedriver.make_free_space')
def make_free_space(datastore, percent):
pass
@celery.task(name='storagedriver.move_to_trash')
#@celery.task(name='storagedriver.move_to_trash')
def move_to_trash(datastore, disk_path):
pass
@celery.task(name='storagedriver.recover_from_trash')
#@celery.task(name='storagedriver.recover_from_trash')
def recover_from_trash(datastore, disk_path):
pass
@celery.task(name='storagedriver.get_storage_stat')
#@celery.task(name='storagedriver.get_storage_stat')
def get_storage_stat(path):
pass
@celery.task(name='storagedriver.get_file_statistics')
#@celery.task(name='storagedriver.get_file_statistics')
def get_file_statistics(datastore):
pass
circle/vm/models/activity.py
......@@ -32,7 +32,7 @@ from common.models import (
HumanReadableObject, HumanReadableException,
)
from manager.mancelery import celery
# from manager.mancelery import celery
logger = getLogger(__name__)
......
circle/vm/operations.py
......@@ -45,7 +45,7 @@ from common.models import (
create_readable, humanize_exception, HumanReadableException
)
from common.operations import Operation, register_operation
from manager.scheduler import SchedulerError
# from manager.scheduler import SchedulerError
from .tasks.local_tasks import (
abortable_async_instance_operation, abortable_async_node_operation,
)
......
circle/vm/tasks/__init__.py
from test import *
\ No newline at end of file
circle/vm/tasks/agent_tasks.py
......@@ -15,79 +15,79 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from manager.mancelery import celery
# from manager.mancelery import celery
@celery.task(name='agent.change_password')
#@celery.task(name='agent.change_password')
def change_password(vm, password):
pass
@celery.task(name='agent.restart_networking')
#@celery.task(name='agent.restart_networking')
def restart_networking(vm):
pass
@celery.task(name='agent.set_time')
#@celery.task(name='agent.set_time')
def set_time(vm, time):
pass
@celery.task(name='agent.set_hostname')
#@celery.task(name='agent.set_hostname')
def set_hostname(vm, time):
pass
@celery.task(name='agent.mount_store')
#@celery.task(name='agent.mount_store')
def mount_store(vm, host, username, password):
pass
@celery.task(name='agent.cleanup')
#@celery.task(name='agent.cleanup')
def cleanup(vm):
pass
@celery.task(name='agent.start_access_server')
#@celery.task(name='agent.start_access_server')
def start_access_server(vm):
pass
@celery.task(name='agent.update_legacy')
#@celery.task(name='agent.update_legacy')
def update_legacy(vm, data, executable=None):
pass
@celery.task(name='agent.append')
#@celery.task(name='agent.append')
def append(vm, data, filename, chunk_number):
pass
@celery.task(name='agent.update')
#@celery.task(name='agent.update')
def update(vm, filename, executable, checksum):
pass
@celery.task(name='agent.add_keys')
#@celery.task(name='agent.add_keys')
def add_keys(vm, keys):
pass
@celery.task(name='agent.del_keys')
#@celery.task(name='agent.del_keys')
def del_keys(vm, keys):
pass
@celery.task(name='agent.get_keys')
#@celery.task(name='agent.get_keys')
def get_keys(vm):
pass
@celery.task(name='agent.send_expiration')
#@celery.task(name='agent.send_expiration')
def send_expiration(vm, url):
pass
@celery.task(name='agent.change_ip')
#@celery.task(name='agent.change_ip')
def change_ip(vm, interfaces, dns):
pass
circle/vm/tasks/local_tasks.py
......@@ -16,10 +16,10 @@
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from celery.contrib.abortable import AbortableTask
from manager.mancelery import celery
# from manager.mancelery import celery
@celery.task(base=AbortableTask, bind=True)
#@celery.task(base=AbortableTask, bind=True)
def abortable_async_instance_operation(task, operation_id, instance_pk,
activity_pk, allargs, auxargs):
from vm.models import Instance, InstanceActivity
......@@ -37,7 +37,7 @@ def abortable_async_instance_operation(task, operation_id, instance_pk,
return operation._exec_op(allargs, auxargs)
@celery.task(base=AbortableTask, bind=True)
#@celery.task(base=AbortableTask, bind=True)
def abortable_async_node_operation(task, operation_id, node_pk, activity_pk,
allargs, auxargs):
from vm.models import Node, NodeActivity
......
circle/vm/tasks/net_tasks.py
......@@ -15,14 +15,14 @@
# You should have received a copy of the GNU General Public License along
# with CIRCLE. If not, see <http://www.gnu.org/licenses/>.
from manager.mancelery import celery
# from manager.mancelery import celery
@celery.task(name='netdriver.create')
#@celery.task(name='netdriver.create')
def create(params):
pass
@celery.task(name='netdriver.delete')
#@celery.task(name='netdriver.delete')
def destroy(params):
pass
circle/vm/tasks/vm_tasks.py
......@@ -18,7 +18,7 @@
from django.core.cache import cache
from logging import getLogger
from manager.mancelery import celery
# from manager.mancelery import celery
logger = getLogger(__name__)
......@@ -62,126 +62,126 @@ def get_queues():
return result
@celery.task(name='vmdriver.attach_disk')
#@celery.task(name='vmdriver.attach_disk')
def attach_disk(vm, disk):
pass
@celery.task(name='vmdriver.detach_disk')
#@celery.task(name='vmdriver.detach_disk')
def detach_disk(vm, disk):
pass
@celery.task(name='vmdriver.attach_network')
#@celery.task(name='vmdriver.attach_network')
def attach_network(vm, net):
pass
@celery.task(name='vmdriver.detach_network')
#@celery.task(name='vmdriver.detach_network')
def detach_network(vm, net):
pass
@celery.task(name='vmdriver.create')
#@celery.task(name='vmdriver.create')
def deploy(params):
pass
@celery.task(name='vmdriver.delete')
#@celery.task(name='vmdriver.delete')
def destroy(params):
pass
@celery.task(name='vmdriver.save')
#@celery.task(name='vmdriver.save')
def sleep(params):
pass
@celery.task(name='vmdriver.restore')
#@celery.task(name='vmdriver.restore')
def wake_up(params):
pass
@celery.task(name='vmdriver.suspend')
#@celery.task(name='vmdriver.suspend')
def suspend(params):
pass
@celery.task(name='vmdriver.resume')
#@celery.task(name='vmdriver.resume')
def resume(params):
pass
@celery.task(name='vmdriver.shutdown')
#@celery.task(name='vmdriver.shutdown')
def shutdown(params):
pass
@celery.task(name='vmdriver.reset')
#@celery.task(name='vmdriver.reset')
def reset(params):
pass
@celery.task(name='vmdriver.reboot')
#@celery.task(name='vmdriver.reboot')
def reboot(params):
pass
@celery.task(name='vmdriver.migrate')
#@celery.task(name='vmdriver.migrate')
def migrate(params):
pass
@celery.task(name='vmdriver.resize_disk')
#@celery.task(name='vmdriver.resize_disk')
def resize_disk(params):
pass
@celery.task(name='vmdriver.domain_info')
#@celery.task(name='vmdriver.domain_info')
def domain_info(params):
pass
@celery.task(name='vmdriver.list_domains')
#@celery.task(name='vmdriver.list_domains')
def list_domains(params):
pass
@celery.task(name='vmdriver.list_domains_info')
#@celery.task(name='vmdriver.list_domains_info')
def list_domains_info(params):
pass
@celery.task(name='vmdriver.ping')
#@celery.task(name='vmdriver.ping')
def ping(params):
pass
@celery.task(name='vmdriver.get_core_num')
#@celery.task(name='vmdriver.get_core_num')
def get_core_num(params):
pass
@celery.task(name='vmdriver.get_architecture')
#@celery.task(name='vmdriver.get_architecture')
def get_architecture():
pass
@celery.task(name='vmdriver.get_ram_size')
#@celery.task(name='vmdriver.get_ram_size')
def get_ram_size(params):
pass
@celery.task(name='vmdriver.get_info')
#@celery.task(name='vmdriver.get_info')
def get_info(params):
pass
@celery.task(name='vmdriver.get_node_metrics')
#@celery.task(name='vmdriver.get_node_metrics')
def get_node_metrics(params):
pass
@celery.task(name='vmdriver.screenshot')
#@celery.task(name='vmdriver.screenshot')
def screenshot(params):
pass
requirements/circlestack.txt
......@@ -26,6 +26,7 @@ Django==1.11.6
django-appconf==1.0.2
django-autocomplete-light==3.2.9
django-braces==1.11.0
django-celery==3.2.2
django-crispy-forms==1.6.1
django-model-utils==3.0.0
django-pipeline==1.6.13
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment