Commit cec0fef7 by cloud

Added periodic task for vm auto migration

parent eae91fdd
Pipeline #636 passed with stage
in 0 seconds
...@@ -590,3 +590,5 @@ TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE") ...@@ -590,3 +590,5 @@ TWO_FACTOR_ISSUER = get_env_variable("TWO_FACTOR_ISSUER", "CIRCLE")
# Default value is every day at midnight # Default value is every day at midnight
AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *") AUTO_MIGRATION_CRONTAB = get_env_variable("AUTO_MIGRATION_CRONTAB", "0 0 * * *")
AUTO_MIGRATION_TIME_LIMIT_IN_HOURS = (
get_env_variable("AUTO_MIGRATION_TIME_LIMIT_IN_HOURS", "2"))
...@@ -865,6 +865,53 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin, ...@@ -865,6 +865,53 @@ class Instance(AclBase, VirtualMachineDescModel, StatusModel, OperatedMixin,
def metric_prefix(self): def metric_prefix(self):
return 'vm.%s' % self.vm_name return 'vm.%s' % self.vm_name
class MonitorUnavailableException(Exception):
"""Exception for monitor_info()
Indicates the unavailability of the monitoring server.
"""
pass
def monitor_info(self):
metrics = ('cpu.percent', 'memory.usage')
prefix = self.metric_prefix
params = [('target', '%s.%s' % (prefix, metric))
for metric in metrics]
params.append(('from', '-5min'))
params.append(('format', 'json'))
try:
logger.info('%s %s', settings.GRAPHITE_URL, params)
response = requests.get(settings.GRAPHITE_URL, params=params)
retval = {}
for target in response.json():
# Example:
# {"target": "circle.vm.{name}.cpu.usage",
# "datapoints": [[0.6, 1403045700], [0.5, 1403045760]
try:
metric = target['target']
if metric.startswith(prefix):
metric = metric[len(prefix):]
else:
continue
value = target['datapoints'][-2][0]
retval[metric] = float(value)
except (KeyError, IndexError, ValueError):
continue
return retval
except Exception:
logger.exception('Monitor server unavailable: ')
raise Instance.MonitorUnavailableException()
def cpu_usage(self):
return self.monitor_info().get('cpu.percent')
def ram_usage(self):
return self.monitor_info().get('memory.usage')
@contextmanager @contextmanager
def activity(self, code_suffix, readable_name, on_abort=None, def activity(self, code_suffix, readable_name, on_abort=None,
on_commit=None, task_uuid=None, user=None, on_commit=None, task_uuid=None, user=None,
......
...@@ -17,7 +17,9 @@ ...@@ -17,7 +17,9 @@
import logging import logging
from django.utils import timezone from django.utils import timezone
from datetime import timedelta
from django.utils.translation import ugettext_noop from django.utils.translation import ugettext_noop
from django.conf import settings
from manager.mancelery import celery from manager.mancelery import celery
from vm.models import Node, Instance from vm.models import Node, Instance
...@@ -79,8 +81,40 @@ def garbage_collector(timeout=15): ...@@ -79,8 +81,40 @@ def garbage_collector(timeout=15):
@celery.task(ignore_result=True) @celery.task(ignore_result=True)
def auto_migrate(): # Dummy implementation def auto_migrate():
import time """Auto migration task for runtime scaling
logger.info("Auto migration has started.") """
time.sleep(10) time_limit = settings.AUTO_MIGRATION_TIME_LIMIT_IN_HOURS
logger.info("Auto migration has finished.") available_time = timedelta(hours=int(time_limit))
deadline = timezone.now() + available_time
while timezone.now() < deadline:
migrate_one()
def migrate_one():
"""Migrate a VM syncronously.
The target node chosen by the scheduler.
"""
nodes = [n for n in Node.objects.filter(enabled=True) if n.online]
node_max_cpu = max(nodes, key=lambda x: x.cpu_usage / x.cpu_weight)
node_max_ram = max(nodes, key=lambda x: x.ram_usage / x.ram_weight)
if node_max_cpu.cpu_usage > node_max_ram.ram_usage:
try:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=lambda x: x.cpu_usage())
instance_to_migrate.migrate(system=True)
except Instance.MonitorUnavailableException:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=(lambda x: x.get_vm_desc()["vcpu"] *
x.get_vm_desc()["cpu_share"]))
instance_to_migrate.migrate(system=True)
else:
try:
instance_to_migrate = max(Instance.objects.filter(node=node_max_ram.pk),
key=lambda x: x.ram_usage())
instance_to_migrate.migrate(system=True)
except Instance.MonitorUnavailableException:
instance_to_migrate = max(Instance.objects.filter(node=node_max_cpu.pk),
key=lambda x: x.get_vm_desc()["memory"])
instance_to_migrate.migrate(system=True)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment