Commit a077aed6 by Bach Dániel

Merge branch 'feature-virtio' into 'master'

Feature Virtio

Enhance CIRCLE to use virtio-serial ports to communicate with the agent.

 WARNING 

Requires (virtio branch):
🆕 agentdriver
🆕 vmdriver

Tests:
* new-windows agent    (Working virtio with update watchdog for service restart)
* new-linux agent 
* legacy-windows agent  (Old windows agent can't update)
* legacy-linux:   Working update with less than 16k tar

See merge request !205
parents 1b16674e c8916780
...@@ -431,9 +431,18 @@ LOGIN_REDIRECT_URL = "/" ...@@ -431,9 +431,18 @@ LOGIN_REDIRECT_URL = "/"
AGENT_DIR = get_env_variable( AGENT_DIR = get_env_variable(
'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent')) 'DJANGO_AGENT_DIR', join(unicode(expanduser("~")), 'agent'))
# AGENT_DIR is the root directory for the agent.
# The directory structure SHOULD be:
# /home/username/agent
# |-- agent-linux
# | |-- .git
# | +-- ...
# |-- agent-win
# | +-- agent-win-%(version).exe
#
try: try:
git_env = {'GIT_DIR': join(AGENT_DIR, '.git')} git_env = {'GIT_DIR': join(join(AGENT_DIR, "agent-linux"), '.git')}
AGENT_VERSION = check_output( AGENT_VERSION = check_output(
('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env) ('git', 'log', '-1', r'--pretty=format:%h', 'HEAD'), env=git_env)
except: except:
......
...@@ -53,8 +53,18 @@ def start_access_server(vm): ...@@ -53,8 +53,18 @@ def start_access_server(vm):
pass pass
@celery.task(name='agent.update_legacy')
def update_legacy(vm, data, executable=None):
pass
@celery.task(name='agent.append')
def append(vm, data, filename, chunk_number):
pass
@celery.task(name='agent.update') @celery.task(name='agent.update')
def update(vm, data): def update(vm, filename, executable, checksum):
pass pass
......
...@@ -19,11 +19,14 @@ from common.models import create_readable ...@@ -19,11 +19,14 @@ from common.models import create_readable
from manager.mancelery import celery from manager.mancelery import celery
from vm.tasks.agent_tasks import (restart_networking, change_password, from vm.tasks.agent_tasks import (restart_networking, change_password,
set_time, set_hostname, start_access_server, set_time, set_hostname, start_access_server,
cleanup, update, change_ip) cleanup, update, append,
change_ip, update_legacy)
from firewall.models import Host from firewall.models import Host
import time import time
import os
from base64 import encodestring from base64 import encodestring
from hashlib import md5
from StringIO import StringIO from StringIO import StringIO
from tarfile import TarFile, TarInfo from tarfile import TarFile, TarInfo
from django.conf import settings from django.conf import settings
...@@ -61,17 +64,34 @@ def send_networking_commands(instance, act): ...@@ -61,17 +64,34 @@ def send_networking_commands(instance, act):
restart_networking.apply_async(queue=queue, args=(instance.vm_name, )) restart_networking.apply_async(queue=queue, args=(instance.vm_name, ))
def create_agent_tar(): def create_linux_tar():
def exclude(tarinfo): def exclude(tarinfo):
if tarinfo.name.startswith('./.git'): ignored = ('./.', './misc', './windows')
if any(tarinfo.name.startswith(x) for x in ignored):
return None return None
else: else:
return tarinfo return tarinfo
f = StringIO() f = StringIO()
with TarFile.open(fileobj=f, mode='w:gz') as tar:
agent_path = os.path.join(settings.AGENT_DIR, "agent-linux")
tar.add(agent_path, arcname='.', filter=exclude)
version_fileobj = StringIO(settings.AGENT_VERSION)
version_info = TarInfo(name='version.txt')
version_info.size = len(version_fileobj.buf)
tar.addfile(version_info, version_fileobj)
return encodestring(f.getvalue()).replace('\n', '')
def create_windows_tar():
f = StringIO()
agent_path = os.path.join(settings.AGENT_DIR, "agent-win")
with TarFile.open(fileobj=f, mode='w|gz') as tar: with TarFile.open(fileobj=f, mode='w|gz') as tar:
tar.add(settings.AGENT_DIR, arcname='.', filter=exclude) tar.add(agent_path, arcname='.')
version_fileobj = StringIO(settings.AGENT_VERSION) version_fileobj = StringIO(settings.AGENT_VERSION)
version_info = TarInfo(name='version.txt') version_info = TarInfo(name='version.txt')
...@@ -82,7 +102,7 @@ def create_agent_tar(): ...@@ -82,7 +102,7 @@ def create_agent_tar():
@celery.task @celery.task
def agent_started(vm, version=None): def agent_started(vm, version=None, system=None):
from vm.models import Instance, instance_activity, InstanceActivity from vm.models import Instance, instance_activity, InstanceActivity
instance = Instance.objects.get(id=int(vm.split('-')[-1])) instance = Instance.objects.get(id=int(vm.split('-')[-1]))
queue = instance.get_remote_queue_name("agent") queue = instance.get_remote_queue_name("agent")
...@@ -105,7 +125,7 @@ def agent_started(vm, version=None): ...@@ -105,7 +125,7 @@ def agent_started(vm, version=None):
if version and version != settings.AGENT_VERSION: if version and version != settings.AGENT_VERSION:
try: try:
update_agent(instance, act) update_agent(instance, act, system, settings.AGENT_VERSION)
except TimeoutError: except TimeoutError:
pass pass
else: else:
...@@ -147,11 +167,16 @@ def measure_boot_time(instance): ...@@ -147,11 +167,16 @@ def measure_boot_time(instance):
@celery.task @celery.task
def agent_stopped(vm): def agent_stopped(vm):
from vm.models import Instance, InstanceActivity from vm.models import Instance, InstanceActivity
from vm.models.activity import ActivityInProgressError
instance = Instance.objects.get(id=int(vm.split('-')[-1])) instance = Instance.objects.get(id=int(vm.split('-')[-1]))
qs = InstanceActivity.objects.filter(instance=instance, qs = InstanceActivity.objects.filter(instance=instance,
activity_code='vm.Instance.agent') activity_code='vm.Instance.agent')
act = qs.latest('id') act = qs.latest('id')
with act.sub_activity('stopping', readable_name=ugettext_noop('stopping')): try:
with act.sub_activity('stopping',
readable_name=ugettext_noop('stopping')):
pass
except ActivityInProgressError:
pass pass
...@@ -162,7 +187,7 @@ def get_network_configs(instance): ...@@ -162,7 +187,7 @@ def get_network_configs(instance):
return (interfaces, settings.FIREWALL_SETTINGS['rdns_ip']) return (interfaces, settings.FIREWALL_SETTINGS['rdns_ip'])
def update_agent(instance, act=None): def update_agent(instance, act=None, system=None, version=None):
if act: if act:
act = act.sub_activity( act = act.sub_activity(
'update', 'update',
...@@ -178,6 +203,40 @@ def update_agent(instance, act=None): ...@@ -178,6 +203,40 @@ def update_agent(instance, act=None):
version=settings.AGENT_VERSION)) version=settings.AGENT_VERSION))
with act: with act:
queue = instance.get_remote_queue_name("agent") queue = instance.get_remote_queue_name("agent")
if system == "Windows":
executable = os.listdir(os.path.join(settings.AGENT_DIR,
"agent-win"))[0]
# executable = "agent-winservice-%(version)s.exe" % {
# 'version': version}
data = create_windows_tar()
elif system == "Linux":
executable = ""
data = create_linux_tar()
else:
executable = ""
# Legacy update method
return update_legacy.apply_async(
queue=queue,
args=(instance.vm_name, create_linux_tar())
).get(timeout=60)
checksum = md5(data).hexdigest()
chunk_size = 1024 * 1024
chunk_number = 0
index = 0
filename = version + ".tar"
while True:
chunk = data[index:index+chunk_size]
if chunk:
append.apply_async(
queue=queue,
args=(instance.vm_name, chunk,
filename, chunk_number)).get(timeout=60)
index = index + chunk_size
chunk_number = chunk_number + 1
else:
update.apply_async( update.apply_async(
queue=queue, queue=queue,
args=(instance.vm_name, create_agent_tar())).get(timeout=10) args=(instance.vm_name, filename, executable, checksum)
).get(timeout=60)
break
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment