Changes and fixes for deployment

3dc93ded · Sulyok Gabor · 97c4dbde · 3dc93ded · 3dc93ded · 3dc93ded
Commit 3dc93ded authored Nov 06, 2016 by Sulyok Gabor
9 changed files
--- a/circle/media/setty/apache.jpg
+++ b/circle/media/setty/apache.jpg
--- a/circle/media/setty/mysql.jpg
+++ b/circle/media/setty/mysql.jpg
--- a/circle/media/setty/wordpress.jpg
+++ b/circle/media/setty/wordpress.jpg
--- a/circle/setty/controller.py
+++ b/circle/setty/controller.py
-from .models import *
+from models import ElementTemplate, Machine, Service, ServiceNode, Element, ElementConnection
 from django.core.exceptions import PermissionDenied
 from django.db.models import Q
 from django.db.models.loading import get_model
 from django.db import transaction
-from saltstackhelper import *
+from saltstackhelper import SaltStackHelper
-import os
 from vm.models import Instance
 import logging
+logger = logging.getLogger(__name__)
+salthelper = SaltStackHelper()
 class SettyController:
    salthelper = SaltStackHelper()
+    '''
+    Save the service data given by the designer. It includes all 
+    Machines, ServiceNodes and connections between them. If the
+    saving fails, the last saved status will not be lost.
+    '''
    @staticmethod
    @transaction.atomic
    def saveService(serviceId, serviceName, serviceNodes, machines, elementConnections):
@@ -18,7 +24,7 @@ class SettyController:
        try:
            service = Service.objects.get(id=serviceId)
        except Service.DoesNotExist:
-            return JsonResponse({'error': 'Service not found'})
+            return {'status': 'error', 'errors': 'SERVICE_NOT_FOUND'}
        service.name = serviceName
        service.save()
@@ -61,16 +67,21 @@ class SettyController:
            connectionObject.save()
-        return {"serviceName": serviceName}
+        return {'status': 'success', 'serviceName': serviceName}
+    '''
+    Load all ServiceNode derivatives, ElementConnections and saved Machines
+    for the Service with the given id and give it back to the caller.
+    If the service doesn't exists, error is returned.
+    '''
    @staticmethod
    def loadService(serviceId):
        service = None
        try:
            service = Service.objects.get(id=serviceId)
        except Service.DoesNotExist:
-            return JsonResponse({'error': 'Service not found'})
+            return {'status': 'error', 'errors': 'SERVICE_NOT_FOUND'}
        machineList = Machine.objects.filter(service=service)
        serviceNodes = []
@@ -95,6 +106,12 @@ class SettyController:
                'serviceNodes': serviceNodes,
                'machines': machines}
+    '''
+    Queiry information -- field names, type of that field -- about 
+    an ElemenTemplate or Machine. If the ElementTemplate doesn't exist
+    or the prototype couldn't be retrieved 
+    '''
    @staticmethod
    def getInformation(elementTemplateId, hostname):
        if elementTemplateId:
@@ -104,19 +121,24 @@ class SettyController:
                model = get_model('setty', elementTemplate.prototype)
                return model.getInformation()
            except ElementTemplate.DoesNotExist:
-                return
+                return {'status': 'error', 'errors': 'ELEMENTTEMPLATE_DOESNT_EXISTS'}
            except LookupError:
-                return
+                return {'status': 'error', 'errors': 'ELEMENTTEMPLATE_COULDNT_GET_PROTOTYPE'}
        elif hostname:
            return Machine.getInformation()
        elif hostname and elementTemplateId:
-            raise PermissionDenied  # TODO: something more meaningful
+            return {'status': 'error', 'errors': 'BOTH_ELEMENTEMPLATE_HOSTNAME_FILLED'}
        else:
-            raise PermissionDenied  # TODO: something more meaningful
+            return {'status': 'error', 'errors': 'UNKNOWN_ERROR'}
+    ''' 
+    Return the available hostnames to add to the Service 
+    based on already used / saved hostanames and Instances 
+    owned by current user which is also known by SaltStack
+    '''
    @staticmethod
    def getMachineAvailableList(serviceId, usedHostnames, current_user):
-        saltMinions = SettyController.salthelper.getAllMinionsUngrouped()
        savedMachines = Machine.objects.filter(service=serviceId)
        savedHostNames = []
@@ -131,27 +153,41 @@ class SettyController:
                userMachines.append(instance.vm_name)
        usedHostnamesByUser = set(savedHostNames + usedHostnames)
+        availableInstances = set(set(userMachines) - usedHostnamesByUser)
+        saltMinions = SettyController.salthelper.getAllMinionsUngrouped()
        if not usedHostnamesByUser:
+            return {'machinedata': [machineName for machineName in userMachines if machineName in saltMinions]}
-            return {'machinedata': [machineName for machineName in userMachines if machineName in saltMinions] }
-        availableInstances = list(set(userMachines) - usedHostnamesByUser)
+        return {'machinedata': [machineName for machineName in availableInstances if machineName in saltMinions]}
-        return {'machinedata': [ machineName for machineName in availableInstances if machineName in saltMinions ]}
+    '''
+    Add a machine with the given hostname to the Service. If there is a 
+    saved Machine with the given hostname or the SaltStack doesn't know about
+    the machine give an error back. If it exists and not already saved give 
+    back the new Machine instance 
+    '''
+     #TODO: addMachine requires usedHostnames too for safety
    @staticmethod
    def addMachine(hostname):
        try:
            Machine.objects.get(hostname=hostname)
-            return {'error': 'already added or doesnt exists'}
        except:
-            pass
+            return {'status': 'error', 'errors': 'MACHINE_ALREADY_ADDED'}
        if SettyController.salthelper.checkMinionExists(hostname):
            machine = Machine.clone()
            machine.hostname = hostname
            return machine.getDataDictionary()
        else:
-            return {'error': 'already added or doesnt exists'}
+            return {'status': 'error', 'errors': 'MACHINE_DOESNT_EXISTS'}
+    '''
+    Add a new service node based on the given ElementTemplate ID to 
+    the current service and return the data member of it.
+    If no ElementTemplate exists with the given ID or it couldn't be
+    institiated give an error.
+    '''
    @staticmethod
    def addServiceNode(elementTemplateId):
@@ -162,12 +198,13 @@ class SettyController:
                model = get_model('setty', elementTemplate.prototype)
                return model.clone().getDataDictionary()
            except ElementTemplate.DoesNotExist:
-                return {'error': "ElementTemplate doesn't exists"}
+                return {'status': 'error', 'errors': 'ELEMENTTEMPLATE_DOESNT_EXISTS'}
            except:
-                return {'error': 'Can not get prototype'}
+                return {'status': 'error', 'errors': 'ELEMENTTEMPLATE_COULDNT_GET_PROTOTYPE'}
        else:
-            return {'error': 'templateid'}
+            return {'status': 'error', 'errors': 'INVALID_ELEMENTTEMPLATE_ID'}
+    ''' Deploy a service using SaltStack. The steps are described inline.'''
    @staticmethod
    def deploy(serviceId):
        service = Service.objects.get(id=serviceId)
@@ -177,7 +214,7 @@ class SettyController:
        nodesToBeDeployed = []
        for serviceNode in serviveNodeList:
            castedServiceNode = serviceNode.cast()
-            nodesToBeDeployed.append( castedServiceNode )
+            nodesToBeDeployed.append(castedServiceNode)
            errorMessage = castedServiceNode.checkDependenciesAndAttributes()
            if errorMessage:
                errorMessages.append(errorMessage)
@@ -186,8 +223,9 @@ class SettyController:
            return {'status': 'error',
                    'errors': errorMessages}
-        # phase one: ask the servicenodes to generate their needed salt commands
+        # phase one: ask the servicenodes to generate their needed salt
+        # commands
        for serviceNode in nodesToBeDeployed:
            serviceNode.generateSaltCommands()
@@ -196,17 +234,19 @@ class SettyController:
        nodesToBeDeployed.sort(reverse=True)
-#        dbgCheck = []
+        dbgCheck = []
 #        for node in nodesToBeDeployed:
 #            commandArray = []
+#
 #            for command in node.generatedCommands:
+#                logger.error( "salt '"+ command.hostname +"' state.sls " + command.command + " pillar=\"" + str(command.parameters) + '"'  )
 #                commandArray.append( command.toDict() )
 #
 #            dbgCheck.append({ "nodeName": str(node.__class__.__name__),
-#                "hostingMachineName": str(node.hostingMachine.hostname),
+#                "hostingMachineName": str(node.getHostingMachine().hostname ),
 #                "commands": commandArray })
-#
-#        return {"status": "error", "errors":dbgCheck}
+        #return {"status": "error", "errors":dbgCheck}
        # phase three: deploy the nodes
        for node in nodesToBeDeployed:
@@ -215,10 +255,9 @@ class SettyController:
            if errorMessages:
                errorMessages.append(deployErrorMessages)
-        # phase four: cleanup generated commands
+        # phase four: cleanup generated commands 
        for serviceNode in nodesToBeDeployed:
            serviceNode.generatedCommands = None
-            serviceNode.hostingMachine = None
        if errorMessages:
            return {'status': 'error',

--- a/circle/setty/models.py
+++ b/circle/setty/models.py
--- a/circle/setty/saltstackhelper.py
+++ b/circle/setty/saltstackhelper.py
@@ -3,55 +3,59 @@ import salt.config
 import salt.runner
 import salt.client
+import logging
+logger = logging.getLogger(__name__)
 class SaltCommand:
    def __init__(self):
        self.hostname = ""
        self.command = ""
-        self.parameters = ""
+        self.parameters = None
    # For debugging purposes only
-    def __str__(self):
-	   return "Command: " + self.hostname + " - " + self.command + " - " + str(self.parameters)
    def toDict(self):
-        return { 'hostname': self.hostname, 'command': self.command, 'parameters': self.parameters }
+        return {'hostname': self.hostname,
+                'command': self.command,
+                'parameters': self.parameters}
 class SaltStackHelper:
    def __init__(self):
        self.master_opts = salt.config.client_config('/etc/salt/master')
        self.salt_runner = salt.runner.RunnerClient(self.master_opts)
        self.salt_localclient = salt.client.LocalClient()
    def getAllMinionsGrouped(self):
-        query_result = self.salt_runner.cmd('manage.status', []);
+        query_result = self.salt_runner.cmd('manage.status', [])
        return query_result
    def getAllMinionsUngrouped(self):
-        query_result = self.salt_runner.cmd('manage.status', []);
+        query_result = self.salt_runner.cmd('manage.status', [])
        return query_result["up"] + query_result["down"]
    def getRunningMinions(self):
-        return self.salt_runner.cmd('manage.up', []);
+        return self.salt_runner.cmd('manage.up', [])
    def getUnavailableMinions(self):
-        return self.salt_runner.cmd('manage.down', []);
+        return self.salt_runner.cmd('manage.down', [])
-    def getMinionBasicHardwareInfo(self, hostname):
-        query_res = self.salt_localclient.cmd( hostname,'grains.items' );
-        if query_res:
-            return {
-                'CpuModel': query_res[hostname]['cpu_model'],
-                'CpuArch': query_res[hostname]['cpuarch'],
-                'TotalMemory': query_res[hostname]['mem_total'],
-                'OSDescription': query_res[hostname]['lsb_distrib_description'] }
-        return query_res
    def checkMinionExists(self, hostname):
-        query_res = self.salt_localclient.cmd( hostname,'network.get_hostname' );
+        query_res = self.salt_localclient.cmd(hostname, 'network.get_hostname')
        return query_res != {}
+    def getIpAddressOfMinion(self, hostname):
+        query_res = self.salt_localclient.cmd(hostname, 'network.ip_addrs')
+        return query_res[hostname][0]
    def executeCommand(self, saltCommands):
-        for saltCommand in saltCommands:
+        for command in saltCommands:
-            self.salt_localclient.cmd(saltCommand.hostname, "state.sls",[saltCommand.command],kwarg={"pillar":saltCommand.parameters} )
+            if command.parameters:
+                self.salt_localclient.cmd(command.hostname, "state.sls",
+                                      [command.command],
+                                      kwarg={"pillar": command.parameters})
+            else:
+                self.salt_localclient.cmd(command.hostname, "state.sls",
+                                      [command.command])
--- a/circle/setty/static/setty/setty.js
+++ b/circle/setty/static/setty/setty.js
@@ -29,6 +29,15 @@ $.ajaxSetup({
    }
 });
+showErrorDialog = function( errors ){
+    $("#errorListContainer").empty();
+    $.each( errors, function(index, message){
+        icon = $("<i/>").addClass("fa").addClass("fa-remove");
+        item = $("<li/>").addClass("list-group-item").addClass("list-group-item-danger").append(icon).append( message );
+        $("#errorListContainer").append( item );
+    });
+    $("#errorDialog").modal('show');
+}
 /* Setty implementation starts here. */
@@ -101,11 +110,12 @@ jsPlumb.ready(function() {
            event: "addMachine",
            data: JSON.stringify({ "hostname": machineHostname } )
        }, function(result) {
-            if(result.error)
+            if(result.errors)
            {
-                alert(result.error)
+                showErrorDialog( result.errors );
                return;
            }
            addMachine( result );
            undoStack.splice(stackIndexer, 0, removeElement);
            redoStack.splice(stackIndexer, 0, addElement);
@@ -793,14 +803,14 @@ jsPlumb.ready(function() {
    $('body').on('click', '#deployService',function() {
        if($("#serviceStatus").text() == "Unsaved") {
-            alert("Only saved services can be deployed");
+            showErrorDialog( ["Only saved services can be deployed"]);
            return;
        }
        $.post("", {
            event: "deploy",
        }, function(result) {
-            if ( result.status ) 
+            if ( result.errors ) 
-                alert( result.errors );
+                showErrorDialog( result.errors );
            else
                alert("Deploying....");
        });

--- a/circle/setty/templates/setty/index.html
+++ b/circle/setty/templates/setty/index.html
@@ -97,6 +97,29 @@
  </div>
 </div>
+<div class="modal fade" id="errorDialog" role="dialog">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <button class="close" type="button" data-dismiss="modal">&times;</button>
+        <h4 class="modal-title"><i class="fa fa-trash-o"></i> {% trans 'Error occured during operation' %}</h4>
+      </div>
+      <div class="modal-body">
+        <p>{% trans 'The following errors occured' %}:</p>
+        <ul id="errorListContainer" class="list-group">
+        </ul>
+      </div>
+      <div class="modal-footer">
+        <div class="row">
+          <div class="col-xs-2">
+            <button class="btn btn-primary btn-md" type="button" data-dismiss="modal">{% trans 'Close' %}</button>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
 <!-- Adding element dialog -->
 <div class="modal fade" id="addElementDialog" role="dialog">

--- a/circle/setty/views.py
+++ b/circle/setty/views.py
@@ -18,14 +18,13 @@
 from django.contrib import messages
 from django.core.exceptions import PermissionDenied
 from django.core.urlresolvers import reverse, reverse_lazy
-from django.db.models import Q
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect
 from braces.views import LoginRequiredMixin
 from django.views.generic import TemplateView, DeleteView
 from django_tables2 import SingleTableView
-from saltstackhelper import *
+from controller import SettyController
-from controller import *
+from models import Service, ElementTemplate
 from dashboard.views.util import FilterMixin
 from django.utils.translation import ugettext as _
 import json
@@ -86,7 +85,7 @@ class DetailView(LoginRequiredMixin, TemplateView):
                                                     'serviceNodes'], data['machines'], data['elementConnections'])
            elif eventName == "getMachineAvailableList":
                result = SettyController.getMachineAvailableList(
-                    serviceId, data["usedHostnames"], self.request.user )
+                    serviceId, data["usedHostnames"], self.request.user)
            elif eventName == "addServiceNode":
                result = SettyController.addServiceNode(
                    data["elementTemplateId"])
@@ -103,7 +102,7 @@ class DetailView(LoginRequiredMixin, TemplateView):
                result = SettyController.getInformation(
                    templateId, hostname)
        return JsonResponse(result)
@@ -151,7 +150,7 @@ class CreateView(LoginRequiredMixin, TemplateView):
            service_name = "Noname"
        try:
-            serviceNameAvailable = Service.objects.get(name=service_name)
+            Service.objects.get(name=service_name)
            raise PermissionDenied
        except Service.DoesNotExist:
            pass