Skip to content

CIRCLE / cloud

Go to a project
Commit 97c4dbde by Sulyok Gabor
Options

SaltCommand generation and deployement fixes

parent 5e8e9dca
Pipeline #260 failed with stage
in 0 seconds
Showing with 177 additions and 129 deletions
circle/setty/controller.py
......@@ -171,12 +171,14 @@ class SettyController:
@staticmethod
def deploy(serviceId):
service = Service.objects.get(id=serviceId)
machines = Machine.objects.filter(service=service)
serviveNodeList = ServiceNode.objects.filter(service=service)
errorMessages = []
nodesToBeDeployed = []
for serviceNode in serviveNodeList:
errorMessage = serviceNode.cast().checkDependenciesAndAttributes()
castedServiceNode = serviceNode.cast()
nodesToBeDeployed.append( castedServiceNode )
errorMessage = castedServiceNode.checkDependenciesAndAttributes()
if errorMessage:
errorMessages.append(errorMessage)
......@@ -184,52 +186,39 @@ class SettyController:
return {'status': 'error',
'errors': errorMessages}
elementConnections = ElementConnection.objects.filter(
Q(target__in=machines) | Q(source__in=machines))
# phase one: set the machine ptr in serviceNodes which can be accessed by
# connections from machines
logger = logging.getLogger('project.interesting.stuff')
for machine in machines:
for connection in elementConnections:
serviceNode = None
if connection.target.cast() == machine:
serviceNode = connection.source.cast()
serviceNode.setMachineForDeploy(machine)
elif connection.source.cast() == machine:
serviceNode = connection.target.cast()
serviceNode.setMachineForDeploy(machine)
# phase two: let the nodes create configurations recursively
configuratedNodes = list()
for serviceNode in serviveNodeList:
node = serviceNode.cast()
node.generateSaltCommands()
configuratedNodes.append( node )
# phase one: ask the servicenodes to generate their needed salt commands
for serviceNode in nodesToBeDeployed:
serviceNode.generateSaltCommands()
# phase three: sort the nodes by deployment priority(lower the prio,
# phase two: sort the nodes by deployment priority(lower the prio,
# later in the deployement)
configuratedNodes.sort(reverse=True)
nodesToBeDeployed.sort(reverse=True)
# dbgCheck = []
# for node in configuratedNodes:
# commandDict = []
# for node in nodesToBeDeployed:
# commandArray = []
# for command in node.generatedCommands:
# commandDict.append( command.__dict__ )
# dbgCheck.append({ "nodeName": my_instance.__class__.__name__,
# "commands": commandDict })
# return dbgCheck
# phase four: deploy the nodes
for node in configuratedNodes:
# commandArray.append( command.toDict() )
#
# dbgCheck.append({ "nodeName": str(node.__class__.__name__),
# "hostingMachineName": str(node.hostingMachine.hostname),
# "commands": commandArray })
#
# return {"status": "error", "errors":dbgCheck}
# phase three: deploy the nodes
for node in nodesToBeDeployed:
deployErrorMessages = SettyController.salthelper.executeCommand(
node.generatedCommands)
if errorMessages:
errorMessages.append(deployErrorMessages)
# phase five: cleanup generated commands
for serviceNode in firstLevelServiceNodes:
# phase four: cleanup generated commands
for serviceNode in nodesToBeDeployed:
serviceNode.generatedCommands = None
serviceNode.hostingMachine = None
if errorMessages:
return {'status': 'error',
......
circle/setty/models.py
......@@ -17,6 +17,7 @@
from django.db import models
from django.db.models import Model, Q
from django.db.models.signals import post_init
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User
from taggit.managers import TaggableManager
......@@ -29,9 +30,11 @@ from saltstackhelper import SaltCommand
SALTSTACK_PILLAR_FOLDER = "/srv/pillar"
# Replacer method for configuration generation
def replaceParameter(config, parameterToReplace, newValue):
configEdited = config.replace(parameterToReplace, str(newValue))
return configEdited
def replaceParameter(pillar, parameterToReplace, newValue):
pillarEdited = pillar.replace(parameterToReplace, str(newValue))
return pillarEdited
class Service(models.Model):
......@@ -139,7 +142,6 @@ class ElementConnection(models.Model):
# Represents an CIRCLE VM Instance which is known by Salt-Master and used
# in Setty configuration
class Machine(Element):
MACHINE_STATUS_CHOICES = (
(1, 'Running'),
......@@ -189,8 +191,11 @@ class ServiceNode(Element):
config_file = models.FileField(
default=None, upload_to='setty/node_configs/', storage=OverwriteStorage())
description = models.TextField(default="")
machine = None # for deploying
generatedCommands = []
def __init__(self, *args, **kwargs):
super(ServiceNode, self).__init__(*args, **kwargs)
self.hostingMachine = None
self.generatedCommands = []
def __unicode__(self):
return "%s" % self.name
......@@ -221,12 +226,12 @@ class ServiceNode(Element):
def checkDependenciesAndAttributes(self):
return []
# functions for deployement
def checkDependecy(self, ObjOther):
elementConnections = ElementConnection.objects.filter(
Q(target=self) | Q(source=self))
for connection in elementConnections:
serviceNode = None
if connection.target.cast() == self:
if isinstance(connection.source.cast(), ObjOther):
return connection.source.cast()
......@@ -236,11 +241,27 @@ class ServiceNode(Element):
return None
def __cmp__(self, other):
if not isinstance( other, ServiceNode ):
raise PermissionDenied
return self.getDeploymentPriority(self).__cmp__(other.getDeploymentPriority(other))
# functions for deployement
def setMachineForDeploy(self, machine):
self.machine = machine
def getHostingMachine(self):
if self.hostingMachine:
return self.hostingMachine
elementConnections = ElementConnection.objects.filter(
Q(target=self) | Q(source=self))
for connection in elementConnections:
if isinstance(connection.target.cast(), Machine):
self.hostingMachine = connection.target.cast()
return self.hostingMachine
if isinstance(connection.source.cast(), Machine):
self.hostingMachine = connection.source.cast()
return self.hostingMachine
raise PermissionDenied
def getDeploymentPriority(self):
return 0
......@@ -251,7 +272,6 @@ class ServiceNode(Element):
def replacePillarParameters(self, pillar):
raise PermissionDenied
class WordpressNode(ServiceNode):
# DB related fields
databaseName = models.TextField(default="")
......@@ -345,52 +365,75 @@ class WordpressNode(ServiceNode):
return errorMessages
def getHostingMachine(self):
if self.hostingMachine:
return hostingMachine
apacheNode = self.checkDependecy(ApacheNode)
if not apacheNode:
raise PermissionDenied
self.hostingMachine = apacheNode.getHostingMachine()
if not self.hostingMachine:
raise PermissionDenied
return self.hostingMachine
@staticmethod
def getDeploymentPriority(self):
return 10
return 1
def generateConfiguration(self, config=""):
config = replaceParameter(
config, r'%%DATABASE_NAME%%', self.databaseName)
config = replaceParameter(
config, r'%%DATABASE_HOST%%', self.databaseHost)
config = replaceParameter(
config, r'%%DATABASE_USER%%', self.databaseUser)
config = replaceParameter(
config, r'%%DATABASE_PASS%%', self.databasePass)
config = replaceParameter(
config, r'%%ADMIN_USERNAME%%', self.adminUsername)
config = replaceParameter(
config, r'%%ADMIN_PASSWORD%%', self.adminPassword)
config = replaceParameter(config, r'%%ADMIN_EMAIL%%', self.adminEmail)
config = replaceParameter(config, r'%%SITE_TITLE%%', self.siteTitle)
config = replaceParameter(config, r'%%SITE_URL%%', self.siteUrl)
return config
def replacePillarParameters(self, pillar):
pillar = replaceParameter(
pillar, r'%%DATABASE_NAME%%', self.databaseName)
pillar = replaceParameter(
pillar, r'%%DATABASE_HOST%%', self.databaseHost)
pillar = replaceParameter(
pillar, r'%%DATABASE_USER%%', self.databaseUser)
pillar = replaceParameter(
pillar, r'%%DATABASE_PASS%%', self.databasePass)
pillar = replaceParameter(
pillar, r'%%ADMIN_USERNAME%%', self.adminUsername)
pillar = replaceParameter(
pillar, r'%%ADMIN_PASSWORD%%', self.adminPassword)
pillar = replaceParameter(pillar, r'%%ADMIN_EMAIL%%', self.adminEmail)
pillar = replaceParameter(pillar, r'%%SITE_TITLE%%', self.siteTitle)
pillar = replaceParameter(pillar, r'%%SITE_URL%%', self.siteUrl)
return pillar
def generateSaltCommands(self):
mysqlNode = self.checkDependecy(MySQLNode)
apacheNode = self.checkDependecy(ApacheNode)
if not mysqlNode:
raise PermissionDenied
if not apacheNode:
raise PermissionDenied
self.hostingMachine = apacheNode.getHostingMachine()
createMySQLUserCommand = mysqlNode.makeCreateDatabaseCommand(
self.databaseName)
createMySQLDatabaseCommand = mysqlNode.makeCreateUserCommand(
self.databaseUser, self.databasePass, self.databaseName)
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "wordpress.sls")
with open(pillarFilePath, 'r') as pillar:
mysqlNode = self.checkDependecy(MySQLNode)
apacheNode = self.checkDependecy(ApacheNode)
if not mysqlNode:
raise PermissionDenied
if not apacheNode:
raise PermissionDenied
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = self.replacePillarParameters(pillar)
self.machine = apacheNode.machine
saltCommand = SaltCommand()
saltCommand.hostname = self.hostingMachine.hostname
saltCommand.command = "wordpress"
saltCommand.parameters = [eval(pillar)]
createMySQLUserCommand = mysqlNode.makeCreateDatabaseCommand(
self.databaseName)
createMySQLUserCommand = mysqlNode.makeCreateUserCommand(
self.databaseUser, self.databasePass)
config = str(yaml.load(pillar))
config = replacePillarParameters(pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="wordpress", parameters=[eval(config)])
self.generatedCommands = [createMySQLUserCommand, saltCommand]
self.generatedCommands = []
self.generatedCommands.append(createMySQLDatabaseCommand)
self.generatedCommands.append(createMySQLUserCommand)
self.generatedCommands.append(saltCommand)
class WebServerNode(ServiceNode):
......@@ -434,17 +477,11 @@ class WebServerNode(ServiceNode):
def getDeploymentPriority(self):
return 10
def generateConfiguration(self, config=""):
config = replaceParameter(config, r"%%USE_SSL%%", self.useSSL)
config = replaceParameter(config,
r"%%LISTENING_PORT%%", self.listeningPort)
return config
def replacePillarParameters(self, pillar):
config = replaceParameter(config, r"%%USE_SSL%%", self.useSSL)
config = replaceParameter(config,
pillar = replaceParameter(pillar, r"%%USE_SSL%%", self.useSSL)
pillar = replaceParameter(pillar,
r"%%LISTENING_PORT%%", self.listeningPort)
return config
return pillar
class ApacheNode(WebServerNode):
......@@ -456,12 +493,17 @@ class ApacheNode(WebServerNode):
def generateSaltCommands(self):
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "apache.sls")
with open(pillarFilePath, 'r') as pillar:
config = str(yaml.load(pillar))
config = WebServerNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="apache", parameters=eval(config))
self.generatedCommands = [saltCommand]
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = WebServerNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand()
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "apache"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class NginxNode(WebServerNode):
......@@ -499,16 +541,19 @@ class NginxNode(WebServerNode):
def generateSaltCommands(self):
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls")
with open(pillarFilePath, 'r') as pillar:
config = str(yaml.load(pillar))
config = WebServerNode.replacePillarParameters(self, pillar)
config = replaceParameter(config,
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = WebServerNode.replacePillarParameters(self, pillar)
pillar = replaceParameter(pillar,
r"%%WORKER_CONNECTIONS%%", self.worker_connections)
saltCommand = SaltCommand(
hostname=machine.hostname, command="nginx", parameters=eval(config))
saltCommand = SaltCommand()
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "nginx"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = [saltCommand]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class DatabaseNode(ServiceNode):
......@@ -576,13 +621,18 @@ class PostgreSQLNode(DatabaseNode):
def generateSaltCommands(self):
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls")
with open(pillarFilePath, 'r') as pillar:
config = str(yaml.load(pillar))
config = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="postgresql", parameters=eval(config))
self.generatedCommands = [saltCommand]
SALTSTACK_PILLAR_FOLDER, "postgresql.sls")
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand()
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "postgresql"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class MySQLNode(DatabaseNode):
......@@ -593,23 +643,30 @@ class MySQLNode(DatabaseNode):
def makeCreateDatabaseCommand(self, databaseName):
saltCommand = SaltCommand()
saltCommand.hostname = self.machine.hostname
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "mysql.database"
saltCommand.parameters = [databaseName]
return saltCommand
def makeCreateUserCommand(self, databaseUser, databasePass, availableDatabases):
saltCommand = SaltCommand()
saltCommand.hostname = self.machine.hostname
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "mysql.user"
saltCommand.parameters = {databaseUser: {'password': databasePass, 'host': 'localhost', 'databases': [
{'database': availableDatabases, 'grants': ['all privileges']}]}}
return saltCommand
def generateSaltCommands(self):
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls")
with open(pillarFilePath, 'r') as pillar:
config = str(yaml.load(pillar))
config = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="mysql.server", parameters=eval(config))
self.generatedCommands = [saltCommand]
\ No newline at end of file
SALTSTACK_PILLAR_FOLDER, "mysql.sls")
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand()
saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "mysql.server"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
circle/setty/saltstackhelper.py
......@@ -13,6 +13,10 @@ class SaltCommand:
def __str__(self):
return "Command: " + self.hostname + " - " + self.command + " - " + str(self.parameters)
def toDict(self):
return { 'hostname': self.hostname, 'command': self.command, 'parameters': self.parameters }
class SaltStackHelper:
def __init__(self):
self.master_opts = salt.config.client_config('/etc/salt/master')
......@@ -48,5 +52,6 @@ class SaltStackHelper:
query_res = self.salt_localclient.cmd( hostname,'network.get_hostname' );
return query_res != {}
def executeCommand(self, saltCommand):
return self.salt_localclient.cmd(saltCommand.hostname, "state.sls",[saltCommand.command],kwarg={"pillar":saltCommand.parameters} )
def executeCommand(self, saltCommands):
for saltCommand in saltCommands:
self.salt_localclient.cmd(saltCommand.hostname, "state.sls",[saltCommand.command],kwarg={"pillar":saltCommand.parameters} )
circle/setty/views.py
......@@ -103,9 +103,7 @@ class DetailView(LoginRequiredMixin, TemplateView):
result = SettyController.getInformation(
templateId, hostname)
print '------------'
print result
print '------------'
return JsonResponse(result)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment