Skip to content

CIRCLE / cloud

Go to a project
Commit 97c4dbde by Sulyok Gabor
Options

SaltCommand generation and deployement fixes

parent 5e8e9dca
Pipeline #260 failed with stage
in 0 seconds
Showing with 170 additions and 122 deletions
circle/setty/controller.py
...@@ -171,12 +171,14 @@ class SettyController: ...@@ -171,12 +171,14 @@ class SettyController:
@staticmethod @staticmethod
def deploy(serviceId): def deploy(serviceId):
service = Service.objects.get(id=serviceId) service = Service.objects.get(id=serviceId)
machines = Machine.objects.filter(service=service)
serviveNodeList = ServiceNode.objects.filter(service=service) serviveNodeList = ServiceNode.objects.filter(service=service)
errorMessages = [] errorMessages = []
nodesToBeDeployed = []
for serviceNode in serviveNodeList: for serviceNode in serviveNodeList:
errorMessage = serviceNode.cast().checkDependenciesAndAttributes() castedServiceNode = serviceNode.cast()
nodesToBeDeployed.append( castedServiceNode )
errorMessage = castedServiceNode.checkDependenciesAndAttributes()
if errorMessage: if errorMessage:
errorMessages.append(errorMessage) errorMessages.append(errorMessage)
...@@ -184,52 +186,39 @@ class SettyController: ...@@ -184,52 +186,39 @@ class SettyController:
return {'status': 'error', return {'status': 'error',
'errors': errorMessages} 'errors': errorMessages}
elementConnections = ElementConnection.objects.filter( # phase one: ask the servicenodes to generate their needed salt commands
Q(target__in=machines) | Q(source__in=machines))
for serviceNode in nodesToBeDeployed:
# phase one: set the machine ptr in serviceNodes which can be accessed by serviceNode.generateSaltCommands()
# connections from machines
logger = logging.getLogger('project.interesting.stuff')
for machine in machines:
for connection in elementConnections:
serviceNode = None
if connection.target.cast() == machine:
serviceNode = connection.source.cast()
serviceNode.setMachineForDeploy(machine)
elif connection.source.cast() == machine:
serviceNode = connection.target.cast()
serviceNode.setMachineForDeploy(machine)
# phase two: let the nodes create configurations recursively
configuratedNodes = list()
for serviceNode in serviveNodeList:
node = serviceNode.cast()
node.generateSaltCommands()
configuratedNodes.append( node )
# phase three: sort the nodes by deployment priority(lower the prio, # phase two: sort the nodes by deployment priority(lower the prio,
# later in the deployement) # later in the deployement)
configuratedNodes.sort(reverse=True) nodesToBeDeployed.sort(reverse=True)
# dbgCheck = [] # dbgCheck = []
# for node in configuratedNodes: # for node in nodesToBeDeployed:
# commandDict = [] # commandArray = []
# for command in node.generatedCommands: # for command in node.generatedCommands:
# commandDict.append( command.__dict__ ) # commandArray.append( command.toDict() )
# dbgCheck.append({ "nodeName": my_instance.__class__.__name__, #
# "commands": commandDict }) # dbgCheck.append({ "nodeName": str(node.__class__.__name__),
# return dbgCheck # "hostingMachineName": str(node.hostingMachine.hostname),
# phase four: deploy the nodes # "commands": commandArray })
for node in configuratedNodes: #
# return {"status": "error", "errors":dbgCheck}
# phase three: deploy the nodes
for node in nodesToBeDeployed:
deployErrorMessages = SettyController.salthelper.executeCommand( deployErrorMessages = SettyController.salthelper.executeCommand(
node.generatedCommands) node.generatedCommands)
if errorMessages: if errorMessages:
errorMessages.append(deployErrorMessages) errorMessages.append(deployErrorMessages)
# phase five: cleanup generated commands # phase four: cleanup generated commands
for serviceNode in firstLevelServiceNodes: for serviceNode in nodesToBeDeployed:
serviceNode.generatedCommands = None serviceNode.generatedCommands = None
serviceNode.hostingMachine = None
if errorMessages: if errorMessages:
return {'status': 'error', return {'status': 'error',
......
circle/setty/models.py
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
from django.db import models from django.db import models
from django.db.models import Model, Q from django.db.models import Model, Q
from django.db.models.signals import post_init
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User from django.contrib.auth.models import User
from taggit.managers import TaggableManager from taggit.managers import TaggableManager
...@@ -29,9 +30,11 @@ from saltstackhelper import SaltCommand ...@@ -29,9 +30,11 @@ from saltstackhelper import SaltCommand
SALTSTACK_PILLAR_FOLDER = "/srv/pillar" SALTSTACK_PILLAR_FOLDER = "/srv/pillar"
# Replacer method for configuration generation # Replacer method for configuration generation
def replaceParameter(config, parameterToReplace, newValue):
configEdited = config.replace(parameterToReplace, str(newValue))
return configEdited def replaceParameter(pillar, parameterToReplace, newValue):
pillarEdited = pillar.replace(parameterToReplace, str(newValue))
return pillarEdited
class Service(models.Model): class Service(models.Model):
...@@ -139,7 +142,6 @@ class ElementConnection(models.Model): ...@@ -139,7 +142,6 @@ class ElementConnection(models.Model):
# Represents an CIRCLE VM Instance which is known by Salt-Master and used # Represents an CIRCLE VM Instance which is known by Salt-Master and used
# in Setty configuration # in Setty configuration
class Machine(Element): class Machine(Element):
MACHINE_STATUS_CHOICES = ( MACHINE_STATUS_CHOICES = (
(1, 'Running'), (1, 'Running'),
...@@ -189,8 +191,11 @@ class ServiceNode(Element): ...@@ -189,8 +191,11 @@ class ServiceNode(Element):
config_file = models.FileField( config_file = models.FileField(
default=None, upload_to='setty/node_configs/', storage=OverwriteStorage()) default=None, upload_to='setty/node_configs/', storage=OverwriteStorage())
description = models.TextField(default="") description = models.TextField(default="")
machine = None # for deploying
generatedCommands = [] def __init__(self, *args, **kwargs):
super(ServiceNode, self).__init__(*args, **kwargs)
self.hostingMachine = None
self.generatedCommands = []
def __unicode__(self): def __unicode__(self):
return "%s" % self.name return "%s" % self.name
...@@ -221,12 +226,12 @@ class ServiceNode(Element): ...@@ -221,12 +226,12 @@ class ServiceNode(Element):
def checkDependenciesAndAttributes(self): def checkDependenciesAndAttributes(self):
return [] return []
# functions for deployement
def checkDependecy(self, ObjOther): def checkDependecy(self, ObjOther):
elementConnections = ElementConnection.objects.filter( elementConnections = ElementConnection.objects.filter(
Q(target=self) | Q(source=self)) Q(target=self) | Q(source=self))
for connection in elementConnections: for connection in elementConnections:
serviceNode = None
if connection.target.cast() == self: if connection.target.cast() == self:
if isinstance(connection.source.cast(), ObjOther): if isinstance(connection.source.cast(), ObjOther):
return connection.source.cast() return connection.source.cast()
...@@ -236,11 +241,27 @@ class ServiceNode(Element): ...@@ -236,11 +241,27 @@ class ServiceNode(Element):
return None return None
def __cmp__(self, other): def __cmp__(self, other):
if not isinstance( other, ServiceNode ):
raise PermissionDenied
return self.getDeploymentPriority(self).__cmp__(other.getDeploymentPriority(other)) return self.getDeploymentPriority(self).__cmp__(other.getDeploymentPriority(other))
# functions for deployement def getHostingMachine(self):
def setMachineForDeploy(self, machine): if self.hostingMachine:
self.machine = machine return self.hostingMachine
elementConnections = ElementConnection.objects.filter(
Q(target=self) | Q(source=self))
for connection in elementConnections:
if isinstance(connection.target.cast(), Machine):
self.hostingMachine = connection.target.cast()
return self.hostingMachine
if isinstance(connection.source.cast(), Machine):
self.hostingMachine = connection.source.cast()
return self.hostingMachine
raise PermissionDenied
def getDeploymentPriority(self): def getDeploymentPriority(self):
return 0 return 0
...@@ -251,7 +272,6 @@ class ServiceNode(Element): ...@@ -251,7 +272,6 @@ class ServiceNode(Element):
def replacePillarParameters(self, pillar): def replacePillarParameters(self, pillar):
raise PermissionDenied raise PermissionDenied
class WordpressNode(ServiceNode): class WordpressNode(ServiceNode):
# DB related fields # DB related fields
databaseName = models.TextField(default="") databaseName = models.TextField(default="")
...@@ -345,52 +365,75 @@ class WordpressNode(ServiceNode): ...@@ -345,52 +365,75 @@ class WordpressNode(ServiceNode):
return errorMessages return errorMessages
def getHostingMachine(self):
if self.hostingMachine:
return hostingMachine
apacheNode = self.checkDependecy(ApacheNode)
if not apacheNode:
raise PermissionDenied
self.hostingMachine = apacheNode.getHostingMachine()
if not self.hostingMachine:
raise PermissionDenied
return self.hostingMachine
@staticmethod @staticmethod
def getDeploymentPriority(self): def getDeploymentPriority(self):
return 10 return 1
def generateConfiguration(self, config=""): def replacePillarParameters(self, pillar):
config = replaceParameter( pillar = replaceParameter(
config, r'%%DATABASE_NAME%%', self.databaseName) pillar, r'%%DATABASE_NAME%%', self.databaseName)
config = replaceParameter( pillar = replaceParameter(
config, r'%%DATABASE_HOST%%', self.databaseHost) pillar, r'%%DATABASE_HOST%%', self.databaseHost)
config = replaceParameter( pillar = replaceParameter(
config, r'%%DATABASE_USER%%', self.databaseUser) pillar, r'%%DATABASE_USER%%', self.databaseUser)
config = replaceParameter( pillar = replaceParameter(
config, r'%%DATABASE_PASS%%', self.databasePass) pillar, r'%%DATABASE_PASS%%', self.databasePass)
config = replaceParameter( pillar = replaceParameter(
config, r'%%ADMIN_USERNAME%%', self.adminUsername) pillar, r'%%ADMIN_USERNAME%%', self.adminUsername)
config = replaceParameter( pillar = replaceParameter(
config, r'%%ADMIN_PASSWORD%%', self.adminPassword) pillar, r'%%ADMIN_PASSWORD%%', self.adminPassword)
config = replaceParameter(config, r'%%ADMIN_EMAIL%%', self.adminEmail)
config = replaceParameter(config, r'%%SITE_TITLE%%', self.siteTitle) pillar = replaceParameter(pillar, r'%%ADMIN_EMAIL%%', self.adminEmail)
config = replaceParameter(config, r'%%SITE_URL%%', self.siteUrl) pillar = replaceParameter(pillar, r'%%SITE_TITLE%%', self.siteTitle)
pillar = replaceParameter(pillar, r'%%SITE_URL%%', self.siteUrl)
return config
return pillar
def generateSaltCommands(self): def generateSaltCommands(self):
pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "wordpress.sls")
with open(pillarFilePath, 'r') as pillar:
mysqlNode = self.checkDependecy(MySQLNode) mysqlNode = self.checkDependecy(MySQLNode)
apacheNode = self.checkDependecy(ApacheNode) apacheNode = self.checkDependecy(ApacheNode)
if not mysqlNode: if not mysqlNode:
raise PermissionDenied raise PermissionDenied
if not apacheNode: if not apacheNode:
raise PermissionDenied raise PermissionDenied
self.machine = apacheNode.machine self.hostingMachine = apacheNode.getHostingMachine()
createMySQLUserCommand = mysqlNode.makeCreateDatabaseCommand( createMySQLUserCommand = mysqlNode.makeCreateDatabaseCommand(
self.databaseName) self.databaseName)
createMySQLUserCommand = mysqlNode.makeCreateUserCommand( createMySQLDatabaseCommand = mysqlNode.makeCreateUserCommand(
self.databaseUser, self.databasePass) self.databaseUser, self.databasePass, self.databaseName)
config = str(yaml.load(pillar))
config = replacePillarParameters(pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="wordpress", parameters=[eval(config)])
self.generatedCommands = [createMySQLUserCommand, saltCommand] pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "wordpress.sls")
with open(pillarFilePath, 'r') as pillarFile:
pillar = str(yaml.load(pillarFile))
pillar = self.replacePillarParameters(pillar)
saltCommand = SaltCommand()
saltCommand.hostname = self.hostingMachine.hostname
saltCommand.command = "wordpress"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(createMySQLDatabaseCommand)
self.generatedCommands.append(createMySQLUserCommand)
self.generatedCommands.append(saltCommand)
class WebServerNode(ServiceNode): class WebServerNode(ServiceNode):
...@@ -434,17 +477,11 @@ class WebServerNode(ServiceNode): ...@@ -434,17 +477,11 @@ class WebServerNode(ServiceNode):
def getDeploymentPriority(self): def getDeploymentPriority(self):
return 10 return 10
def generateConfiguration(self, config=""):
config = replaceParameter(config, r"%%USE_SSL%%", self.useSSL)
config = replaceParameter(config,
r"%%LISTENING_PORT%%", self.listeningPort)
return config
def replacePillarParameters(self, pillar): def replacePillarParameters(self, pillar):
config = replaceParameter(config, r"%%USE_SSL%%", self.useSSL) pillar = replaceParameter(pillar, r"%%USE_SSL%%", self.useSSL)
config = replaceParameter(config, pillar = replaceParameter(pillar,
r"%%LISTENING_PORT%%", self.listeningPort) r"%%LISTENING_PORT%%", self.listeningPort)
return config return pillar
class ApacheNode(WebServerNode): class ApacheNode(WebServerNode):
...@@ -456,12 +493,17 @@ class ApacheNode(WebServerNode): ...@@ -456,12 +493,17 @@ class ApacheNode(WebServerNode):
def generateSaltCommands(self): def generateSaltCommands(self):
pillarFilePath = os.path.join( pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "apache.sls") SALTSTACK_PILLAR_FOLDER, "apache.sls")
with open(pillarFilePath, 'r') as pillar: with open(pillarFilePath, 'r') as pillarFile:
config = str(yaml.load(pillar)) pillar = str(yaml.load(pillarFile))
config = WebServerNode.replacePillarParameters(self, pillar) pillar = WebServerNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="apache", parameters=eval(config)) saltCommand = SaltCommand()
self.generatedCommands = [saltCommand] saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "apache"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class NginxNode(WebServerNode): class NginxNode(WebServerNode):
...@@ -499,16 +541,19 @@ class NginxNode(WebServerNode): ...@@ -499,16 +541,19 @@ class NginxNode(WebServerNode):
def generateSaltCommands(self): def generateSaltCommands(self):
pillarFilePath = os.path.join( pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls") SALTSTACK_PILLAR_FOLDER, "nginx.sls")
with open(pillarFilePath, 'r') as pillar: with open(pillarFilePath, 'r') as pillarFile:
config = str(yaml.load(pillar)) pillar = str(yaml.load(pillarFile))
config = WebServerNode.replacePillarParameters(self, pillar) pillar = WebServerNode.replacePillarParameters(self, pillar)
config = replaceParameter(config, pillar = replaceParameter(pillar,
r"%%WORKER_CONNECTIONS%%", self.worker_connections) r"%%WORKER_CONNECTIONS%%", self.worker_connections)
saltCommand = SaltCommand( saltCommand = SaltCommand()
hostname=machine.hostname, command="nginx", parameters=eval(config)) saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "nginx"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = [saltCommand] self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class DatabaseNode(ServiceNode): class DatabaseNode(ServiceNode):
...@@ -576,13 +621,18 @@ class PostgreSQLNode(DatabaseNode): ...@@ -576,13 +621,18 @@ class PostgreSQLNode(DatabaseNode):
def generateSaltCommands(self): def generateSaltCommands(self):
pillarFilePath = os.path.join( pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls") SALTSTACK_PILLAR_FOLDER, "postgresql.sls")
with open(pillarFilePath, 'r') as pillar: with open(pillarFilePath, 'r') as pillarFile:
config = str(yaml.load(pillar)) pillar = str(yaml.load(pillarFile))
config = DatabaseNode.replacePillarParameters(self, pillar) pillar = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="postgresql", parameters=eval(config)) saltCommand = SaltCommand()
self.generatedCommands = [saltCommand] saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "postgresql"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
class MySQLNode(DatabaseNode): class MySQLNode(DatabaseNode):
...@@ -593,23 +643,30 @@ class MySQLNode(DatabaseNode): ...@@ -593,23 +643,30 @@ class MySQLNode(DatabaseNode):
def makeCreateDatabaseCommand(self, databaseName): def makeCreateDatabaseCommand(self, databaseName):
saltCommand = SaltCommand() saltCommand = SaltCommand()
saltCommand.hostname = self.machine.hostname saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "mysql.database" saltCommand.command = "mysql.database"
saltCommand.parameters = [databaseName] saltCommand.parameters = [databaseName]
return saltCommand
def makeCreateUserCommand(self, databaseUser, databasePass, availableDatabases): def makeCreateUserCommand(self, databaseUser, databasePass, availableDatabases):
saltCommand = SaltCommand() saltCommand = SaltCommand()
saltCommand.hostname = self.machine.hostname saltCommand.hostname = self.getHostingMachine().hostname
saltCommand.command = "mysql.user" saltCommand.command = "mysql.user"
saltCommand.parameters = {databaseUser: {'password': databasePass, 'host': 'localhost', 'databases': [ saltCommand.parameters = {databaseUser: {'password': databasePass, 'host': 'localhost', 'databases': [
{'database': availableDatabases, 'grants': ['all privileges']}]}} {'database': availableDatabases, 'grants': ['all privileges']}]}}
return saltCommand
def generateSaltCommands(self): def generateSaltCommands(self):
pillarFilePath = os.path.join( pillarFilePath = os.path.join(
SALTSTACK_PILLAR_FOLDER, "nginx.sls") SALTSTACK_PILLAR_FOLDER, "mysql.sls")
with open(pillarFilePath, 'r') as pillar: with open(pillarFilePath, 'r') as pillarFile:
config = str(yaml.load(pillar)) pillar = str(yaml.load(pillarFile))
config = DatabaseNode.replacePillarParameters(self, pillar) pillar = DatabaseNode.replacePillarParameters(self, pillar)
saltCommand = SaltCommand(
hostname=machine.hostname, command="mysql.server", parameters=eval(config)) saltCommand = SaltCommand()
self.generatedCommands = [saltCommand] saltCommand.hostname = self.getHostingMachine().hostname
\ No newline at end of file saltCommand.command = "mysql.server"
saltCommand.parameters = [eval(pillar)]
self.generatedCommands = []
self.generatedCommands.append(saltCommand)
circle/setty/saltstackhelper.py
...@@ -13,6 +13,10 @@ class SaltCommand: ...@@ -13,6 +13,10 @@ class SaltCommand:
def __str__(self): def __str__(self):
return "Command: " + self.hostname + " - " + self.command + " - " + str(self.parameters) return "Command: " + self.hostname + " - " + self.command + " - " + str(self.parameters)
def toDict(self):
return { 'hostname': self.hostname, 'command': self.command, 'parameters': self.parameters }
class SaltStackHelper: class SaltStackHelper:
def __init__(self): def __init__(self):
self.master_opts = salt.config.client_config('/etc/salt/master') self.master_opts = salt.config.client_config('/etc/salt/master')
...@@ -48,5 +52,6 @@ class SaltStackHelper: ...@@ -48,5 +52,6 @@ class SaltStackHelper:
query_res = self.salt_localclient.cmd( hostname,'network.get_hostname' ); query_res = self.salt_localclient.cmd( hostname,'network.get_hostname' );
return query_res != {} return query_res != {}
def executeCommand(self, saltCommand): def executeCommand(self, saltCommands):
return self.salt_localclient.cmd(saltCommand.hostname, "state.sls",[saltCommand.command],kwarg={"pillar":saltCommand.parameters} ) for saltCommand in saltCommands:
self.salt_localclient.cmd(saltCommand.hostname, "state.sls",[saltCommand.command],kwarg={"pillar":saltCommand.parameters} )
circle/setty/views.py
...@@ -103,9 +103,7 @@ class DetailView(LoginRequiredMixin, TemplateView): ...@@ -103,9 +103,7 @@ class DetailView(LoginRequiredMixin, TemplateView):
result = SettyController.getInformation( result = SettyController.getInformation(
templateId, hostname) templateId, hostname)
print '------------'
print result
print '------------'
return JsonResponse(result) return JsonResponse(result)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment