Skip to content

Gelencsér Szabolcs / monitor-client

Go to a project
Commit ec91e24f by Gregory Nagy
Options

pep8 conventions

parent 3faa2b8f
Showing with 161 additions and 135 deletions
src/client.py
...@@ -13,6 +13,15 @@ logging.basicConfig() ...@@ -13,6 +13,15 @@ logging.basicConfig()
class Client: class Client:
env_config = {
"host": "GRAPHITE_HOST",
"port": "GRAPHITE_PORT",
"amqp_user": "GRAPHITE_AMQP_USER",
"amqp_pass": "GRAPHITE_AMQP_PASSWORD",
"amqp_queue": "GRAPHITE_AMQP_QUEUE",
"amqp_vhost": "GRAPHITE_AMQP_VHOST",
}
def __init__(self, config): def __init__(self, config):
""" """
Constructor of the client class that is responsible for handling the Constructor of the client class that is responsible for handling the
...@@ -29,40 +38,42 @@ class Client: ...@@ -29,40 +38,42 @@ class Client:
""" """
hostname = socket.gethostname().split('.') hostname = socket.gethostname().split('.')
hostname.reverse() hostname.reverse()
self.name = "circle." + ".".join(hostname) separator = '.'
if os.getenv("GRAPHITE_SERVER_ADDRESS") is "": self.name = 'circle.%(host)s' % {'host': separator.join(hostname)}
print("GRAPHITE_SERVER_ADDRESS cannot be found in environmental " self.server_address = str(os.getenv(self.env_config['host']))
"variables" self.server_port = int(os.getenv(self.env_config['port']))
self.amqp_user = str(os.getenv(self.env_config['amqp_user']))
self.amqp_pass = str(os.getenv(self.env_config['amqp_pass']))
self.amqp_queue = str(os.getenv(self.env_config['amqp_queue']))
self.amqp_vhost = str(os.getenv(self.env_config['amqp_vhost']))
if self.server_address is "":
print(('%(host)s cannot be found in environmental variables.')
% {'host': self.env_config['host']}
) )
self.valid = False self.valid = False
return return
if os.getenv("GRAPHITE_SERVER_PORT") is "": if self.server_port is "":
print("GRAPHITE_SERVER_PORT cannot be found in environmental " print(('%(port)s cannot be found in environmental variables. ')
"variables. (AMQP standard is: 5672" % {'port': self.env_config['port']}
) )
self.valid = False self.valid = False
return return
if os.getenv("GRAPHITE_AMQP_USER") is "" or os.getenv( if self.amqp_user is "" or self.amqp_pass is "":
"GRAPHITE_AMQP_PASSWORD") is "": print(('%(user)s or %(pass)s cannot be '
print("GRAPHITE_AMQP_USER or GRAPHITE_AMQP_PASSWORD cannot be " 'found in environmental variables.')
"found in environmental variables. (AMQP standard is: " % {'user': self.env_config['amqp_user'],
"guest-guest)" 'pass': self.env_config['amqp_pass']}
) )
self.valid = False self.valid = False
return return
if os.getenv("GRAPHITE_AMQP_QUEUE") is "" or os.getenv( if self.amqp_queue is "" or self.amqp_vhost is "":
"GRAPHITE_AMQP_VHOST") is "": print(('%(queue)s or %(vhost)s cannot be '
print("GRAPHITE_AMQP_QUEUE or GRAPHITE_AMQP_VHOST cannot be " 'found in environmental variables.')
"found in environmental variables." % {'queue': self.env_config['amqp_queue'],
'vhost': self.env_config['amqp_vhost']}
) )
self.valid = False self.valid = False
return return
self.server_address = str(os.getenv("GRAPHITE_SERVER_ADDRESS"))
self.server_port = int(os.getenv("GRAPHITE_SERVER_PORT"))
self.amqp_user = str(os.getenv("GRAPHITE_AMQP_USER"))
self.amqp_pass = str(os.getenv("GRAPHITE_AMQP_PASSWORD"))
self.amqp_queue = str(os.getenv("GRAPHITE_AMQP_QUEUE"))
self.amqp_vhost = str(os.getenv("GRAPHITE_AMQP_VHOST"))
self.debugMode = config["debugMode"] self.debugMode = config["debugMode"]
self.kvmCPU = int(config["kvmCpuUsage"]) self.kvmCPU = int(config["kvmCpuUsage"])
self.kvmMem = int(config["kvmMemoryUsage"]) self.kvmMem = int(config["kvmMemoryUsage"])
...@@ -70,7 +81,7 @@ class Client: ...@@ -70,7 +81,7 @@ class Client:
self.beat = 1 self.beat = 1
self.valid = True self.valid = True
def __connect(self): def connect(self):
""" """
This method creates the connection to the queue of the graphite This method creates the connection to the queue of the graphite
server using the environmental variables given in the constructor. server using the environmental variables given in the constructor.
...@@ -86,17 +97,17 @@ class Client: ...@@ -86,17 +97,17 @@ class Client:
self.channel = self.connection.channel() self.channel = self.connection.channel()
return True return True
except RuntimeError: except RuntimeError:
print ("[ERROR] Cannot connect to the server. " print ('[ERROR] Cannot connect to the server. '
"Parameters could be wrong." 'Parameters could be wrong.'
) )
return False return False
except: except:
print ("[ERROR] Cannot connect to the server. There is no one " print ('[ERROR] Cannot connect to the server. There is no one '
"listening on the other side." 'listening on the other side.'
) )
return False return False
def __disconnect(self): def disconnect(self):
""" """
Break up the connection to the graphite server. If something went Break up the connection to the graphite server. If something went
wrong while disconnecting it simply cut the connection up. wrong while disconnecting it simply cut the connection up.
...@@ -105,11 +116,11 @@ class Client: ...@@ -105,11 +116,11 @@ class Client:
self.channel.close() self.channel.close()
self.connection.close() self.connection.close()
except RuntimeError: except RuntimeError:
print("[ERROR] An error has occured while disconnecting from the " print('[ERROR] An error has occured '
"server." 'while disconnecting from the server.'
) )
def __send(self, message): def send(self, message):
""" """
Send the message given in the parameters given in the message Send the message given in the parameters given in the message
parameter. This function expects that the graphite server want the parameter. This function expects that the graphite server want the
...@@ -121,12 +132,12 @@ class Client: ...@@ -121,12 +132,12 @@ class Client:
routing_key='', body="\n".join(message)) routing_key='', body="\n".join(message))
return True return True
except: except:
print("[ERROR] An error has occured while sending metrics to the " print('[ERROR] An error has occured '
"server." 'while sending metrics to the server.'
) )
return False return False
def __collectFromNode(self, metricCollectors): def collect_node(self, metricCollectors):
""" """
It harvests the given metrics in the metricCollectors list. This list It harvests the given metrics in the metricCollectors list. This list
should be provided by the collectables modul. It is important that should be provided by the collectables modul. It is important that
...@@ -134,15 +145,18 @@ class Client: ...@@ -134,15 +145,18 @@ class Client:
""" """
metrics = [] metrics = []
for collector in metricCollectors: for collector in metricCollectors:
if (self.beat % collector[1]) is 0: collector_function = collector[0]
stat = collector[0]() phase = collector[1]
metrics.append((self.name + "." + if (self.beat % phase) is 0:
stat.name + " %d" % (stat.value) stat = collector_function()
+ " %d" % (time.time()) metrics.append(('%(hostname)s.%(name)s %(value)f %(time)d') %
)) {'hostname': self.name,
'name': stat.name,
'value': stat.value,
'time': time.time()})
return metrics return metrics
def __collectFromVMs(self): def collect_vms(self):
""" """
This method is used for fetching the kvm processes running on the This method is used for fetching the kvm processes running on the
node and using the cmdline parameters calculates different types of node and using the cmdline parameters calculates different types of
...@@ -154,80 +168,80 @@ class Client: ...@@ -154,80 +168,80 @@ class Client:
for entry in procList: for entry in procList:
try: try:
entry_name = entry.name entry_name = entry.name
except psutil._error.NoSuchProcess: if entry_name in "kvm":
entry_name = "" cmdLine = entry.as_dict()["cmdline"]
if entry_name in "kvm": search = [cmd_param_index for cmd_param_index, cmd_param in
cmdLine = entry.as_dict()["cmdline"] enumerate(cmdLine)
search = [cmd_param_index for cmd_param_index, cmd_param in if cmd_param == "-name"]
enumerate(cmdLine) if not entry.is_running():
if cmd_param == "-name"] break
if not entry.is_running(): memory = [cmd_param_index for cmd_param_index, cmd_param in
break enumerate(cmdLine)
memory = [cmd_param_index for cmd_param_index, cmd_param in if cmd_param == "-m"]
enumerate(cmdLine) if not entry.is_running():
if cmd_param == "-m"] break
if not entry.is_running(): try:
break running_vms.append([cmdLine[search[0] + 1],
try: entry.pid,
running_vms.append([cmdLine[search[0] + 1], int(entry.as_dict()["cmdline"][
entry.pid, memory[0] + 1])])
int(entry.as_dict()["cmdline"][ except IndexError:
memory[0] + 1])]) pass
except IndexError: if ((self.beat % 30) is 0):
pass metrics.append("%s.vmcount %d %d"
if ((self.beat % 30) is 0): % (self.name, len(running_vms), time.time()))
metrics.append("%s.vmcount %d %d" for vm in running_vms:
% (self.name, len(running_vms), time.time())) vm_proc = psutil.Process(vm[1])
for vm in running_vms: if (((self.beat % self.kvmCPU) is 0) and vm_proc.is_running()):
vm_proc = psutil.Process(vm[1]) mem_perc = vm_proc.get_memory_percent() / 100 * vm[2]
if (((self.beat % self.kvmCPU) is 0) and vm_proc.is_running()): metrics.append("vm.%s.memory.usage %f %d"
mem_perc = vm_proc.get_memory_percent() / 100 * vm[2] % (vm[0], mem_perc, time.time()))
metrics.append("vm.%s.memory.usage %f %d" if (((self.beat % self.kvmMem) is 0) and vm_proc.is_running()):
% (vm[0], mem_perc, time.time())) systemtime = vm_proc.get_cpu_times().system
if (((self.beat % self.kvmMem) is 0) and vm_proc.is_running()): usertime = vm_proc.get_cpu_times().user
systemtime = vm_proc.get_cpu_times().system sumCpu = systemtime + usertime
usertime = vm_proc.get_cpu_times().user metrics.append("vm.%s.cpu.usage %f %d"
sumCpu = systemtime + usertime % (vm[0], sumCpu, time.time()))
metrics.append("vm.%s.cpu.usage %f %d" interfaces_list = psutil.network_io_counters(
% (vm[0], sumCpu, time.time())) pernic=True)
interfaces_list = psutil.network_io_counters( if ((self.beat % self.kvmNet) is 0):
pernic=True) for vm in running_vms:
if ((self.beat % self.kvmNet) is 0): interfaces_list_enum = enumerate(interfaces_list)
for vm in running_vms: for iname_index, iname in interfaces_list_enum:
interfaces_list_enum = enumerate(interfaces_list) if vm[0] in iname:
for iname_index, iname in interfaces_list_enum: metrics.append(
if vm[0] in iname: ('vm.%(name)s.network.packets_sent_%(interface)s '
metrics.append( '%(data)f %(time)d') %
('vm.%(name)s.network.packets_sent_%(interface)s ' {'name': vm[0],
'%(data)f %(time)d') % { 'interface': iname,
'name': vm[0], 'time': time.time(),
'interface': iname, 'data': interfaces_list[iname].packets_sent})
'time': time.time(), metrics.append(
'data': interfaces_list[iname].packets_sent}) ('vm.%(name)s.network.packets_recv_%(interface)s '
metrics.append( '%(data)f %(time)d') %
('vm.%(name)s.network.packets_recv_%(interface)s ' {'name': vm[0],
'%(data)f %(time)d') % { 'interface': iname,
'name': vm[0], 'time': time.time(),
'interface': iname, 'data': interfaces_list[iname].packets_recv})
'time': time.time(), metrics.append(
'data': interfaces_list[iname].packets_recv}) ('vm.%(name)s.network.bytes_sent_%(interface)s '
metrics.append( '%(data)f %(time)d') %
('vm.%(name)s.network.bytes_sent_%(interface)s ' {'name': vm[0],
'%(data)f %(time)d') % { 'interface': iname,
'name': vm[0], 'time': time.time(),
'interface': iname, 'data': interfaces_list[iname].bytes_sent})
'time': time.time(), metrics.append(
'data': interfaces_list[iname].bytes_sent}) ('vm.%(name)s.network.bytes_recv_%(interface)s '
metrics.append( '%(data)f %(time)d') %
('vm.%(name)s.network.bytes_recv_%(interface)s ' {'name': vm[0],
'%(data)f %(time)d') % { 'interface': iname,
'name': vm[0], 'time': time.time(),
'interface': iname, 'data': interfaces_list[iname].bytes_recv})
'time': time.time(), except psutil.NoSuchProcess:
'data': interfaces_list[iname].bytes_recv}) print('[ERROR LOG] Process lost.')
return metrics return metrics
def getMaxFrequency(self, metricCollectors=[]): def get_frequency(self, metricCollectors=[]):
""" """
""" """
items = metricCollectors + [["kvmCpuUsage", self.kvmMem], [ items = metricCollectors + [["kvmCpuUsage", self.kvmMem], [
...@@ -238,7 +252,7 @@ class Client: ...@@ -238,7 +252,7 @@ class Client:
max = item[1] max = item[1]
return max return max
def startReporting(self, metricCollectors=[]): def run(self, metricCollectors=[]):
""" """
Call this method to start reporting to the server, it needs the Call this method to start reporting to the server, it needs the
metricCollectors parameter that should be provided by the collectables metricCollectors parameter that should be provided by the collectables
...@@ -247,20 +261,20 @@ class Client: ...@@ -247,20 +261,20 @@ class Client:
if self.valid is False: if self.valid is False:
print("[ERROR] The client cannot be started.") print("[ERROR] The client cannot be started.")
raise RuntimeError raise RuntimeError
if self.__connect() is False: if self.connect() is False:
hostdata = self.server_address + ':' + self.server_port
print("[ERROR] An error has occured while connecting to the " print("[ERROR] An error has occured while connecting to the "
"server on %s." "server on %(host)s."
% (self.server_address + ":" + str(self.server_port))) % {'host': hostdata})
else: else:
print("[SUCCESS] Connection established to %s on port %s. \ print('[SUCCESS] Connection established to %(host)s:%(port)s.'
Clientname: %s" % {'host': self.server_address,
% (self.server_address, self.server_port, 'port': self.server_port})
self.name))
try: try:
maxFrequency = self.getMaxFrequency(metricCollectors) maxFrequency = self.get_frequency(metricCollectors)
while True: while True:
nodeMetrics = self.__collectFromNode(metricCollectors) nodeMetrics = self.collect_node(metricCollectors)
vmMetrics = self.__collectFromVMs() vmMetrics = self.collect_vms()
metrics = nodeMetrics + vmMetrics metrics = nodeMetrics + vmMetrics
if self.debugMode == "True": if self.debugMode == "True":
print(metrics) print(metrics)
...@@ -274,4 +288,4 @@ class Client: ...@@ -274,4 +288,4 @@ class Client:
except KeyboardInterrupt: except KeyboardInterrupt:
print("[x] Reporting has stopped by the user. Exiting...") print("[x] Reporting has stopped by the user. Exiting...")
finally: finally:
self.__disconnect() self.disconnect()
src/cnfparse.py
import ConfigParser as configparser import sys
if sys.version_info < (3, 0):
import ConfigParser as configparser
else:
import configparser
def importConf(path_to_file): def import_conf(path_to_file):
config = configparser.RawConfigParser(allow_no_value=False) config = configparser.RawConfigParser(allow_no_value=False)
try: try:
config.read(path_to_file) config.read(path_to_file)
params = {} params = {}
metrics = {} metrics = {}
params["debugMode"] = config.get("Client", "Debug") params["debugMode"] = config.get("Client", "Debug")
##
## Metrics
##
metrics["cpu.usage"] = int(config.get("Metrics", "cpuUsage")) metrics["cpu.usage"] = int(config.get("Metrics", "cpuUsage"))
metrics["cpu.times"] = int(config.get("Metrics", "cpuTimes")) metrics["cpu.times"] = int(config.get("Metrics", "cpuTimes"))
metrics["memory.usage"] = int(config.get("Metrics", "memoryUsage")) metrics["memory.usage"] = int(config.get("Metrics", "memoryUsage"))
...@@ -16,17 +23,23 @@ def importConf(path_to_file): ...@@ -16,17 +23,23 @@ def importConf(path_to_file):
metrics["system.boot_time"] = int(config.get("Metrics", metrics["system.boot_time"] = int(config.get("Metrics",
"systemBootTime")) "systemBootTime"))
metrics["network"] = int(config.get("Metrics", "dataTraffic")) metrics["network"] = int(config.get("Metrics", "dataTraffic"))
##
## Params
##
params["kvmCpuUsage"] = int(config.get("KVM", "cpuUsage")) params["kvmCpuUsage"] = int(config.get("KVM", "cpuUsage"))
params["kvmMemoryUsage"] = int(config.get("KVM", "memoryUsage")) params["kvmMemoryUsage"] = int(config.get("KVM", "memoryUsage"))
params["kvmNetworkUsage"] = int(config.get("KVM", "networkUsage")) params["kvmNetworkUsage"] = int(config.get("KVM", "networkUsage"))
except configparser.NoSectionError: except configparser.NoSectionError:
print("Config file contains error! Reason: Missing section.") print("[ERROR] Config file contains error! "
"Reason: Missing section.")
raise raise
except configparser.ParsingError: except configparser.ParsingError:
print("Config file contains error! Reason: Cannot parse.") print("[ERROR] Config file contains error! "
"Reason: Cannot parse.")
raise raise
except configparser.MissingSectionHeaderError: except configparser.MissingSectionHeaderError:
print("Config file contains error! Reason: Missing section-header.") print("[ERROR] Config file contains error! "
"Reason: Missing section-header.")
raise raise
return params, metrics return params, metrics
src/collectables.py
...@@ -18,20 +18,19 @@ class collectables: ...@@ -18,20 +18,19 @@ class collectables:
} }
@staticmethod @staticmethod
def listKeys(): def list_keys():
return list(collectables.__collectables.keys()) return list(collectables.__collectables.keys())
@staticmethod @staticmethod
def listMetricsToKey(key): def list_metrics_to_key(key):
return collectables.__collectables[key] return collectables.__collectables[key]
@staticmethod @staticmethod
def listMetricsNameToKey(key): def list_metrics_name_to_key(key):
return [x.name for x in collectables.__collectables[key]] return [x.name for x in collectables.__collectables[key]]
@staticmethod @staticmethod
def provide(requests=[]): def provide(requests=[]):
#valid_keys = collectables.listKeys()
reqs = [] reqs = []
for requests, value in requests.items(): for requests, value in requests.items():
if value > 0: if value > 0:
...@@ -44,4 +43,4 @@ class collectables: ...@@ -44,4 +43,4 @@ class collectables:
@staticmethod @staticmethod
def provideAll(): def provideAll():
return collectables.provide(collectables.listKeys()) return collectables.provide(collectables.list_keys())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment