Skip to content

CIRCLE / monitor-client

Go to a project
Commit 606df697 by Bach Dániel
Options

add disk io metric

parent 5a251e2c
Showing with 48 additions and 268 deletions
.gitignore 0 → 100644
*.pyc
*.swp
config/client.conf deleted 100644 → 0
[Client]
Debug = False
[Metrics]
cpuUsage = 5
cpuTimes = 5
memoryUsage = 5
userCount = 60
swapUsage = 30
systemBootTime = 120
packageTraffic = 10
dataTraffic = 10
[KVM]
cpuUsage = 5
memoryUsage = 5
networkUsage = 5
manage.py
import logging
import sys
from src import cnfparse
from src import client
from src.collectables import collectables
from src.client import Client
def main():
......@@ -16,10 +14,8 @@ def main():
if len(sys.argv) is not 2 and sys.argv[1] is not "run":
print("[ERROR] Command cannot be parsed. Exiting...")
return
configuration, metrics = cnfparse.import_conf("config/client.conf")
cli = client.Client(configuration)
metricCollectors = collectables.provide(metrics)
cli.run(metricCollectors)
Client().run()
if __name__ == "__main__":
......
src/client.py
#!/usr/bin/python
from datetime import datetime
from itertools import islice
from socket import gethostname
import argparse
import logging
import operator
import os
import pika
import psutil
......@@ -25,7 +23,7 @@ class Client:
"amqp_vhost": "GRAPHITE_AMQP_VHOST",
}
def __init__(self, config):
def __init__(self):
"""
Constructor of the client class that is responsible for handling the
communication between the graphite server and the data source. In
......@@ -46,11 +44,6 @@ class Client:
setattr(self, var, value)
else:
raise RuntimeError('%s environment variable missing' % env_var)
self.debugMode = config["debugMode"]
self.kvmCPU = int(config["kvmCpuUsage"])
self.kvmMem = int(config["kvmMemoryUsage"])
self.kvmNet = int(config["kvmNetworkUsage"])
self.beat = 0
def connect(self):
"""
......@@ -86,7 +79,8 @@ class Client:
self.channel.close()
self.connection.close()
except RuntimeError as e:
logger.error('An error has occured while disconnecting. %s', unicode(e))
logger.error('An error has occured while disconnecting. %s',
unicode(e))
raise
def send(self, message):
......@@ -105,24 +99,39 @@ class Client:
len(body))
raise
def collect_node(self, metricCollectors):
def collect_node(self):
"""
It harvests the given metrics in the metricCollectors list. This list
should be provided by the collectables modul. It is important that
only the information collected from the node is provided here.
"""
metrics = []
for collector in metricCollectors:
collector_function = collector[0]
phase = collector[1]
if self.beat % phase == 0:
stat = collector_function()
metrics.append(('%(hostname)s.%(name)s %(value)f %(time)d') %
{'hostname': self.name,
'name': stat.name,
'value': stat.value,
'time': time.time()})
return metrics
now = time.time()
metrics = {
'cpu.usage': psutil.cpu_percent(interval=0.0),
'cpu.times': psutil.cpu_times().user + psutil.cpu_times().system,
'memory.usage': psutil.virtual_memory().percent,
'swap.usage': psutil.swap_memory().percent,
'user.count': len(psutil.get_users()),
'system.boot_time': psutil.get_boot_time()
}
for k, v in psutil.disk_io_counters().__dict__.items():
metrics['disk.%s' % k] = v
interfaces = psutil.network_io_counters(pernic=True)
for interface, data in interfaces.iteritems():
if not interface.startswith('cloud-'):
for metric in ('packets_sent', 'packets_recv',
'bytes_sent', 'bytes_recv'):
metrics['network.%s-%s' %
(metric, interface)] = getattr(data, metric)
return ['%(host)s.%(name)s %(val)f %(time)d' % {'host': self.name,
'name': name,
'val': value,
'time': now}
for name, value in metrics.iteritems()]
def collect_vms(self):
"""
......@@ -133,24 +142,20 @@ class Client:
metrics = []
now = time.time()
running_vms = []
beats = {
'mem': self.beat % self.kvmMem == 0,
'cpu': self.beat % self.kvmCPU == 0,
'net': self.beat % self.kvmNet == 0
}
if beats['cpu'] or beats['mem']:
for entry in psutil.get_process_list():
try:
if entry.name == 'kvm':
parser = argparse.ArgumentParser()
parser.add_argument('-name')
parser.add_argument('--memory-size', '-m ', type=int)
args, unknown = parser.parse_known_args(entry.cmdline[1:])
args, unknown = parser.parse_known_args(
entry.cmdline[1:])
process = psutil.Process(entry.pid)
mem_perc = process.get_memory_percent() / 100 * args.memory_size
mem_perc = (process.get_memory_percent()
/ 100 * args.memory_size)
metrics.append('vm.%(name)s.memory.usage %(value)f '
'%(time)d' % {'name': args.name,
'value': mem_perc,
......@@ -165,9 +170,8 @@ class Client:
except psutil.NoSuchProcess:
logger.warning('Process %d lost.', entry.pid)
interfaces_list = psutil.network_io_counters(pernic=True)
if beats['net']:
for interface, data in interfaces_list.iteritems():
interfaces = psutil.network_io_counters(pernic=True)
for interface, data in interfaces.iteritems():
try:
vm, vlan = interface.rsplit('-', 1)
except ValueError:
......@@ -184,21 +188,13 @@ class Client:
'time': now,
'data': getattr(data, metric)})
if (self.beat % 30) == 0:
metrics.append(
('%(host)s.vmcount %(data)d %(time)d') %
{'host': self.name,
'%(host)s.vmcount %(data)d %(time)d' % {
'host': self.name,
'data': len(running_vms),
'time': time.time()})
return metrics
def get_frequency(self, metricCollectors=[]):
"""
"""
items = metricCollectors + [["kvmCpuUsage", self.kvmMem], [
"kvmMemoryUsage", self.kvmCPU], ["kvmNetworkUsage", self.kvmNet]]
freqs = set([i[1] for i in items if i[1]>0])
return reduce(operator.mul, freqs, 1)
return metrics
@staticmethod
def _chunker(seq, size):
......@@ -207,7 +203,7 @@ class Client:
for pos in xrange(0, len(seq), size):
yield islice(seq, pos, pos + size)
def run(self, metricCollectors=[]):
def run(self):
"""
Call this method to start reporting to the server, it needs the
metricCollectors parameter that should be provided by the collectables
......@@ -215,22 +211,13 @@ class Client:
"""
self.connect()
try:
maxFrequency = self.get_frequency(metricCollectors)
while True:
nodeMetrics = self.collect_node(metricCollectors)
vmMetrics = self.collect_vms()
metrics = nodeMetrics + vmMetrics
metrics = self.collect_node() + self.collect_vms()
if metrics:
if self.debugMode == "True":
for i in metrics:
logger.debug('Metric to send: %s', i)
for chunk in self._chunker(metrics, 100):
self.send(chunk)
logger.info("%d metrics sent", len(metrics))
time.sleep(1)
self.beat += 1
if self.beat % maxFrequency == 0:
self.beat = 0
time.sleep(10)
except KeyboardInterrupt:
logger.info("Reporting has stopped by the user. Exiting...")
finally:
......
src/cnfparse.py deleted 100644 → 0
import sys
if sys.version_info < (3, 0):
import ConfigParser as configparser
else:
import configparser
def import_conf(path_to_file):
config = configparser.RawConfigParser(allow_no_value=False)
try:
config.read(path_to_file)
params = {}
metrics = {}
params["debugMode"] = config.get("Client", "Debug")
##
## Metrics
##
metrics["cpu.usage"] = int(config.get("Metrics", "cpuUsage"))
metrics["cpu.times"] = int(config.get("Metrics", "cpuTimes"))
metrics["memory.usage"] = int(config.get("Metrics", "memoryUsage"))
metrics["user.count"] = int(config.get("Metrics", "userCount"))
metrics["swap.usage"] = int(config.get("Metrics", "swapUsage"))
metrics["system.boot_time"] = int(config.get("Metrics",
"systemBootTime"))
metrics["network"] = int(config.get("Metrics", "dataTraffic"))
##
## Params
##
params["kvmCpuUsage"] = int(config.get("KVM", "cpuUsage"))
params["kvmMemoryUsage"] = int(config.get("KVM", "memoryUsage"))
params["kvmNetworkUsage"] = int(config.get("KVM", "networkUsage"))
except configparser.NoSectionError:
print("[ERROR] Config file contains error! "
"Reason: Missing section.")
raise
except configparser.ParsingError:
print("[ERROR] Config file contains error! "
"Reason: Cannot parse.")
raise
except configparser.MissingSectionHeaderError:
print("[ERROR] Config file contains error! "
"Reason: Missing section-header.")
raise
return params, metrics
src/collectables.py deleted 100644 → 0
from metrics import std
class collectables:
__collectables = {
std.cpu.usage.name: [std.cpu.usage],
std.cpu.times.name: [std.cpu.times],
std.memory.usage.name: [std.memory.usage],
std.swap.usage.name: [std.swap.usage],
std.user.count.name: [std.user.count],
std.network.packages_sent.name: [std.network.packages_sent],
std.network.packages_received.name: [std.network.packages_received],
std.network.bytes_sent.name: [std.network.bytes_sent],
std.network.bytes_received.name: [std.network.bytes_received],
std.system.boot_time.name: [std.system.boot_time],
"network": [std.network.bytes_sent, std.network.bytes_received,
std.network.packages_sent, std.network.packages_received],
}
@staticmethod
def list_keys():
return list(collectables.__collectables.keys())
@staticmethod
def list_metrics_to_key(key):
return collectables.__collectables[key]
@staticmethod
def list_metrics_name_to_key(key):
return [x.name for x in collectables.__collectables[key]]
@staticmethod
def provide(requests=[]):
reqs = []
for requests, value in requests.items():
if value > 0:
reqs.append([requests, value])
collectors = []
for request in reqs:
for item in collectables.__collectables[request[0]]:
collectors.append([item.harvest, request[1]])
return collectors
@staticmethod
def provideAll():
return collectables.provide(collectables.list_keys())
src/metrics.py deleted 100644 → 0
#!/usr/bin/python
import psutil as ps
import collections
#############################################################################
Metrics = collections.namedtuple("Metrics", ["name", "value"])
class Collection(object):
class Group(object):
class Metric(object):
name = "unknown"
collector_function = 0
collector_function_arguments = {}
collector_function_result_attr = ""
@classmethod
def harvest(cls):
query = cls.collector_function.im_func(
**cls.collector_function_arguments)
if ((isinstance(query, list)) or (isinstance(query, dict))):
return Metrics(cls.name,
query[cls.collector_function_result_attr])
elif (isinstance(query, tuple)):
return Metrics(cls.name, query.__getattribute__(
cls.collector_function_result_attr))
else:
return Metrics(cls.name, query)
##############################################################################
class std(Collection):
class cpu(Collection.Group):
class usage(Collection.Group.Metric):
name = "cpu.usage"
collector_function = ps.cpu_percent
collector_function_arguments = {'interval': 0.0}
class times(Collection.Group.Metric):
name = "cpu.times"
@classmethod
def harvest(cls):
return Metrics(cls.name,
ps.cpu_times().user + ps.cpu_times().system)
class memory(Collection.Group):
class usage(Collection.Group.Metric):
name = "memory.usage"
collector_function = ps.virtual_memory
collector_function_result_attr = "percent"
class swap(Collection.Group):
class usage(Collection.Group.Metric):
name = "swap.usage"
collector_function = ps.swap_memory
collector_function_result_attr = "percent"
class user(Collection.Group):
class count(Collection.Group.Metric):
name = "user.count"
@classmethod
def harvest(cls):
return Metrics(cls.name, len(ps.get_users()))
class network(Collection.Group):
class packages_sent(Collection.Group.Metric):
name = "network.packets_sent"
collector_function = ps.network_io_counters
collector_function_result_attr = "packets_sent"
class packages_received(Collection.Group.Metric):
name = "network.packets_recv"
collector_function = ps.network_io_counters
collector_function_result_attr = "packets_recv"
class bytes_sent(Collection.Group.Metric):
name = "network.bytes_sent"
collector_function = ps.network_io_counters
collector_function_result_attr = "bytes_sent"
class bytes_received(Collection.Group.Metric):
name = "network.bytes_received"
collector_function = ps.network_io_counters
collector_function_result_attr = "bytes_recv"
class system(Collection.Group):
class boot_time(Collection.Group.Metric):
name = "system.boot_time"
collector_function = ps.get_boot_time
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment