dc stats utils

balancer/stats.py

@@ -5,11 +5,11 @@ import time
 
 logger = logging.getLogger(__name__)
 
-PREFIX = "RBstat?"
-MEM_ALLOCATED_TARGET = "target=circle.*.memory.allocated&format=json&time=-1min"
-MEM_USAGE_TARGET = "target=circle.*.memory.usage&format=json&time=-1min"
-VMCOUNT_TARGET = "target=circle.*.vmcount&format=json&time=-1min"
-CPU_PERCENT_TARGET = "target=circle.*.cpu.percent&format=json&time=-1min"
+MEM_ALLOCATED_TARGET = "RBstat?target=circle.*.memory.allocated&format=json&time=-2min"
+MEM_USAGE_TARGET = "RBstat?target=circle.*.memory.usage&format=json&time=-2min"
+VMCOUNT_TARGET = "RBstat?target=circle.*.vmcount&format=json&time=-2min"
+CPU_PERCENT_TARGET = "RBstat?target=circle.*.cpu.percent&format=json&time=-2min"
+STATIC_INFO = "dashboard/acpi/nodesum/"
 
 STAT_TARGETS = {
     "mem_allocated": MEM_ALLOCATED_TARGET,
@@ -18,11 +18,32 @@ STAT_TARGETS = {
     "cpu_percent": CPU_PERCENT_TARGET,
 }
 
+def get_static_info_from_dc(datacenter):
+    url: str = f"{datacenter}/{STATIC_INFO}"
+    req = requests.request(
+        method="GET",
+        url=url,
+        verify=False,
+    )
+    if int(req.status_code / 100) != 2:
+        logger.warning(
+            f"cannot retrieve the data, from dc: {datacenter}  ({req.status_code})"
+        )
+        return 0, 0
+    else:
+        try:
+            data = json.loads(req.content)
+            core_num = data["core_num"]
+            max_ram = data["max_ram"]   
+            return core_num, max_ram
+        except json.decoder.JSONDecodeError:
+            logger.warning(f"JSON decoding error: Static infos")
+            return 0, 0  
 
 def get_stat_from_dc(datacenter, retry = False):
     stat = {}
     for key, target in STAT_TARGETS.items():
-        url: str = f"{datacenter}/{PREFIX}{target}"
+        url: str = f"{datacenter}/{target}"
         req = requests.request(
             method="GET",
             url=url,
@@ -33,17 +54,26 @@ def get_stat_from_dc(datacenter, retry = False):
                 f"cannot retrieve the data, from dc: {datacenter}  ({req.status_code})"
             )
         else:
-            data = json.loads(req.content)
-            stat[key] = [i["datapoints"][0][0] for i in data]
-
-            if retry and stat[key][0] is None:
-                time.sleep(1)
-                req = requests.request(
-                    method="GET",
-                    url=url,
-                    verify=False,
-                )
+            try:
                 data = json.loads(req.content)
-                stat[key] = [i["datapoints"][0][0] for i in data]
-    print(stat)
+                stat[key] = [max(
+                        i["datapoints"][0][0] if i["datapoints"][0][0] else 0.0, 
+                        i["datapoints"][1][0] if i["datapoints"][1][0] else 0.0
+                    ) for i in data]
+
+                if retry and stat[key][0] is None:
+                    time.sleep(1)
+                    req = requests.request(
+                        method="GET",
+                        url=url,
+                        verify=False,
+                    )
+                    data = json.loads(req.content)
+                    stat[key] = [max(
+                        i["datapoints"][0][0] if i["datapoints"][0][0] else 0.0, 
+                        i["datapoints"][1][0] if i["datapoints"][1][0] else 0.0
+                    ) for i in data]
+            except json.decoder.JSONDecodeError:
+                logger.warning(f"JSON decoding error: key {key}")
+                stat[key] = 0.0  
     return stat

core/models.py

@@ -53,4 +53,5 @@ class ResourceStat(BaseModel):
     mem_max: int
     mem_used: int
     cpu_usage: float
+    cpu_core: int
     vm_count: int

main.py

@@ -14,6 +14,7 @@ from sredis.sredis import (
     update_datacenter_stats,
     update_datacenter_status,
     update_stats,
+    valid_ram_datacenters,
 )
 import logging
 import requests
@@ -141,11 +142,19 @@ def get_datacenter_stat(
             mem_max=x.mem_max,
             mem_used=x.mem_used,
             cpu_usage=x.cpu_usage,
-            vm_count=x.vm_count
+            vm_count=x.vm_count,
+            cpu_core=x.cpu_core
         ) for x in stats
     ]
     return stats
 
+@app.get("/stats/ram/")
+def get_datacenter_stat(
+    username=Depends(get_current_user), vm_ram: int = 1024
+): 
+    stats = valid_ram_datacenters(vm_ram)
+    return stats
+
 
 
 @app.get("/tokens/")

sredis/models.py

@@ -29,3 +29,4 @@ class DataCenterResource(HashModel):
     mem_used: int = Field()
     cpu_usage: float = Field()
     vm_count: int = Field()
+    cpu_core: int = Field()

sredis/sredis.py

 import redis
 import requests
-from balancer.stats import get_stat_from_dc
+from balancer.stats import get_stat_from_dc, get_static_info_from_dc
 from core.models import UserLoginSchema, ResourceStat
 from .models import PUser, DataCenterResource, ResourcePool
 from passlib.hash import pbkdf2_sha256
@@ -63,12 +63,15 @@ def update_datacenter_stats():
     for dc in all_keys.values():
         logger.info(f"Get stats from {dc}")
         stats = get_stat_from_dc(dc, retry=True)
+        cpu_core, mem_max = get_static_info_from_dc(dc)
         rs = ResourceStat(
             datacenter_name=dc, 
-            mem_max=4096,
-            mem_used=int(stats["mem_usage"][0] * 4096 / 100.0) if stats["mem_usage"][0] else None,
-            cpu_usage=stats["cpu_percent"][0],
-            vm_count=int(stats["vmcount"][0])  
+            cpu_core=cpu_core,
+            mem_max=int(mem_max / 1048576) if mem_max else 4096,
+            mem_allocated=int(stats["mem_allocated"][0] / 1048576.0) if stats["mem_allocated"] else 0,
+            mem_used=int(stats["mem_usage"][0] * 4096 / 100.0) if stats["mem_usage"][0] else 0,
+            cpu_usage=float(stats["cpu_percent"][0]) if stats["cpu_percent"][0] else 0.0,
+            vm_count=int(stats["vmcount"][0])  if stats["vmcount"][0] else 0 
         )
         update_stats(rs)
 
@@ -157,7 +160,8 @@ def update_stats(stat: ResourceStat):
             mem_max=stat.mem_max,
             mem_used=stat.mem_used,
             cpu_usage=stat.cpu_usage,
-            vm_count=stat.vm_count
+            vm_count=stat.vm_count,
+            cpu_core=stat.cpu_core
         )
         dc.save()
     else:
@@ -166,8 +170,28 @@ def update_stats(stat: ResourceStat):
         dc.mem_used = stat.mem_used if stat.mem_used else dc.mem_used
         dc.cpu_usage = stat.cpu_usage if stat.cpu_usage else dc.cpu_usage
         dc.vm_count = stat.vm_count if stat.vm_count else dc.vm_count
+        dc.cpu_core = stat.cpu_core if stat.cpu_core else dc.cpu_core
         dc.save()
 
+def get_stats(datacenter_name :str):
+    dc = DataCenterResource.find(DataCenterResource.datacenter_name == datacenter_name).get_item()
+    return dc
+
+def valid_ram_datacenter(datacenter_name :str, vm_mem: int):
+    dc = DataCenterResource.find(DataCenterResource.datacenter_name == datacenter_name).get_item()
+    if dc is None:
+        return False
+    if dc.mem_max - dc.mem_used > vm_mem:
+        return True
+    return False
+
+def valid_ram_datacenters(vm_mem: int):
+    dcs = DataCenterResource.find().all()
+    valid_dcs = []
+    for dc in dcs:
+        if dc.mem_max - dc.mem_used > vm_mem:
+            valid_dcs.append(dc.datacenter_name)
+    return valid_dcs
 
 def get_all_stats():
     dcs = DataCenterResource.find().all()