RC: (update) script to use threading library and a deque (a highly efficient rolling array).

By using Python's threading library and a deque (a highly efficient rolling array). A background thread will wake up every 1 second, take a snapshot, and push it into the array. When your website hits the API, it just grabs whatever is currently in the arrays.

pve-metrics/homelab-api.py

@@ -2,26 +2,94 @@
 import json
 import subprocess
 import os
+import time
+import threading
+from collections import deque
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
-# --- CONFIGURATION FLAGS ---
+# --- CONFIGURATION ---
 PORT = 8080
-IFACE = "bond0"
+IFACE = "vmbr0"
+ENABLE_PVE_STATS = True
+# ---------------------
 
-# Toggle features True/False depending on your setup
+# Rolling memory buffers (stores exactly 60 seconds of data)
-ENABLE_ZFS = False
+hist_cpu = deque([0]*60, maxlen=60)
-ZPOOL = "rpool"
+hist_ram = deque([0]*60, maxlen=60)
+hist_rx  = deque([0.0]*60, maxlen=60)
+hist_tx  = deque([0.0]*60, maxlen=60)
 
-ENABLE_PVE_STATS = True  # Set to False if not running on Proxmox
+# Dictionary to hold the most recent slow-changing data
-# ---------------------------
+latest_state = {}
 
 def run_cmd(cmd):
-    """Runs a shell command and silently returns None if it fails."""
     try:
         return subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL).decode('utf-8').strip()
     except Exception:
         return None
 
+def get_cpu_times():
+    """Reads raw CPU ticks directly from the kernel (ultra-fast, no blocking)"""
+    with open('/proc/stat', 'r') as f:
+        for line in f:
+            if line.startswith('cpu '):
+                parts = list(map(int, line.split()[1:]))
+                idle = parts[3] + parts[4] # idle + iowait
+                total = sum(parts)
+                return idle, total
+    return 0, 0
+
+def background_poller():
+    """Runs in the background, gathering data every 1 second."""
+    global latest_state
+    
+    # Baseline for 1-second delta calculations
+    prev_idle, prev_total = get_cpu_times()
+    rx_prev = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/rx_bytes") or 0)
+    tx_prev = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/tx_bytes") or 0)
+
+    while True:
+        time.sleep(1) # The 1-second heartbeat
+
+        try:
+            # 1. CPU Delta Calculation
+            idle, total = get_cpu_times()
+            diff_idle = idle - prev_idle
+            diff_total = total - prev_total
+            cpu_pct = (1.0 - (diff_idle / diff_total)) * 100.0 if diff_total > 0 else 0.0
+            hist_cpu.append(round(cpu_pct, 1))
+            prev_idle, prev_total = idle, total
+
+            # 2. Network Delta Calculation
+            rx_now = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/rx_bytes") or 0)
+            tx_now = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/tx_bytes") or 0)
+            hist_rx.append(round((rx_now - rx_prev) / (1024 * 1024), 2))
+            hist_tx.append(round((tx_now - tx_prev) / (1024 * 1024), 2))
+            rx_prev, tx_prev = rx_now, tx_now
+
+            # 3. RAM Snapshot
+            ram_raw = run_cmd("free -b | grep Mem | awk '{print $2, $3}'")
+            if ram_raw:
+                ram_total, ram_used = map(int, ram_raw.split())
+                ram_pct = (ram_used / ram_total) * 100.0
+                hist_ram.append(round(ram_pct, 1))
+                latest_state["ram"] = {
+                    "percent": round(ram_pct, 1),
+                    "used_gb": round(ram_used / (1024**3), 1),
+                    "total_gb": round(ram_total / (1024**3), 1)
+                }
+
+            # 4. Storage & System Health
+            st = os.statvfs('/')
+            latest_state["root_disk_pct"] = round((1.0 - (st.f_bavail / st.f_blocks)) * 100.0, 1)
+
+            if ENABLE_PVE_STATS:
+                latest_state["vms_running"] = int(run_cmd("qm list | grep running | wc -l") or 0)
+                latest_state["lxcs_running"] = int(run_cmd("pct list | grep running | wc -l") or 0)
+
+        except Exception as e:
+            print(f"Poller error: {e}")
+
 class MetricsHandler(BaseHTTPRequestHandler):
     def do_GET(self):
         if self.path == '/api/homelab':
@@ -30,73 +98,26 @@ class MetricsHandler(BaseHTTPRequestHandler):
             self.send_header('Access-Control-Allow-Origin', '*') 
             self.end_headers()
 
-            data = {}
+            # Package the latest instantaneous data + the full 60s history arrays
+            data = latest_state.copy()
+            data["cpu"] = hist_cpu[-1] if hist_cpu else 0
+            data["rx_mb"] = hist_rx[-1] if hist_rx else 0
+            data["tx_mb"] = hist_tx[-1] if hist_tx else 0
+            
+            data["cpu_hist"] = list(hist_cpu)
+            data["ram_hist"] = list(hist_ram)
+            data["rx_hist"] = list(hist_rx)
+            data["tx_hist"] = list(hist_tx)
 
-            try:
+            self.wfile.write(json.dumps(data).encode())
-                # 1. Network & CPU (Includes 1-second delay for accuracy)
-                rx1 = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/rx_bytes") or 0)
-                tx1 = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/tx_bytes") or 0)
-
-                cpu_idle_str = run_cmd("vmstat 1 2 | tail -1 | awk '{print $15}'")
-                cpu_idle = float(cpu_idle_str) if cpu_idle_str else 100.0
-                data["cpu"] = round(100.0 - cpu_idle, 1)
-
-                rx2 = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/rx_bytes") or 0)
-                tx2 = int(run_cmd(f"cat /sys/class/net/{IFACE}/statistics/tx_bytes") or 0)
-                
-                data["rx_mb"] = round((rx2 - rx1) / (1024 * 1024), 2)
-                data["tx_mb"] = round((tx2 - tx1) / (1024 * 1024), 2)
-
-                # 2. RAM Usage
-                ram_str = run_cmd("free | grep Mem | awk '{print $3/$2 * 100.0}'")
-                data["ram"] = round(float(ram_str), 1) if ram_str else 0.0
-
-                # 3. Load Average (Pure Python)
-                load1, load5, load15 = os.getloadavg()
-                data["load_avg"] = [round(load1, 2), round(load5, 2), round(load15, 2)]
-
-                # 4. Uptime in Days (Pure Python)
-                with open('/proc/uptime', 'r') as f:
-                    uptime_seconds = float(f.readline().split()[0])
-                    data["uptime_days"] = round(uptime_seconds / 86400, 1)
-
-                # 5. Root Disk Usage (Pure Python, lightning fast)
-                st = os.statvfs('/')
-                total = st.f_blocks * st.f_frsize
-                free = st.f_bavail * st.f_frsize
-                data["root_disk_pct"] = round(((total - free) / total) * 100.0, 1)
-
-                # 6. CPU Temperature (Reads native thermal zone)
-                try:
-                    with open('/sys/class/thermal/thermal_zone0/temp', 'r') as f:
-                        data["temp_c"] = round(int(f.read()) / 1000.0, 1)
-                except Exception:
-                    data["temp_c"] = "N/A" # Fails safely if hardware doesn't expose it
-
-                # 7. ZFS Array (Optional)
-                if ENABLE_ZFS:
-                    zfs_alloc = run_cmd(f"zpool list -H -p -o allocated {ZPOOL}")
-                    zfs_size = run_cmd(f"zpool list -H -p -o size {ZPOOL}")
-                    if zfs_alloc and zfs_size:
-                        data["zfs_pct"] = round((float(zfs_alloc) / float(zfs_size)) * 100.0, 1)
-                    else:
-                        data["zfs_pct"] = "Error"
-
-                # 8. Proxmox VM/LXC Counts (Optional)
-                if ENABLE_PVE_STATS:
-                    data["vms_running"] = int(run_cmd("qm list | grep running | wc -l") or 0)
-                    data["lxcs_running"] = int(run_cmd("pct list | grep running | wc -l") or 0)
-
-                # Send payload
-                self.wfile.write(json.dumps(data).encode())
-                
-            except Exception as e:
-                self.wfile.write(json.dumps({"error": str(e)}).encode())
         else:
             self.send_response(404)
             self.end_headers()
 
 if __name__ == '__main__':
+    # Start the data collector in the background
+    threading.Thread(target=background_poller, daemon=True).start()
+    
     server = HTTPServer(('0.0.0.0', PORT), MetricsHandler)
-    print(f"Starting homelab API on port {PORT}...")
+    print(f"Starting API on port {PORT}...")
     server.serve_forever()