Собственная система мониторинга парка ВМ 2.0

Итак, задача стара: заказчик запустил в офисе целое стадо виртуальных машин, которые выполняют однотипные задачи в несколько потоков каждая. Нужно прикрутить аскетичный мониторинг, который покажет нагрузку на ЦП, свободное мето на диске и примерно прикинет количество активных потоков по состоянию на сейчас.

Решение показано ниже.

1. Серверная часть

Был арендован сервер для сбора метрик, как всегда, было выбрано надёжное и бюджетное решение.

2. Установлен форк nginx под названием angie

Добавлен поддомен:

server {
listen 8080;
location / {
proxy_pass http://localhost:5000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr; # <-- Передаём реальный IP
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # <-- Поддержка цепочки прокси
# proxy_set_header X-Forwarded-Proto $scheme; # Полезно, если в будущем добавите HTTPS

# Таймауты для надёжности
proxy_connect_timeout 10s;
proxy_send_timeout 30s;
proxy_read_timeout 30s;

# Запрещаем кэширование ответов (если API)
proxy_cache off;
proxy_store off;
}
}

Серверная часть написана на Питоне:

# server.py
from flask import Flask, request, jsonify, render_template
import mysql.connector
from datetime import datetime
from zoneinfo import ZoneInfo

app = Flask(__name__)
MOSCOW_TZ = ZoneInfo("Europe/Moscow")

def get_db_connection(database):
return mysql.connector.connect(
host="localhost",
user="root",
password="Moscow@7",
database=database
)

@app.route('/metrics', methods=['POST'])
def receive_metrics():
db = None
try:
data = request.get_json()
if not data:
return jsonify({"error": "No JSON data received"}), 400

#moscow_time = datetime.now(MOSCOW_TZ).strftime('%Y-%m-%d %H:%M:%S+03:00')
moscow_time = datetime.now(MOSCOW_TZ).strftime('%Y-%m-%d %H:%M:%S')

vm_id = str(data.get('vm_id', 'UNKNOWN'))[:100]

# Нормализация группы при приёме
vm_profile = data.get('vm_profile')
if not vm_profile or str(vm_profile).strip().lower() in ('null', 'none', ''):
vm_profile = 'Cyber'
else:
vm_profile = str(vm_profile)[:50]

vm_threads_raw = data.get('vm_threads', '0')
try:
threads = int(vm_threads_raw)
except (ValueError, TypeError):
threads = 0

cpu = float(data.get('cpu', 0.0)) if data.get('cpu') is not None else 0.0
disk_free = int(data.get('disk_free', 0)) if data.get('disk_free') is not None else 0

bas_title_raw = data.get('bas_title')
if bas_title_raw is None:
bas_title = ""
elif isinstance(bas_title_raw, (dict, list)):
bas_title = str(bas_title_raw)[:255]
else:
bas_title = str(bas_title_raw)[:255]

success_events = data.get('success_events', [])
if not isinstance(success_events, list):
success_events = []
success_count = len(success_events)

# --- ИЗМЕНЕНИЕ: Работаем только с одной БД ---
db = get_db_connection("bas_monitor_2")
cursor = db.cursor()
cursor.execute("""
INSERT INTO metrics (vm_id, vm_group, timestamp, cpu, disk_free, threads, bas_title, success)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (vm_id, vm_profile, moscow_time, cpu, disk_free, threads, bas_title, success_count))

db.commit()
cursor.close()

return jsonify({"status": "ok"}), 200

except Exception as e:
print(f"[ERROR] {type(e).__name__}: {e}")
import traceback
traceback.print_exc()
return jsonify({"error": "Internal server error"}), 500

finally:
if db is not None:
try:
db.close()
except:
pass

@app.route('/dashboard')
def dashboard():
try:
conn = get_db_connection("bas_monitor_2")
cursor = conn.cursor(dictionary=True)

cursor.execute("""
SELECT vm_id, vm_group, cpu, disk_free, threads, timestamp
FROM metrics
WHERE timestamp >= NOW() - INTERVAL 12 HOUR
ORDER BY timestamp ASC
""")
raw_data = cursor.fetchall()

vms_by_group = {}

for row in raw_data:
vm = row['vm_id']

# ? Критически важная нормализация: всё непонятное → Cyber
group_raw = row['vm_group']
if not group_raw or str(group_raw).strip().lower() in ('null', 'none', ''):
group = 'Cyber'
else:
group = str(group_raw)

if group not in vms_by_group:
vms_by_group[group] = {}
if vm not in vms_by_group[group]:
vms_by_group[group][vm] = {'cpu': [], 'disk': []}

time_local = row['timestamp'].strftime('%Y-%m-%dT%H:%M:%S') + '+03:00'

vms_by_group[group][vm]['cpu'].append({'x': time_local, 'y': float(row['cpu'])})
vms_by_group[group][vm]['disk'].append({'x': time_local, 'y': row['disk_free']})

# Порядок групп: Cyber — гарантированно первым
group_order = []
if 'Cyber' in vms_by_group:
group_order.append('Cyber')
for group in vms_by_group:
if group != 'Cyber':
group_order.append(group)

# Подсчёт потоков
cursor.execute("""
SELECT vm_id, vm_group, threads
FROM (
SELECT vm_id, vm_group, threads,
ROW_NUMBER() OVER (PARTITION BY vm_id ORDER BY timestamp DESC) as rn
FROM metrics
WHERE timestamp >= NOW() - INTERVAL 20 MINUTE
) ranked
WHERE rn = 1
""")
latest_vms = cursor.fetchall()
conn.close()

group_threads = {}
for row in latest_vms:
group_raw = row['vm_group']
if not group_raw or str(group_raw).strip().lower() in ('null', 'none', ''):
group = 'Cyber'
else:
group = str(group_raw)

# --- ИЗМЕНЕНИЕ: Добавлена защита от NULL ---
threads = row['threads'] if row['threads'] is not None else 0
group_threads[group] = group_threads.get(group, 0) + threads

group_rows = [{'group': k, 'threads': v} for k, v in group_threads.items()]
group_rows.sort(key=lambda x: -x['threads'])

return render_template(
'dashboard.html',
vms_by_group=vms_by_group,
group_order=group_order,
group_rows=group_rows
)

except Exception as e:
import traceback
traceback.print_exc()
return f"<h1>Dashboard Error</h1><pre>{e}</pre>", 500

if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)

Добавлен шаблон в ./templates/

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Mega VM Monitoring Dashboard</title>
<!-- Chart.js и адаптер времени (обязательно!) -->
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
<style>
body { font-family: Arial, sans-serif; margin: 20px; background: #f8f9fa; }
.chart-container { width: 98%; height: 400px; margin-bottom: 30px; background: white; padding: 15px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
.table-container { background: white; padding: 15px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
table { width: 100%; border-collapse: collapse; }
th, td { padding: 10px; text-align: left; border-bottom: 1px solid #ddd; }
th { background-color: #f1f1f1; }
h1, h2 { color: #333; }
h1 { margin-top: 0; }
</style>
</head>
<body>
<h1>Giga VM Monitoring Dashboard</h1>

<div class="chart-container">
<h2>CPU Load (%) over Time</h2>
<canvas id="cpuChart"></canvas>
</div>

<div class="chart-container">
<h2>Free Disk Space (bytes) over Time</h2>
<canvas id="diskChart"></canvas>
</div>

<div class="table-container">
<h2>VM Groups – Total Threads</h2>
<table>
<thead>
<tr><th>VM Group</th><th>Total Threads</th></tr>
</thead>
<tbody>
{% for row in group_rows %}
<tr><td>{{ row.group }}</td><td>{{ row.threads }}</td></tr>
{% endfor %}
</tbody>
</table>
</div>

<script>
// === Вспомогательные функции для цветов ===
function interpolateColor(color1, color2, factor) {
const hex = (c) => c.toString(16).padStart(2, '0');
const r1 = parseInt(color1.slice(1, 3), 16);
const g1 = parseInt(color1.slice(3, 5), 16);
const b1 = parseInt(color1.slice(5, 7), 16);
const r2 = parseInt(color2.slice(1, 3), 16);
const g2 = parseInt(color2.slice(3, 5), 16);
const b2 = parseInt(color2.slice(5, 7), 16);

const r = Math.round(r1 + factor * (r2 - r1));
const g = Math.round(g1 + factor * (g2 - g1));
const b = Math.round(b1 + factor * (b2 - b1));

return `#${hex(r)}${hex(g)}${hex(b)}`;
}

function randomColor() {
return '#' + Math.floor(Math.random() * 16777215).toString(16).padStart(6, '0');
}

function getPaletteForGroup(index) {
const palettes = [
['#DA70D6', '#00008B'], // Cyber: orchid → dark blue
['#00FFFF', '#006400'], // cyan → dark green
['#FF0000', '#8B4513'], // red → saddlebrown
['#00FF00', '#A08000'] // lime → olive brown
];
if (index < palettes.length) {
return palettes[index];
} else {
return [randomColor(), randomColor()];
}
}

// === Форматирование байтов ===
function formatBytes(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}

// === Подготовка данных ===
// 1. Выгружаем все данные из Python в JavaScript-объекты
const vmData = {{ vms_by_group | tojson }};
const groupOrder = {{ group_order | tojson }};

// 2. Создаем пустые массивы для датасетов графиков
const cpuDatasets = [];
const diskDatasets = [];

// 3. Итерируем по данным с помощью ЧИСТОГО JavaScript
groupOrder.forEach((group, groupIndex) => {
const vmsInGroup = Object.keys(vmData[group]);
const vmCount = vmsInGroup.length;
const [startColor, endColor] = getPaletteForGroup(groupIndex);

vmsInGroup.forEach((vm, vmIndex) => {
const factor = vmCount > 1 ? vmIndex / (vmCount - 1) : 0;
const color = interpolateColor(startColor, endColor, factor);

// CPU
cpuDatasets.push({
label: vm + ' (' + group + ')',
data: vmData[group][vm]['cpu'],
borderColor: color,
backgroundColor: 'rgba(0,0,0,0)',
tension: 0.2
});

// Disk
diskDatasets.push({
label: vm + ' (' + group + ')',
data: vmData[group][vm]['disk'],
borderColor: color,
backgroundColor: 'rgba(0,0,0,0)',
tension: 0.2
});
});
});

// === CPU Chart ===
const cpuCtx = document.getElementById('cpuChart').getContext('2d');
new Chart(cpuCtx, {
type: 'line',
data: { datasets: cpuDatasets },
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: {
type: 'time',
time: {
unit: 'minute',
tooltipFormat: 'MMM d, HH:mm',
displayFormats: { minute: 'HH:mm' }
},
title: { display: true, text: 'Time (Moscow)' }
},
y: {
min: 0,
max: 100,
title: { display: true, text: 'CPU %' }
}
},
plugins: { legend: { position: 'top' } }
}
});

// === Disk Chart ===
const diskCtx = document.getElementById('diskChart').getContext('2d');
new Chart(diskCtx, {
type: 'line',
data: { datasets: diskDatasets },
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: {
type: 'time',
time: {
unit: 'minute',
tooltipFormat: 'MMM d, HH:mm',
displayFormats: { minute: 'HH:mm' }
},
title: { display: true, text: 'Time (Moscow)' }
},
y: {
title: { display: true, text: 'Free Disk' },
ticks: {
callback: function(value) {
return formatBytes(value);
}
}
}
},
plugins: { legend: { position: 'top' } }
}
});

</script>
</body>
</html>

 В итоге видим в браузере адекватные графики!

И загрузку диска

Клиентская часть:

# send-metrics.ps1

$vm_id = "VM-209-06" # Уникальный ID ВМ
$vm_profile = "" # Уникальный profile ВМ
$vm_threads = "8" # How many threads are allowed for ВМ

$server_ip = "47.82.5.187"
$api_endpoint = "http://$server_ip`:8080/metrics"

Write-Host "[+] Начинаю сбор метрик для $vm_id..." -ForegroundColor Cyan

# 1. Сбор метрик
try {
$cpu = (Get-WmiObject Win32_Processor | Measure-Object -Property LoadPercentage -Average).Average
if ($null -eq $cpu) { $cpu = 0 }

$diskObj = Get-WmiObject Win32_LogicalDisk -Filter "DeviceID='C:'"
$disk_free = if ($diskObj) { $diskObj.FreeSpace } else { 0 }

# Check if BAS process is running (regardless of window title)
$basProc = Get-Process BAS -ErrorAction SilentlyContinue
$bas_running = if ($basProc) { $true } else { $false }

# Still capture title if available (for debugging)
$bas_title = if ($basProc -and $basProc.MainWindowTitle) {
$basProc.MainWindowTitle
} else {
"BAS running (no window)" # More accurate than "not running"
}

Write-Host "[+] CPU: $cpu%, Disk free: $disk_free bytes, BAS running: $bas_running, Title: '$bas_title'" -ForegroundColor Green
} catch {
Write-Host "[-] Ошибка при сборе метрик: $($_.Exception.Message)" -ForegroundColor Red
Start-Sleep -Seconds 15
exit 1
}

# 2. Анализ логов BAS
$log_path = "C:\tmp\success.log"
$success_events = @()

if (Test-Path $log_path) {
try {
$lines = Get-Content $log_path -ErrorAction Stop | Select-Object -Last 100
$matched = $lines | Where-Object { $_ -match "ПРОГОН ЗАКОНЧЕН!!!" }
$success_events = foreach ($line in $matched) {
@{
timestamp = Get-Date -Format "o"
message = $line
}
}
Write-Host "[+] Найдено $($success_events.Count) успешных событий в логе." -ForegroundColor Green
} catch {
Write-Host "[-] Ошибка при чтении лога: $($_.Exception.Message)" -ForegroundColor Yellow
}
} else {
Write-Host "[ ] Лог-файл $log_path не найден." -ForegroundColor Gray
}

# 3. Формирование JSON — NOW INCLUDES vm_profile and vm_threads
try {
$bodyObj = @{
vm_id = $vm_id
vm_profile = $vm_profile # ← Added
vm_threads = $vm_threads # ← Added (sent as string; server can parse as int if needed)
timestamp = Get-Date -Format "o"
cpu = $cpu
disk_free = $disk_free
bas_running = $bas_running # More reliable status
bas_title = $bas_title # For debugging
success_events = $success_events
}

$body = $bodyObj | ConvertTo-Json -Depth 3 -Compress
Write-Host "[+] Сформирован JSON (первые 500 символов): $($body.Substring(0, [Math]::Min(500, $body.Length)))..." -ForegroundColor Cyan
} catch {
Write-Host "[-] Ошибка при создании JSON: $($_.Exception.Message)" -ForegroundColor Red
Start-Sleep -Seconds 15
exit 1
}

# 4. Отправка на сервер
try {
Write-Host "[+] Отправляю данные на $api_endpoint..." -ForegroundColor Cyan
$response = Invoke-RestMethod -Uri $api_endpoint -Method Post -Body $body -ContentType "application/json" -TimeoutSec 15
Write-Host "[+] Успешный ответ от сервера: $($response | ConvertTo-Json -Depth 2)" -ForegroundColor Green
} catch {
$errorMessage = $_.Exception.Message
if ($_.Exception.Response) {
$statusCode = $_.Exception.Response.StatusCode.value__
Write-Host "[-] HTTP ошибка $statusCode при отправке: $errorMessage" -ForegroundColor Red
} else {
Write-Host "[-] Ошибка подключения: $errorMessage" -ForegroundColor Red
}
}

# Пауза для ручного запуска
Write-Host "`n[=] Скрипт завершён. Пауза на 3-5 секунд..." -ForegroundColor Magenta
Start-Sleep -Seconds 1