Files
ServerKit/backend/app/api/fleet_monitor.py
T
Juan Denis f6f08ae9d1 Fleet monitor + security and agent hardening
Adds cross-server Fleet Monitor endpoints and models, plus broad security and robustness improvements across agent, backend, and tooling.

Key changes:
- New fleet monitor API (endpoints for heatmap, comparison, alerts, thresholds, anomalies, forecast, search, exports, and Prometheus) and service skeleton.
- New DB models for ServerAlertThreshold and MetricAlert to support metric alerts.
- Agent discovery/registration hardening: HMAC-signed discovery requests, signature/timestamp validation, minimal discovery responses, HMAC-based unregister headers, and enforced command timeout cap in agent command handling.
- WebSocket and agent client TLS handling gated by SERVERKIT_INSECURE_TLS env var; added auth rate limiting on agent gateway.
- Docker-compose: parameterize agent image via AGENT_VERSION and avoid running as root by adding docker group access.
- Backend security & policy: tightened CSP script/style sources, request size limit, 2FA pending-token rejection on non-2FA endpoints, registration flow/guards, admin-only SSO admin routes via middleware, rate limits on sensitive endpoints, and logging for auth/2FA events.
- Database service hardening: identifier validation, avoid passing passwords on CLI (use env vars), safer subprocess usage, and docker exec handling improvements.
- CronService: command validation to prevent shell injection and run commands without shell=True; build_service and other subprocess usages updated similarly.
- Misc: .env.example notes clarified for encryption key generation, invitation token generation improved, agent registry timestamp window reduced, various logging and input validations added, frontend FleetMonitor page and styles stubbed in, and Vite/config/style updates.

Overall this commit implements fleet monitoring scaffolding and a set of security, validation, and operational improvements to make agent discovery, registration, command execution, and backend services safer and more robust.
2026-03-23 01:58:28 -04:00

199 lines
6.4 KiB
Python

"""
Fleet Monitor API
Endpoints for fleet-wide monitoring: heatmaps, comparisons,
alert thresholds, anomaly detection, capacity forecasting,
fleet search, and metrics export.
"""
import os
from flask import Blueprint, request, jsonify, Response
from flask_jwt_extended import jwt_required, get_jwt_identity
from app.services.fleet_monitor_service import fleet_monitor_service
from app.middleware.rbac import admin_required
fleet_monitor_bp = Blueprint('fleet_monitor', __name__)
# ==================== Heatmap ====================
@fleet_monitor_bp.route('/heatmap', methods=['GET'])
@jwt_required()
def get_fleet_heatmap():
"""Get latest metrics for all servers in heatmap format."""
group_id = request.args.get('group_id')
return jsonify(fleet_monitor_service.get_fleet_heatmap(group_id))
# ==================== Comparison ====================
@fleet_monitor_bp.route('/comparison', methods=['GET'])
@jwt_required()
def get_comparison():
"""Get multi-server comparison time-series."""
ids_param = request.args.get('ids', '')
server_ids = [s.strip() for s in ids_param.split(',') if s.strip()]
metric = request.args.get('metric', 'cpu')
period = request.args.get('period', '24h')
if not server_ids:
return jsonify({'error': 'ids parameter is required'}), 400
return jsonify(fleet_monitor_service.get_comparison_timeseries(server_ids, metric, period))
# ==================== Alerts ====================
@fleet_monitor_bp.route('/alerts', methods=['GET'])
@jwt_required()
def get_alerts():
"""Get metric alerts."""
status = request.args.get('status')
severity = request.args.get('severity')
server_id = request.args.get('server_id')
limit = request.args.get('limit', 50, type=int)
return jsonify(fleet_monitor_service.get_alerts(status, severity, server_id, limit))
@fleet_monitor_bp.route('/alerts/<alert_id>/acknowledge', methods=['POST'])
@jwt_required()
def acknowledge_alert(alert_id):
"""Acknowledge an active alert."""
user_id = get_jwt_identity()
success = fleet_monitor_service.acknowledge_alert(alert_id, user_id)
if not success:
return jsonify({'error': 'Cannot acknowledge alert'}), 400
return jsonify({'success': True})
@fleet_monitor_bp.route('/alerts/<alert_id>/resolve', methods=['POST'])
@jwt_required()
def resolve_alert(alert_id):
"""Resolve an alert."""
success = fleet_monitor_service.resolve_alert(alert_id)
if not success:
return jsonify({'error': 'Cannot resolve alert'}), 400
return jsonify({'success': True})
# ==================== Thresholds ====================
@fleet_monitor_bp.route('/thresholds', methods=['GET'])
@jwt_required()
def get_thresholds():
"""Get alert thresholds."""
server_id = request.args.get('server_id')
return jsonify(fleet_monitor_service.get_thresholds(server_id))
@fleet_monitor_bp.route('/thresholds', methods=['POST'])
@jwt_required()
@admin_required
def create_threshold():
"""Create or update an alert threshold."""
data = request.get_json()
result = fleet_monitor_service.upsert_threshold(data)
if 'error' in result:
return jsonify(result), 400
return jsonify(result), 201
@fleet_monitor_bp.route('/thresholds/<threshold_id>', methods=['DELETE'])
@jwt_required()
@admin_required
def delete_threshold(threshold_id):
"""Delete an alert threshold."""
success = fleet_monitor_service.delete_threshold(threshold_id)
if not success:
return jsonify({'error': 'Threshold not found'}), 404
return jsonify({'success': True})
# ==================== Anomalies ====================
@fleet_monitor_bp.route('/anomalies', methods=['GET'])
@jwt_required()
def get_anomalies():
"""Get anomaly detection results."""
server_id = request.args.get('server_id')
return jsonify(fleet_monitor_service.detect_anomalies(server_id))
# ==================== Capacity Forecast ====================
@fleet_monitor_bp.route('/forecast/<server_id>', methods=['GET'])
@jwt_required()
def get_forecast(server_id):
"""Get capacity forecast for a server."""
metric = request.args.get('metric', 'disk')
return jsonify(fleet_monitor_service.forecast_capacity(server_id, metric))
# ==================== Fleet Search ====================
@fleet_monitor_bp.route('/search', methods=['GET'])
@jwt_required()
def search_fleet():
"""Search across fleet for servers, containers, services, ports."""
query = request.args.get('q', '')
search_type = request.args.get('type', 'any')
if not query or len(query) < 2:
return jsonify({'error': 'Query must be at least 2 characters'}), 400
return jsonify(fleet_monitor_service.search_fleet(query, search_type))
# ==================== Export ====================
@fleet_monitor_bp.route('/export/csv', methods=['GET'])
@jwt_required()
def export_csv():
"""Export metrics as CSV download."""
ids_param = request.args.get('ids', '')
server_ids = [s.strip() for s in ids_param.split(',') if s.strip()]
metric = request.args.get('metric', 'cpu')
period = request.args.get('period', '24h')
if not server_ids:
return jsonify({'error': 'ids parameter is required'}), 400
csv_data = fleet_monitor_service.export_metrics_csv(server_ids, metric, period)
return Response(
csv_data,
mimetype='text/csv',
headers={'Content-Disposition': f'attachment; filename=fleet_{metric}_{period}.csv'}
)
@fleet_monitor_bp.route('/export/json', methods=['GET'])
@jwt_required()
def export_json():
"""Export metrics as JSON."""
ids_param = request.args.get('ids', '')
server_ids = [s.strip() for s in ids_param.split(',') if s.strip()]
metric = request.args.get('metric', 'cpu')
period = request.args.get('period', '24h')
if not server_ids:
return jsonify({'error': 'ids parameter is required'}), 400
return jsonify(fleet_monitor_service.get_comparison_timeseries(server_ids, metric, period))
# ==================== Prometheus ====================
@fleet_monitor_bp.route('/prometheus', methods=['GET'])
def prometheus_metrics():
"""Prometheus-compatible metrics endpoint (no JWT, uses token param)."""
token = request.args.get('token') or request.headers.get('X-Prometheus-Token')
expected = os.environ.get('PROMETHEUS_TOKEN')
if expected and token != expected:
return Response('Unauthorized', status=401)
metrics = fleet_monitor_service.get_prometheus_metrics()
return Response(metrics, mimetype='text/plain; version=0.0.4; charset=utf-8')