ServerKit/backend/app/services/crash_loop_service.py

"""Crash-loop visibility.

A container with a restart policy that keeps dying looked "running" in the
panel: Docker restarts it each time, and nothing counted. A one-minute
builtin tick now reads each live app's containers' ``RestartCount`` and keeps
a short sample window per container. More than ``RESTARTS`` restarts within
``WINDOW`` marks the app crash-looping and opens an incident through the
monitor incident path; ``WINDOW`` without a restart resolves it.

State lives in one SystemSettings JSON row, keyed by app id, holding only
live apps and samples inside the window, so it stays bounded.
"""
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Optional

logger = logging.getLogger(__name__)

STATE_KEY = 'crash_loop_state'
RESTARTS = 3
WINDOW = timedelta(minutes=10)


def _restart_counts(app) -> Dict[str, int]:
    """``{container name: RestartCount}`` for every container of ``app``."""
    from app.services.docker_service import DockerService
    names = [c.get('name') or c.get('id') for c in DockerService.get_all_app_containers(app)]
    if not names:
        names = [n for n in (getattr(app, 'container_id', None), app.name,
                             f'serverkit-app-{app.id}') if n]
    result = DockerService.run(['inspect', '--format', '{{.Name}} {{.RestartCount}}', *names],
                               timeout=30)
    counts = {}
    # inspect prints the ones it found even when another name is missing.
    for line in (result.get('output') or '').splitlines():
        parts = line.strip().lstrip('/').rsplit(' ', 1)
        if len(parts) == 2 and parts[1].isdigit():
            counts[parts[0]] = int(parts[1])
    return counts


class CrashLoopService:
    restart_counts = staticmethod(_restart_counts)

    @classmethod
    def sweep(cls, now: Optional[datetime] = None) -> Dict:
        from app import db
        from app.models.application import Application
        from app.models.system_settings import SystemSettings

        now = now or datetime.utcnow()
        cutoff = (now - WINDOW).isoformat()
        state = SystemSettings.get(STATE_KEY) or {}
        if not isinstance(state, dict):
            state = {}
        new_state, opened, resolved = {}, [], []

        for app in Application.query_active().filter(Application.app_type == 'docker').all():
            if app.status not in ('running', 'error'):
                continue
            entry = state.get(str(app.id)) or {}
            samples = {name: [s for s in rows if s[0] >= cutoff]
                       for name, rows in (entry.get('samples') or {}).items()}
            for name, count in cls.restart_counts(app).items():
                samples.setdefault(name, []).append([now.isoformat(), count])
            restarts = max((rows[-1][1] - rows[0][1] for rows in samples.values() if rows),
                           default=0)
            looping = restarts >= RESTARTS
            last_restart = entry.get('last_restart')
            if restarts > 0:
                changed = any(len(rows) > 1 and rows[-1][1] > rows[-2][1] for rows in samples.values())
                if changed:
                    last_restart = now.isoformat()

            incident_id = entry.get('incident_id')
            if looping and not incident_id:
                incident_id = cls._open_incident(app, restarts)
                opened.append(app.id)
            elif incident_id and not looping and (not last_restart or last_restart < cutoff):
                cls._resolve_incident(incident_id)
                resolved.append(app.id)
                incident_id = None

            new_state[str(app.id)] = {
                'samples': samples, 'restarts_in_window': restarts,
                'looping': bool(incident_id), 'incident_id': incident_id,
                'last_restart': last_restart,
            }

        if new_state != state:
            SystemSettings.set(STATE_KEY, new_state, value_type='json',
                               description='Crash-loop sample windows')
            db.session.commit()
        return {'apps': len(new_state), 'opened': opened, 'resolved': resolved}

    @staticmethod
    def _open_incident(app, restarts) -> Optional[int]:
        from app.services.monitor_service import MonitorService
        try:
            incident = MonitorService.create_incident(None, {
                'title': f'{app.name} is crash-looping',
                'status': 'investigating',
                'impact': 'major',
                'body': (f'Its containers restarted {restarts} times within '
                         f'{int(WINDOW.total_seconds() // 60)} minutes. Check the app logs; '
                         'Docker keeps restarting it, so it looks running between crashes.'),
            })
            return incident.id
        except Exception as exc:  # noqa: BLE001 - visibility must not break the sweep
            logger.warning('crash-loop incident for app %s failed: %s', app.id, exc)
            return None

    @staticmethod
    def _resolve_incident(incident_id) -> None:
        from app.services.monitor_service import MonitorService
        try:
            MonitorService.update_incident(incident_id, {
                'status': 'resolved',
                'update_body': f'No restarts for {int(WINDOW.total_seconds() // 60)} minutes.',
            })
        except Exception as exc:  # noqa: BLE001
            logger.warning('crash-loop incident %s resolve failed: %s', incident_id, exc)

    @staticmethod
    def state_for(app) -> Dict:
        from app.models.system_settings import SystemSettings
        state = SystemSettings.get(STATE_KEY) or {}
        return (state.get(str(app.id)) or {}) if isinstance(state, dict) else {}