mirror of
https://github.com/jhd3197/ServerKit.git
synced 2026-11-02 18:17:11 +09:00
513 lines
20 KiB
Python
513 lines
20 KiB
Python
"""Deploy preflight — prove a deploy can work BEFORE anything live is stopped.
|
|
|
|
A redeploy's point of no return is ``compose down`` / ``docker stop``. An
|
|
invalid compose file, an image that no longer pulls, or a port somebody else
|
|
took must be caught before it, or the live site goes down and stays down.
|
|
|
|
Everything that can be checked while the old containers keep serving is
|
|
checked here, and every caller aborts on a blocking finding, at which point the
|
|
live deployment is byte-for-byte untouched. Nothing in this module stops,
|
|
removes, recreates or rewrites anything; the only side effects are image
|
|
pulls/builds, which are additive by construction.
|
|
|
|
The checks, in the order they run (a broken compose makes the rest meaningless):
|
|
|
|
1. **Compose validity** — :meth:`DockerService.validate_compose_file`
|
|
(``docker compose config --quiet``).
|
|
2. **Image resolution** — build (when the project declares ``build:``) or pull,
|
|
to completion. A resolution failure is downgraded to advisory when every
|
|
referenced image is already present locally, so a locally-tagged image that
|
|
was never pushed to a registry still deploys.
|
|
3. **Host port conflicts** — reuses ``ManifestApplyService._port_bound`` (the
|
|
same probe behind the manifest-apply ``kind: 'port_conflict'`` blocker) and
|
|
only for ports the *new* configuration wants that the *currently running*
|
|
project does not already publish. Probing a port the deploy's own live
|
|
container holds would fail every ordinary redeploy, so owned ports are
|
|
excluded; when ownership cannot be determined the check is skipped (a false
|
|
abort is worse than a missed warning).
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, Dict, List, Optional, Set
|
|
|
|
from app.services.docker_service import DockerService
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BLOCKING = 'error'
|
|
ADVISORY = 'warning'
|
|
|
|
# Finding kinds. ``port_conflict`` deliberately matches the manifest-apply
|
|
# blocker kind so the two surfaces speak the same vocabulary.
|
|
KIND_COMPOSE_INVALID = 'compose_invalid'
|
|
KIND_IMAGE_UNRESOLVED = 'image_unresolved'
|
|
KIND_BUILD_FAILED = 'build_failed'
|
|
KIND_PORT_CONFLICT = 'port_conflict'
|
|
KIND_DISK_LOW = 'disk_low'
|
|
|
|
# Free space a pull/build needs. Under the floor the deploy is
|
|
# refused — images cannot fit and a full disk also stops the panel's database
|
|
# writing. Under the warning line it proceeds with a warning.
|
|
DISK_BLOCK_BYTES = 512 * 1024 ** 2
|
|
DISK_WARN_BYTES = 2 * 1024 ** 3
|
|
DISK_WARN_FRACTION = 0.05
|
|
DOCKER_DATA_ROOT = '/var/lib/docker'
|
|
|
|
LogFn = Optional[Callable[[str], None]]
|
|
|
|
#: Appended to every blocking log line. This is the user-visible payoff: the
|
|
#: deploy failed and the site is still up.
|
|
UNTOUCHED = 'The running deployment was left untouched.'
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PreflightFinding:
|
|
"""One thing the preflight noticed. ``severity`` decides whether it aborts."""
|
|
|
|
kind: str
|
|
message: str
|
|
severity: str = BLOCKING
|
|
detail: str = ''
|
|
|
|
def to_dict(self) -> Dict[str, str]:
|
|
return {'kind': self.kind, 'message': self.message,
|
|
'severity': self.severity, 'detail': self.detail}
|
|
|
|
|
|
@dataclass
|
|
class PreflightResult:
|
|
"""Structured pass/fail plus the findings behind it."""
|
|
|
|
findings: List[PreflightFinding] = field(default_factory=list)
|
|
checks: List[str] = field(default_factory=list)
|
|
skipped: List[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def blockers(self) -> List[PreflightFinding]:
|
|
return [f for f in self.findings if f.severity == BLOCKING]
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
"""True when nothing blocking was found — i.e. it is safe to switch over."""
|
|
return not self.blockers
|
|
|
|
@property
|
|
def error(self) -> Optional[str]:
|
|
blockers = self.blockers
|
|
return blockers[0].message if blockers else None
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return {
|
|
'ok': self.ok,
|
|
'checks': list(self.checks),
|
|
'skipped': list(self.skipped),
|
|
'findings': [f.to_dict() for f in self.findings],
|
|
}
|
|
|
|
def add(self, finding: PreflightFinding, log: LogFn = None) -> 'PreflightResult':
|
|
self.findings.append(finding)
|
|
prefix = 'Preflight failed' if finding.severity == BLOCKING else 'Preflight warning'
|
|
suffix = f' {UNTOUCHED}' if finding.severity == BLOCKING else ''
|
|
_emit(log, f'{prefix}: {finding.message}{suffix}')
|
|
return self
|
|
|
|
|
|
def _emit(log: LogFn, message: str) -> None:
|
|
"""Push one human-readable line at the caller's log sink. Never raises.
|
|
|
|
Deploy jobs pass the run-log callback here, so preflight lines land in the
|
|
Deploy Console next to the compose transcript.
|
|
"""
|
|
logger.info(message)
|
|
if log is None:
|
|
return
|
|
try:
|
|
log(message)
|
|
except Exception: # pragma: no cover - a broken sink must not fail a deploy
|
|
pass
|
|
|
|
|
|
def _first_line(text: Any, limit: int = 400) -> str:
|
|
"""Condense a subprocess error blob into one readable line."""
|
|
if not text:
|
|
return ''
|
|
lines = [ln.strip() for ln in str(text).splitlines() if ln.strip()]
|
|
if not lines:
|
|
return ''
|
|
joined = lines[0] if len(lines) == 1 else ' | '.join(lines[:3])
|
|
return joined[:limit]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# compose preflight
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def preflight_compose_project(project_path: str, compose_file: str = None, *,
|
|
resolve_images: bool = True,
|
|
check_ports: bool = True,
|
|
log: LogFn = None,
|
|
label: str = None) -> PreflightResult:
|
|
"""Validate a compose project before its live containers are stopped.
|
|
|
|
``compose_file`` may name a candidate file that is not yet the project's
|
|
real compose (that is how a template update validates the *next* compose
|
|
while the current one keeps running).
|
|
|
|
Returns a :class:`PreflightResult`; callers must abort on ``not ok`` and
|
|
must not have stopped anything before calling.
|
|
"""
|
|
result = PreflightResult()
|
|
what = label or (compose_file or 'docker-compose.yml')
|
|
_emit(log, f'Preflight: validating {what} before stopping the running containers')
|
|
|
|
# 0. free disk ------------------------------------------------------------
|
|
_check_disk(project_path, result, log)
|
|
if not result.ok:
|
|
return result
|
|
|
|
# 1. compose validity -----------------------------------------------------
|
|
try:
|
|
validation = DockerService.validate_compose_file(project_path, compose_file)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
validation = {'valid': False, 'error': str(exc)}
|
|
if not validation.get('valid'):
|
|
return result.add(PreflightFinding(
|
|
kind=KIND_COMPOSE_INVALID,
|
|
message=(f'{what} is not a valid compose file — '
|
|
f'{_first_line(validation.get("error")) or "docker compose config rejected it"}'),
|
|
detail=str(validation.get('error') or ''),
|
|
), log)
|
|
result.checks.append('compose_config')
|
|
|
|
# The parsed config drives both remaining checks; one `docker compose
|
|
# config` for both rather than one each.
|
|
config = _compose_config(project_path, compose_file)
|
|
|
|
# 2. image resolution -----------------------------------------------------
|
|
if resolve_images:
|
|
if config is None:
|
|
result.skipped.append('image_resolution')
|
|
else:
|
|
_resolve_images(project_path, compose_file, config, result, log)
|
|
if not result.ok:
|
|
return result
|
|
|
|
# 3. port conflicts -------------------------------------------------------
|
|
if check_ports:
|
|
if config is None:
|
|
result.skipped.append('port_conflicts')
|
|
else:
|
|
_check_ports(project_path, config, result, log)
|
|
if not result.ok:
|
|
return result
|
|
|
|
_emit(log, 'Preflight passed — safe to switch over')
|
|
return result
|
|
|
|
|
|
def _disk_usage(path: str):
|
|
return shutil.disk_usage(path)
|
|
|
|
|
|
def _check_disk(project_path: str, result: PreflightResult, log: LogFn = None) -> None:
|
|
"""Refuse a deploy the disk cannot hold; warn when it is getting close.
|
|
Measured where images land (Docker's data root), else the project.
|
|
Adds a finding only when space is short, so a healthy disk is silent."""
|
|
from app.services.disk_reclaim_service import human_bytes
|
|
path = DOCKER_DATA_ROOT if os.path.isdir(DOCKER_DATA_ROOT) else project_path
|
|
try:
|
|
du = _disk_usage(path)
|
|
except OSError:
|
|
return
|
|
warn_at = max(DISK_WARN_BYTES, int(du.total * DISK_WARN_FRACTION))
|
|
if du.free < DISK_BLOCK_BYTES:
|
|
result.add(PreflightFinding(
|
|
kind=KIND_DISK_LOW,
|
|
message=(f'Only {human_bytes(du.free)} free on {path}: not enough to pull '
|
|
'or build images. Free up space from Storage, then deploy again.'),
|
|
detail=f'free={du.free} total={du.total}',
|
|
), log)
|
|
elif du.free < warn_at:
|
|
result.add(PreflightFinding(
|
|
kind=KIND_DISK_LOW, severity=ADVISORY,
|
|
message=(f'Only {human_bytes(du.free)} free on {path}; a large build may '
|
|
'fail. Storage shows what is using the disk.'),
|
|
detail=f'free={du.free} total={du.total}',
|
|
), log)
|
|
|
|
|
|
def _compose_config(project_path: str, compose_file: str = None) -> Optional[Dict[str, Any]]:
|
|
"""Parsed ``docker compose config``, or None when it cannot be read.
|
|
|
|
None means "unknown", and every caller of this treats unknown as
|
|
fail-open: an inability to introspect must never abort a deploy that the
|
|
old code would have run.
|
|
"""
|
|
try:
|
|
parsed = DockerService.get_compose_config(project_path, compose_file)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
logger.debug('compose config read failed for %s: %s', project_path, exc)
|
|
return None
|
|
if not parsed.get('success'):
|
|
return None
|
|
config = parsed.get('config')
|
|
return config if isinstance(config, dict) else None
|
|
|
|
|
|
def _resolve_images(project_path: str, compose_file: str, config: Dict[str, Any],
|
|
result: PreflightResult, log: LogFn) -> None:
|
|
"""Build and pull every image the project needs, to completion.
|
|
|
|
Services with a ``build:`` section are built (the failure mode git deploys
|
|
actually hit); the rest are pulled. Doing both before the stop is the whole
|
|
point — ``compose up`` would otherwise discover the failure with nothing
|
|
left serving. In a mixed project the pull is issued per service, because a
|
|
whole-project ``compose pull`` also tries to pull the buildable ones and
|
|
fails on images that only exist once built.
|
|
"""
|
|
services = config.get('services') or {}
|
|
buildable = [name for name, svc in services.items()
|
|
if isinstance(svc, dict) and svc.get('build')]
|
|
pullable = {name: svc['image'] for name, svc in services.items()
|
|
if isinstance(svc, dict) and svc.get('image') and not svc.get('build')}
|
|
|
|
if buildable:
|
|
_emit(log, f'Preflight: building {len(buildable)} image(s) before the switchover')
|
|
try:
|
|
build = DockerService.compose_build(project_path, compose_file=compose_file)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
build = {'success': False, 'error': str(exc)}
|
|
if not build.get('success'):
|
|
result.add(PreflightFinding(
|
|
kind=KIND_BUILD_FAILED,
|
|
message=(f'image build failed — '
|
|
f'{_first_line(build.get("error")) or "docker compose build returned a non-zero exit"}'),
|
|
detail=str(build.get('error') or ''),
|
|
), log)
|
|
return
|
|
result.checks.append('image_build')
|
|
|
|
images = sorted(set(pullable.values()))
|
|
if not images:
|
|
if not buildable:
|
|
result.skipped.append('image_resolution')
|
|
return
|
|
|
|
_emit(log, f'Preflight: pulling {len(images)} image(s) before the switchover')
|
|
pull = _pull_images(project_path, compose_file,
|
|
list(pullable) if buildable else None)
|
|
if pull.get('success'):
|
|
result.checks.append('image_pull')
|
|
return
|
|
|
|
# A pull failure is only fatal when the image genuinely isn't available.
|
|
# Locally-built/tagged images that were never pushed cannot be pulled and
|
|
# deployed fine before this check existed — do not start failing them.
|
|
missing = [ref for ref in images if not _image_present(ref)]
|
|
if not missing:
|
|
result.add(PreflightFinding(
|
|
kind=KIND_IMAGE_UNRESOLVED,
|
|
severity=ADVISORY,
|
|
message=('could not pull images, but every one is already present '
|
|
f'locally — continuing ({_first_line(pull.get("error"))})'),
|
|
detail=str(pull.get('error') or ''),
|
|
), log)
|
|
result.checks.append('image_pull')
|
|
return
|
|
|
|
result.add(PreflightFinding(
|
|
kind=KIND_IMAGE_UNRESOLVED,
|
|
message=(f'image {missing[0]} could not be resolved — '
|
|
f'{_first_line(pull.get("error")) or "docker compose pull failed"}'),
|
|
detail=str(pull.get('error') or ''),
|
|
), log)
|
|
|
|
|
|
def _pull_images(project_path: str, compose_file: str,
|
|
services: Optional[List[str]]) -> Dict[str, Any]:
|
|
"""``compose pull`` for the whole project, or service by service."""
|
|
targets = services if services is not None else [None]
|
|
for service in targets:
|
|
try:
|
|
pull = DockerService.compose_pull(project_path, service=service,
|
|
compose_file=compose_file)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
pull = {'success': False, 'error': str(exc)}
|
|
if not pull.get('success'):
|
|
return pull
|
|
return {'success': True}
|
|
|
|
|
|
def _image_present(image_ref: str) -> bool:
|
|
"""True when ``image_ref`` already exists in the local image store."""
|
|
try:
|
|
return bool(DockerService.image_exists(image_ref))
|
|
except Exception: # pragma: no cover - defensive
|
|
return False
|
|
|
|
|
|
def _check_ports(project_path: str, config: Dict[str, Any],
|
|
result: PreflightResult, log: LogFn) -> None:
|
|
"""Flag host ports the new config wants that somebody else already holds."""
|
|
desired = _desired_host_ports(config)
|
|
if not desired:
|
|
result.skipped.append('port_conflicts')
|
|
return
|
|
|
|
owned = published_host_ports(project_path)
|
|
if owned is None:
|
|
# Cannot tell which ports are ours; probing would flag our own live
|
|
# containers and abort a perfectly good redeploy.
|
|
result.skipped.append('port_conflicts')
|
|
return
|
|
|
|
for port in sorted(desired - owned):
|
|
if _port_bound(port):
|
|
result.add(PreflightFinding(
|
|
kind=KIND_PORT_CONFLICT,
|
|
message=(f'host port {port} is already in use by something else — '
|
|
f'free the port or publish this service on another one'),
|
|
detail=f'port {port} is bound on 0.0.0.0 and is not published by this project',
|
|
), log)
|
|
if result.ok:
|
|
result.checks.append('port_conflicts')
|
|
|
|
|
|
def _port_bound(port: int) -> bool:
|
|
"""Reuse of the manifest-apply port probe (its ``port_conflict`` blocker).
|
|
|
|
Called through the class so the existing test seam
|
|
(``ManifestApplyService._port_bound``) keeps working for both surfaces.
|
|
"""
|
|
try:
|
|
from app.services.manifest_apply_service import ManifestApplyService
|
|
return bool(ManifestApplyService._port_bound(int(port)))
|
|
except Exception: # pragma: no cover - defensive; unknown means "not a conflict"
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# port bookkeeping
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_PORTS_RE = re.compile(r'(?:(?:\d{1,3}(?:\.\d{1,3}){3}|\[[^\]]+\]|::):)?(\d+)(?:-\d+)?->')
|
|
|
|
|
|
def _desired_host_ports(config: Dict[str, Any]) -> Set[int]:
|
|
"""Host ports published by the compose config (long and short syntax)."""
|
|
ports: Set[int] = set()
|
|
for svc in (config.get('services') or {}).values():
|
|
if not isinstance(svc, dict):
|
|
continue
|
|
for entry in svc.get('ports') or []:
|
|
port = (_published_from_mapping(entry) if isinstance(entry, dict)
|
|
else _published_from_short(str(entry)))
|
|
if port is not None:
|
|
ports.add(port)
|
|
return ports
|
|
|
|
|
|
def _published_from_mapping(entry: Dict[str, Any]) -> Optional[int]:
|
|
published = entry.get('published')
|
|
if published in (None, ''):
|
|
return None
|
|
try:
|
|
return int(str(published).split('-')[0])
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _published_from_short(spec: str) -> Optional[int]:
|
|
"""`8080:80`, `127.0.0.1:8080:80`, `8080:80/tcp` -> 8080. `80` -> None."""
|
|
spec = spec.strip().split('/')[0]
|
|
parts = spec.split(':')
|
|
if len(parts) < 2:
|
|
return None # container-only port: docker picks the host port
|
|
candidate = parts[-2]
|
|
if '-' in candidate:
|
|
return None # a range; fail open rather than guess
|
|
try:
|
|
return int(candidate)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def published_host_ports(project_path: str, compose_file: str = None) -> Optional[Set[int]]:
|
|
"""Host ports the project's *currently running* containers publish.
|
|
|
|
Returns None when it cannot be determined — callers must then skip the
|
|
port check rather than assume the ports are free.
|
|
"""
|
|
try:
|
|
# strict: a failed listing must read as "unknown", not "owns nothing" —
|
|
# otherwise every redeploy flags the app's own ports as taken.
|
|
containers = DockerService.compose_ps(project_path, compose_file, strict=True)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
logger.debug('compose ps failed for %s: %s', project_path, exc)
|
|
return None
|
|
if containers is None:
|
|
return None
|
|
|
|
ports: Set[int] = set()
|
|
for container in containers:
|
|
if not isinstance(container, dict):
|
|
continue
|
|
for publisher in container.get('Publishers') or []:
|
|
if not isinstance(publisher, dict):
|
|
continue
|
|
try:
|
|
published = int(publisher.get('PublishedPort') or 0)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if published:
|
|
ports.add(published)
|
|
raw = container.get('Ports')
|
|
if isinstance(raw, str):
|
|
ports.update(int(m) for m in _PORTS_RE.findall(raw))
|
|
return ports
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# single-container preflight
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def preflight_image(image_ref: str, *, pull: Callable[[], Dict[str, Any]] = None,
|
|
log: LogFn = None) -> PreflightResult:
|
|
"""Resolve a single container image before its predecessor is stopped.
|
|
|
|
The ``docker run`` deploy path has no compose file to validate; its one
|
|
pre-traffic requirement is that the image exists by the time the old
|
|
container is gone. ``pull`` is supplied by the caller (it owns registry
|
|
authentication); without it the image came from the build step that already
|
|
ran, and only local presence is asserted.
|
|
"""
|
|
result = PreflightResult()
|
|
if not image_ref:
|
|
return result.add(PreflightFinding(
|
|
kind=KIND_IMAGE_UNRESOLVED,
|
|
message='no image was produced for this deployment'), log)
|
|
|
|
if pull is not None:
|
|
_emit(log, f'Preflight: pulling {image_ref} before the switchover')
|
|
try:
|
|
pulled = pull()
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
pulled = {'success': False, 'error': str(exc)}
|
|
if not pulled.get('success'):
|
|
return result.add(PreflightFinding(
|
|
kind=KIND_IMAGE_UNRESOLVED,
|
|
message=(f'image {image_ref} could not be pulled — '
|
|
f'{_first_line(pulled.get("error")) or "registry pull failed"}'),
|
|
detail=str(pulled.get('error') or ''),
|
|
), log)
|
|
result.checks.append('image_pull')
|
|
else:
|
|
result.checks.append('image_from_build')
|
|
|
|
_emit(log, f'Preflight passed — {image_ref} is resolved, safe to switch over')
|
|
return result
|