"""Deploy preflight — prove a deploy can work BEFORE anything live is stopped. A redeploy's point of no return is ``compose down`` / ``docker stop``. An invalid compose file, an image that no longer pulls, or a port somebody else took must be caught before it, or the live site goes down and stays down. Everything that can be checked while the old containers keep serving is checked here, and every caller aborts on a blocking finding, at which point the live deployment is byte-for-byte untouched. Nothing in this module stops, removes, recreates or rewrites anything; the only side effects are image pulls/builds, which are additive by construction. The checks, in the order they run (a broken compose makes the rest meaningless): 1. **Compose validity** — :meth:`DockerService.validate_compose_file` (``docker compose config --quiet``). 2. **Image resolution** — build (when the project declares ``build:``) or pull, to completion. A resolution failure is downgraded to advisory when every referenced image is already present locally, so a locally-tagged image that was never pushed to a registry still deploys. 3. **Host port conflicts** — reuses ``ManifestApplyService._port_bound`` (the same probe behind the manifest-apply ``kind: 'port_conflict'`` blocker) and only for ports the *new* configuration wants that the *currently running* project does not already publish. Probing a port the deploy's own live container holds would fail every ordinary redeploy, so owned ports are excluded; when ownership cannot be determined the check is skipped (a false abort is worse than a missed warning). """ import logging import os import re import shutil from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Set from app.services.docker_service import DockerService logger = logging.getLogger(__name__) BLOCKING = 'error' ADVISORY = 'warning' # Finding kinds. ``port_conflict`` deliberately matches the manifest-apply # blocker kind so the two surfaces speak the same vocabulary. KIND_COMPOSE_INVALID = 'compose_invalid' KIND_IMAGE_UNRESOLVED = 'image_unresolved' KIND_BUILD_FAILED = 'build_failed' KIND_PORT_CONFLICT = 'port_conflict' KIND_DISK_LOW = 'disk_low' # Free space a pull/build needs. Under the floor the deploy is # refused — images cannot fit and a full disk also stops the panel's database # writing. Under the warning line it proceeds with a warning. DISK_BLOCK_BYTES = 512 * 1024 ** 2 DISK_WARN_BYTES = 2 * 1024 ** 3 DISK_WARN_FRACTION = 0.05 DOCKER_DATA_ROOT = '/var/lib/docker' LogFn = Optional[Callable[[str], None]] #: Appended to every blocking log line. This is the user-visible payoff: the #: deploy failed and the site is still up. UNTOUCHED = 'The running deployment was left untouched.' @dataclass(frozen=True) class PreflightFinding: """One thing the preflight noticed. ``severity`` decides whether it aborts.""" kind: str message: str severity: str = BLOCKING detail: str = '' def to_dict(self) -> Dict[str, str]: return {'kind': self.kind, 'message': self.message, 'severity': self.severity, 'detail': self.detail} @dataclass class PreflightResult: """Structured pass/fail plus the findings behind it.""" findings: List[PreflightFinding] = field(default_factory=list) checks: List[str] = field(default_factory=list) skipped: List[str] = field(default_factory=list) @property def blockers(self) -> List[PreflightFinding]: return [f for f in self.findings if f.severity == BLOCKING] @property def ok(self) -> bool: """True when nothing blocking was found — i.e. it is safe to switch over.""" return not self.blockers @property def error(self) -> Optional[str]: blockers = self.blockers return blockers[0].message if blockers else None def to_dict(self) -> Dict[str, Any]: return { 'ok': self.ok, 'checks': list(self.checks), 'skipped': list(self.skipped), 'findings': [f.to_dict() for f in self.findings], } def add(self, finding: PreflightFinding, log: LogFn = None) -> 'PreflightResult': self.findings.append(finding) prefix = 'Preflight failed' if finding.severity == BLOCKING else 'Preflight warning' suffix = f' {UNTOUCHED}' if finding.severity == BLOCKING else '' _emit(log, f'{prefix}: {finding.message}{suffix}') return self def _emit(log: LogFn, message: str) -> None: """Push one human-readable line at the caller's log sink. Never raises. Deploy jobs pass the run-log callback here, so preflight lines land in the Deploy Console next to the compose transcript. """ logger.info(message) if log is None: return try: log(message) except Exception: # pragma: no cover - a broken sink must not fail a deploy pass def _first_line(text: Any, limit: int = 400) -> str: """Condense a subprocess error blob into one readable line.""" if not text: return '' lines = [ln.strip() for ln in str(text).splitlines() if ln.strip()] if not lines: return '' joined = lines[0] if len(lines) == 1 else ' | '.join(lines[:3]) return joined[:limit] # --------------------------------------------------------------------------- # compose preflight # --------------------------------------------------------------------------- def preflight_compose_project(project_path: str, compose_file: str = None, *, resolve_images: bool = True, check_ports: bool = True, log: LogFn = None, label: str = None) -> PreflightResult: """Validate a compose project before its live containers are stopped. ``compose_file`` may name a candidate file that is not yet the project's real compose (that is how a template update validates the *next* compose while the current one keeps running). Returns a :class:`PreflightResult`; callers must abort on ``not ok`` and must not have stopped anything before calling. """ result = PreflightResult() what = label or (compose_file or 'docker-compose.yml') _emit(log, f'Preflight: validating {what} before stopping the running containers') # 0. free disk ------------------------------------------------------------ _check_disk(project_path, result, log) if not result.ok: return result # 1. compose validity ----------------------------------------------------- try: validation = DockerService.validate_compose_file(project_path, compose_file) except Exception as exc: # pragma: no cover - defensive validation = {'valid': False, 'error': str(exc)} if not validation.get('valid'): return result.add(PreflightFinding( kind=KIND_COMPOSE_INVALID, message=(f'{what} is not a valid compose file — ' f'{_first_line(validation.get("error")) or "docker compose config rejected it"}'), detail=str(validation.get('error') or ''), ), log) result.checks.append('compose_config') # The parsed config drives both remaining checks; one `docker compose # config` for both rather than one each. config = _compose_config(project_path, compose_file) # 2. image resolution ----------------------------------------------------- if resolve_images: if config is None: result.skipped.append('image_resolution') else: _resolve_images(project_path, compose_file, config, result, log) if not result.ok: return result # 3. port conflicts ------------------------------------------------------- if check_ports: if config is None: result.skipped.append('port_conflicts') else: _check_ports(project_path, config, result, log) if not result.ok: return result _emit(log, 'Preflight passed — safe to switch over') return result def _disk_usage(path: str): return shutil.disk_usage(path) def _check_disk(project_path: str, result: PreflightResult, log: LogFn = None) -> None: """Refuse a deploy the disk cannot hold; warn when it is getting close. Measured where images land (Docker's data root), else the project. Adds a finding only when space is short, so a healthy disk is silent.""" from app.services.disk_reclaim_service import human_bytes path = DOCKER_DATA_ROOT if os.path.isdir(DOCKER_DATA_ROOT) else project_path try: du = _disk_usage(path) except OSError: return warn_at = max(DISK_WARN_BYTES, int(du.total * DISK_WARN_FRACTION)) if du.free < DISK_BLOCK_BYTES: result.add(PreflightFinding( kind=KIND_DISK_LOW, message=(f'Only {human_bytes(du.free)} free on {path}: not enough to pull ' 'or build images. Free up space from Storage, then deploy again.'), detail=f'free={du.free} total={du.total}', ), log) elif du.free < warn_at: result.add(PreflightFinding( kind=KIND_DISK_LOW, severity=ADVISORY, message=(f'Only {human_bytes(du.free)} free on {path}; a large build may ' 'fail. Storage shows what is using the disk.'), detail=f'free={du.free} total={du.total}', ), log) def _compose_config(project_path: str, compose_file: str = None) -> Optional[Dict[str, Any]]: """Parsed ``docker compose config``, or None when it cannot be read. None means "unknown", and every caller of this treats unknown as fail-open: an inability to introspect must never abort a deploy that the old code would have run. """ try: parsed = DockerService.get_compose_config(project_path, compose_file) except Exception as exc: # pragma: no cover - defensive logger.debug('compose config read failed for %s: %s', project_path, exc) return None if not parsed.get('success'): return None config = parsed.get('config') return config if isinstance(config, dict) else None def _resolve_images(project_path: str, compose_file: str, config: Dict[str, Any], result: PreflightResult, log: LogFn) -> None: """Build and pull every image the project needs, to completion. Services with a ``build:`` section are built (the failure mode git deploys actually hit); the rest are pulled. Doing both before the stop is the whole point — ``compose up`` would otherwise discover the failure with nothing left serving. In a mixed project the pull is issued per service, because a whole-project ``compose pull`` also tries to pull the buildable ones and fails on images that only exist once built. """ services = config.get('services') or {} buildable = [name for name, svc in services.items() if isinstance(svc, dict) and svc.get('build')] pullable = {name: svc['image'] for name, svc in services.items() if isinstance(svc, dict) and svc.get('image') and not svc.get('build')} if buildable: _emit(log, f'Preflight: building {len(buildable)} image(s) before the switchover') try: build = DockerService.compose_build(project_path, compose_file=compose_file) except Exception as exc: # pragma: no cover - defensive build = {'success': False, 'error': str(exc)} if not build.get('success'): result.add(PreflightFinding( kind=KIND_BUILD_FAILED, message=(f'image build failed — ' f'{_first_line(build.get("error")) or "docker compose build returned a non-zero exit"}'), detail=str(build.get('error') or ''), ), log) return result.checks.append('image_build') images = sorted(set(pullable.values())) if not images: if not buildable: result.skipped.append('image_resolution') return _emit(log, f'Preflight: pulling {len(images)} image(s) before the switchover') pull = _pull_images(project_path, compose_file, list(pullable) if buildable else None) if pull.get('success'): result.checks.append('image_pull') return # A pull failure is only fatal when the image genuinely isn't available. # Locally-built/tagged images that were never pushed cannot be pulled and # deployed fine before this check existed — do not start failing them. missing = [ref for ref in images if not _image_present(ref)] if not missing: result.add(PreflightFinding( kind=KIND_IMAGE_UNRESOLVED, severity=ADVISORY, message=('could not pull images, but every one is already present ' f'locally — continuing ({_first_line(pull.get("error"))})'), detail=str(pull.get('error') or ''), ), log) result.checks.append('image_pull') return result.add(PreflightFinding( kind=KIND_IMAGE_UNRESOLVED, message=(f'image {missing[0]} could not be resolved — ' f'{_first_line(pull.get("error")) or "docker compose pull failed"}'), detail=str(pull.get('error') or ''), ), log) def _pull_images(project_path: str, compose_file: str, services: Optional[List[str]]) -> Dict[str, Any]: """``compose pull`` for the whole project, or service by service.""" targets = services if services is not None else [None] for service in targets: try: pull = DockerService.compose_pull(project_path, service=service, compose_file=compose_file) except Exception as exc: # pragma: no cover - defensive pull = {'success': False, 'error': str(exc)} if not pull.get('success'): return pull return {'success': True} def _image_present(image_ref: str) -> bool: """True when ``image_ref`` already exists in the local image store.""" try: return bool(DockerService.image_exists(image_ref)) except Exception: # pragma: no cover - defensive return False def _check_ports(project_path: str, config: Dict[str, Any], result: PreflightResult, log: LogFn) -> None: """Flag host ports the new config wants that somebody else already holds.""" desired = _desired_host_ports(config) if not desired: result.skipped.append('port_conflicts') return owned = published_host_ports(project_path) if owned is None: # Cannot tell which ports are ours; probing would flag our own live # containers and abort a perfectly good redeploy. result.skipped.append('port_conflicts') return for port in sorted(desired - owned): if _port_bound(port): result.add(PreflightFinding( kind=KIND_PORT_CONFLICT, message=(f'host port {port} is already in use by something else — ' f'free the port or publish this service on another one'), detail=f'port {port} is bound on 0.0.0.0 and is not published by this project', ), log) if result.ok: result.checks.append('port_conflicts') def _port_bound(port: int) -> bool: """Reuse of the manifest-apply port probe (its ``port_conflict`` blocker). Called through the class so the existing test seam (``ManifestApplyService._port_bound``) keeps working for both surfaces. """ try: from app.services.manifest_apply_service import ManifestApplyService return bool(ManifestApplyService._port_bound(int(port))) except Exception: # pragma: no cover - defensive; unknown means "not a conflict" return False # --------------------------------------------------------------------------- # port bookkeeping # --------------------------------------------------------------------------- _PORTS_RE = re.compile(r'(?:(?:\d{1,3}(?:\.\d{1,3}){3}|\[[^\]]+\]|::):)?(\d+)(?:-\d+)?->') def _desired_host_ports(config: Dict[str, Any]) -> Set[int]: """Host ports published by the compose config (long and short syntax).""" ports: Set[int] = set() for svc in (config.get('services') or {}).values(): if not isinstance(svc, dict): continue for entry in svc.get('ports') or []: port = (_published_from_mapping(entry) if isinstance(entry, dict) else _published_from_short(str(entry))) if port is not None: ports.add(port) return ports def _published_from_mapping(entry: Dict[str, Any]) -> Optional[int]: published = entry.get('published') if published in (None, ''): return None try: return int(str(published).split('-')[0]) except (TypeError, ValueError): return None def _published_from_short(spec: str) -> Optional[int]: """`8080:80`, `127.0.0.1:8080:80`, `8080:80/tcp` -> 8080. `80` -> None.""" spec = spec.strip().split('/')[0] parts = spec.split(':') if len(parts) < 2: return None # container-only port: docker picks the host port candidate = parts[-2] if '-' in candidate: return None # a range; fail open rather than guess try: return int(candidate) except ValueError: return None def published_host_ports(project_path: str, compose_file: str = None) -> Optional[Set[int]]: """Host ports the project's *currently running* containers publish. Returns None when it cannot be determined — callers must then skip the port check rather than assume the ports are free. """ try: # strict: a failed listing must read as "unknown", not "owns nothing" — # otherwise every redeploy flags the app's own ports as taken. containers = DockerService.compose_ps(project_path, compose_file, strict=True) except Exception as exc: # pragma: no cover - defensive logger.debug('compose ps failed for %s: %s', project_path, exc) return None if containers is None: return None ports: Set[int] = set() for container in containers: if not isinstance(container, dict): continue for publisher in container.get('Publishers') or []: if not isinstance(publisher, dict): continue try: published = int(publisher.get('PublishedPort') or 0) except (TypeError, ValueError): continue if published: ports.add(published) raw = container.get('Ports') if isinstance(raw, str): ports.update(int(m) for m in _PORTS_RE.findall(raw)) return ports # --------------------------------------------------------------------------- # single-container preflight # --------------------------------------------------------------------------- def preflight_image(image_ref: str, *, pull: Callable[[], Dict[str, Any]] = None, log: LogFn = None) -> PreflightResult: """Resolve a single container image before its predecessor is stopped. The ``docker run`` deploy path has no compose file to validate; its one pre-traffic requirement is that the image exists by the time the old container is gone. ``pull`` is supplied by the caller (it owns registry authentication); without it the image came from the build step that already ran, and only local presence is asserted. """ result = PreflightResult() if not image_ref: return result.add(PreflightFinding( kind=KIND_IMAGE_UNRESOLVED, message='no image was produced for this deployment'), log) if pull is not None: _emit(log, f'Preflight: pulling {image_ref} before the switchover') try: pulled = pull() except Exception as exc: # pragma: no cover - defensive pulled = {'success': False, 'error': str(exc)} if not pulled.get('success'): return result.add(PreflightFinding( kind=KIND_IMAGE_UNRESOLVED, message=(f'image {image_ref} could not be pulled — ' f'{_first_line(pulled.get("error")) or "registry pull failed"}'), detail=str(pulled.get('error') or ''), ), log) result.checks.append('image_pull') else: result.checks.append('image_from_build') _emit(log, f'Preflight passed — {image_ref} is resolved, safe to switch over') return result