feat(backup): trust round 2 — restore drills, offsite/verify (plan 30)

backend/app/models/backup_policy.py

@@ -60,6 +60,13 @@ class BackupPolicy(db.Model):
     # Remote
     remote_copy = db.Column(db.Boolean, default=False, nullable=False)
 
+    # Restore-drill cadence (plan 23). 'off' | 'weekly' | 'monthly'. A single
+    # daily sweep enqueues due drills; last_drill_* is the denormalized cache
+    # that powers the "last proven restore" badge + doctor staleness check.
+    drill_cadence = db.Column(db.String(12), default='off', nullable=False)
+    last_drill_at = db.Column(db.DateTime, nullable=True)
+    last_drill_status = db.Column(db.String(24), nullable=True)  # success|failed|skipped_no_space
+
     # Hooks (optional shell snippets run before/after a backup)
     pre_backup_hook = db.Column(db.Text, nullable=True)
     post_backup_hook = db.Column(db.Text, nullable=True)
@@ -112,6 +119,9 @@ class BackupPolicy(db.Model):
             'full_every_n_days': self.full_every_n_days,
             'compression': self.compression,
             'remote_copy': self.remote_copy,
+            'drill_cadence': self.drill_cadence or 'off',
+            'last_drill_at': self.last_drill_at.isoformat() if self.last_drill_at else None,
+            'last_drill_status': self.last_drill_status,
             'pre_backup_hook': self.pre_backup_hook,
             'post_backup_hook': self.post_backup_hook,
             'last_run_at': self.last_run_at.isoformat() if self.last_run_at else None,

backend/app/models/backup_run.py

@@ -42,11 +42,39 @@ class BackupRun(db.Model):
 
     storage_path = db.Column(db.Text, nullable=True)  # local archive directory/path
     remote_key = db.Column(db.Text, nullable=True)
-    verified = db.Column(db.Boolean, default=False)
+    verified = db.Column(db.Boolean, default=False)  # remote-side evidence (legacy S3 HEAD)
+
+    # Verification ladder (plan 23). 'none' → 'listed' (tar -t readable) → 'hashed'
+    # (manifest sha256 spot-check passed) → 'drilled' (a real restore drill of a
+    # chain ending at this run succeeded). Only a drill ever sets 'drilled'.
+    verify_level = db.Column(db.String(12), nullable=False, default='none')
+    verified_at = db.Column(db.DateTime, nullable=True)
+    verify_error = db.Column(db.Text, nullable=True)
+    # sha256 of the primary archive, computed at build time (Decision 3).
+    checksum_sha256 = db.Column(db.String(64), nullable=True)
 
     error_message = db.Column(db.Text, nullable=True)
     metadata_json = db.Column(db.Text, default='{}')  # tables, files, compression, etc.
 
+    # Ordered verification levels, weakest → strongest.
+    VERIFY_LEVELS = ('none', 'listed', 'hashed', 'drilled')
+
+    @classmethod
+    def _verify_rank(cls, level):
+        try:
+            return cls.VERIFY_LEVELS.index(level or 'none')
+        except ValueError:
+            return 0
+
+    def effective_verify_level(self):
+        """Verify level mapping the legacy ``verified`` boolean forward: a run
+        with no explicit level but a remote-verified copy reads as 'listed' (the
+        remote HEAD proves it exists + size-matches) so old rows aren't 'none'."""
+        level = self.verify_level or 'none'
+        if level == 'none' and self.verified:
+            return 'listed'
+        return level
+
     def get_metadata(self):
         if not self.metadata_json:
             return {}
@@ -80,6 +108,10 @@ class BackupRun(db.Model):
             'storage_path': self.storage_path,
             'remote_key': self.remote_key,
             'verified': self.verified,
+            'verify_level': self.effective_verify_level(),
+            'verified_at': self.verified_at.isoformat() if self.verified_at else None,
+            'verify_error': self.verify_error,
+            'checksum_sha256': self.checksum_sha256,
             'error_message': self.error_message,
             'compression': meta.get('compression'),
             'tables': meta.get('tables'),

backend/app/models/restore_drill.py

@@ -0,0 +1,76 @@
+import json
+from datetime import datetime
+
+from app import db
+
+
+class RestoreDrill(db.Model):
+    """A single restore *drill* — a real restore of a policy's latest restorable
+    point into a scratch location (temp dir / scratch DB), probe-verified, then
+    torn down. Never touches the live target.
+
+    A successful drill is the strongest proof a backup is restorable: it promotes
+    the drilled :class:`BackupRun` to ``verify_level='drilled'`` and stamps the
+    policy's ``last_drill_*`` cache. Drilling the latest point implicitly proves
+    the whole chain (incrementals are useless without their base).
+
+    See docs/plans/23_BACKUP_TRUST_RESTORE_DRILLS_PLAN.md (Decision 4).
+    """
+
+    __tablename__ = 'restore_drills'
+
+    id = db.Column(db.Integer, primary_key=True)
+    policy_id = db.Column(db.Integer, db.ForeignKey('backup_policies.id'), nullable=False, index=True)
+    # The run whose restore-point (chain endpoint) was drilled.
+    run_id = db.Column(db.Integer, db.ForeignKey('backup_runs.id'), nullable=True, index=True)
+    job_id = db.Column(db.String(36), nullable=True, index=True)
+
+    # 'running' | 'success' | 'failed' | 'skipped_no_space'
+    status = db.Column(db.String(24), nullable=False, default='running')
+    trigger = db.Column(db.String(20), nullable=True)  # 'manual' | 'scheduled'
+
+    started_at = db.Column(db.DateTime, default=datetime.utcnow)
+    finished_at = db.Column(db.DateTime, nullable=True)
+    duration_seconds = db.Column(db.Integer, nullable=True)
+
+    # Space accounting (operator's own disk — raw bytes, never a fee).
+    bytes_required = db.Column(db.BigInteger, nullable=True)   # estimated extracted size
+    bytes_free = db.Column(db.BigInteger, nullable=True)       # free space at precheck
+    bytes_restored = db.Column(db.BigInteger, nullable=True)   # actually written to scratch
+
+    scratch_ref = db.Column(db.Text, nullable=True)  # scratch dir path / scratch DB name
+    probes_json = db.Column(db.Text, default='{}')   # probe results + error tail
+    error = db.Column(db.Text, nullable=True)
+
+    def get_probes(self):
+        if not self.probes_json:
+            return {}
+        try:
+            return json.loads(self.probes_json)
+        except (json.JSONDecodeError, TypeError):
+            return {}
+
+    def set_probes(self, value):
+        self.probes_json = json.dumps(value or {})
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'policy_id': self.policy_id,
+            'run_id': self.run_id,
+            'job_id': self.job_id,
+            'status': self.status,
+            'trigger': self.trigger,
+            'started_at': self.started_at.isoformat() if self.started_at else None,
+            'finished_at': self.finished_at.isoformat() if self.finished_at else None,
+            'duration_seconds': self.duration_seconds,
+            'bytes_required': self.bytes_required,
+            'bytes_free': self.bytes_free,
+            'bytes_restored': self.bytes_restored,
+            'scratch_ref': self.scratch_ref,
+            'probes': self.get_probes(),
+            'error': self.error,
+        }
+
+    def __repr__(self):
+        return f'<RestoreDrill {self.id} policy={self.policy_id} {self.status}>'

backend/app/services/backup_alert_service.py

@@ -0,0 +1,114 @@
+"""Edge-triggered restore-proof alerts (plan 23 Phase 5).
+
+Drills and verification run on a schedule, so a naive "notify on failure" would
+re-alert every sweep. These alerts are STATE-TRANSITION only: a failure fires
+once (the first sweep that flips healthy→failed for a policy) and a recovery
+fires once (the first success after a failure). The previous outcome per policy
+is remembered in SettingsService, keyed by policy id.
+
+Events (category ``backups``, admin-targeted):
+  * ``backup.drill_failed`` / ``backup.drill_recovered``
+  * ``backup.verify_failed`` / ``backup.verify_recovered``
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+_DRILL_KEY = 'backup_alert_drill_{policy_id}'
+_VERIFY_KEY = 'backup_alert_verify_{policy_id}'
+
+
+class BackupAlertService:
+
+    # ------------------------------------------------------------------ #
+    # State store (best-effort)
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _get_state(key):
+        try:
+            from app.services.settings_service import SettingsService
+            return SettingsService.get(key)
+        except Exception:  # noqa: BLE001
+            return None
+
+    @staticmethod
+    def _set_state(key, value):
+        try:
+            from app.services.settings_service import SettingsService
+            SettingsService.set(key, value)
+        except Exception as exc:  # noqa: BLE001
+            logger.debug('could not persist backup alert state %s: %s', key, exc)
+
+    @staticmethod
+    def _policy_label(policy):
+        return f'{policy.target_type}:{policy.target_id}'
+
+    @staticmethod
+    def _notify(event, data, policy=None):
+        # Carry the policy target so the notification deep-links straight to its
+        # protection panel (focus=policy:<target_type>:<target_id>, plan 33).
+        if policy is not None:
+            data = {**data, 'target_type': policy.target_type, 'target_id': policy.target_id}
+        try:
+            from app.plugins_sdk import notify
+            notify.send(event, to='admins', data=data, category='backups')
+        except Exception as exc:  # noqa: BLE001 — a notification must never break a sweep
+            logger.debug('notify %s failed: %s', event, exc)
+
+    # ------------------------------------------------------------------ #
+    # Drill outcome
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def on_drill_result(cls, policy, status, error=None):
+        """Record a drill's outcome and fire a failed/recovered notice only on a
+        state transition. ``status`` is the RestoreDrill status."""
+        outcome = 'failed' if status in ('failed', 'skipped_no_space') else 'success'
+        key = _DRILL_KEY.format(policy_id=policy.id)
+        prev = cls._get_state(key)
+        label = cls._policy_label(policy)
+        if outcome == 'failed' and prev != 'failed':
+            cls._notify('backup.drill_failed', {
+                'policy': label, 'target': label, 'status': status,
+                'error_message': (error or status or 'drill failed')[:300],
+                'message': (f'A restore drill for {label} {status}. The latest backup '
+                            'may not be restorable — open Backups → the policy to '
+                            'inspect the drill, or Monitoring → Doctor to re-run it.'),
+            }, policy=policy)
+        elif outcome == 'success' and prev == 'failed':
+            cls._notify('backup.drill_recovered', {
+                'policy': label, 'target': label,
+                'message': f'Restore drills for {label} are passing again.',
+            }, policy=policy)
+        cls._set_state(key, outcome)
+
+    # ------------------------------------------------------------------ #
+    # Verification outcome
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def on_verify_result(cls, policy, run):
+        """Record a run's verification outcome and fire on transition. A run is a
+        verification failure if it has a ``verify_error`` or its offsite deep
+        check reported a mismatch."""
+        offsite = (run.get_metadata() or {}).get('offsite') or {}
+        failed = bool(run.verify_error) or (offsite.get('deep_match') is False)
+        outcome = 'failed' if failed else 'success'
+        key = _VERIFY_KEY.format(policy_id=policy.id)
+        prev = cls._get_state(key)
+        label = cls._policy_label(policy)
+        if outcome == 'failed' and prev != 'failed':
+            detail = run.verify_error or 'The offsite copy did not match its stored checksum.'
+            cls._notify('backup.verify_failed', {
+                'policy': label, 'target': label,
+                'error_message': detail[:300],
+                'message': (f'The latest backup for {label} failed verification: {detail} '
+                            'Open Backups → the policy to inspect it.'),
+            }, policy=policy)
+        elif outcome == 'success' and prev == 'failed':
+            cls._notify('backup.verify_recovered', {
+                'policy': label, 'target': label,
+                'message': f'Backups for {label} are verifying cleanly again.',
+            }, policy=policy)
+        cls._set_state(key, outcome)

backend/app/services/backup_offsite_service.py

@@ -0,0 +1,235 @@
+"""Offsite verification hardening (plan 23 Phase 4).
+
+For remote-copy policies, prove the offsite object actually matches — not just
+"the upload call returned":
+
+  * **weekly shallow** (`backup.offsite.verify`) — HEAD + size + single-part
+    ETag/MD5 compare for the newest run of every remote-copy policy, recording
+    an honest evidence label (``checksum`` where a single-part ETag confirms
+    content, ``size_only`` where a multipart ETag can only confirm size).
+  * **monthly deep** — ONE deterministically sampled run per destination gets a
+    full download-and-hash against the STORED sha256 (Decision 3 — never a hash
+    recomputed from the same possibly-corrupt local file). Egress-bounded to one
+    object/month and the downloaded bytes are reported raw (Decision 8).
+
+Everything here is best-effort: a verification failure is recorded on the run's
+offsite evidence + surfaced to the drawer, never raised out of the sweep.
+"""
+import logging
+import os
+import random
+import shutil
+from datetime import datetime
+
+from app import db
+
+logger = logging.getLogger(__name__)
+
+OFFSITE_VERIFY_JOB_KIND = 'backup.offsite.verify'
+OFFSITE_VERIFY_SCHEDULE_NAME = 'backup-offsite-verify'
+
+
+class BackupOffsiteService:
+
+    # ------------------------------------------------------------------ #
+    # Candidate selection
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def _remote_copy_policies(cls):
+        from app.models.backup_policy import BackupPolicy
+        return BackupPolicy.query.filter_by(remote_copy=True).all()
+
+    @classmethod
+    def _newest_remote_run(cls, policy):
+        from app.models.backup_run import BackupRun
+        return (BackupRun.query
+                .filter(BackupRun.policy_id == policy.id,
+                        BackupRun.status == 'success',
+                        BackupRun.remote_key.isnot(None))
+                .order_by(BackupRun.started_at.desc()).first())
+
+    @staticmethod
+    def _is_deep_week(now):
+        """Monthly cadence on a weekly schedule: deep-check on ~1 week in 4."""
+        return (now.isocalendar()[1] % 4) == 0
+
+    @classmethod
+    def _sample_one(cls, runs, seed):
+        """Deterministically pick one run (seeded — reproducible for a given
+        seed, so the monthly egress is bounded AND testable)."""
+        if not runs:
+            return None
+        ordered = sorted(runs, key=lambda r: r.id)
+        return ordered[random.Random(seed).randrange(len(ordered))]
+
+    @staticmethod
+    def _destination_of(run):
+        """A stable key for the destination a run's offsite copy lives in. Today
+        every run shares one configured provider (one group), but the day a
+        second bucket/provider lands (plan 36) each destination samples on its
+        own — so a full backup is never left unproven because a different
+        destination happened to win the single-sample lottery."""
+        meta = run.get_metadata() or {}
+        dest = (meta.get('offsite') or {}).get('destination')
+        if dest:
+            return str(dest)
+        try:
+            from app.services.backup_cost_service import BackupCostService
+            prov = BackupCostService.configured_remote_provider()
+            if prov is not None:
+                return str(getattr(prov, 'name', None) or getattr(prov, 'id', None) or prov)
+        except Exception:  # noqa: BLE001
+            pass
+        return 'default'
+
+    @classmethod
+    def _sample_per_destination(cls, runs, seed):
+        """One deterministically sampled run per distinct destination (plan 30
+        #10 / Decision 7). A no-op vs the old one-run-total behaviour while
+        storage is single-provider; correct the day a second destination exists."""
+        groups = {}
+        for r in runs:
+            groups.setdefault(cls._destination_of(r), []).append(r)
+        picks = []
+        for dest in sorted(groups.keys()):
+            pick = cls._sample_one(groups[dest], f'{seed}:{dest}')
+            if pick is not None:
+                picks.append(pick)
+        return picks
+
+    # ------------------------------------------------------------------ #
+    # Verify handlers
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def run_offsite_verify(cls, job=None):
+        payload = (job.get_payload() if job else None) or {}
+        now = datetime.utcnow()
+        deep = payload.get('deep')
+        if deep is None:
+            deep = cls._is_deep_week(now)
+        seed = payload.get('seed', now.isocalendar()[1])
+
+        results = {'shallow': [], 'deep': []}
+        candidates = []
+        for policy in cls._remote_copy_policies():
+            run = cls._newest_remote_run(policy)
+            if not run:
+                continue
+            candidates.append(run)
+            ev = cls._shallow_verify(policy, run)
+            results['shallow'].append({'policy_id': policy.id, 'run_id': run.id,
+                                       'evidence': ev.get('evidence'),
+                                       'verified': ev.get('verified')})
+
+        if deep and candidates:
+            for pick in cls._sample_per_destination(candidates, seed):
+                deep_res = cls._deep_verify(pick)
+                results['deep'].append({'run_id': pick.id, **deep_res})
+                # A deep mismatch is a verification failure — alert on transition.
+                try:
+                    from app.models.backup_policy import BackupPolicy
+                    from app.services.backup_alert_service import BackupAlertService
+                    pol = BackupPolicy.query.get(pick.policy_id)
+                    if pol:
+                        BackupAlertService.on_verify_result(pol, pick)
+                except Exception as exc:  # noqa: BLE001
+                    logger.debug('offsite verify alert skipped: %s', exc)
+
+        db.session.commit()
+        logger.info('offsite verify: %s shallow, %s deep',
+                    len(results['shallow']), len(results['deep']))
+        return results
+
+    @classmethod
+    def _shallow_verify(cls, policy, run):
+        """HEAD + size + single-part content compare via verify_run, which records
+        the honest evidence label ('checksum' | 'size_only') on the run."""
+        from app.services.backup_policy_service import BackupPolicyService, BackupPolicyError
+        try:
+            res = BackupPolicyService.verify_run(policy, run.id)
+            return {'verified': res.get('verified'), 'evidence': res.get('evidence')}
+        except BackupPolicyError as exc:
+            return {'verified': False, 'evidence': 'error', 'error': str(exc)}
+        except Exception as exc:  # noqa: BLE001
+            logger.warning('shallow offsite verify failed for run %s: %s', run.id, exc)
+            return {'verified': False, 'evidence': 'error', 'error': str(exc)}
+
+    @classmethod
+    def _deep_verify(cls, run):
+        """Download the remote object and hash it against the STORED sha256 — the
+        only check that can promote a multipart ``size_only`` run to real content
+        proof. Egress-bounded (one object) and the bytes are reported raw."""
+        from app.services.storage_provider_service import StorageProviderService
+        from app.services.backup_service import BackupService
+        from app.services.backup_verify_service import sha256_file
+
+        meta = run.get_metadata() or {}
+        offsite = dict(meta.get('offsite') or {})
+        stored = offsite.get('our_sha256') or run.checksum_sha256
+        remote_key = run.remote_key
+        if not remote_key:
+            return {'ok': False, 'reason': 'no remote key'}
+        if not stored:
+            offsite['deep_evidence'] = 'no_stored_hash'
+            meta['offsite'] = offsite
+            run.set_metadata(meta)
+            return {'ok': False, 'reason': 'no stored sha256 to compare against'}
+
+        tmp_dir = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', 'offsite-verify')
+        os.makedirs(tmp_dir, exist_ok=True)
+        tmp_path = os.path.join(tmp_dir, f'deep-{run.id}.bin')
+        bytes_downloaded = 0
+        try:
+            dl = StorageProviderService.download_file(remote_key, tmp_path)
+            if not dl.get('success'):
+                offsite['deep_evidence'] = 'download_failed'
+                offsite['deep_error'] = dl.get('error')
+                meta['offsite'] = offsite
+                run.set_metadata(meta)
+                return {'ok': False, 'reason': dl.get('error')}
+            bytes_downloaded = dl.get('size') or (os.path.getsize(tmp_path)
+                                                  if os.path.exists(tmp_path) else 0)
+            actual = sha256_file(tmp_path)
+            match = bool(actual and actual == stored)
+            offsite['deep_evidence'] = 'deep_sha256'
+            offsite['deep_match'] = match
+            offsite['deep_bytes'] = bytes_downloaded
+            offsite['deep_checked_at'] = datetime.utcnow().isoformat() + 'Z'
+            if match:
+                # A confirmed offsite content match is at least as strong as a
+                # single-part checksum — the run's remote copy is proven.
+                run.verified = True
+                offsite['evidence'] = 'checksum'
+            meta['offsite'] = offsite
+            run.set_metadata(meta)
+            return {'ok': match, 'bytes': bytes_downloaded, 'evidence': 'deep_sha256'}
+        except Exception as exc:  # noqa: BLE001
+            logger.warning('deep offsite verify failed for run %s: %s', run.id, exc)
+            offsite['deep_evidence'] = 'error'
+            offsite['deep_error'] = str(exc)[:300]
+            meta['offsite'] = offsite
+            run.set_metadata(meta)
+            return {'ok': False, 'reason': str(exc)}
+        finally:
+            try:
+                if os.path.exists(tmp_path):
+                    os.remove(tmp_path)
+            except OSError:
+                pass
+            # keep the dir tidy
+            try:
+                if os.path.isdir(tmp_dir) and not os.listdir(tmp_dir):
+                    shutil.rmtree(tmp_dir, ignore_errors=True)
+            except OSError:
+                pass
+
+    # ------------------------------------------------------------------ #
+    # Registration
+    # ------------------------------------------------------------------ #
+
+    @classmethod
+    def register_jobs(cls):
+        from app.jobs import registry
+        registry.register(OFFSITE_VERIFY_JOB_KIND, cls.run_offsite_verify, replace=True)

backend/migrations/versions/058_backup_restore_drills.py

@@ -0,0 +1,104 @@
+"""Backup Trust — verification ladder, checksums, drill cadence + restore_drills.
+
+Adds the "restore proof" columns (plan 23):
+  * backup_runs: verify_level, verified_at, verify_error, checksum_sha256
+  * backup_policies: drill_cadence, last_drill_at, last_drill_status
+  * new restore_drills table (a scratch-mode restore, probe-verified, torn down)
+
+Backfill: existing runs keep verify_level='none' (the column default); the
+legacy `verified` boolean is preserved as-is and mapped forward in serialization
+(BackupRun.effective_verify_level), so old rows aren't silently downgraded.
+
+Idempotent: MigrationService runs _fix_missing_columns() + db.create_all() on
+boot before Alembic, so guard on the live schema.
+
+Revision ID: 058_backup_restore_drills
+Revises: 057_user_setup_snoozes
+Create Date: 2026-07-05
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '058_backup_restore_drills'
+down_revision = '057_user_setup_snoozes'
+branch_labels = None
+depends_on = None
+
+
+def _cols(inspector, table):
+    return {c['name'] for c in inspector.get_columns(table)}
+
+
+def upgrade():
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    tables = set(inspector.get_table_names())
+
+    if 'backup_runs' in tables:
+        cols = _cols(inspector, 'backup_runs')
+        if 'verify_level' not in cols:
+            op.add_column('backup_runs', sa.Column(
+                'verify_level', sa.String(length=12), nullable=False, server_default='none'))
+        if 'verified_at' not in cols:
+            op.add_column('backup_runs', sa.Column('verified_at', sa.DateTime(), nullable=True))
+        if 'verify_error' not in cols:
+            op.add_column('backup_runs', sa.Column('verify_error', sa.Text(), nullable=True))
+        if 'checksum_sha256' not in cols:
+            op.add_column('backup_runs', sa.Column('checksum_sha256', sa.String(length=64), nullable=True))
+
+    if 'backup_policies' in tables:
+        cols = _cols(inspector, 'backup_policies')
+        if 'drill_cadence' not in cols:
+            op.add_column('backup_policies', sa.Column(
+                'drill_cadence', sa.String(length=12), nullable=False, server_default='off'))
+        if 'last_drill_at' not in cols:
+            op.add_column('backup_policies', sa.Column('last_drill_at', sa.DateTime(), nullable=True))
+        if 'last_drill_status' not in cols:
+            op.add_column('backup_policies', sa.Column('last_drill_status', sa.String(length=24), nullable=True))
+
+    if 'restore_drills' not in tables:
+        op.create_table(
+            'restore_drills',
+            sa.Column('id', sa.Integer(), nullable=False),
+            sa.Column('policy_id', sa.Integer(), nullable=False),
+            sa.Column('run_id', sa.Integer(), nullable=True),
+            sa.Column('job_id', sa.String(length=36), nullable=True),
+            sa.Column('status', sa.String(length=24), nullable=False, server_default='running'),
+            sa.Column('trigger', sa.String(length=20), nullable=True),
+            sa.Column('started_at', sa.DateTime(), nullable=True),
+            sa.Column('finished_at', sa.DateTime(), nullable=True),
+            sa.Column('duration_seconds', sa.Integer(), nullable=True),
+            sa.Column('bytes_required', sa.BigInteger(), nullable=True),
+            sa.Column('bytes_free', sa.BigInteger(), nullable=True),
+            sa.Column('bytes_restored', sa.BigInteger(), nullable=True),
+            sa.Column('scratch_ref', sa.Text(), nullable=True),
+            sa.Column('probes_json', sa.Text(), nullable=True),
+            sa.Column('error', sa.Text(), nullable=True),
+            sa.ForeignKeyConstraint(['policy_id'], ['backup_policies.id']),
+            sa.ForeignKeyConstraint(['run_id'], ['backup_runs.id']),
+            sa.PrimaryKeyConstraint('id'),
+        )
+        op.create_index('ix_restore_drills_policy_id', 'restore_drills', ['policy_id'])
+        op.create_index('ix_restore_drills_run_id', 'restore_drills', ['run_id'])
+        op.create_index('ix_restore_drills_job_id', 'restore_drills', ['job_id'])
+
+
+def downgrade():
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    tables = set(inspector.get_table_names())
+
+    if 'restore_drills' in tables:
+        op.drop_table('restore_drills')
+
+    if 'backup_policies' in tables:
+        cols = _cols(inspector, 'backup_policies')
+        for col in ('drill_cadence', 'last_drill_at', 'last_drill_status'):
+            if col in cols:
+                op.drop_column('backup_policies', col)
+
+    if 'backup_runs' in tables:
+        cols = _cols(inspector, 'backup_runs')
+        for col in ('verify_level', 'verified_at', 'verify_error', 'checksum_sha256'):
+            if col in cols:
+                op.drop_column('backup_runs', col)

backend/tests/test_backup_alerts.py

@@ -0,0 +1,144 @@
+"""Plan 23 Phase 5 — edge-triggered restore-proof alerts + support-bundle section."""
+from datetime import datetime
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_alert_service import BackupAlertService
+
+
+def _capture_notify(monkeypatch):
+    import app.plugins_sdk as sdk
+    sent = []
+    monkeypatch.setattr(sdk.notify, 'send',
+                        lambda event, to=None, data=None, **kw: sent.append((event, data)))
+    return sent
+
+
+def _policy(target_id):
+    return BackupPolicyService.get_or_create_policy('files', target_id,
+                                                    target_meta={'paths': ['/x']})
+
+
+# --------------------------------------------------------------------------- #
+# Drill alerts — fire once on transition (#17)
+# --------------------------------------------------------------------------- #
+
+def test_drill_failed_fires_once_then_throttles(app, monkeypatch):
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5501)
+        BackupAlertService.on_drill_result(policy, 'failed', 'boom')
+        assert [e for e, _ in sent] == ['backup.drill_failed']
+        # A second consecutive failure does NOT re-alert.
+        BackupAlertService.on_drill_result(policy, 'failed', 'boom again')
+        assert [e for e, _ in sent] == ['backup.drill_failed']
+
+
+def test_drill_recovery_fires_after_failure(app, monkeypatch):
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5502)
+        BackupAlertService.on_drill_result(policy, 'failed', 'boom')
+        BackupAlertService.on_drill_result(policy, 'success', None)
+        assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']
+        # Staying healthy doesn't repeat the recovery notice.
+        BackupAlertService.on_drill_result(policy, 'success', None)
+        assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']
+
+
+def test_first_success_is_silent(app, monkeypatch):
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5503)
+        # A healthy-from-the-start policy never had a failure -> no recovery spam.
+        BackupAlertService.on_drill_result(policy, 'success', None)
+        assert sent == []
+
+
+def test_skipped_no_space_counts_as_failure(app, monkeypatch):
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5504)
+        BackupAlertService.on_drill_result(policy, 'skipped_no_space', 'no space')
+        assert [e for e, _ in sent] == ['backup.drill_failed']
+
+
+# --------------------------------------------------------------------------- #
+# Verify alerts (#17)
+# --------------------------------------------------------------------------- #
+
+def test_verify_failed_and_recovered(app, monkeypatch):
+    from app.models.backup_run import BackupRun
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5505)
+        bad = BackupRun(policy_id=policy.id, kind='full', status='success',
+                        verify_level='listed', verify_error='Checksum mismatch')
+        db.session.add(bad)
+        db.session.commit()
+        BackupAlertService.on_verify_result(policy, bad)
+        assert [e for e, _ in sent] == ['backup.verify_failed']
+
+        good = BackupRun(policy_id=policy.id, kind='full', status='success',
+                         verify_level='hashed')
+        db.session.add(good)
+        db.session.commit()
+        BackupAlertService.on_verify_result(policy, good)
+        assert [e for e, _ in sent] == ['backup.verify_failed', 'backup.verify_recovered']
+
+
+def test_verify_failed_on_deep_offsite_mismatch(app, monkeypatch):
+    from app.models.backup_run import BackupRun
+    with app.app_context():
+        sent = _capture_notify(monkeypatch)
+        policy = _policy(5506)
+        run = BackupRun(policy_id=policy.id, kind='full', status='success',
+                        verify_level='hashed')
+        run.set_metadata({'offsite': {'evidence': 'size_only', 'deep_match': False}})
+        db.session.add(run)
+        db.session.commit()
+        BackupAlertService.on_verify_result(policy, run)
+        assert [e for e, _ in sent] == ['backup.verify_failed']
+
+
+# --------------------------------------------------------------------------- #
+# Catalog registration (#17)
+# --------------------------------------------------------------------------- #
+
+def test_catalog_has_restore_proof_events():
+    from app.notifications import catalog
+    for key in ('backup.drill_failed', 'backup.drill_recovered',
+                'backup.verify_failed', 'backup.verify_recovered'):
+        entry = catalog.get(key)
+        assert entry is not None
+        assert entry['category'] == 'backups'
+
+
+# --------------------------------------------------------------------------- #
+# Support bundle restore_confidence section (#18)
+# --------------------------------------------------------------------------- #
+
+def test_support_bundle_restore_confidence(app):
+    from app.services.support_bundle_service import _collect_restore_confidence
+    from app.models.backup_run import BackupRun
+    with app.app_context():
+        policy = _policy(5507)
+        policy.drill_cadence = 'weekly'
+        policy.last_drill_status = 'success'
+        policy.last_drill_at = datetime.utcnow()
+        run = BackupRun(policy_id=policy.id, kind='full', status='success',
+                        verify_level='drilled')
+        run.set_metadata({'offsite': {'evidence': 'checksum', 'deep_evidence': 'deep_sha256'}})
+        db.session.add(run)
+        db.session.commit()
+
+        section = _collect_restore_confidence()
+        assert section['total'] >= 1
+        entry = next(p for p in section['policies']
+                     if p['target_type'] == 'files' and p['target_id'] == 5507)
+        assert entry['drill_cadence'] == 'weekly'
+        assert entry['latest_verify_level'] == 'drilled'
+        assert entry['last_drill_status'] == 'success'
+        assert entry['offsite_evidence'] == 'checksum'
+        # No paths / secrets leak into the section.
+        assert 'storage_path' not in entry and 'remote_key' not in entry

backend/tests/test_backup_drill.py

@@ -0,0 +1,251 @@
+"""Plan 23 Phase 2 — restore-drill engine proving tests.
+
+End-to-end files drill on tmp dirs (real archive, cross-platform), a database
+drill against a stubbed engine (asserts create -> restore -> probe -> drop call
+order), the no-space skip path, teardown/record-on-exception, and single-flight
+refusal."""
+import io
+import os
+import tarfile
+from datetime import datetime
+
+import pytest
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_drill_service import BackupDrillService, BackupDrillError
+from app.services import backup_verify_service as bv
+
+
+class _FakeJob:
+    def __init__(self, payload, jid='job-drill-1'):
+        self.id = jid
+        self._payload = payload
+
+    def get_payload(self):
+        return self._payload
+
+
+def _write_tar_gz_tree(dest, files):
+    with tarfile.open(dest, 'w:gz') as tar:
+        for name, content in files.items():
+            data = io.BytesIO(content)
+            info = tarfile.TarInfo(name)
+            info.size = len(content)
+            tar.addfile(info, data)
+
+
+def _make_files_run(policy, tmp_path, name='run', files=None):
+    """A successful files BackupRun with a real .tar.gz + manifest on disk."""
+    from app.models.backup_run import BackupRun
+    run_dir = tmp_path / name
+    run_dir.mkdir()
+    primary = str(run_dir / 'files.tar.gz')
+    _write_tar_gz_tree(primary, files if files is not None else {'a.txt': b'alpha', 'b.txt': b'beta'})
+    run = BackupRun(policy_id=policy.id, kind='full', status='success',
+                    storage_path=str(run_dir), started_at=datetime.utcnow())
+    meta = {'engine': 'files', 'kind': 'full', 'primary_archive': primary,
+            'incremental': False}
+    run.set_metadata(meta)
+    db.session.add(run)
+    db.session.commit()
+    bv.write_manifest(run, meta)
+    run.set_metadata(meta)
+    db.session.commit()
+    return run
+
+
+# --------------------------------------------------------------------------- #
+# End-to-end files drill (#9)
+# --------------------------------------------------------------------------- #
+
+def test_files_drill_success_promotes_run_and_stamps_policy(app, tmp_path, monkeypatch):
+    from app.services.backup_service import BackupService
+    from app.models.backup_run import BackupRun
+    from app.models.backup_policy import BackupPolicy
+    monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+    os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy(
+            'files', 8201, target_meta={'paths': ['/etc/x'], 'label': 'configs'})
+        run = _make_files_run(policy, tmp_path)
+
+        BackupDrillService.run_restore_drill(_FakeJob(
+            {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'manual'}))
+
+        from app.models.restore_drill import RestoreDrill
+        drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+        assert drill.status == 'success'
+        assert drill.get_probes()['file_count'] == 2
+        assert drill.get_probes()['sampled_ok'] == drill.get_probes()['sampled']
+
+        # The drilled run is promoted; the policy cache is stamped.
+        assert BackupRun.query.get(run.id).verify_level == 'drilled'
+        pol = BackupPolicy.query.get(policy.id)
+        assert pol.last_drill_status == 'success'
+        assert pol.last_drill_at is not None
+
+        # Scratch is torn down on success.
+        scratch = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', f'drill-{drill.id}')
+        assert not os.path.exists(scratch)
+
+
+# --------------------------------------------------------------------------- #
+# Database drill call order against a stubbed engine (#9)
+# --------------------------------------------------------------------------- #
+
+def test_database_drill_create_restore_probe_drop_order(app, tmp_path, monkeypatch):
+    from app.services.backup_service import BackupService
+    from app.services import database_service as dbsvc
+    from app.models.backup_run import BackupRun
+    from app.models.backup_policy import BackupPolicy
+    monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+
+    calls = []
+    monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',
+                        staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))
+    monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_get_tables',
+                        staticmethod(lambda name, **kw: calls.append(('tables', name)) or ['wp_posts', 'wp_users']))
+    monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',
+                        staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))
+    monkeypatch.setattr(BackupService, 'restore_database',
+                        staticmethod(lambda **kw: calls.append(('restore', kw['db_name'])) or {'success': True}))
+
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy(
+            'database', 8202, target_subtype='mysql', target_meta={'db_name': 'shop'})
+        dump = tmp_path / 'shop.sql.gz'
+        dump.write_bytes(b'dummy')
+        run = BackupRun(policy_id=policy.id, kind='full', status='success',
+                        storage_path=str(dump))
+        run.set_metadata({'engine': 'database', 'primary_archive': str(dump),
+                          'db_type': 'mysql', 'db_name': 'shop'})
+        db.session.add(run)
+        db.session.commit()
+
+        scratch_name = f'skdrill_{run.id}'
+        BackupDrillService.run_restore_drill(_FakeJob(
+            {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'scheduled'}))
+
+        kinds = [c[0] for c in calls]
+        assert kinds == ['create', 'restore', 'tables', 'drop']
+        # All operate on the scratch DB, never the live 'shop'.
+        assert all(c[1] == scratch_name for c in calls)
+        from app.models.restore_drill import RestoreDrill
+        drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+        assert drill.status == 'success'
+        assert drill.get_probes()['table_count'] == 2
+
+
+def test_database_drill_drops_scratch_even_on_restore_failure(app, tmp_path, monkeypatch):
+    from app.services.backup_service import BackupService
+    from app.services import database_service as dbsvc
+    from app.models.backup_run import BackupRun
+    monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+
+    calls = []
+    monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',
+                        staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))
+    monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',
+                        staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))
+    monkeypatch.setattr(BackupService, 'restore_database',
+                        staticmethod(lambda **kw: {'success': False, 'error': 'boom'}))
+
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy(
+            'database', 8203, target_subtype='mysql', target_meta={'db_name': 'shop'})
+        dump = tmp_path / 'shop2.sql.gz'
+        dump.write_bytes(b'dummy')
+        run = BackupRun(policy_id=policy.id, kind='full', status='success',
+                        storage_path=str(dump))
+        run.set_metadata({'engine': 'database', 'primary_archive': str(dump),
+                          'db_type': 'mysql', 'db_name': 'shop'})
+        db.session.add(run)
+        db.session.commit()
+
+        scratch_name = f'skdrill_{run.id}'
+        with pytest.raises(Exception):
+            BackupDrillService.run_restore_drill(_FakeJob(
+                {'policy_id': policy.id, 'run_id': run.id}))
+
+        # Scratch DB was created then dropped despite the restore failing.
+        assert ('create', scratch_name) in calls
+        assert ('drop', scratch_name) in calls
+        from app.models.restore_drill import RestoreDrill
+        drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+        assert drill.status == 'failed'
+        assert 'boom' in (drill.error or '')
+
+
+# --------------------------------------------------------------------------- #
+# No-space skip (#9)
+# --------------------------------------------------------------------------- #
+
+def test_drill_skipped_no_space(app, tmp_path, monkeypatch):
+    from app.services.backup_service import BackupService
+    from app.models.backup_policy import BackupPolicy
+    monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+    os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+    # Huge requirement, tiny free space -> loud skip, never a silent pass.
+    monkeypatch.setattr(BackupDrillService, '_estimate_required', classmethod(lambda cls, r, m: 10 ** 12))
+    monkeypatch.setattr(BackupDrillService, '_free_bytes', staticmethod(lambda p: 1024))
+
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 8204,
+                                                          target_meta={'paths': ['/x']})
+        run = _make_files_run(policy, tmp_path)
+        result = BackupDrillService.run_restore_drill(_FakeJob(
+            {'policy_id': policy.id, 'run_id': run.id}))
+
+        assert result['status'] == 'skipped_no_space'
+        from app.models.restore_drill import RestoreDrill
+        drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+        assert drill.status == 'skipped_no_space'
+        assert BackupPolicy.query.get(policy.id).last_drill_status == 'skipped_no_space'
+
+
+# --------------------------------------------------------------------------- #
+# Teardown / record on exception (#9)
+# --------------------------------------------------------------------------- #
+
+def test_drill_records_failure_on_empty_extract(app, tmp_path, monkeypatch):
+    from app.services.backup_service import BackupService
+    from app.models.backup_run import BackupRun
+    monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+    os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 8205,
+                                                          target_meta={'paths': ['/x']})
+        # An empty archive -> the drill must fail loudly (0 files restored).
+        run = _make_files_run(policy, tmp_path, files={})
+        with pytest.raises(Exception):
+            BackupDrillService.run_restore_drill(_FakeJob(
+                {'policy_id': policy.id, 'run_id': run.id}))
+
+        from app.models.restore_drill import RestoreDrill
+        drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+        assert drill.status == 'failed'
+        assert drill.error
+        # The run is NOT promoted to drilled on a failed drill.
+        assert BackupRun.query.get(run.id).verify_level != 'drilled'
+
+
+# --------------------------------------------------------------------------- #
+# Single-flight refusal (#9)
+# --------------------------------------------------------------------------- #
+
+def test_single_flight_refusal(app, tmp_path):
+    from app.models.restore_drill import RestoreDrill
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 8206,
+                                                          target_meta={'paths': ['/x']})
+        _make_files_run(policy, tmp_path)
+        # A running drill already exists -> request is refused.
+        db.session.add(RestoreDrill(policy_id=policy.id, status='running',
+                                    started_at=datetime.utcnow()))
+        db.session.commit()
+        assert BackupDrillService.is_drilling() is True
+        with pytest.raises(BackupDrillError):
+            BackupDrillService.request_drill(policy)

backend/tests/test_backup_drill_cadence.py

@@ -0,0 +1,154 @@
+"""Plan 23 Phase 3 — cadence sweep, badge serialization, doctor check matrix."""
+from datetime import datetime, timedelta
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_drill_service import BackupDrillService
+
+
+def _policy_with_run(target_id, cadence='weekly', enabled=True, last_drill_at=None,
+                     last_drill_status=None):
+    from app.models.backup_run import BackupRun
+    policy = BackupPolicyService.get_or_create_policy('files', target_id,
+                                                      target_meta={'paths': ['/x']})
+    policy.enabled = enabled
+    policy.drill_cadence = cadence
+    policy.last_drill_at = last_drill_at
+    policy.last_drill_status = last_drill_status
+    db.session.add(BackupRun(policy_id=policy.id, kind='full', status='success',
+                             started_at=datetime.utcnow()))
+    db.session.commit()
+    return policy
+
+
+# --------------------------------------------------------------------------- #
+# Cadence due-selection (#13)
+# --------------------------------------------------------------------------- #
+
+def test_is_due_matrix(app):
+    with app.app_context():
+        now = datetime.utcnow()
+        # never drilled + cadenced + enabled -> due
+        p_never = _policy_with_run(7301, cadence='weekly', last_drill_at=None)
+        assert BackupDrillService.is_due(p_never, now) is True
+
+        # drilled recently -> not due
+        p_fresh = _policy_with_run(7302, cadence='weekly',
+                                   last_drill_at=now - timedelta(days=1))
+        assert BackupDrillService.is_due(p_fresh, now) is False
+
+        # drilled long ago (> weekly interval incl. jitter) -> due
+        p_old = _policy_with_run(7303, cadence='weekly',
+                                 last_drill_at=now - timedelta(days=30))
+        assert BackupDrillService.is_due(p_old, now) is True
+
+        # cadence off -> never due
+        p_off = _policy_with_run(7304, cadence='off', last_drill_at=None)
+        assert BackupDrillService.is_due(p_off, now) is False
+
+        # disabled -> never due
+        p_dis = _policy_with_run(7305, cadence='weekly', enabled=False, last_drill_at=None)
+        assert BackupDrillService.is_due(p_dis, now) is False
+
+
+def test_due_policies_and_sweep_enqueues(app, monkeypatch):
+    enqueued = []
+    monkeypatch.setattr(BackupDrillService, 'request_drill',
+                        classmethod(lambda cls, policy, trigger='scheduled':
+                                    enqueued.append(policy.id) or type('J', (), {'id': f'j{policy.id}'})()))
+    monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: False))
+    with app.app_context():
+        due_policy = _policy_with_run(7311, cadence='weekly', last_drill_at=None)  # due
+        fresh_policy = _policy_with_run(7312, cadence='weekly',
+                                        last_drill_at=datetime.utcnow() - timedelta(hours=1))  # fresh
+        off_policy = _policy_with_run(7313, cadence='off', last_drill_at=None)   # off
+        due = {p.id for p in BackupDrillService.due_policies()}
+        assert due_policy.id in due
+        assert fresh_policy.id not in due and off_policy.id not in due
+
+        result = BackupDrillService.run_drill_sweep()
+        assert due_policy.id in enqueued
+        assert result['count'] == len(enqueued)
+
+
+def test_sweep_defers_when_a_drill_is_in_flight(app, monkeypatch):
+    monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: True))
+    called = []
+    monkeypatch.setattr(BackupDrillService, 'request_drill',
+                        classmethod(lambda cls, policy, trigger='scheduled': called.append(policy.id)))
+    with app.app_context():
+        _policy_with_run(7321, cadence='weekly', last_drill_at=None)
+        result = BackupDrillService.run_drill_sweep()
+        assert result['count'] == 0
+        assert called == []  # deferred — nothing enqueued while one is running
+
+
+# --------------------------------------------------------------------------- #
+# Badge serialization states (#13)
+# --------------------------------------------------------------------------- #
+
+def test_badge_states(app):
+    with app.app_context():
+        now = datetime.utcnow()
+        never = _policy_with_run(7331, cadence='weekly', last_drill_at=None)
+        assert BackupPolicyService._restore_proof_view(never)['badge'] == 'never'
+
+        ok = _policy_with_run(7332, cadence='weekly', last_drill_status='success',
+                              last_drill_at=now - timedelta(days=1))
+        assert BackupPolicyService._restore_proof_view(ok)['badge'] == 'ok'
+
+        stale = _policy_with_run(7333, cadence='weekly', last_drill_status='success',
+                                 last_drill_at=now - timedelta(days=60))
+        assert BackupPolicyService._restore_proof_view(stale)['badge'] == 'stale'
+
+        failed = _policy_with_run(7334, cadence='weekly', last_drill_status='failed',
+                                  last_drill_at=now - timedelta(days=1))
+        assert BackupPolicyService._restore_proof_view(failed)['badge'] == 'failed'
+
+
+def test_serialize_policy_view_includes_restore_proof(app):
+    with app.app_context():
+        policy = _policy_with_run(7341, cadence='monthly', last_drill_status='success',
+                                  last_drill_at=datetime.utcnow())
+        view = BackupPolicyService.serialize_policy_view(policy)
+        assert 'restore_proof' in view
+        proof = view['restore_proof']
+        assert proof['drill_cadence'] == 'monthly'
+        assert proof['badge'] == 'ok'
+        assert 'recent_drills' in proof
+
+
+# --------------------------------------------------------------------------- #
+# Doctor check matrix (#13)
+# --------------------------------------------------------------------------- #
+
+def test_doctor_backup_proof_matrix(app):
+    from app.services.doctor_service import DoctorService
+    with app.app_context():
+        now = datetime.utcnow()
+        _policy_with_run(7351, cadence='weekly', last_drill_status='success',
+                         last_drill_at=now - timedelta(days=1))          # fresh -> ok
+        _policy_with_run(7352, cadence='weekly', last_drill_status='success',
+                         last_drill_at=now - timedelta(days=60))         # stale -> warn
+        _policy_with_run(7353, cadence='weekly', last_drill_status='failed',
+                         last_drill_at=now - timedelta(days=1))          # failed -> fail
+        _policy_with_run(7354, cadence='off', last_drill_at=None)        # no cadence -> no drill check
+
+        checks = DoctorService._backup_proof_checks()
+        by_key = {c['key']: c for c in checks}
+
+        # Locate the drill checks by policy.
+        from app.models.backup_policy import BackupPolicy
+        pid = {tid: BackupPolicy.query.filter_by(target_type='files', target_id=tid).first().id
+               for tid in (7351, 7352, 7353, 7354)}
+        assert by_key[f'backup_drill_stale.{pid[7351]}']['status'] == 'ok'
+        assert by_key[f'backup_drill_stale.{pid[7352]}']['status'] == 'warn'
+        assert by_key[f'backup_drill_stale.{pid[7353]}']['status'] == 'fail'
+        assert by_key[f'backup_drill_stale.{pid[7353]}']['repairable'] is True
+        # A no-cadence policy has no drill-stale check.
+        assert f'backup_drill_stale.{pid[7354]}' not in by_key
+
+        # Every policy with a successful run gets an unverified check; the runs
+        # here were created verify_level='none' -> warn + repairable.
+        assert by_key[f'backup_unverified.{pid[7351]}']['status'] == 'warn'
+        assert by_key[f'backup_unverified.{pid[7351]}']['repair_ref']['kind'] == 'backup_verify'

backend/tests/test_backup_verify.py

@@ -0,0 +1,203 @@
+"""Plan 23 Phase 1 — Tier-1 backup verification proving tests.
+
+Covers the manifest writer + post-run verifier (backup_verify_service): manifest
+shape, corrupted-archive detection (flip a byte -> verify_error), incremental
+chain metadata intact, and legacy-run serialization compatibility. All tests use
+real (small) tar archives so they run cross-platform (the verifier falls back to
+Python's tarfile where GNU tar is unavailable, e.g. Windows dev)."""
+import os
+import tarfile
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services import backup_verify_service as bv
+
+
+def _mk_run(policy_id, storage_path, primary, kind='full', meta_extra=None):
+    from app.models.backup_run import BackupRun
+    run = BackupRun(policy_id=policy_id, kind=kind, status='verifying',
+                    storage_path=storage_path)
+    meta = {'engine': 'files', 'kind': kind, 'primary_archive': primary,
+            'incremental': False}
+    if meta_extra:
+        meta.update(meta_extra)
+    run.set_metadata(meta)
+    db.session.add(run)
+    db.session.commit()
+    return run, meta
+
+
+def _write_tar_gz(dest, payload=b'hello world'):
+    """A tiny valid .tar.gz containing one file."""
+    import io
+    with tarfile.open(dest, 'w:gz') as tar:
+        data = io.BytesIO(payload)
+        info = tarfile.TarInfo('data.txt')
+        info.size = len(payload)
+        tar.addfile(info, data)
+
+
+def _write_tar_plain(dest, payload=b'A' * 1024):
+    """An uncompressed .tar — flipping a content byte stays listable but changes
+    the sha256, so it isolates the checksum spot-check from the readability one."""
+    import io
+    with tarfile.open(dest, 'w') as tar:
+        data = io.BytesIO(payload)
+        info = tarfile.TarInfo('data.bin')
+        info.size = len(payload)
+        tar.addfile(info, data)
+
+
+# --------------------------------------------------------------------------- #
+# Manifest shape (#5)
+# --------------------------------------------------------------------------- #
+
+def test_manifest_shape(app, tmp_path):
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 9101,
+                                                          target_meta={'paths': ['/x']})
+        run_dir = tmp_path / 'run1'
+        run_dir.mkdir()
+        primary = str(run_dir / 'files.tar.gz')
+        _write_tar_gz(primary)
+        # a second artifact to prove the manifest lists everything
+        (run_dir / 'notes.txt').write_text('side file')
+
+        run, meta = _mk_run(policy.id, str(run_dir), primary)
+        manifest = bv.write_manifest(run, meta)
+
+        assert manifest['version'] == bv.MANIFEST_VERSION
+        assert manifest['primary_archive'] == 'files.tar.gz'
+        assert manifest['primary_sha256'] and len(manifest['primary_sha256']) == 64
+        names = {a['name'] for a in manifest['artifacts']}
+        assert names == {'files.tar.gz', 'notes.txt'}
+        for art in manifest['artifacts']:
+            assert art['sha256'] and len(art['sha256']) == 64
+            assert art['size'] >= 0
+        assert manifest['totals']['count'] == 2
+        assert manifest['chain'] == {'incremental': False, 'full_run_id': None}
+        # sha256 is denormalized onto the run row.
+        assert run.checksum_sha256 == manifest['primary_sha256']
+        # manifest.json actually lands next to the archives.
+        assert os.path.exists(run_dir / bv.MANIFEST_NAME)
+
+
+# --------------------------------------------------------------------------- #
+# Good archive -> hashed (#5)
+# --------------------------------------------------------------------------- #
+
+def test_verify_promotes_good_archive_to_hashed(app, tmp_path):
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 9102,
+                                                          target_meta={'paths': ['/x']})
+        run_dir = tmp_path / 'run2'
+        run_dir.mkdir()
+        primary = str(run_dir / 'files.tar.gz')
+        _write_tar_gz(primary)
+        run, meta = _mk_run(policy.id, str(run_dir), primary)
+        bv.write_manifest(run, meta)
+
+        probes = bv.verify_run_tier1(run, meta)
+        assert run.verify_level == 'hashed'
+        assert run.verify_error is None
+        assert run.verified_at is not None
+        assert probes['hashed'] is True
+        assert probes['listed'][0]['ok'] is True
+
+
+# --------------------------------------------------------------------------- #
+# Corrupted archive -> verify_error, not 'hashed' (#5)
+# --------------------------------------------------------------------------- #
+
+def test_corrupted_archive_flags_checksum_mismatch(app, tmp_path):
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 9103,
+                                                          target_meta={'paths': ['/x']})
+        run_dir = tmp_path / 'run3'
+        run_dir.mkdir()
+        primary = str(run_dir / 'files.tar')  # uncompressed: stays listable
+        _write_tar_plain(primary)
+        run, meta = _mk_run(policy.id, str(run_dir), primary)
+        bv.write_manifest(run, meta)  # records the good sha256
+
+        # Flip a byte inside the file-content region (offset well past the header).
+        with open(primary, 'r+b') as f:
+            f.seek(512 + 256)
+            b = f.read(1)
+            f.seek(512 + 256)
+            f.write(bytes([b[0] ^ 0xFF]))
+
+        probes = bv.verify_run_tier1(run, meta)
+        # The archive is still readable (listing ok) but the hash no longer matches.
+        assert run.verify_level == 'listed'
+        assert run.verify_error and 'mismatch' in run.verify_error.lower()
+        assert probes['hashed'] is False
+
+
+def test_unreadable_archive_flags_verify_error(app, tmp_path):
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 9104,
+                                                          target_meta={'paths': ['/x']})
+        run_dir = tmp_path / 'run4'
+        run_dir.mkdir()
+        primary = str(run_dir / 'files.tar.gz')
+        # Not a real gzip/tar at all -> listing must fail.
+        with open(primary, 'wb') as f:
+            f.write(b'this is not a tar archive at all')
+        run, meta = _mk_run(policy.id, str(run_dir), primary)
+        bv.write_manifest(run, meta)
+
+        bv.verify_run_tier1(run, meta)
+        assert run.verify_level == 'none'
+        assert run.verify_error and 'readable' in run.verify_error.lower()
+
+
+# --------------------------------------------------------------------------- #
+# Incremental chain metadata intact (#5)
+# --------------------------------------------------------------------------- #
+
+def test_incremental_chain_metadata_intact(app, tmp_path):
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('application', 9105)
+        run_dir = tmp_path / 'incr'
+        run_dir.mkdir()
+        primary = str(run_dir / 'files.tar.gz')
+        _write_tar_gz(primary)
+        run, meta = _mk_run(policy.id, str(run_dir), primary, kind='incremental',
+                            meta_extra={'incremental': True, 'full_run_id': 4242})
+        manifest = bv.write_manifest(run, meta)
+        assert manifest['chain'] == {'incremental': True, 'full_run_id': 4242}
+        assert manifest['kind'] == 'incremental'
+
+
+# --------------------------------------------------------------------------- #
+# Legacy-run serialization compatibility (#5)
+# --------------------------------------------------------------------------- #
+
+def test_legacy_run_serialization_maps_forward(app):
+    from app.models.backup_run import BackupRun
+    with app.app_context():
+        policy = BackupPolicyService.get_or_create_policy('files', 9106,
+                                                          target_meta={'paths': ['/x']})
+        # A pre-plan-23 run: verify_level defaulted to 'none' but it had a
+        # remote-verified copy — must read forward as 'listed', not 'none'.
+        legacy = BackupRun(policy_id=policy.id, kind='full', status='success',
+                           verified=True)
+        db.session.add(legacy)
+        db.session.commit()
+        assert legacy.verify_level == 'none'           # stored value untouched
+        assert legacy.effective_verify_level() == 'listed'
+        assert legacy.to_dict()['verify_level'] == 'listed'
+
+        # A run explicitly hashed serializes as hashed.
+        fresh = BackupRun(policy_id=policy.id, kind='full', status='success',
+                          verify_level='hashed')
+        db.session.add(fresh)
+        db.session.commit()
+        assert fresh.to_dict()['verify_level'] == 'hashed'
+
+        # A never-verified local-only run stays 'none'.
+        plain = BackupRun(policy_id=policy.id, kind='full', status='success')
+        db.session.add(plain)
+        db.session.commit()
+        assert plain.to_dict()['verify_level'] == 'none'

docs/BACKUP_PROTECTION.md

@@ -81,6 +81,50 @@ Open a backup from the history → **Restore** → choose the scope and confirm:
 
 Restore runs as a `restore.run` job; you're notified on completion.
 
+## How restore proof works
+
+A backup you have never restored is an assumption, not a safety net. ServerKit
+records how strongly each backup has been *proven*, on a ladder:
+
+| Level | What it means |
+|-------|---------------|
+| `none` | Produced, nothing checked yet. |
+| `listed` | Every archive is readable end to end (`tar -t`). |
+| `hashed` | The primary archive still matches the sha256 recorded in its `manifest.json` — catches silent corruption. |
+| `drilled` | A real restore of this point actually succeeded (strongest proof). |
+
+`listed` and `hashed` run automatically at the end of **every** backup (inside
+the existing `verifying` window). Each run writes a `manifest.json` next to its
+archives — file list, per-artifact sha256, chain refs, tool versions — and the
+primary archive's hash is stored on the run.
+
+A **restore drill** earns `drilled`. On a per-policy cadence (**Off / Weekly /
+Monthly**, set on the Schedule card) — or on demand with **Run drill now** — the
+latest restorable point is restored into a throwaway scratch location (a
+`restores/drill-<id>/` dir for files/apps/WordPress, a `skdrill_<run_id>`
+scratch database for databases), probe-verified (file count + bytes + a hashed
+sample; table listing for databases), then torn down. It never touches the live
+target. Drilling the *latest* point proves the whole incremental chain by
+construction. A free-space precheck marks an under-provisioned drill
+`skipped_no_space` rather than failing mid-extract, and only one drill runs at a
+time per host.
+
+The Protection panel shows a **Last proven restore** badge (never / ok / stale /
+failed) and the Doctor raises a repairable warning when a cadenced policy is
+overdue (> 1.5× its cadence) or its latest backup is unverified.
+
+For **remote copies**, a weekly check compares the offsite object's size and —
+for single-part uploads — its ETag/MD5; multipart uploads are honestly labelled
+`size_only` until a **monthly deep check** downloads one sampled object and
+hashes it against the *stored* sha256 (never a hash recomputed from the local
+file, which could share the same corruption). Egress stays bounded to one object
+per month.
+
+Drill/verify failures and recoveries notify admins **once per transition**
+(`backup.drill_failed` / `backup.verify_failed` and their recovery notices), not
+on every sweep. The diagnostic support bundle includes a scrubbed
+`restore_confidence` summary (levels + states only, no paths or secrets).
+
 ## Under the hood
 
 - **Models**: `BackupPolicy` (one per target) and `BackupRun` (one per run, the
@@ -89,6 +133,10 @@ Restore runs as a `restore.run` job; you're notified on completion.
 - **Services**: `backup_policy_service.py` (policy CRUD, schedule wiring, the
   `backup.policy.run` / `restore.run` job handlers, retention),
   `backup_cost_service.py` (cost), and the smart-backup helpers in
-  `backup_service.py`.
+  `backup_service.py`. Restore proof lives in `backup_verify_service.py`
+  (manifest + tier-1 verifier), `backup_drill_service.py` (`backup.drill.run` +
+  cadence/orphan sweeps, `RestoreDrill` model), `backup_offsite_service.py`
+  (`backup.offsite.verify`), and `backup_alert_service.py` (edge-triggered
+  drill/verify notices).
 - **Scheduling**: the policy's cron is mirrored into a `ScheduledJob` on the
   unified job bus, so firing is observable and retryable like any other job.

frontend/src/components/backups/BackupHistoryList.jsx

@@ -1,7 +1,11 @@
+import { useState } from 'react';
 import { Pill } from '@/components/ds';
 import { Button } from '@/components/ui/button';
 import EmptyState from '@/components/EmptyState';
-import { Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers } from 'lucide-react';
+import {
+    Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers,
+    FlaskConical, ChevronRight, ChevronDown,
+} from 'lucide-react';
 import { humanSize, formatMoney, formatWhen, statusKind, storageLabel } from './format';
 
 // Card 3 of the backup "Protection" panel: a data-table of backup runs.
@@ -29,72 +33,167 @@ function storageIcon(run) {
     );
 }
 
+// Pill kind for a restore-drill outcome. skipped_no_space is a soft warning
+// (no disk headroom to drill), never a hard failure.
+function drillStatusKind(status) {
+    switch (status) {
+        case 'success': return 'green';
+        case 'failed': return 'red';
+        case 'skipped_no_space': return 'amber';
+        default: return 'gray';
+    }
+}
+
+function drillStatusLabel(status) {
+    if (status === 'skipped_no_space') return 'skipped (no space)';
+    return status || 'unknown';
+}
+
+// Render whatever shape `probes` arrives in (array of checks, keyed object, or
+// a plain string) as compact key/value rows without assuming a schema.
+function renderProbes(probes) {
+    if (!probes) return <span className="backup-drills__probe-empty">No probe details recorded.</span>;
+    const entries = Array.isArray(probes)
+        ? probes.map((p, i) => [p?.name || `Probe ${i + 1}`, p?.detail ?? p?.result ?? p?.ok ?? p])
+        : (typeof probes === 'object' ? Object.entries(probes) : [['result', probes]]);
+    return (
+        <dl className="backup-drills__probes">
+            {entries.map(([key, value]) => (
+                <div key={key} className="backup-drills__probe">
+                    <dt>{key}</dt>
+                    <dd>{typeof value === 'object' ? JSON.stringify(value) : String(value)}</dd>
+                </div>
+            ))}
+        </dl>
+    );
+}
+
+function RestoreDrills({ drills }) {
+    const [expanded, setExpanded] = useState(null);
+    if (!drills || drills.length === 0) return null;
+
+    return (
+        <div className="backup-drills">
+            <div className="backup-drills__head">
+                <FlaskConical size={14} />
+                <span>Restore drills</span>
+                <span className="backup-drills__sub">Proof that these backups can actually be recovered.</span>
+            </div>
+            <ul className="backup-drills__list">
+                {drills.map((drill) => {
+                    const isOpen = expanded === drill.id;
+                    return (
+                        <li key={drill.id} className="backup-drills__item">
+                            <button
+                                type="button"
+                                className="backup-drills__row"
+                                onClick={() => setExpanded(isOpen ? null : drill.id)}
+                                aria-expanded={isOpen}
+                            >
+                                <span className="backup-drills__chev">
+                                    {isOpen ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+                                </span>
+                                <Pill kind={drillStatusKind(drill.status)}>{drillStatusLabel(drill.status)}</Pill>
+                                <span className="backup-drills__trigger">{drill.trigger || 'manual'}</span>
+                                <span className="backup-drills__when">{formatWhen(drill.started_at)}</span>
+                                {drill.duration_seconds != null && (
+                                    <span className="backup-drills__meta">{drill.duration_seconds}s</span>
+                                )}
+                                {drill.bytes_restored != null && (
+                                    <span className="backup-drills__meta">{humanSize(drill.bytes_restored)}</span>
+                                )}
+                            </button>
+                            {isOpen && (
+                                <div className="backup-drills__detail">
+                                    {drill.error && <p className="backup-drills__error">{drill.error}</p>}
+                                    {renderProbes(drill.probes)}
+                                </div>
+                            )}
+                        </li>
+                    );
+                })}
+            </ul>
+        </div>
+    );
+}
+
 export default function BackupHistoryList({
     runs,
+    drills,
     loading,
     onRestore,
     onVerify,
     onDelete,
     onRowClick,
 }) {
+    const hasDrills = drills && drills.length > 0;
+
     if (loading && (!runs || runs.length === 0)) {
         return <EmptyState icon={Archive} title="Loading backups…" loading />;
     }
 
     if (!loading && (!runs || runs.length === 0)) {
         return (
-            <EmptyState
-                icon={Archive}
-                title="No backups yet"
-                description="Turn on protection or click Back up now."
-            />
+            <>
+                <RestoreDrills drills={drills} />
+                {!hasDrills && (
+                    <EmptyState
+                        icon={Archive}
+                        title="No backups yet"
+                        description="Turn on protection or click Back up now."
+                    />
+                )}
+            </>
         );
     }
 
     return (
-        <table className="sk-dtable backup-history-list">
-            <thead>
-                <tr>
-                    <th>Backup</th>
-                    <th>Date</th>
-                    <th>Size</th>
-                    <th>Cost</th>
-                    <th>Status</th>
-                    <th>Storage</th>
-                    <th aria-label="Actions" />
-                </tr>
-            </thead>
-            <tbody>
-                {runs.map((run) => (
-                    <tr
-                        key={run.id}
-                        className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}
-                        onClick={onRowClick ? () => onRowClick(run) : undefined}
-                    >
-                        <td>
-                            <div className="sk-cell-name">
-                                <span className="backup-history-list__ico"><Archive size={14} /></span>
-                                <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>
-                                <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>
-                            </div>
-                        </td>
-                        <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>
-                        <td className="sk-cell-mono">{humanSize(run.size_total)}</td>
-                        <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>
-                        <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>
-                        <td>{storageIcon(run)}</td>
-                        <td>
-                            <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>
-                                <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>
-                                {run.remote_key && (
-                                    <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>
-                                )}
-                                <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>
-                            </div>
-                        </td>
+        <>
+            <RestoreDrills drills={drills} />
+            <table className="sk-dtable backup-history-list">
+                <thead>
+                    <tr>
+                        <th>Backup</th>
+                        <th>Date</th>
+                        <th>Size</th>
+                        <th>Cost</th>
+                        <th>Status</th>
+                        <th>Storage</th>
+                        <th aria-label="Actions" />
                     </tr>
-                ))}
-            </tbody>
-        </table>
+                </thead>
+                <tbody>
+                    {runs.map((run) => (
+                        <tr
+                            key={run.id}
+                            className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}
+                            onClick={onRowClick ? () => onRowClick(run) : undefined}
+                        >
+                            <td>
+                                <div className="sk-cell-name">
+                                    <span className="backup-history-list__ico"><Archive size={14} /></span>
+                                    <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>
+                                    <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>
+                                </div>
+                            </td>
+                            <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>
+                            <td className="sk-cell-mono">{humanSize(run.size_total)}</td>
+                            <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>
+                            <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>
+                            <td>{storageIcon(run)}</td>
+                            <td>
+                                <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>
+                                    <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>
+                                    {run.remote_key && (
+                                        <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>
+                                    )}
+                                    <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>
+                                </div>
+                            </td>
+                        </tr>
+                    ))}
+                </tbody>
+            </table>
+        </>
     );
 }

frontend/src/components/backups/ProtectionPanel.jsx

@@ -36,6 +36,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
     const [loading, setLoading] = useState(true);
     const [saving, setSaving] = useState(false);
     const [backingUp, setBackingUp] = useState(false);
+    const [drilling, setDrilling] = useState(false);
     const [historyView, setHistoryView] = useState('list');  // 'list' | 'calendar'
     const [dayFilter, setDayFilter] = useState(null);        // Date | null
     const [detailRun, setDetailRun] = useState(null);
@@ -117,6 +118,19 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
         }
     }, [targetType, targetId, toast, scheduleReloads]);
 
+    const handleRunDrill = useCallback(async () => {
+        setDrilling(true);
+        try {
+            await api.runBackupDrill(targetType, targetId);
+            toast.success('Restore drill started');
+            scheduleReloads();
+        } catch (err) {
+            toast.error(err.message || 'Failed to start restore drill');
+        } finally {
+            setDrilling(false);
+        }
+    }, [targetType, targetId, toast, scheduleReloads]);
+
     // Restore is always initiated from a run — opening the restore drawer.
     const openRestore = useCallback((run) => setRestoreRun(run), []);
 
@@ -177,10 +191,12 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
                     policyView={view}
                     onToggle={handleToggle}
                     onBackupNow={handleBackupNow}
+                    onRunDrill={handleRunDrill}
                     onViewGlobal={() => navigate('/backups')}
                     onViewJobs={() => navigate('/jobs')}
                     busy={saving}
                     backingUp={backingUp}
+                    drilling={drilling || !!view?.restore_proof?.is_drilling}
                 />
 
                 <ScheduleCard
@@ -236,6 +252,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
                         ) : (
                             <BackupHistoryList
                                 runs={visibleRuns}
+                                drills={dayFilter ? [] : view?.restore_proof?.recent_drills}
                                 loading={loading}
                                 onRestore={openRestore}
                                 onVerify={handleVerify}

frontend/src/components/backups/ScheduleCard.jsx

@@ -1,32 +1,22 @@
-import { useState, useEffect, useMemo } from 'react';
+import { useState, useEffect } from 'react';
 import { SegControl } from '@/components/ds';
 import { Switch } from '@/components/ui/switch';
 import { Button } from '@/components/ui/button';
 import { Input } from '@/components/ui/input';
 import { Calendar, Save, Loader2 } from 'lucide-react';
+import SchedulePicker from '../SchedulePicker';
 
 // Card 2 of the backup "Protection" panel: the editable schedule form.
-// The saved policy carries a raw cron string; this card translates it into a
-// friendly Daily / Weekly / Custom UI and back. Local state mirrors the form
-// while a useEffect re-seeds it whenever the parent reloads the policy, so an
-// external save stays in sync. Save is disabled until something actually
+// The saved policy carries a raw cron string; this card now mounts the shared
+// SchedulePicker (compact) so backups get the same Presets/Builder/Advanced UI
+// and server-side preview as every other cron surface. Local state mirrors the
+// form while a useEffect re-seeds it whenever the parent reloads the policy, so
+// an external save stays in sync. Save is disabled until something actually
 // changes (dirty tracking) and shows a spinner while the request is in flight.
 
-// Canonical cron strings for the two preset frequencies (02:00 server time).
-const DAILY_CRON = '0 2 * * *';
-const WEEKLY_CRON = '0 2 * * 0'; // Sunday
-// Day-of-week chip labels; index 0 = Sunday … 6 = Saturday (cron dow order).
-const DOW_LABELS = ['S', 'M', 'T', 'W', 'T', 'F', 'S'];
-
-const DEFAULT_TIME = '02:00';
+const DEFAULT_CRON = '0 2 * * *'; // Daily at 02:00 server time.
 const MIN_COUNT = 1; // floor shared by every numeric field (count/days/full-every)
 
-// Two-digit zero-pad for assembling/displaying HH:MM from cron integers.
-const pad = (n) => String(n).padStart(2, '0');
-
-// True when a cron token is a plain non-negative integer (e.g. "2", not "*").
-const isInt = (token) => /^\d+$/.test(token);
-
 // Coerce a numeric form field (kept as a string) to a number, treating empty
 // or sub-floor values as the minimum so we never persist 0 / NaN.
 const toNum = (value) => {
@@ -34,81 +24,30 @@ const toNum = (value) => {
     return Number.isFinite(n) && n >= MIN_COUNT ? n : MIN_COUNT;
 };
 
-// Expand a cron day-of-week field into a sorted int array (0..6).
-// "*"/"?" -> [] (every day); supports comma lists and "a-b" ranges.
-function parseDow(field) {
-    if (!field || field === '*' || field === '?') return [];
-    const out = new Set();
-    for (const part of field.split(',')) {
-        const range = part.split('-');
-        if (range.length === 2 && isInt(range[0]) && isInt(range[1])) {
-            const lo = Number(range[0]);
-            const hi = Number(range[1]);
-            for (let d = lo; d <= hi; d += 1) {
-                if (d >= 0 && d <= 6) out.add(d);
-            }
-        } else if (isInt(part)) {
-            const d = Number(part);
-            if (d >= 0 && d <= 6) out.add(d);
-        }
-    }
-    return Array.from(out).sort((a, b) => a - b);
-}
-
-// Translate a saved cron string into the form's { frequency, time, days }.
-function deriveFromCron(cron) {
-    if (cron === DAILY_CRON) return { frequency: 'daily', time: DEFAULT_TIME, days: [] };
-    if (cron === WEEKLY_CRON) return { frequency: 'weekly', time: DEFAULT_TIME, days: [0] };
-
-    const fields = (cron || '').trim().split(/\s+/);
-    const min = fields[0];
-    const hour = fields[1];
-    const time = isInt(hour) && isInt(min) ? `${pad(Number(hour))}:${pad(Number(min))}` : DEFAULT_TIME;
-    return { frequency: 'custom', time, days: parseDow(fields[4]) };
-}
-
-// Assemble a cron string from the form state.
-function buildCron(frequency, time, days) {
-    if (frequency === 'daily') return DAILY_CRON;
-    if (frequency === 'weekly') return WEEKLY_CRON;
-
-    const [hour = '02', min = '00'] = (time || DEFAULT_TIME).split(':');
-    const dowField =
-        days.length > 0 && days.length < 7
-            ? days.slice().sort((a, b) => a - b).join(',')
-            : '*';
-    return `${Number(min)} ${Number(hour)} * * ${dowField}`;
-}
-
 const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
     // All hooks run unconditionally, before any early return (Rules of Hooks).
     // The form is only shown once a policy exists, so the seed values used when
     // `policy` is null are throwaway and get re-seeded by the effect on load.
-    const initial = deriveFromCron(policy?.schedule_cron);
-
-    const [frequency, setFrequency] = useState(initial.frequency);
-    const [time, setTime] = useState(initial.time);
-    const [days, setDays] = useState(initial.days);
+    const [cron, setCron] = useState(policy?.schedule_cron || DEFAULT_CRON);
     const [retentionCount, setRetentionCount] = useState(String(policy?.retention_count ?? ''));
     const [retentionDays, setRetentionDays] = useState(String(policy?.retention_days ?? ''));
     const [fullEvery, setFullEvery] = useState(String(policy?.full_every_n_days ?? ''));
     const [compression, setCompression] = useState(policy?.compression ?? 'balanced');
     const [remoteCopy, setRemoteCopy] = useState(!!policy?.remote_copy);
+    const [drillCadence, setDrillCadence] = useState(policy?.drill_cadence ?? 'off');
 
     // Re-seed the form when the saved policy changes (e.g. external reload after
     // a successful save). Keyed on the persisted values so it only fires when
     // the source of truth actually moves, not on every render.
     useEffect(() => {
         if (!policy) return;
-        const next = deriveFromCron(policy.schedule_cron);
-        setFrequency(next.frequency);
-        setTime(next.time);
-        setDays(next.days);
+        setCron(policy.schedule_cron || DEFAULT_CRON);
         setRetentionCount(String(policy.retention_count));
         setRetentionDays(String(policy.retention_days));
         setFullEvery(String(policy.full_every_n_days));
         setCompression(policy.compression);
         setRemoteCopy(!!policy.remote_copy);
+        setDrillCadence(policy.drill_cadence || 'off');
     }, [
         policy,
         policy?.schedule_cron,
@@ -117,9 +56,10 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
         policy?.full_every_n_days,
         policy?.compression,
         policy?.remote_copy,
+        policy?.drill_cadence,
     ]);
 
-    const currentCron = useMemo(() => buildCron(frequency, time, days), [frequency, time, days]);
+    const currentCron = cron;
 
     // Conditional render AFTER all hooks have run.
     if (!policy) {
@@ -136,12 +76,6 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
         );
     }
 
-    const toggleDay = (i) => {
-        setDays((prev) =>
-            prev.includes(i) ? prev.filter((d) => d !== i) : [...prev, i].sort((a, b) => a - b)
-        );
-    };
-
     // The form is dirty when any field diverges from the saved policy. Numeric
     // fields are compared via toNum so an empty box reads as the floor (= what
     // would actually be persisted), avoiding a phantom-dirty Save button.
@@ -151,7 +85,8 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
         toNum(retentionDays) !== policy.retention_days ||
         toNum(fullEvery) !== policy.full_every_n_days ||
         compression !== policy.compression ||
-        remoteCopy !== !!policy.remote_copy;
+        remoteCopy !== !!policy.remote_copy ||
+        drillCadence !== (policy.drill_cadence || 'off');
 
     const handleSave = () => {
         onSave({
@@ -161,6 +96,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
             full_every_n_days: toNum(fullEvery),
             compression,
             remote_copy: remoteCopy,
+            drill_cadence: drillCadence,
         });
     };
 
@@ -176,44 +112,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
             <div className="app-panel-body">
                 <div className="schedule-card__field">
                     <label>Frequency</label>
-                    <SegControl
-                        options={[
-                            { value: 'daily', label: 'Daily' },
-                            { value: 'weekly', label: 'Weekly' },
-                            { value: 'custom', label: 'Custom' },
-                        ]}
-                        value={frequency}
-                        onChange={setFrequency}
-                    />
-                </div>
-
-                {frequency === 'custom' && (
-                    <>
-                        <div className="schedule-card__field">
-                            <label>Time</label>
-                            <Input type="time" value={time} onChange={(e) => setTime(e.target.value)} />
-                        </div>
-                        <div className="schedule-card__field">
-                            <label>Days</label>
-                            <div className="schedule-card__days">
-                                {DOW_LABELS.map((d, i) => (
-                                    <button
-                                        type="button"
-                                        key={i}
-                                        className={`schedule-card__day ${days.includes(i) ? 'is-active' : ''}`}
-                                        onClick={() => toggleDay(i)}
-                                    >
-                                        {d}
-                                    </button>
-                                ))}
-                            </div>
-                        </div>
-                    </>
-                )}
-
-                <div className="schedule-card__preview">
-                    <span className="schedule-card__preview-label">Cron</span>
-                    <code>{currentCron}</code>
+                    <SchedulePicker value={cron} onChange={setCron} compact />
                 </div>
 
                 <div className="schedule-card__field schedule-card__retention">
@@ -273,6 +172,25 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
                     />
                 </div>
 
+                <div className="schedule-card__field">
+                    <label>Restore drill cadence</label>
+                    <SegControl
+                        options={[
+                            { value: 'off', label: 'Off' },
+                            { value: 'weekly', label: 'Weekly' },
+                            { value: 'monthly', label: 'Monthly' },
+                        ]}
+                        value={drillCadence}
+                        onChange={setDrillCadence}
+                    />
+                    <p className="app-panel-hint">
+                        Periodically test-restore the latest backup to prove it can actually be
+                        recovered. Drills run into a throwaway location and never touch this site.
+                        Replaying an incremental backup chain during a drill needs GNU tar, so
+                        those drills run on Linux hosts only.
+                    </p>
+                </div>
+
                 <div className="schedule-card__field schedule-card__remote">
                     <Switch
                         id="remote-copy"

frontend/src/services/api/backupProtection.js

@@ -43,6 +43,17 @@ export async function verifyBackupRun(targetType, targetId, runId) {
     return this.request(`${runsBase(targetType, targetId)}/${runId}/verify`, { method: 'POST' });
 }
 
+// Trigger a restore drill (a proving test-restore into a throwaway location).
+// Applications and WordPress sites use their own edit-gated, target-scoped
+// endpoint (the same owner who can back up a target can drill it); any other
+// target routes through the admin-only generic policy-by-target endpoint.
+export async function runBackupDrill(targetType, targetId) {
+    if (targetType === 'application' || targetType === 'wordpress_site') {
+        return this.request(`${runsBase(targetType, targetId)}/drill`, { method: 'POST' });
+    }
+    return this.request(`/backups/policies/${targetType}/${targetId}/drill`, { method: 'POST' });
+}
+
 export async function deleteBackupRun(targetType, targetId, runId) {
     return this.request(`${runsBase(targetType, targetId)}/${runId}`, { method: 'DELETE' });
 }

frontend/src/styles/pages/_backups.scss

@@ -1,732 +1,10 @@
-// ============================================
-// BACKUPS PAGE (redesign — infra console look)
-// ============================================
-
-.backups-page {
-
-    // ── KPI strip (4 tiles, +1 when a remote provider is set) ──
-    .bk-kpis {
-        display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
-        gap: $space-4;
-        margin-bottom: $space-5;
-
-        @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
-    }
-
-    // ── list head: mono eyebrow + segmented filter + actions ─
-    .bk-listhead {
-        display: flex;
-        align-items: center;
-        gap: $space-3;
-        margin-bottom: $space-3;
-
-        h2 {
-            flex: 1;
-            min-width: 0;
-            margin: 0;
-            font-family: var(--mono);
-            font-size: 10.5px;
-            font-weight: $font-weight-semibold;
-            letter-spacing: 0.08em;
-            text-transform: uppercase;
-            color: var(--text-faint);
-        }
-
-        @media (max-width: $breakpoint-md) {
-            flex-wrap: wrap;
-
-            h2 { flex-basis: 100%; }
-        }
-    }
-
-    // ── dense tables (backup archive + schedules) ───────────
-    .bk-card {
-        background: var(--surface);
-        border: 1px solid var(--border-soft);
-        border-radius: var(--radius);
-        overflow: hidden;
-    }
-
-    .bk-empty {
-        padding: $space-10;
-        text-align: center;
-        color: var(--text-faint);
-        font-size: 13px;
-        background: var(--surface);
-        border: 1px solid var(--border-soft);
-        border-radius: var(--radius);
-    }
-
-    .bk-table {
-        // actions column shrinks to fit, buttons never wrap
-        th:last-child { width: 1%; }
-        td:last-child { white-space: nowrap; }
-
-        tbody tr.is-disabled {
-            opacity: 0.55;
-
-            .bk-ico {
-                background: var(--surface-3);
-                color: var(--text-faint);
-            }
-        }
-    }
-
-    // backup kind tile (tinted per type)
-    .bk-ico {
-        width: 30px;
-        height: 30px;
-        border-radius: 8px;
-        display: grid;
-        place-items: center;
-        flex-shrink: 0;
-        background: var(--surface-2);
-        color: var(--text-dim);
-
-        &--application { background: var(--accent-bg); color: var(--accent-bright); }
-        &--database { background: var(--violet-bg); color: var(--violet); }
-        &--files { background: var(--amber-bg); color: var(--amber); }
-    }
-
-    // mono type chip (application / database / files) — %sk-state shape
-    // (_design-system.scss) with local sizing + bordered per-type tints kept
-    .bk-type {
-        @extend %sk-state;
-        padding: 2px 7px;
-        border-radius: 5px;
-        font-size: 10.5px;
-        background: var(--surface-2);
-        border: 1px solid var(--border);
-        color: var(--text-dim);
-
-        &--application {
-            background: var(--accent-bg);
-            border-color: fade($accent-primary-raw, 30%);
-            color: var(--accent-bright);
-        }
-
-        &--database {
-            background: var(--violet-bg);
-            border-color: rgba(176, 123, 245, 0.3);
-            color: var(--violet);
-        }
-
-        &--files {
-            background: var(--amber-bg);
-            border-color: rgba(245, 185, 69, 0.3);
-            color: var(--amber);
-        }
-    }
-
-    .bk-when {
-        font-family: var(--mono);
-        font-size: 12px;
-        color: var(--text-faint);
-        white-space: nowrap;
-    }
-
-    .bk-actions {
-        display: flex;
-        justify-content: flex-end;
-        gap: $space-1;
-    }
-
-    // schedule time chip (matches the cron expression chip)
-    .bk-sched {
-        display: inline-flex;
-        align-items: center;
-        gap: 5px;
-        padding: 2px 8px;
-        font-family: var(--mono);
-        font-size: 11.5px;
-        color: var(--text-dim);
-        background: var(--surface-2);
-        border: 1px solid var(--border);
-        border-radius: 6px;
-        white-space: nowrap;
-
-        svg {
-            color: var(--text-faint);
-            flex-shrink: 0;
-        }
-    }
-
-    // remote-upload indicator on schedule rows
-    .bk-remote-on { color: var(--green); }
-    .bk-remote-off { color: var(--text-ghost); }
-
-    // storage pills carry a small lucide icon instead of the dot
-    .sk-pill svg { flex-shrink: 0; }
-
-    @media (max-width: $breakpoint-md) {
-        .bk-card { overflow-x: auto; }
-        .bk-table { min-width: 720px; }
-        .bk-table--schedules { min-width: 880px; }
-    }
-
-    // ── storage tab: real-data spec strip ───────────────────
-    .bk-specs {
-        display: grid;
-        grid-template-columns: repeat(3, 1fr);
-        gap: $space-3;
-        margin-bottom: $space-4;
-
-        @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
-    }
-
-    // storage/settings cards → surface chrome + eyebrow headers
-    .card {
-        background: var(--surface);
-        border: 1px solid var(--border-soft);
-        border-radius: var(--radius);
-    }
-
-    .card-header h3 {
-        margin: 0;
-        font-family: var(--mono);
-        font-size: 10.5px;
-        font-weight: $font-weight-semibold;
-        letter-spacing: 0.08em;
-        text-transform: uppercase;
-        color: var(--text-faint);
-    }
-
-    .storage-provider-config {
-        margin: $space-4 0;
-        padding: $space-4;
-        background: var(--surface-2);
-        border: 1px solid var(--border-soft);
-        border-radius: var(--radius-sm);
-
-        h4 {
-            margin: 0 0 $space-4;
-            font-family: var(--mono);
-            font-size: 10.5px;
-            font-weight: $font-weight-semibold;
-            letter-spacing: 0.08em;
-            text-transform: uppercase;
-            color: var(--text-faint);
-        }
-    }
-
-    // ── forms (cards + modals): mono eyebrow labels ─────────
-    .form-group > label:not(.checkbox-label) {
-        font-family: var(--mono);
-        font-size: 10.5px;
-        font-weight: $font-weight-semibold;
-        letter-spacing: 0.08em;
-        text-transform: uppercase;
-        color: var(--text-faint);
-    }
-
-    .form-row {
-        display: grid;
-        grid-template-columns: 1fr 1fr;
-        gap: $space-4;
-
-        @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
-    }
-
-    .form-help {
-        display: block;
-        margin-top: $space-2;
-        font-family: var(--mono);
-        font-size: 11px;
-        color: var(--text-faint);
-    }
-
-    // inline hint inside an eyebrow label — undo the uppercase
-    .form-help-inline {
-        font-family: var(--mono);
-        font-size: 10px;
-        font-weight: $font-weight-normal;
-        letter-spacing: normal;
-        text-transform: none;
-        color: var(--text-ghost);
-    }
-
-    .form-actions {
-        display: flex;
-        gap: $space-2;
-        margin-top: $space-5;
-    }
-
-    .checkbox-label {
-        display: flex;
-        align-items: center;
-        gap: $space-2;
-        cursor: pointer;
-
-        input[type="checkbox"] {
-            width: 16px;
-            height: 16px;
-            cursor: pointer;
-            accent-color: var(--accent);
-        }
-
-        span {
-            font-size: 13px;
-            color: var(--text);
-        }
-    }
-
-    // ── restore modal ───────────────────────────────────────
-    .bk-restore-warn {
-        display: flex;
-        gap: 11px;
-        padding: 13px 15px;
-        margin-bottom: $space-5;
-        border-radius: 11px;
-        background: var(--amber-bg);
-        border: 1px solid rgba(245, 185, 69, 0.28);
-        font-size: 12.5px;
-        line-height: 1.5;
-        color: var(--text-dim);
-
-        svg {
-            color: var(--amber);
-            flex-shrink: 0;
-            margin-top: 1px;
-        }
-
-        b { color: var(--text); }
-    }
-
-    .bk-restore-details {
-        padding: $space-1 $space-4;
-        background: var(--surface-2);
-        border: 1px solid var(--border-soft);
-        border-radius: var(--radius-sm);
-    }
-
-    // ── spinners (upload / test-connection buttons) ─────────
-    .spinning {
-        animation: spin 1s linear infinite;
-    }
-}
-
-@keyframes spin {
-    from { transform: rotate(0deg); }
-    to { transform: rotate(360deg); }
-}
-
-// Reusable inline spinner for the protection components (lucide <Loader2 className="spin" />).
-.spin { animation: spin 1s linear infinite; }
-
-// ============================================
-// PROTECTION PANEL (backup protection — WordPress + Services settings)
-// Rendered inside .app-overview-grid > .app-overview-left, so these styles are
-// top-level (not nested under .backups-page).
-// ============================================
-.protection-panel {
-    // Single column inside a settings pane (no right rail).
-    grid-template-columns: 1fr;
-
-    .app-overview-left {
-        display: flex;
-        flex-direction: column;
-        gap: $space-4;
-    }
-}
-
-// ── Card 1: status ──────────────────────────────────────────
-.protection-status-card {
-    &__toggle {
-        display: flex;
-        align-items: flex-start;
-        gap: $space-3;
-        padding-bottom: $space-4;
-        border-bottom: 1px solid var(--border-soft);
-
-        label {
-            display: flex;
-            flex-direction: column;
-            gap: 2px;
-            cursor: pointer;
-        }
-    }
-
-    &__toggle-title { font-weight: $font-weight-semibold; color: var(--text); }
-    &__toggle-sub { font-size: 12px; color: var(--text-faint); }
-
-    &__status {
-        display: flex;
-        align-items: center;
-        flex-wrap: wrap;
-        gap: $space-2;
-        margin: $space-4 0;
-        font-size: 13px;
-        color: var(--text-dim);
-    }
-
-    &__kpis {
-        display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
-        gap: $space-3;
-        margin-bottom: $space-4;
-
-        @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
-    }
-
-    &__actions {
-        display: flex;
-        align-items: center;
-        flex-wrap: wrap;
-        gap: $space-2;
-    }
-}
-
-// ── Card 2: schedule ────────────────────────────────────────
-.schedule-card {
-    .app-panel-body {
-        display: flex;
-        flex-direction: column;
-        gap: $space-4;
-    }
-
-    &__field {
-        display: flex;
-        flex-direction: column;
-        gap: $space-2;
-
-        > label {
-            font-size: 12px;
-            font-weight: $font-weight-semibold;
-            color: var(--text);
-        }
-    }
-
-    &__days {
-        display: flex;
-        gap: $space-2;
-    }
-
-    &__day {
-        width: 32px;
-        height: 32px;
-        border: 1px solid var(--border-soft);
-        background: var(--surface-2);
-        color: var(--text-dim);
-        border-radius: $radius-sm;
-        font-size: 12px;
-        font-weight: $font-weight-semibold;
-        cursor: pointer;
-        transition: border-color .12s ease, background .12s ease, color .12s ease;
-
-        &:hover { border-color: var(--accent-bright); }
-
-        &.is-active {
-            background: var(--accent-bg);
-            color: var(--accent-bright);
-            border-color: var(--accent-bright);
-        }
-    }
-
-    &__preview {
-        display: flex;
-        align-items: center;
-        gap: $space-2;
-
-        code {
-            font-family: var(--mono);
-            font-size: 12px;
-            color: var(--text);
-            background: var(--surface-2);
-            border: 1px solid var(--border-soft);
-            border-radius: $radius-sm;
-            padding: 2px 8px;
-        }
-    }
-
-    &__preview-label {
-        font-family: var(--mono);
-        font-size: 10.5px;
-        text-transform: uppercase;
-        letter-spacing: 0.04em;
-        color: var(--text-faint);
-    }
-
-    &__retention-row {
-        display: flex;
-        align-items: center;
-        gap: $space-2;
-        font-size: 13px;
-        color: var(--text-dim);
-
-        input { width: 80px; }
-    }
-
-    &__remote {
-        flex-direction: row;
-        align-items: flex-start;
-        gap: $space-3;
-
-        label {
-            display: flex;
-            flex-direction: column;
-            gap: 2px;
-            cursor: pointer;
-        }
-    }
-
-    &__actions { margin-top: $space-1; }
-}
-
-// ── Card 3: history list ────────────────────────────────────
-.backup-history-list {
-    width: 100%;
-
-    &__row.is-clickable { cursor: pointer; }
-
-    &__ico {
-        display: inline-flex;
-        color: var(--text-faint);
-    }
-
-    &__when {
-        color: var(--text-dim);
-        font-size: 13px;
-        white-space: nowrap;
-    }
-
-    &__actions {
-        display: flex;
-        justify-content: flex-end;
-        gap: $space-1;
-    }
-
-    &__storage {
-        display: inline-flex;
-        align-items: center;
-        gap: 4px;
-        font-size: 12px;
-        color: var(--text-dim);
-    }
-}
-
-// ── History card chrome (list/calendar toggle + day-filter chip) ────
-.backup-history-card {
-    &__tools {
-        display: flex;
-        align-items: center;
-        gap: $space-2;
-    }
-
-    &__toggle {
-        display: inline-flex;
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-        overflow: hidden;
-
-        button {
-            display: inline-flex;
-            align-items: center;
-            justify-content: center;
-            width: 28px;
-            height: 26px;
-            background: var(--surface-2);
-            color: var(--text-dim);
-            border: none;
-            cursor: pointer;
-
-            &.is-active { background: var(--accent-bg); color: var(--accent-bright); }
-            & + button { border-left: 1px solid var(--border-soft); }
-        }
-    }
-
-    &__chip {
-        display: inline-flex;
-        align-items: center;
-        gap: 4px;
-        padding: 2px 8px;
-        font-size: 12px;
-        color: var(--text-dim);
-        background: var(--surface-2);
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-full;
-        cursor: pointer;
-
-        &:hover { color: var(--text); }
-    }
-}
-
-// ── Calendar view ───────────────────────────────────────────
-.backup-calendar {
-    &__head {
-        display: flex;
-        align-items: center;
-        gap: $space-2;
-        margin-bottom: $space-3;
-    }
-
-    &__month {
-        flex: 1;
-        font-weight: $font-weight-semibold;
-        color: var(--text);
-    }
-
-    &__nav,
-    &__today {
-        display: inline-flex;
-        align-items: center;
-        justify-content: center;
-        height: 28px;
-        padding: 0 $space-2;
-        background: var(--surface-2);
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-        color: var(--text-dim);
-        cursor: pointer;
-
-        &:hover { border-color: var(--accent-bright); color: var(--text); }
-    }
-
-    &__grid {
-        display: grid;
-        grid-template-columns: repeat(7, 1fr);
-        gap: $space-1;
-    }
-
-    &__dow {
-        text-align: center;
-        font-family: var(--mono);
-        font-size: 10px;
-        text-transform: uppercase;
-        letter-spacing: 0.04em;
-        color: var(--text-faint);
-        padding-bottom: $space-1;
-    }
-
-    &__cell {
-        position: relative;
-        aspect-ratio: 1 / 1;
-        display: flex;
-        flex-direction: column;
-        align-items: center;
-        justify-content: center;
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-        background: var(--surface-2);
-        color: var(--text-dim);
-        font-size: 12px;
-
-        &--empty { background: transparent; border-color: transparent; }
-
-        &--has {
-            cursor: pointer;
-
-            &:hover { border-color: var(--accent-bright); }
-            &.is-success { border-color: var(--green); background: var(--green-bg); }
-            &.is-failed { border-color: var(--red); background: var(--red-bg); }
-            &.is-running { border-color: var(--cyan); background: var(--cyan-bg); }
-        }
-    }
-
-    &__badge {
-        position: absolute;
-        top: 2px;
-        right: 3px;
-        font-size: 9px;
-        font-weight: $font-weight-semibold;
-        color: var(--text-faint);
-    }
-}
-
-// ── Detail drawer ───────────────────────────────────────────
-.backup-detail-drawer {
-    display: flex;
-    flex-direction: column;
-    gap: $space-4;
-
-    &__error {
-        padding: $space-2 $space-3;
-        font-size: 13px;
-        color: var(--red);
-        background: var(--red-bg);
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-    }
-
-    &__meta {
-        display: grid;
-        gap: $space-1;
-    }
-
-    &__row {
-        display: flex;
-        justify-content: space-between;
-        gap: $space-3;
-        padding: 6px 0;
-        border-bottom: 1px solid var(--border-soft);
-        font-size: 13px;
-
-        span:first-child { color: var(--text-faint); }
-        span:last-child { color: var(--text); }
-    }
-
-    &__cost {
-        font-size: 13px;
-        color: var(--text-dim);
-        font-family: var(--mono);
-    }
-
-    &__actions {
-        display: flex;
-        flex-wrap: wrap;
-        gap: $space-2;
-    }
-
-    &__joblink {
-        display: inline-flex;
-        align-items: center;
-        gap: 4px;
-        font-size: 13px;
-        color: var(--accent-bright);
-    }
-}
-
-// ── Restore drawer ──────────────────────────────────────────
-.restore-drawer {
-    display: flex;
-    flex-direction: column;
-    gap: $space-4;
-
-    &__section { display: flex; flex-direction: column; gap: $space-2; }
-    &__label { font-size: 12px; font-weight: $font-weight-semibold; color: var(--text); }
-
-    &__tables {
-        display: flex;
-        flex-direction: column;
-        gap: $space-1;
-        max-height: 180px;
-        overflow-y: auto;
-        padding: $space-2;
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-
-        label { display: flex; align-items: center; gap: $space-2; font-size: 13px; color: var(--text-dim); }
-    }
-
-    &__options { display: flex; flex-direction: column; gap: $space-3; }
-
-    &__option {
-        display: flex;
-        align-items: flex-start;
-        gap: $space-3;
-
-        label { display: flex; flex-direction: column; font-size: 13px; color: var(--text); cursor: pointer; }
-    }
-
-    &__warning {
-        display: flex;
-        align-items: flex-start;
-        gap: $space-2;
-        padding: $space-2 $space-3;
-        font-size: 13px;
-        color: var(--amber);
-        background: var(--amber-bg);
-        border: 1px solid var(--border-soft);
-        border-radius: $radius-sm;
-    }
-}
+LdbCls2cOX529YMxMz86/EK7TsxOpe4t6kpI7BC641eG9mHH7NVye8FM7vi+tL9v0qu9bYJ6pzb0CwUHAy0ZKWSdJx7Gq29zY1TnVmlBarE5P3L/w/HFv7EXeXsAr0VfhTb85V3tfcHlMR/QLbzcVKIbHfPua1b7b8+cqRWAdFXvv7agazNpvEoUB5VKONznN17CxjvmNZfykjN1aWBsLP6CUTaNXpwuvakQdiMSJTS5pGOVKAcXtcST+63/46nJ2Dmc+Vcvd1bTggna7Svr0/fuA7r6lHaLcGnK6DO1qwGGy3t7lHve4KniN4YcLKI7TYpMjMxV85FMMyRobTU69k71xSyJT0YwaHbidW068FnQZ+L3DzoMlGTu1qHwN4iZH9yL4p2lcgN5EEED8dGLLW5uLeeeAb0wJbrSg9sJQH8qKkGO5DuhjcmnigNZH0oJIb7/W+hRg4p4cviVksEKOgjRALuucGqXuq+R9g5mdimKauZXzaDPx5aeGzSgDcBihc36BS5MrIeYh7G/k7mllUUebh6g52CSgjj0gKxn4SkrzCyvM7yXod9MicpxG5r81n8HAgbZo1ACDVVyslkv2YgbkKNz39Aio521nZHXgTwtRGHjd/hSWBFAHpxW1VPpbsNvteNza+Qd0gKuaFeNyheF/R0kPQop6+SEDXk1bpJdXjDn3QrzvBCUM+JdZcTcRkDVQyI4Kg8mpIOp46JixyKXCh1bULPiD93t6w93/bxbKfOcGudX/d2IX2chD0yT7gSfV9B4/lZnshkEQFH71G4I9TMJJ2d/MfTLv9dQL4WU7UjQXOMTpwDQS4LfQSU5oin7Pw4dNyJ3RhWvMzi57Kx8UXrr6iou3/ZI4O8iWbq1Do3Ev1HInHFjBnRosDzwMnN/5LQNzVEhnerPmpW/GFQ0uEadQ3n3ahBHMuhrap2qUELMP28uwHXoYP61twxle2IL69Y+itfAGNKk0j/TkOXSyrpKvMkI3vyyWe9+GV8EZDkF6RhNrjgH9c2R4lUMOimFc59spWVyQrnANcBZ1TJExSQnrXQ2gYBw1i0bjiCf88Xt82eQ4XJYirDwOpac163g+R2b+uXFqGkvGeb7gVpZpP8Q65jwcRwShs+Do/N5yI0DtzNZVXrbJHyDuehfpPYgqGgemOg/rrcWNyER33TgYLwbIDDPkQZsl4xHBtFrbj3ISoLuACKbr7pZXXF8D/K1t6jWeSIPoebs7WjS5POT5dkbYhTx01dlTYrvTkzwXhieqmh9Oz/qdzCcSIR6beiL9x5wCeEtnxYaW2faEiCOATUJOuRJP2CrVYoWrcsG0AFj0/InNf2DDJmG1xEAWHBI05vPIRnNCuFi7SM0DRI6aIoa61/ysJ8kdkSJtbOSAv+cdOiiw2rOrpXiQkPDN3x94psoa4opVSSqifOrU4byOcbWXOJ7I1Nq/cd0Jim8H4Ptz6m0tP836TULUiOsk6UR/0/b/CWXQ4y7XLdm8mpIouWlFObBsGLrBwYCVttdnd8sWb3Be9eJJ2W8H3gb0C+Ai9kH4GZC2D4GWmoW1wHTS09wUqYZMkqhGf0RJXaPAi5sRnmPzVLl2c5P0QQWq0XYatcnm4l0grE4WnF8VMzqBN15rhb0KRWMRjS/WjPL4OKMR6xlWNuUmSjLh8lA769QqXaUQarIAivoHL/+cHkw9oRPwGGGICts1CQhNJ7GvfFaLn+ZOVHLuZ4sV9Sq+MGs7nwPFHCiX7LXHt/3xkOS+IhxUlTBrMQXsO75SkSltyqtZuNliegA1g/vIWYlsw0Qz2IjDRDTdSVoJUY7TOxBQ/GeNEjzBNsur7dZqxxwpSbus6B8uFcMg6L+NkqCUOZDZyOhn0MX6A6Mj4ogJpdIedZrHIG6MlRaibbqUUqXOH69k5riaoGtVxeSvXitaDkCiVE0ds9tPuwRpSRWgdiK9ow9V73vP0tGED8SjdyOjLNZ283EXEFaArM4A2bpl62DO9vGYxogQWjlIRnIghmJ+Skj0bPD5LueD1xG+dx/Zl9OIFK/6EphQAz+3DH249uXTja0w0tqeEOc4eNd6SbuZZh1wDcjwhJUIlJHEYEuxB+yL2Joj9Pw7sVl2or13I6QFzUKDucSUfjktAqYdUWeRrnewG2cuz3ptm/ng4hutM7OkL74k0WEhVXDrDVv0rrqDYcNByLb0lzgOhtaBHsyR/Nel/z4EZPEY471bT7islXgsewHjWg5aSzmKlHO9O0mrtRIX1PwlwXSNTRbhMMIluBxydlOq2oWXQEIqhoKc/JhFKge6czzPTXOVi8sNrghvmEpzGeYcG6hbKzhzIaF3HTWh/57VyD2N2VLML7365/Jtyz5u7W7lCbyPueH+8Eq1voBLUK3+W6HbuKPagr39M0nXFazmUXIeLtB5oPawaejea5LLVIng2fAYVFf1CnbpzF61H5tSDZps/KmlF9+S7pSceACd4EaxSDuTYcWUaf6R+5WCJIlHv0AnMzN/A+eu/7xTUhIlFV4LeSqhBAzr+HZjsfMustVFDkxHmZU8syK+7ydIRpGbAm1jk+ACxAyhP2OUJWPX36Ul1gWBzJN4cpAdSdzabfUFpkBV2tgHjGj5sWTnjC/wyoyEyTEo2ZENJkxSXLb4DKFudvRtogIsvmFTfIKtaMe7efHW7frTp3b5y7AWkZXZqXGrxXufm8Wi/cYck2UxeqMnVrTQrTimynACFfr3ByMYTsp0MZQGgQ2OL2qdPJVePlEosk3rYrKB4DXUJ0DS1A+r/+/upvWRZfqLY4YHz//fd5BF+86FCAnm57TJV9yggF438GKiGXz0oWxyDO/AdJnPn+N7KKb9thLp0b4WcNf8RjgdCjMT+zJGTjBuwz6WwvISD3Pi0QahIkpAKtu38JHbKo5S2u2QMDmGAVQHuxcB2s8QKuBOQ7u8uJailqWcBUNQB8/pqBKSAjUcZueRdNJ8/76yttgFJaLhnaWXTEe2joA55qX71f0Hn+VUmY7GX+KRr3ZJp+TuJItEXhxlOMwvIcALyCco+OkZ9IN58A7IqGFqJBNdkLUjPDpBEODvjkBSIQbu1U67tARfnCq2LZijGt8AJlDEG130gPac28wPVkUobkZ7QTOOiIDFlWQ+8oUGB3MaOCGn34rxc1t0yOiL0rreI14vfDxv8Idgj+yP2C82WUkpojU8i/+oTrDLM42hIgy5x5fcjTdLWYoyYsxJog2U55nKeUmeJu3lvduwV/CXTL69dfRhLEZPn7MfSt7U33bp/QdwDy9HfCjj9A7rCTs4PXiGrJjZ3HfNvVdF7xHQu262yXP0xcOBceK5+RiF0Xc/SDctNHh5zvIQIFbgHJFBSxaA+a7Sko+Nt49TU/aOWV0SdhAtO4/BreOv9/fvR6aiwlmd9Qunw4I40u4I1QSaZr7gG7IB9nt6BRaJBg548DFiVufmlWwIqHg4wX6yaFDGg6BieSquZourJ7w4Z0GsH8m3tTMq2B3fUBDpev4IXN3QyVHdQVCPU5pmcbVxoxs1O/ZX++Wr1DJHrZsvkB/KvMZEpB0AMnhC8eLbsszefUQk9xcJCFR5XKssAjs1IbBDyYMjLZqI50UxMF8QAaoGktFXPLD9AUKgAdqFP2TlxhmaBixEN1rXk6UNQikK6pWFIFclsGj0SZo/oSYxaIe9O785JzXfpjJzlH0V8gbeAlRTwz8BTFJPexJMVJv0DZA8hjxgR4t8dgYxLwg+XP68vb6M03VhLug6hcep4yRoB3QsJI+FQMKVaQRS3vQcsyUmtZi6x9zWf/OhohTwQ6bZRRzl0EwBpivxoxdCnifLaxAqS9RV9gCGnWxzhpfwv2gNhj5I1uY+Wp6MyGFdno2sVIRWChMODmsR2k0hkJZ4yfdJgFK9extMH8R96sa+SOlPYMQAPLL4r8W6fyagyfZFwjFMVYy7c66XJIyoNsy7QCw1bxZvkFj+MvsqcCYcp1H1hH0gSLxL8vsgtDKYrzm4m32rxBKyYdqx7U9udwlaiHPS3n02CRwXnMr4PvoQ4OfVPovU1dG6SLPuBLt0kqpyZInhdBBQgDr+4vlhyve/WgNrGvwGgYEcSIkdldlpGHau4OhI3dAm/v3yYYbHLzULH3Sxdpkd8x4rEIJpRrAKprgxCEhO3kw3T5Ob4vXxdXgrRVHPGd40z4SmFkIfVj0a5EdGxe7vlt7NR7ABTLrdvamJ44nht9T/qqyhfbicpj2bNUkEtomqnzusyLcBTyJhBclIxMKnpYSITx0fMEPosCc3ghVx3yvUYAQ=="}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","Line truncated
+{"parentUuid":"dd93c7f8-826a-4eb8-8674-31eae9f9214b","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"text","text":"Critical findings: Documents is **not** OneDrive-backed, Recycle Bin bypassed. The unpushed commits only survive if there's a Volume Shadow Copy / restore point, or the GitHub remote. Let me check both recovery avenues:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"0310623c-db69-4a0d-8002-967d7527480d","timestamp":"2026-07-07T15:42:38.597Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"0310623c-db69-4a0d-8002-967d7527480d","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01N4eBeWoctiXCTxxptNQgm7","name":"PowerShell","input":{"command":"Write-Output \"=== Restore points ===\"; try { Get-ComputerRestorePoint -ErrorAction Stop | Format-Table SequenceNumber, CreationTime, Description -Auto } catch { \"restore-point query: $($_.Exception.Message)\" }\nWrite-Output \"=== vssadmin shadows ===\"; & vssadmin list shadows 2>&1 | Select-Object -First 30\nWrite-Output \"=== Previous Versions dir listing via wmic ===\"; & cmd /c \"vssadmin list shadowstorage\" 2>&1 | Select-Object -First 15\nWrite-Output \"=== git remote reachable? ===\"; & git ls-remote https://github.com/jhd3197/ServerKit.git 2>&1 | Select-Object -First 15","description":"Check restore points, shadow copies, remote"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","timestamp":"2026-07-07T15:42:42.234Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"type":"last-prompt","lastPrompt":"# ServerKit Plan Executor  > Set the plan path below, paste this whole file as your message. Claude executes every > phase, one by one, with subagents — no questions, no stopping early.  ## PLAN  ```…","leafUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"type":"ai-title","aiTitle":"Execute ServerKit staging testbed plan","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"type":"mode","mode":"normal","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"parentUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","isSidechain":false,"promptId":"487ba573-d3df-4194-aced-415c39485354","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command.  Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command.  Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","is_error":true,"tool_use_id":"toolu_01N4eBeWoctiXCTxxptNQgm7"}]},"uuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","timestamp":"2026-07-07T15:42:46.010Z","toolUseResult":"Error: Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command.  Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command.  Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","sourceToolAssistantUUID":"e1c3e155-b961-4b17-8951-00a5b7570c2c","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EpIiCmMIDxgCKkCM83hkUpKsTAdmjQp4vaBEBrTs/q6o49h7HnmFvPTNFVbiGnNcjzlGZQKnB3yabCxxeN+4KfpbPbWdBrwpeEHHMg9jbGF1ZGUtb3B1cy00LTg4AEIIdGhpbmtpbmcSDNo1GA/83N9NkvkQ9RoMo5aU3Kqe56UyV/CkIjBX8QcqeyjcOI2tVS8kHUG9BhsEUc0iJlFz5c8n7vjSP4MFScy6QFD/jjsSpTeZjUYq3CCUKlpZbUpsH4poqn9OY4X5Xeh4vFU+JCmMgedHIBx3xMjzEu2mBmaC0wOPH21TF5Mk4piWemSm8oVOEniiOAf2FC4tCsvOAWK3hdXKmxeBiKKI2G8z8j6lGf4etRGWbYiA6VdwTBiUPAB7bEJfRLbC1Yl+vsqTaCtx/VSj9bEGtjS9UaJdnU629rJmTZauDkfylfLbBaZKzDeaa4UVxQZt1FKhHFLlEvaZ9A31kxalIGnSWA36z4/vL0gBi0bpWecj+9Sct/zPYcSVD4edSUq1za9TTKjB57Sy5Ycv1xhvntTYoFxvKPSvSKqRycLYsKQkx0+BL5mVJvloE6mL1aOR4IcqJco3/uyOzxwITjBy3T9Hp8yqwNT2/2kYr1x3aKxU+XelNTjb5bTRwf3Q2iqin9rlggdEgwO7MpPqpHSxXIDL0Cq5PU4x/7cM1mCF0FX5P2kk6jD7ZhO9sZFOCp3UntusycVrQYgthciBedqnHT1UcvAWMJ7sLIqxmtqax50oUF++7plG69QXNEUbzzahJ5uLJmx7oOUuSX/CmD4H4j0yqjmUSR48m5EtdTv6hZJS5lmapQms93Nf4Bva0aSBPNVJ5spKSzYPY8Jm3ksOof/mnQl3t0iFRTIP5JxKFl4waT3aCVslD2IpcrnEjkDxyHs+c+OXViU2C2P+dPodnpKHEdFShZnWahR4FxB3TkhcGePnPVb0+t5/2jT+8ZflgvDMfSOp79vzypgcPBwxaHHiYGQx/xRBJm+W/COvYnub9U0JeBq5Q6N/dg8ss7ieWCf1hohTHMjeY74NDb8VBBYU6iliD9xMOh+yZSXOv+NIdjLCXZ3ievPFvxHUT4q5Yh46XEYfcfBj1iiSCPvf1xMM+oKjoBMGDhm/c9VhBHEitSKzcmdQ380tPho5F0vSx1sfhVbjU2Hx8QTlCDzKwvDvg8FOW26dbjCr182M4Rqs7y9r/82uZMBBi2tfnx0A+VVr+d4tA2tcoexWBKfgEt80//yRMtScgMXpsQibkxf/rlPjUCO8gGJql5dKB0BS+CTqQGhaL9q9kDoL1fjFdfzpyGt3rhTxjynRrgJsQBVsVymuafrhsehl6+bQQYiQp0NYW+PtoadbPYcErSZoPJBDMezG0zLwf1pj566vtd896tHP7VJduEC2Vq376M0N/nE/VQXzVnEikeR/v5XZH/y/ZcB+JLrs63Ng3Kht/vFUncJp71OAmhKZD8slEs78i0XYOzty1OxgaGQJzZ1Qy2y/4bvW9k+xvQllgm5WzmFtLYspP63WEFSlBNu8SNUKw8iC9M/sFg3G2xu0q56WbiM6bYQ4i63tzLVH7JqEG87CbdejCdv6VXco1QUNH7BW8x+IX9toaPYsd2jbyHqv3vThsue3mK0U+KDibDbzKc27SHmhR1re8CTLrINaKb+u+U7A1Cp0nvPyIZxVA+d34E3FCG3ctqMN49tAXybRp+3FWOMx9iQlR9Hf9CUxf6Syfox0mvUG2qBnog+bprOh1fZgIdj+imEYfXcyXedEljl7ZQmmC1nmfiHFcN8T6CGZRrTrxLYKUG3Oj9bo9YwG3TQRpWy91HzdZZ1S974eB8cplHDfAjjRVjhmuAeB+t0zr5FBokxODUjHpVism43WHH+tknTtqHUkk6HA5sFyOhfHi0AODbiBGAIkKUz2v4yrLKXSQWuTgbxRU+9ejCge4F1PlFwDB8pmUc83y+06LIZH+AEpxAWcWwYQ1m980bGnpXwgU19zMqMgDXP02pM1/G4zOeK9I197OP+6GT0kkduBcVDOS1Nrd4c2LcvWUBYCFhfsJUXIO13tON9vwTCom1dJTFYvZUHj7op5iwYcn1StqbhglXDro+ZQf0utu7vkRrCaM7ZBZWrzXcK8XMLyoChS4bwUk1p5teyQbBRypp8L/sV1vuYYShmfoxewPzlVYvCD1O3nWtP63DwR1K6MHboojE1cue/6qucVViLamob76T3QRe5RJ1/o2Ec9ZFq1BBlSnDMNVqapEhSQJURfVesxsM+UUuGYGkj+278wT79/D9YFo+cKlUn2gk1S8xZ+cNIFSYDuIb9Fypby533cVXk+mVwzTzpNzzOER/Da4zHWNDkmSSUhQktUNP3hs8KaYuNkXdZzeIVkDxU5sXtD0yfv33ir2zRYDGoDveEs9KJ4IUhueqBD/PYj++Ur9MhZ3W+sZG7W94jC3hwNrldc1FxgP8GnkEyUUjYscLzmD5Lc1Ii2ahQKpGS4n5CaTd5+pC1a5k5HbG+O38rrlzzY4ocuaAht0MH5Ya0babz2TKS673sjPtX2OrIguY67x7PE9ffG17qhC8lDK4UJXYYHK5Vs27Qx/AcKXKj070N6GfqV+k79yPRor239fBG3qn0xQKb1WuWbDfrsNeWL8j0Dh/5s/A9VJxuue2ExlusdBkFCsuM8eFdDbn79fMqTDFoWles1Rb3cHWnJZ4DxqzA5X8B3VKmTGi24CL9KnP9NWMgYjKFtzhVSR8eaa3zJNpcRktMDz5nRw8CV7jx+yuL0QK4YsnOwSOSjR0kqNKMvaaHX+59F875aOjJc8S9rk+0hsQaBgCOmCAjRqp2iAR6Pp3q2qdetNVGWRD9Y+rqq5v9i2tWpuV7v8bsLF8kjGOxyo+r8zwDtYaKuHxCrwisviaXrSd42/XGsbpdMG52wN4oSqd9FjEs+xVui9LNxMGy7glmASKDeyf/eQBrFFHMe82SAYzP+Z41RgW/pXwofoYxs5+aFodL18lpoJp+TLQoVV/Q1gPEJ+u0j9C34nzhbspF2q14DZLKJa9tULjxVaa0wLsKjKWiSE0pusL78zny/02LmFXlYm+Y4Em6v02nzusvYC56lP6l6bKe317D+J1YN8dyQO1LM5E55wUXkZIfF2pw4sdbupNFvnZC3YxgoR/7+4JPdkMe6KLkxSMZtrsMr44HmVm/F/xJQWC38c0EmWMzczX8YlXTxR4/72SAEeO6uNdvo+jzc2Wlh6qOZBlQN/O4LuDfdPu/bpcF9V7VHLsAq74L8uWRwqpMdp4KY5SUyX6YH8kLm818vdo5fkDyiEmy5VujBVlwrZmb3TslcY2zKag3Ga+aEtF18efaHVseoQphsH4mV/O8l7GH8simO0TysDXTCYwEkboZpXGFkvAmKaQRhwSWMXS6XF0CZkdfwHWo9sLQTrLq7CSeblwbpAvmrjSCb5cIe0Ngr5jtoJA/c0eCLh7ZS2OG2QKTgyQeRSOaXWxQsVGzMHyF/Xv1UnajStSSktgJYOKX0QM+Oxy6egaJuS4Mb5IRssYLKSvQ2w4hydViXEop/wUyI3v7NOz/js7YzXZ7zPvFjYwM+4flijh7KH5A0QMCp4BQQ7YOOOzN/snLEqLKSOOy32WKEyCnNL698dWhNfZOT3SzRhtinYJU3aL9HPV36apg3TgNWfBfM8IfSbq2Q8hGa0B6VjT4wmjIhmt5Sl6yKxxNho4XBok9gjTi8aNdse+ZRSUBet0uDlObrm+ZqX58ppfHLlF6LFp3+zpefiYGxlhZQu78ykIdBXAeU1e1sxi55yHSkYfsaE6Dt5MmBZu31WABjRC/hYn1nM7D/jvXqq+6W3TP9r6ES8mCE/JgiNS+bBTmu7VfIRhDKUZKvAxT7Sq5kdY9Sqa7rck4ICK6roEv6dfkjCqfURlwseGzb7EVni8oHVBp7qMoUT3hqt0XiIPttcKjOVqiAXZ9ArWW91A1gpin2DQRKoQDmPpWa5SMdjxmhxPzdp6MtjFsCCXOEf/wU1r94y8V84U+hlKxzZQqWnrIumpFTUdyqa0oa9zdMxlBGGy4AZIgHgKAlNA9CD30CVD29Cr0oYw3ZML3a5+HAeMGjwJ+e5Lpjx7S7tGldYc/TD4s3uZi6vbw8SlHSwfxskYdXdFq6DffMP5dKep2QlvKSUClODXtKnj7ETM6rg+4nvv1DtJsPHse2tRodJ2vtNlrvTZlCku15Hizi8GlZwX+NmFVI/y+bHvVlylnsAMKSH68IHkqy2YY5a0LWzihQp0dbMPGzSPXLH+zC8TmbRNJGRI/N725YgNkSm0fjE1AEmW7tKRSg1pOauJ8Ld/vUASy1nKsaBMvAjQvRSgrdN+djfSt8eM3J6srvpbyUGMhmNNXCmcVfRr2jCKnr1uxTRBRpn96LvMa4k6RJ2wKL+/rI9N9Mrpsj2IrgKJmOC/zUo5ijtuBjuelV9lxaB+iWpXIiVKz5wsPKcrcmC4Q7mNPQDtlctQFmlpiDNhwIyuaNEOSXmVseuMrGBoUpgPp90p7BmEikcggYLObwaQ2C3fYUJ0aWXJrTgKwYoIGn4JFYpl/5VOQ7cRk30942wUtdevm3WvieyS2CynQ6To74noZ7bO3qLeFRrA97nadPuyRmo1vaHXTb8bxO+ljXPgaWJD6QBCXFTTkQZGctOzIYnlHDIiYXl1D/SzTDdD8ZGJdxiEll3OaNb8qNdsCefBdaE+y6eut6ATHHZssDTUBGCiX3yV8+lydxLine truncated
+{"parentUuid":"1a34a991-1644-4726-8618-96c08a2cba2a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"text","text":"The situation is now clear and serious. Let me do one more **non-elevated** recovery check (File History), then I must stop and report to you — this needs your decision and elevated access:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2797,"cache_read_input_tokens":261476,"output_tokens":2090,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2797,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":2090,"cache_read_input_tokens":261476,"cache_creation_input_tokens":2797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2797},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxCdtcwMwH94on8fJ6v","type":"assistant","uuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","timestamp":"2026-07-07T15:43:14.826Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01MWz2Ja4hPnZG7qF1PHtNT2","name":"PowerShell","input":{"command":"Write-Output \"=== File History configured? ===\"\n$fh = Get-ItemProperty 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\FileHistory' -ErrorAction SilentlyCon