cozystack/internal/backupcontroller/restorejob_controller.go

package backupcontroller

import (
	"context"
	"fmt"
	"net/http"
	"slices"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/record"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/log"

	strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)

const restoreJobFinalizer = "backups.cozystack.io/cleanup"

// RestoreJobReconciler reconciles RestoreJob objects.
// It routes RestoreJobs to strategy-specific handlers based on the strategy
// referenced in the Backup that the RestoreJob is restoring from.
type RestoreJobReconciler struct {
	client.Client
	dynamic.Interface
	meta.RESTMapper
	Scheme   *runtime.Scheme
	Recorder record.EventRecorder
	// Clientset reads Pod logs (the controller-runtime cache client cannot);
	// the CNPG restore driver uses it to read a bootstrap-recovery pod's log
	// and tell an unreachable point-in-time target apart from a transient
	// recovery-pod failure. Wired in SetupWithManager; nil-safe at call sites.
	Clientset kubernetes.Interface
	// readPodLog is the seam the CNPG restore driver reads a recovery pod's log
	// through. Defaults to readPodContainerLog (which uses Clientset); tests
	// inject a stub so the log-classification path - including the Forbidden
	// (missing pods/log RBAC) branch - is unit-testable without a live cluster.
	readPodLog        func(ctx context.Context, namespace, podName, container string) (string, error)
	CredentialsConfig BackupCredentialsConfig
	// APIReader is the manager's uncached reader. The keepOriginalPVC rename
	// reads back objects it has just written, which the cache may not have
	// seen yet. Wired in SetupWithManager.
	APIReader client.Reader
}

func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	logger := log.FromContext(ctx)
	logger.Info("reconciling RestoreJob", "namespace", req.Namespace, "name", req.Name)

	restoreJob := &backupsv1alpha1.RestoreJob{}
	err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, restoreJob)
	if err != nil {
		if apierrors.IsNotFound(err) {
			logger.V(1).Info("RestoreJob not found, skipping")
			return ctrl.Result{}, nil
		}
		logger.Error(err, "failed to get RestoreJob")
		return ctrl.Result{}, err
	}

	// Handle deletion: dispatch cleanup by strategy. The Velero strategy
	// owns side-state (Velero Restore + ConfigMap in cozy-velero); the CNPG
	// and Job strategies own no namespaced artifacts that survive RestoreJob
	// deletion, so their cleanup is a no-op. Reading the referenced Backup
	// to identify the strategy is best-effort - a missing/unreadable Backup
	// must not block finalizer removal, otherwise the RestoreJob would be
	// stuck in Terminating forever.
	if !restoreJob.DeletionTimestamp.IsZero() {
		if controllerutil.ContainsFinalizer(restoreJob, restoreJobFinalizer) {
			r.cleanupOnDelete(ctx, restoreJob)
			controllerutil.RemoveFinalizer(restoreJob, restoreJobFinalizer)
			if err := r.Update(ctx, restoreJob); err != nil {
				return ctrl.Result{}, err
			}
			logger.V(1).Info("removed finalizer and cleaned up strategy-owned side state", "restoreJob", restoreJob.Name)
		}
		return ctrl.Result{}, nil
	}

	// Ensure finalizer is present
	if !controllerutil.ContainsFinalizer(restoreJob, restoreJobFinalizer) {
		controllerutil.AddFinalizer(restoreJob, restoreJobFinalizer)
		if err := r.Update(ctx, restoreJob); err != nil {
			return ctrl.Result{}, err
		}
	}

	// If already completed, no need to reconcile
	if restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded ||
		restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
		logger.V(1).Info("RestoreJob already completed, skipping", "phase", restoreJob.Status.Phase)
		return ctrl.Result{}, nil
	}

	// Step 1: Fetch the referenced Backup
	backup := &backupsv1alpha1.Backup{}
	backupKey := types.NamespacedName{Namespace: req.Namespace, Name: restoreJob.Spec.BackupRef.Name}
	if err := r.Get(ctx, backupKey, backup); err != nil {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to get Backup: %v", err))
	}

	// Step 2: Determine effective strategy from backup.spec.strategyRef.
	// Resolve and filter BEFORE projecting credentials so RestoreJobs
	// targeting third-party drivers do not materialise cozy-backups-creds
	// in the tenant namespace.
	if backup.Spec.StrategyRef.APIGroup == nil {
		return r.markRestoreJobFailed(ctx, restoreJob, "Backup has nil StrategyRef.APIGroup")
	}

	if *backup.Spec.StrategyRef.APIGroup != strategyv1alpha1.GroupVersion.Group {
		return r.markRestoreJobFailed(ctx, restoreJob,
			fmt.Sprintf("StrategyRef.APIGroup doesn't match: %s", *backup.Spec.StrategyRef.APIGroup))
	}

	// Reject unsupported Kinds inside the platform APIGroup BEFORE
	// credentials projection (mirrors BackupJob path). Without this guard
	// a RestoreJob against a Backup whose strategyRef.Kind is unknown to
	// this controller would leak cozy-backups-creds into the tenant
	// namespace and then fall through to markRestoreJobFailed with the
	// Secret already in place.
	{
		supported := slices.Contains(supportedBackupStrategyKinds(), backup.Spec.StrategyRef.Kind)
		if !supported {
			return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind))
		}
	}

	// Step 3: Project the platform-managed S3 credentials into the tenant
	// namespace so default Strategy CRs (re-rendered at restore time) can
	// reference a deterministic Secret name. Idempotent / no-op when
	// projection is not configured. Transient errors requeue; terminal
	// misconfig (target owned by someone else, source malformed) is
	// surfaced as Failed.
	if err := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, restoreJob.Namespace); err != nil {
		return r.handleProjectionError(ctx, restoreJob, err)
	}

	logger.Info("processing RestoreJob", "restorejob", restoreJob.Name, "backup", backup.Name, "strategyKind", backup.Spec.StrategyRef.Kind)
	switch backup.Spec.StrategyRef.Kind {
	case strategyv1alpha1.JobStrategyKind:
		return r.reconcileJobRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.VeleroStrategyKind:
		return r.reconcileVeleroRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.CNPGStrategyKind:
		return r.reconcileCNPGRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.AltinityStrategyKind:
		return r.reconcileAltinityRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.MariaDBStrategyKind:
		return r.reconcileMariaDBRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.MongoDBStrategyKind:
		return r.reconcileMongoDBRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.FoundationDBStrategyKind:
		return r.reconcileFoundationDBRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.EtcdStrategyKind:
		return r.reconcileEtcdRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.RabbitmqStrategyKind:
		return r.reconcileRabbitmqRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.RedisStrategyKind:
		return r.reconcileRedisRestore(ctx, restoreJob, backup)
	case strategyv1alpha1.KafkaStrategyKind:
		return r.reconcileKafkaRestore(ctx, restoreJob, backup)
	default:
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind))
	}
}

// SetupWithManager registers our controller with the Manager and sets up watches.
func (r *RestoreJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
	r.APIReader = mgr.GetAPIReader()
	cfg := mgr.GetConfig()
	var err error
	if r.Interface, err = dynamic.NewForConfig(cfg); err != nil {
		return err
	}
	if r.Clientset, err = kubernetes.NewForConfig(cfg); err != nil {
		return err
	}
	var h *http.Client
	if h, err = rest.HTTPClientFor(cfg); err != nil {
		return err
	}
	if r.RESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, h); err != nil {
		return err
	}
	return ctrl.NewControllerManagedBy(mgr).
		For(&backupsv1alpha1.RestoreJob{}).
		Complete(r)
}

// apiReader returns the uncached reader. Reconcilers built by hand in unit
// tests have none and fall back to their fake Client, which has no cache to
// bypass.
func (r *RestoreJobReconciler) apiReader() client.Reader {
	if r.APIReader != nil {
		return r.APIReader
	}
	return r.Client
}

// handleProjectionError classifies a credentials-projection error as
// transient (requeue with Ready=False/CredentialsProjectionPending) or
// terminal (markRestoreJobFailed). Mirrors the BackupJob variant — both
// rely on IsTransient + CredentialsProjectionRequeue so a single source
// of truth governs the backoff and the transient-vs-terminal split.
func (r *RestoreJobReconciler) handleProjectionError(ctx context.Context, rj *backupsv1alpha1.RestoreJob, err error) (ctrl.Result, error) {
	logger := getLogger(ctx)
	credentialsProjectionFailures.WithLabelValues(rj.Namespace, classifyReason(err)).Inc()
	if IsTransient(err) {
		meta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{
			Type:    "Ready",
			Status:  metav1.ConditionFalse,
			Reason:  "CredentialsProjectionPending",
			Message: err.Error(),
		})
		if updateErr := r.Status().Update(ctx, rj); updateErr != nil {
			logger.Error(updateErr, "failed to update RestoreJob status to projection-pending")
		}
		logger.Info("restore credentials projection transient failure; requeueing", "message", err.Error())
		return ctrl.Result{RequeueAfter: CredentialsProjectionRequeue}, nil
	}
	return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf("failed to project backup credentials: %v", err))
}

// markRestoreJobFailed updates the RestoreJob status to Failed with the given message.
//
// Coupling note: a failure that fires before reconcileCNPG's StartedAt block
// has set restoreJob.Status.StartedAt leaves StartedAt nil. The CNPG path's
// deadline gates (cnpgWALArchiveDeadline, options.effectiveRestoreDeadline)
// only fire once StartedAt is set, so an early-failure retry that gets all
// the way through to the StartedAt block restarts the deadline budget from
// the retry's StartedAt - intentional, since retries are user-driven and a
// fresh budget is what users expect after correcting the cause of the
// previous failure.
func (r *RestoreJobReconciler) markRestoreJobFailed(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) {
	return r.markRestoreJobFailedReason(ctx, restoreJob, "RestoreFailed", message)
}

// markRestoreJobFailedReason is markRestoreJobFailed with a caller-chosen
// Ready-condition Reason so a driver can distinguish failure classes a tenant
// acts on differently (e.g. RecoveryTargetUnreachable, which points at the
// recoverable window rather than at the controller).
func (r *RestoreJobReconciler) markRestoreJobFailedReason(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, reason, message string) (ctrl.Result, error) {
	logger := getLogger(ctx)
	now := metav1.Now()
	restoreJob.Status.CompletedAt = &now
	restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseFailed
	restoreJob.Status.Message = message

	// SetStatusCondition keeps Conditions matching the +listType=map +listMapKey=type
	// CRD contract: a previously-set Ready condition (e.g. from a transient
	// retry path that flipped through ConditionFalse with a different
	// Reason) is updated in-place rather than appended, and LastTransitionTime
	// is preserved unless Status changes.
	meta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
		Type:    "Ready",
		Status:  metav1.ConditionFalse,
		Reason:  reason,
		Message: message,
	})

	if err := r.Status().Update(ctx, restoreJob); err != nil {
		logger.Error(err, "failed to update RestoreJob status to Failed")
		return ctrl.Result{}, err
	}
	logger.Debug("RestoreJob failed", "message", message)
	return ctrl.Result{}, nil
}

// requeueRestoreStrategyNotReady mirrors BackupJobReconciler.requeueStrategyNotReady
// for the restore path. Surfaces a transient Ready=False/StrategyNotReady
// when the Strategy CR referenced by Backup.spec.strategyRef.name is
// not (yet) present — same bootstrap-window self-heal contract as the
// backup path, and the same StrategyNotReadyDeadline bound so a permanently
// missing strategy fails closed instead of requeuing forever.
func (r *RestoreJobReconciler) requeueRestoreStrategyNotReady(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, strategyName string) (ctrl.Result, error) {
	logger := getLogger(ctx)
	if strategyNotReadyDeadlineExceeded(restoreJob.Status.StartedAt) {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
			"Strategy %q referenced by Backup was not provisioned within %s; check the strategyRef name or the platform backup-storage bootstrap",
			strategyName, StrategyNotReadyDeadline))
	}
	meta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
		Type:    "Ready",
		Status:  metav1.ConditionFalse,
		Reason:  "StrategyNotReady",
		Message: fmt.Sprintf("Strategy %q referenced by Backup is not yet provisioned; the platform may still be initialising backup storage", strategyName),
	})
	if updateErr := r.Status().Update(ctx, restoreJob); updateErr != nil {
		logger.Error(updateErr, "failed to update RestoreJob status to StrategyNotReady")
	}
	return ctrl.Result{RequeueAfter: CredentialsProjectionRequeue}, nil
}

// cleanupResourceModifierConfigMaps deletes resource modifier ConfigMaps owned
// by this RestoreJob. Called on completion (success or failure) to avoid leaking
// ConfigMaps in cozy-velero when RestoreJobs are not immediately deleted.
func (r *RestoreJobReconciler) cleanupResourceModifierConfigMaps(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {
	logger := log.FromContext(ctx)
	opts := []client.DeleteAllOfOption{
		client.InNamespace(veleroNamespace),
		client.MatchingLabels{
			backupsv1alpha1.OwningJobNameLabel:      restoreJob.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
		},
	}
	if err := r.DeleteAllOf(ctx, &corev1.ConfigMap{}, opts...); err != nil {
		logger.Error(err, "failed to clean up resourceModifiers ConfigMap(s)")
	}
}

// cleanupOnDelete dispatches RestoreJob deletion cleanup to the strategy
// driver that produced the side state. Velero RestoreJobs created Velero
// Restore CRs and resourceModifiers ConfigMaps in cozy-velero; CNPG and
// Job RestoreJobs do not. Dispatching avoids issuing pointless DeleteAllOf
// calls against cozy-velero for non-Velero RestoreJobs and prevents the
// "finalizer name says velero but the job is CNPG" UX papercut.
func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {
	logger := log.FromContext(ctx)
	kind, err := strategyKindForRestoreJob(ctx, r.Client, restoreJob)
	if err != nil {
		// The referenced Backup is gone or unreadable, so we cannot tell
		// which driver produced this RestoreJob. Speculatively reap any
		// Velero Restore it may have created, but quietly: this is not
		// necessarily a Velero job, so a failure (or a cluster with no
		// Velero CRDs) must not surface a CleanupFailed event.
		logger.V(1).Info("RestoreJob Backup unreadable; attempting best-effort Velero cleanup",
			"restoreJob", restoreJob.Name, "error", err.Error())
		r.cleanupStrayVeleroRestore(ctx, restoreJob)
		return
	}
	logger.V(1).Info("dispatching RestoreJob cleanup", "restoreJob", restoreJob.Name, "strategy", kind)
	switch kind {
	case strategyv1alpha1.VeleroStrategyKind:
		r.cleanupVeleroRestore(ctx, restoreJob)

	case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind:
		// Nothing to clean up: these drivers don't materialise namespaced
		// artifacts that outlive the RestoreJob. (Etcd: the operator-side
		// EtcdCluster is owned by the source HelmRelease, and the
		// EtcdClusterSpecCaptured / TargetPurged conditions live on the
		// RestoreJob itself - all gone with the parent. Kafka: the metadata
		// restore runs a one-shot Job owned by the RestoreJob and writes only
		// to the Kafka cluster via the Admin API.)
	default:
		// Readable Backup, but an unrecognised strategy kind — not Velero
		// as far as we can tell. Speculatively reap a stray labelled Velero
		// Restore if one exists, quietly: don't emit a CleanupFailed event
		// for a strategy this controller doesn't own.
		logger.V(1).Info("RestoreJob has unrecognised strategy kind; attempting best-effort Velero cleanup",
			"restoreJob", restoreJob.Name, "kind", kind)
		r.cleanupStrayVeleroRestore(ctx, restoreJob)
	}
}

// strategyKindForRestoreJob looks up the strategy kind via the referenced
// Backup. Returns a non-nil error when the Backup is missing or unreadable
// so callers can distinguish "cannot tell" from a readable Backup whose
// strategy kind is simply one they don't handle.
func strategyKindForRestoreJob(ctx context.Context, c client.Client, restoreJob *backupsv1alpha1.RestoreJob) (string, error) {
	backup := &backupsv1alpha1.Backup{}
	key := types.NamespacedName{Namespace: restoreJob.Namespace, Name: restoreJob.Spec.BackupRef.Name}
	if err := c.Get(ctx, key, backup); err != nil {
		return "", err
	}
	return backup.Spec.StrategyRef.Kind, nil
}

// cleanupVeleroRestore deletes all Velero Restores and resourceModifier
// ConfigMaps owned by this RestoreJob (identified by labels). Used on the
// known-Velero path, so a failed Restore delete is surfaced as a
// CleanupFailed event.
func (r *RestoreJobReconciler) cleanupVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {
	r.deleteVeleroRestoreArtifacts(ctx, restoreJob, true)
}

// cleanupStrayVeleroRestore is the speculative variant used when we cannot
// positively identify the RestoreJob as Velero (Backup unreadable, or a
// readable Backup with an unrecognised strategy kind). It acts only when a
// labelled Velero Restore actually exists, so on a cluster without the
// Velero CRDs — a supported configuration, see values.yaml velero.bslEnabled
// — the List fails and we skip silently. It never emits a CleanupFailed
// event, because the RestoreJob may not be a Velero job at all.
func (r *RestoreJobReconciler) cleanupStrayVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {
	logger := log.FromContext(ctx)
	list := &velerov1.RestoreList{}
	if err := r.List(ctx, list,
		client.InNamespace(veleroNamespace),
		client.MatchingLabels{
			backupsv1alpha1.OwningJobNameLabel:      restoreJob.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
		},
	); err != nil {
		logger.V(1).Info("skipping speculative Velero cleanup (Velero absent or List failed)",
			"restoreJob", restoreJob.Name, "reason", err.Error())
		return
	}
	if len(list.Items) == 0 {
		return
	}
	// A stray labelled Velero Restore exists — this RestoreJob was a Velero
	// job after all. Reap it (and the resourceModifiers ConfigMaps), still
	// best-effort and event-free.
	r.deleteVeleroRestoreArtifacts(ctx, restoreJob, false)
}

// deleteVeleroRestoreArtifacts deletes the Velero Restores and
// resourceModifier ConfigMaps owned by this RestoreJob. When emitEvent is
// true a failed Restore delete is surfaced as a CleanupFailed Warning;
// speculative callers pass false to stay quiet.
func (r *RestoreJobReconciler) deleteVeleroRestoreArtifacts(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, emitEvent bool) {
	logger := log.FromContext(ctx)
	opts := []client.DeleteAllOfOption{
		client.InNamespace(veleroNamespace),
		client.MatchingLabels{
			backupsv1alpha1.OwningJobNameLabel:      restoreJob.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
		},
	}

	if err := r.DeleteAllOf(ctx, &velerov1.Restore{}, opts...); err != nil {
		logger.Error(err, "failed to delete Velero Restore(s)")
		if emitEvent {
			r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "CleanupFailed",
				fmt.Sprintf("Failed to delete Velero Restore: %v", err))
		}
	}

	if err := r.DeleteAllOf(ctx, &corev1.ConfigMap{}, opts...); err != nil {
		logger.Error(err, "failed to delete resourceModifiers ConfigMap(s)")
	}
}