cozystack/internal/backupcontroller/jobstrategy_controller.go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller

import (
	"context"
	"fmt"
	"maps"
	"strings"
	"time"

	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apimeta "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

	strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
	"github.com/cozystack/cozystack/internal/template"
)

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const (
	// Mode label / values applied to the rendered batch/v1.Job so RBAC and
	// debugging tools can distinguish backup vs restore runs at a glance.
	// Mirrors the Altinity driver's altinityLabelMode convention.
	jobStrategyLabelMode   = "job.strategy.backups.cozystack.io/mode"
	jobStrategyModeBackup  = "backup"
	jobStrategyModeRestore = "restore"

	// Driver-metadata key prefix used to round-trip BackupClassStrategy
	// parameters through the Backup artifact, so a later RestoreJob can
	// re-render the strategy template with the same parameter values that
	// were in effect at backup time.
	jobStrategyParamPrefix = "job.strategy.backups.cozystack.io/parameter/"

	// Polling cadence for the Job lifecycle. Mirrors the Altinity / CNPG /
	// Velero strategy cadence so behaviour is uniform across drivers.
	jobStrategyPollInterval = 5 * time.Second
)

// jobStrategyParameters extracts BackupClassStrategy parameters from a Backup's
// DriverMetadata. Round-trips the values written by reconcileJob at backup time
// so a later RestoreJob can render the strategy template with the parameter
// snapshot in effect when the backup was taken. Mirrors backupParameters from
// the Altinity driver.
func jobStrategyParameters(b *backupsv1alpha1.Backup) map[string]string {
	out := map[string]string{}
	for k, v := range b.Spec.DriverMetadata {
		if !strings.HasPrefix(k, jobStrategyParamPrefix) {
			continue
		}
		paramKey := strings.TrimPrefix(k, jobStrategyParamPrefix)
		if paramKey == "" {
			continue
		}
		out[paramKey] = v
	}
	return out
}

// renderJobTemplate runs the strategy's PodTemplateSpec through the
// repository's text/template engine with the same context shape as the
// Altinity / CNPG drivers: every string field is templated against the
// application object, the release shorthand, the run mode, and the resolved
// parameters. Mirrors renderAltinityTemplate; the Job strategy carries the
// same corev1.PodTemplateSpec shape and reuses the identical context, but
// keeps its own per-driver render entrypoint so each strategy file owns its
// rendering surface (matches the renderCNPGTemplate / renderEtcdTemplate /
// renderFoundationDBTemplate convention).
func renderJobTemplate(
	tmpl corev1.PodTemplateSpec,
	app map[string]any,
	releaseName, releaseNamespace, mode string,
	parameters map[string]string,
	backup *backupsv1alpha1.Backup,
) (*corev1.PodTemplateSpec, error) {
	ctxMap := map[string]any{
		"Application": app,
		"Release": map[string]string{
			"Name":      releaseName,
			"Namespace": releaseNamespace,
		},
		"Mode":       mode,
		"Parameters": parameters,
	}
	if backup != nil {
		// Expose ApplicationRef so restore strategies can reference the
		// SOURCE release name even when restoring into a differently-named
		// target. Mirrors the .Backup context exposed by the Altinity
		// driver - see renderAltinityTemplate for the longer rationale.
		sourceAPIGroup := ""
		if backup.Spec.ApplicationRef.APIGroup != nil {
			sourceAPIGroup = *backup.Spec.ApplicationRef.APIGroup
		}
		ctxMap["Backup"] = map[string]any{
			"Name":      backup.Name,
			"Namespace": backup.Namespace,
			"ApplicationRef": map[string]string{
				"APIGroup": sourceAPIGroup,
				"Kind":     backup.Spec.ApplicationRef.Kind,
				"Name":     backup.Spec.ApplicationRef.Name,
			},
		}
	}
	return template.Template(&tmpl, ctxMap)
}

// ---------------------------------------------------------------------------
// BackupJob path
// ---------------------------------------------------------------------------

// reconcileJob drives a BackupJob whose resolved strategy is a
// strategy.backups.cozystack.io/Job. The Job strategy is the generic, app-
// agnostic driver: it renders the strategy's PodTemplateSpec against the live
// application object and runs it once as a batch/v1.Job, treating Job
// completion as backup success. The Altinity driver is a specialisation of
// this flow with a ClickHouse-specific applicationRef gate; the Job driver
// applies no such gate and runs for any applicationRef.
func (r *BackupJobReconciler) reconcileJob(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
	logger := getLogger(ctx)
	logger.Debug("reconciling Job strategy", "backupjob", j.Name, "phase", j.Status.Phase)

	if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
		j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
		return ctrl.Result{}, nil
	}

	// First-reconcile bookkeeping. Refetch before writing StartedAt so a
	// stale informer cache cannot silently slide the timestamp forward
	// across reconciles, mirroring reconcileAltinity / reconcileCNPG.
	if j.Status.StartedAt == nil {
		fresh := &backupsv1alpha1.BackupJob{}
		if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil {
			return ctrl.Result{}, err
		}
		if fresh.Status.StartedAt != nil {
			j.Status.StartedAt = fresh.Status.StartedAt
			j.Status.Phase = fresh.Status.Phase
		} else {
			base := fresh.DeepCopy()
			now := metav1.Now()
			fresh.Status.StartedAt = &now
			fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
			if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
				return ctrl.Result{}, err
			}
			// Requeue so the next reconcile re-Gets j with the post-patch
			// ResourceVersion. Copying StartedAt/Phase back into the local j
			// here would leave any subsequent r.Status().Update calls in the
			// same reconcile carrying the pre-patch ResourceVersion and
			// failing with Conflict. Mirrors reconcileAltinity / reconcileCNPG.
			return ctrl.Result{RequeueAfter: jobStrategyPollInterval}, nil
		}
	}

	strategy := &strategyv1alpha1.Job{}
	if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil {
		if apierrors.IsNotFound(err) {
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Job strategy not found: %s", resolved.StrategyRef.Name))
		}
		return ctrl.Result{}, err
	}

	app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef)
	if err != nil {
		// A missing object (IsNotFound) and an unmappable kind
		// (IsNoMatchError - a typo in applicationRef.kind, or a CRD that
		// isn't installed) are both terminal: retrying cannot fix them.
		// The Job driver, unlike the specialised drivers, has no
		// applicationRef.kind gate, so it is the path that forwards
		// arbitrary kinds into RESTMapping and must classify NoKindMatch
		// itself - otherwise the BackupJob requeues forever in Running.
		if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf("application not found or kind not registered: %s/%s (kind=%q)", j.Namespace, j.Spec.ApplicationRef.Name, j.Spec.ApplicationRef.Kind))
		}
		return ctrl.Result{}, err
	}

	rendered, err := renderJobTemplate(
		strategy.Spec.Template,
		app,
		j.Spec.ApplicationRef.Name,
		j.Namespace,
		jobStrategyModeBackup,
		resolved.Parameters,
		nil,
	)
	if err != nil {
		return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Job strategy: %v", err))
	}

	batchJob, err := r.ensureJobStrategyJob(ctx, j, j.Namespace, jobNameForBackupJob(j),
		jobStrategyModeBackup,
		map[string]string{
			backupsv1alpha1.OwningJobNameLabel:      j.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
		},
		rendered,
	)
	if err != nil {
		return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure batch/v1.Job: %v", err))
	}

	switch jobConditionState(batchJob) {
	case batchv1.JobComplete:
		if j.Status.BackupRef != nil {
			return ctrl.Result{}, nil
		}
		artifact, err := r.createJobBackupArtifact(ctx, j, resolved)
		if err != nil {
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
		}
		now := metav1.Now()
		j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
		j.Status.CompletedAt = &now
		j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
		apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
			Type:    "Ready",
			Status:  metav1.ConditionTrue,
			Reason:  "BackupCompleted",
			Message: "backup Job completed",
		})
		// A full Status().Update (rather than a MergeFrom patch) is safe here:
		// j is the object the parent Reconcile freshly Get()s on every pass, and
		// the only other status write in this driver - the StartedAt block above
		// - returns a requeue instead of writing j in the same pass. So by the
		// time control reaches this branch, j carries the current
		// ResourceVersion and no concurrent write from this driver can have
		// staled it. Mirrors reconcileAltinity.
		if err := r.Status().Update(ctx, j); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil

	case batchv1.JobFailed:
		message := jobFailureMessage(batchJob)
		if message == "" {
			message = "backup Job reported Failed"
		}
		return r.markBackupJobFailed(ctx, j, message)

	default:
		return ctrl.Result{RequeueAfter: jobStrategyPollInterval}, nil
	}
}

// ensureJobStrategyJob materialises a batch/v1.Job from the rendered
// PodTemplateSpec, idempotently. The Job is named deterministically per owning
// BackupJob, so a retried reconcile observes the existing Job rather than
// creating a duplicate, and carries a controllerRef so kube-gc collects it
// (and the running Pod) when the BackupJob is deleted. Mirrors
// ensureAltinityJob.
func (r *BackupJobReconciler) ensureJobStrategyJob(
	ctx context.Context,
	owner client.Object,
	namespace, name, mode string,
	ownerLabels map[string]string,
	rendered *corev1.PodTemplateSpec,
) (*batchv1.Job, error) {
	existing := &batchv1.Job{}
	err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
	if err == nil {
		return existing, nil
	}
	if !apierrors.IsNotFound(err) {
		return nil, err
	}

	labels := map[string]string{jobStrategyLabelMode: mode}
	maps.Copy(labels, ownerLabels)

	desired := buildJobStrategyBatchJob(namespace, name, labels, rendered)
	if err := controllerutil.SetControllerReference(owner, desired, r.Scheme); err != nil {
		return nil, fmt.Errorf("set controller reference on backup Job: %w", err)
	}
	if err := r.Create(ctx, desired); err != nil {
		if apierrors.IsAlreadyExists(err) {
			if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
				return nil, err
			}
			return existing, nil
		}
		return nil, err
	}
	return desired, nil
}

// buildJobStrategyBatchJob assembles the typed batch/v1.Job that wraps the
// rendered PodTemplateSpec. RestartPolicyNever and a small backoff cap match
// what a one-shot backup needs - the tool either succeeds or fails, retries
// inside the same Pod don't add value. Mirrors buildAltinityBatchJob.
func buildJobStrategyBatchJob(namespace, name string, labels map[string]string, rendered *corev1.PodTemplateSpec) *batchv1.Job {
	pod := *rendered.DeepCopy()
	if pod.Spec.RestartPolicy == "" {
		pod.Spec.RestartPolicy = corev1.RestartPolicyNever
	}
	if pod.Labels == nil {
		pod.Labels = map[string]string{}
	}
	maps.Copy(pod.Labels, labels)
	// backoffLimit is fixed at 2 (inherited from the Altinity driver). For an
	// app-agnostic driver this is a footgun - an S3-heavy dump may want 0
	// retries while a sidecar wants more - so making it operator-configurable
	// is tracked in https://github.com/cozystack/cozystack/issues/2742.
	backoffLimit := int32(2)
	return &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      name,
			Labels:    labels,
		},
		Spec: batchv1.JobSpec{
			BackoffLimit: &backoffLimit,
			Template:     pod,
		},
	}
}

// createJobBackupArtifact materialises a Cozystack Backup carrying the strategy
// reference and the BackupClassStrategy parameters in effect at backup time.
// Parameters round-trip via DriverMetadata under jobStrategyParamPrefix so a
// later RestoreJob can re-render the strategy template against the same values.
// Mirrors createAltinityBackupArtifact - see that function for why Status is
// populated on the struct but not promoted from this driver.
func (r *BackupJobReconciler) createJobBackupArtifact(
	ctx context.Context,
	j *backupsv1alpha1.BackupJob,
	resolved *ResolvedBackupConfig,
) (*backupsv1alpha1.Backup, error) {
	driverMD := map[string]string{}
	for k, v := range resolved.Parameters {
		driverMD[jobStrategyParamPrefix+k] = v
	}

	backup := &backupsv1alpha1.Backup{
		ObjectMeta: metav1.ObjectMeta{
			Name:      j.Name,
			Namespace: j.Namespace,
		},
		Spec: backupsv1alpha1.BackupSpec{
			ApplicationRef: j.Spec.ApplicationRef,
			StrategyRef:    resolved.StrategyRef,
			TakenAt:        metav1.Now(),
			DriverMetadata: driverMD,
		},
		Status: backupsv1alpha1.BackupStatus{
			Phase: backupsv1alpha1.BackupPhaseReady,
		},
	}
	if j.Spec.PlanRef != nil {
		backup.Spec.PlanRef = j.Spec.PlanRef
	}
	if err := r.Create(ctx, backup); err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return nil, err
		}
		existing := &backupsv1alpha1.Backup{}
		if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
			return nil, getErr
		}
		return existing, nil
	}
	return backup, nil
}

// ---------------------------------------------------------------------------
// RestoreJob path
// ---------------------------------------------------------------------------

// reconcileJobRestore drives a RestoreJob whose source Backup was produced by
// the Job strategy. It re-renders the same PodTemplateSpec with Mode=restore
// and runs it once as a batch/v1.Job, treating Job completion as restore
// success. Symmetric with reconcileJob, mirroring the Altinity restore path.
//
// Cross-namespace restore is intentionally unsupported: RestoreJob.spec.
// targetApplicationRef is corev1.TypedLocalObjectReference ("Local" => same
// namespace as the RestoreJob and the source Backup), so the restore Job
// always runs in restoreJob.Namespace.
func (r *RestoreJobReconciler) reconcileJobRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
	logger := getLogger(ctx)
	logger.Debug("reconciling Job restore", "restorejob", restoreJob.Name, "backup", backup.Name)

	if restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded ||
		restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
		return ctrl.Result{}, nil
	}

	// Resolve the effective restore target. Defaults to the source
	// application recorded on the Backup; targetApplicationRef overrides any
	// of name/kind/apiGroup when set (e.g. restore-as-copy into a differently
	// named application in the same namespace).
	targetNamespace := restoreJob.Namespace
	targetAppName := backup.Spec.ApplicationRef.Name
	targetAppKind := backup.Spec.ApplicationRef.Kind
	targetAPIGroup := ""
	if backup.Spec.ApplicationRef.APIGroup != nil {
		targetAPIGroup = *backup.Spec.ApplicationRef.APIGroup
	}
	if restoreJob.Spec.TargetApplicationRef != nil {
		if restoreJob.Spec.TargetApplicationRef.Name != "" {
			targetAppName = restoreJob.Spec.TargetApplicationRef.Name
		}
		if restoreJob.Spec.TargetApplicationRef.Kind != "" {
			targetAppKind = restoreJob.Spec.TargetApplicationRef.Kind
		}
		if restoreJob.Spec.TargetApplicationRef.APIGroup != nil {
			targetAPIGroup = *restoreJob.Spec.TargetApplicationRef.APIGroup
		}
	}
	targetRef := corev1.TypedLocalObjectReference{
		APIGroup: new(targetAPIGroup),
		Kind:     targetAppKind,
		Name:     targetAppName,
	}

	if restoreJob.Status.StartedAt == nil {
		fresh := &backupsv1alpha1.RestoreJob{}
		if err := r.Get(ctx, types.NamespacedName{Namespace: restoreJob.Namespace, Name: restoreJob.Name}, fresh); err != nil {
			return ctrl.Result{}, err
		}
		if fresh.Status.StartedAt != nil {
			restoreJob.Status.StartedAt = fresh.Status.StartedAt
			restoreJob.Status.Phase = fresh.Status.Phase
		} else {
			base := fresh.DeepCopy()
			now := metav1.Now()
			fresh.Status.StartedAt = &now
			fresh.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning
			if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
				return ctrl.Result{}, err
			}
			// Requeue so the next reconcile re-Gets restoreJob with the
			// post-patch ResourceVersion. See reconcileJob for the rationale.
			return ctrl.Result{RequeueAfter: jobStrategyPollInterval}, nil
		}
	}

	strategy := &strategyv1alpha1.Job{}
	if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
		if apierrors.IsNotFound(err) {
			return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("Job strategy not found: %s", backup.Spec.StrategyRef.Name))
		}
		return ctrl.Result{}, err
	}

	app, err := r.getApplicationUnstructured(ctx, targetNamespace, targetRef)
	if err != nil {
		// IsNotFound (target app not deployed) and IsNoMatchError
		// (unmappable target kind) are both terminal - see the matching
		// note on the BackupJob path in reconcileJob.
		if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
			return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
				"target application not found or kind not registered: %s/%s (kind=%q; deploy it before requesting a restore)",
				targetNamespace, targetAppName, targetAppKind))
		}
		return ctrl.Result{}, err
	}

	rendered, err := renderJobTemplate(
		strategy.Spec.Template,
		app,
		targetAppName,
		targetNamespace,
		jobStrategyModeRestore,
		jobStrategyParameters(backup),
		backup,
	)
	if err != nil {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Job strategy: %v", err))
	}

	batchJob, err := r.ensureJobStrategyRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob),
		jobStrategyModeRestore,
		map[string]string{
			backupsv1alpha1.OwningJobNameLabel:      restoreJob.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
		},
		rendered,
	)
	if err != nil {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure batch/v1.Job: %v", err))
	}

	switch jobConditionState(batchJob) {
	case batchv1.JobComplete:
		now := metav1.Now()
		restoreJob.Status.CompletedAt = &now
		restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
		apimeta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
			Type:    "Ready",
			Status:  metav1.ConditionTrue,
			Reason:  "RestoreCompleted",
			Message: "restore Job completed",
		})
		// Full Status().Update is safe for the same reason as the BackupJob
		// success branch in reconcileJob: restoreJob is freshly Get() by the
		// parent Reconcile each pass, and the StartedAt block returns a requeue
		// rather than writing restoreJob in the same pass.
		if err := r.Status().Update(ctx, restoreJob); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil

	case batchv1.JobFailed:
		message := jobFailureMessage(batchJob)
		if message == "" {
			message = "restore Job reported Failed"
		}
		return r.markRestoreJobFailed(ctx, restoreJob, message)

	default:
		return ctrl.Result{RequeueAfter: jobStrategyPollInterval}, nil
	}
}

// ensureJobStrategyRestoreJob is the RestoreJob-side mirror of
// ensureJobStrategyJob. The Job lives in the RestoreJob's namespace and gets a
// controllerRef on the RestoreJob. Mirrors ensureAltinityRestoreJob.
func (r *RestoreJobReconciler) ensureJobStrategyRestoreJob(
	ctx context.Context,
	owner client.Object,
	namespace, name, mode string,
	ownerLabels map[string]string,
	rendered *corev1.PodTemplateSpec,
) (*batchv1.Job, error) {
	existing := &batchv1.Job{}
	err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
	if err == nil {
		return existing, nil
	}
	if !apierrors.IsNotFound(err) {
		return nil, err
	}

	labels := map[string]string{jobStrategyLabelMode: mode}
	maps.Copy(labels, ownerLabels)

	desired := buildJobStrategyBatchJob(namespace, name, labels, rendered)
	if err := controllerutil.SetControllerReference(owner, desired, r.Scheme); err != nil {
		return nil, fmt.Errorf("set controller reference on restore Job: %w", err)
	}
	if err := r.Create(ctx, desired); err != nil {
		if apierrors.IsAlreadyExists(err) {
			if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
				return nil, err
			}
			return existing, nil
		}
		return nil, err
	}
	return desired, nil
}