cozystack/internal/backupcontroller/mariadbstrategy_controller.go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apimeta "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"

	strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
	backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
	"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
	"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
	"github.com/cozystack/cozystack/internal/template"
)

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const (
	mariadbFieldManager = "cozystack-mariadb-backup-driver"

	// mariadb-operator's MariaDB ApplicationDefinition (release.prefix). The
	// cozystack mariadb-rd renders the HelmRelease with releaseName =
	// "mariadb-" + appName (see packages/system/mariadb-rd/cozyrds/mariadb.yaml,
	// release.prefix). The chart's templates/mariadb.yaml then materialises
	// the operator-side k8s.mariadb.com/MariaDB CR with metadata.name set to
	// .Release.Name, so the driver must look up by the prefixed name.
	// Mirrors the CNPG driver's postgresAppPrefix / cnpgClusterNameForApp.
	mariadbAppKind   = "MariaDB"
	mariadbAppPrefix = "mariadb-"

	// Driver-metadata keys persisted on Cozystack Backup artifacts. Restore
	// path reads these to drive the operator Restore CR.
	mariadbBackupNameKey      = "k8s.mariadb.com/backup-name"
	mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace"
	mariadbStorageURIKey      = "k8s.mariadb.com/storage-uri"

	// Polling cadence for the operator Backup/Restore lifecycle.
	mariadbPollInterval = 5 * time.Second

	// Wall-clock cap on a BackupJob waiting for the operator Backup to
	// flip Complete. CNPG uses the same number for the same reason: a
	// permanently-stuck Backup must not pin the BackupJob in Running and
	// wedge the Plan-controller queue.
	mariadbDefaultBackupDeadline = 30 * time.Minute

	// Default deadline on a RestoreJob waiting for the operator Restore
	// to terminate. Tenants override via spec.options.restoreTimeoutSeconds.
	mariadbDefaultRestoreDeadline = 30 * time.Minute

	// mariadbBackupSnapshotKind is the Kind stamped onto the snapshot
	// persisted in Backup.status.underlyingResources. The same shape carries
	// the storage descriptor and the rendered strategy parameters used at
	// backup time, so restore-time re-rendering produces deterministic
	// values.
	mariadbBackupSnapshotKind = "MariaDBBackupSnapshot"
)

// mariadbBackupSnapshotAPIVersion is the apiVersion stamped onto the snapshot.
// Borrows the Cozystack backups group so the field is self-typed within the
// existing API surface.
var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String()

// mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a
// cozystack MariaDB application instance. The mapping mirrors the mariadb
// ApplicationDefinition (release.prefix=mariadb-).
func mariadbNameForApp(appName string) string {
	return mariadbAppPrefix + appName
}

// validateMariaDBApplicationRef rejects ApplicationRefs that name a
// Kind/APIGroup the MariaDB driver does not own. The driver assumes
// apps.cozystack.io/MariaDB; without this gate a ref like
// other.example.com/MariaDB would be accepted by the Kind check alone and
// then reconciled against the wrong CRD via the typed client.
func validateMariaDBApplicationRef(ref corev1.TypedLocalObjectReference) error {
	if ref.Kind != mariadbAppKind {
		return fmt.Errorf("MariaDB strategy supports applicationRef.kind=%q, got %q", mariadbAppKind, ref.Kind)
	}
	apiGroup := ""
	if ref.APIGroup != nil {
		apiGroup = *ref.APIGroup
	}
	if apiGroup != "" && apiGroup != mariadbapp.GroupName {
		return fmt.Errorf("MariaDB strategy supports applicationRef.apiGroup=%q, got %q", mariadbapp.GroupName, apiGroup)
	}
	return nil
}

// ---------------------------------------------------------------------------
// BackupJob path
// ---------------------------------------------------------------------------

func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
	logger := getLogger(ctx)
	logger.Debug("reconciling MariaDB strategy", "backupjob", j.Name, "phase", j.Status.Phase)

	if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
		j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
		return ctrl.Result{}, nil
	}

	if err := validateMariaDBApplicationRef(j.Spec.ApplicationRef); err != nil {
		return r.markBackupJobFailed(ctx, j, err.Error())
	}

	if j.Status.StartedAt == nil {
		// Refetch the latest persisted state before writing StartedAt: a
		// stale informer cache that returns StartedAt==nil after we already
		// persisted it would otherwise let the deadline gate slide forward
		// on every poll. Same idempotency pattern as the CNPG driver.
		fresh := &backupsv1alpha1.BackupJob{}
		if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil {
			return ctrl.Result{}, err
		}
		if fresh.Status.StartedAt != nil {
			j.Status.StartedAt = fresh.Status.StartedAt
		} else {
			base := fresh.DeepCopy()
			now := metav1.Now()
			fresh.Status.StartedAt = &now
			if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
				return ctrl.Result{}, err
			}
			return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
		}
	}

	strategy := &strategyv1alpha1.MariaDB{}
	if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil {
		if apierrors.IsNotFound(err) {
			return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name)
		}
		return ctrl.Result{}, err
	}

	app, err := r.getMariaDBApp(ctx, j.Namespace, j.Spec.ApplicationRef.Name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf("MariaDB application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name))
		}
		return ctrl.Result{}, err
	}

	rendered, err := renderMariaDBTemplate(strategy.Spec.Template, app, resolved.Parameters)
	if err != nil {
		return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template MariaDB strategy: %v", err))
	}

	// Operator-side MariaDB CR carries the prefixed release name (see
	// mariadbNameForApp); verify it exists before we ask the operator to
	// back it up. NotFound is normally transient (the chart may still be
	// rendering on a fresh app), but bounded by mariadbDefaultBackupDeadline
	// — otherwise a BackupJob whose applicationRef.name typo never resolves
	// would pin in Running forever. StartedAt was persisted at line 132,
	// so the deadline clock already started ticking on the first reconcile.
	mdbName := mariadbNameForApp(j.Spec.ApplicationRef.Name)
	if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: mdbName},
		&mariadbtypes.MariaDB{}); err != nil {
		if apierrors.IsNotFound(err) {
			if mariadbBackupDeadlineExceeded(j.Status.StartedAt) {
				return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
					"k8s.mariadb.com/MariaDB %s/%s never reached existence within %s",
					j.Namespace, mdbName, mariadbDefaultBackupDeadline))
			}
			apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
				Type:    "Ready",
				Status:  metav1.ConditionFalse,
				Reason:  "MariaDBNotReady",
				Message: fmt.Sprintf("waiting for k8s.mariadb.com/MariaDB %s/%s to exist", j.Namespace, mdbName),
			})
			if err := r.Status().Update(ctx, j); err != nil {
				return ctrl.Result{}, err
			}
			return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
		}
		return ctrl.Result{}, err
	}

	mdbBackup, err := r.ensureMariaDBBackup(ctx, j, rendered)
	if err != nil {
		return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure k8s.mariadb.com/Backup: %v", err))
	}

	if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
		j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
		if err := r.Status().Update(ctx, j); err != nil {
			return ctrl.Result{}, err
		}
	}

	cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete)

	switch {
	case cond != nil && cond.Status == metav1.ConditionTrue:
		// Backup completed successfully. Materialise the Cozystack artifact
		// and finalise the BackupJob. Idempotent: if the artifact already
		// exists (a previous reconcile created it and then raced on the
		// status update), reuse the existing object.
		if j.Status.BackupRef != nil {
			return ctrl.Result{}, nil
		}
		artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered)
		if err != nil {
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
		}
		now := metav1.Now()
		j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
		j.Status.CompletedAt = &now
		j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
		apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
			Type:    "Ready",
			Status:  metav1.ConditionTrue,
			Reason:  "BackupCompleted",
			Message: "k8s.mariadb.com Backup completed",
		})
		if err := r.Status().Update(ctx, j); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil

	case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed:
		// Terminal failure. The operator marks Complete=False/JobFailed only
		// after the underlying Job exhausts its backoff budget, so we can
		// fail the BackupJob immediately rather than waiting for the
		// driver-side deadline.
		message := cond.Message
		if message == "" {
			message = "k8s.mariadb.com Backup terminal failure"
		}
		return r.markBackupJobFailed(ctx, j, message)

	default:
		// Still running (Complete=False/JobRunning, Complete=False/JobNotComplete,
		// JobSuspended, or no Complete condition yet). Apply a wall-clock
		// deadline so a permanently-stuck Backup eventually fails the
		// BackupJob instead of pinning it Running forever.
		if mariadbBackupDeadlineExceeded(j.Status.StartedAt) {
			detail := "no Complete condition observed"
			if cond != nil {
				detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
			}
			return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
				"k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, detail))
		}
		return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
	}
}

// mariadbBackupDeadlineExceeded reports whether enough wall-clock time has
// elapsed since the BackupJob started that we should give up on a stuck
// operator Backup. Returns false when StartedAt is nil so the very first
// reconcile (which sets StartedAt) does not trip the gate.
func mariadbBackupDeadlineExceeded(startedAt *metav1.Time) bool {
	if startedAt == nil {
		return false
	}
	return time.Since(startedAt.Time) > mariadbDefaultBackupDeadline
}

// ensureMariaDBBackup creates a one-shot k8s.mariadb.com/Backup CR labelled
// with the BackupJob, or returns the existing one if a previous reconcile
// already created it. Idempotency relies on the OwningJob labels.
func (r *BackupJobReconciler) ensureMariaDBBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, rendered *strategyv1alpha1.MariaDBTemplate) (*mariadbtypes.Backup, error) {
	existing, err := r.findMariaDBBackupForJob(ctx, j)
	if err != nil {
		return nil, err
	}
	if existing != nil {
		return existing, nil
	}

	storage, err := buildMariaDBBackupStorage(rendered.Storage)
	if err != nil {
		return nil, err
	}

	obj := &mariadbtypes.Backup{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:    j.Namespace,
			GenerateName: fmt.Sprintf("%s-", j.Name),
			Labels: map[string]string{
				backupsv1alpha1.OwningJobNameLabel:      j.Name,
				backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
			},
		},
		Spec: mariadbtypes.BackupSpec{
			MariaDBRef:   mariadbtypes.MariaDBObjectRef{Name: mariadbNameForApp(j.Spec.ApplicationRef.Name)},
			Storage:      storage,
			Databases:    append([]string(nil), rendered.Databases...),
			Compression:  rendered.Compression,
			LogLevel:     rendered.LogLevel,
			MaxRetention: rendered.MaxRetention.DeepCopy(),
		},
	}

	if err := r.Create(ctx, obj); err != nil {
		return nil, err
	}
	return obj, nil
}

// findMariaDBBackupForJob returns the k8s.mariadb.com/Backup labelled with
// the BackupJob's OwningJob{Name,Namespace}, if any. Returns (nil, nil)
// when no match is found.
//
// In steady state at most one Backup CR carries a given OwningJob label
// pair (ensureMariaDBBackup is idempotent by this label). The list path
// can in theory observe two if two controller replicas raced Create
// between the first list returning empty and the cache catching up. The
// driver picks list.Items[0] deterministically; the V(1) breadcrumb on
// duplicates exists so an operator triaging "wrong Backup got reused"
// has a log line to grep for instead of staring at silent reuse.
func (r *BackupJobReconciler) findMariaDBBackupForJob(ctx context.Context, j *backupsv1alpha1.BackupJob) (*mariadbtypes.Backup, error) {
	list := &mariadbtypes.BackupList{}
	if err := r.List(ctx, list,
		client.InNamespace(j.Namespace),
		client.MatchingLabels{
			backupsv1alpha1.OwningJobNameLabel:      j.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
		},
	); err != nil {
		return nil, err
	}
	if len(list.Items) == 0 {
		return nil, nil
	}
	if len(list.Items) > 1 {
		names := make([]string, 0, len(list.Items))
		for i := range list.Items {
			names = append(names, list.Items[i].Name)
		}
		getLogger(ctx).Debug("multiple k8s.mariadb.com/Backup CRs match BackupJob OwningJob labels; reusing first",
			"backupjob", j.Name, "namespace", j.Namespace, "matches", names, "picked", names[0])
	}
	return &list.Items[0], nil
}

// createMariaDBBackupArtifact materialises a Cozystack Backup resource
// carrying the metadata callers need to drive a future restore. The
// rendered storage descriptor plus the BackupClass parameters are persisted
// in status.underlyingResources so a future fallback path (see
// marshalMariaDBBackupSnapshot) can re-render them when the operator-side
// Backup CR has been reaped.
func (r *BackupJobReconciler) createMariaDBBackupArtifact(
	ctx context.Context,
	j *backupsv1alpha1.BackupJob,
	resolved *ResolvedBackupConfig,
	mdbBackup *mariadbtypes.Backup,
	rendered *strategyv1alpha1.MariaDBTemplate,
) (*backupsv1alpha1.Backup, error) {
	takenAt := metav1.Now()
	if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() {
		takenAt = cond.LastTransitionTime
	}

	driverMD := map[string]string{
		mariadbBackupNameKey:      mdbBackup.Name,
		mariadbBackupNamespaceKey: mdbBackup.Namespace,
	}
	if uri := mariadbBackupURI(rendered, mdbBackup); uri != "" {
		driverMD[mariadbStorageURIKey] = uri
	}

	underlyingResources, err := marshalMariaDBBackupSnapshot(rendered, resolved.Parameters)
	if err != nil {
		return nil, fmt.Errorf("encode source snapshot for Backup.status.underlyingResources: %w", err)
	}

	status := backupsv1alpha1.BackupStatus{
		Phase:               backupsv1alpha1.BackupPhaseReady,
		UnderlyingResources: underlyingResources,
	}
	// Status.Artifact carries a "this backup has an artefact at URI X"
	// signal. PVC/Volume storage has no stable URI (the location is the
	// operator's PVC ID), so an Artifact{URI: ""} is a worse signal than
	// no Artifact at all - leave it nil instead.
	if uri := driverMD[mariadbStorageURIKey]; uri != "" {
		status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri}
	}
	backup := &backupsv1alpha1.Backup{
		ObjectMeta: metav1.ObjectMeta{
			Name:      j.Name,
			Namespace: j.Namespace,
		},
		Spec: backupsv1alpha1.BackupSpec{
			ApplicationRef: j.Spec.ApplicationRef,
			StrategyRef:    resolved.StrategyRef,
			TakenAt:        takenAt,
			DriverMetadata: driverMD,
		},
		Status: status,
	}
	if j.Spec.PlanRef != nil {
		backup.Spec.PlanRef = j.Spec.PlanRef
	}
	if err := r.Create(ctx, backup); err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return nil, err
		}
		existing := &backupsv1alpha1.Backup{}
		if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
			return nil, getErr
		}
		return existing, nil
	}
	return backup, nil
}

// mariadbBackupURI synthesises a human-readable URI for the Cozystack
// Backup artifact. Empty when the storage shape doesn't resolve to a stable
// URI (PVC/Volume, where the location is the operator's PVC ID).
func mariadbBackupURI(rendered *strategyv1alpha1.MariaDBTemplate, mdbBackup *mariadbtypes.Backup) string {
	if rendered.Storage.S3 == nil {
		return ""
	}
	s := rendered.Storage.S3
	prefix := s.Prefix
	if prefix != "" && prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}
	return fmt.Sprintf("s3://%s/%s%s", s.Bucket, prefix, mdbBackup.Name)
}

// ---------------------------------------------------------------------------
// RestoreJob path
// ---------------------------------------------------------------------------

func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
	logger := getLogger(ctx)
	logger.Debug("reconciling MariaDB restore", "restorejob", restoreJob.Name, "backup", backup.Name)

	if err := validateMariaDBApplicationRef(backup.Spec.ApplicationRef); err != nil {
		return r.markRestoreJobFailed(ctx, restoreJob, err.Error())
	}

	// Validate the resolved target shape before any apiserver call. This is
	// cheap (in-memory state only) and lets a malformed TargetApplicationRef
	// fail fast instead of after parseMariaDBRestoreOptions + the srcBackup
	// Get round-trip.
	target := r.resolveMariaDBRestoreTarget(restoreJob, backup)
	if target.Kind != mariadbAppKind {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
			"target applicationRef.kind=%q is not supported by the MariaDB driver", target.Kind))
	}
	if target.APIGroup != "" && target.APIGroup != mariadbapp.GroupName {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
			"target applicationRef.apiGroup=%q is not supported by the MariaDB driver", target.APIGroup))
	}

	if restoreJob.Status.StartedAt == nil {
		// Refetch the latest persisted state before writing StartedAt; same
		// stale-cache idempotency pattern the BackupJob path uses.
		fresh := &backupsv1alpha1.RestoreJob{}
		if err := r.Get(ctx, types.NamespacedName{Namespace: restoreJob.Namespace, Name: restoreJob.Name}, fresh); err != nil {
			return ctrl.Result{}, err
		}
		if fresh.Status.StartedAt != nil {
			// A previous reconcile (or another replica) already persisted
			// StartedAt - adopt it locally and continue inline so we don't
			// waste a reconcile cycle bouncing through the cache. Mirrors
			// the BackupJob path above.
			restoreJob.Status.StartedAt = fresh.Status.StartedAt
			if fresh.Status.Phase != "" {
				restoreJob.Status.Phase = fresh.Status.Phase
			}
		} else {
			base := fresh.DeepCopy()
			now := metav1.Now()
			fresh.Status.StartedAt = &now
			fresh.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning
			if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
				return ctrl.Result{}, err
			}
			restoreJob.Status.StartedAt = fresh.Status.StartedAt
			restoreJob.Status.Phase = fresh.Status.Phase
			// Requeue so the patched status round-trips via the informer
			// cache before subsequent reconciles read it back.
			return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
		}
	}

	// parseMariaDBRestoreOptions is intentionally permissive (event + log,
	// proceed with defaults). This diverges from the structural failures
	// below (missing driverMetadata, MariaDB CR genuinely absent) because a
	// bad spec.options is best-effort tenant-supplied override, not a
	// load-bearing piece of the restore contract - the defaults are still
	// the right behaviour, so surfacing a transient breadcrumb beats
	// failing the restore on a typo.
	options, err := parseMariaDBRestoreOptions(restoreJob.Spec.Options)
	if err != nil {
		logger.Info("malformed restoreJob.spec.options; falling back to defaults", "error", err)
		r.Recorder.Eventf(restoreJob, corev1.EventTypeWarning, "MalformedOptions",
			"spec.options is not valid JSON; falling back to defaults: %v", err)
	}

	sourceBackupName := backup.Spec.DriverMetadata[mariadbBackupNameKey]
	if sourceBackupName == "" {
		return r.markRestoreJobFailed(ctx, restoreJob,
			"Backup driverMetadata is missing k8s.mariadb.com/backup-name (re-take the backup with a controller version that persists the operator Backup name)")
	}

	// Verify the operator Backup CR still exists. The operator's Restore CR
	// only resolves backupRef in the same namespace as the Restore itself,
	// so cross-namespace restores aren't supported here. NotFound surfaces
	// as a terminal failure with a clear migration path (re-run the
	// BackupJob to materialise a fresh artifact).
	srcBackup := &mariadbtypes.Backup{}
	if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, srcBackup); err != nil {
		if apierrors.IsNotFound(err) {
			return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
				"k8s.mariadb.com/Backup %s/%s not found; the operator-side artifact has been reaped",
				backup.Namespace, sourceBackupName))
		}
		return ctrl.Result{}, err
	}

	// The operator Restore CR replays the dump against an existing live
	// MariaDB; for to-copy this requires the target MariaDB to exist. A
	// tenant who fires a RestoreJob seconds before their target
	// HelmRelease materialises the MariaDB CR shouldn't have to recreate
	// the RestoreJob - mirror the backup path's transient handling and let
	// the operator-side StartedAt + effectiveRestoreDeadline guard against
	// a target that never shows up.
	targetMDBName := mariadbNameForApp(target.AppName)
	if err := r.Get(ctx, types.NamespacedName{Namespace: target.Namespace, Name: targetMDBName},
		&mariadbtypes.MariaDB{}); err != nil {
		if apierrors.IsNotFound(err) {
			deadline := options.effectiveRestoreDeadline()
			if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline {
				return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
					"target k8s.mariadb.com/MariaDB %s/%s not found within %s (deploy the target MariaDB application before requesting the restore; override via spec.options.restoreTimeoutSeconds)",
					target.Namespace, targetMDBName, deadline))
			}
			apimeta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
				Type:   "Ready",
				Status: metav1.ConditionFalse,
				Reason: "TargetMariaDBNotReady",
				Message: fmt.Sprintf("waiting for target k8s.mariadb.com/MariaDB %s/%s to exist",
					target.Namespace, targetMDBName),
			})
			if err := r.Status().Update(ctx, restoreJob); err != nil {
				return ctrl.Result{}, err
			}
			return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
		}
		return ctrl.Result{}, err
	}

	mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName)
	if err != nil {
		return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
	}

	cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete)
	switch {
	case cond != nil && cond.Status == metav1.ConditionTrue:
		now := metav1.Now()
		restoreJob.Status.CompletedAt = &now
		restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
		apimeta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
			Type:    "Ready",
			Status:  metav1.ConditionTrue,
			Reason:  "RestoreCompleted",
			Message: fmt.Sprintf("k8s.mariadb.com/Restore %s/%s completed", mdbRestore.Namespace, mdbRestore.Name),
		})
		if err := r.Status().Update(ctx, restoreJob); err != nil {
			return ctrl.Result{}, err
		}
		return ctrl.Result{}, nil

	case cond != nil && cond.Status == metav1.ConditionFalse && cond.Reason == mariadbtypes.ConditionReasonJobFailed:
		message := cond.Message
		if message == "" {
			message = fmt.Sprintf("k8s.mariadb.com/Restore %s/%s terminal failure", mdbRestore.Namespace, mdbRestore.Name)
		}
		return r.markRestoreJobFailed(ctx, restoreJob, message)

	default:
		deadline := options.effectiveRestoreDeadline()
		if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline {
			detail := "no Complete condition observed"
			if cond != nil {
				detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
			}
			return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
				"k8s.mariadb.com/Restore did not complete within %s (%s; override via spec.options.restoreTimeoutSeconds)",
				deadline, detail))
		}
		return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
	}
}

// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with
// the RestoreJob, or returns the existing one if a previous reconcile
// already created it.
func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) {
	list := &mariadbtypes.RestoreList{}
	if err := r.List(ctx, list,
		client.InNamespace(rj.Namespace),
		client.MatchingLabels{
			backupsv1alpha1.OwningJobNameLabel:      rj.Name,
			backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace,
		},
	); err != nil {
		return nil, err
	}
	if len(list.Items) > 0 {
		return &list.Items[0], nil
	}

	obj := &mariadbtypes.Restore{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:    rj.Namespace,
			GenerateName: fmt.Sprintf("%s-", rj.Name),
			Labels: map[string]string{
				backupsv1alpha1.OwningJobNameLabel:      rj.Name,
				backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace,
			},
		},
		Spec: mariadbtypes.RestoreSpec{
			MariaDBRef: mariadbtypes.MariaDBObjectRef{Name: targetMariaDBName},
			BackupRef: &mariadbtypes.BackupReference{
				Kind: mariadbtypes.BackupKind,
				Name: sourceBackupName,
			},
		},
	}
	if err := r.Create(ctx, obj); err != nil {
		return nil, err
	}
	return obj, nil
}

// mariadbRestoreTarget captures the resolved target for a MariaDB restore.
// The reconcile path treats in-place and to-copy identically: both create a
// k8s.mariadb.com/Restore CR pointing at the named target MariaDB. Callers
// infer the mode from AppName != backup.Spec.ApplicationRef.Name.
type mariadbRestoreTarget struct {
	Namespace string
	AppName   string
	Kind      string
	APIGroup  string
}

func (r *RestoreJobReconciler) resolveMariaDBRestoreTarget(restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) mariadbRestoreTarget {
	t := mariadbRestoreTarget{
		Namespace: backup.Namespace,
		AppName:   backup.Spec.ApplicationRef.Name,
		Kind:      backup.Spec.ApplicationRef.Kind,
	}
	if backup.Spec.ApplicationRef.APIGroup != nil {
		t.APIGroup = *backup.Spec.ApplicationRef.APIGroup
	}
	if restoreJob.Spec.TargetApplicationRef != nil {
		if restoreJob.Spec.TargetApplicationRef.Name != "" {
			t.AppName = restoreJob.Spec.TargetApplicationRef.Name
		}
		if restoreJob.Spec.TargetApplicationRef.Kind != "" {
			t.Kind = restoreJob.Spec.TargetApplicationRef.Kind
		}
		if restoreJob.Spec.TargetApplicationRef.APIGroup != nil {
			t.APIGroup = *restoreJob.Spec.TargetApplicationRef.APIGroup
		}
	}
	return t
}

// ---------------------------------------------------------------------------
// Snapshot persisted on Cozystack Backup.status.underlyingResources
// ---------------------------------------------------------------------------

// mariadbBackupSnapshot is the MariaDB-specific payload persisted in
// Backup.status.underlyingResources at backup time. Carries the rendered
// storage descriptor and BackupClass parameters. The current restore path
// resolves the operator-side k8s.mariadb.com/Backup CR directly and fails
// terminally when it has been reaped; this snapshot is reserved for a
// future fallback path that would re-materialise the operator Backup from
// the persisted state. Keep the shape stable so an older snapshot stays
// machine-readable when that fallback lands.
//
// SECRET-HANDLING CONTRACT (this is plaintext-readable to anyone with read
// access to backups.cozystack.io/Backups):
//
//   - Storage carries only Secret REFERENCES (Name + Key) - the
//     MariaDBS3Template / MariaDBSecretKeySelector schema has no field for
//     a raw credential value, so the snapshot can never contain an access
//     key or secret access key. The mariadb-operator dereferences these
//     names at runtime against the apiserver's Secret cache.
//   - Parameters lands here verbatim from BackupClassStrategy.parameters.
//     Callers MUST NOT put credentials in Parameters (it is documented on
//     MariaDBSpec.Template that those values are not credential-bearing);
//     a tenant who templates `{{ .Parameters.secretAccessKey }}` into a
//     string field would persist that value here plaintext. The schema's
//     credential-by-reference shape is the enforcement; this comment is
//     the contract.
type mariadbBackupSnapshot struct {
	Kind       string                                   `json:"kind"`
	APIVersion string                                   `json:"apiVersion"`
	Storage    *strategyv1alpha1.MariaDBStorageTemplate `json:"storage,omitempty"`
	Databases  []string                                 `json:"databases,omitempty"`
	Parameters map[string]string                        `json:"parameters,omitempty"`
}

// marshalMariaDBBackupSnapshot encodes the snapshot for persistence on the
// Cozystack Backup. See mariadbBackupSnapshot godoc for the secret-handling
// contract: Storage carries only Secret references (Name + Key), never
// raw credential values; Parameters is persisted verbatim and is plaintext.
func marshalMariaDBBackupSnapshot(rendered *strategyv1alpha1.MariaDBTemplate, parameters map[string]string) (*runtime.RawExtension, error) {
	storage := rendered.Storage.DeepCopy()
	snap := mariadbBackupSnapshot{
		Kind:       mariadbBackupSnapshotKind,
		APIVersion: mariadbBackupSnapshotAPIVersion,
		Storage:    storage,
		Databases:  append([]string(nil), rendered.Databases...),
		Parameters: parameters,
	}
	raw, err := json.Marshal(snap)
	if err != nil {
		return nil, err
	}
	return &runtime.RawExtension{Raw: raw}, nil
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

// renderMariaDBTemplate templates the strategy template against a context
// containing the live application object and the BackupClass parameters.
// Reuses the same templating helper as the CNPG / Velero strategies.
func renderMariaDBTemplate(t strategyv1alpha1.MariaDBTemplate, app *mariadbapp.MariaDB, parameters map[string]string) (*strategyv1alpha1.MariaDBTemplate, error) {
	appAsMap, err := toJSONMapMariaDB(app)
	if err != nil {
		return nil, fmt.Errorf("encode application for templating: %w", err)
	}
	templateContext := map[string]interface{}{
		"Application": appAsMap,
		"Parameters":  parameters,
	}
	return template.Template(&t, templateContext)
}

// toJSONMapMariaDB converts a typed object to a generic map via JSON tags
// so user-authored go-templates address fields by their JSON names (e.g.
// .Application.metadata.name). Mirrors the helper in the CNPG controller;
// scoped here to avoid cross-strategy import.
func toJSONMapMariaDB(obj interface{}) (map[string]interface{}, error) {
	raw, err := json.Marshal(obj)
	if err != nil {
		return nil, err
	}
	out := map[string]interface{}{}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// getMariaDBApp fetches the apps.cozystack.io MariaDB instance via the
// shared typed client. The MariaDB scheme is registered in main.go so the
// controller-runtime cache serves it directly.
func (r *BackupJobReconciler) getMariaDBApp(ctx context.Context, namespace, name string) (*mariadbapp.MariaDB, error) {
	app := &mariadbapp.MariaDB{}
	if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, app); err != nil {
		return nil, err
	}
	return app, nil
}

// buildMariaDBBackupStorage translates the typed strategy template into the
// k8s.mariadb.com/v1alpha1 Backup.spec.storage shape. Exactly one of S3, PVC
// or Volume must be set; the CRD's XValidation rule enforces this at
// admission, but a defence-in-depth check here keeps the driver's behaviour
// well-defined for any pre-existing strategy CR that pre-dates the
// admission rule (otherwise the switch below would silently pick whichever
// branch comes first in source order).
func buildMariaDBBackupStorage(in strategyv1alpha1.MariaDBStorageTemplate) (mariadbtypes.BackupStorage, error) {
	set := 0
	if in.S3 != nil {
		set++
	}
	if in.PersistentVolumeClaim != nil {
		set++
	}
	if in.Volume != nil {
		set++
	}
	if set != 1 {
		return mariadbtypes.BackupStorage{}, fmt.Errorf(
			"strategy.spec.template.storage requires exactly one of s3, persistentVolumeClaim or volume; got %d", set)
	}
	switch {
	case in.S3 != nil:
		out := &mariadbtypes.S3Storage{
			Bucket:   in.S3.Bucket,
			Endpoint: in.S3.Endpoint,
			Prefix:   in.S3.Prefix,
			Region:   in.S3.Region,
			AccessKeyIdSecretKeyRef: mariadbtypes.SecretKeySelector{
				Name: in.S3.AccessKeyIDSecretKeyRef.Name,
				Key:  in.S3.AccessKeyIDSecretKeyRef.Key,
			},
			SecretAccessKeySecretKeyRef: mariadbtypes.SecretKeySelector{
				Name: in.S3.SecretAccessKeySecretKeyRef.Name,
				Key:  in.S3.SecretAccessKeySecretKeyRef.Key,
			},
		}
		if in.S3.SessionTokenSecretKeyRef != nil {
			out.SessionTokenSecretKeyRef = &mariadbtypes.SecretKeySelector{
				Name: in.S3.SessionTokenSecretKeyRef.Name,
				Key:  in.S3.SessionTokenSecretKeyRef.Key,
			}
		}
		if in.S3.TLS != nil {
			tls := &mariadbtypes.S3TLS{Enabled: in.S3.TLS.Enabled}
			if in.S3.TLS.CASecretKeyRef != nil {
				tls.CASecretKeyRef = &mariadbtypes.SecretKeySelector{
					Name: in.S3.TLS.CASecretKeyRef.Name,
					Key:  in.S3.TLS.CASecretKeyRef.Key,
				}
			}
			out.TLS = tls
		}
		return mariadbtypes.BackupStorage{S3: out}, nil
	case in.PersistentVolumeClaim != nil:
		return mariadbtypes.BackupStorage{PersistentVolumeClaim: in.PersistentVolumeClaim.DeepCopy()}, nil
	case in.Volume != nil:
		return mariadbtypes.BackupStorage{Volume: in.Volume.DeepCopy()}, nil
	default:
		return mariadbtypes.BackupStorage{}, fmt.Errorf("strategy.spec.template.storage requires exactly one of s3, persistentVolumeClaim or volume")
	}
}

// MariaDBRestoreOptions is the typed shape of RestoreJob.Spec.Options for
// the MariaDB driver. Mirrors the CNPG strategy's RestoreOptions pattern
// so the boundary parses lazily and keeps behaviour permissive.
type MariaDBRestoreOptions struct {
	// RestoreTimeoutSeconds caps the time the driver waits for the
	// k8s.mariadb.com/Restore to terminate before it marks the RestoreJob
	// Failed. Zero or unset falls back to mariadbDefaultRestoreDeadline.
	// +optional
	RestoreTimeoutSeconds int64 `json:"restoreTimeoutSeconds,omitempty"`
}

func parseMariaDBRestoreOptions(opts *runtime.RawExtension) (MariaDBRestoreOptions, error) {
	var out MariaDBRestoreOptions
	if opts == nil || len(opts.Raw) == 0 {
		return out, nil
	}
	if err := json.Unmarshal(opts.Raw, &out); err != nil {
		return MariaDBRestoreOptions{}, fmt.Errorf("decode restoreJob.spec.options: %w", err)
	}
	return out, nil
}

func (o MariaDBRestoreOptions) effectiveRestoreDeadline() time.Duration {
	if o.RestoreTimeoutSeconds > 0 {
		return time.Duration(o.RestoreTimeoutSeconds) * time.Second
	}
	return mariadbDefaultRestoreDeadline
}