Files
cozystack/internal/backupcontroller/kafkastrategy_controller.go
T
Andrey Kolkov ca2a6d6429 fix(backups): warn when a Kafka backupTimeout is rejected
kafkaRunDeadline silently substituted the 30m default for a backupTimeout
that time.ParseDuration rejects or that falls below the one-minute floor,
with no Event, condition or log line. An operator who set an unparseable
or sub-minute value, then raised it, saw every run keep dying at
DeadlineExceeded with nothing naming the discarded parameter.

Return the applied deadline together with a message when the configured
value is rejected, and record it as a BackupTimeoutInvalid Warning Event
on the BackupJob and RestoreJob (the reconcilers already hold a Recorder).
An honoured or unset value stays silent. TestKafkaRunDeadline now asserts
the warning fires for the rejected cases and not otherwise.

Signed-off-by: Andrey Kolkov <androndo@gmail.com>
Assisted-by: LLM
2026-09-14 17:55:48 +04:00

952 lines
40 KiB
Go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"errors"
"fmt"
"strings"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/backupcontroller/kafkatypes"
"github.com/cozystack/cozystack/internal/template"
)
// The Kafka strategy is a typed variant of the generic Job strategy for the
// topic METADATA of a Cozystack Kafka application: the driver renders
// spec.template into a one-shot batch/v1.Job that talks to the Kafka Admin API
// (export topic definitions + configs on backup, recreate them on restore).
// It adds three things on top of the Job strategy's render-a-Job flow: an
// applicationRef Kind gate, a Ready precondition on the Strimzi Kafka cluster,
// and a per-run object key via .BackupName + ArtifactURITemplate.
const (
kafkaStrategyLabelMode = "kafka.strategy.backups.cozystack.io/mode"
kafkaStrategyModeBackup = "backup"
kafkaStrategyModeRestore = "restore"
kafkaStrategyModeCleanup = "cleanup"
// kafkaSkipArtifactCleanupAnnotation, set to "true" on a Backup, releases it
// without deleting (or waiting to delete) its S3 object - the operator escape
// hatch for a Backup left Terminating because object storage is unreachable.
// Shares the platform-wide key with the RabbitMQ driver's identical hatch.
kafkaSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup"
// Round-trips BackupClass parameters through the Backup's DriverMetadata so
// a later RestoreJob re-renders with the backup-time values.
kafkaStrategyParamPrefix = "kafka.strategy.backups.cozystack.io/parameter/"
// Stamped on every produced Backup so `kubectl describe backup` and any
// consumer make the narrow scope explicit: this strategy captures topic
// definitions/configs, not message data, offsets, ACLs or users.
kafkaScopeKey = "strategy.backups.cozystack.io/scope"
kafkaScopeValue = "topic-metadata"
kafkaStrategyPollInterval = 5 * time.Second
// Default wall-clock bound, used for two things: the Ready-precondition wait
// (controller-timed, before any Job exists) and the run itself (the Job's
// activeDeadlineSeconds). Export time scales with topic count, so operators
// whose cluster legitimately runs past this raise it per BackupClass with the
// kafkaBackupTimeoutParam parameter rather than being capped at a fixed value.
kafkaDefaultBackupDeadline = 30 * time.Minute
// Floor for an operator-supplied backupTimeout: a value below this (or an
// unparseable one) falls back to kafkaDefaultBackupDeadline, so a typo cannot
// wedge every run at a near-zero deadline.
kafkaMinBackupDeadline = time.Minute
// BackupClass parameter overriding the wall-clock deadline (a Go duration
// string, e.g. "2h"). It round-trips through the Backup's DriverMetadata like
// every other parameter, so a restore is bounded by the same value.
kafkaBackupTimeoutParam = "backupTimeout"
// The Cozystack chart names the Strimzi Kafka cluster "kafka-<app>".
kafkaClusterNamePrefix = "kafka-"
kafkaApplicationKind = "Kafka"
// The Strimzi cluster-operator stamps every pod of a Kafka cluster with the
// cluster name; a broker pod carries a container named "kafka". The driver
// resolves the client image from a live broker instead of a pinned value, so
// the Admin-API CLI it runs always matches the broker it talks to (any Kafka
// version the operator provisions, not only the chart default).
kafkaBrokerClusterLabel = "strimzi.io/cluster"
kafkaBrokerContainerName = "kafka"
// kafkaStrategyClientImageKey records, on the produced Backup, the broker
// image the backup Job ran. Cleanup reuses it to run the S3 delete once the
// source cluster (and its brokers) may be gone and no broker is left to
// resolve an image from; the image itself still resolves in the registry.
kafkaStrategyClientImageKey = "kafka.strategy.backups.cozystack.io/client-image"
)
// kafkaClusterName maps an apps.cozystack.io/Kafka name to the Strimzi Kafka
// cluster CR name the chart creates.
func kafkaClusterName(appName string) string {
return kafkaClusterNamePrefix + appName
}
// validateKafkaApplicationRef enforces that the driver only ever acts on an
// apps.cozystack.io/Kafka application.
func validateKafkaApplicationRef(ref corev1.TypedLocalObjectReference) error {
if ref.Kind != kafkaApplicationKind {
return fmt.Errorf("Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind)
}
if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup {
return fmt.Errorf("Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
}
return nil
}
// kafkaNotReadyMessage returns "" when the cluster reports Ready=True, otherwise
// a legible reason for the BackupJob/RestoreJob precondition.
func kafkaNotReadyMessage(cluster *kafkatypes.Kafka) string {
cond := apimeta.FindStatusCondition(cluster.Status.Conditions, kafkatypes.ConditionTypeReady)
if cond == nil {
return fmt.Sprintf("Kafka cluster %s has no Ready condition yet", cluster.Name)
}
if cond.Status == metav1.ConditionTrue {
return ""
}
msg := fmt.Sprintf("Kafka cluster %s is Ready=%s", cluster.Name, cond.Status)
if cond.Message != "" {
msg += ": " + cond.Message
}
return msg
}
// kafkaStrategyParameters extracts the round-tripped BackupClass parameters from
// a Backup's DriverMetadata. Mirrors jobStrategyParameters.
func kafkaStrategyParameters(b *backupsv1alpha1.Backup) map[string]string {
out := map[string]string{}
for k, v := range b.Spec.DriverMetadata {
if !strings.HasPrefix(k, kafkaStrategyParamPrefix) {
continue
}
paramKey := strings.TrimPrefix(k, kafkaStrategyParamPrefix)
if paramKey == "" {
continue
}
out[paramKey] = v
}
return out
}
// kafkaRunDeadline resolves the wall-clock bound for a run from its parameters,
// falling back to kafkaDefaultBackupDeadline when unset, unparseable, or below
// the floor. When a backupTimeout is set but rejected, the second return value is
// a message naming the rejected value and the deadline actually applied, so the
// silent fallback becomes a Warning Event the caller records; it is empty when the
// value is honoured or unset.
func kafkaRunDeadline(parameters map[string]string) (time.Duration, string) {
v := parameters[kafkaBackupTimeoutParam]
if v == "" {
return kafkaDefaultBackupDeadline, ""
}
d, err := time.ParseDuration(v)
switch {
case err != nil:
return kafkaDefaultBackupDeadline, fmt.Sprintf("backupTimeout %q is not a valid Go duration; applying the default %s", v, kafkaDefaultBackupDeadline)
case d < kafkaMinBackupDeadline:
return kafkaDefaultBackupDeadline, fmt.Sprintf("backupTimeout %q is below the %s floor; applying the default %s", v, kafkaMinBackupDeadline, kafkaDefaultBackupDeadline)
default:
return d, ""
}
}
// kafkaRenderContext builds the template context. Unlike the Job strategy it
// carries .BackupName (the per-run identity) so the strategy can scope the S3
// object key and a restore reads the exact object its Backup wrote.
func kafkaRenderContext(
app map[string]interface{},
releaseName, releaseNamespace, mode, backupName, artifactURI, clientImage string,
parameters map[string]string,
backup *backupsv1alpha1.Backup,
) map[string]any {
ctxMap := map[string]any{
"Application": app,
"Release": map[string]string{
"Name": releaseName,
"Namespace": releaseNamespace,
},
"Mode": mode,
"BackupName": backupName,
// ClientImage is the container image the rendered Job runs. The chart
// leaves the strategy template's image as the placeholder "{{ .ClientImage }}"
// and the driver fills it with the target Kafka broker's own image, so the
// Admin-API CLI version always matches the broker.
"ClientImage": clientImage,
// ArtifactURI is the single source of truth for the stored object's
// location. On backup it is the rendered artifactURITemplate (also
// recorded on the Backup); on restore and cleanup it is that recorded
// value read back from the Backup, so both act on the exact object the
// backup wrote and a later key-layout change cannot orphan old backups.
"ArtifactURI": artifactURI,
"Parameters": parameters,
}
if backup != nil {
sourceAPIGroup := ""
if backup.Spec.ApplicationRef.APIGroup != nil {
sourceAPIGroup = *backup.Spec.ApplicationRef.APIGroup
}
ctxMap["Backup"] = map[string]any{
"Name": backup.Name,
"Namespace": backup.Namespace,
"ApplicationRef": map[string]string{
"APIGroup": sourceAPIGroup,
"Kind": backup.Spec.ApplicationRef.Kind,
"Name": backup.Spec.ApplicationRef.Name,
},
}
}
return ctxMap
}
func renderKafkaTemplate(tmpl corev1.PodTemplateSpec, ctxMap map[string]any) (*corev1.PodTemplateSpec, error) {
return template.Template(&tmpl, ctxMap)
}
// renderKafkaArtifactURI renders the strategy's ArtifactURITemplate against the
// same context, surfacing template errors so a broken URI fails the backup
// rather than recording a half-rendered location.
func renderKafkaArtifactURI(tmplStr string, ctxMap map[string]any) (string, error) {
uri, err := template.String(tmplStr, ctxMap)
if err != nil {
return "", fmt.Errorf("artifact URI template: %w", err)
}
return strings.TrimSpace(uri), nil
}
// errNoKafkaBroker distinguishes "the cluster is Ready but no broker pod is
// listable" (terminal - the callers gate on Ready first) from a transient List
// error, which the caller requeues instead of failing the whole run.
var errNoKafkaBroker = errors.New("no Kafka broker pod to resolve the client image from")
// resolveKafkaBrokerImage returns the container image of a running broker of the
// Strimzi Kafka cluster, so the backup/restore Job runs the exact Admin-API CLI
// the broker ships rather than a separately pinned guess that drifts from it.
// Callers gate on the cluster being Ready first, so at least one broker pod
// exists; an empty result is a hard error, never a silent fallback.
func resolveKafkaBrokerImage(ctx context.Context, c client.Client, namespace, clusterName string) (string, error) {
pods := &corev1.PodList{}
if err := c.List(ctx, pods, client.InNamespace(namespace),
client.MatchingLabels{kafkaBrokerClusterLabel: clusterName}); err != nil {
return "", err
}
for i := range pods.Items {
for _, ct := range pods.Items[i].Spec.Containers {
if ct.Name == kafkaBrokerContainerName && ct.Image != "" {
return ct.Image, nil
}
}
}
return "", fmt.Errorf("%w (looked for a %q container in cluster %s/%s)", errNoKafkaBroker, kafkaBrokerContainerName, namespace, clusterName)
}
// ---------------------------------------------------------------------------
// BackupJob path
// ---------------------------------------------------------------------------
func (r *BackupJobReconciler) reconcileKafka(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
logger := getLogger(ctx)
logger.Debug("reconciling Kafka strategy", "backupjob", j.Name, "phase", j.Status.Phase)
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
return ctrl.Result{}, nil
}
if err := validateKafkaApplicationRef(j.Spec.ApplicationRef); err != nil {
return r.markBackupJobFailed(ctx, j, err.Error())
}
deadline, deadlineWarn := kafkaRunDeadline(resolved.Parameters)
if deadlineWarn != "" && r.Recorder != nil {
r.Recorder.Event(j, corev1.EventTypeWarning, "BackupTimeoutInvalid", deadlineWarn)
}
// First-reconcile bookkeeping (refetch guards against a stale informer
// sliding StartedAt forward). Mirrors reconcileJob.
if j.Status.StartedAt == nil {
fresh := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil {
return ctrl.Result{}, err
}
if fresh.Status.StartedAt != nil {
j.Status.StartedAt = fresh.Status.StartedAt
j.Status.Phase = fresh.Status.Phase
} else {
base := fresh.DeepCopy()
now := metav1.Now()
fresh.Status.StartedAt = &now
fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
}
strategy := &strategyv1alpha1.Kafka{}
if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil {
if apierrors.IsNotFound(err) {
return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name)
}
return ctrl.Result{}, err
}
app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef)
if err != nil {
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("application not found or kind not registered: %s/%s (kind=%q)", j.Namespace, j.Spec.ApplicationRef.Name, j.Spec.ApplicationRef.Kind))
}
return ctrl.Result{}, err
}
// Ready precondition on the Strimzi Kafka cluster: the Admin API is only
// reachable once it is up. Deadline-gate the wait so a cluster that never
// becomes Ready fails legibly.
appName := j.Spec.ApplicationRef.Name
cluster := &kafkatypes.Kafka{}
clusterErr := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: kafkaClusterName(appName)}, cluster)
if clusterErr != nil {
if !apierrors.IsNotFound(clusterErr) {
return ctrl.Result{}, clusterErr
}
return r.kafkaBackupNotReady(ctx, j, deadline, "KafkaClusterNotReady",
fmt.Sprintf("Kafka cluster %s not found yet", kafkaClusterName(appName)))
}
if msg := kafkaNotReadyMessage(cluster); msg != "" {
return r.kafkaBackupNotReady(ctx, j, deadline, "KafkaClusterNotReady", msg)
}
// Run the exact CLI the broker ships by resolving its image now that the
// cluster is Ready (so a broker pod exists), rather than from a chart pin that
// drifts from the broker version. A transient List error requeues; only a
// Ready cluster with no listable broker is terminal.
clientImage, err := resolveKafkaBrokerImage(ctx, r.Client, j.Namespace, kafkaClusterName(appName))
if err != nil {
if errors.Is(err, errNoKafkaBroker) {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("cannot resolve Kafka client image: %v", err))
}
return ctrl.Result{}, err
}
// The artifact URI is the single source of truth for where the export is
// stored: render it once, inject it into the Job (which writes exactly that
// object), and record the same value on the Backup so restore/cleanup act on
// it rather than reconstructing a key a later layout change could orphan.
if strategy.Spec.ArtifactURITemplate == "" {
return r.markBackupJobFailed(ctx, j, "Kafka strategy has no spec.artifactURITemplate; nothing records where the export is stored")
}
artifactURI, err := renderKafkaArtifactURI(strategy.Spec.ArtifactURITemplate,
kafkaRenderContext(app, appName, j.Namespace, kafkaStrategyModeBackup, j.Name, "", clientImage, resolved.Parameters, nil))
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to render Kafka strategy artifact URI: %v", err))
}
ctxMap := kafkaRenderContext(app, appName, j.Namespace, kafkaStrategyModeBackup, j.Name, artifactURI, clientImage, resolved.Parameters, nil)
rendered, err := renderKafkaTemplate(strategy.Spec.Template, ctxMap)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kafka strategy: %v", err))
}
batchJob, err := r.ensureKafkaJob(ctx, j, j.Namespace, jobNameForBackupJob(j),
kafkaStrategyModeBackup, deadline,
map[string]string{
backupsv1alpha1.OwningJobNameLabel: j.Name,
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
},
rendered,
)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure batch/v1.Job: %v", err))
}
switch jobConditionState(batchJob) {
case batchv1.JobComplete:
if j.Status.BackupRef != nil {
return ctrl.Result{}, nil
}
artifact, err := r.createKafkaBackupArtifact(ctx, j, resolved, artifactURI, clientImage)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
}
now := metav1.Now()
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
j.Status.CompletedAt = &now
j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionTrue,
Reason: "BackupCompleted",
Message: "Kafka topic-metadata backup completed (topic definitions and configs only; no message data, offsets, ACLs or users)",
})
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
case batchv1.JobFailed:
message := jobFailureMessage(batchJob)
if message == "" {
message = "Kafka metadata backup Job reported Failed"
}
return r.markBackupJobFailed(ctx, j, message)
default:
// The Job carries activeDeadlineSeconds (set in ensureKafkaJob), so a run
// that never completes - an unschedulable pod or a wedged export - is
// failed by Kubernetes as DeadlineExceeded and lands in the JobFailed
// branch above; the pod is killed before the script's final upload, so no
// object is left orphaned. Here we only poll while it makes progress.
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
}
// kafkaBackupNotReady surfaces a precise Ready=False on the BackupJob and
// requeues, or fails once the deadline since StartedAt is exceeded.
func (r *BackupJobReconciler) kafkaBackupNotReady(ctx context.Context, j *backupsv1alpha1.BackupJob, deadline time.Duration, reason, message string) (ctrl.Result, error) {
if j.Status.StartedAt != nil && time.Since(j.Status.StartedAt.Time) > deadline {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("timed out waiting for Kafka cluster to become Ready: %s", message))
}
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
})
if err := r.Status().Update(ctx, j); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
func (r *BackupJobReconciler) ensureKafkaJob(
ctx context.Context,
owner client.Object,
namespace, name, mode string,
deadline time.Duration,
ownerLabels map[string]string,
rendered *corev1.PodTemplateSpec,
) (*batchv1.Job, error) {
existing := &batchv1.Job{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
if err == nil {
return existing, nil
}
if !apierrors.IsNotFound(err) {
return nil, err
}
labels := map[string]string{kafkaStrategyLabelMode: mode}
for k, v := range ownerLabels {
labels[k] = v
}
desired := buildJobStrategyBatchJob(namespace, name, labels, rendered)
// Bound the run at the Job layer: Kubernetes fails an unschedulable or wedged
// Job as DeadlineExceeded and kills its pod before the script's final upload,
// so a timed-out run cannot both leave the BackupJob terminal and go on to
// write an object no Backup ever references.
setKafkaJobDeadline(desired, deadline)
if err := controllerutil.SetControllerReference(owner, desired, r.Scheme); err != nil {
return nil, fmt.Errorf("set controller reference on backup Job: %w", err)
}
if err := r.Create(ctx, desired); err != nil {
if apierrors.IsAlreadyExists(err) {
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
return nil, err
}
return existing, nil
}
return nil, err
}
return desired, nil
}
func (r *BackupJobReconciler) createKafkaBackupArtifact(
ctx context.Context,
j *backupsv1alpha1.BackupJob,
resolved *ResolvedBackupConfig,
artifactURI, clientImage string,
) (*backupsv1alpha1.Backup, error) {
driverMD := map[string]string{kafkaScopeKey: kafkaScopeValue}
for k, v := range resolved.Parameters {
driverMD[kafkaStrategyParamPrefix+k] = v
}
// Record the broker image this backup ran so cleanup can reuse it once the
// source cluster is gone (no broker left to resolve one from).
if clientImage != "" {
driverMD[kafkaStrategyClientImageKey] = clientImage
}
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: j.Name,
Namespace: j.Namespace,
},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: j.Spec.ApplicationRef,
StrategyRef: resolved.StrategyRef,
TakenAt: metav1.Now(),
DriverMetadata: driverMD,
},
Status: backupsv1alpha1.BackupStatus{
Phase: backupsv1alpha1.BackupPhaseReady,
},
}
if artifactURI != "" {
backup.Status.Artifact = &backupsv1alpha1.BackupArtifact{URI: artifactURI}
}
if j.Spec.PlanRef != nil {
backup.Spec.PlanRef = j.Spec.PlanRef
}
if err := r.Create(ctx, backup); err != nil {
if !apierrors.IsAlreadyExists(err) {
return nil, err
}
existing := &backupsv1alpha1.Backup{}
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, getErr
}
// Adopt an existing Backup only when it describes the same application.
// A retained Backup whose BackupJob was reaped, then reused for a
// different application of the same name, would otherwise be reported as
// this run's success while describing something else (mirrors the
// RabbitMQ driver's guard).
if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind ||
existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name {
return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)",
backup.Namespace, backup.Name,
existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name,
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name)
}
return existing, nil
}
return backup, nil
}
// ---------------------------------------------------------------------------
// RestoreJob path
// ---------------------------------------------------------------------------
func (r *RestoreJobReconciler) reconcileKafkaRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
logger := getLogger(ctx)
logger.Debug("reconciling Kafka restore", "restorejob", restoreJob.Name, "backup", backup.Name)
if restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded ||
restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
return ctrl.Result{}, nil
}
if err := validateKafkaApplicationRef(backup.Spec.ApplicationRef); err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, err.Error())
}
// The deadline round-trips from backup time via the Backup's parameters, so a
// restore is bounded by the same value the operator set for the backup.
deadline, deadlineWarn := kafkaRunDeadline(kafkaStrategyParameters(backup))
if deadlineWarn != "" && r.Recorder != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "BackupTimeoutInvalid", deadlineWarn)
}
// A Backup with no recorded object can never restore; fail fast, before the
// readiness gate, so it does not wait out the whole deadline and then report a
// misleading "timed out waiting for Kafka cluster to become Ready".
if backup.Status.Artifact == nil || backup.Status.Artifact.URI == "" {
return r.markRestoreJobFailed(ctx, restoreJob, "Backup has no recorded artifact URI (status.artifact.uri); cannot locate the metadata object to restore")
}
// Resolve the effective target: source app by default, overridden per-field
// by targetApplicationRef for a to-copy restore.
targetNamespace := restoreJob.Namespace
targetAppName := backup.Spec.ApplicationRef.Name
targetAppKind := backup.Spec.ApplicationRef.Kind
targetAPIGroup := ""
if backup.Spec.ApplicationRef.APIGroup != nil {
targetAPIGroup = *backup.Spec.ApplicationRef.APIGroup
}
if restoreJob.Spec.TargetApplicationRef != nil {
if restoreJob.Spec.TargetApplicationRef.Name != "" {
targetAppName = restoreJob.Spec.TargetApplicationRef.Name
}
if restoreJob.Spec.TargetApplicationRef.Kind != "" {
targetAppKind = restoreJob.Spec.TargetApplicationRef.Kind
}
if restoreJob.Spec.TargetApplicationRef.APIGroup != nil {
targetAPIGroup = *restoreJob.Spec.TargetApplicationRef.APIGroup
}
}
targetRef := corev1.TypedLocalObjectReference{
APIGroup: stringPtr(targetAPIGroup),
Kind: targetAppKind,
Name: targetAppName,
}
if err := validateKafkaApplicationRef(targetRef); err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, err.Error())
}
if restoreJob.Status.StartedAt == nil {
fresh := &backupsv1alpha1.RestoreJob{}
if err := r.Get(ctx, types.NamespacedName{Namespace: restoreJob.Namespace, Name: restoreJob.Name}, fresh); err != nil {
return ctrl.Result{}, err
}
if fresh.Status.StartedAt != nil {
restoreJob.Status.StartedAt = fresh.Status.StartedAt
restoreJob.Status.Phase = fresh.Status.Phase
} else {
base := fresh.DeepCopy()
now := metav1.Now()
fresh.Status.StartedAt = &now
fresh.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning
if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
}
strategy := &strategyv1alpha1.Kafka{}
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
if apierrors.IsNotFound(err) {
return r.requeueRestoreStrategyNotReady(ctx, restoreJob, backup.Spec.StrategyRef.Name)
}
return ctrl.Result{}, err
}
app, err := r.getApplicationUnstructured(ctx, targetNamespace, targetRef)
if err != nil {
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
"target Kafka application not found or kind not registered: %s/%s (kind=%q; deploy it before requesting a restore)",
targetNamespace, targetAppName, targetAppKind))
}
return ctrl.Result{}, err
}
// Ready precondition on the TARGET cluster.
cluster := &kafkatypes.Kafka{}
clusterErr := r.Get(ctx, types.NamespacedName{Namespace: targetNamespace, Name: kafkaClusterName(targetAppName)}, cluster)
if clusterErr != nil {
if !apierrors.IsNotFound(clusterErr) {
return ctrl.Result{}, clusterErr
}
return r.kafkaRestoreNotReady(ctx, restoreJob, deadline, "KafkaClusterNotReady",
fmt.Sprintf("target Kafka cluster %s not found yet", kafkaClusterName(targetAppName)))
}
if msg := kafkaNotReadyMessage(cluster); msg != "" {
return r.kafkaRestoreNotReady(ctx, restoreJob, deadline, "KafkaClusterNotReady", msg)
}
// Resolve the client image from the TARGET broker (Ready above, so a broker
// pod exists), so a to-copy restore into a differently-versioned cluster runs
// that cluster's own CLI rather than the source's. Transient List errors
// requeue; only a Ready cluster with no listable broker is terminal.
clientImage, err := resolveKafkaBrokerImage(ctx, r.Client, targetNamespace, kafkaClusterName(targetAppName))
if err != nil {
if errors.Is(err, errNoKafkaBroker) {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("cannot resolve Kafka client image: %v", err))
}
return ctrl.Result{}, err
}
// Restore reads the object the Backup recorded (guarded non-empty above), not a
// reconstruction, so a later change to the key layout cannot make older backups
// unrestorable.
artifactURI := backup.Status.Artifact.URI
ctxMap := kafkaRenderContext(app, targetAppName, targetNamespace, kafkaStrategyModeRestore, backup.Name, artifactURI, clientImage, kafkaStrategyParameters(backup), backup)
rendered, err := renderKafkaTemplate(strategy.Spec.Template, ctxMap)
if err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Kafka strategy: %v", err))
}
batchJob, err := r.ensureKafkaRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob),
kafkaStrategyModeRestore, deadline,
map[string]string{
backupsv1alpha1.OwningJobNameLabel: restoreJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
},
rendered,
)
if err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure batch/v1.Job: %v", err))
}
switch jobConditionState(batchJob) {
case batchv1.JobComplete:
now := metav1.Now()
restoreJob.Status.CompletedAt = &now
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
apimeta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionTrue,
Reason: "RestoreCompleted",
Message: "Kafka topic-metadata restore completed (topic definitions and configs only)",
})
if err := r.Status().Update(ctx, restoreJob); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
case batchv1.JobFailed:
message := jobFailureMessage(batchJob)
if message == "" {
message = "Kafka metadata restore Job reported Failed"
}
return r.markRestoreJobFailed(ctx, restoreJob, message)
default:
// Same bound as the backup path, at the Job layer: the restore Job carries
// activeDeadlineSeconds, so an unschedulable or wedged pod is failed by
// Kubernetes and lands in the JobFailed branch above rather than requeuing
// in Running forever.
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
}
func (r *RestoreJobReconciler) kafkaRestoreNotReady(ctx context.Context, rj *backupsv1alpha1.RestoreJob, deadline time.Duration, reason, message string) (ctrl.Result, error) {
if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline {
return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf("timed out waiting for Kafka cluster to become Ready: %s", message))
}
apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
})
if err := r.Status().Update(ctx, rj); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
func (r *RestoreJobReconciler) ensureKafkaRestoreJob(
ctx context.Context,
owner client.Object,
namespace, name, mode string,
deadline time.Duration,
ownerLabels map[string]string,
rendered *corev1.PodTemplateSpec,
) (*batchv1.Job, error) {
existing := &batchv1.Job{}
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing)
if err == nil {
return existing, nil
}
if !apierrors.IsNotFound(err) {
return nil, err
}
labels := map[string]string{kafkaStrategyLabelMode: mode}
for k, v := range ownerLabels {
labels[k] = v
}
desired := buildJobStrategyBatchJob(namespace, name, labels, rendered)
setKafkaJobDeadline(desired, deadline)
if err := controllerutil.SetControllerReference(owner, desired, r.Scheme); err != nil {
return nil, fmt.Errorf("set controller reference on restore Job: %w", err)
}
if err := r.Create(ctx, desired); err != nil {
if apierrors.IsAlreadyExists(err) {
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
return nil, err
}
return existing, nil
}
return nil, err
}
return desired, nil
}
// ---------------------------------------------------------------------------
// Backup cleanup path (best-effort object deletion on Backup delete)
// ---------------------------------------------------------------------------
// cleanupKafkaBackup deletes the metadata object a Backup recorded, and WAITS
// for that delete to finish before the Backup is removed, so no object is
// orphaned. Like the RabbitMQ driver, this one uniquely owns its artifact (a
// plain object it wrote with curl, keyed on status.artifact.uri) and no engine
// retention prunes it. The controller has no S3 client, so the delete runs as a
// one-shot Job through the strategy's image; this returns a requeue until the
// Job succeeds. The script treats "object already gone" as success; a genuine
// failure keeps the Backup Terminating rather than orphaning the object.
func (r *BackupReconciler) cleanupKafkaBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
logger := getLogger(ctx)
// Escape hatch: an operator releases a Backup stuck Terminating (object
// storage unreachable) by setting this annotation. Cleanup then skips the
// delete - reaping any in-flight delete Job - and lets the Backup go.
if backup.Annotations[kafkaSkipArtifactCleanupAnnotation] == "true" {
existing := &batchv1.Job{}
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name + "-cleanup"}, existing); err == nil {
_ = r.deleteKafkaCleanupJob(ctx, existing)
}
logger.Debug("skipping Kafka artifact cleanup per annotation; object left in bucket", "backup", backup.Name, "annotation", kafkaSkipArtifactCleanupAnnotation)
return ctrl.Result{}, nil
}
if backup.Status.Artifact == nil || backup.Status.Artifact.URI == "" {
return ctrl.Result{}, nil
}
uri := backup.Status.Artifact.URI
// The delete runs through the image the backup recorded: the source cluster
// (and its brokers) may already be gone, so there is no broker left to resolve
// one from. Absent it, there is nothing to run the delete with - release the
// Backup rather than wedge it Terminating forever.
clientImage := backup.Spec.DriverMetadata[kafkaStrategyClientImageKey]
if clientImage == "" {
return r.releaseKafkaCleanup(ctx, backup, uri, "no recorded client image on the Backup"), nil
}
strategy := &strategyv1alpha1.Kafka{}
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil {
if apierrors.IsNotFound(err) {
// No strategy to render the delete Job from (the shipped strategy is
// gated on a resolved bucket name and stops rendering if that lookup
// fails). Release the Backup rather than wedge it forever.
return r.releaseKafkaCleanup(ctx, backup, uri, "strategy CR is gone"), nil
}
return ctrl.Result{}, err
}
jobName := backup.Name + "-cleanup"
job := &batchv1.Job{}
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job)
switch {
case apierrors.IsNotFound(err):
// Spawning the delete Job and projecting the credentials Secret it needs
// are CREATEs, which NamespaceLifecycle admission forbids in a Terminating
// namespace. Release the Backup then, so teardown can finish; the object
// cannot be deleted from a namespace that no longer exists.
terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace)
if nsErr != nil {
return ctrl.Result{}, nsErr
}
if terminating {
return r.releaseKafkaCleanup(ctx, backup, uri, "namespace is terminating"), nil
}
if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil {
return r.releaseKafkaCleanup(ctx, backup, uri, fmt.Sprintf("cannot project credentials: %v", perr)), nil
}
// The template reads only .Mode, .ArtifactURI and .ClientImage in cleanup
// mode (never .Application), so the source app being gone does not matter.
renderCtx := kafkaRenderContext(nil, backup.Spec.ApplicationRef.Name, backup.Namespace, kafkaStrategyModeCleanup, backup.Name, uri, clientImage, nil, nil)
rendered, rerr := renderKafkaTemplate(strategy.Spec.Template, renderCtx)
if rerr != nil {
return r.releaseKafkaCleanup(ctx, backup, uri, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil
}
if cerr := r.Create(ctx, buildKafkaCleanupJob(backup.Namespace, jobName, rendered)); cerr != nil && !apierrors.IsAlreadyExists(cerr) {
// Forbidden (namespace went Terminating after the check) or Invalid
// (an over-long <name>-cleanup Job name) cannot be fixed by retrying;
// release. Other errors are transient - requeue.
if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) {
return r.releaseKafkaCleanup(ctx, backup, uri, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil
}
return ctrl.Result{}, cerr
}
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
case err != nil:
return ctrl.Result{}, err
}
if !job.DeletionTimestamp.IsZero() {
// A prior failed attempt is being collected; wait, then the NotFound
// branch recreates a fresh one.
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
switch jobConditionState(job) {
case batchv1.JobComplete:
_ = r.deleteKafkaCleanupJob(ctx, job)
logger.Debug("Kafka backup object deleted", "backup", backup.Name, "uri", uri)
return ctrl.Result{}, nil
case batchv1.JobFailed:
// A genuine delete failure (a missing object is success in the script).
// Collect the failed Job so the NotFound branch recreates a fresh one,
// and keep the Backup Terminating.
logger.Debug("Kafka cleanup Job failed; retrying", "backup", backup.Name, "job", jobName)
_ = r.deleteKafkaCleanupJob(ctx, job)
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
default:
return ctrl.Result{RequeueAfter: kafkaStrategyPollInterval}, nil
}
}
// deleteKafkaCleanupJob removes a finished cleanup Job together with its Pod.
func (r *BackupReconciler) deleteKafkaCleanupJob(ctx context.Context, job *batchv1.Job) error {
policy := metav1.DeletePropagationBackground
return client.IgnoreNotFound(r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &policy}))
}
// releaseKafkaCleanup gives up on deleting the backup object and lets the Backup
// be removed - for the conditions under which the delete cannot run here
// (namespace terminating, credentials/permissions unavailable, an unrenderable
// or absent strategy). It records a Warning Event naming the object left behind
// so the give-up is visible, then returns a zero Result so the finalizer is
// released. The delete is best-effort, not guaranteed.
func (r *BackupReconciler) releaseKafkaCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, uri, reason string) ctrl.Result {
getLogger(ctx).Info("releasing Backup without deleting its object", "backup", backup.Name, "uri", uri, "reason", reason)
if r.Recorder != nil {
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left object %s in the bucket: %s", uri, reason)
}
return ctrl.Result{}
}
// setKafkaJobDeadline stamps activeDeadlineSeconds on a backup/restore Job. No
// TTLSecondsAfterFinished: the controller must observe the Job's terminal
// condition (to create the Backup, or mark the owner Failed) before it is
// collected, and the owning BackupJob/RestoreJob garbage-collects it.
func setKafkaJobDeadline(job *batchv1.Job, deadline time.Duration) {
secs := int64(deadline / time.Second)
if secs < 1 {
secs = 1
}
job.Spec.ActiveDeadlineSeconds = &secs
}
// buildKafkaCleanupJob wraps the cleanup-mode pod in a one-shot, ownerless Job.
// Ownerless because cleanupKafkaBackup manages its lifecycle explicitly and it
// must outlive the Backup being deleted; activeDeadlineSeconds + TTL are
// backstops if the controller stops mid-wait.
func buildKafkaCleanupJob(namespace, name string, rendered *corev1.PodTemplateSpec) *batchv1.Job {
pod := *rendered.DeepCopy()
if pod.Spec.RestartPolicy == "" {
pod.Spec.RestartPolicy = corev1.RestartPolicyNever
}
if pod.Labels == nil {
pod.Labels = map[string]string{}
}
pod.Labels[kafkaStrategyLabelMode] = kafkaStrategyModeCleanup
backoffLimit := int32(1)
activeDeadline := int64(300)
ttl := int32(300)
return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
Labels: map[string]string{kafkaStrategyLabelMode: kafkaStrategyModeCleanup},
},
Spec: batchv1.JobSpec{
BackoffLimit: &backoffLimit,
ActiveDeadlineSeconds: &activeDeadline,
TTLSecondsAfterFinished: &ttl,
Template: pod,
},
}
}