Files
cozystack/internal/backupcontroller/velerostrategy_controller.go
T
Andrey Kolkov d4ad5612d3 fix(backups): report sizeBytes for MariaDB backup artifacts
Backup.status.artifact.sizeBytes was only ever set by the etcd strategy,
so MariaDB backups always arrived without a size.

mariadb-operator writes each logical dump as
<prefix>/backup.<RFC3339>[.<compression>].sql under a prefix shared by
every backup of the application, and the URI the driver records is
synthetic, so there is no single object to HEAD. On completion the driver
now lists the prefix with the tenant credentials the operator Backup was
rendered with and takes the size of the one dump whose timestamp falls
inside the operator Backup's lifetime. When the bucket cannot be read or
the window does not match exactly one dump, the field stays unset.

The API doc now states that an absent sizeBytes means "unknown", and the
Velero strategy documents why it leaves the field unset: kopia stores
data content-addressed across backups, so there is no per-backup stored
size.

Refs #4316

Assisted-by: LLM
Signed-off-by: Andrey Kolkov <androndo@gmail.com>
2026-11-04 13:04:22 +00:00

1993 lines
82 KiB
Go

package backupcontroller
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"sigs.k8s.io/yaml"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/template"
"github.com/go-logr/logr"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
func getLogger(ctx context.Context) loggerWithDebug {
return loggerWithDebug{Logger: log.FromContext(ctx)}
}
// loggerWithDebug wraps a logr.Logger and provides a Debug() method
// that maps to V(1).Info() for convenience.
type loggerWithDebug struct {
logr.Logger
}
// Debug logs at debug level (equivalent to V(1).Info())
func (l loggerWithDebug) Debug(msg string, keysAndValues ...any) {
l.Logger.V(1).Info(msg, keysAndValues...)
}
const (
defaultRequeueAfter = 5 * time.Second
defaultActiveJobPollingInterval = defaultRequeueAfter
defaultRestoreRequeueAfter = 5 * time.Second
defaultActiveRestorePollingInterval = defaultRestoreRequeueAfter
// Condition recorded on a RestoreJob before prepareForRestore first changes
// the target. From then on a failure has to say what preparation left
// behind, and its LastTransitionTime starts the preparation deadline.
restoreCondPreparationStarted = "PreparationStarted"
// How long preparation may wait (VM shutdown, Pods releasing the disks, the
// DataVolume and a recreated PVC going away) before the RestoreJob fails.
restorePreparationDeadline = 30 * time.Minute
// Velero requires API objects and secrets to be in the cozy-velero namespace
veleroNamespace = "cozy-velero"
veleroBackupNameMetadataKey = "velero.io/backup-name"
veleroBackupNamespaceMetadataKey = "velero.io/backup-namespace"
// Annotation key for persisting underlying resources on the Velero Backup object
underlyingResourcesAnnotation = "backups.cozystack.io/underlying-resources"
// keepOriginalPVC moves a disk's PV to Retain so it survives the deletion
// of its PVC. The policy it had is kept here, on the PV and on the
// <name>-orig-<hash> PVC, because nothing in the controller restores it:
// whoever deletes the -orig PVC must set it back first, or a PV that was
// Delete stays Retain and its volume is never reclaimed.
originalReclaimPolicyAnnotation = "backups.cozystack.io/original-reclaim-policy"
// The -orig suffix hashes the RestoreJob name, and a name can be reused
// once the previous RestoreJob is deleted, leaving its -orig PVC behind.
// The UID tells a rename this RestoreJob started apart from that one.
origPVCRestoreJobUIDAnnotation = "backups.cozystack.io/restore-job-uid"
// VM-specific constants
vmInstanceKind = "VMInstance"
vmDiskAppKind = "VMDisk"
vmNamePrefix = "vm-instance-"
vmDiskNamePrefix = "vm-disk-"
appKindLabel = "apps.cozystack.io/application.kind"
appNameLabel = "apps.cozystack.io/application.name"
vmPodNameLabel = "vm.kubevirt.io/name"
ovnIPAnnotation = "ovn.kubernetes.io/ip_address"
ovnMACAnnotation = "ovn.kubernetes.io/mac_address"
cdiAllowClaimAdoption = "cdi.kubevirt.io/allowClaimAdoption"
)
// boolDefault returns the value of a *bool pointer, or the given default if nil.
func boolDefault(p *bool, def bool) bool {
if p != nil {
return *p
}
return def
}
// CommonRestoreOptions contains driver-agnostic restore options shared across
// all application kinds.
type CommonRestoreOptions struct {
// TargetNamespace is the namespace to restore into. When set (and differs
// from the backup namespace), a cross-namespace restore (copy) is performed
// using Velero's namespaceMapping.
TargetNamespace string `json:"targetNamespace,omitempty"`
// FailIfTargetExists makes the restore fail if the target resource already
// exists. Defaults to true when omitted.
FailIfTargetExists *bool `json:"failIfTargetExists,omitempty"`
}
// RestoreOptions is the typed representation of RestoreJob.Spec.Options for the
// Velero driver. The struct is deserialized from runtime.RawExtension and used
// for all application kinds. KeepOriginalPVC applies to VMInstance and VMDisk
// targets; KeepOriginalIpAndMac only to VMInstance.
type RestoreOptions struct {
CommonRestoreOptions `json:",inline"`
// KeepOriginalPVC renames the original PVC to <name>-orig-<hash> before restore;
// when false, the original PVC is deleted so that Velero recreates it.
// Only effective for an in-place VMInstance or VMDisk restore (no targetNamespace). Defaults to true when omitted.
KeepOriginalPVC *bool `json:"keepOriginalPVC,omitempty"`
// KeepOriginalIpAndMac preserves the original IP and MAC address via OVN
// annotations. Only effective for VMInstance restores. Defaults to true when omitted.
KeepOriginalIpAndMac *bool `json:"keepOriginalIpAndMac,omitempty"`
}
// GetFailIfTargetExists returns the effective value (default: true).
func (o *CommonRestoreOptions) GetFailIfTargetExists() bool {
return boolDefault(o.FailIfTargetExists, true)
}
// GetKeepOriginalPVC returns the effective value (default: true).
func (o *RestoreOptions) GetKeepOriginalPVC() bool {
return boolDefault(o.KeepOriginalPVC, true)
}
// GetKeepOriginalIpAndMac returns the effective value (default: true).
func (o *RestoreOptions) GetKeepOriginalIpAndMac() bool {
return boolDefault(o.KeepOriginalIpAndMac, true)
}
// parseRestoreOptions deserializes RestoreJob.Spec.Options into RestoreOptions.
// Returns zero-value RestoreOptions if options is nil.
func parseRestoreOptions(opts *runtime.RawExtension) (RestoreOptions, error) {
var ro RestoreOptions
if opts == nil || len(opts.Raw) == 0 {
return ro, nil
}
if err := json.Unmarshal(opts.Raw, &ro); err != nil {
return ro, fmt.Errorf("failed to parse restore options: %w", err)
}
return ro, nil
}
// restoreTarget holds the resolved target namespace and app identity for a restore operation.
type restoreTarget struct {
Namespace string
AppName string
AppKind string
IsCopy bool // true when targetNamespace differs from backup namespace
IsRenamed bool // true when target app name differs from source app name
}
// resolveRestoreTarget computes the effective restore target from RestoreJob, Backup, and options.
func resolveRestoreTarget(restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts RestoreOptions) restoreTarget {
targetNS := backup.Namespace
isCopy := false
if opts.TargetNamespace != "" && opts.TargetNamespace != backup.Namespace {
targetNS = opts.TargetNamespace
isCopy = true
}
targetAppName := backup.Spec.ApplicationRef.Name
if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Name != "" {
targetAppName = restoreJob.Spec.TargetApplicationRef.Name
}
targetAppKind := backup.Spec.ApplicationRef.Kind
if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Kind != "" {
targetAppKind = restoreJob.Spec.TargetApplicationRef.Kind
}
return restoreTarget{
Namespace: targetNS,
AppName: targetAppName,
AppKind: targetAppKind,
IsCopy: isCopy,
IsRenamed: targetAppName != backup.Spec.ApplicationRef.Name,
}
}
// vmInstanceResources contains VM-specific underlying resources discovered during backup.
type vmInstanceResources struct {
DataVolumes []backupsv1alpha1.DataVolumeResource `json:"dataVolumes,omitempty"`
IP string `json:"ip,omitempty"`
MAC string `json:"mac,omitempty"`
}
// marshalUnderlyingResources serializes application-specific data into a
// runtime.RawExtension suitable for Backup.Status.UnderlyingResources.
func marshalUnderlyingResources(data any) (*runtime.RawExtension, error) {
raw, err := json.Marshal(data)
if err != nil {
return nil, err
}
return &runtime.RawExtension{Raw: raw}, nil
}
// getVMInstanceResources extracts VMInstance-specific resources from the opaque blob.
// The caller is responsible for checking that the application kind is VMInstance
// (via backup.Spec.ApplicationRef.Kind) before calling this function.
// Returns nil if ur is nil or has no VM-specific data.
func getVMInstanceResources(ur *runtime.RawExtension) *vmInstanceResources {
if ur == nil || len(ur.Raw) == 0 {
return nil
}
var res vmInstanceResources
if err := json.Unmarshal(ur.Raw, &res); err != nil {
return nil
}
if len(res.DataVolumes) == 0 && res.IP == "" && res.MAC == "" {
return nil
}
return &res
}
func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) {
logger := getLogger(ctx)
logger.Debug("reconciling Velero strategy", "backupjob", j.Name, "phase", j.Status.Phase)
// If already completed, no need to reconcile
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded ||
j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
logger.Debug("BackupJob already completed, skipping", "phase", j.Status.Phase)
return ctrl.Result{}, nil
}
// Step 1: On first reconcile, set startedAt (but not phase yet - phase will be set after backup creation)
logger.Debug("checking BackupJob status", "startedAt", j.Status.StartedAt, "phase", j.Status.Phase)
if j.Status.StartedAt == nil {
logger.Debug("setting BackupJob StartedAt")
now := metav1.Now()
j.Status.StartedAt = &now
// Don't set phase to Running yet - will be set after Velero backup is successfully created
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob status")
return ctrl.Result{}, err
}
logger.Debug("set BackupJob StartedAt", "startedAt", j.Status.StartedAt)
} else {
logger.Debug("BackupJob already started", "startedAt", j.Status.StartedAt, "phase", j.Status.Phase)
}
// Step 2: Resolve inputs - Read Strategy from resolved config
logger.Debug("fetching Velero strategy", "strategyName", resolved.StrategyRef.Name)
veleroStrategy := &strategyv1alpha1.Velero{}
if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, veleroStrategy); err != nil {
if errors.IsNotFound(err) {
return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name)
}
logger.Error(err, "failed to get Velero strategy")
return ctrl.Result{}, err
}
logger.Debug("fetched Velero strategy", "strategyName", veleroStrategy.Name)
// Step 3: Execute backup logic
// Check if we already created a Velero Backup
if j.Status.StartedAt == nil {
logger.Error(nil, "StartedAt is nil after status update, this should not happen")
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
logger.Debug("checking for existing Velero Backup", "namespace", veleroNamespace)
veleroBackupList := &velerov1.BackupList{}
opts := []client.ListOption{
client.InNamespace(veleroNamespace),
client.MatchingLabels{
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
backupsv1alpha1.OwningJobNameLabel: j.Name,
},
}
if err := r.List(ctx, veleroBackupList, opts...); err != nil {
logger.Error(err, "failed to get Velero Backup")
return ctrl.Result{}, err
}
if len(veleroBackupList.Items) == 0 {
// Create Velero Backup
logger.Debug("Velero Backup not found, creating new one")
if err := r.createVeleroBackup(ctx, j, veleroStrategy, resolved); err != nil {
logger.Error(err, "failed to create Velero Backup")
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Velero Backup: %v", err))
}
// After successful Velero backup creation, set phase to Running
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
logger.Debug("setting BackupJob phase to Running after successful Velero backup creation")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob phase to Running")
return ctrl.Result{}, err
}
}
logger.Debug("created Velero Backup, requeuing")
// Requeue to check status
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
if len(veleroBackupList.Items) > 1 {
logger.Error(fmt.Errorf("too many Velero backups for BackupJob"), "found more than one Velero Backup referencing a single BackupJob as owner")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseFailed
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob status")
}
return ctrl.Result{}, nil
}
veleroBackup := veleroBackupList.Items[0].DeepCopy()
logger.Debug("found existing Velero Backup", "phase", veleroBackup.Status.Phase)
// If Velero backup exists but phase is not Running, set it to Running
// This handles the case where the backup was created but phase wasn't set yet
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
logger.Debug("setting BackupJob phase to Running (Velero backup already exists)")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob phase to Running")
return ctrl.Result{}, err
}
}
// Check Velero Backup status
phase := string(veleroBackup.Status.Phase)
if phase == "" {
// Still in progress, requeue
return ctrl.Result{RequeueAfter: defaultActiveJobPollingInterval}, nil
}
// Step 4: On success - Create Backup resource and update status
if phase == "Completed" {
// Check if we already created the Backup resource
if j.Status.BackupRef == nil {
backup, err := r.createBackupResource(ctx, j, veleroBackup, resolved)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup resource: %v", err))
}
now := metav1.Now()
j.Status.BackupRef = &corev1.LocalObjectReference{Name: backup.Name}
j.Status.CompletedAt = &now
j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob status")
return ctrl.Result{}, err
}
logger.Debug("BackupJob succeeded", "backup", backup.Name)
}
return ctrl.Result{}, nil
}
// Step 5: On failure
if phase == "Failed" || phase == "PartiallyFailed" {
message := formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, veleroBackup)
return r.markBackupJobFailed(ctx, j, message)
}
// Still in progress (InProgress, New, etc.)
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// collectUnderlyingResources discovers resources associated with an application
// that need to be backed up and restored. Returns a map keyed by application kind.
// Returns nil if the application type has no underlying resources to collect.
func (r *BackupJobReconciler) collectUnderlyingResources(ctx context.Context, app *unstructured.Unstructured, appKind, ns string) (*runtime.RawExtension, error) {
logger := getLogger(ctx)
appName := app.GetName()
// A standalone VMDisk is its own underlying disk. Capturing it lets the
// restore controller prepare it as it prepares a VMInstance's disks: keep
// the live PVC aside and delete the DataVolume, so the restored PVC is
// created fresh under the original name. The HelmRelease is suspended only
// until Velero restores it; Flux then recreates the DataVolume and CDI
// adopts the restored PVC because the vm-disk chart sets allowClaimAdoption
// on that DataVolume. A standalone VMDisk has no VM, so there is no IP/MAC
// to collect.
if appKind == vmDiskAppKind {
return marshalUnderlyingResources(vmInstanceResources{
DataVolumes: []backupsv1alpha1.DataVolumeResource{{
DataVolumeName: vmDiskNamePrefix + appName,
ApplicationName: appName,
}},
})
}
if appKind != vmInstanceKind {
logger.Debug("application has no underlying resources to collect", "kind", appKind)
return nil, nil
}
// Extract disk names from VMInstance spec.disks[].name
disks, found, err := unstructured.NestedSlice(app.Object, "spec", "disks")
if err != nil {
return nil, fmt.Errorf("failed to read spec.disks from application: %w", err)
}
var dataVolumes []backupsv1alpha1.DataVolumeResource
if found {
for _, d := range disks {
disk, ok := d.(map[string]any)
if !ok {
continue
}
name, ok := disk["name"].(string)
if !ok || name == "" {
continue
}
dataVolumes = append(dataVolumes, backupsv1alpha1.DataVolumeResource{
DataVolumeName: vmDiskNamePrefix + name,
ApplicationName: name,
})
}
}
logger.Debug("collected dataVolumes from VMInstance", "count", len(dataVolumes), "appName", appName)
// Find VM Pod to extract OVN IP/MAC addresses
vmName := vmNamePrefix + appName
podList := &corev1.PodList{}
if err := r.List(ctx, podList,
client.InNamespace(ns),
client.MatchingLabels{vmPodNameLabel: vmName},
); err != nil {
logger.Error(err, "failed to list VM pods for IP/MAC collection", "vmName", vmName)
// Non-fatal: we can still proceed without IP/MAC
}
var ip, mac string
if len(podList.Items) > 0 {
pod := podList.Items[0]
ip = pod.Annotations[ovnIPAnnotation]
mac = pod.Annotations[ovnMACAnnotation]
logger.Debug("collected OVN network info from VM pod", "ip", ip, "mac", mac, "pod", pod.Name)
} else {
logger.Debug("no VM pod found for OVN info", "vmName", vmName)
}
if len(dataVolumes) == 0 && ip == "" && mac == "" {
return nil, nil
}
return marshalUnderlyingResources(vmInstanceResources{
DataVolumes: dataVolumes,
IP: ip,
MAC: mac,
})
}
func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Velero, resolved *ResolvedBackupConfig) error {
logger := getLogger(ctx)
logger.Debug("createVeleroBackup called", "strategy", strategy.Name)
mapping, err := r.RESTMapping(schema.GroupKind{Group: *backupJob.Spec.ApplicationRef.APIGroup, Kind: backupJob.Spec.ApplicationRef.Kind})
if err != nil {
return err
}
ns := backupJob.Namespace
if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
ns = ""
}
app, err := r.Resource(mapping.Resource).Namespace(ns).Get(ctx, backupJob.Spec.ApplicationRef.Name, metav1.GetOptions{})
if err != nil {
return err
}
// Collect underlying resources (VM disks, IP/MAC)
underlyingResources, err := r.collectUnderlyingResources(ctx, app, backupJob.Spec.ApplicationRef.Kind, backupJob.Namespace)
if err != nil {
logger.Error(err, "failed to collect underlying resources, proceeding without them")
// Non-fatal: proceed with backup even if collection fails
}
templateContext := map[string]any{
"Application": app.Object,
"Parameters": resolved.Parameters,
}
veleroBackupSpec, err := template.Template(&strategy.Spec.Template.Spec, templateContext)
if err != nil {
return err
}
// Add label selectors for a VMInstance's VMDisk HelmReleases. A VMDisk's own
// strategy already selects the disk, and Velero rejects a spec that carries
// both labelSelector and orLabelSelectors, so a VMDisk strategy written with
// labelSelector must be left as it is.
if vmRes := getVMInstanceResources(underlyingResources); vmRes != nil && backupJob.Spec.ApplicationRef.Kind == vmInstanceKind {
for _, dv := range vmRes.DataVolumes {
veleroBackupSpec.OrLabelSelectors = append(veleroBackupSpec.OrLabelSelectors, &metav1.LabelSelector{
MatchLabels: map[string]string{
appKindLabel: vmDiskAppKind,
appNameLabel: dv.ApplicationName,
},
})
}
if len(vmRes.DataVolumes) > 0 {
logger.Debug("added VMDisk label selectors to Velero backup", "count", len(vmRes.DataVolumes))
}
}
// Serialize underlying resources as annotation to persist across reconcile cycles
annotations := map[string]string{}
if underlyingResources != nil && len(underlyingResources.Raw) > 0 {
annotations[underlyingResourcesAnnotation] = string(underlyingResources.Raw)
}
veleroBackup := &velerov1.Backup{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s.%s-", backupJob.Namespace, backupJob.Name),
Namespace: veleroNamespace,
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: backupJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace,
},
Annotations: annotations,
},
Spec: *veleroBackupSpec,
}
name := veleroBackup.GenerateName
if err := r.Create(ctx, veleroBackup); err != nil {
if veleroBackup.Name != "" {
name = veleroBackup.Name
}
logger.Error(err, "failed to create Velero Backup", "name", veleroBackup.Name)
r.Recorder.Event(backupJob, corev1.EventTypeWarning, "VeleroBackupCreationFailed",
fmt.Sprintf("Failed to create Velero Backup %s/%s: %v", veleroNamespace, name, err))
return err
}
logger.Debug("created Velero Backup", "name", veleroBackup.Name, "namespace", veleroBackup.Namespace)
r.Recorder.Event(backupJob, corev1.EventTypeNormal, "VeleroBackupCreated",
fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, name))
return nil
}
func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, veleroBackup *velerov1.Backup, resolved *ResolvedBackupConfig) (*backupsv1alpha1.Backup, error) {
logger := getLogger(ctx)
// Get takenAt from Velero Backup creation timestamp or status
takenAt := metav1.Now()
if veleroBackup.Status.StartTimestamp != nil {
takenAt = *veleroBackup.Status.StartTimestamp
} else if !veleroBackup.CreationTimestamp.IsZero() {
takenAt = veleroBackup.CreationTimestamp
}
// Extract driver metadata (e.g., Velero backup name)
driverMetadata := map[string]string{
veleroBackupNameMetadataKey: veleroBackup.Name,
veleroBackupNamespaceMetadataKey: veleroBackup.Namespace,
}
// SizeBytes stays unset: data lands in a content-addressed kopia
// repository shared across backups, so there is no per-backup stored
// size, and DataUpload totalBytes counts bytes read, not bytes stored.
artifact := &backupsv1alpha1.BackupArtifact{
URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name),
}
// Read underlying resources from Velero Backup annotation
var underlyingResources *runtime.RawExtension
if urJSON, ok := veleroBackup.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" {
underlyingResources = &runtime.RawExtension{Raw: []byte(urJSON)}
}
// Note: No OwnerReferences set on Backup. The Backup must survive BackupJob deletion
// so users don't lose their backup artifacts when cleaning up completed jobs.
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: backupJob.Name,
Namespace: backupJob.Namespace,
},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: backupJob.Spec.ApplicationRef,
StrategyRef: resolved.StrategyRef,
TakenAt: takenAt,
DriverMetadata: driverMetadata,
},
Status: backupsv1alpha1.BackupStatus{
Phase: backupsv1alpha1.BackupPhaseReady,
Artifact: artifact,
UnderlyingResources: underlyingResources,
},
}
if backupJob.Spec.PlanRef != nil {
backup.Spec.PlanRef = backupJob.Spec.PlanRef
}
if err := r.Create(ctx, backup); err != nil {
logger.Error(err, "failed to create Backup resource")
return nil, err
}
logger.Debug("created Backup resource", "name", backup.Name,
"hasUnderlyingResources", underlyingResources != nil)
return backup, nil
}
// reconcileVeleroRestore handles restore operations for Velero strategy.
func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
logger := getLogger(ctx)
logger.Debug("reconciling Velero strategy restore", "restorejob", restoreJob.Name, "backup", backup.Name)
// Parse restore options from the opaque blob
restoreOpts, err := parseRestoreOptions(restoreJob.Spec.Options)
if err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("invalid restore options: %v", err))
}
target := resolveRestoreTarget(restoreJob, backup, restoreOpts)
logger.Debug("resolved restore target", "targetNS", target.Namespace, "targetApp", target.AppName, "isCopy", target.IsCopy)
// Validate: target namespace must exist for cross-namespace copies
if target.IsCopy {
targetNS := &corev1.Namespace{}
if err := r.Get(ctx, client.ObjectKey{Name: target.Namespace}, targetNS); err != nil {
if errors.IsNotFound(err) {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
"target namespace %q does not exist; create it before requesting a cross-namespace restore",
target.Namespace))
}
return ctrl.Result{}, fmt.Errorf("failed to check target namespace %q: %w", target.Namespace, err)
}
}
// Validate: same-namespace restore with a different app name is not supported
// due to Velero DataUpload always writing to PVCs with the original name.
if !target.IsCopy && target.AppName != backup.Spec.ApplicationRef.Name {
return r.markRestoreJobFailed(ctx, restoreJob,
"restoring to the same namespace with a different application name is not supported "+
"due to Velero DataUpload limitations: data is always uploaded to PVCs with the original name. "+
"Use options.targetNamespace to restore into a different namespace")
}
// Validate: rename is only implemented for VMInstance. Failing here beats a
// Succeeded RestoreJob whose application silently keeps the source name.
if target.IsRenamed && !veleroRenameSupported(backup.Spec.ApplicationRef.Kind) {
msg := fmt.Sprintf(
"restoring %s %q under a different name (%q) is not supported by the Velero strategy: "+
"Velero DataUpload always restores volumes under their original names and only a VMInstance "+
"can adopt them. Omit targetApplicationRef.name to keep the source name",
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name, target.AppName)
if backup.Spec.ApplicationRef.Kind != vmDiskAppKind {
msg += ", or use the application's engine-native strategy to restore into a new application"
}
if restoreJob.Status.StartedAt != nil {
// The job predates this check: its Velero Restore may already have
// restored the application under the source name.
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
msg += fmt.Sprintf(". This RestoreJob was already running, so %q may have been restored into namespace %q under its source name",
backup.Spec.ApplicationRef.Name, target.Namespace)
}
return r.markRestoreJobFailed(ctx, restoreJob, msg)
}
// Step 1: On first reconcile, set startedAt and phase = Running
if restoreJob.Status.StartedAt == nil {
logger.Debug("setting RestoreJob StartedAt and phase to Running")
now := metav1.Now()
restoreJob.Status.StartedAt = &now
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning
if err := r.Status().Update(ctx, restoreJob); err != nil {
logger.Error(err, "failed to update RestoreJob status")
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
// Step 2: Resolve inputs - Read Strategy, Storage, target Application
logger.Debug("fetching Velero strategy", "strategyName", backup.Spec.StrategyRef.Name)
veleroStrategy := &strategyv1alpha1.Velero{}
if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, veleroStrategy); err != nil {
if errors.IsNotFound(err) {
return r.requeueRestoreStrategyNotReady(ctx, restoreJob, backup.Spec.StrategyRef.Name)
}
logger.Error(err, "failed to get Velero strategy")
return ctrl.Result{}, err
}
logger.Debug("fetched Velero strategy", "strategyName", veleroStrategy.Name)
// Get Velero backup name from Backup's driverMetadata
veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey]
if !ok {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("Backup missing Velero backup name in driverMetadata (key: %s)", veleroBackupNameMetadataKey))
}
// Step 3: Execute restore logic
// Check if we already created a Velero Restore
logger.Debug("checking for existing Velero Restore", "namespace", veleroNamespace)
veleroRestoreList := &velerov1.RestoreList{}
opts := []client.ListOption{
client.InNamespace(veleroNamespace),
client.MatchingLabels{
backupsv1alpha1.OwningJobNameLabel: restoreJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
},
}
if err := r.List(ctx, veleroRestoreList, opts...); err != nil {
logger.Error(err, "failed to get Velero Restore")
return ctrl.Result{}, err
}
if len(veleroRestoreList.Items) == 0 {
// For copy restores, enforce failIfTargetExists before touching anything.
// In-place restores are excluded: the source application is expected to exist
// and will be halted/overwritten deliberately.
if target.IsCopy && restoreOpts.GetFailIfTargetExists() {
targetHRName := helmReleaseNameForApp(target.AppKind, target.AppName)
exists, err := r.targetHelmReleaseExists(ctx, target.AppKind, targetHRName, target.Namespace)
if err != nil {
logger.Error(err, "failed to check whether target HelmRelease exists")
return ctrl.Result{}, err
}
if exists {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
"target application %q already exists in namespace %q; "+
"set options.failIfTargetExists=false to overwrite",
target.AppName, target.Namespace))
}
}
// The Backup artifact outlives the Velero Backup it points to, which
// Velero garbage-collects once its TTL expires. Checked on every pass
// before prepareForRestore halts the application and deletes its disks,
// so a restore that cannot run leaves them in place, or, when preparation
// already started, names what it left behind.
veleroBackup, unrestorable, err := r.veleroBackupForRestore(ctx, veleroBackupName)
if err != nil {
return ctrl.Result{}, err
}
// Resolve underlying resources once; prefer Backup status, fall back to Velero annotation.
ur := resolveUnderlyingResourcesForRestore(backup, veleroBackup)
// Preparation spans several passes. Once it has changed the target, a
// failure must name what it left behind instead of claiming nothing changed.
failBeforeRestore := func(msg string) (ctrl.Result, error) {
if meta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondPreparationStarted) {
return r.markPreparedRestoreFailed(ctx, restoreJob, msg+r.preparedRestoreLeftovers(ctx, restoreJob, backup, ur, target, restoreOpts))
}
return r.markRestoreJobFailed(ctx, restoreJob, msg+"; nothing was changed")
}
if unrestorable != "" {
return failBeforeRestore(unrestorable)
}
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy)
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts)
if err != nil {
return failBeforeRestore(fmt.Sprintf("pre-restore preparation failed: %v", err))
}
if !ready {
if cond := meta.FindStatusCondition(restoreJob.Status.Conditions, restoreCondPreparationStarted); cond != nil &&
cond.Status == metav1.ConditionTrue && time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline {
return failBeforeRestore(fmt.Sprintf("pre-restore preparation did not finish within %s; see the RestoreJob events for what it was waiting on", restorePreparationDeadline))
}
logger.Debug("pre-restore preparation in progress, requeuing")
return result, nil
}
// Create Velero Restore
logger.Debug("Velero Restore not found, creating new one")
if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur, target, restoreOpts); err != nil {
logger.Error(err, "failed to create Velero Restore")
return r.markPreparedRestoreFailed(ctx, restoreJob, fmt.Sprintf("failed to create Velero Restore: %v", err)+
r.preparedRestoreLeftovers(ctx, restoreJob, backup, ur, target, restoreOpts))
}
logger.Debug("created Velero Restore, requeuing")
// Requeue to check status
return ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
if len(veleroRestoreList.Items) > 1 {
logger.Error(fmt.Errorf("too many Velero restores for RestoreJob"), "found more than one Velero Restore referencing a single RestoreJob as owner")
return r.markPreparedRestoreFailed(ctx, restoreJob, "found multiple Velero Restores for this RestoreJob"+
r.preparedRestoreLeftovers(ctx, restoreJob, backup, r.restoredUnderlyingResources(ctx, backup, veleroBackupName), target, restoreOpts))
}
veleroRestore := veleroRestoreList.Items[0].DeepCopy()
logger.Debug("found existing Velero Restore", "phase", veleroRestore.Status.Phase)
// Check Velero Restore status
phase := string(veleroRestore.Status.Phase)
if phase == "" {
// Still in progress, requeue
return ctrl.Result{RequeueAfter: defaultActiveRestorePollingInterval}, nil
}
// Step 4: On success
if phase == "Completed" {
// Post-restore: rename resources if target app name differs from source.
// Velero resource modifiers cannot change metadata.name, so we do it after restore.
if target.IsRenamed {
if err := r.postRestoreRename(ctx, restoreJob, backup, target); err != nil {
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("post-restore rename failed: %v", err))
}
}
// Clean up resource modifier ConfigMaps now that the restore is complete.
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
now := metav1.Now()
restoreJob.Status.CompletedAt = &now
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
if err := r.Status().Update(ctx, restoreJob); err != nil {
logger.Error(err, "failed to update RestoreJob status")
return ctrl.Result{}, err
}
logger.Debug("RestoreJob succeeded")
return ctrl.Result{}, nil
}
// Step 5: On failure. FailedValidation is terminal too; left to the polling
// branch below it would keep a target that preparation already took apart
// Running forever.
if phase == string(velerov1.RestorePhaseFailed) || phase == string(velerov1.RestorePhasePartiallyFailed) ||
phase == string(velerov1.RestorePhaseFailedValidation) {
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
message := fmt.Sprintf("Velero Restore failed with phase: %s", phase)
if veleroRestore.Status.FailureReason != "" {
message = fmt.Sprintf("%s: %s", message, veleroRestore.Status.FailureReason)
}
if len(veleroRestore.Status.ValidationErrors) > 0 {
message = fmt.Sprintf("%s: %s", message, strings.Join(veleroRestore.Status.ValidationErrors, "; "))
}
return r.markPreparedRestoreFailed(ctx, restoreJob, message+
r.preparedRestoreLeftovers(ctx, restoreJob, backup, r.restoredUnderlyingResources(ctx, backup, veleroBackupName), target, restoreOpts))
}
// Still in progress (InProgress, New, etc.)
return ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
// Velero resource modifier types (local mirrors of the internal Velero types).
type resourceModifiers struct {
Version string `yaml:"version"`
ResourceModifierRules []resourceModifierRule `yaml:"resourceModifierRules"`
}
type resourceModifierRule struct {
Conditions resourceModifierConditions `yaml:"conditions"`
MergePatches []mergePatch `yaml:"mergePatches,omitempty"`
Patches []jsonPatch `yaml:"patches,omitempty"`
}
type resourceModifierConditions struct {
GroupResource string `yaml:"groupResource"`
ResourceNameRegex string `yaml:"resourceNameRegex,omitempty"`
Namespaces []string `yaml:"namespaces,omitempty"`
}
type mergePatch struct {
PatchData string `yaml:"patchData"`
}
type jsonPatch struct {
Operation string `yaml:"operation"`
Path string `yaml:"path"`
Value string `yaml:"value,omitempty"`
}
// marshalPatchData marshals an arbitrary object to YAML for use as
// mergePatch.PatchData in Velero resource modifiers.
func marshalPatchData(v any) (string, error) {
b, err := yaml.Marshal(v)
if err != nil {
return "", fmt.Errorf("failed to marshal patch data: %w", err)
}
return string(b), nil
}
// createResourceModifiersConfigMap creates a Velero resource modifiers ConfigMap
// that patches VM resources during restore:
// - PVC adoption: adds cdi.kubevirt.io/allowClaimAdoption=true to all
// restored PVCs. CDI v1.66 does not read it there, only on the DataVolume,
// so adoption of a restored PVC rests on the vm-disk chart annotating the
// DataVolume it recreates. Without it, CDI refuses that DataVolume unless
// the DataVolumeClaimAdoption feature gate is on.
// - OVN IP/MAC: sets OVN annotations on the VirtualMachine for correct ssh access to restored VM.
func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (*corev1.ConfigMap, error) {
logger := getLogger(ctx)
targetNS := target.Namespace
var rules []resourceModifierRule
// Inert for CDI, which reads allowClaimAdoption only on the DataVolume.
pvcPatch, err := marshalPatchData(map[string]any{
"metadata": map[string]any{
"annotations": map[string]string{
cdiAllowClaimAdoption: "true",
},
},
})
if err != nil {
return nil, err
}
rules = append(rules, resourceModifierRule{
Conditions: resourceModifierConditions{
GroupResource: "persistentvolumeclaims",
ResourceNameRegex: ".*",
Namespaces: []string{targetNS},
},
MergePatches: []mergePatch{{PatchData: pvcPatch}},
})
// For cross-namespace restore: strip Velero's dynamic PV restore selector and
// volumeName from PVCs so the storage provisioner can dynamically provision new PVs.
// Without this, Velero's CSI PVCAction adds spec.selector with a velero.io/dynamic-pv-restore
// label that prevents dynamic provisioning when PVs are not included in the restore.
// Uses merge patch (null values) instead of JSON Patch remove to avoid RFC 6902
// failures when the fields don't exist on the PVC (e.g. statically provisioned PVCs).
if target.IsCopy {
pvcStripPatch, err := marshalPatchData(map[string]any{
"spec": map[string]any{
"selector": nil,
"volumeName": nil,
},
})
if err != nil {
return nil, err
}
rules = append(rules, resourceModifierRule{
Conditions: resourceModifierConditions{
GroupResource: "persistentvolumeclaims",
ResourceNameRegex: ".*",
Namespaces: []string{targetNS},
},
MergePatches: []mergePatch{{PatchData: pvcStripPatch}},
})
}
// OVN IP/MAC annotations on VirtualMachine for correct network identity after restore.
// Only applied when keepOriginalIpAndMac is true; for restore-to-copy the copy
// should get new IP/MAC from the network to avoid conflicts.
if opts.GetKeepOriginalIpAndMac() {
if vmRes := getVMInstanceResources(ur); vmRes != nil && (vmRes.IP != "" || vmRes.MAC != "") {
ovnAnnotations := map[string]string{}
if vmRes.IP != "" {
ovnAnnotations[ovnIPAnnotation] = vmRes.IP
}
if vmRes.MAC != "" {
ovnAnnotations[ovnMACAnnotation] = vmRes.MAC
}
vmPatch, err := marshalPatchData(map[string]any{
"spec": map[string]any{
"template": map[string]any{
"metadata": map[string]any{
"annotations": ovnAnnotations,
},
},
},
})
if err != nil {
return nil, err
}
rules = append(rules, resourceModifierRule{
Conditions: resourceModifierConditions{
GroupResource: "virtualmachines.kubevirt.io",
ResourceNameRegex: ".*",
Namespaces: []string{targetNS},
},
MergePatches: []mergePatch{{PatchData: vmPatch}},
})
}
}
rulesYAML, err := yaml.Marshal(resourceModifiers{
Version: "v1",
ResourceModifierRules: rules,
})
if err != nil {
return nil, fmt.Errorf("failed to marshal resource modifier rules: %w", err)
}
cmName := fmt.Sprintf("restore-modifiers-%s-%s", restoreJob.Namespace, restoreJob.Name)
// Truncate name to fit Kubernetes 253-char limit
if len(cmName) > 253 {
cmName = cmName[:253]
}
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: cmName,
Namespace: veleroNamespace,
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: restoreJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
},
},
Data: map[string]string{
"resource-modifier-rules.yaml": string(rulesYAML),
},
}
if err := r.Create(ctx, cm); err != nil {
if errors.IsAlreadyExists(err) {
// ConfigMap already exists (e.g. RestoreJob recreated with same name).
// Update its data to reflect the current backup's underlying resources.
existing := &corev1.ConfigMap{}
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: cmName}, existing); err != nil {
return nil, fmt.Errorf("failed to get existing resourceModifiers ConfigMap: %w", err)
}
existing.Data = cm.Data
if err := r.Update(ctx, existing); err != nil {
return nil, fmt.Errorf("failed to update existing resourceModifiers ConfigMap: %w", err)
}
logger.Debug("updated existing resourceModifiers ConfigMap", "name", cmName)
return existing, nil
}
return nil, fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err)
}
logger.Debug("created resourceModifiers ConfigMap", "name", cm.Name, "namespace", cm.Namespace)
return cm, nil
}
// resolveUnderlyingResourcesForRestore returns underlying resources for symmetric
// restore label selectors. Velero Backup annotation is used when Backup.status was empty
// (e.g. CRD without underlyingResources in schema).
func resolveUnderlyingResourcesForRestore(backup *backupsv1alpha1.Backup, vb *velerov1.Backup) *runtime.RawExtension {
if backup.Status.UnderlyingResources != nil && len(backup.Status.UnderlyingResources.Raw) > 0 {
return backup.Status.UnderlyingResources
}
if urJSON, ok := vb.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" {
return &runtime.RawExtension{Raw: []byte(urJSON)}
}
return backup.Status.UnderlyingResources
}
// GVRs used during pre-restore preparation.
var (
helmReleaseGVR = schema.GroupVersionResource{Group: "helm.toolkit.fluxcd.io", Version: "v2", Resource: "helmreleases"}
virtualMachineGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"}
vmiGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachineinstances"}
dataVolumeGVR = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"}
)
// veleroRenameSupported reports whether postRestoreRename can restore the
// given application kind under a new name. Velero DataUpload writes volumes
// under their original names, and only a VMInstance re-attaches them after its
// HelmRelease is renamed: its disks keep their names and CDI adopts the restored
// PVCs. Every other kind (a standalone VMDisk, a database operator's Cluster)
// renders fresh volumes for the new name and comes up empty.
func veleroRenameSupported(appKind string) bool {
return appKind == vmInstanceKind
}
// helmReleaseNameForApp returns the HelmRelease name for the given application
// kind and name. Returns empty string for unsupported kinds.
func helmReleaseNameForApp(appKind, appName string) string {
switch appKind {
case vmInstanceKind:
return vmNamePrefix + appName
case vmDiskAppKind:
return vmDiskNamePrefix + appName
default:
return ""
}
}
// targetHelmReleaseExists returns true when a HelmRelease with the given name
// already exists in namespace. Returns false for unsupported application kinds
// (those where helmReleaseNameForApp returns "").
func (r *RestoreJobReconciler) targetHelmReleaseExists(ctx context.Context, appKind, hrName, namespace string) (bool, error) {
if hrName == "" {
return false, nil
}
_, err := r.Resource(helmReleaseGVR).Namespace(namespace).Get(ctx, hrName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
return true, nil
}
// shortHash returns the first 4 hex characters of sha256(input).
func shortHash(input string) string {
h := sha256.Sum256([]byte(input))
return hex.EncodeToString(h[:])[:4]
}
// prepareForRestore performs graceful pre-restore cleanup:
// 1. Suspends HelmReleases that belong to the backup scope.
// 2. Halts the VirtualMachine (sets spec.runStrategy=Halted).
// 3. Waits for the VMI to disappear (graceful shutdown complete).
// 4. Deletes DataVolumes so CDI doesn't recreate PVCs after rename.
// 5. Renames existing PVCs to <name>-orig-<hash> so Velero can create fresh
// ones via Data Movement.
//
// Missing resources are expected (e.g. restore requested when app was already
// deleted), and a failure to suspend or halt is non-fatal. A failed rename
// holds the restore back and is retried, since the DataVolume deletion after
// it would hand a disk that was not kept to the restore, as
// keepOriginalPVC=false does, and an -orig name another PVC holds fails the
// restore. A DataVolume or a replaced PVC that cannot be deleted fails the
// restore, since Velero would skip the claim left in place. Each action emits
// a Kubernetes Event on the RestoreJob for observability.
//
// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
// Velero resource modifiers cannot change metadata.name, so this step creates
// a new HelmRelease with the target name and deletes the old one.
// Flux will reconcile the renamed HelmRelease and recreate downstream resources
// (VM, VMI) with the new name.
func (r *RestoreJobReconciler) postRestoreRename(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, target restoreTarget) error {
logger := getLogger(ctx)
sourceAppName := backup.Spec.ApplicationRef.Name
sourceHRName := vmNamePrefix + sourceAppName
targetHRName := vmNamePrefix + target.AppName
logger.Debug("post-restore rename", "from", sourceHRName, "to", targetHRName, "namespace", target.Namespace)
// Get the restored HelmRelease with the original name
hrClient := r.Resource(helmReleaseGVR).Namespace(target.Namespace)
oldHR, err := hrClient.Get(ctx, sourceHRName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
logger.Debug("source HelmRelease not found, skipping rename", "name", sourceHRName)
return nil
}
return fmt.Errorf("failed to get HelmRelease %s: %w", sourceHRName, err)
}
// Create new HelmRelease with the target name
newHR := oldHR.DeepCopy()
newHR.SetName(targetHRName)
newHR.SetResourceVersion("")
newHR.SetUID("")
newHR.SetCreationTimestamp(metav1.Time{})
newHR.SetManagedFields(nil)
newHR.SetGeneration(0)
// Update labels
labels := newHR.GetLabels()
if labels == nil {
labels = map[string]string{}
}
labels[appNameLabel] = target.AppName
labels["app.kubernetes.io/instance"] = targetHRName
labels["helm.toolkit.fluxcd.io/name"] = targetHRName
newHR.SetLabels(labels)
// Remove Velero restore annotations/labels that tie it to the old restore
annotations := newHR.GetAnnotations()
delete(annotations, "velero.io/restore-name")
newHR.SetAnnotations(annotations)
// Clear status so Flux reconciles fresh
unstructured.RemoveNestedField(newHR.Object, "status")
if _, err := hrClient.Create(ctx, newHR, metav1.CreateOptions{}); err != nil {
if errors.IsAlreadyExists(err) {
logger.Debug("target HelmRelease already exists, skipping create", "name", targetHRName)
} else {
return fmt.Errorf("failed to create renamed HelmRelease %s: %w", targetHRName, err)
}
} else {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename",
fmt.Sprintf("Created renamed HelmRelease %s (from %s)", targetHRName, sourceHRName))
}
// Delete the old HelmRelease
if err := hrClient.Delete(ctx, sourceHRName, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to delete old HelmRelease %s: %w", sourceHRName, err)
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename",
fmt.Sprintf("Deleted old HelmRelease %s", sourceHRName))
logger.Debug("post-restore rename complete", "from", sourceHRName, "to", targetHRName)
return nil
}
// Returns true when preparation is complete and the Velero Restore can be created.
// Returns false (with a requeue) when still waiting for VM shutdown.
func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (ready bool, result ctrl.Result, err error) {
// For restore-to-copy, skip all source-app preparation.
// The source application remains untouched; we're restoring a copy into another namespace.
if target.IsCopy {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
"Restore to copy: skipping source application preparation")
return true, ctrl.Result{}, nil
}
ns := restoreJob.Namespace
appName := backup.Spec.ApplicationRef.Name
appKind := backup.Spec.ApplicationRef.Kind
origSuffix := "-orig-" + shortHash(restoreJob.Name)
vmRes := restoreDisks(backup, ur)
// A VMInstance is halted below, which releases its disks; a VMDisk target is
// not. Renaming a PVC that a running VM still mounts leaves the original
// Terminating behind pvc-protection, and Velero skips restoring a PVC that
// still exists, so the restore would report success having moved nothing
// while the VM loses its disk on the next restart. Refuse before changing
// anything.
if appKind != vmInstanceKind && vmRes != nil {
inUse, leaving, err := r.podsMountingDisks(ctx, ns, vmRes.DataVolumes)
if err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to check whether the disk is in use: %v", err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
if len(inUse) > 0 {
return false, ctrl.Result{}, fmt.Errorf(
"the disk is in use by %s; stop the VM that uses it, or restore that VMInstance instead, then create a new RestoreJob",
strings.Join(inUse, ", "))
}
if len(leaving) > 0 {
// Not covered by the preparation deadline, which starts once
// preparation changes the target; a Pod stuck terminating on a lost
// node would otherwise keep the job Running forever.
if started := restoreJob.Status.StartedAt; started != nil && time.Since(started.Time) > restorePreparationDeadline {
return false, ctrl.Result{}, fmt.Errorf("%s still held the disk %s after the RestoreJob started",
strings.Join(leaving, ", "), restorePreparationDeadline)
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Waiting for %s to release the disk", strings.Join(leaving, ", ")))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
// Refuse a taken -orig name before anything is halted or deleted, rather
// than fail halfway through a multi-disk rename.
if opts.GetKeepOriginalPVC() && vmRes != nil {
for _, dv := range vmRes.DataVolumes {
origPVC, err := r.getOrigPVC(ctx, ns, dv.DataVolumeName+origSuffix)
if err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to check PVC %s: %v", dv.DataVolumeName+origSuffix, err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
if err := foreignOrigPVC(origPVC, restoreJob); err != nil {
return false, ctrl.Result{}, err
}
}
}
if !meta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondPreparationStarted) {
before := restoreJob.DeepCopy()
meta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
Type: restoreCondPreparationStarted,
Status: metav1.ConditionTrue,
Reason: "Preparing",
Message: "suspending the target's HelmReleases and keeping its disks aside",
})
// Nothing has been changed yet: on a failed patch drop the in-memory
// condition and retry, or the failure path would report leftovers that
// preparation never made.
if err := r.Status().Patch(ctx, restoreJob, client.MergeFrom(before)); err != nil {
restoreJob.Status.Conditions = before.Status.Conditions
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to record that preparation started: %v", err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
// --- Step 1: Suspend HelmReleases ---
hrNames := []string{}
if appKind == vmInstanceKind {
hrNames = append(hrNames, vmNamePrefix+appName)
}
if vmRes != nil {
for _, dv := range vmRes.DataVolumes {
hrNames = append(hrNames, dv.DataVolumeName)
}
}
// A HelmRelease left running would have Flux and CDI recreate the
// DataVolume and PVC under the data mover, so a failed suspend stops here.
for _, hrName := range hrNames {
if err := r.suspendHelmRelease(ctx, ns, hrName); err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to suspend HelmRelease %s: %v", hrName, err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Suspended HelmRelease %s", hrName))
}
// --- Step 2: Halt VM and wait for shutdown ---
if appKind == vmInstanceKind {
vmName := vmNamePrefix + appName
// haltVirtualMachine reports a missing VM as halted, so an error here is
// a VM that may still be running on the disks.
halted, err := r.haltVirtualMachine(ctx, ns, vmName)
if err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to halt VM %s: %v", vmName, err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
} else if !halted {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Waiting for VM %s to shut down", vmName))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
} else {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("VM %s is halted", vmName))
}
// The VMI is gone before its launcher Pod is; renaming a PVC that Pod still
// mounts would leave it Terminating behind pvc-protection.
if vmRes != nil {
inUse, leaving, err := r.podsMountingDisks(ctx, ns, vmRes.DataVolumes)
if err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to check whether the disks are in use: %v", err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
if holders := append(inUse, leaving...); len(holders) > 0 {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Waiting for %s to release the disks", strings.Join(holders, ", ")))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
}
// --- Step 3: Rename PVCs to <name>-orig-<hash> ---
// Must happen BEFORE deleting DVs: the PVC has an ownerReference to the DV,
// so deleting the DV first would cascade-delete the PVC via garbage collection.
// Only when keepOriginalPVC is true (default for in-place restore).
if opts.GetKeepOriginalPVC() && vmRes != nil {
for _, dv := range vmRes.DataVolumes {
if err := r.renamePVC(ctx, restoreJob, ns, dv.DataVolumeName, dv.DataVolumeName+origSuffix); err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to keep old PVC %s: %v", dv.DataVolumeName, err))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
}
// --- Step 4: Delete DataVolumes so CDI doesn't recreate PVCs ---
// A DataVolume still present makes CDI recreate its claim under the original
// name, which Velero then skips as an existing PVC: the restore would end
// Succeeded on a freshly imported disk, so nothing goes further until every
// DataVolume is gone.
if vmRes != nil {
var terminating []string
for _, dv := range vmRes.DataVolumes {
gone, err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName)
if err != nil {
return false, ctrl.Result{}, fmt.Errorf("failed to delete DataVolume %s: %w", dv.DataVolumeName, err)
}
if !gone {
terminating = append(terminating, dv.DataVolumeName)
continue
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName))
}
if len(terminating) > 0 {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Waiting for DataVolumes %s to be deleted", strings.Join(terminating, ", ")))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
// --- Step 5: Delete the PVCs a restore without keepOriginalPVC replaces ---
// Velero restores a PVC only by creating it: its CSI restore action skips the
// data mover for a PVC that already exists, and existingResourcePolicy=update
// cannot patch the immutable spec of a bound claim. A PVC left in place keeps
// its old data, and, orphaned from its DataVolume in step 4 and without the
// cdi.kubevirt.io/storage.populatedFor the backed-up copy carries, it makes
// CDI refuse the DataVolume the resumed HelmRelease creates.
if !opts.GetKeepOriginalPVC() && vmRes != nil {
var terminating []string
for _, dv := range vmRes.DataVolumes {
gone, err := r.deletePVC(ctx, ns, dv.DataVolumeName)
if err != nil {
return false, ctrl.Result{}, fmt.Errorf("failed to delete PVC %s: %w", dv.DataVolumeName, err)
}
if !gone {
terminating = append(terminating, dv.DataVolumeName)
}
}
if len(terminating) > 0 {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Waiting for PVCs %s to be deleted", strings.Join(terminating, ", ")))
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
// --- Step 6: Clear a PVC CDI recreated under the original name ---
// Between the rename and the DataVolume delete, CDI can recreate an empty PVC
// for the still-present DataVolume. Velero skips restoring a PVC that exists,
// so the restore would report success and leave the disk blank. The original
// is kept under its -orig name by now, so a PVC under the original name is
// such a copy and holds no data.
if opts.GetKeepOriginalPVC() && vmRes != nil {
for _, dv := range vmRes.DataVolumes {
clear, err := r.clearRecreatedPVC(ctx, restoreJob, ns, dv.DataVolumeName)
if err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
fmt.Sprintf("Failed to clear PVC %s: %v", dv.DataVolumeName, err))
}
if err != nil || !clear {
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
}
}
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", "Pre-restore preparation complete")
return true, ctrl.Result{}, nil
}
// clearRecreatedPVC reports whether the original name of a kept disk is free
// for Velero to restore into, and deletes a PVC found under the name; the
// caller requeues until it is gone. Step 4 has already waited for the
// DataVolume to be gone, so CDI cannot recreate the PVC again.
func (r *RestoreJobReconciler) clearRecreatedPVC(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, name string) (bool, error) {
// Uncached: the cache can still hold the original PVC the rename just deleted.
pvc := &corev1.PersistentVolumeClaim{}
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, pvc); err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
if !pvc.DeletionTimestamp.IsZero() {
return false, nil
}
if err := r.Delete(ctx, pvc); err != nil && !errors.IsNotFound(err) {
return false, err
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Deleted PVC %s that CDI recreated after the original was kept aside", name))
return false, nil
}
// markPreparedRestoreFailed fails a RestoreJob whose target was already
// prepared, and repeats the message as a Warning event, since recovering the
// target is left to the user.
func (r *RestoreJobReconciler) markPreparedRestoreFailed(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "RestoreFailed", message)
return r.markRestoreJobFailed(ctx, restoreJob, message)
}
// restoreDisks returns the disks an in-place restore prepares. A VMDisk Backup
// taken before its underlying resources were recorded has none, but its only
// disk is the VMDisk itself, named as collectUnderlyingResources names it.
func restoreDisks(backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) *vmInstanceResources {
if vmRes := getVMInstanceResources(ur); vmRes != nil {
return vmRes
}
if backup.Spec.ApplicationRef.Kind != vmDiskAppKind {
return nil
}
name := backup.Spec.ApplicationRef.Name
return &vmInstanceResources{DataVolumes: []backupsv1alpha1.DataVolumeResource{{
DataVolumeName: vmDiskNamePrefix + name,
ApplicationName: name,
}}}
}
// preparedRestoreLeftovers describes, for the failure message of an in-place
// restore that fails after prepareForRestore ran, what preparation left behind.
// Nothing is rolled back automatically: resuming a disk's HelmRelease while its
// PVC is renamed and its DataVolume deleted would have CDI import a fresh disk
// under the original name, which looks healthy and hides that the data sits in
// the -orig- PVC. Preparation can fail before it reaches every step, so only a
// HelmRelease that is suspended and an -orig- PVC that exists are named; one
// that cannot be read is named anyway rather than hidden.
func (r *RestoreJobReconciler) preparedRestoreLeftovers(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) string {
if target.IsCopy {
return ""
}
vmRes := restoreDisks(backup, ur)
var hrs, kept []string
if backup.Spec.ApplicationRef.Kind == vmInstanceKind {
hrs = append(hrs, vmNamePrefix+backup.Spec.ApplicationRef.Name)
}
if vmRes != nil {
for _, dv := range vmRes.DataVolumes {
hrs = append(hrs, dv.DataVolumeName)
if opts.GetKeepOriginalPVC() {
kept = append(kept, dv.DataVolumeName+"-orig-"+shortHash(restoreJob.Name))
}
}
}
ns := target.Namespace
hrs = slices.DeleteFunc(hrs, func(name string) bool {
hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return errors.IsNotFound(err)
}
suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend")
return !suspended
})
kept = slices.DeleteFunc(kept, func(name string) bool {
err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &corev1.PersistentVolumeClaim{})
return errors.IsNotFound(err)
})
if len(hrs) == 0 && len(kept) == 0 {
return ""
}
var parts []string
if len(hrs) > 0 {
parts = append(parts, fmt.Sprintf("suspended HelmRelease %s", strings.Join(hrs, ", ")))
}
if len(kept) > 0 {
parts = append(parts, fmt.Sprintf("kept the original data in PVC %s", strings.Join(kept, ", ")))
}
return "; preparation " + strings.Join(parts, " and ") + "; if Velero restored the HelmRelease it is resumed and the disk may have been recreated, so check what the disk holds before relying on it"
}
// podsMountingDisks returns "pod/<pod> (PVC <claim>)" for every Pod in ns that
// has not terminated and mounts one of the disks' PVCs, split into Pods that
// are running and Pods already being deleted. A terminated Pod no longer holds
// pvc-protection, so it does not count.
func (r *RestoreJobReconciler) podsMountingDisks(ctx context.Context, ns string, disks []backupsv1alpha1.DataVolumeResource) (inUse, leaving []string, err error) {
claims := make(map[string]bool, len(disks))
for _, dv := range disks {
claims[dv.DataVolumeName] = true
}
pods := &corev1.PodList{}
if err := r.List(ctx, pods, client.InNamespace(ns)); err != nil {
return nil, nil, err
}
for i := range pods.Items {
pod := &pods.Items[i]
if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
continue
}
for _, v := range pod.Spec.Volumes {
if v.PersistentVolumeClaim == nil || !claims[v.PersistentVolumeClaim.ClaimName] {
continue
}
holder := fmt.Sprintf("pod/%s (PVC %s)", pod.Name, v.PersistentVolumeClaim.ClaimName)
if pod.DeletionTimestamp.IsZero() {
inUse = append(inUse, holder)
} else {
leaving = append(leaving, holder)
}
}
}
sort.Strings(inUse)
sort.Strings(leaving)
return inUse, leaving, nil
}
// veleroBackupForRestore returns the named Velero Backup and why it cannot be
// restored, or "" when it can. Only a Completed Backup is restored: Velero
// itself validates only that the Backup exists and its storage location is
// usable, so a Restore from one in another phase would take the target apart
// for nothing. A Backup that is gone comes back empty rather than nil.
func (r *RestoreJobReconciler) veleroBackupForRestore(ctx context.Context, name string) (*velerov1.Backup, string, error) {
vb := &velerov1.Backup{}
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb); err != nil {
if errors.IsNotFound(err) {
return &velerov1.Backup{}, fmt.Sprintf("Velero Backup %s/%s not found; it may have expired", veleroNamespace, name), nil
}
return nil, "", fmt.Errorf("failed to get Velero Backup %s: %w", name, err)
}
if vb.Status.Phase != velerov1.BackupPhaseCompleted {
return vb, fmt.Sprintf("Velero Backup %s/%s is %q, not %q", veleroNamespace, name, vb.Status.Phase, velerov1.BackupPhaseCompleted), nil
}
return vb, "", nil
}
// restoredUnderlyingResources resolves the underlying resources of a restore
// whose Velero Restore already exists, for naming what preparation left
// behind. A Velero Backup that cannot be read leaves the Backup status.
func (r *RestoreJobReconciler) restoredUnderlyingResources(ctx context.Context, backup *backupsv1alpha1.Backup, name string) *runtime.RawExtension {
vb := &velerov1.Backup{}
_ = r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb)
return resolveUnderlyingResourcesForRestore(backup, vb)
}
// suspendHelmRelease sets spec.suspend=true on a HelmRelease.
func (r *RestoreJobReconciler) suspendHelmRelease(ctx context.Context, ns, name string) error {
hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return err
}
suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend")
if suspended {
return nil
}
if err := unstructured.SetNestedField(hr.Object, true, "spec", "suspend"); err != nil {
return err
}
if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Update(ctx, hr, metav1.UpdateOptions{}); err != nil {
return err
}
return nil
}
// haltVirtualMachine sets runStrategy=Halted and returns true when the VMI is gone.
func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmName string) (bool, error) {
vm, err := r.Resource(virtualMachineGVR).Namespace(ns).Get(ctx, vmName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
currentStrategy, _, _ := unstructured.NestedString(vm.Object, "spec", "runStrategy")
if currentStrategy != "Halted" {
if err := unstructured.SetNestedField(vm.Object, "Halted", "spec", "runStrategy"); err != nil {
return false, err
}
if _, err := r.Resource(virtualMachineGVR).Namespace(ns).Update(ctx, vm, metav1.UpdateOptions{}); err != nil {
return false, err
}
}
// VMI gone = shutdown complete
_, err = r.Resource(vmiGVR).Namespace(ns).Get(ctx, vmName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
}
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after
// rename, and reports whether it is gone. Orphan propagation holds the DV behind
// the orphan finalizer until garbage collection has released its PVC, and leaves
// that PVC to prepareForRestore, which waits for its deletion, rather than to
// background garbage collection, which nothing waits for before the Velero
// Restore starts.
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) (bool, error) {
dvClient := r.Resource(dataVolumeGVR).Namespace(ns)
dv, err := dvClient.Get(ctx, name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
if dv.GetDeletionTimestamp() == nil {
orphan := metav1.DeletePropagationOrphan
err := dvClient.Delete(ctx, name, metav1.DeleteOptions{PropagationPolicy: &orphan})
if err != nil && !errors.IsNotFound(err) {
return false, err
}
}
if _, err := dvClient.Get(ctx, name, metav1.GetOptions{}); err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
}
// getOrigPVC returns the PVC named name, or nil when there is none. It reads
// past the cache: a retried rename must see the -orig PVC it created, not
// take the name for free and fail to create it again.
func (r *RestoreJobReconciler) getOrigPVC(ctx context.Context, ns, name string) (*corev1.PersistentVolumeClaim, error) {
pvc := &corev1.PersistentVolumeClaim{}
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, pvc); err != nil {
return nil, client.IgnoreNotFound(err)
}
return pvc, nil
}
// foreignOrigPVC fails when pvc exists and restoreJob did not create it.
// Proceeding would hand the live disk to the restore under its own name, as
// keepOriginalPVC=false does, while the -orig name holds some other disk.
func foreignOrigPVC(pvc *corev1.PersistentVolumeClaim, restoreJob *backupsv1alpha1.RestoreJob) error {
if pvc == nil || pvc.Annotations[origPVCRestoreJobUIDAnnotation] == string(restoreJob.UID) {
return nil
}
return fmt.Errorf("PVC %s/%s already exists and was not created by this RestoreJob, so the current disk cannot be kept under that name: "+
"remove that PVC or create the RestoreJob under another name", pvc.Namespace, pvc.Name)
}
// deletePVC deletes a PVC and reports whether it is gone. A claim still mounted
// stays Terminating behind kubernetes.io/pvc-protection, and Velero would find
// it still there.
func (r *RestoreJobReconciler) deletePVC(ctx context.Context, ns, name string) (bool, error) {
key := client.ObjectKey{Namespace: ns, Name: name}
pvc := &corev1.PersistentVolumeClaim{}
if err := r.Get(ctx, key, pvc); err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
if pvc.DeletionTimestamp == nil {
if err := r.Delete(ctx, pvc); err != nil && !errors.IsNotFound(err) {
return false, err
}
}
if err := r.Get(ctx, key, pvc); err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
}
// renamePVC preserves an existing PVC by rebinding it under a new name.
// The original PVC is deleted and a new one pointing to the same PV is created.
// Missing resources are silently skipped (non-fatal). Each step can be
// resumed, so a rename interrupted by a controller restart completes on the
// next reconcile.
func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, oldName, newName string) error {
logger := getLogger(ctx)
newPVC, err := r.getOrigPVC(ctx, ns, newName)
if err != nil {
return err
}
if err := foreignOrigPVC(newPVC, restoreJob); err != nil {
return err
}
resuming := newPVC != nil
oldPVC := &corev1.PersistentVolumeClaim{}
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: oldName}, oldPVC); err != nil {
if !errors.IsNotFound(err) {
return err
}
oldPVC = nil
}
pv := &corev1.PersistentVolume{}
var pvName string
switch {
case resuming:
pvName = newPVC.Spec.VolumeName
case oldPVC == nil:
return nil // nothing to rename
case oldPVC.Spec.VolumeName == "":
logger.Debug("PVC not bound, deleting", "name", oldName)
return r.Delete(ctx, oldPVC)
default:
pvName = oldPVC.Spec.VolumeName
// Patch PV reclaim policy to Retain so it survives PVC deletion.
// A policy already recorded is the one from before any rename.
if err := r.apiReader().Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
return fmt.Errorf("failed to get PV %s: %w", pvName, err)
}
originalPolicy, recorded := pv.Annotations[originalReclaimPolicyAnnotation]
if !recorded {
originalPolicy = string(pv.Spec.PersistentVolumeReclaimPolicy)
metav1.SetMetaDataAnnotation(&pv.ObjectMeta, originalReclaimPolicyAnnotation, originalPolicy)
}
if !recorded || pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain {
pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain
if err := r.Update(ctx, pv); err != nil {
return fmt.Errorf("failed to set Retain policy on PV %s: %w", pvName, err)
}
}
// Create the new -orig PVC first (unbound, just the object)
newPVC = &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: newName,
Namespace: ns,
Annotations: map[string]string{
originalReclaimPolicyAnnotation: originalPolicy,
origPVCRestoreJobUIDAnnotation: string(restoreJob.UID),
},
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: oldPVC.Spec.AccessModes,
Resources: oldPVC.Spec.Resources,
StorageClassName: oldPVC.Spec.StorageClassName,
VolumeMode: oldPVC.Spec.VolumeMode,
VolumeName: pvName,
},
}
// Create fills in the UID the claimRef needs. Reading the PVC back
// through the cache can miss it, as it is this reconcile's own write.
if err := r.Create(ctx, newPVC); err != nil {
return fmt.Errorf("failed to create -orig PVC %s: %w", newName, err)
}
}
// Delete the original PVC. Only while it still claims the kept PV: after
// an interrupted rename, CDI may have recreated an empty one under the old
// name, and that one is left to the restore.
if oldPVC != nil && oldPVC.Spec.VolumeName == pvName {
if err := r.Delete(ctx, oldPVC); err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to delete PVC %s: %w", oldName, err)
}
}
// Point the PV's claimRef directly to the new -orig PVC.
// This is atomic — no window where the PV is Available for other PVCs to grab.
// The PV controller writes the PV too once the original PVC is gone, so
// the read is uncached and a conflict is retried.
rebound := false
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
if err := r.apiReader().Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
return err
}
if ref := pv.Spec.ClaimRef; ref != nil && ref.Namespace == ns && ref.Name == newName && ref.UID == newPVC.UID {
return nil
}
pv.Spec.ClaimRef = &corev1.ObjectReference{
APIVersion: "v1",
Kind: "PersistentVolumeClaim",
Namespace: ns,
Name: newName,
UID: newPVC.UID,
}
rebound = true
return r.Update(ctx, pv)
})
if err != nil {
return fmt.Errorf("failed to rebind PV %s to %s: %w", pvName, newName, err)
}
if !rebound {
return nil
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
fmt.Sprintf("Keep old PVC %s as %s (PV: %s)", oldName, newName, pvName))
return nil
}
// createVeleroRestore creates a Velero Restore resource.
func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) error {
logger := getLogger(ctx)
logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName, "targetNS", target.Namespace, "isCopy", target.IsCopy)
// For restore template context, always use the source (backup) namespace and app name.
// The strategy template uses includedNamespaces and orLabelSelectors to select
// resources from the backup tarball, which are stored under the source namespace
// and labeled with the source app name.
// Velero's namespaceMapping handles redirecting to the target namespace;
// resource modifiers handle renaming when the target app name differs.
templateContext := map[string]any{
"Application": map[string]any{
"metadata": map[string]any{
"name": backup.Spec.ApplicationRef.Name,
"namespace": backup.Namespace,
},
"kind": backup.Spec.ApplicationRef.Kind,
},
// TODO: Parameters are not currently stored on Backup, so they're unavailable during restore.
// This is a design limitation that should be addressed by persisting Parameters on the Backup object.
"Parameters": map[string]string{},
}
// Template the restore spec from the strategy, or use defaults if not specified
var veleroRestoreSpec velerov1.RestoreSpec
if strategy.Spec.Template.RestoreSpec != nil {
templatedSpec, err := template.Template(strategy.Spec.Template.RestoreSpec, templateContext)
if err != nil {
return fmt.Errorf("failed to template Velero Restore spec: %w", err)
}
veleroRestoreSpec = *templatedSpec
}
// Set the backupName in the spec (required by Velero)
veleroRestoreSpec.BackupName = veleroBackupName
// For restore-to-copy, set Velero namespaceMapping to redirect resources
// from the source namespace to the target namespace.
if target.IsCopy {
if veleroRestoreSpec.NamespaceMapping == nil {
veleroRestoreSpec.NamespaceMapping = make(map[string]string)
}
veleroRestoreSpec.NamespaceMapping[backup.Namespace] = target.Namespace
logger.Debug("set namespaceMapping on Velero Restore", "from", backup.Namespace, "to", target.Namespace)
}
// Match backup: add OR selectors for each underlying VMDisk so restore applies the same
// scope as the intended backup (see createVeleroBackup).
if vmRes := getVMInstanceResources(ur); vmRes != nil && backup.Spec.ApplicationRef.Kind == vmInstanceKind {
for _, dv := range vmRes.DataVolumes {
veleroRestoreSpec.OrLabelSelectors = append(veleroRestoreSpec.OrLabelSelectors, &metav1.LabelSelector{
MatchLabels: map[string]string{
appKindLabel: vmDiskAppKind,
appNameLabel: dv.ApplicationName,
},
})
}
if len(vmRes.DataVolumes) > 0 {
logger.Debug("added VMDisk label selectors to Velero restore", "count", len(vmRes.DataVolumes))
}
}
// Create resourceModifiers ConfigMap
resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts)
if err != nil {
return fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err)
}
if resourceModifierCM != nil {
veleroRestoreSpec.ResourceModifier = &corev1.TypedLocalObjectReference{
APIGroup: new(""),
Kind: "ConfigMap",
Name: resourceModifierCM.Name,
}
logger.Debug("set resourceModifier on Velero Restore", "configMap", resourceModifierCM.Name)
}
generateName := fmt.Sprintf("%s.%s-", restoreJob.Namespace, restoreJob.Name)
veleroRestore := &velerov1.Restore{
ObjectMeta: metav1.ObjectMeta{
GenerateName: generateName,
Namespace: veleroNamespace,
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: restoreJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
},
},
Spec: veleroRestoreSpec,
}
if err := r.Create(ctx, veleroRestore); err != nil {
logger.Error(err, "failed to create Velero Restore", "generateName", generateName)
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "VeleroRestoreCreationFailed",
fmt.Sprintf("Failed to create Velero Restore %s/%s: %v", veleroNamespace, generateName, err))
return err
}
logger.Debug("created Velero Restore", "name", veleroRestore.Name, "namespace", veleroRestore.Namespace)
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "VeleroRestoreCreated",
fmt.Sprintf("Created Velero Restore %s/%s", veleroNamespace, veleroRestore.Name))
return nil
}
// dataUploadListGVK is the API version shipped with Velero data mover CRDs (see velero datauploads CRD).
var dataUploadListGVK = schema.GroupVersionKind{Group: "velero.io", Version: "v2alpha1", Kind: "DataUploadList"}
// formatVeleroBackupFailureMessageForBackupJob builds a BackupJob status message from Velero Backup
// status plus failed DataUpload resources (CSI data mover), similar to `velero backup describe`.
func formatVeleroBackupFailureMessageForBackupJob(ctx context.Context, c client.Client, veleroBackup *velerov1.Backup) string {
var b strings.Builder
fmt.Fprintf(&b, "Velero Backup failed with phase %s", veleroBackup.Status.Phase)
if fr := strings.TrimSpace(veleroBackup.Status.FailureReason); fr != "" {
fmt.Fprintf(&b, ": %s", fr)
}
if len(veleroBackup.Status.ValidationErrors) > 0 {
fmt.Fprintf(&b, "; validation: %v", veleroBackup.Status.ValidationErrors)
}
if h := veleroBackup.Status.HookStatus; h != nil && h.HooksFailed > 0 {
fmt.Fprintf(&b, "; hooks failed %d/%d", h.HooksFailed, h.HooksAttempted)
}
if veleroBackup.Status.BackupItemOperationsFailed > 0 {
fmt.Fprintf(&b, "; async item operations failed %d (completed %d, attempted %d)",
veleroBackup.Status.BackupItemOperationsFailed,
veleroBackup.Status.BackupItemOperationsCompleted,
veleroBackup.Status.BackupItemOperationsAttempted)
}
b.WriteString(appendFailedDataUploadMessages(ctx, c, veleroBackup.Name))
return b.String()
}
func appendFailedDataUploadMessages(ctx context.Context, c client.Client, veleroBackupName string) string {
ul := unstructured.UnstructuredList{}
ul.SetGroupVersionKind(dataUploadListGVK)
if err := c.List(ctx, &ul, client.InNamespace(veleroNamespace)); err != nil {
return ""
}
prefix := veleroBackupName + "-"
var b strings.Builder
for _, item := range ul.Items {
if !strings.HasPrefix(item.GetName(), prefix) {
continue
}
phase, _, _ := unstructured.NestedString(item.Object, "status", "phase")
if phase != "Failed" {
continue
}
msg, _, _ := unstructured.NestedString(item.Object, "status", "message")
if strings.TrimSpace(msg) == "" {
msg = "(empty status.message)"
}
srcPVC, _, _ := unstructured.NestedString(item.Object, "spec", "sourcePVC")
if srcPVC != "" {
fmt.Fprintf(&b, "; DataUpload %s failed for PVC %s: %s", item.GetName(), srcPVC, msg)
} else {
fmt.Fprintf(&b, "; DataUpload %s failed: %s", item.GetName(), msg)
}
}
return b.String()
}