mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-23 03:27:50 +09:00
fix(backups): prepare standalone VMDisk restore targets before the data mover runs
collectUnderlyingResources returned nothing for a non-VMInstance kind, so prepareForRestore did nothing for a standalone VMDisk: no HelmRelease suspend and no DataVolume delete. An in-place VMDisk restore then raced Flux/CDI — the restored HelmRelease reinstalled and CDI recreated the DataVolume over the same PVC name the data mover was repopulating, wedging the RestoreJob. Report a standalone VMDisk's own disk as its underlying resource, so the existing (VMInstance-proven) restore preparation suspends the vm-disk-<name> HelmRelease, deletes its DataVolume and — with keepOriginalPVC — renames the live PVC before the data mover recreates and repopulates it. No VM, so no IP/MAC is collected. A VMInstance is halted before its disks are touched; a VMDisk target is not, and every VMInstance disk is a VMDisk. Renaming the PVC of a disk a running VM mounts leaves the original Terminating behind pvc-protection, Velero skips a PVC that still exists, and the RestoreJob would report Succeeded having moved nothing while the VM loses its disk on the next restart. An in-place VMDisk restore now fails before changing anything when a Pod that has not terminated mounts the disk's PVC, naming the Pod and telling the user to stop the VM or restore the VMInstance instead. A VMDisk Backup taken before the disk was recorded as its underlying resource has none, so its disk is now derived from the application name and such a restore is prepared and checked for a mounting Pod as well. The per-disk orLabelSelectors stay a VMInstance concern. A VMDisk strategy already selects its disk, and Velero rejects a Backup or Restore that carries both labelSelector and orLabelSelectors, so appending them for a VMDisk would break a strategy written with labelSelector. When an in-place restore fails after preparation, nothing is rolled back: resuming the disk's HelmRelease while its PVC is renamed and its DataVolume deleted would have CDI import a fresh disk under the original name, which looks healthy and hides that the data sits in the -orig- PVC. The failure message and a Warning event now name the suspended HelmReleases and the -orig- PVCs so the user can recover them, read back from the cluster, so a failure before a step ran does not name what it would have left. Preparation takes the target apart before Velero validates the Restore, so it is refused up front when the Velero Backup is gone or not Completed. That check now runs on every pass behind the same failure path as the rest of preparation, and a FailedValidation Restore fails the RestoreJob like a Failed one instead of leaving it Running. A HelmRelease suspend or a VM halt that did not happen now stops preparation instead of renaming and deleting under a running release or VM, and a VMInstance's disks are renamed only once its launcher Pod has let go of them. A VMDisk held only by a Pod already being deleted is waited for rather than refused, for up to 30 minutes after the RestoreJob started. Between renaming a kept disk's PVC and deleting its DataVolume, CDI can recreate an empty PVC under the original name. Velero then skips the PVC as already present, the RestoreJob reports Succeeded and the VM boots a blank disk. Once the DataVolume is gone, preparation now deletes a PVC found under the original name, read uncached, before the Velero Restore is created. Preparation spans several reconciles, so the RestoreJob records a PreparationStarted condition before it first changes the target, and retries without changing anything when recording it fails. A refusal on a later pass (the Velero Backup expired, a VM started on the disk) then names what preparation left behind instead of claiming that nothing changed, and a preparation still waiting 30 minutes after it started fails the same way instead of polling forever. Now that a VMDisk target is prepared, keepOriginalPVC=false reaches it too: its DataVolume is deleted with orphan propagation and the bound PVC is deleted before the Restore, as for a VMInstance disk. Left in place, Velero would skip it and CDI would refuse the DataVolume the resumed HelmRelease creates. Assisted-by: LLM Signed-off-by: Andrey Kolkov <androndo@gmail.com>
This commit is contained in:
1 parent
c0449eeabb
commit
9af1c0f902
4 files changed
+1427
-36
No files matched your search
@@ -6,6 +6,8 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -51,6 +53,14 @@ const (
|
||||
defaultActiveJobPollingInterval = defaultRequeueAfter
|
||||
defaultRestoreRequeueAfter = 5 * time.Second
|
||||
defaultActiveRestorePollingInterval = defaultRestoreRequeueAfter
|
||||
|
||||
// Condition recorded on a RestoreJob before prepareForRestore first changes
|
||||
// the target. From then on a failure has to say what preparation left
|
||||
// behind, and its LastTransitionTime starts the preparation deadline.
|
||||
restoreCondPreparationStarted = "PreparationStarted"
|
||||
// How long preparation may wait (VM shutdown, Pods releasing the disks, the
|
||||
// DataVolume and a recreated PVC going away) before the RestoreJob fails.
|
||||
restorePreparationDeadline = 30 * time.Minute
|
||||
// Velero requires API objects and secrets to be in the cozy-velero namespace
|
||||
veleroNamespace = "cozy-velero"
|
||||
veleroBackupNameMetadataKey = "velero.io/backup-name"
|
||||
@@ -105,13 +115,13 @@ type CommonRestoreOptions struct {
|
||||
|
||||
// RestoreOptions is the typed representation of RestoreJob.Spec.Options for the
|
||||
// Velero driver. The struct is deserialized from runtime.RawExtension and used
|
||||
// for all application kinds. VMInstance-specific fields (KeepOriginalPVC,
|
||||
// KeepOriginalIpAndMac) are only effective when the application kind is VMInstance.
|
||||
// for all application kinds. KeepOriginalPVC applies to VMInstance and VMDisk
|
||||
// targets; KeepOriginalIpAndMac only to VMInstance.
|
||||
type RestoreOptions struct {
|
||||
CommonRestoreOptions `json:",inline"`
|
||||
// KeepOriginalPVC renames the original PVC to <name>-orig-<hash> before restore;
|
||||
// when false, the original PVC is deleted so that Velero recreates it.
|
||||
// Only effective for in-place VMInstance restore (no targetNamespace). Defaults to true when omitted.
|
||||
// Only effective for an in-place VMInstance or VMDisk restore (no targetNamespace). Defaults to true when omitted.
|
||||
KeepOriginalPVC *bool `json:"keepOriginalPVC,omitempty"`
|
||||
// KeepOriginalIpAndMac preserves the original IP and MAC address via OVN
|
||||
// annotations. Only effective for VMInstance restores. Defaults to true when omitted.
|
||||
@@ -363,13 +373,29 @@ func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1a
|
||||
// Returns nil if the application type has no underlying resources to collect.
|
||||
func (r *BackupJobReconciler) collectUnderlyingResources(ctx context.Context, app *unstructured.Unstructured, appKind, ns string) (*runtime.RawExtension, error) {
|
||||
logger := getLogger(ctx)
|
||||
appName := app.GetName()
|
||||
|
||||
if appKind != vmInstanceKind {
|
||||
logger.Debug("application is not a VMInstance, skipping underlying resource collection", "kind", appKind)
|
||||
return nil, nil
|
||||
// A standalone VMDisk is its own underlying disk. Capturing it lets the
|
||||
// restore controller prepare it as it prepares a VMInstance's disks: keep
|
||||
// the live PVC aside and delete the DataVolume, so the restored PVC is
|
||||
// created fresh under the original name. The HelmRelease is suspended only
|
||||
// until Velero restores it; Flux then recreates the DataVolume and CDI
|
||||
// adopts the restored PVC through the allowClaimAdoption annotation the
|
||||
// restore's resource modifiers set. A standalone VMDisk has no VM, so there
|
||||
// is no IP/MAC to collect.
|
||||
if appKind == vmDiskAppKind {
|
||||
return marshalUnderlyingResources(vmInstanceResources{
|
||||
DataVolumes: []backupsv1alpha1.DataVolumeResource{{
|
||||
DataVolumeName: vmDiskNamePrefix + appName,
|
||||
ApplicationName: appName,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
appName := app.GetName()
|
||||
if appKind != vmInstanceKind {
|
||||
logger.Debug("application has no underlying resources to collect", "kind", appKind)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Extract disk names from VMInstance spec.disks[].name
|
||||
disks, found, err := unstructured.NestedSlice(app.Object, "spec", "disks")
|
||||
@@ -462,8 +488,11 @@ func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob
|
||||
return err
|
||||
}
|
||||
|
||||
// Add label selectors for underlying VMDisk HelmReleases
|
||||
if vmRes := getVMInstanceResources(underlyingResources); vmRes != nil {
|
||||
// Add label selectors for a VMInstance's VMDisk HelmReleases. A VMDisk's own
|
||||
// strategy already selects the disk, and Velero rejects a spec that carries
|
||||
// both labelSelector and orLabelSelectors, so a VMDisk strategy written with
|
||||
// labelSelector must be left as it is.
|
||||
if vmRes := getVMInstanceResources(underlyingResources); vmRes != nil && backupJob.Spec.ApplicationRef.Kind == vmInstanceKind {
|
||||
for _, dv := range vmRes.DataVolumes {
|
||||
veleroBackupSpec.OrLabelSelectors = append(veleroBackupSpec.OrLabelSelectors, &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
@@ -699,32 +728,41 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
|
||||
}
|
||||
|
||||
// The Backup artifact outlives the Velero Backup it points to, which
|
||||
// Velero garbage-collects once its TTL expires. Checked before
|
||||
// prepareForRestore halts the application and deletes its disks, so a
|
||||
// restore that cannot run leaves them in place.
|
||||
veleroBackup := &velerov1.Backup{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, veleroBackup); err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
|
||||
"Velero Backup %s/%s not found; it may have expired", veleroNamespace, veleroBackupName))
|
||||
}
|
||||
return ctrl.Result{}, fmt.Errorf("failed to get Velero Backup %s: %w", veleroBackupName, err)
|
||||
}
|
||||
if veleroBackup.Status.Phase != velerov1.BackupPhaseCompleted {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
|
||||
"Velero Backup %s/%s is %q, not %q", veleroNamespace, veleroBackupName,
|
||||
veleroBackup.Status.Phase, velerov1.BackupPhaseCompleted))
|
||||
// Velero garbage-collects once its TTL expires. Checked on every pass
|
||||
// before prepareForRestore halts the application and deletes its disks,
|
||||
// so a restore that cannot run leaves them in place, or, when preparation
|
||||
// already started, names what it left behind.
|
||||
veleroBackup, unrestorable, err := r.veleroBackupForRestore(ctx, veleroBackupName)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
// Resolve underlying resources once; prefer Backup status, fall back to Velero annotation.
|
||||
ur := resolveUnderlyingResourcesForRestore(backup, veleroBackup)
|
||||
|
||||
// Preparation spans several passes. Once it has changed the target, a
|
||||
// failure must name what it left behind instead of claiming nothing changed.
|
||||
failBeforeRestore := func(msg string) (ctrl.Result, error) {
|
||||
if meta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondPreparationStarted) {
|
||||
return r.markPreparedRestoreFailed(ctx, restoreJob, msg+r.preparedRestoreLeftovers(ctx, restoreJob, backup, ur, target, restoreOpts))
|
||||
}
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, msg+"; nothing was changed")
|
||||
}
|
||||
|
||||
if unrestorable != "" {
|
||||
return failBeforeRestore(unrestorable)
|
||||
}
|
||||
|
||||
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy)
|
||||
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts)
|
||||
if err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("pre-restore preparation failed: %v", err))
|
||||
return failBeforeRestore(fmt.Sprintf("pre-restore preparation failed: %v", err))
|
||||
}
|
||||
if !ready {
|
||||
if cond := meta.FindStatusCondition(restoreJob.Status.Conditions, restoreCondPreparationStarted); cond != nil &&
|
||||
cond.Status == metav1.ConditionTrue && time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline {
|
||||
return failBeforeRestore(fmt.Sprintf("pre-restore preparation did not finish within %s; see the RestoreJob events for what it was waiting on", restorePreparationDeadline))
|
||||
}
|
||||
logger.Debug("pre-restore preparation in progress, requeuing")
|
||||
return result, nil
|
||||
}
|
||||
@@ -733,7 +771,8 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
|
||||
logger.Debug("Velero Restore not found, creating new one")
|
||||
if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur, target, restoreOpts); err != nil {
|
||||
logger.Error(err, "failed to create Velero Restore")
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to create Velero Restore: %v", err))
|
||||
return r.markPreparedRestoreFailed(ctx, restoreJob, fmt.Sprintf("failed to create Velero Restore: %v", err)+
|
||||
r.preparedRestoreLeftovers(ctx, restoreJob, backup, ur, target, restoreOpts))
|
||||
}
|
||||
logger.Debug("created Velero Restore, requeuing")
|
||||
// Requeue to check status
|
||||
@@ -742,7 +781,8 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
|
||||
|
||||
if len(veleroRestoreList.Items) > 1 {
|
||||
logger.Error(fmt.Errorf("too many Velero restores for RestoreJob"), "found more than one Velero Restore referencing a single RestoreJob as owner")
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, "found multiple Velero Restores for this RestoreJob")
|
||||
return r.markPreparedRestoreFailed(ctx, restoreJob, "found multiple Velero Restores for this RestoreJob"+
|
||||
r.preparedRestoreLeftovers(ctx, restoreJob, backup, r.restoredUnderlyingResources(ctx, backup, veleroBackupName), target, restoreOpts))
|
||||
}
|
||||
|
||||
veleroRestore := veleroRestoreList.Items[0].DeepCopy()
|
||||
@@ -780,14 +820,21 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// Step 5: On failure
|
||||
if phase == "Failed" || phase == "PartiallyFailed" {
|
||||
// Step 5: On failure. FailedValidation is terminal too; left to the polling
|
||||
// branch below it would keep a target that preparation already took apart
|
||||
// Running forever.
|
||||
if phase == string(velerov1.RestorePhaseFailed) || phase == string(velerov1.RestorePhasePartiallyFailed) ||
|
||||
phase == string(velerov1.RestorePhaseFailedValidation) {
|
||||
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
|
||||
message := fmt.Sprintf("Velero Restore failed with phase: %s", phase)
|
||||
if veleroRestore.Status.FailureReason != "" {
|
||||
message = fmt.Sprintf("%s: %s", message, veleroRestore.Status.FailureReason)
|
||||
}
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, message)
|
||||
if len(veleroRestore.Status.ValidationErrors) > 0 {
|
||||
message = fmt.Sprintf("%s: %s", message, strings.Join(veleroRestore.Status.ValidationErrors, "; "))
|
||||
}
|
||||
return r.markPreparedRestoreFailed(ctx, restoreJob, message+
|
||||
r.preparedRestoreLeftovers(ctx, restoreJob, backup, r.restoredUnderlyingResources(ctx, backup, veleroBackupName), target, restoreOpts))
|
||||
}
|
||||
|
||||
// Still in progress (InProgress, New, etc.)
|
||||
@@ -1148,8 +1195,39 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
|
||||
appName := backup.Spec.ApplicationRef.Name
|
||||
appKind := backup.Spec.ApplicationRef.Kind
|
||||
origSuffix := "-orig-" + shortHash(restoreJob.Name)
|
||||
vmRes := restoreDisks(backup, ur)
|
||||
|
||||
vmRes := getVMInstanceResources(ur)
|
||||
// A VMInstance is halted below, which releases its disks; a VMDisk target is
|
||||
// not. Renaming a PVC that a running VM still mounts leaves the original
|
||||
// Terminating behind pvc-protection, and Velero skips restoring a PVC that
|
||||
// still exists, so the restore would report success having moved nothing
|
||||
// while the VM loses its disk on the next restart. Refuse before changing
|
||||
// anything.
|
||||
if appKind != vmInstanceKind && vmRes != nil {
|
||||
inUse, leaving, err := r.podsMountingDisks(ctx, ns, vmRes.DataVolumes)
|
||||
if err != nil {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to check whether the disk is in use: %v", err))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
if len(inUse) > 0 {
|
||||
return false, ctrl.Result{}, fmt.Errorf(
|
||||
"the disk is in use by %s; stop the VM that uses it, or restore that VMInstance instead, then create a new RestoreJob",
|
||||
strings.Join(inUse, ", "))
|
||||
}
|
||||
if len(leaving) > 0 {
|
||||
// Not covered by the preparation deadline, which starts once
|
||||
// preparation changes the target; a Pod stuck terminating on a lost
|
||||
// node would otherwise keep the job Running forever.
|
||||
if started := restoreJob.Status.StartedAt; started != nil && time.Since(started.Time) > restorePreparationDeadline {
|
||||
return false, ctrl.Result{}, fmt.Errorf("%s still held the disk %s after the RestoreJob started",
|
||||
strings.Join(leaving, ", "), restorePreparationDeadline)
|
||||
}
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Waiting for %s to release the disk", strings.Join(leaving, ", ")))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Refuse a taken -orig name before anything is halted or deleted, rather
|
||||
// than fail halfway through a multi-disk rename.
|
||||
@@ -1167,6 +1245,25 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
|
||||
}
|
||||
}
|
||||
|
||||
if !meta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondPreparationStarted) {
|
||||
before := restoreJob.DeepCopy()
|
||||
meta.SetStatusCondition(&restoreJob.Status.Conditions, metav1.Condition{
|
||||
Type: restoreCondPreparationStarted,
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: "Preparing",
|
||||
Message: "suspending the target's HelmReleases and keeping its disks aside",
|
||||
})
|
||||
// Nothing has been changed yet: on a failed patch drop the in-memory
|
||||
// condition and retry, or the failure path would report leftovers that
|
||||
// preparation never made.
|
||||
if err := r.Status().Patch(ctx, restoreJob, client.MergeFrom(before)); err != nil {
|
||||
restoreJob.Status.Conditions = before.Status.Conditions
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to record that preparation started: %v", err))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 1: Suspend HelmReleases ---
|
||||
hrNames := []string{}
|
||||
if appKind == vmInstanceKind {
|
||||
@@ -1177,24 +1274,28 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
|
||||
hrNames = append(hrNames, dv.DataVolumeName)
|
||||
}
|
||||
}
|
||||
// A HelmRelease left running would have Flux and CDI recreate the
|
||||
// DataVolume and PVC under the data mover, so a failed suspend stops here.
|
||||
for _, hrName := range hrNames {
|
||||
if err := r.suspendHelmRelease(ctx, ns, hrName); err != nil {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to suspend HelmRelease %s: %v", hrName, err))
|
||||
} else {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Suspended HelmRelease %s", hrName))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Suspended HelmRelease %s", hrName))
|
||||
}
|
||||
|
||||
// --- Step 2: Halt VM and wait for shutdown ---
|
||||
if appKind == vmInstanceKind {
|
||||
vmName := vmNamePrefix + appName
|
||||
// haltVirtualMachine reports a missing VM as halted, so an error here is
|
||||
// a VM that may still be running on the disks.
|
||||
halted, err := r.haltVirtualMachine(ctx, ns, vmName)
|
||||
if err != nil {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to halt VM %s: %v", vmName, err))
|
||||
// Non-fatal: proceed even if halting fails (VM might not exist)
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
} else if !halted {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Waiting for VM %s to shut down", vmName))
|
||||
@@ -1203,6 +1304,21 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("VM %s is halted", vmName))
|
||||
}
|
||||
// The VMI is gone before its launcher Pod is; renaming a PVC that Pod still
|
||||
// mounts would leave it Terminating behind pvc-protection.
|
||||
if vmRes != nil {
|
||||
inUse, leaving, err := r.podsMountingDisks(ctx, ns, vmRes.DataVolumes)
|
||||
if err != nil {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to check whether the disks are in use: %v", err))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
if holders := append(inUse, leaving...); len(holders) > 0 {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Waiting for %s to release the disks", strings.Join(holders, ", ")))
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 3: Rename PVCs to <name>-orig-<hash> ---
|
||||
@@ -1270,10 +1386,192 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
|
||||
}
|
||||
}
|
||||
|
||||
// --- Step 6: Clear a PVC CDI recreated under the original name ---
|
||||
// Between the rename and the DataVolume delete, CDI can recreate an empty PVC
|
||||
// for the still-present DataVolume. Velero skips restoring a PVC that exists,
|
||||
// so the restore would report success and leave the disk blank. The original
|
||||
// is kept under its -orig name by now, so a PVC under the original name is
|
||||
// such a copy and holds no data.
|
||||
if opts.GetKeepOriginalPVC() && vmRes != nil {
|
||||
for _, dv := range vmRes.DataVolumes {
|
||||
clear, err := r.clearRecreatedPVC(ctx, restoreJob, ns, dv.DataVolumeName)
|
||||
if err != nil {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
|
||||
fmt.Sprintf("Failed to clear PVC %s: %v", dv.DataVolumeName, err))
|
||||
}
|
||||
if err != nil || !clear {
|
||||
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", "Pre-restore preparation complete")
|
||||
return true, ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// clearRecreatedPVC reports whether the original name of a kept disk is free
|
||||
// for Velero to restore into, and deletes a PVC found under the name; the
|
||||
// caller requeues until it is gone. Step 4 has already waited for the
|
||||
// DataVolume to be gone, so CDI cannot recreate the PVC again.
|
||||
func (r *RestoreJobReconciler) clearRecreatedPVC(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, name string) (bool, error) {
|
||||
// Uncached: the cache can still hold the original PVC the rename just deleted.
|
||||
pvc := &corev1.PersistentVolumeClaim{}
|
||||
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, pvc); err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return true, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if !pvc.DeletionTimestamp.IsZero() {
|
||||
return false, nil
|
||||
}
|
||||
if err := r.Delete(ctx, pvc); err != nil && !errors.IsNotFound(err) {
|
||||
return false, err
|
||||
}
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
|
||||
fmt.Sprintf("Deleted PVC %s that CDI recreated after the original was kept aside", name))
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// markPreparedRestoreFailed fails a RestoreJob whose target was already
|
||||
// prepared, and repeats the message as a Warning event, since recovering the
|
||||
// target is left to the user.
|
||||
func (r *RestoreJobReconciler) markPreparedRestoreFailed(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) {
|
||||
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "RestoreFailed", message)
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, message)
|
||||
}
|
||||
|
||||
// restoreDisks returns the disks an in-place restore prepares. A VMDisk Backup
|
||||
// taken before its underlying resources were recorded has none, but its only
|
||||
// disk is the VMDisk itself, named as collectUnderlyingResources names it.
|
||||
func restoreDisks(backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) *vmInstanceResources {
|
||||
if vmRes := getVMInstanceResources(ur); vmRes != nil {
|
||||
return vmRes
|
||||
}
|
||||
if backup.Spec.ApplicationRef.Kind != vmDiskAppKind {
|
||||
return nil
|
||||
}
|
||||
name := backup.Spec.ApplicationRef.Name
|
||||
return &vmInstanceResources{DataVolumes: []backupsv1alpha1.DataVolumeResource{{
|
||||
DataVolumeName: vmDiskNamePrefix + name,
|
||||
ApplicationName: name,
|
||||
}}}
|
||||
}
|
||||
|
||||
// preparedRestoreLeftovers describes, for the failure message of an in-place
|
||||
// restore that fails after prepareForRestore ran, what preparation left behind.
|
||||
// Nothing is rolled back automatically: resuming a disk's HelmRelease while its
|
||||
// PVC is renamed and its DataVolume deleted would have CDI import a fresh disk
|
||||
// under the original name, which looks healthy and hides that the data sits in
|
||||
// the -orig- PVC. Preparation can fail before it reaches every step, so only a
|
||||
// HelmRelease that is suspended and an -orig- PVC that exists are named; one
|
||||
// that cannot be read is named anyway rather than hidden.
|
||||
func (r *RestoreJobReconciler) preparedRestoreLeftovers(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) string {
|
||||
if target.IsCopy {
|
||||
return ""
|
||||
}
|
||||
vmRes := restoreDisks(backup, ur)
|
||||
var hrs, kept []string
|
||||
if backup.Spec.ApplicationRef.Kind == vmInstanceKind {
|
||||
hrs = append(hrs, vmNamePrefix+backup.Spec.ApplicationRef.Name)
|
||||
}
|
||||
if vmRes != nil {
|
||||
for _, dv := range vmRes.DataVolumes {
|
||||
hrs = append(hrs, dv.DataVolumeName)
|
||||
if opts.GetKeepOriginalPVC() {
|
||||
kept = append(kept, dv.DataVolumeName+"-orig-"+shortHash(restoreJob.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
ns := target.Namespace
|
||||
hrs = slices.DeleteFunc(hrs, func(name string) bool {
|
||||
hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return errors.IsNotFound(err)
|
||||
}
|
||||
suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend")
|
||||
return !suspended
|
||||
})
|
||||
kept = slices.DeleteFunc(kept, func(name string) bool {
|
||||
err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &corev1.PersistentVolumeClaim{})
|
||||
return errors.IsNotFound(err)
|
||||
})
|
||||
if len(hrs) == 0 && len(kept) == 0 {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
if len(hrs) > 0 {
|
||||
parts = append(parts, fmt.Sprintf("suspended HelmRelease %s", strings.Join(hrs, ", ")))
|
||||
}
|
||||
if len(kept) > 0 {
|
||||
parts = append(parts, fmt.Sprintf("kept the original data in PVC %s", strings.Join(kept, ", ")))
|
||||
}
|
||||
return "; preparation " + strings.Join(parts, " and ") + "; if Velero restored the HelmRelease it is resumed and the disk may have been recreated, so check what the disk holds before relying on it"
|
||||
}
|
||||
|
||||
// podsMountingDisks returns "pod/<pod> (PVC <claim>)" for every Pod in ns that
|
||||
// has not terminated and mounts one of the disks' PVCs, split into Pods that
|
||||
// are running and Pods already being deleted. A terminated Pod no longer holds
|
||||
// pvc-protection, so it does not count.
|
||||
func (r *RestoreJobReconciler) podsMountingDisks(ctx context.Context, ns string, disks []backupsv1alpha1.DataVolumeResource) (inUse, leaving []string, err error) {
|
||||
claims := make(map[string]bool, len(disks))
|
||||
for _, dv := range disks {
|
||||
claims[dv.DataVolumeName] = true
|
||||
}
|
||||
pods := &corev1.PodList{}
|
||||
if err := r.List(ctx, pods, client.InNamespace(ns)); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for i := range pods.Items {
|
||||
pod := &pods.Items[i]
|
||||
if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
|
||||
continue
|
||||
}
|
||||
for _, v := range pod.Spec.Volumes {
|
||||
if v.PersistentVolumeClaim == nil || !claims[v.PersistentVolumeClaim.ClaimName] {
|
||||
continue
|
||||
}
|
||||
holder := fmt.Sprintf("pod/%s (PVC %s)", pod.Name, v.PersistentVolumeClaim.ClaimName)
|
||||
if pod.DeletionTimestamp.IsZero() {
|
||||
inUse = append(inUse, holder)
|
||||
} else {
|
||||
leaving = append(leaving, holder)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(inUse)
|
||||
sort.Strings(leaving)
|
||||
return inUse, leaving, nil
|
||||
}
|
||||
|
||||
// veleroBackupForRestore returns the named Velero Backup and why it cannot be
|
||||
// restored, or "" when it can. Only a Completed Backup is restored: Velero
|
||||
// itself validates only that the Backup exists and its storage location is
|
||||
// usable, so a Restore from one in another phase would take the target apart
|
||||
// for nothing. A Backup that is gone comes back empty rather than nil.
|
||||
func (r *RestoreJobReconciler) veleroBackupForRestore(ctx context.Context, name string) (*velerov1.Backup, string, error) {
|
||||
vb := &velerov1.Backup{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb); err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return &velerov1.Backup{}, fmt.Sprintf("Velero Backup %s/%s not found; it may have expired", veleroNamespace, name), nil
|
||||
}
|
||||
return nil, "", fmt.Errorf("failed to get Velero Backup %s: %w", name, err)
|
||||
}
|
||||
if vb.Status.Phase != velerov1.BackupPhaseCompleted {
|
||||
return vb, fmt.Sprintf("Velero Backup %s/%s is %q, not %q", veleroNamespace, name, vb.Status.Phase, velerov1.BackupPhaseCompleted), nil
|
||||
}
|
||||
return vb, "", nil
|
||||
}
|
||||
|
||||
// restoredUnderlyingResources resolves the underlying resources of a restore
|
||||
// whose Velero Restore already exists, for naming what preparation left
|
||||
// behind. A Velero Backup that cannot be read leaves the Backup status.
|
||||
func (r *RestoreJobReconciler) restoredUnderlyingResources(ctx context.Context, backup *backupsv1alpha1.Backup, name string) *runtime.RawExtension {
|
||||
vb := &velerov1.Backup{}
|
||||
_ = r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb)
|
||||
return resolveUnderlyingResourcesForRestore(backup, vb)
|
||||
}
|
||||
|
||||
// suspendHelmRelease sets spec.suspend=true on a HelmRelease.
|
||||
func (r *RestoreJobReconciler) suspendHelmRelease(ctx context.Context, ns, name string) error {
|
||||
hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
|
||||
@@ -1578,7 +1876,7 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ
|
||||
|
||||
// Match backup: add OR selectors for each underlying VMDisk so restore applies the same
|
||||
// scope as the intended backup (see createVeleroBackup).
|
||||
if vmRes := getVMInstanceResources(ur); vmRes != nil {
|
||||
if vmRes := getVMInstanceResources(ur); vmRes != nil && backup.Spec.ApplicationRef.Kind == vmInstanceKind {
|
||||
for _, dv := range vmRes.DataVolumes {
|
||||
veleroRestoreSpec.OrLabelSelectors = append(veleroRestoreSpec.OrLabelSelectors, &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
|
||||
@@ -3,6 +3,7 @@ package backupcontroller
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -355,6 +356,7 @@ func newTestRestoreJobReconcilerWithDynamic(t *testing.T, dynamicObjects []runti
|
||||
fakeClient := clientfake.NewClientBuilder().
|
||||
WithScheme(testScheme).
|
||||
WithObjects(objects...).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
|
||||
Build()
|
||||
|
||||
dynamicClient := dynamicfake.NewSimpleDynamicClient(testScheme, dynamicObjects...)
|
||||
@@ -385,6 +387,7 @@ func newTestRestoreJobReconciler(t *testing.T, objects ...client.Object) *Restor
|
||||
fakeClient := clientfake.NewClientBuilder().
|
||||
WithScheme(testScheme).
|
||||
WithObjects(objects...).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
|
||||
Build()
|
||||
|
||||
dynamicClient := dynamicfake.NewSimpleDynamicClient(testScheme)
|
||||
@@ -730,6 +733,72 @@ func TestReconcileVeleroRestore_UnusableVeleroBackupTouchesNothing(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectUnderlyingResources_VMDisk(t *testing.T) {
|
||||
// A standalone VMDisk must report its own disk as an underlying resource so
|
||||
// the restore controller suspends the vm-disk-<name> HelmRelease and deletes
|
||||
// its DataVolume before the data mover repopulates the PVC. Without this,
|
||||
// prepareForRestore does nothing for a VMDisk (vmRes == nil) and Flux/CDI
|
||||
// race the data mover over the same PVC name, wedging the restore.
|
||||
r := &BackupJobReconciler{}
|
||||
app := &unstructured.Unstructured{Object: map[string]interface{}{
|
||||
"apiVersion": "apps.cozystack.io/v1alpha1",
|
||||
"kind": "VMDisk",
|
||||
"metadata": map[string]interface{}{"name": "backup-src", "namespace": "tenant-root"},
|
||||
}}
|
||||
|
||||
ur, err := r.collectUnderlyingResources(context.Background(), app, "VMDisk", "tenant-root")
|
||||
if err != nil {
|
||||
t.Fatalf("collectUnderlyingResources returned error: %v", err)
|
||||
}
|
||||
res := getVMInstanceResources(ur)
|
||||
if res == nil {
|
||||
t.Fatal("expected non-nil underlying resources for a VMDisk, got nil (prepareForRestore would skip the HR suspend and DataVolume delete)")
|
||||
}
|
||||
if len(res.DataVolumes) != 1 {
|
||||
t.Fatalf("expected exactly one DataVolume, got %d", len(res.DataVolumes))
|
||||
}
|
||||
if got, want := res.DataVolumes[0].DataVolumeName, "vm-disk-backup-src"; got != want {
|
||||
t.Errorf("DataVolumeName = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := res.DataVolumes[0].ApplicationName, "backup-src"; got != want {
|
||||
t.Errorf("ApplicationName = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectUnderlyingResources_VMInstance(t *testing.T) {
|
||||
testScheme := runtime.NewScheme()
|
||||
_ = scheme.AddToScheme(testScheme)
|
||||
r := &BackupJobReconciler{Client: clientfake.NewClientBuilder().WithScheme(testScheme).Build()}
|
||||
app := &unstructured.Unstructured{Object: map[string]interface{}{
|
||||
"apiVersion": "apps.cozystack.io/v1alpha1",
|
||||
"kind": "VMInstance",
|
||||
"metadata": map[string]interface{}{"name": "test", "namespace": "tenant-root"},
|
||||
"spec": map[string]interface{}{
|
||||
"disks": []interface{}{
|
||||
map[string]interface{}{"name": "system"},
|
||||
map[string]interface{}{"name": "data"},
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
ur, err := r.collectUnderlyingResources(context.Background(), app, "VMInstance", "tenant-root")
|
||||
if err != nil {
|
||||
t.Fatalf("collectUnderlyingResources returned error: %v", err)
|
||||
}
|
||||
res := getVMInstanceResources(ur)
|
||||
if res == nil {
|
||||
t.Fatal("expected non-nil underlying resources for a VMInstance with disks, got nil")
|
||||
}
|
||||
var got []string
|
||||
for _, dv := range res.DataVolumes {
|
||||
got = append(got, dv.DataVolumeName+"/"+dv.ApplicationName)
|
||||
}
|
||||
want := []string{"vm-disk-system/system", "vm-disk-data/data"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("DataVolumes = %v, want %v (one per spec.disks[] entry, not the VMInstance itself)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRestoreTarget_VMDisk_CommonRestoreOptions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
)
|
||||
|
||||
// An in-place restore of a VMDisk that a running VM mounts must be refused
|
||||
// before anything changes: renaming the PVC under the VM leaves it Terminating
|
||||
// behind pvc-protection, and Velero then skips the PVC as already present.
|
||||
func TestPrepareForRestore_VMDiskInUse(t *testing.T) {
|
||||
const (
|
||||
ns = "tenant-root"
|
||||
claim = "vm-disk-data"
|
||||
)
|
||||
mountingPod := func(name, claimName string, phase corev1.PodPhase) *corev1.Pod {
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{{Name: "compute", Image: "virt-launcher"}},
|
||||
Volumes: []corev1.Volume{{
|
||||
Name: "disk",
|
||||
VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claimName}},
|
||||
}},
|
||||
},
|
||||
Status: corev1.PodStatus{Phase: phase},
|
||||
}
|
||||
}
|
||||
urRaw, _ := json.Marshal(vmInstanceResources{
|
||||
DataVolumes: []backupsv1alpha1.DataVolumeResource{{DataVolumeName: claim, ApplicationName: "data"}},
|
||||
})
|
||||
ur := &runtime.RawExtension{Raw: urRaw}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "restore-disk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
}
|
||||
backupOf := func(kind, name string) *backupsv1alpha1.Backup {
|
||||
return &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: kind, Name: name},
|
||||
},
|
||||
}
|
||||
}
|
||||
pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: claim, Namespace: ns}}
|
||||
keep := RestoreOptions{KeepOriginalPVC: new(true)}
|
||||
|
||||
t.Run("a running pod that mounts the disk fails the restore and changes nothing", func(t *testing.T) {
|
||||
backup := backupOf("VMDisk", "data")
|
||||
hr := makeUnstructuredHelmRelease(claim, ns, nil)
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{hr},
|
||||
restoreJob.DeepCopy(), backup, pvc.DeepCopy(), mountingPod("virt-launcher-vm-instance-web-abcde", claim, corev1.PodRunning))
|
||||
ctx := context.Background()
|
||||
|
||||
ready, _, err := r.prepareForRestore(ctx, restoreJob.DeepCopy(), backup, ur, restoreTarget{Namespace: ns, AppName: "data", AppKind: "VMDisk"}, keep)
|
||||
if err == nil || ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want an error for a mounted disk", ready, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pod/virt-launcher-vm-instance-web-abcde") {
|
||||
t.Errorf("error %q does not name the pod that mounts the disk", err)
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: claim}, &corev1.PersistentVolumeClaim{}); err != nil {
|
||||
t.Errorf("the mounted PVC must keep its name, got %v", err)
|
||||
}
|
||||
got, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, claim, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspended {
|
||||
t.Errorf("the HelmRelease must not be suspended for a refused restore")
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
pod *corev1.Pod
|
||||
}{
|
||||
{name: "a pod that has finished does not hold the disk", pod: mountingPod("importer", claim, corev1.PodSucceeded)},
|
||||
{name: "a pod that mounts another disk does not hold this one", pod: mountingPod("other", "vm-disk-other", corev1.PodRunning)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
backup := backupOf("VMDisk", "data")
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pvc.DeepCopy(), tc.pod)
|
||||
ready, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: "VMDisk"}, RestoreOptions{KeepOriginalPVC: new(false)})
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A VMInstance target is halted, so its own launcher Pod is not a reason to
|
||||
// refuse; the rename waits until that Pod has let go of the disks.
|
||||
t.Run("a VMInstance target waits for its own launcher pod instead of being refused", func(t *testing.T) {
|
||||
backup := backupOf("VMInstance", "web")
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pvc.DeepCopy(),
|
||||
mountingPod("virt-launcher-vm-instance-web-abcde", claim, corev1.PodRunning))
|
||||
ready, res, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: "web", AppKind: "VMInstance"}, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a requeue", ready, res.RequeueAfter, err)
|
||||
}
|
||||
if err := r.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: claim}, &corev1.PersistentVolumeClaim{}); err != nil {
|
||||
t.Errorf("the PVC was renamed while the launcher still mounts it: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a VMInstance target proceeds once no pod mounts its disks", func(t *testing.T) {
|
||||
backup := backupOf("VMInstance", "web")
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pvc.DeepCopy())
|
||||
ready, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: "web", AppKind: "VMInstance"}, RestoreOptions{KeepOriginalPVC: new(false)})
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
})
|
||||
|
||||
// A launcher Pod already being deleted (the VM was just stopped) releases the
|
||||
// disk on its own, so the restore waits rather than fails.
|
||||
t.Run("a VMDisk held by a pod being deleted waits instead of failing", func(t *testing.T) {
|
||||
backup := backupOf("VMDisk", "data")
|
||||
pod := mountingPod("virt-launcher-vm-instance-web-abcde", claim, corev1.PodRunning)
|
||||
now := metav1.Now()
|
||||
pod.DeletionTimestamp = &now
|
||||
pod.Finalizers = []string{"kubernetes"}
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pvc.DeepCopy(), pod)
|
||||
ready, res, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: "VMDisk"}, keep)
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a requeue", ready, res.RequeueAfter, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a pod that failed does not hold the disk", func(t *testing.T) {
|
||||
backup := backupOf("VMDisk", "data")
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pvc.DeepCopy(), mountingPod("evicted", claim, corev1.PodFailed))
|
||||
ready, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: "VMDisk"}, RestoreOptions{KeepOriginalPVC: new(false)})
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,870 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
dynamicfake "k8s.io/client-go/dynamic/fake"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
k8stesting "k8s.io/client-go/testing"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
)
|
||||
|
||||
func vmDiskTestScheme() *runtime.Scheme {
|
||||
s := runtime.NewScheme()
|
||||
_ = scheme.AddToScheme(s)
|
||||
_ = backupsv1alpha1.AddToScheme(s)
|
||||
_ = strategyv1alpha1.AddToScheme(s)
|
||||
_ = velerov1.AddToScheme(s)
|
||||
return s
|
||||
}
|
||||
|
||||
func diskSelector() *metav1.LabelSelector {
|
||||
return &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/instance": "vm-disk-{{ .Application.metadata.name }}"}}
|
||||
}
|
||||
|
||||
// Velero rejects a Backup or Restore that carries both labelSelector and
|
||||
// orLabelSelectors, so a VMDisk strategy written with labelSelector must not
|
||||
// get the per-disk orLabelSelectors a VMInstance needs.
|
||||
func TestCreateVeleroBackup_VMDiskKeepsStrategySelector(t *testing.T) {
|
||||
s := vmDiskTestScheme()
|
||||
app := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "data", Namespace: "tenant-root"},
|
||||
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "i"}}},
|
||||
}
|
||||
mapping := &meta.RESTMapping{
|
||||
Resource: schema.GroupVersionResource{Version: "v1", Resource: "pods"},
|
||||
GroupVersionKind: schema.GroupVersionKind{Version: "v1", Kind: "Pod"},
|
||||
Scope: meta.RESTScopeNamespace,
|
||||
}
|
||||
bj := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bj", Namespace: "tenant-root"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
strategy := &strategyv1alpha1.Velero{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "s"},
|
||||
Spec: strategyv1alpha1.VeleroSpec{Template: strategyv1alpha1.VeleroTemplate{
|
||||
Spec: velerov1.BackupSpec{LabelSelector: diskSelector()},
|
||||
}},
|
||||
}
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).WithObjects(bj, strategy).Build()
|
||||
r := &BackupJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(s, app),
|
||||
RESTMapper: &mockRESTMapper{mapping: mapping}, Scheme: s, Recorder: record.NewFakeRecorder(10)}
|
||||
|
||||
if err := r.createVeleroBackup(context.Background(), bj, strategy, &ResolvedBackupConfig{}); err != nil {
|
||||
t.Fatalf("createVeleroBackup: %v", err)
|
||||
}
|
||||
backups := &velerov1.BackupList{}
|
||||
if err := c.List(context.Background(), backups, client.InNamespace(veleroNamespace)); err != nil || len(backups.Items) != 1 {
|
||||
t.Fatalf("list Velero Backups: %v (%d items)", err, len(backups.Items))
|
||||
}
|
||||
spec := backups.Items[0].Spec
|
||||
if spec.LabelSelector == nil || len(spec.OrLabelSelectors) != 0 {
|
||||
t.Fatalf("labelSelector %v, orLabelSelectors %d; want the strategy's labelSelector alone", spec.LabelSelector, len(spec.OrLabelSelectors))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateVeleroRestore_VMDiskKeepsStrategySelector(t *testing.T) {
|
||||
s := vmDiskTestScheme()
|
||||
ns := "tenant-root"
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns, UID: "uid-rj"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
}
|
||||
strategy := &strategyv1alpha1.Velero{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "s"},
|
||||
Spec: strategyv1alpha1.VeleroSpec{Template: strategyv1alpha1.VeleroTemplate{
|
||||
RestoreSpec: &velerov1.RestoreSpec{LabelSelector: diskSelector()},
|
||||
}},
|
||||
}
|
||||
urRaw, _ := json.Marshal(vmInstanceResources{
|
||||
DataVolumes: []backupsv1alpha1.DataVolumeResource{{DataVolumeName: "vm-disk-data", ApplicationName: "data"}},
|
||||
})
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).WithObjects(backup, restoreJob, strategy).Build()
|
||||
r := &RestoreJobReconciler{Client: c, Scheme: s, Recorder: record.NewFakeRecorder(10)}
|
||||
|
||||
if err := r.createVeleroRestore(context.Background(), restoreJob, backup, strategy, "velero-bk", &runtime.RawExtension{Raw: urRaw},
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: vmDiskAppKind}, RestoreOptions{}); err != nil {
|
||||
t.Fatalf("createVeleroRestore: %v", err)
|
||||
}
|
||||
restores := &velerov1.RestoreList{}
|
||||
if err := c.List(context.Background(), restores, client.InNamespace(veleroNamespace)); err != nil || len(restores.Items) != 1 {
|
||||
t.Fatalf("list Velero Restores: %v (%d items)", err, len(restores.Items))
|
||||
}
|
||||
spec := restores.Items[0].Spec
|
||||
if spec.LabelSelector == nil || len(spec.OrLabelSelectors) != 0 {
|
||||
t.Fatalf("labelSelector %v, orLabelSelectors %d; want the strategy's labelSelector alone", spec.LabelSelector, len(spec.OrLabelSelectors))
|
||||
}
|
||||
}
|
||||
|
||||
// A VMDisk Backup taken before underlying resources were recorded still
|
||||
// restores its disk prepared, and the in-use refusal still applies to it.
|
||||
func TestPrepareForRestore_VMDiskBackupWithoutUnderlyingResources(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
}
|
||||
target := restoreTarget{Namespace: ns, AppName: "data", AppKind: vmDiskAppKind}
|
||||
|
||||
t.Run("its disk is prepared", func(t *testing.T) {
|
||||
hr := makeUnstructuredHelmRelease("vm-disk-data", ns, nil)
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{hr}, restoreJob.DeepCopy(), backup)
|
||||
ready, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, nil, target, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
got, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), "vm-disk-data", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); !suspended {
|
||||
t.Fatalf("the disk's HelmRelease was not suspended")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a mounted disk is refused", func(t *testing.T) {
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "virt-launcher-web", Namespace: ns},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{{Name: "compute", Image: "virt-launcher"}},
|
||||
Volumes: []corev1.Volume{{Name: "disk", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "vm-disk-data"},
|
||||
}}},
|
||||
},
|
||||
Status: corev1.PodStatus{Phase: corev1.PodRunning},
|
||||
}
|
||||
r := newTestRestoreJobReconciler(t, restoreJob.DeepCopy(), backup, pod)
|
||||
if _, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, nil, target, RestoreOptions{KeepOriginalPVC: new(true)}); err == nil {
|
||||
t.Fatalf("prepareForRestore accepted a mounted disk")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Failing to list Pods must not read as "not in use": the restore waits and
|
||||
// touches nothing.
|
||||
func TestPrepareForRestore_VMDiskInUseCheckFailsClosed(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
s := vmDiskTestScheme()
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns}}
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).WithObjects(backup, restoreJob).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
|
||||
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
||||
if _, ok := list.(*corev1.PodList); ok {
|
||||
return errors.New("pods is forbidden")
|
||||
}
|
||||
return cl.List(ctx, list, opts...)
|
||||
}}).Build()
|
||||
hr := makeUnstructuredHelmRelease("vm-disk-data", ns, nil)
|
||||
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(s, hr), Scheme: s, Recorder: record.NewFakeRecorder(10)}
|
||||
|
||||
ready, res, err := r.prepareForRestore(context.Background(), restoreJob, backup, nil,
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: vmDiskAppKind}, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a requeue", ready, res.RequeueAfter, err)
|
||||
}
|
||||
got, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), "vm-disk-data", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspended {
|
||||
t.Fatalf("the HelmRelease was suspended although the in-use check could not run")
|
||||
}
|
||||
}
|
||||
|
||||
// A restore that fails after preparation leaves the disk suspended and its data
|
||||
// in the -orig- PVC; the failure has to say so, since nothing is rolled back,
|
||||
// and must not name what preparation did not get to.
|
||||
func TestPreparedRestoreLeftovers(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns}}
|
||||
vmDisk := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: vmDiskAppKind, Name: "data"}}}
|
||||
orig := "vm-disk-data-orig-" + shortHash("rj")
|
||||
origPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: orig, Namespace: ns}}
|
||||
keep := RestoreOptions{KeepOriginalPVC: new(true)}
|
||||
inPlace := restoreTarget{Namespace: ns}
|
||||
ctx := context.Background()
|
||||
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{suspendedHelmRelease(ns, "vm-disk-data", true)}, origPVC)
|
||||
got := r.preparedRestoreLeftovers(ctx, restoreJob, vmDisk, nil, inPlace, keep)
|
||||
for _, want := range []string{"suspended HelmRelease vm-disk-data", "PVC " + orig} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("leftovers %q does not mention %q", got, want)
|
||||
}
|
||||
}
|
||||
if got := r.preparedRestoreLeftovers(ctx, restoreJob, vmDisk, nil, inPlace, RestoreOptions{KeepOriginalPVC: new(false)}); strings.Contains(got, "-orig-") {
|
||||
t.Errorf("leftovers %q names an -orig- PVC although keepOriginalPVC is off", got)
|
||||
}
|
||||
if got := r.preparedRestoreLeftovers(ctx, restoreJob, vmDisk, nil, restoreTarget{Namespace: ns, IsCopy: true}, keep); got != "" {
|
||||
t.Errorf("a restore to a copy prepares nothing, got %q", got)
|
||||
}
|
||||
|
||||
suspendedOnly := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{suspendedHelmRelease(ns, "vm-disk-data", true)})
|
||||
if got := suspendedOnly.preparedRestoreLeftovers(ctx, restoreJob, vmDisk, nil, inPlace, keep); !strings.Contains(got, "suspended HelmRelease vm-disk-data") || strings.Contains(got, "-orig-") {
|
||||
t.Errorf("leftovers %q: want the suspended HelmRelease and no -orig- PVC, which was never created", got)
|
||||
}
|
||||
|
||||
untouched := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{makeUnstructuredHelmRelease("vm-disk-data", ns, nil)})
|
||||
if got := untouched.preparedRestoreLeftovers(ctx, restoreJob, vmDisk, nil, inPlace, keep); got != "" {
|
||||
t.Errorf("leftovers %q for a target preparation never changed, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Every way an in-place restore can fail once its target was prepared has to
|
||||
// name what preparation left behind.
|
||||
func TestReconcileVeleroRestore_FailureAfterPreparationNamesLeftovers(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
orig := "vm-disk-data-orig-" + shortHash("rj")
|
||||
veleroRestore := func(name string, phase velerov1.RestorePhase) *velerov1.Restore {
|
||||
return &velerov1.Restore{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: veleroNamespace, Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: ns,
|
||||
}},
|
||||
Status: velerov1.RestoreStatus{Phase: phase},
|
||||
}
|
||||
}
|
||||
failRestoreCreate := interceptor.Funcs{Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
||||
if _, ok := obj.(*velerov1.Restore); ok {
|
||||
return errors.New("admission webhook denied the request")
|
||||
}
|
||||
return cl.Create(ctx, obj, opts...)
|
||||
}}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
restores []client.Object
|
||||
funcs interceptor.Funcs
|
||||
cause string
|
||||
}{
|
||||
{name: "the Velero Restore partially failed", restores: []client.Object{veleroRestore("rj-a", velerov1.RestorePhasePartiallyFailed)}, cause: "PartiallyFailed"},
|
||||
{name: "several Velero Restores own the job", restores: []client.Object{veleroRestore("rj-a", ""), veleroRestore("rj-b", "")}, cause: "multiple Velero Restores"},
|
||||
{name: "Velero rejected the Restore", restores: []client.Object{func() client.Object {
|
||||
vr := veleroRestore("rj-a", velerov1.RestorePhaseFailedValidation)
|
||||
vr.Status.ValidationErrors = []string{"backup velero-bk not found"}
|
||||
return vr
|
||||
}()}, cause: "FailedValidation: backup velero-bk not found"},
|
||||
{name: "the Velero Restore could not be created", funcs: failRestoreCreate, cause: "failed to create Velero Restore"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := vmDiskTestScheme()
|
||||
started := metav1.Now()
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns, UID: "rj-uid"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{Phase: backupsv1alpha1.RestoreJobPhaseRunning, StartedAt: &started},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data"},
|
||||
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new(strategyv1alpha1.GroupVersion.Group), Kind: strategyv1alpha1.VeleroStrategyKind, Name: "s"},
|
||||
DriverMetadata: map[string]string{veleroBackupNameMetadataKey: "velero-bk"},
|
||||
},
|
||||
}
|
||||
strategy := &strategyv1alpha1.Velero{ObjectMeta: metav1.ObjectMeta{Name: "s"}}
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).
|
||||
WithObjects(append([]client.Object{restoreJob, backup, strategy, completedVeleroBackup("velero-bk")}, append(keptOrigPVC(ns), tc.restores...)...)...).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
|
||||
WithInterceptorFuncs(tc.funcs).Build()
|
||||
recorder := record.NewFakeRecorder(20)
|
||||
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(s, suspendedHelmRelease(ns, "vm-disk-data", true)), Scheme: s, Recorder: recorder}
|
||||
|
||||
rj := &backupsv1alpha1.RestoreJob{}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
if _, err := r.reconcileVeleroRestore(context.Background(), rj, backup); err != nil {
|
||||
t.Fatalf("reconcileVeleroRestore: %v", err)
|
||||
}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("Phase = %q, want Failed", rj.Status.Phase)
|
||||
}
|
||||
for _, want := range []string{tc.cause, "suspended HelmRelease vm-disk-data", "PVC " + orig} {
|
||||
if !strings.Contains(rj.Status.Message, want) {
|
||||
t.Errorf("message %q does not mention %q", rj.Status.Message, want)
|
||||
}
|
||||
}
|
||||
warned := false
|
||||
for len(recorder.Events) > 0 {
|
||||
ev := <-recorder.Events
|
||||
if strings.HasPrefix(ev, "Warning RestoreFailed") && strings.Contains(ev, orig) {
|
||||
warned = true
|
||||
}
|
||||
}
|
||||
if !warned {
|
||||
t.Errorf("no Warning RestoreFailed event naming %s", orig)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func completedVeleroBackup(name string) *velerov1.Backup {
|
||||
return &velerov1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: veleroNamespace},
|
||||
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseCompleted},
|
||||
}
|
||||
}
|
||||
|
||||
// A Velero Backup that is gone or failed cannot be restored; the job fails
|
||||
// before preparation takes the target apart.
|
||||
func TestReconcileVeleroRestore_UnrestorableVeleroBackupFailsBeforePreparation(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
vb []client.Object
|
||||
cause string
|
||||
}{
|
||||
{name: "the Velero Backup is gone", cause: "not found; it may have expired"},
|
||||
{name: "the Velero Backup failed", vb: []client.Object{&velerov1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "velero-bk", Namespace: veleroNamespace},
|
||||
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseFailed},
|
||||
}}, cause: `is "Failed"`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := vmDiskTestScheme()
|
||||
started := metav1.Now()
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns, UID: "rj-uid"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{Phase: backupsv1alpha1.RestoreJobPhaseRunning, StartedAt: &started},
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data"},
|
||||
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new(strategyv1alpha1.GroupVersion.Group), Kind: strategyv1alpha1.VeleroStrategyKind, Name: "s"},
|
||||
DriverMetadata: map[string]string{veleroBackupNameMetadataKey: "velero-bk"},
|
||||
},
|
||||
}
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).
|
||||
WithObjects(append([]client.Object{restoreJob, backup, &strategyv1alpha1.Velero{ObjectMeta: metav1.ObjectMeta{Name: "s"}}}, tc.vb...)...).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).Build()
|
||||
hr := makeUnstructuredHelmRelease("vm-disk-data", ns, nil)
|
||||
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(s, hr), Scheme: s, Recorder: record.NewFakeRecorder(10)}
|
||||
|
||||
rj := &backupsv1alpha1.RestoreJob{}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
if _, err := r.reconcileVeleroRestore(context.Background(), rj, backup); err != nil {
|
||||
t.Fatalf("reconcileVeleroRestore: %v", err)
|
||||
}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(rj.Status.Message, tc.cause) {
|
||||
t.Fatalf("phase %q, message %q; want Failed mentioning %q", rj.Status.Phase, rj.Status.Message, tc.cause)
|
||||
}
|
||||
got, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), "vm-disk-data", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspended {
|
||||
t.Fatalf("the disk was prepared although its Velero Backup cannot be restored")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A suspend or halt that failed must stop preparation: renaming the PVC and
|
||||
// deleting the DataVolume under a running HelmRelease or VM is the race the
|
||||
// preparation exists to prevent.
|
||||
func TestPrepareForRestore_StopsWhenSuspendOrHaltFails(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
urRaw, _ := json.Marshal(vmInstanceResources{
|
||||
DataVolumes: []backupsv1alpha1.DataVolumeResource{{DataVolumeName: "vm-disk-data", ApplicationName: "data"}},
|
||||
})
|
||||
ur := &runtime.RawExtension{Raw: urRaw}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns}}
|
||||
pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-data", Namespace: ns}}
|
||||
dataVolume := &unstructured.Unstructured{}
|
||||
dataVolume.SetAPIVersion("cdi.kubevirt.io/v1beta1")
|
||||
dataVolume.SetKind("DataVolume")
|
||||
dataVolume.SetName("vm-disk-data")
|
||||
dataVolume.SetNamespace(ns)
|
||||
vm := &unstructured.Unstructured{}
|
||||
vm.SetAPIVersion("kubevirt.io/v1")
|
||||
vm.SetKind("VirtualMachine")
|
||||
vm.SetName("vm-instance-web")
|
||||
vm.SetNamespace(ns)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
kind string
|
||||
app string
|
||||
resource string
|
||||
}{
|
||||
{name: "the HelmRelease suspend fails", kind: vmDiskAppKind, app: "data", resource: "helmreleases"},
|
||||
{name: "the VM halt fails", kind: vmInstanceKind, app: "web", resource: "virtualmachines"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: tc.kind, Name: tc.app}},
|
||||
}
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{
|
||||
makeUnstructuredHelmRelease("vm-disk-data", ns, nil),
|
||||
makeUnstructuredHelmRelease("vm-instance-web", ns, nil),
|
||||
dataVolume.DeepCopy(), vm.DeepCopy(),
|
||||
}, restoreJob.DeepCopy(), backup, pvc.DeepCopy())
|
||||
r.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("update", tc.resource, func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, errors.New("conflict")
|
||||
})
|
||||
|
||||
ready, res, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, ur,
|
||||
restoreTarget{Namespace: ns, AppName: tc.app, AppKind: tc.kind}, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a requeue", ready, res.RequeueAfter, err)
|
||||
}
|
||||
if err := r.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "vm-disk-data"}, &corev1.PersistentVolumeClaim{}); err != nil {
|
||||
t.Errorf("the PVC was renamed: %v", err)
|
||||
}
|
||||
if _, err := r.Resource(dataVolumeGVR).Namespace(ns).Get(context.Background(), "vm-disk-data", metav1.GetOptions{}); err != nil {
|
||||
t.Errorf("the DataVolume was deleted: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDisks(t *testing.T) {
|
||||
backupOf := func(kind string) *backupsv1alpha1.Backup {
|
||||
return &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: kind, Name: "data"}}}
|
||||
}
|
||||
if got := restoreDisks(backupOf(vmDiskAppKind), nil); got == nil || len(got.DataVolumes) != 1 || got.DataVolumes[0].DataVolumeName != "vm-disk-data" {
|
||||
t.Errorf("VMDisk without underlying resources: %+v, want vm-disk-data", got)
|
||||
}
|
||||
for _, kind := range []string{vmInstanceKind, "Postgres"} {
|
||||
if got := restoreDisks(backupOf(kind), nil); got != nil {
|
||||
t.Errorf("%s without underlying resources: %+v, want none; a disk must not be derived for another kind", kind, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Kinds other than VMDisk and VMInstance have no disks to prepare and must pass
|
||||
// through preparation untouched.
|
||||
func TestPrepareForRestore_OtherKindWithoutDisks(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: "Postgres", Name: "data"}},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns}}
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{makeUnstructuredHelmRelease("vm-disk-data", ns, nil)}, restoreJob.DeepCopy(), backup)
|
||||
ready, _, err := r.prepareForRestore(context.Background(), restoreJob.DeepCopy(), backup, nil,
|
||||
restoreTarget{Namespace: ns, AppName: "data", AppKind: "Postgres"}, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
got, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(context.Background(), "vm-disk-data", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend"); suspended {
|
||||
t.Fatalf("an unrelated vm-disk-data HelmRelease was suspended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRestoreLeftovers_VMInstanceNamesItsHelmRelease(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{ObjectMeta: metav1.ObjectMeta{Name: "rj"}}
|
||||
backup := &backupsv1alpha1.Backup{Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: vmInstanceKind, Name: "web"}}}
|
||||
urRaw, _ := json.Marshal(vmInstanceResources{
|
||||
DataVolumes: []backupsv1alpha1.DataVolumeResource{{DataVolumeName: "vm-disk-root", ApplicationName: "root"}},
|
||||
})
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{suspendedHelmRelease(ns, "vm-instance-web", true), suspendedHelmRelease(ns, "vm-disk-root", true)})
|
||||
got := r.preparedRestoreLeftovers(context.Background(), restoreJob, backup, &runtime.RawExtension{Raw: urRaw}, restoreTarget{Namespace: ns}, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if !strings.Contains(got, "vm-instance-web, vm-disk-root") {
|
||||
t.Errorf("leftovers %q do not name the VM's HelmRelease and its disk's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// CDI can recreate an empty PVC under the original name between the rename and
|
||||
// the DataVolume delete. Velero skips restoring a PVC that exists, so the
|
||||
// restore would report success with a blank disk; preparation has to clear it.
|
||||
func TestPrepareForRestore_ClearsPVCRecreatedAfterRename(t *testing.T) {
|
||||
f := newKeepPVCFixture(t)
|
||||
r := f.reconciler(t, f.livePVC, f.pv)
|
||||
ctx := context.Background()
|
||||
recreated := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: f.livePVC.Name, Namespace: f.ns}}
|
||||
recreatedOnce := false
|
||||
r.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("delete", "datavolumes", func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
if !recreatedOnce {
|
||||
recreatedOnce = true
|
||||
if err := r.Create(ctx, recreated.DeepCopy()); err != nil {
|
||||
t.Errorf("recreate PVC: %v", err)
|
||||
}
|
||||
}
|
||||
return false, nil, nil
|
||||
})
|
||||
|
||||
ready, res, err := r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("pass 1: prepareForRestore = ready %v, requeue %v, err %v; want a requeue while the recreated PVC goes", ready, res.RequeueAfter, err)
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(recreated), &corev1.PersistentVolumeClaim{}); !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("the recreated PVC is still there (err %v); Velero would skip restoring it", err)
|
||||
}
|
||||
|
||||
ready, _, err = r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
|
||||
if err != nil || !ready {
|
||||
t.Fatalf("pass 2: prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
kept := &corev1.PersistentVolumeClaim{}
|
||||
if err := r.Get(ctx, client.ObjectKey{Namespace: f.ns, Name: f.origName}, kept); err != nil || kept.Spec.VolumeName != "pv-current" {
|
||||
t.Fatalf("the original disk is not kept as %s on pv-current: %+v, err %v", f.origName, kept.Spec, err)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcilePreparedVMDisk runs reconcileVeleroRestore for an in-place VMDisk
|
||||
// restore that has no Velero Restore yet; started, when set, is the time the
|
||||
// RestoreJob recorded PreparationStarted.
|
||||
// keptOrigPVC is the -orig- PVC, bound to its retained PV, that preparation
|
||||
// leaves for the RestoreJob "rj" with UID "rj-uid".
|
||||
func keptOrigPVC(ns string) []client.Object {
|
||||
name := "vm-disk-data-orig-" + shortHash("rj")
|
||||
return []client.Object{
|
||||
&corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Annotations: map[string]string{origPVCRestoreJobUIDAnnotation: "rj-uid"}},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "pv-data"},
|
||||
},
|
||||
&corev1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "pv-data"},
|
||||
Spec: corev1.PersistentVolumeSpec{
|
||||
PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain,
|
||||
ClaimRef: &corev1.ObjectReference{Namespace: ns, Name: name},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func reconcilePreparedVMDisk(t *testing.T, started *metav1.Time, extra ...client.Object) (*backupsv1alpha1.RestoreJob, *record.FakeRecorder) {
|
||||
t.Helper()
|
||||
const ns = "tenant-root"
|
||||
s := vmDiskTestScheme()
|
||||
now := metav1.Now()
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns, UID: "rj-uid"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{Phase: backupsv1alpha1.RestoreJobPhaseRunning, StartedAt: &now},
|
||||
}
|
||||
hr := makeUnstructuredHelmRelease("vm-disk-data", ns, nil)
|
||||
if started != nil {
|
||||
hr = suspendedHelmRelease(ns, "vm-disk-data", true)
|
||||
extra = append(extra, keptOrigPVC(ns)...)
|
||||
restoreJob.Status.Conditions = []metav1.Condition{{
|
||||
Type: restoreCondPreparationStarted, Status: metav1.ConditionTrue, Reason: "Preparing", LastTransitionTime: *started,
|
||||
}}
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data"},
|
||||
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new(strategyv1alpha1.GroupVersion.Group), Kind: strategyv1alpha1.VeleroStrategyKind, Name: "s"},
|
||||
DriverMetadata: map[string]string{veleroBackupNameMetadataKey: "velero-bk"},
|
||||
},
|
||||
}
|
||||
c := clientfake.NewClientBuilder().WithScheme(s).
|
||||
WithObjects(append([]client.Object{restoreJob, backup, &strategyv1alpha1.Velero{ObjectMeta: metav1.ObjectMeta{Name: "s"}}}, extra...)...).
|
||||
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).Build()
|
||||
recorder := record.NewFakeRecorder(20)
|
||||
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(s, hr), Scheme: s, Recorder: recorder}
|
||||
rj := &backupsv1alpha1.RestoreJob{}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
if _, err := r.reconcileVeleroRestore(context.Background(), rj, backup); err != nil {
|
||||
t.Fatalf("reconcileVeleroRestore: %v", err)
|
||||
}
|
||||
if err := c.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
return rj, recorder
|
||||
}
|
||||
|
||||
func mountingVMDiskPod(name string, deleting bool) *corev1.Pod {
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-root"},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{{Name: "compute", Image: "virt-launcher"}},
|
||||
Volumes: []corev1.Volume{{Name: "disk", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "vm-disk-data"},
|
||||
}}},
|
||||
},
|
||||
Status: corev1.PodStatus{Phase: corev1.PodRunning},
|
||||
}
|
||||
if deleting {
|
||||
now := metav1.Now()
|
||||
pod.DeletionTimestamp = &now
|
||||
pod.Finalizers = []string{"kubernetes"}
|
||||
}
|
||||
return pod
|
||||
}
|
||||
|
||||
// Preparation spans several passes. A refusal on a later pass must say what
|
||||
// preparation already left behind, not that nothing was changed.
|
||||
func TestReconcileVeleroRestore_RefusalAfterPreparationStartedNamesLeftovers(t *testing.T) {
|
||||
started := metav1.Now()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
extra []client.Object
|
||||
cause string
|
||||
}{
|
||||
{name: "the Velero Backup expired meanwhile", cause: "not found; it may have expired"},
|
||||
{name: "a VM started on the disk meanwhile", extra: []client.Object{completedVeleroBackup("velero-bk"), mountingVMDiskPod("virt-launcher-web", false)}, cause: "in use by pod/virt-launcher-web"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rj, recorder := reconcilePreparedVMDisk(t, &started, tc.extra...)
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("Phase = %q, want Failed", rj.Status.Phase)
|
||||
}
|
||||
for _, want := range []string{tc.cause, "suspended HelmRelease vm-disk-data", "PVC vm-disk-data-orig-" + shortHash("rj")} {
|
||||
if !strings.Contains(rj.Status.Message, want) {
|
||||
t.Errorf("message %q does not mention %q", rj.Status.Message, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(rj.Status.Message, "nothing was changed") {
|
||||
t.Errorf("message %q claims nothing was changed after preparation started", rj.Status.Message)
|
||||
}
|
||||
warned := false
|
||||
for len(recorder.Events) > 0 {
|
||||
if strings.HasPrefix(<-recorder.Events, "Warning RestoreFailed") {
|
||||
warned = true
|
||||
}
|
||||
}
|
||||
if !warned {
|
||||
t.Error("no Warning RestoreFailed event")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Before preparation starts, a refusal really has changed nothing and must
|
||||
// leave PreparationStarted unset.
|
||||
func TestReconcileVeleroRestore_RefusalBeforePreparationChangesNothing(t *testing.T) {
|
||||
rj, _ := reconcilePreparedVMDisk(t, nil, completedVeleroBackup("velero-bk"), mountingVMDiskPod("virt-launcher-web", false))
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(rj.Status.Message, "nothing was changed") {
|
||||
t.Fatalf("phase %q, message %q; want Failed saying nothing was changed", rj.Status.Phase, rj.Status.Message)
|
||||
}
|
||||
if meta.IsStatusConditionTrue(rj.Status.Conditions, restoreCondPreparationStarted) {
|
||||
t.Error("PreparationStarted was recorded although the restore was refused first")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileVeleroRestore_PreparationDeadline(t *testing.T) {
|
||||
waiting := []client.Object{completedVeleroBackup("velero-bk"), mountingVMDiskPod("virt-launcher-web", true)}
|
||||
|
||||
t.Run("a preparation still waiting past the deadline fails and names the leftovers", func(t *testing.T) {
|
||||
long := metav1.NewTime(time.Now().Add(-restorePreparationDeadline - time.Minute))
|
||||
rj, _ := reconcilePreparedVMDisk(t, &long, waiting...)
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(rj.Status.Message, "did not finish within") ||
|
||||
!strings.Contains(rj.Status.Message, "suspended HelmRelease vm-disk-data") {
|
||||
t.Fatalf("phase %q, message %q; want Failed past the deadline, naming the leftovers", rj.Status.Phase, rj.Status.Message)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a preparation waiting within the deadline keeps running", func(t *testing.T) {
|
||||
recent := metav1.Now()
|
||||
rj, _ := reconcilePreparedVMDisk(t, &recent, waiting...)
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseRunning {
|
||||
t.Fatalf("Phase = %q, message %q; want Running", rj.Status.Phase, rj.Status.Message)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPrepareForRestore_RecordsPreparationStarted(t *testing.T) {
|
||||
f := newKeepPVCFixture(t)
|
||||
r := f.reconciler(t, f.livePVC, f.pv)
|
||||
if ready, _, err := r.prepareForRestore(context.Background(), f.restoreJob, f.backup, f.ur, f.target, f.opts); err != nil || !ready {
|
||||
t.Fatalf("prepareForRestore = ready %v, err %v; want ready", ready, err)
|
||||
}
|
||||
got := &backupsv1alpha1.RestoreJob{}
|
||||
if err := r.Get(context.Background(), client.ObjectKeyFromObject(f.restoreJob), got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !meta.IsStatusConditionTrue(got.Status.Conditions, restoreCondPreparationStarted) {
|
||||
t.Errorf("PreparationStarted not recorded: %+v", got.Status.Conditions)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5 waits for the DataVolume to be gone: while it exists, CDI can create
|
||||
// the PVC again after Step 5 cleared it.
|
||||
func TestPrepareForRestore_WaitsForDataVolumeGone(t *testing.T) {
|
||||
f := newKeepPVCFixture(t)
|
||||
r := f.reconciler(t, f.livePVC, f.pv)
|
||||
r.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("delete", "datavolumes", func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, nil
|
||||
})
|
||||
ready, res, err := r.prepareForRestore(context.Background(), f.restoreJob, f.backup, f.ur, f.target, f.opts)
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a requeue while the DataVolume exists", ready, res.RequeueAfter, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Without keepOriginalPVC, a VMDisk PVC left in place would be skipped by
|
||||
// Velero and, orphaned from its DataVolume, make CDI refuse the DataVolume
|
||||
// the resumed HelmRelease creates: preparation has to delete it.
|
||||
func TestPrepareForRestore_VMDiskWithoutKeepDeletesOriginalPVC(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
}
|
||||
target := restoreTarget{Namespace: ns, AppName: "data", AppKind: vmDiskAppKind}
|
||||
pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-data", Namespace: ns}}
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t,
|
||||
[]runtime.Object{makeUnstructuredHelmRelease("vm-disk-data", ns, nil), makeUnstructuredDataVolume("vm-disk-data", ns)},
|
||||
restoreJob.DeepCopy(), backup, pvc)
|
||||
ctx := context.Background()
|
||||
|
||||
ready := false
|
||||
for pass := 0; pass < 5 && !ready; pass++ {
|
||||
var err error
|
||||
if ready, _, err = r.prepareForRestore(ctx, restoreJob.DeepCopy(), backup, nil, target, RestoreOptions{KeepOriginalPVC: new(false)}); err != nil {
|
||||
t.Fatalf("prepareForRestore: %v", err)
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("preparation did not finish")
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}); !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("original PVC still present (err %v); Velero would skip restoring it", err)
|
||||
}
|
||||
pvcs := &corev1.PersistentVolumeClaimList{}
|
||||
if err := r.List(ctx, pvcs, client.InNamespace(ns)); err != nil {
|
||||
t.Fatalf("list PVCs: %v", err)
|
||||
}
|
||||
for _, p := range pvcs.Items {
|
||||
t.Errorf("unexpected PVC %s left behind without keepOriginalPVC", p.Name)
|
||||
}
|
||||
if _, err := r.Resource(dataVolumeGVR).Namespace(ns).Get(ctx, "vm-disk-data", metav1.GetOptions{}); !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("DataVolume still present (err %v)", err)
|
||||
}
|
||||
hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, "vm-disk-data", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get HelmRelease: %v", err)
|
||||
}
|
||||
if suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend"); !suspended {
|
||||
t.Fatal("the disk's HelmRelease was not suspended")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed PreparationStarted patch must leave the target and the in-memory
|
||||
// condition untouched and retry, so a later failure does not report leftovers.
|
||||
func TestPrepareForRestore_PreparationStartedPatchFailureRetries(t *testing.T) {
|
||||
f := newKeepPVCFixture(t)
|
||||
r := f.reconciler(t, f.livePVC, f.pv)
|
||||
r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{
|
||||
SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error {
|
||||
return errors.New("the object has been modified")
|
||||
},
|
||||
})
|
||||
ctx := context.Background()
|
||||
ready, res, err := r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
|
||||
if err != nil || ready || res.RequeueAfter == 0 {
|
||||
t.Fatalf("prepareForRestore = ready %v, requeue %v, err %v; want a retry", ready, res.RequeueAfter, err)
|
||||
}
|
||||
if meta.FindStatusCondition(f.restoreJob.Status.Conditions, restoreCondPreparationStarted) != nil {
|
||||
t.Errorf("PreparationStarted kept in memory after the patch failed: %+v", f.restoreJob.Status.Conditions)
|
||||
}
|
||||
if err := r.Get(ctx, client.ObjectKeyFromObject(f.livePVC), &corev1.PersistentVolumeClaim{}); err != nil {
|
||||
t.Errorf("original PVC touched although preparation did not start: %v", err)
|
||||
}
|
||||
if !f.dataVolumeExists(t, r) {
|
||||
t.Error("DataVolume deleted although preparation did not start")
|
||||
}
|
||||
}
|
||||
|
||||
// A Pod stuck terminating holds the disk before preparation changes anything,
|
||||
// so the preparation deadline does not cover it; the wait needs its own.
|
||||
func TestPrepareForRestore_VMDiskLeavingPodWaitHasDeadline(t *testing.T) {
|
||||
const ns = "tenant-root"
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns},
|
||||
Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: new("apps.cozystack.io"), Kind: vmDiskAppKind, Name: "data",
|
||||
}},
|
||||
}
|
||||
target := restoreTarget{Namespace: ns, AppName: "data", AppKind: vmDiskAppKind}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
age time.Duration
|
||||
wantFail bool
|
||||
}{
|
||||
{name: "within the deadline it waits", age: time.Minute},
|
||||
{name: "past the deadline it fails", age: restorePreparationDeadline + time.Minute, wantFail: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
started := metav1.NewTime(time.Now().Add(-tc.age))
|
||||
restoreJob := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: ns},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started},
|
||||
}
|
||||
pod := mountingVMDiskPod("virt-launcher-web", true)
|
||||
pod.Namespace = ns
|
||||
r := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{makeUnstructuredHelmRelease("vm-disk-data", ns, nil)}, restoreJob.DeepCopy(), backup, pod)
|
||||
ready, res, err := r.prepareForRestore(context.Background(), restoreJob, backup, nil, target, RestoreOptions{KeepOriginalPVC: new(true)})
|
||||
if ready {
|
||||
t.Fatal("prepareForRestore went ahead while a Pod still holds the disk")
|
||||
}
|
||||
if tc.wantFail {
|
||||
if err == nil || !strings.Contains(err.Error(), "still held the disk") {
|
||||
t.Fatalf("err = %v, want a failure naming the Pod that still holds the disk", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || res.RequeueAfter == 0 {
|
||||
t.Fatalf("requeue %v, err %v; want a requeue while within the deadline", res.RequeueAfter, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user