From a042f335900b0a2340d27d3a235bb7ea2748c0ec Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:49:35 +0000 Subject: [PATCH 01/22] fix(mariadb): record the dump a Backup was taken as Every run of a MariaDB application writes its dump under the same storage prefix, and the Backup did not say which of those dumps is its own. A restore had nothing to give mariadb-operator but the operator Backup, which resolves to the newest dump under the prefix. The Backup now records the dump's file name and the timestamp in that name in driverMetadata, taken from the match the size probe already makes. Nothing is recorded when that match cannot prove the dump belongs to the operator Backup, so a Backup never names another run's dump. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../backupcontroller/mariadb_artifact_size.go | 16 ++-- .../mariadb_artifact_size_test.go | 95 ++++++++++++++++--- .../mariadbstrategy_controller.go | 14 ++- 3 files changed, 104 insertions(+), 21 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 6397325ca..950303d78 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -39,19 +39,19 @@ type mariadbDump struct { LastModified time.Time } -// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for -// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump -// cannot be found or told apart: an unknown size must not be reported as a -// measured one. -func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { +// attributedMariaDBDump returns the dump mariadb-operator uploaded for +// mdbBackup. ok is false when the dump cannot be found or told apart, and +// the Backup then reports neither a size nor a dump: an unknown dump must +// not be reported as a measured one. +func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (dump mariadbDump, ok bool) { ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) if err != nil { - getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) - return 0 + getLogger(ctx).Info("MariaDB dump not attributed; leaving its size and name unset", "backup", mdbBackup.Name, "error", err) + return mariadbDump{}, false } - return dump.Size + return dump, true } // findMariaDBDump lists the bucket mdbBackup wrote to and returns the one diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 9d649c15c..80e9e4617 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -29,6 +29,14 @@ func dumpKey(prefix string, ts time.Time, compression string) string { return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) } +// probedSize is the size a Backup reports for a probe result. +func probedSize(dump mariadbDump, attributed bool) int64 { + if !attributed { + return 0 + } + return dump.Size +} + // mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a // Backup whose Job succeeded. A failed Job also gets Status=True, with // Reason=JobFailed, so a fixture without the reason is not a success. @@ -105,7 +113,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) + got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)) if got != 4242 { t.Errorf("size = %d, want 4242", got) } @@ -130,7 +138,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed)); got != 3 { t.Errorf("size = %d, want 3", got) } if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { @@ -148,7 +156,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed)); got != 4 { t.Errorf("size = %d, want 4", got) } if l.conn.SessionToken != "TOKEN" { @@ -160,7 +168,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -173,7 +181,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) } }) @@ -182,7 +190,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 5 { t.Errorf("size = %d, want 5", got) } }) @@ -191,7 +199,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } withShortProbeTimeout(t, func() { - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -202,7 +210,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -211,14 +219,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -226,7 +234,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -481,7 +489,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { s3.Prefix = "tenant/db/" s3.Region = "us-east-1" - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute))); got != 31337 { t.Fatalf("size = %d, want 31337", got) } if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { @@ -523,6 +531,69 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { } } +// TestCreateMariaDBBackupArtifact_RecordsDump pins that the Backup names +// its dump only when the probe attributed one to the operator Backup. +func TestCreateMariaDBBackupArtifact_RecordsDump(t *testing.T) { + apps := mariadbapp.GroupName + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := metav1.NewTime(created.Add(3 * time.Minute)) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + own := s3ObjectInfo{Key: "src/backup.2026-10-21T10:01:00Z.gzip.sql", Size: 2048, LastModified: created.Add(2 * time.Minute)} + second := s3ObjectInfo{Key: "src/backup.2026-10-21T10:02:00Z.gzip.sql", Size: 1, LastModified: created.Add(2 * time.Minute)} + offset := s3ObjectInfo{Key: "src/backup.2026-10-21T14:01:00+04:00.sql", Size: 7, LastModified: created.Add(2 * time.Minute)} + + cases := []struct { + name string + lister *fakeS3Lister + wantFile, wantTime string + }{ + {name: "an attributed dump is recorded", lister: &fakeS3Lister{objects: []s3ObjectInfo{own}}, + wantFile: "backup.2026-10-21T10:01:00Z.gzip.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "a dump named with an offset is timed in UTC", lister: &fakeS3Lister{objects: []s3ObjectInfo{offset}}, + wantFile: "backup.2026-10-21T14:01:00+04:00.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "two dumps in the lifetime record none", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second}}}, + {name: "no dump in the lifetime records none", lister: &fakeS3Lister{}}, + {name: "a failed listing records none", lister: &fakeS3Lister{err: errors.New("AccessDenied")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, created) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-dump"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, tc.lister, job, s3CredsSecret("tenant", "creds")) + if _, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered); err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + persisted := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-dump"}, persisted); err != nil { + t.Fatalf("get Backup: %v", err) + } + md := persisted.Spec.DriverMetadata + file, hasFile := md[mariadbDumpFileKey] + at, hasTime := md[mariadbDumpTimeKey] + if tc.wantFile == "" { + if hasFile || hasTime { + t.Errorf("driverMetadata = %v; want no dump recorded", md) + } + return + } + if file != tc.wantFile || at != tc.wantTime { + t.Errorf("recorded dump %q at %q; want %q at %q", file, at, tc.wantFile, tc.wantTime) + } + }) + } +} + // TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the // probe reads where the operator Backup wrote its dump, not where the // strategy, edited while the backup ran, now points. diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 4d88ab8a1..50c871db3 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -47,6 +47,14 @@ const ( mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace" mariadbStorageURIKey = "k8s.mariadb.com/storage-uri" + // The dump the operator Backup wrote: its file name under the storage + // prefix, and the RFC3339 timestamp in that name. Every run of an + // application writes under the same prefix, so only this name tells its + // dump from the others. Recorded only when findMariaDBDump proved the + // dump belongs to the operator Backup. + mariadbDumpFileKey = "k8s.mariadb.com/dump-file" + mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" + // Polling cadence for the operator Backup/Restore lifecycle. mariadbPollInterval = 5 * time.Second @@ -398,7 +406,11 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} if probe { - status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) + if dump, ok := r.attributedMariaDBDump(ctx, mdbBackup, takenAt.Time); ok { + status.Artifact.SizeBytes = dump.Size + driverMD[mariadbDumpFileKey] = dump.Name + driverMD[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + } } } backup := &backupsv1alpha1.Backup{ From 21d1e9f02b4dafeefab3acece372cc8f077b7701 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:53:38 +0000 Subject: [PATCH 02/22] fix(mariadb): restore the dump a Backup records A RestoreJob created a mariadb-operator Restore that named only the operator Backup. The operator then lists the prefix that Backup wrote to and restores the dump nearest to the moment the restore started: the application's newest dump, not the one the Backup was taken as. When the Backup records its dump, the Restore now carries the timestamp in that dump's name as targetRecoveryTime. mariadb-operator returns the dump whose timestamp equals the target before it looks for the nearest one, and the nearest one is exactly what it would restore if the recorded dump were gone. So the controller first checks, with its own bounded call, that the dump is still in the operator Backup's storage. A dump that is gone fails the RestoreJob, naming the dump, instead of restoring a neighbour in its place. A check that cannot be made keeps the RestoreJob waiting with Ready=False until the restore deadline, then fails it with the cause: the Backup could record its dump only because the controller reached that storage when it was taken, so an unreachable storage is not a reason to guess. The dump is chosen once, as the operator Restore is created; a Restore that already exists is followed as before. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- internal/backupcontroller/artifact_size.go | 4 + .../backupcontroller/mariadb_artifact_size.go | 52 ++- .../mariadb_restore_dump_test.go | 304 ++++++++++++++++++ .../mariadbstrategy_controller.go | 111 ++++++- .../mariadbstrategy_controller_test.go | 19 +- .../backupcontroller/restorejob_controller.go | 5 + 6 files changed, 467 insertions(+), 28 deletions(-) create mode 100644 internal/backupcontroller/mariadb_restore_dump_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 941b00323..9197f07be 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -90,6 +90,10 @@ func (r *BackupJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *RestoreJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 950303d78..a61ec4a43 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,22 +60,14 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - s3 := mdbBackup.Spec.Storage.S3 - if s3 == nil { - return mariadbDump{}, errors.New("the operator Backup does not store to S3") + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return mariadbDump{}, err } others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } - conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) - if err != nil { - return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) - } - lister, err := store.lister(conn) - if err != nil { - return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) - } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same // name shares it and a recursive listing would walk the WAL archive. @@ -88,6 +80,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt return matchMariaDBDump(objects, prefix, own, others) } +// mariadbDumpExists reports whether the dump file name is still in the +// storage mdbBackup wrote to. +func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) { + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return false, err + } + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + for _, obj := range objects { + if obj.Key == key { + return true, nil + } + } + return false, nil +} + +// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials +// its own spec names. +func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return nil, nil, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return nil, nil, fmt.Errorf("build S3 client: %w", err) + } + return lister, s3, nil +} + // mariadbRun is the span in which an operator Backup can have written its // dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. type mariadbRun struct { diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go new file mode 100644 index 000000000..0375def33 --- /dev/null +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// The source run of the restore fixture: operator Backup op-bk of +// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing +// under tenant/src/ with mdbRestoreDump as its dump. +var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + +const ( + mdbRestorePrefix = "tenant/src/" + mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql" + mdbRestoreTime = "2026-10-21T10:00:05Z" + mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql" +) + +type mariadbRestoreFixture struct { + c client.Client + r *RestoreJobReconciler + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup + src *mariadbtypes.Backup + lister *fakeS3Lister + events *record.FakeRecorder +} + +// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose +// driverMetadata is driverMD plus the operator Backup's name, into the +// existing application target. extra objects are seeded too. +func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture { + t.Helper() + apps := mariadbapp.GroupName + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/src" + src := operatorBackupStoringTo(t, s3, mdbRestoreT0) + src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))} + + md := map[string]string{mariadbBackupNameKey: src.Name} + for k, v := range driverMD { + md[k] = v + } + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)), + DriverMetadata: md, + }, + } + started := metav1.Now() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: backup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}} + objs := append([]client.Object{src, backup, rj, target}, extra...) + c := newMariaDBStrategyTestClient(t, objs...) + fetched := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + events := record.NewFakeRecorder(10) + return &mariadbRestoreFixture{ + c: c, + rj: fetched, + backup: backup, + src: src, + lister: lister, + events: events, + r: &RestoreJobReconciler{ + Client: c, + Recorder: events, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + lister.conn = conn + return lister, nil + }, + }, + } +} + +func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result { + t.Helper() + res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + return res +} + +func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore { + t.Helper() + list := &mariadbtypes.RestoreList{} + if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { + t.Fatalf("list operator Restores: %v", err) + } + return list.Items +} + +// wantFailed asserts the RestoreJob failed with a message holding every +// fragment, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) { + t.Helper() + if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase) + } + for _, s := range fragments { + if !strings.Contains(f.rj.Status.Message, s) { + t.Errorf("message %q does not contain %q", f.rj.Status.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantWaiting asserts the RestoreJob is kept waiting on the dump check +// with Ready=False saying why, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) { + t.Helper() + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message) + } + if res.RequeueAfter != mariadbDumpCheckRetryInterval { + t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval) + } + persisted := &backupsv1alpha1.RestoreJob{} + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" { + t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond) + } + for _, s := range fragments { + if !strings.Contains(cond.Message, s) { + t.Errorf("Ready message %q does not contain %q", cond.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantRestore asserts exactly one operator Restore of op-bk into target, +// asking for targetTime. +func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) { + t.Helper() + got := f.restores(t) + if len(got) != 1 { + t.Fatalf("created %d operator Restores; want one", len(got)) + } + spec := got[0].Spec + if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") { + t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target")) + } + if spec.TargetRecoveryTime != targetTime { + t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime) + } +} + +func recordedDump(name string) map[string]string { + return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} +} + +func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + + t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}, + {Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" { + t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump) + } + }) + + t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix) + }) + + t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.") + }) + + t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials") + if l.calls != 0 { + t.Errorf("S3 listed %d times without credentials", l.calls) + } + }) + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var res ctrl.Result + var err error + withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error()) + }) + + t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute)) + f.rj.Status.StartedAt = &long + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after") + }) + + t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds) + f.reconcile(t) + f.wantFailed(t, `"dump.sql"`) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a name that is not a dump", l.calls) + } + }) + + t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing) + res := f.reconcile(t) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls) + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval { + t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter) + } + if got := f.restores(t); len(got) != 1 { + t.Errorf("%d operator Restores; want only the existing one", len(got)) + } + }) +} + +func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds")) + f.reconcile(t) + f.wantRestore(t, "") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls) + } +} + +// withShortDumpCheckTimeout shortens the restore's dump check timeout and +// fails the test if fn is not back well before the default would expire. +func withShortDumpCheckTimeout(t *testing.T, fn func()) { + t.Helper() + saved := mariadbRestoreDumpCheckTimeout + mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond + defer func() { mariadbRestoreDumpCheckTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the dump check did not give up at its timeout") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 50c871db3..86b42d33f 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -50,8 +51,9 @@ const ( // The dump the operator Backup wrote: its file name under the storage // prefix, and the RFC3339 timestamp in that name. Every run of an // application writes under the same prefix, so only this name tells its - // dump from the others. Recorded only when findMariaDBDump proved the - // dump belongs to the operator Backup. + // dump from the others, and a restore asks for exactly that dump. + // Recorded only when findMariaDBDump proved the dump belongs to the + // operator Backup. mariadbDumpFileKey = "k8s.mariadb.com/dump-file" mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" @@ -68,6 +70,12 @@ const ( // to terminate. Tenants override via spec.options.restoreTimeoutSeconds. mariadbDefaultRestoreDeadline = 30 * time.Minute + // Retry cadence while the dump a restore asks for cannot be checked. + // Longer than mariadbPollInterval because each attempt can hold the + // RestoreJob controller's single worker for + // mariadbRestoreDumpCheckTimeout. + mariadbDumpCheckRetryInterval = 30 * time.Second + // mariadbBackupSnapshotKind is the Kind stamped onto the snapshot // persisted in Backup.status.underlyingResources. The same shape carries // the storage descriptor and the rendered strategy parameters used at @@ -81,6 +89,12 @@ const ( // existing API surface. var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String() +// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a +// restore makes before it creates the operator Restore. The RestoreJob +// controller has a single worker, so an unreachable endpoint stalls every +// other RestoreJob for this long. A variable so tests can shorten it. +var mariadbRestoreDumpCheckTimeout = 10 * time.Second + // mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a // cozystack MariaDB application instance. The mapping mirrors the mariadb // ApplicationDefinition (release.prefix=mariadb-). @@ -582,10 +596,25 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest return ctrl.Result{}, err } - mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName) + mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) } + // The dump is chosen once, as the operator Restore is created; one that + // already exists has made its choice. + if mdbRestore == nil { + targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup) + var refused mariadbRestoreRefused + if errors.As(err, &refused) { + return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) + } + if err != nil { + return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err) + } + if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + } + } cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { @@ -626,10 +655,68 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } } -// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with -// the RestoreJob, or returns the existing one if a previous reconcile -// already created it. -func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) { +// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be +// restored. It fails the RestoreJob, where any other error from choosing +// the dump is retried. +type mariadbRestoreRefused string + +func (e mariadbRestoreRefused) Error() string { return string(e) } + +// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator +// Restore for backup: the timestamp in the name of the dump backup +// records. mariadb-operator restores the dump whose timestamp equals the +// target and, when there is none, the nearest one, so the dump is first +// checked to still exist. Empty when backup records no dump, which leaves +// the operator to take the newest dump under the prefix. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name == "" { + return "", nil + } + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() + exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) + if err != nil { + return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) + } + if !exists { + s3 := src.Spec.Storage.S3 + return "", mariadbRestoreRefused(fmt.Sprintf( + "dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump", + name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix))) + } + return ts.UTC().Format(time.RFC3339), nil +} + +// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False +// saying why, while the dump it asks for cannot be checked, and fails it +// with the cause once the restore deadline has passed. +func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) { + deadline := options.effectiveRestoreDeadline() + if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf( + "%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline)) + } + getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause) + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DumpCheckPending", + Message: cause.Error(), + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil +} + +// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled +// with the RestoreJob, if a previous reconcile already created one. +func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} if err := r.List(ctx, list, client.InNamespace(rj.Namespace), @@ -640,10 +727,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac ); err != nil { return nil, err } - if len(list.Items) > 0 { - return &list.Items[0], nil + if len(list.Items) == 0 { + return nil, nil } + return &list.Items[0], nil +} +// createMariaDBRestore creates the k8s.mariadb.com/Restore of the +// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty. +func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) { obj := &mariadbtypes.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: rj.Namespace, @@ -659,6 +751,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac Kind: mariadbtypes.BackupKind, Name: sourceBackupName, }, + TargetRecoveryTime: targetRecoveryTime, }, } if err := r.Create(ctx, obj); err != nil { diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index 0e4f63b5e..fa033fb22 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -313,7 +313,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) { // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- -func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { +func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newMariaDBRestoreJob("rj-1", "tenant") @@ -321,16 +321,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { t.Fatalf("seed RestoreJob: %v", err) } - first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") - if err != nil { - t.Fatalf("first ensureMariaDBRestore: %v", err) + if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil { + t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err) } - second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") + created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z") if err != nil { - t.Fatalf("second ensureMariaDBRestore: %v", err) + t.Fatalf("createMariaDBRestore: %v", err) } - if first.Name != second.Name { - t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) + found, err := r.findMariaDBRestoreForJob(context.Background(), rj) + if err != nil || found == nil || found.Name != created.Name { + t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err) } list := &mariadbtypes.RestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { @@ -346,6 +346,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind { t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef) } + if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" { + t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime) + } } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..db7348923 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -57,6 +57,11 @@ type RestoreJobReconciler struct { // see a peer's StartedAt before the cache does. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to check that a + // backup's object still exists; nil means a real minio client. Set by + // unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { From 195de033e8156876885d685cd1cced3d2dfb1142 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:55:36 +0000 Subject: [PATCH 03/22] fix(mariadb): pin legacy Backups to a proven dump, refuse failed runs A Backup taken before Backups recorded their dump gives a restore nothing to pin. Refusing such Backups would leave every existing one unrestorable, so a restore of one now makes the same ownership-proven match the backup side makes, against the operator Backup it names, and restores that dump by its timestamp. When no dump can be proven to be the run's own, or the storage cannot be read, mariadb-operator restores the newest dump under the prefix as it always did, and a BestEffortDump Warning on the RestoreJob says so and why. An operator Backup whose Job failed is Complete=True with reason JobFailed, and a BackupJob that took Complete=True for success made a Backup of it. That run left no dump of its own, and restoring it would quietly bring back another run's dump, so the RestoreJob now fails instead. The condition is read by its reason, because its status does not tell a failure from a success. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../mariadb_restore_dump_test.go | 129 +++++++++++++++++- .../mariadbstrategy_controller.go | 41 +++++- 2 files changed, 156 insertions(+), 14 deletions(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 0375def33..6c8202adf 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -274,13 +274,128 @@ func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { }) } -func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) { - l := &fakeS3Lister{err: errors.New("Access Denied.")} - f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds")) - f.reconcile(t) - f.wantRestore(t, "") - if l.calls != 0 { - t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls) +// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no +// dump: one proven to belong to the source run is restored by its +// timestamp, and anything else restores the newest dump with a Warning. +func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)} + newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)} + second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)} + + t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if ev := drainEvents(f.events); len(ev) != 0 { + t.Errorf("events %q; want none for a proven dump", ev) + } + }) + + cases := []struct { + name string + lister *fakeS3Lister + cause string + }{ + {name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"}, + {name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"}, + {name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."}, + } + wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) { + t.Helper() + f.wantRestore(t, "") + ev := drainEvents(f.events) + if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") { + t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause) + } + } + for _, tc := range cases { + t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, tc.lister, creds) + f.reconcile(t) + wantBestEffort(t, f, tc.cause) + }) + } + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var err error + withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + wantBestEffort(t, f, context.DeadlineExceeded.Error()) + }) +} + +// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an +// operator Backup whose Job failed: there is no dump of its own to +// restore, whatever the Backup records. +func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + failed := func(status metav1.ConditionStatus) []metav1.Condition { + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed, + Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)), + }} + } + listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}} + + cases := []struct { + name string + driverMD map[string]string + status metav1.ConditionStatus + }{ + {name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue}, + {name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue}, + {name: "Complete=False/JobFailed", status: metav1.ConditionFalse}, + } + for _, tc := range cases { + t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: listing} + f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds) + f.src.Status.Conditions = failed(tc.status) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a failed backup run", l.calls) + } + }) + } + + t.Run("an operator Restore created earlier is followed", func(t *testing.T) { + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing) + f.src.Status.Conditions = failed(metav1.ConditionTrue) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message) + } + }) +} + +func drainEvents(r *record.FakeRecorder) []string { + var out []string + for { + select { + case e := <-r.Events: + out = append(out, e) + default: + return out + } } } diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 86b42d33f..6d6312eb4 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -603,7 +603,7 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice. if mdbRestore == nil { - targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup) + targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup) var refused mariadbRestoreRefused if errors.As(err, &refused) { return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) @@ -666,19 +666,26 @@ func (e mariadbRestoreRefused) Error() string { return string(e) } // Restore for backup: the timestamp in the name of the dump backup // records. mariadb-operator restores the dump whose timestamp equals the // target and, when there is none, the nearest one, so the dump is first -// checked to still exist. Empty when backup records no dump, which leaves -// the operator to take the newest dump under the prefix. -func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { +// checked to still exist. A Backup that records no dump is resolved by +// legacyMariaDBRestoreTargetTime. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // A failed Job is Complete=True too, with reason JobFailed, and a + // BackupJob that took Complete=True for success made a Backup of it. + if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { + return "", mariadbRestoreRefused(fmt.Sprintf( + "the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump", + src.Namespace, src.Name, cond.Status, cond.Reason)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() name := backup.Spec.DriverMetadata[mariadbDumpFileKey] if name == "" { - return "", nil + return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil } ts, ok := parseMariaDBLogicalBackupFile(name) if !ok { return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) } - ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) - defer cancel() exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) if err != nil { return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) @@ -692,6 +699,26 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, bac return ts.UTC().Format(time.RFC3339), nil } +// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before +// Backups recorded theirs: the one findMariaDBDump proves belongs to src, +// with the completion time the size probe was given, which takenAt keeps. +// When none is proven, for any reason, it returns empty, and the operator +// restores the newest dump under the prefix as such a restore always did: +// refusing would leave these Backups unrestorable, and the controller may +// never have been able to reach their storage. A Warning event says the +// dump is a best effort. +func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string { + dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time) + if err != nil { + getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err) + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump", + "Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's", + backup.Name, src.Namespace, src.Name, err) + return "" + } + return dump.Timestamp.UTC().Format(time.RFC3339) +} + // awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False // saying why, while the dump it asks for cannot be checked, and fails it // with the cause once the restore deadline has passed. From 55f1fd33a8dd8b73c196d4b45bb328e4c1791a74 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:04 +0300 Subject: [PATCH 04/22] fix(backups): let the controller delete operator MariaDB Backups Deleting a MariaDB Backup now deletes the k8s.mariadb.com/Backup that wrote its dump, once the dump itself is gone, so the controller needs the delete verb on that resource. Restores stay create-only. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../templates/rbac.yaml | 11 ++++---- .../tests/mariadb_cleanup_rbac_test.yaml | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..e9e35b066 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -146,15 +146,16 @@ rules: # MariaDB strategy: read the operator-side MariaDB CR to surface "not yet # rendered" preconditions, create one-shot k8s.mariadb.com/Backup CRs on # the BackupJob path, and create k8s.mariadb.com/Restore CRs on the -# RestoreJob path. The driver only ever Creates/Lists/Gets these objects - -# it does not patch or delete them; their lifecycle is owned by -# mariadb-operator and (for cleanup) by tenants / explicit teardown flows, -# never by this controller. +# RestoreJob path. A Backup is deleted once the dump it wrote is, when its +# Cozystack Backup is deleted; a Restore is never deleted. - apiGroups: ["k8s.mariadb.com"] resources: ["mariadbs"] verbs: ["get", "list", "watch"] - apiGroups: ["k8s.mariadb.com"] - resources: ["backups", "restores"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] verbs: ["get", "list", "watch", "create"] # MongoDB strategy: read the operator-side PerconaServerMongoDB CR to gate on # the backup preconditions, create one-shot PerconaServerMongoDBBackup CRs on the diff --git a/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml new file mode 100644 index 000000000..21ac6e807 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml @@ -0,0 +1,28 @@ +suite: RBAC of the MariaDB dump cleanup + +# Deleting a MariaDB Backup deletes its operator Backup once the dump is gone. +# The Go unit tests run with no authorization, so a missing verb only shows up +# on a cluster, where every such Backup would stay Terminating. + +templates: + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "ClusterRole can delete operator Backups but not Restores" + asserts: + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] + verbs: ["get", "list", "watch", "create"] From f147b22d86343eb25aa8ff88651efc5ce7993a95 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:11 +0300 Subject: [PATCH 05/22] fix(backups): delete a MariaDB dump when its Backup is deleted Deleting a MariaDB Backup left its dump in the bucket for good: mariadb-operator's maxRetention prunes only from inside the next run of the same prefix, and a Plan never deletes a Backup, so on the platform bucket nothing else ever removes it. Every run of an application writes under one prefix, so the key deleted has to be proven to be this Backup's. It is the dump the Backup records, or, for a Backup taken before dumps were recorded, the one the existing ownership match attributes to its operator Backup. That match is recorded on the Backup before the delete, so a retry deletes the same key by name instead of matching again against a bucket that no longer holds it. Without proof the Backup is released with a Warning and nothing is deleted, the operator Backup included, since any later attribution starts from it. The operator Backup is deleted only after its dump, and the delete waits while a RestoreJob still restores from the Backup. The delete runs in the controller through the same minio client the size probe uses, so it verifies TLS the way mariadb-operator did when it wrote the dump. A dump on the platform bucket, which the operator Backup names by its bucket and by the projected credentials Secret, is reached through the platform's current endpoint and keys, still trusting a CA the strategy pinned, so a Backup taken before the endpoint moved is still deleted. Platform credentials that cannot be read fall back to the ones the operator Backup names. A failed delete holds the Backup and retries with a Warning naming the skip-artifact-cleanup annotation, except in a terminating namespace, where the Backup is released with a Warning so it does not hold the namespace. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 38 +- .../backupcontroller/backup_controller.go | 33 +- .../backup_controller_test.go | 7 +- .../backupcontroller/mariadb_artifact_size.go | 15 +- .../mariadbstrategy_cleanup.go | 268 ++++++ .../mariadbstrategy_cleanup_test.go | 886 ++++++++++++++++++ 6 files changed, 1223 insertions(+), 24 deletions(-) create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup.go create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 9197f07be..40d827c62 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -38,6 +38,11 @@ type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key; a key that does not exist is not an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -63,8 +68,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -74,6 +81,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +112,10 @@ func (r *RestoreJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +125,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -152,6 +182,10 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..f817e696b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,16 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +133,18 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and the tenant deleting the Backup is asking for exactly that + // (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +157,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index a61ec4a43..f9b9973ad 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,11 +60,22 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + lister, _, err := mariadbStorage(ctx, store, mdbBackup) if err != nil { return mariadbDump{}, err } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..483ded502 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + if name == "" { + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// activeRestoreOf returns the name of a RestoreJob that restores from +// backup and has not finished, or "". +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return rj.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials cannot be read, is reached as op recorded it; a +// delete that fails there is handled by retryMariaDBCleanup like any other. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + switch { + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..745f7aef4 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket answers only when its context ends, as an unreachable +// endpoint does. +type blockingBucket struct{ fakeS3Bucket } + +func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { + <-ctx.Done() + return ctx.Err() +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + bucket := &blockingBucket{} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not give up on an unreachable endpoint") + } +} + +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +} From e66b764734a505a85009922c36629a4d674512c6 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:21:22 +0300 Subject: [PATCH 06/22] fix(backups): let an unfinished MariaDB run block a legacy dump delete A Backup taken before dumps were recorded finds its dump by window, and only finished neighbour runs were allowed to contest a candidate. A run uploads its dump and prunes older ones through maxRetention before its operator Backup turns Complete, so in that gap the pruned run's cleanup saw the newer run's dump as the only candidate and deleted it, leaving that Backup Ready with nothing to restore. The size probe and the restore keep ignoring unfinished runs, where a wrong guess costs a size or a refused restore. Deletion now treats a run that has not completed as one whose window is still open, so the dump is left behind with a Warning, as for any other unproven attribution. Holding the Backup would not help: once that run completes its window still covers the candidate, and the attribution stays ambiguous. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/mariadb_artifact_size.go | 37 ++++++++----- .../mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 52 +++++++++++++++++++ 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index f9b9973ad..4e0525517 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -64,18 +64,19 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt if err != nil { return mariadbDump{}, err } - return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false) } // findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, // for a caller that reaches the bucket by another endpoint than the one -// mdbBackup names. -func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { +// mdbBackup names. withUnfinished lets runs that have not completed contest +// the dump too, as neighbourMariaDBRuns describes. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) { s3 := mdbBackup.Spec.Storage.S3 if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } - others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) + others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } @@ -136,6 +137,10 @@ type mariadbRun struct { lower, upper time.Time } +// mariadbUnfinishedRunEnd closes the window of a run that has not completed, +// which can still write its dump at any time. +var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC) + func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { return mariadbRun{ name: b.Namespace + "/" + b.Name, @@ -161,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { // dump in the window on one clock only may be this run's own, misplaced by // a skewed clock, which leaves any other candidate unproven. A neighbour's // dump is still taken if this run's own is missing from the listing, or -// misplaced on both clocks, and the neighbour's operator Backup is gone or -// has not completed. +// misplaced on both clocks, and the neighbour's operator Backup is gone or, +// unless others includes unfinished runs, has not completed. func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { var near []mariadbDump for _, obj := range objects { @@ -194,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth return d, nil } -// finishedMariaDBRuns returns the runs of the other operator Backups that +// neighbourMariaDBRuns returns the runs of the other operator Backups that // write under the same bucket and prefix as b and have completed, whether -// their Job succeeded or failed. One still running is left out: its window -// has no end yet, and an operator Backup stuck short of Complete, as one -// whose Job pod never starts is, would otherwise contest every later run of -// the application for good. The endpoint is not compared, since one has -// several spellings, and a false neighbour only leaves a size unset. -func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { +// their Job succeeded or failed. One still running is left out unless +// withUnfinished is set: its window has no end yet, and an operator Backup +// stuck short of Complete, as one whose Job pod never starts is, would +// otherwise contest every later run of the application for good. That is +// the price of deleting a dump, since a run can upload its dump and prune +// older ones through maxRetention before it completes. The endpoint is not +// compared, since one has several spellings, and a false neighbour only +// keeps a dump unattributed. +func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) { list := &mariadbtypes.BackupList{} if err := reader.List(ctx, list); err != nil { return nil, err @@ -218,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty } cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) if cond == nil || cond.Status != metav1.ConditionTrue { + if withUnfinished { + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd)) + } continue } runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 483ded502..0a2fc1e34 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -117,7 +117,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 745f7aef4..9b41f4701 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -385,6 +385,58 @@ func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased( assertReleasedUntouched(t, r, rec, bucket, res, err) } +// A run started before this one completed can upload its dump and prune +// this run's through maxRetention before its own operator Backup completes. +// Its dump is then the only one in this run's window, and only its open +// window says it may be its own. +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpAnUnfinishedRunCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(2 * time.Minute) + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, started.Add(90*time.Second), "gzip"), LastModified: started.Add(100 * time.Second)}, + }} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(started) + running.Status.Conditions = conditions + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// An unfinished run contests only what it could have written: a dump +// that predates it, on both clocks, is still this run's. +func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + backup := cleanupBackup("") + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 0972690f0d31426f892e7d7e04ad48cf33d5e9c3 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 13:52:38 +0300 Subject: [PATCH 07/22] fix(backups): contest a recorded MariaDB dump before deleting it The size probe records the dump it attributes to a Backup, and it lets only runs that had already completed contest the attribution. A run can upload its dump and prune an older run's through maxRetention before its own operator Backup completes, so the probe of the pruned run can record the newer run's dump, and deleting that Backup then deleted a dump that another Backup still restores from. Deletion now contests the recorded dump against every run on the same bucket and prefix as they stand at deletion time, the unfinished ones included, and leaves it behind with a Warning when any could have written it, as the unrecorded path already does. A run that never completes therefore keeps every later dump its lifetime covers, which leaks storage rather than losing a backup. The probe is unchanged here. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 53 ++++- .../mariadbstrategy_cleanup_test.go | 215 ++++++++++++++++++ 2 files changed, 264 insertions(+), 4 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 0a2fc1e34..94a7d10f4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "time" @@ -37,7 +38,8 @@ var mariadbDumpCleanupTimeout = 10 * time.Second // cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then // the operator Backup that wrote it. Every run of an application writes // under one prefix, so the only key deleted is one proven to be this -// Backup's: the dump it records, or, for a Backup taken before dumps were +// Backup's: the dump it records, unless contestedMariaDBDump finds another +// run that could have written it, or, for a Backup taken before dumps were // recorded, the one findMariaDBDump attributes to its operator Backup. When // no dump is proven the Backup is released with a Warning and nothing is // deleted, the operator Backup included, since it is what any later @@ -112,11 +114,20 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) defer cancel() store := r.objectStore() - if name == "" { - lister, err := store.lister(conn) + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if name != "" { + other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) if err != nil { - return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + return r.retryMariaDBCleanup(ctx, backup, location+name, err) } + if other != "" { + return r.releaseMariaDBCleanup(ctx, backup, location+name, + fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil + } + } else { dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, @@ -151,6 +162,40 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac return ctrl.Result{}, nil } +// contestedMariaDBDump returns the name of another operator Backup on op's +// bucket and prefix whose run, finished or not, could have written the dump +// name, or "" when none could. The size probe that recorded name let only +// runs finished by then contest it, and a run can upload its dump and prune +// op's through maxRetention before it completes. A dump no longer in the +// bucket is contested by nobody: deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + } + s3 := op.Spec.Storage.S3 + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) + if i < 0 { + return "", nil + } + others, err := neighbourMariaDBRuns(ctx, reader, op, true) + if err != nil { + return "", fmt.Errorf("list operator Backups: %w", err) + } + d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} + for _, other := range others { + if other.placesOnEitherClock(d) { + return other.name, nil + } + } + return "", nil +} + // activeRestoreOf returns the name of a RestoreJob that restores from // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 9b41f4701..ddca34523 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -22,6 +22,7 @@ import ( strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) @@ -437,6 +438,220 @@ func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedR } } +// The size probe that records a dump lets only finished runs contest it, so +// a run still unfinished then, which uploaded its dump and pruned this +// run's through maxRetention, can have its dump recorded here. Deletion +// contests the recorded dump again, with every run as it stands now. +func TestCleanupMariaDBBackup_RecordedDumpARunUnfinishedAtTheProbeCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + "completed since": {mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted.Add(5 * time.Minute)))}, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(started) + other.Status.Conditions = conditions + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// Either clock placing the recorded dump in another run's lifetime is +// enough: the backup pod's clock can be skewed while the store's is not. +func TestCleanupMariaDBBackup_RecordedDumpAnotherRunPlacesOnOneClockIsReleased(t *testing.T) { + stamped := cleanupCreated.Add(20 * time.Second) + uploaded := cleanupCreated.Add(2 * time.Minute) + theirs := dumpKey(cleanupPrefix, stamped, "gzip") + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: uploaded}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(uploaded.Add(30 * time.Second)) + other.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// The listing is by prefix, so a longer key sharing it is not the dump and +// contests nothing. +func TestCleanupMariaDBBackup_RecordedDumpIgnoresAKeyItIsOnlyAPrefixOf(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key + ".tmp", LastModified: cleanupCompleted.Add(2 * time.Hour)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// A name the dump time cannot be read from leaves one clock out of the +// contest, so it is an error rather than a contest on half the evidence. +func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { + op := cleanupOperatorBackup(t) + name := "backup.not-a-time.sql" + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} + c := newMariaDBStrategyTestClient(t, op) + if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + t.Fatal("err = nil, want the unreadable dump time reported") + } +} + +// A retry after the dump was deleted but the operator Backup was not finds +// nothing to contest, whichever run's lifetime the dump's name falls in. +func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T) { + bucket := &fakeS3Bucket{} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCreated) + running.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted") + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no ArtifactNotDeleted", events) + } +} + +// A run started after the recorded dump was uploaded cannot have written it. +func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } +} + +func TestCleanupMariaDBBackup_RecordedDumpListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// Without the list of runs nothing is known about who else could have +// written the dump, so it is kept and the delete retried. +func TestCleanupMariaDBBackup_RecordedDumpWithUnreadableRunsRetries(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.BackupList); ok { + return errors.New("apiserver unavailable") + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// The whole sequence: run B uploads its dump and prunes this run's before +// B completes, this run's Backup is created and probed, then deleted. B's +// dump must survive whatever the probe recorded. +func TestMariaDBBackup_ProbeThenCleanupKeepsADumpAnUnfinishedRunUploaded(t *testing.T) { + apps := mariadbapp.GroupName + started := cleanupCreated.Add(time.Minute) + theirs := s3ObjectInfo{Key: dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip"), Size: 42, LastModified: started.Add(time.Minute)} + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "tenant/db" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "db", APIGroup: &apps}, + }, + } + other := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-other" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + + prober := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{theirs}}, job, other(), s3CredsSecret("tenant", "creds")) + created, err := prober.createMariaDBBackupArtifact(context.Background(), job, resolved, cleanupOperatorBackup(t), rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if got, want := created.Spec.DriverMetadata[mariadbDumpFileKey], strings.TrimPrefix(theirs.Key, cleanupPrefix); got != want { + t.Fatalf("probe recorded dump %q, want %q: this test covers the recorded path", got, want) + } + + backup := created.DeepCopy() + backup.ResourceVersion = "" + now := metav1.Now() + backup.DeletionTimestamp = &now + backup.Finalizers = []string{backupFinalizer} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{theirs}} + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), other(), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 74fbb34aca35beabe6c42a7c02f89fd3bc7a40e8 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 18:47:08 +0300 Subject: [PATCH 08/22] docs(backups): say who deletes a MariaDB Backup The comment justified deleting a dump in a tenant's own bucket by the tenant asking for it, but tenant roles grant no delete on Backups: only an administrator or the namespace teardown deletes one, and the reader of this branch should know whose request the delete answers. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index f817e696b..9d755059e 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -137,8 +137,8 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // backup run of the same prefix and a Plan never deletes a Backup, so // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns - // it, and the tenant deleting the Backup is asking for exactly that - // (see cleanupMariaDBBackup). + // it, and tenants cannot delete Backups, so the delete comes from an + // administrator or from namespace teardown (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 320d0299ad8e4ebb4d34568fdc561473f2808b86 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:03:59 +0300 Subject: [PATCH 09/22] fix(backups): refuse a MariaDB restore from a Backup being deleted Deleting a MariaDB Backup now deletes its dump, and mariadb-operator restores the dump nearest the requested time when the exact one is gone. A restore that starts while the Backup's cleanup runs can lose its dump between the check and the operator's read, and then completes on another run's data without any error. The restore now reads the Backup from the API server before creating the operator Restore and fails when it is being deleted or is gone. The copy the reconcile was handed comes from the cache and can predate the delete; a transient read error keeps the RestoreJob waiting like any other failed dump check. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 52 +++++++++++++++++++ .../mariadbstrategy_controller.go | 14 +++++ 2 files changed, 66 insertions(+) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 6c8202adf..218c91de6 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" @@ -184,6 +185,57 @@ func recordedDump(name string) map[string]string { return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} } +// A Backup being deleted is deleting its dump, so a restore that starts +// now can find it gone once the operator reads it, and the operator then +// restores the nearest other dump. The Backup is read from the API server, +// since the copy the reconcile was handed can predate the delete. +func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + present := func() *fakeS3Lister { + return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}} + } + cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil} + for name, md := range cases { + t.Run(name+", being deleted", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "being deleted") + }) + t.Run(name+", already gone", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "is gone") + }) + } + t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable") + }) +} + func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { creds := s3CredsSecret("tenant", "creds") diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 6d6312eb4..e3e00dbec 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -669,6 +669,20 @@ func (e mariadbRestoreRefused) Error() string { return string(e) } // checked to still exist. A Backup that records no dump is resolved by // legacyMariaDBRestoreTargetTime. func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // Deleting a Backup deletes its dump, and the operator restores the + // nearest other dump once its own is gone, so a restore must not start + // from a Backup being deleted. The copy in hand can predate the delete; + // the cleanup holds the Backup only for RestoreJobs it can already see. + live := &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { + if apierrors.IsNotFound(err) { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name)) + } + return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err) + } + if !live.DeletionTimestamp.IsZero() { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name)) + } // A failed Job is Complete=True too, with reason JobFailed, and a // BackupJob that took Complete=True for success made a Backup of it. if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { From 654c543264bbcba9c939a7c8b436ffbc02340c4d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:04:53 +0300 Subject: [PATCH 10/22] fix(backups): see a MariaDB restore the cache has not caught up with A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 15 +++++++++++++++ .../backupcontroller/mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 9d755059e..d8c69bf1c 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -43,6 +43,10 @@ type BackupReconciler struct { // cleanups project them into the Backup's namespace, and the MariaDB // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs of a Backup through it: a restore that started before + // the delete may not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the // MariaDB cleanup deletes a dump with; nil means a real minio client. @@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 94a7d10f4..531b806c9 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { list := &backupsv1alpha1.RestoreJobList{} - if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) } for i := range list.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ddca34523..20f2757d4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { } }) } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From 37a74e59394a69700e99b04a697f50b9a78e2625 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:05:51 +0300 Subject: [PATCH 11/22] test(backups): reach the S3 calls the MariaDB cleanup timeout bounds Contesting a recorded dump lists the bucket before the delete, and the timeout test supplied no lister, so a real client failed on DNS before the blocking delete was ever called and the suite stayed green with the timeout removed. The test now lists the dump through the fake bucket, blocks in the listing and in the delete in turn, and requires that the blocking call was entered. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 75 +++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 20f2757d4..0ce6b2d01 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -961,36 +961,69 @@ func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *tes } } -// blockingBucket answers only when its context ends, as an unreachable -// endpoint does. -type blockingBucket struct{ fakeS3Bucket } +// blockingBucket lists the dump it holds and answers the call named by +// blockOn only when its context ends, as an unreachable endpoint does. +type blockingBucket struct { + fakeS3Bucket + blockOn string + entered chan struct{} +} -func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { +func (b *blockingBucket) block(ctx context.Context, call string) bool { + if b.blockOn != call { + return false + } + close(b.entered) <-ctx.Done() - return ctx.Err() + return true +} + +func (b *blockingBucket) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + if b.block(ctx, "list") { + return nil, ctx.Err() + } + return b.fakeS3Bucket.ListObjects(ctx, bucket, prefix, recursive) +} + +func (b *blockingBucket) RemoveObject(ctx context.Context, bucket, key string) error { + if b.block(ctx, "remove") { + return ctx.Err() + } + return b.fakeS3Bucket.RemoveObject(ctx, bucket, key) } func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { prev := mariadbDumpCleanupTimeout mariadbDumpCleanupTimeout = 50 * time.Millisecond t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) - bucket := &blockingBucket{} - c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) - r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), - newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + for _, call := range []string{"list", "remove"} { + t.Run(call, func(t *testing.T) { + bucket := &blockingBucket{blockOn: call, entered: make(chan struct{})} + bucket.objects = []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectLister: func(s3Connection) (s3ObjectLister, error) { return bucket, nil }, + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} - done := make(chan ctrl.Result, 1) - go func() { - res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) - done <- res - }() - select { - case res := <-done: - if !requeues(res) { - t.Errorf("res = %+v, want the Backup held and retried", res) - } - case <-time.After(5 * time.Second): - t.Fatal("cleanup did not give up on an unreachable endpoint") + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + select { + case <-bucket.entered: + default: + t.Fatalf("cleanup returned without reaching the blocking %s", call) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatalf("cleanup did not give up on an unreachable endpoint blocking %s", call) + } + }) } } From bbeca3453534124975682bda008f0d025aaed65c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:07:03 +0300 Subject: [PATCH 12/22] fix(backups): retry a failed read of the platform backup credentials The MariaDB cleanup reaches the platform bucket through the platform's current endpoint so that a Backup taken before the endpoint moved is still deleted. A failed read of the platform credentials made it fall back to the endpoint recorded on the operator Backup, which may be the old address: there the dump is not found, a missing dump counts as already deleted, and the operator Backup that pointed at the real dump was deleted with a success log. A read that fails with an API error now holds the Backup and retries. A missing or malformed platform Secret still falls back, since it is a standing state that a retry would not change. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 10 +++++-- .../mariadbstrategy_cleanup_test.go | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 531b806c9..b4617918e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv // Secret the platform credentials are projected to, is reached as the // platform reaches it now, so a Backup taken before the endpoint moved is // still deleted. Any other bucket, or the platform bucket while the -// platform credentials cannot be read, is reached as op recorded it; a -// delete that fails there is handled by retryMariaDBCleanup like any other. +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { if r.CredentialsConfig.IsEnabled() { conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err case err != nil: getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 0ce6b2d01..1eecfeaff 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator } } +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 3334152da7d78e755ab7d3cc85ed8d459c0ba86e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:08:43 +0300 Subject: [PATCH 13/22] fix(backups): report a recorded MariaDB dump that was not found A recorded dump missing from the listing was logged as deleted, with no event. That is right when an earlier attempt or maxRetention removed it, but the listing goes through the platform's current endpoint, and after an endpoint move the dump can still sit on the old storage: the cleanup then deleted the operator Backup that pointed at it and said it had deleted the dump. The cleanup now records a Normal ArtifactAlreadyAbsent event naming the key and the endpoint it listed, so whoever later looks for that dump can tell where it was not found. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 36 ++++++++++++------- .../mariadbstrategy_cleanup_test.go | 14 ++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index b4617918e..25775185b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -118,8 +118,10 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } + absent := false if name != "" { - other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) } @@ -158,42 +160,52 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) } + if absent { + logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent", + "%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted", + location+name, conn.Endpoint, opName) + } + return ctrl.Result{}, nil + } logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) return ctrl.Result{}, nil } // contestedMariaDBDump returns the name of another operator Backup on op's // bucket and prefix whose run, finished or not, could have written the dump -// name, or "" when none could. The size probe that recorded name let only -// runs finished by then contest it, and a run can upload its dump and prune -// op's through maxRetention before it completes. A dump no longer in the -// bucket is contested by nobody: deleting it again removes nothing. -func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { +// name, or "" when none could, and whether the listing found the dump. The +// size probe that recorded name let only runs finished by then contest it, +// and a run can upload its dump and prune op's through maxRetention before +// it completes. A dump no longer in the bucket is contested by nobody: +// deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) { ts, ok := parseMariaDBLogicalBackupFile(name) if !ok { - return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name) } s3 := op.Spec.Storage.S3 key := mariadbOperatorS3Prefix(s3.Prefix) + name objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) if err != nil { - return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) } i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) if i < 0 { - return "", nil + return "", false, nil } others, err := neighbourMariaDBRuns(ctx, reader, op, true) if err != nil { - return "", fmt.Errorf("list operator Backups: %w", err) + return "", true, fmt.Errorf("list operator Backups: %w", err) } d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} for _, other := range others { if other.placesOnEitherClock(d) { - return other.name, nil + return other.name, true, nil } } - return "", nil + return "", true, nil } // activeRestoreOf returns the name of a RestoreJob that restores from diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 1eecfeaff..da9e0902e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -165,7 +165,7 @@ func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.Fake } func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { - bucket := &fakeS3Bucket{} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) @@ -512,7 +512,7 @@ func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { name := "backup.not-a-time.sql" bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} c := newMariaDBStrategyTestClient(t, op) - if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { t.Fatal("err = nil, want the unreadable dump time reported") } } @@ -534,9 +534,17 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T if operatorBackupExists(t, r.Client) { t.Error("operator Backup kept, want it deleted") } - if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") { t.Errorf("events = %v, want no ArtifactNotDeleted", events) } + // The listing may have reached other storage than the dump went to, as + // after an endpoint move, so the absence is reported with where it was + // looked for rather than logged as a delete. + if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") || + !strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") { + t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events) + } } // A run started after the recorded dump was uploaded cannot have written it. From d57ebbe808e831637062a920ba9f12fd5580a541 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:09:15 +0300 Subject: [PATCH 14/22] docs(backups): state what the MariaDB dump delete does not cover The comment on who deletes a MariaDB Backup listed namespace teardown without saying that teardown can remove the operator Backup first, in which case the dump is left behind with a Warning, so it promised a delete the code does not make. The remover contract claimed a missing key is never an error, which holds only for a backend that answers the delete with 204, the one status minio-go accepts. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 4 +++- internal/backupcontroller/backup_controller.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 40d827c62..97a67a729 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -39,7 +39,9 @@ type s3ObjectLister interface { } type s3ObjectRemover interface { - // RemoveObject deletes key; a key that does not exist is not an error. + // RemoveObject deletes key. S3 answers 204 for a key that does not + // exist and minio-go takes 204 as success; a backend answering a + // missing key with any other status makes it an error. RemoveObject(ctx context.Context, bucket, key string) error } diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index d8c69bf1c..a632f851d 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -142,7 +142,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns // it, and tenants cannot delete Backups, so the delete comes from an - // administrator or from namespace teardown (see cleanupMariaDBBackup). + // administrator or from namespace teardown. Teardown can remove the + // operator Backup first, and the dump is then left behind with a + // Warning, since its storage is read from it (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 14a842ad168c0fbe3c650a6ff541c4af163ee114 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:15:18 +0300 Subject: [PATCH 15/22] fix(backups): find a MariaDB operator Restore the cache has not seen The MariaDB restore creates its operator Restore once and then follows it, finding it by label in the informer cache. Nothing watches operator Restores, so a reconcile that ran before the new Restore reached the cache saw none: it created a second one, or, now that a restore refuses a Backup being deleted, failed the RestoreJob while its Restore was still running, after which the cleanup deleted the dump that Restore was reading. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 38 +++++++++++++++++++ .../mariadbstrategy_controller.go | 6 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 218c91de6..1919d41c9 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -222,6 +222,44 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { f.wantFailed(t, "src-bk", "is gone") }) } + // The refusal applies only before the operator Restore exists. One + // created by an earlier reconcile but not yet in the cache must be + // found and followed, or the RestoreJob fails under a running restore + // and the cleanup deletes the dump that restore reads. + t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() { + t.Fatalf("Backup not left being deleted: %v", err) + } + live.ResourceVersion = "" + apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }}) + f.r.APIReader = apiServer + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want the existing one followed", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index e3e00dbec..b97f7548b 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -756,10 +756,12 @@ func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *ba } // findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled -// with the RestoreJob, if a previous reconcile already created one. +// with the RestoreJob, if a previous reconcile already created one. It is +// read uncached: one the cache has not caught up with would be created +// again, or, once its Backup is being deleted, refused while it runs. func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} - if err := r.List(ctx, list, + if err := r.apiReader().List(ctx, list, client.InNamespace(rj.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: rj.Name, From ee6404a0c88af0850910b9bbb25ad8cc292f30ed Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:28:45 +0300 Subject: [PATCH 16/22] fix(backups): retry a failed lookup of a MariaDB operator Restore The lookup now goes to the API server, which can time out or throttle, and a failed lookup failed the RestoreJob. It says nothing about whether the operator Restore exists, so the RestoreJob could be failed while its Restore was running, and the Backup's cleanup would then stop waiting for it and delete the dump the Restore reads. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 22 +++++++++++++++++++ .../mariadbstrategy_controller.go | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 1919d41c9..5ec96e663 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -260,6 +260,28 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { t.Errorf("created %d operator Restores; want the existing one followed", len(got)) } }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index b97f7548b..28a24a14e 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -598,7 +598,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) } // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice. From 82468082890a9e6a587ae6c94f7ac02622f6610d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:30:41 +0300 Subject: [PATCH 17/22] fix(backups): hold a MariaDB dump while an operator Restore reads it The cleanup waited for a restore by the phase of its RestoreJob, but the phase is not the restore: a RestoreJob fails on its deadline while the operator Restore it created keeps running, and that Restore still reads the dump. Deleting the dump then made mariadb-operator restore the nearest other one. The cleanup now also holds the Backup while an operator Restore of its operator Backup has not been marked Complete, which mariadb-operator does once the Restore's Job has completed or failed, so what keeps the dump is the restore that reads it rather than the status of the request. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 32 +++++-- .../mariadbstrategy_cleanup_test.go | 83 +++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 25775185b..17d612955 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -59,15 +59,15 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac // The operator Restore reads the dump through the operator Backup, so // neither may go while a restore from this Backup runs. - restore, err := r.activeRestoreOf(ctx, backup) + restore, err := r.activeRestoreOf(ctx, backup, opName) if err != nil { return ctrl.Result{}, err } if restore != "" { - logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore) if r.Recorder != nil { r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", - "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + "%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", restore, mariadbSkipArtifactCleanupAnnotation) } return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil @@ -208,9 +208,14 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli return "", true, nil } -// activeRestoreOf returns the name of a RestoreJob that restores from -// backup and has not finished, or "". -func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { +// activeRestoreOf names a restore from backup that has not finished: a +// RestoreJob of it, or an operator Restore reading the dump of opName, the +// operator Backup it was made of; "" when there is none. The operator +// Restore is checked on its own because a RestoreJob can fail, on its +// deadline, while the Restore it created keeps reading the dump. +// mariadb-operator marks a Restore Complete=True once its Job has +// completed or failed. +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) { list := &backupsv1alpha1.RestoreJobList{} if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) @@ -221,7 +226,20 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv continue } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { - return rj.Name, nil + return "RestoreJob " + rj.Name, nil + } + } + restores := &mariadbtypes.RestoreList{} + if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) + } + for i := range restores.Items { + restore := &restores.Items[i] + if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName { + continue + } + if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) { + return "k8s.mariadb.com/Restore " + restore.Name, nil } } return "", nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index da9e0902e..ec2dea1a8 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -753,6 +753,89 @@ func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase } } +// operatorRestoreOf is a k8s.mariadb.com/Restore reading the dump of the +// operator Backup backupRef, with the given Complete condition. +func operatorRestoreOf(name, backupRef string, complete *metav1.Condition) *mariadbtypes.Restore { + restore := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}} + if backupRef != "" { + restore.Spec.BackupRef = &mariadbtypes.BackupReference{Name: backupRef} + } + if complete != nil { + restore.Status.Conditions = []metav1.Condition{*complete} + } + return restore +} + +func restoreComplete(status metav1.ConditionStatus, reason string) *metav1.Condition { + return &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: reason, LastTransitionTime: metav1.Now()} +} + +// The RestoreJob's phase is not the restore: one failed on its deadline +// leaves its operator Restore running, and that Restore still reads the +// dump. Until the operator marks it Complete, as it does a finished or a +// failed Job, the dump stays. +func TestCleanupMariaDBBackup_WaitsForAnOperatorRestoreOfItsDump(t *testing.T) { + held := map[string]*metav1.Condition{ + "running": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobRunning), + "suspended": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobSuspended), + "not yet reconciled": nil, + } + for name, cond := range held { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseFailed), operatorRestoreOf("rj-abcde", "op-bk", cond)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the operator Restore runs", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), "k8s.mariadb.com/Restore rj-abcde") { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the operator Restore", events) + } + }) + } + released := map[string]*mariadbtypes.Restore{ + "completed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobComplete)), + "failed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobFailed)), + "of another operator Backup": operatorRestoreOf("rj-abcde", "op-other", nil), + "from storage, not a Backup": operatorRestoreOf("rj-abcde", "", nil), + } + for name, restore := range released { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), restore) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) + } + t.Run("one the cache has not seen yet", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, operatorRestoreOf("rj-abcde", "op-bk", nil)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) +} + func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { t.Run(string(phase), func(t *testing.T) { From 0195a66faeea7a143ccf3aea2ba2f95df67d9178 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 18/22] fix(backups): release a MariaDB Backup when the operator CRDs are gone Listing the operator Restores of a Backup before reading its operator Backup made the cleanup fail on a cluster without the mariadb-operator CRDs, retried for good, so the Backup and its namespace never went away. The operator Backup read that follows already releases the Backup in that case; with no Restore kind no restore can run, so the listing now lets the cleanup reach it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/mariadbstrategy_cleanup.go | 5 +++++ .../backupcontroller/mariadbstrategy_cleanup_test.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 17d612955..3ff8adfbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -231,6 +231,11 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv } restores := &mariadbtypes.RestoreList{} if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + // Without the operator's CRDs no Restore can run; the operator + // Backup read that follows releases the Backup. + if apimeta.IsNoMatchError(err) { + return "", nil + } return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) } for i := range restores.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ec2dea1a8..e9a90029b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -1148,6 +1148,9 @@ func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { } } +// Without the mariadb-operator CRDs every k8s.mariadb.com read fails, the +// listing of operator Restores included: no Restore can run then, so the +// Backup is released rather than held for good. func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { bucket := &fakeS3Bucket{} c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ @@ -1157,6 +1160,12 @@ func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { } return c.Get(ctx, key, obj, opts...) }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Restore").GroupKind()} + } + return c.List(ctx, list, opts...) + }, }) r, rec := cleanupReconcilerWithClient(c, bucket) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) From f2507b4b84f905e13c0dea1565c64f9cf5699a7e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 19/22] docs(backups): name what the MariaDB cleanup reads uncached The reader's comment named only RestoreJobs, and the restore path said the cleanup holds a Backup only for RestoreJobs it can see, while the cleanup now lists both RestoreJobs and operator Restores straight from the API server. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 5 +++-- internal/backupcontroller/mariadbstrategy_controller.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index a632f851d..b67c640ab 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,8 +44,9 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs of a Backup through it: a restore that started before - // the delete may not be in the cache yet. Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup through it: a + // restore that started before the delete may not be in the cache yet. + // Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 28a24a14e..edff9d187 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -674,7 +674,8 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj // Deleting a Backup deletes its dump, and the operator restores the // nearest other dump once its own is gone, so a restore must not start // from a Backup being deleted. The copy in hand can predate the delete; - // the cleanup holds the Backup only for RestoreJobs it can already see. + // the cleanup holds the Backup only for restores that exist when it + // lists them. live := &backupsv1alpha1.Backup{} if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { if apierrors.IsNotFound(err) { From a6a704768ce0deb6013a6ab33772d46d0e16da6e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:41:03 +0300 Subject: [PATCH 20/22] test(backups): pin a failed RestoreJob listing in the MariaDB cleanup A RestoreJob listing that fails says nothing about whether a restore reads the dump. Swallowing that error left the whole suite green, while it would let the cleanup delete the dump under a running restore. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index e9a90029b..354bf83f3 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -866,6 +866,21 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) } }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From b3ca798bc438f0854d624b19d11fb1bab67a73d7 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 12 Nov 2026 17:01:29 +0300 Subject: [PATCH 21/22] fix(backups): read the runs that may own a MariaDB dump uncached Before deleting a dump the cleanup checks every other operator Backup on the same prefix that could have written it, and it read them from the informer cache. A run whose operator Backup had not reached the cache yet but had already uploaded its dump was missed, and its dump deleted as this Backup's. Both the recorded and the legacy paths now list the runs from the API server, as the restore checks already do. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/backup_controller.go | 7 ++--- .../mariadbstrategy_cleanup.go | 4 +-- .../mariadbstrategy_cleanup_test.go | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index b67c640ab..5e6292635 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,9 +44,10 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs and operator Restores of a Backup through it: a - // restore that started before the delete may not be in the cache yet. - // Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 3ff8adfbe..912bb0038 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac } absent := false if name != "" { - other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) @@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil } } else { - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 354bf83f3..1d2f10cbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T } } +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + // A run started after the recorded dump was uploaded cannot have written it. func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { key := cleanupPrefix + cleanupDumpName From 5d453c4d238463244e30d328ddbe862daadbb1c3 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 16:33:57 +0300 Subject: [PATCH 22/22] test(backups): give the fake S3 bucket a StatObject main added StatObject to the object lister interface, so the fake bucket the MariaDB cleanup tests use has to implement it to compile. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../backupcontroller/mariadbstrategy_cleanup_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 1d2f10cbe..7efb3046f 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -1368,3 +1368,12 @@ func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { t.Fatal("RemoveObject = nil, want the AccessDenied error") } } + +func (f *fakeS3Bucket) StatObject(_ context.Context, _, key string) (int64, error) { + for _, o := range f.objects { + if o.Key == key { + return o.Size, nil + } + } + return 0, errors.New("not found") +}