fix(backups): report a recorded MariaDB dump that was not found

A recorded dump missing from the listing was logged as deleted, with no
event. That is right when an earlier attempt or maxRetention removed it,
but the listing goes through the platform's current endpoint, and after
an endpoint move the dump can still sit on the old storage: the cleanup
then deleted the operator Backup that pointed at it and said it had
deleted the dump.

The cleanup now records a Normal ArtifactAlreadyAbsent event naming the
key and the endpoint it listed, so whoever later looks for that dump
can tell where it was not found.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-11 15:08:43 +03:00
1 parent bbeca34535
commit 3334152da7
2 files changed
+35 -15

No files matched your search

@@ -118,8 +118,10 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
if err != nil {
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err))
}
absent := false
if name != "" {
other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name)
other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name)
absent = !found
if err != nil {
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
}
@@ -158,42 +160,52 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) {
return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err)
}
if absent {
logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName)
if r.Recorder != nil {
r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent",
"%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted",
location+name, conn.Endpoint, opName)
}
return ctrl.Result{}, nil
}
logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName)
return ctrl.Result{}, nil
}
// contestedMariaDBDump returns the name of another operator Backup on op's
// bucket and prefix whose run, finished or not, could have written the dump
// name, or "" when none could. The size probe that recorded name let only
// runs finished by then contest it, and a run can upload its dump and prune
// op's through maxRetention before it completes. A dump no longer in the
// bucket is contested by nobody: deleting it again removes nothing.
func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) {
// name, or "" when none could, and whether the listing found the dump. The
// size probe that recorded name let only runs finished by then contest it,
// and a run can upload its dump and prune op's through maxRetention before
// it completes. A dump no longer in the bucket is contested by nobody:
// deleting it again removes nothing.
func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) {
ts, ok := parseMariaDBLogicalBackupFile(name)
if !ok {
return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name)
return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name)
}
s3 := op.Spec.Storage.S3
key := mariadbOperatorS3Prefix(s3.Prefix) + name
objects, err := lister.ListObjects(ctx, s3.Bucket, key, false)
if err != nil {
return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err)
return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err)
}
i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key })
if i < 0 {
return "", nil
return "", false, nil
}
others, err := neighbourMariaDBRuns(ctx, reader, op, true)
if err != nil {
return "", fmt.Errorf("list operator Backups: %w", err)
return "", true, fmt.Errorf("list operator Backups: %w", err)
}
d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified}
for _, other := range others {
if other.placesOnEitherClock(d) {
return other.name, nil
return other.name, true, nil
}
}
return "", nil
return "", true, nil
}
// activeRestoreOf returns the name of a RestoreJob that restores from
@@ -165,7 +165,7 @@ func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.Fake
}
func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) {
bucket := &fakeS3Bucket{}
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}}
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
@@ -512,7 +512,7 @@ func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) {
name := "backup.not-a-time.sql"
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}}
c := newMariaDBStrategyTestClient(t, op)
if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil {
if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil {
t.Fatal("err = nil, want the unreadable dump time reported")
}
}
@@ -534,9 +534,17 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T
if operatorBackupExists(t, r.Client) {
t.Error("operator Backup kept, want it deleted")
}
if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") {
events := drainEvents(rec)
if hasEvent(events, "Warning ArtifactNotDeleted") {
t.Errorf("events = %v, want no ArtifactNotDeleted", events)
}
// The listing may have reached other storage than the dump went to, as
// after an endpoint move, so the absence is reported with where it was
// looked for rather than logged as a delete.
if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") ||
!strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") {
t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events)
}
}
// A run started after the recorded dump was uploaded cannot have written it.