mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): read the runs that may own a MariaDB dump uncached
Before deleting a dump the cleanup checks every other operator Backup on the same prefix that could have written it, and it read them from the informer cache. A run whose operator Backup had not reached the cache yet but had already uploaded its dump was missed, and its dump deleted as this Backup's. Both the recorded and the legacy paths now list the runs from the API server, as the restore checks already do. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
a6a704768c
commit
b3ca798bc4
3 files changed
+33
-5
No files matched your search
@@ -44,9 +44,10 @@ type BackupReconciler struct {
|
||||
// cleanup reads them to reach the platform bucket.
|
||||
CredentialsConfig BackupCredentialsConfig
|
||||
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
|
||||
// the RestoreJobs and operator Restores of a Backup through it: a
|
||||
// restore that started before the delete may not be in the cache yet.
|
||||
// Wired in SetupWithManager.
|
||||
// the RestoreJobs and operator Restores of a Backup, and the operator
|
||||
// Backups that may own its dump, through it: a restore that started
|
||||
// before the delete, or a run that has already uploaded its dump, may
|
||||
// not be in the cache yet. Wired in SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
||||
|
||||
@@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
|
||||
}
|
||||
absent := false
|
||||
if name != "" {
|
||||
other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name)
|
||||
other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name)
|
||||
absent = !found
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
|
||||
@@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac
|
||||
fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil
|
||||
}
|
||||
} else {
|
||||
dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true)
|
||||
dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true)
|
||||
if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil
|
||||
|
||||
@@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
// The runs that may own a dump are read from the API server: a run whose
|
||||
// operator Backup has not reached the cache yet can already have uploaded
|
||||
// its dump, and missing it would delete that dump as this Backup's.
|
||||
func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) {
|
||||
started := cleanupCreated.Add(time.Minute)
|
||||
theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip")
|
||||
unseen := func() *mariadbtypes.Backup {
|
||||
b := cleanupOperatorBackup(t)
|
||||
b.Name = "op-unseen"
|
||||
b.CreationTimestamp = metav1.NewTime(started)
|
||||
b.Status.Conditions = nil
|
||||
return b
|
||||
}
|
||||
cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""}
|
||||
for name, recorded := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}}
|
||||
backup := cleanupBackup(recorded)
|
||||
r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), backup)
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A run started after the recorded dump was uploaded cannot have written it.
|
||||
func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) {
|
||||
key := cleanupPrefix + cleanupDumpName
|
||||
|
||||
Reference in new issue
Block a user