mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): see a MariaDB restore the cache has not caught up with
A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
320d0299ad
commit
654c543264
3 files changed
+28
-1
No files matched your search
@@ -43,6 +43,10 @@ type BackupReconciler struct {
|
||||
// cleanups project them into the Backup's namespace, and the MariaDB
|
||||
// cleanup reads them to reach the platform bucket.
|
||||
CredentialsConfig BackupCredentialsConfig
|
||||
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
|
||||
// the RestoreJobs of a Backup through it: a restore that started before
|
||||
// the delete may not be in the cache yet. Wired in SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
||||
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
||||
@@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup
|
||||
|
||||
// SetupWithManager registers the BackupReconciler with the Manager.
|
||||
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
r.APIReader = mgr.GetAPIReader()
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&backupsv1alpha1.Backup{}).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// apiReader returns the uncached reader. Reconcilers built by hand in unit
|
||||
// tests have none and fall back to their fake Client, which has no cache to
|
||||
// bypass.
|
||||
func (r *BackupReconciler) apiReader() client.Reader {
|
||||
if r.APIReader != nil {
|
||||
return r.APIReader
|
||||
}
|
||||
return r.Client
|
||||
}
|
||||
@@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli
|
||||
// backup and has not finished, or "".
|
||||
func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) {
|
||||
list := &backupsv1alpha1.RestoreJobList{}
|
||||
if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
|
||||
if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
|
||||
return "", fmt.Errorf("list RestoreJobs: %w", err)
|
||||
}
|
||||
for i := range list.Items {
|
||||
|
||||
@@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
// A restore checks that its Backup is not being deleted before it
|
||||
// starts, so one that started first is on the API server before the
|
||||
// delete, though not necessarily in this controller's cache yet.
|
||||
t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning))
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || !requeues(res) || len(bucket.removed) != 0 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed)
|
||||
}
|
||||
})
|
||||
t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
|
||||
Reference in new issue
Block a user