mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): delete a MariaDB dump when its Backup is deleted
Deleting a MariaDB Backup left its dump in the bucket for good: mariadb-operator's maxRetention prunes only from inside the next run of the same prefix, and a Plan never deletes a Backup, so on the platform bucket nothing else ever removes it. Every run of an application writes under one prefix, so the key deleted has to be proven to be this Backup's. It is the dump the Backup records, or, for a Backup taken before dumps were recorded, the one the existing ownership match attributes to its operator Backup. That match is recorded on the Backup before the delete, so a retry deletes the same key by name instead of matching again against a bucket that no longer holds it. Without proof the Backup is released with a Warning and nothing is deleted, the operator Backup included, since any later attribution starts from it. The operator Backup is deleted only after its dump, and the delete waits while a RestoreJob still restores from the Backup. The delete runs in the controller through the same minio client the size probe uses, so it verifies TLS the way mariadb-operator did when it wrote the dump. A dump on the platform bucket, which the operator Backup names by its bucket and by the projected credentials Secret, is reached through the platform's current endpoint and keys, still trusting a CA the strategy pinned, so a Backup taken before the endpoint moved is still deleted. Platform credentials that cannot be read fall back to the ones the operator Backup names. A failed delete holds the Backup and retries with a Warning naming the skip-artifact-cleanup annotation, except in a terminating namespace, where the Backup is released with a Warning so it does not hold the namespace. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
55f1fd33a8
commit
f147b22d86
6 files changed
+1223
-24
No files matched your search
@@ -38,6 +38,11 @@ type s3ObjectLister interface {
|
||||
ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error)
|
||||
}
|
||||
|
||||
type s3ObjectRemover interface {
|
||||
// RemoveObject deletes key; a key that does not exist is not an error.
|
||||
RemoveObject(ctx context.Context, bucket, key string) error
|
||||
}
|
||||
|
||||
// s3Connection is what a driver resolves from its rendered strategy to
|
||||
// read the bucket with the same identity the backup was written with.
|
||||
type s3Connection struct {
|
||||
@@ -63,8 +68,10 @@ type objectStore struct {
|
||||
// reader reads Secrets, which the manager's client fetches uncached,
|
||||
// and lists operator Backups, which it serves from the cache.
|
||||
reader client.Reader
|
||||
// newLister overrides the S3 client; nil means a real minio client.
|
||||
newLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
// newLister and newRemover override the S3 client; nil means a real
|
||||
// minio client.
|
||||
newLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
newRemover func(conn s3Connection) (s3ObjectRemover, error)
|
||||
}
|
||||
|
||||
func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
@@ -74,6 +81,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return newMinioObjectLister(conn)
|
||||
}
|
||||
|
||||
func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) {
|
||||
if s.newRemover != nil {
|
||||
return s.newRemover(conn)
|
||||
}
|
||||
c, err := newMinioObjectClient(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
|
||||
secret := &corev1.Secret{}
|
||||
if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
|
||||
@@ -94,6 +112,10 @@ func (r *RestoreJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *BackupReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover}
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return r.objectStore().lister(conn)
|
||||
}
|
||||
@@ -103,6 +125,14 @@ type minioObjectLister struct {
|
||||
}
|
||||
|
||||
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
|
||||
c, err := newMinioObjectClient(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) {
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken),
|
||||
Region: conn.Region,
|
||||
@@ -152,6 +182,10 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri
|
||||
return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}))
|
||||
}
|
||||
|
||||
func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error {
|
||||
return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
// collectObjects drains a minio listing. minio-go ends a listing whose
|
||||
// context is cancelled between pages by closing the channel without an
|
||||
// error item, so a cancelled context is checked once the channel closes:
|
||||
|
||||
@@ -39,9 +39,16 @@ type BackupReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Recorder record.EventRecorder
|
||||
// CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds
|
||||
// into the Backup's namespace before spawning its best-effort delete Job.
|
||||
// CredentialsConfig locates the platform backup credentials: the Job-based
|
||||
// cleanups project them into the Backup's namespace, and the MariaDB
|
||||
// cleanup reads them to reach the platform bucket.
|
||||
CredentialsConfig BackupCredentialsConfig
|
||||
|
||||
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
||||
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
||||
// Set by unit tests.
|
||||
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error)
|
||||
}
|
||||
|
||||
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
@@ -126,22 +133,18 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
|
||||
// orchestration" for tenant-facing guidance.
|
||||
return ctrl.Result{}, nil
|
||||
case strategyv1alpha1.MariaDBStrategyKind:
|
||||
// Cozystack Backup deletion does NOT delete the operator-side
|
||||
// k8s.mariadb.com/Backup CR or the backing S3/PVC archive.
|
||||
// Lifecycle of that CR is owned by mariadb-operator's
|
||||
// MaxRetention setting and by tenants / explicit teardown flows;
|
||||
// the controller's RBAC has no delete verb on
|
||||
// k8s.mariadb.com/backups for that reason. Surfacing this as an
|
||||
// explicit branch (rather than falling through to the Velero
|
||||
// default) keeps the contract honest: the MariaDB driver does
|
||||
// not own the archive, so it does not delete it on CR removal.
|
||||
// Mirrors the Altinity branch's "we do not own S3" stance.
|
||||
return ctrl.Result{}, nil
|
||||
// mariadb-operator's maxRetention prunes only from inside the next
|
||||
// backup run of the same prefix and a Plan never deletes a Backup, so
|
||||
// a dump on the platform bucket is deleted here or never. A dump in
|
||||
// a tenant's own bucket is deleted too: the Backup proves it owns
|
||||
// it, and the tenant deleting the Backup is asking for exactly that
|
||||
// (see cleanupMariaDBBackup).
|
||||
return r.cleanupMariaDBBackup(ctx, backup)
|
||||
case strategyv1alpha1.MongoDBStrategyKind:
|
||||
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
||||
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
||||
// operator's — that branch is a no-op, the same "we do not own the
|
||||
// archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup
|
||||
// archive" contract as Altinity / FDB. A useSystemBucket backup
|
||||
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
||||
// prune it, so the driver OWNS it: this deletes the operator
|
||||
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
||||
@@ -154,7 +157,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
|
||||
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
||||
// call, by manual operator action, or by
|
||||
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
||||
// the archive" contract as Altinity / MariaDB — the explicit
|
||||
// the archive" contract as Altinity — the explicit
|
||||
// branch guards the seam against a future driver refactor that
|
||||
// might incidentally stamp velero.io/backup-name onto FDB
|
||||
// driverMetadata via a shared helper.
|
||||
|
||||
@@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) {
|
||||
|
||||
// TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same
|
||||
// invariant as the Altinity test above for the MariaDB strategy. The
|
||||
// dispatcher must explicitly route MariaDB-strategy Backups to a no-op
|
||||
// cleanup rather than fall through to the Velero default - the MariaDB
|
||||
// driver does not own the operator-side k8s.mariadb.com/Backup CR or the
|
||||
// S3/PVC archive behind it (rbac.yaml grants no delete verb on
|
||||
// k8s.mariadb.com/backups for that reason). The test seeds the Velero
|
||||
// dispatcher must route MariaDB-strategy Backups to their own cleanup
|
||||
// rather than fall through to the Velero default. The test seeds the Velero
|
||||
// metadata keys plus a matching velero.io/Backup; with the explicit branch
|
||||
// no DeleteBackupRequest is synthesised, and the Velero Backup survives.
|
||||
// Without the branch the default falls through to cleanupVeleroBackup,
|
||||
|
||||
@@ -60,11 +60,22 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack
|
||||
// from can have been edited since, and pointing at another bucket now says
|
||||
// nothing about where this dump went.
|
||||
func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
|
||||
lister, s3, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
lister, _, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, err
|
||||
}
|
||||
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
|
||||
return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt)
|
||||
}
|
||||
|
||||
// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister,
|
||||
// for a caller that reaches the bucket by another endpoint than the one
|
||||
// mdbBackup names.
|
||||
func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
|
||||
}
|
||||
others, err := finishedMariaDBRuns(ctx, reader, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
const (
|
||||
mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup"
|
||||
|
||||
// Retry cadence while a dump cannot be deleted or a restore from the
|
||||
// Backup is still running.
|
||||
mariadbCleanupRetryInterval = 30 * time.Second
|
||||
)
|
||||
|
||||
// mariadbDumpCleanupTimeout bounds the object-store round trips of one
|
||||
// cleanup attempt. The Backup controller has a single worker, so an
|
||||
// unreachable endpoint stalls every other Backup for this long. A variable
|
||||
// so tests can shorten it.
|
||||
var mariadbDumpCleanupTimeout = 10 * time.Second
|
||||
|
||||
// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then
|
||||
// the operator Backup that wrote it. Every run of an application writes
|
||||
// under one prefix, so the only key deleted is one proven to be this
|
||||
// Backup's: the dump it records, or, for a Backup taken before dumps were
|
||||
// recorded, the one findMariaDBDump attributes to its operator Backup. When
|
||||
// no dump is proven the Backup is released with a Warning and nothing is
|
||||
// deleted, the operator Backup included, since it is what any later
|
||||
// attribution would start from. A failed delete holds the Backup and
|
||||
// retries, as the Redis and Rabbitmq cleanups do.
|
||||
func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
||||
logger := getLogger(ctx)
|
||||
if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" {
|
||||
logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
opName := backup.Spec.DriverMetadata[mariadbBackupNameKey]
|
||||
if opName == "" {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil
|
||||
}
|
||||
|
||||
// The operator Restore reads the dump through the operator Backup, so
|
||||
// neither may go while a restore from this Backup runs.
|
||||
restore, err := r.activeRestoreOf(ctx, backup)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if restore != "" {
|
||||
logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting",
|
||||
"RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump",
|
||||
restore, mariadbSkipArtifactCleanupAnnotation)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil
|
||||
}
|
||||
|
||||
op := &mariadbtypes.Backup{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil {
|
||||
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, "the dump",
|
||||
fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
s3 := op.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName),
|
||||
"PVC and Volume storage is not reachable from the controller"), nil
|
||||
}
|
||||
prefix := mariadbOperatorS3Prefix(s3.Prefix)
|
||||
location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix)
|
||||
|
||||
// A failed Job is Complete=True too, with reason JobFailed, and wrote no
|
||||
// dump: whatever its window holds is another run's.
|
||||
if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil ||
|
||||
cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil
|
||||
}
|
||||
|
||||
name := backup.Spec.DriverMetadata[mariadbDumpFileKey]
|
||||
if name != "" {
|
||||
if _, ok := parseMariaDBLogicalBackupFile(name); !ok {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := r.mariadbCleanupConnection(ctx, op)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err))
|
||||
}
|
||||
|
||||
s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout)
|
||||
defer cancel()
|
||||
store := r.objectStore()
|
||||
if name == "" {
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err))
|
||||
}
|
||||
dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time)
|
||||
if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil
|
||||
}
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, err)
|
||||
}
|
||||
name = dump.Name
|
||||
// Recorded first: once the dump is gone a retry could not match it
|
||||
// again, and could match a neighbour's instead.
|
||||
base := backup.DeepCopy()
|
||||
backup.Spec.DriverMetadata[mariadbDumpFileKey] = name
|
||||
backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339)
|
||||
if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
remover, err := store.remover(conn)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err))
|
||||
}
|
||||
if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
|
||||
}
|
||||
policy := metav1.DeletePropagationBackground
|
||||
if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err)
|
||||
}
|
||||
logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// activeRestoreOf returns the name of a RestoreJob that restores from
|
||||
// backup and has not finished, or "".
|
||||
func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) {
|
||||
list := &backupsv1alpha1.RestoreJobList{}
|
||||
if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
|
||||
return "", fmt.Errorf("list RestoreJobs: %w", err)
|
||||
}
|
||||
for i := range list.Items {
|
||||
rj := &list.Items[i]
|
||||
if rj.Spec.BackupRef.Name != backup.Name {
|
||||
continue
|
||||
}
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
return rj.Name, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// mariadbCleanupConnection returns how to reach the bucket op wrote to. A
|
||||
// dump on the platform bucket, which op names by its bucket and by the
|
||||
// Secret the platform credentials are projected to, is reached as the
|
||||
// platform reaches it now, so a Backup taken before the endpoint moved is
|
||||
// still deleted. Any other bucket, or the platform bucket while the
|
||||
// platform credentials cannot be read, is reached as op recorded it; a
|
||||
// delete that fails there is handled by retryMariaDBCleanup like any other.
|
||||
func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) {
|
||||
if r.CredentialsConfig.IsEnabled() {
|
||||
conn, bucket, err := r.platformS3Connection(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err)
|
||||
case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName:
|
||||
// A strategy can pin a private CA for the platform bucket; the
|
||||
// platform credentials carry none.
|
||||
if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil {
|
||||
ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key)
|
||||
if err != nil {
|
||||
return s3Connection{}, err
|
||||
}
|
||||
conn.CAPEM = []byte(ca)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3)
|
||||
}
|
||||
|
||||
// platformS3Connection reads the platform's backup bucket from the Secret
|
||||
// cozy-backups-creds is projected from, and its TLS from the scheme of
|
||||
// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets
|
||||
// tls.enabled. It reads the source rather than projecting it: projecting
|
||||
// writes a Secret, which a terminating namespace refuses.
|
||||
func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) {
|
||||
cfg := r.CredentialsConfig
|
||||
src := &corev1.Secret{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing,
|
||||
Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)}
|
||||
}
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)}
|
||||
}
|
||||
creds, err := parseSourceSecret(src)
|
||||
if err != nil {
|
||||
return s3Connection{}, "", err
|
||||
}
|
||||
if creds.endpoint == "" {
|
||||
creds.endpoint = stripScheme(cfg.Endpoint)
|
||||
}
|
||||
if creds.endpoint == "" {
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed,
|
||||
Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)}
|
||||
}
|
||||
if creds.region == "" {
|
||||
creds.region = cfg.Region
|
||||
}
|
||||
return s3Connection{
|
||||
Endpoint: creds.endpoint,
|
||||
Secure: strings.HasPrefix(cfg.Endpoint, "https://"),
|
||||
Region: creds.region,
|
||||
AccessKey: creds.accessKey,
|
||||
SecretKey: creds.secretKey,
|
||||
}, creds.bucket, nil
|
||||
}
|
||||
|
||||
// retryMariaDBCleanup holds the Backup after a failed attempt and says so,
|
||||
// unless its namespace is going away, where nothing it needs can be read
|
||||
// any longer and holding it would hold the namespace.
|
||||
func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) {
|
||||
terminating, err := r.namespaceTerminating(ctx, backup.Namespace)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if terminating {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil
|
||||
}
|
||||
getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed",
|
||||
"cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump",
|
||||
location, cause, mariadbSkipArtifactCleanupAnnotation)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil
|
||||
}
|
||||
|
||||
// releaseMariaDBCleanup lets the Backup go without deleting anything further
|
||||
// and records a Warning naming what was left, so the give-up is visible.
|
||||
func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result {
|
||||
getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason)
|
||||
}
|
||||
return ctrl.Result{}
|
||||
}
|
||||
@@ -0,0 +1,886 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
|
||||
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
// fakeS3Bucket stands in for the object store on both the listing and the
|
||||
// delete path, and records what was asked of it.
|
||||
type fakeS3Bucket struct {
|
||||
objects []s3ObjectInfo
|
||||
listErr error
|
||||
removeErr error
|
||||
|
||||
conns []s3Connection
|
||||
removed []string
|
||||
}
|
||||
|
||||
func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
|
||||
return f.objects, f.listErr
|
||||
}
|
||||
|
||||
func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error {
|
||||
if f.removeErr != nil {
|
||||
return f.removeErr
|
||||
}
|
||||
f.removed = append(f.removed, bucket+"/"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
cleanupCompleted = cleanupCreated.Add(3 * time.Minute)
|
||||
cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql"
|
||||
)
|
||||
|
||||
const cleanupPrefix = "tenant/db/"
|
||||
|
||||
// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of:
|
||||
// it stored to bkt under tenant/db with the tenant's creds Secret, and its
|
||||
// Job succeeded.
|
||||
func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup {
|
||||
t.Helper()
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/db"
|
||||
op := operatorBackupStoringTo(t, s3, cleanupCreated)
|
||||
op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))}
|
||||
return op
|
||||
}
|
||||
|
||||
// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup
|
||||
// op-bk, recording dumpFile when it is not empty.
|
||||
func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup {
|
||||
apiGroup := strategyv1alpha1.GroupVersion.Group
|
||||
md := map[string]string{
|
||||
mariadbBackupNameKey: "op-bk",
|
||||
mariadbBackupNamespaceKey: "tenant",
|
||||
}
|
||||
if dumpFile != "" {
|
||||
md[mariadbDumpFileKey] = dumpFile
|
||||
}
|
||||
now := metav1.Now()
|
||||
return &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant",
|
||||
Name: "bk",
|
||||
DeletionTimestamp: &now,
|
||||
Finalizers: []string{backupFinalizer},
|
||||
},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{
|
||||
APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb",
|
||||
},
|
||||
TakenAt: metav1.NewTime(cleanupCompleted),
|
||||
DriverMetadata: md,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func tenantNamespace() *corev1.Namespace {
|
||||
return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}}
|
||||
}
|
||||
|
||||
func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) {
|
||||
t.Helper()
|
||||
return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket)
|
||||
}
|
||||
|
||||
func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) {
|
||||
rec := record.NewFakeRecorder(20)
|
||||
return &BackupReconciler{
|
||||
Client: c,
|
||||
Scheme: c.Scheme(),
|
||||
Recorder: rec,
|
||||
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
||||
bucket.conns = append(bucket.conns, conn)
|
||||
return bucket, nil
|
||||
},
|
||||
newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) {
|
||||
bucket.conns = append(bucket.conns, conn)
|
||||
return bucket, nil
|
||||
},
|
||||
}, rec
|
||||
}
|
||||
|
||||
func operatorBackupExists(t *testing.T, c client.Client) bool {
|
||||
t.Helper()
|
||||
err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("get operator Backup: %v", err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func requeues(res ctrl.Result) bool {
|
||||
return res.Requeue || res.RequeueAfter > 0
|
||||
}
|
||||
|
||||
func hasEvent(events []string, prefix string) bool {
|
||||
for _, e := range events {
|
||||
if strings.HasPrefix(e, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// assertReleasedUntouched checks a Backup that is let go with nothing
|
||||
// deleted: no S3 delete, the operator Backup kept, and a Warning saying so.
|
||||
func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) {
|
||||
t.Helper()
|
||||
if err != nil || requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup released", res, err)
|
||||
}
|
||||
if len(bucket.removed) != 0 {
|
||||
t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed)
|
||||
}
|
||||
if !operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup deleted, want it kept")
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup released", res, err)
|
||||
}
|
||||
if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) {
|
||||
t.Errorf("removed %v, want %v", bucket.removed, want)
|
||||
}
|
||||
if operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup kept, want it deleted once its dump is")
|
||||
}
|
||||
if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" {
|
||||
t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns)
|
||||
}
|
||||
if events := drainEvents(rec); len(events) != 0 {
|
||||
t.Errorf("events = %v, want none", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if !requeues(res) {
|
||||
t.Errorf("res = %+v, want the Backup held and retried", res)
|
||||
}
|
||||
if !operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup deleted although its dump was not")
|
||||
}
|
||||
events := drainEvents(rec)
|
||||
if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) {
|
||||
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{
|
||||
Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
|
||||
if _, ok := obj.(*mariadbtypes.Backup); ok {
|
||||
return errors.New("apiserver unavailable")
|
||||
}
|
||||
return c.Delete(ctx, obj, opts...)
|
||||
},
|
||||
})
|
||||
r, _ := cleanupReconcilerWithClient(c, bucket)
|
||||
|
||||
if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil {
|
||||
t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held")
|
||||
}
|
||||
}
|
||||
|
||||
// A legacy Backup records the dump it proved before deleting it, so a retry
|
||||
// after a failed operator Backup delete finds the key by name instead of
|
||||
// matching again against a bucket that no longer holds it.
|
||||
func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) {
|
||||
key := cleanupPrefix + cleanupDumpName
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}}
|
||||
backup := cleanupBackup("")
|
||||
failDelete := true
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{
|
||||
Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
|
||||
if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete {
|
||||
return errors.New("apiserver unavailable")
|
||||
}
|
||||
return c.Delete(ctx, obj, opts...)
|
||||
},
|
||||
})
|
||||
r, rec := cleanupReconcilerWithClient(c, bucket)
|
||||
req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)}
|
||||
|
||||
if _, err := r.Reconcile(context.Background(), req); err == nil {
|
||||
t.Fatal("first pass: err = nil, want the failed operator Backup delete returned")
|
||||
}
|
||||
got := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(context.Background(), req.NamespacedName, got); err != nil {
|
||||
t.Fatalf("Backup gone after a failed operator Backup delete: %v", err)
|
||||
}
|
||||
if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName {
|
||||
t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata)
|
||||
}
|
||||
|
||||
// The dump is gone now, as the first pass deleted it.
|
||||
bucket.objects = nil
|
||||
failDelete = false
|
||||
if _, err := r.Reconcile(context.Background(), req); err != nil {
|
||||
t.Fatalf("second pass: %v", err)
|
||||
}
|
||||
if operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup kept, want it deleted on the retry")
|
||||
}
|
||||
if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) {
|
||||
t.Errorf("Backup still present (err=%v), want its finalizer removed", err)
|
||||
}
|
||||
if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") {
|
||||
t.Errorf("events = %v, want no claim that the deleted dump was left behind", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) {
|
||||
key := cleanupPrefix + cleanupDumpName
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}}
|
||||
backup := cleanupBackup("")
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{
|
||||
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
||||
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
||||
return errors.New("apiserver unavailable")
|
||||
}
|
||||
return c.Patch(ctx, obj, patch, opts...)
|
||||
},
|
||||
})
|
||||
r, _ := cleanupReconcilerWithClient(c, bucket)
|
||||
|
||||
if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil {
|
||||
t.Fatal("err = nil, want the failed record returned so the Backup is held")
|
||||
}
|
||||
if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) {
|
||||
t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) {
|
||||
for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) {
|
||||
key := cleanupPrefix + cleanupDumpName
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)},
|
||||
{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)},
|
||||
}}
|
||||
backup := cleanupBackup("")
|
||||
r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), backup)
|
||||
if err != nil || requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup released", res, err)
|
||||
}
|
||||
if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) {
|
||||
t.Errorf("removed %v, want %v", bucket.removed, want)
|
||||
}
|
||||
if operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup kept, want it deleted once its dump is")
|
||||
}
|
||||
if events := drainEvents(rec); len(events) != 0 {
|
||||
t.Errorf("events = %v, want none", events)
|
||||
}
|
||||
got := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) {
|
||||
t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
// The window is the operator Backup's lifetime, closed by the Backup's
|
||||
// takenAt: a dump stamped after takenAt plus the skew is not this run's.
|
||||
func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) {
|
||||
late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute)
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late},
|
||||
}}
|
||||
op := cleanupOperatorBackup(t)
|
||||
// The operator's own Complete transition is not what bounds the window.
|
||||
op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))}
|
||||
r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(""))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) {
|
||||
inWindow := cleanupCreated.Add(time.Minute)
|
||||
cases := map[string][]s3ObjectInfo{
|
||||
"no dump in its lifetime": {
|
||||
{Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)},
|
||||
},
|
||||
"two dumps in its lifetime": {
|
||||
{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow},
|
||||
{Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)},
|
||||
},
|
||||
"a dump placed by one clock only": {
|
||||
{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)},
|
||||
},
|
||||
}
|
||||
for name, objects := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{objects: objects}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(""))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) {
|
||||
inWindow := cleanupCreated.Add(time.Minute)
|
||||
bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}}
|
||||
neighbour := cleanupOperatorBackup(t)
|
||||
neighbour.Name = "op-neighbour"
|
||||
neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second))
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(""))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{listErr: errors.New("connection refused")}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(""))
|
||||
if err != nil || !requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err)
|
||||
}
|
||||
if !operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup deleted, want it kept")
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) {
|
||||
t.Run("operator Backup is gone", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 0 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed)
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events)
|
||||
}
|
||||
})
|
||||
t.Run("Backup records no operator Backup", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace())
|
||||
backup := cleanupBackup(cleanupDumpName)
|
||||
delete(backup.Spec.DriverMetadata, mariadbBackupNameKey)
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), backup)
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
t.Run("operator Backup's Job failed", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed
|
||||
r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
t.Run("operator Backup has no Complete condition", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Status.Conditions = nil
|
||||
r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
t.Run("operator Backup never completed", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Status.Conditions[0].Status = metav1.ConditionFalse
|
||||
r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
t.Run("dump is on a volume", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage = mariadbtypes.BackupStorage{}
|
||||
r, rec := cleanupReconciler(t, bucket, op, tenantNamespace())
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
backup := cleanupBackup(cleanupDumpName)
|
||||
backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"}
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), backup)
|
||||
if err != nil || requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup released", res, err)
|
||||
}
|
||||
if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) {
|
||||
t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client))
|
||||
}
|
||||
}
|
||||
|
||||
func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob {
|
||||
return &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{Phase: phase},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) {
|
||||
for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} {
|
||||
t.Run(string(phase), func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
restoreJobOf("rj", "bk", phase))
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || !requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err)
|
||||
}
|
||||
if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) {
|
||||
t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client))
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) {
|
||||
t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded),
|
||||
restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed),
|
||||
restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning))
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// platformCredsConfig points the reconciler at the platform's credentials
|
||||
// Secret, the source the per-tenant Secret is projected from. The projected
|
||||
// Secret is "creds", the one cleanupOperatorBackup names, as the default
|
||||
// strategy names cozy-backups-creds.
|
||||
func platformCredsConfig(endpoint string) BackupCredentialsConfig {
|
||||
return BackupCredentialsConfig{
|
||||
SourceNamespace: "cozy-system",
|
||||
SourceSecretName: "cozy-backups-credentials",
|
||||
TargetSecretName: "creds",
|
||||
Endpoint: endpoint,
|
||||
Region: "us-east-1",
|
||||
}
|
||||
}
|
||||
|
||||
func platformCredsSecret(bucketName, endpoint string) *corev1.Secret {
|
||||
return &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"},
|
||||
Data: map[string][]byte{
|
||||
"accessKey": []byte("PLATFORM"),
|
||||
"secretKey": []byte("platform-secret"),
|
||||
"endpoint": []byte(endpoint),
|
||||
"bucketName": []byte(bucketName),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
platformCredsSecret("bkt", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
for _, conn := range bucket.conns {
|
||||
if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" {
|
||||
t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(),
|
||||
platformCredsSecret("another-bucket", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
for _, conn := range bucket.conns {
|
||||
if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" {
|
||||
t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) {
|
||||
for _, bucketName := range []string{"bkt", "tenant-own-bucket"} {
|
||||
t.Run(bucketName, func(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage.S3.Bucket = bucketName
|
||||
// No platform Secret: the projection source is missing.
|
||||
r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed)
|
||||
}
|
||||
if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" {
|
||||
t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || !requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup held", res, err)
|
||||
}
|
||||
if !operatorBackupExists(t, r.Client) {
|
||||
t.Error("operator Backup deleted although its dump was not")
|
||||
}
|
||||
events := drainEvents(rec)
|
||||
if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")}
|
||||
ns := tenantNamespace()
|
||||
now := metav1.Now()
|
||||
ns.DeletionTimestamp = &now
|
||||
ns.Finalizers = []string{"kubernetes"}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns)
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}}
|
||||
ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}}
|
||||
r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" {
|
||||
t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}}
|
||||
r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || !requeues(res) || len(bucket.removed) != 0 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed)
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}}
|
||||
r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("http://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
platform := platformCredsSecret("bkt", "")
|
||||
delete(platform.Data, "endpoint")
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform)
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.configured.example")
|
||||
|
||||
if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bucket.conns[0].Endpoint != "s3.configured.example" {
|
||||
t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0])
|
||||
}
|
||||
}
|
||||
|
||||
// blockingBucket answers only when its context ends, as an unreachable
|
||||
// endpoint does.
|
||||
type blockingBucket struct{ fakeS3Bucket }
|
||||
|
||||
func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) {
|
||||
prev := mariadbDumpCleanupTimeout
|
||||
mariadbDumpCleanupTimeout = 50 * time.Millisecond
|
||||
t.Cleanup(func() { mariadbDumpCleanupTimeout = prev })
|
||||
bucket := &blockingBucket{}
|
||||
c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20),
|
||||
newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }}
|
||||
|
||||
done := make(chan ctrl.Result, 1)
|
||||
go func() {
|
||||
res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
done <- res
|
||||
}()
|
||||
select {
|
||||
case res := <-done:
|
||||
if !requeues(res) {
|
||||
t.Errorf("res = %+v, want the Backup held and retried", res)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not give up on an unreachable endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
||||
if _, ok := obj.(*mariadbtypes.Backup); ok {
|
||||
return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()}
|
||||
}
|
||||
return c.Get(ctx, key, obj, opts...)
|
||||
},
|
||||
})
|
||||
r, rec := cleanupReconcilerWithClient(c, bucket)
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 0 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed)
|
||||
}
|
||||
if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") {
|
||||
t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{
|
||||
Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
|
||||
if _, ok := obj.(*mariadbtypes.Backup); ok {
|
||||
return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName())
|
||||
}
|
||||
return c.Delete(ctx, obj, opts...)
|
||||
},
|
||||
})
|
||||
r, _ := cleanupReconcilerWithClient(c, bucket)
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) {
|
||||
t.Fatalf("got res=%+v err=%v, want the Backup released", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A strategy of its own that happens to use the platform bucket's name, as
|
||||
// an admin-chosen name on external S3 can, is reached as recorded: only the
|
||||
// projected Secret marks a Backup as written to the platform bucket.
|
||||
func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
op := cleanupOperatorBackup(t)
|
||||
op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds"
|
||||
op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds"
|
||||
r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(),
|
||||
platformCredsSecret("bkt", "s3.new.example"))
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" {
|
||||
t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0])
|
||||
}
|
||||
}
|
||||
|
||||
// With no endpoint anywhere the platform credentials are unusable, which
|
||||
// falls back like any other unreadable credentials.
|
||||
func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
platform := platformCredsSecret("bkt", "")
|
||||
delete(platform.Data, "endpoint")
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform)
|
||||
r.CredentialsConfig = platformCredsConfig("")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
if bucket.conns[0].Endpoint != "s3.example:9000" {
|
||||
t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{}
|
||||
malformed := platformCredsSecret("bkt", "s3.new.example")
|
||||
delete(malformed.Data, "secretKey")
|
||||
r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed)
|
||||
r.CredentialsConfig = platformCredsConfig("https://s3.new.example")
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
if err != nil || requeues(res) || len(bucket.removed) != 1 {
|
||||
t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed)
|
||||
}
|
||||
if bucket.conns[0].Endpoint != "s3.example:9000" {
|
||||
t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")}
|
||||
ns := tenantNamespace()
|
||||
now := metav1.Now()
|
||||
ns.DeletionTimestamp = &now
|
||||
ns.Finalizers = []string{"kubernetes"}
|
||||
r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns)
|
||||
|
||||
res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName))
|
||||
assertReleasedUntouched(t, r, rec, bucket, res, err)
|
||||
}
|
||||
|
||||
// Reconcile keeps the finalizer while the dump cannot be deleted, and drops
|
||||
// it once both the dump and the operator Backup are gone.
|
||||
func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) {
|
||||
bucket := &fakeS3Bucket{removeErr: errors.New("503")}
|
||||
backup := cleanupBackup(cleanupDumpName)
|
||||
r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace())
|
||||
req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)}
|
||||
|
||||
if _, err := r.Reconcile(context.Background(), req); err != nil {
|
||||
t.Fatalf("Reconcile: %v", err)
|
||||
}
|
||||
if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil {
|
||||
t.Fatalf("Backup gone while its dump could not be deleted: %v", err)
|
||||
}
|
||||
|
||||
bucket.removeErr = nil
|
||||
if _, err := r.Reconcile(context.Background(), req); err != nil {
|
||||
t.Fatalf("Reconcile: %v", err)
|
||||
}
|
||||
if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) {
|
||||
t.Errorf("Backup still present (err=%v), want its finalizer removed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The minio adapter deletes the exact key, with the colons of the dump's
|
||||
// timestamp escaped in the path; SeaweedFS refuses them raw.
|
||||
func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) {
|
||||
var method, rawPath, auth string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil {
|
||||
t.Fatalf("RemoveObject: %v", err)
|
||||
}
|
||||
if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" {
|
||||
t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath)
|
||||
}
|
||||
if !strings.Contains(auth, "Credential=AKID/") {
|
||||
t.Errorf("request not signed with the access key: %q", auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil {
|
||||
t.Fatal("RemoveObject = nil, want the AccessDenied error")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user