Files
cozystack/internal/backupcontroller/artifact_size_test.go
T
Andrey Kolkov 92fa979690 fix(backups): trust only the endpoint CA where the backup writer does
With an endpoint CA the probe trusted the system roots plus that CA.
barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the
default trust store, so a CNPG probe could accept an endpoint the backup
itself would have refused. Let the connection say whether the CA adds to
the system roots or replaces them, and replace them for CNPG.
mariadb-operator appends its CA to the system pool, so MariaDB keeps
that.

Pin in reconcile tests that a Create the apiserver rejects still fails
the BackupJob, so only the read-back of an existing Backup is retried.

Assisted-by: LLM
Signed-off-by: Andrey Kolkov <androndo@gmail.com>
2026-11-04 13:04:23 +00:00

288 lines
10 KiB
Go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/minio/minio-go/v7"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
type fakeS3Lister struct {
objects []s3ObjectInfo
err error
conn s3Connection
bucket, prefix string
recursive bool
calls int
}
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
f.calls++
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
return f.objects, f.err
}
func s3CredsSecret(namespace, name string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("AKID"),
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
},
}
}
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
// by a fake client seeded with objs (typically the credentials Secret).
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
t.Helper()
c := newMariaDBStrategyTestClient(t, objs...)
return &BackupJobReconciler{
Client: c,
Scheme: c.Scheme(),
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
l.conn = conn
return l, nil
},
}
}
func TestCollectObjects(t *testing.T) {
listing := func() <-chan minio.ObjectInfo {
ch := make(chan minio.ObjectInfo, 1)
ch <- minio.ObjectInfo{Key: "a", Size: 1}
close(ch)
return ch
}
got, err := collectObjects(context.Background(), listing())
if err != nil || len(got) != 1 || got[0].Size != 1 {
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
}
errCh := make(chan minio.ObjectInfo, 1)
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
close(errCh)
if _, err := collectObjects(context.Background(), errCh); err == nil {
t.Error("listing error was not returned")
}
}
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
// Backup the cache does not show yet: the previous reconcile created it
// moments earlier.
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
r := &BackupJobReconciler{
Client: newMariaDBStrategyTestClient(t),
APIReader: newMariaDBStrategyTestClient(t, backup),
}
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
t.Fatal("existingBackup = nil; want the Backup the API server holds")
}
r.APIReader = newMariaDBStrategyTestClient(t)
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
}
}
// staleCache wraps a fake client so that Get reports NotFound for Backups,
// the way an informer cache does right after another writer created one.
func staleCache(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
// the retry check runs before the Backup exists, the refetch after Create
// collides with it.
func notFoundOnce(t *testing.T, c client.Client) client.Client {
t.Helper()
missed := false
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
missed = true
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
}
}
// blockingLister answers nothing until the probe's context is done, the way
// a black-holed endpoint does.
type blockingLister struct{}
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
<-ctx.Done()
return nil, ctx.Err()
}
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
// is not back well before the default timeout would have expired.
func withShortProbeTimeout(t *testing.T, fn func()) {
t.Helper()
saved := artifactSizeProbeTimeout
artifactSizeProbeTimeout = 50 * time.Millisecond
defer func() { artifactSizeProbeTimeout = saved }()
done := make(chan struct{})
go func() {
fn()
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the probe did not give up at its timeout")
}
}
// failingReads wraps a fake client whose every Get fails, like an apiserver
// that times out.
func failingReads(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
}
func TestFailBackupArtifact(t *testing.T) {
newJob := func() *backupsv1alpha1.BackupJob {
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
}
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
if !errors.Is(err, errBackupNotReadBack) {
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Error("the BackupJob was failed although its Backup exists")
}
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
}
})
t.Run("any other error fails the BackupJob", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
t.Fatalf("failBackupArtifact: %v", err)
}
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed", j.Status.Phase)
}
})
}
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
// roots" and "the CA alone". The system pool holds the endpoint's
// certificate, and the configured CA is an unrelated one, so the endpoint
// must stop being trusted only when CAOnly is set.
func TestEndpointTrustPool_CAOnly(t *testing.T) {
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
defer endpoint.Close()
system := x509.NewCertPool()
system.AddCert(endpoint.Certificate())
// Every httptest server shares one certificate, so the unrelated CA is
// minted here.
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
verifies := func(caOnly bool) bool {
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
if err != nil {
t.Fatalf("endpointTrustPool: %v", err)
}
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
return err == nil
}
if !verifies(false) {
t.Error("CA on top of the system roots: the endpoint was not trusted")
}
if verifies(true) {
t.Error("CA alone: the endpoint was trusted through the system roots")
}
}
// forbidBackupCreate wraps a fake client whose Create of a Backup is
// rejected, the way an admission webhook rejects one.
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
}
return cl.Create(ctx, obj, opts...)
},
})
}