Files
cozystack/internal/backupcontroller/artifact_size_test.go
T
Andrey Kolkov e03986e8d0 fix(backups): treat an S3 object without a reported size as unmeasured
minio-go returns size -1 when a HEAD answer carries no Content-Length,
which would land in Backup.status.artifact.sizeBytes as -1. The probe
now reports such an object as unmeasured, so the field stays unset.

The shared client's trust settings also gain a test against an endpoint
whose certificate the configured CA did not sign, with the CA alone and
on top of the system roots: nothing pinned that a configured CA is
actually enforced.

Assisted-by: LLM
Signed-off-by: Andrey Kolkov <androndo@gmail.com>
2026-11-10 11:01:18 +07:00

363 lines
13 KiB
Go

// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/minio/minio-go/v7"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
type fakeS3Lister struct {
objects []s3ObjectInfo
err error
conn s3Connection
bucket, prefix string
recursive bool
calls int
statSize int64
statErr error
statKey string
}
func (f *fakeS3Lister) StatObject(_ context.Context, bucket, key string) (int64, error) {
f.calls++
f.bucket, f.statKey = bucket, key
return f.statSize, f.statErr
}
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
f.calls++
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
return f.objects, f.err
}
func s3CredsSecret(namespace, name string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("AKID"),
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
},
}
}
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
// by a fake client seeded with objs (typically the credentials Secret).
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
t.Helper()
c := newMariaDBStrategyTestClient(t, objs...)
return &BackupJobReconciler{
Client: c,
Scheme: c.Scheme(),
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
l.conn = conn
return l, nil
},
}
}
func TestCollectObjects(t *testing.T) {
listing := func() <-chan minio.ObjectInfo {
ch := make(chan minio.ObjectInfo, 1)
ch <- minio.ObjectInfo{Key: "a", Size: 1}
close(ch)
return ch
}
got, err := collectObjects(context.Background(), listing())
if err != nil || len(got) != 1 || got[0].Size != 1 {
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
}
errCh := make(chan minio.ObjectInfo, 1)
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
close(errCh)
if _, err := collectObjects(context.Background(), errCh); err == nil {
t.Error("listing error was not returned")
}
}
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
// Backup the cache does not show yet: the previous reconcile created it
// moments earlier.
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
r := &BackupJobReconciler{
Client: newMariaDBStrategyTestClient(t),
APIReader: newMariaDBStrategyTestClient(t, backup),
}
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
t.Fatal("existingBackup = nil; want the Backup the API server holds")
}
r.APIReader = newMariaDBStrategyTestClient(t)
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
}
}
// staleCache wraps a fake client so that Get reports NotFound for Backups,
// the way an informer cache does right after another writer created one.
func staleCache(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
// the retry check runs before the Backup exists, the refetch after Create
// collides with it.
func notFoundOnce(t *testing.T, c client.Client) client.Client {
t.Helper()
missed := false
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
missed = true
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
}
}
// blockingLister answers nothing until the probe's context is done, the way
// a black-holed endpoint does.
type blockingLister struct{}
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
<-ctx.Done()
return nil, ctx.Err()
}
func (blockingLister) StatObject(ctx context.Context, _, _ string) (int64, error) {
<-ctx.Done()
return 0, ctx.Err()
}
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
// is not back well before the default timeout would have expired.
func withShortProbeTimeout(t *testing.T, fn func()) {
t.Helper()
saved := artifactSizeProbeTimeout
artifactSizeProbeTimeout = 50 * time.Millisecond
defer func() { artifactSizeProbeTimeout = saved }()
done := make(chan struct{})
go func() {
fn()
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the probe did not give up at its timeout")
}
}
// failingReads wraps a fake client whose every Get fails, like an apiserver
// that times out.
func failingReads(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
}
func TestFailBackupArtifact(t *testing.T) {
newJob := func() *backupsv1alpha1.BackupJob {
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
}
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
if !errors.Is(err, errBackupNotReadBack) {
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Error("the BackupJob was failed although its Backup exists")
}
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
}
})
t.Run("any other error fails the BackupJob", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
t.Fatalf("failBackupArtifact: %v", err)
}
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed", j.Status.Phase)
}
})
}
// unrelatedCAPEM mints a CA that signed nothing an httptest server serves:
// every httptest server shares one certificate.
func unrelatedCAPEM(t *testing.T) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
}
// TestMinioObjectLister_EnforcesTheCA pins that a configured CA is the
// trust the client verifies with, whether alone or on top of the system
// roots: an endpoint whose certificate it did not sign is refused.
func TestMinioObjectLister_EnforcesTheCA(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "1")
w.Header().Set("ETag", `"abc"`)
w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT")
}))
defer srv.Close()
// minio-go retries a TLS failure with backoff; a refusal needs no more
// than the first attempt.
stat := func(ca []byte, caOnly bool) error {
l, err := newMinioObjectLister(s3Connection{
Endpoint: strings.TrimPrefix(srv.URL, "https://"), Secure: true,
AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1",
CAPEM: ca, CAOnly: caOnly,
})
if err != nil {
t.Fatalf("newMinioObjectLister: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_, err = l.StatObject(ctx, "cozy-backups", "k")
return err
}
signer := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
for _, caOnly := range []bool{true, false} {
if err := stat(signer, caOnly); err != nil {
t.Errorf("CAOnly=%v: the CA that signed the endpoint: %v", caOnly, err)
}
if err := stat(unrelatedCAPEM(t), caOnly); err == nil {
t.Errorf("CAOnly=%v: StatObject succeeded against a certificate the CA did not sign, want a TLS error", caOnly)
}
}
}
// TestMinioObjectLister_StatWithoutSize pins that a HEAD answer without
// Content-Length, which minio-go turns into size -1, is not a size.
func TestMinioObjectLister_StatWithoutSize(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("ETag", `"abc"`)
w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT")
}))
defer srv.Close()
l, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1"})
if err != nil {
t.Fatalf("newMinioObjectLister: %v", err)
}
if size, err := l.StatObject(context.Background(), "cozy-backups", "k"); err == nil {
t.Errorf("StatObject = %d, nil; want an error for a response without Content-Length", size)
}
}
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
// roots" and "the CA alone". The system pool holds the endpoint's
// certificate, and the configured CA is an unrelated one, so the endpoint
// must stop being trusted only when CAOnly is set.
func TestEndpointTrustPool_CAOnly(t *testing.T) {
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
defer endpoint.Close()
system := x509.NewCertPool()
system.AddCert(endpoint.Certificate())
unrelatedCA := unrelatedCAPEM(t)
verifies := func(caOnly bool) bool {
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
if err != nil {
t.Fatalf("endpointTrustPool: %v", err)
}
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
return err == nil
}
if !verifies(false) {
t.Error("CA on top of the system roots: the endpoint was not trusted")
}
if verifies(true) {
t.Error("CA alone: the endpoint was trusted through the system roots")
}
}
// forbidBackupCreate wraps a fake client whose Create of a Backup is
// rejected, the way an admission webhook rejects one.
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
}
return cl.Create(ctx, obj, opts...)
},
})
}