mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
minio-go returns size -1 when a HEAD answer carries no Content-Length, which would land in Backup.status.artifact.sizeBytes as -1. The probe now reports such an object as unmeasured, so the field stays unset. The shared client's trust settings also gain a test against an endpoint whose certificate the configured CA did not sign, with the CA alone and on top of the system roots: nothing pinned that a configured CA is actually enforced. Assisted-by: LLM Signed-off-by: Andrey Kolkov <androndo@gmail.com>
363 lines
13 KiB
Go
363 lines
13 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"math/big"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
)
|
|
|
|
type fakeS3Lister struct {
|
|
objects []s3ObjectInfo
|
|
err error
|
|
|
|
conn s3Connection
|
|
bucket, prefix string
|
|
recursive bool
|
|
calls int
|
|
|
|
statSize int64
|
|
statErr error
|
|
statKey string
|
|
}
|
|
|
|
func (f *fakeS3Lister) StatObject(_ context.Context, bucket, key string) (int64, error) {
|
|
f.calls++
|
|
f.bucket, f.statKey = bucket, key
|
|
return f.statSize, f.statErr
|
|
}
|
|
|
|
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
|
|
f.calls++
|
|
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
|
|
return f.objects, f.err
|
|
}
|
|
|
|
func s3CredsSecret(namespace, name string) *corev1.Secret {
|
|
return &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
|
Data: map[string][]byte{
|
|
"AWS_ACCESS_KEY_ID": []byte("AKID"),
|
|
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
|
|
},
|
|
}
|
|
}
|
|
|
|
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
|
|
// by a fake client seeded with objs (typically the credentials Secret).
|
|
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
|
|
t.Helper()
|
|
c := newMariaDBStrategyTestClient(t, objs...)
|
|
return &BackupJobReconciler{
|
|
Client: c,
|
|
Scheme: c.Scheme(),
|
|
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
|
l.conn = conn
|
|
return l, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestCollectObjects(t *testing.T) {
|
|
listing := func() <-chan minio.ObjectInfo {
|
|
ch := make(chan minio.ObjectInfo, 1)
|
|
ch <- minio.ObjectInfo{Key: "a", Size: 1}
|
|
close(ch)
|
|
return ch
|
|
}
|
|
|
|
got, err := collectObjects(context.Background(), listing())
|
|
if err != nil || len(got) != 1 || got[0].Size != 1 {
|
|
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
|
|
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
|
|
}
|
|
|
|
errCh := make(chan minio.ObjectInfo, 1)
|
|
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
|
|
close(errCh)
|
|
if _, err := collectObjects(context.Background(), errCh); err == nil {
|
|
t.Error("listing error was not returned")
|
|
}
|
|
}
|
|
|
|
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
|
|
// Backup the cache does not show yet: the previous reconcile created it
|
|
// moments earlier.
|
|
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
|
|
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
|
r := &BackupJobReconciler{
|
|
Client: newMariaDBStrategyTestClient(t),
|
|
APIReader: newMariaDBStrategyTestClient(t, backup),
|
|
}
|
|
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
|
|
t.Fatal("existingBackup = nil; want the Backup the API server holds")
|
|
}
|
|
r.APIReader = newMariaDBStrategyTestClient(t)
|
|
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
|
|
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
|
|
}
|
|
}
|
|
|
|
// staleCache wraps a fake client so that Get reports NotFound for Backups,
|
|
// the way an informer cache does right after another writer created one.
|
|
func staleCache(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
|
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
}
|
|
|
|
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
|
|
// the retry check runs before the Backup exists, the refetch after Create
|
|
// collides with it.
|
|
func notFoundOnce(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
missed := false
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
|
|
missed = true
|
|
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
}
|
|
|
|
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
|
|
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
|
|
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
|
return errors.New("etcdserver: request timed out")
|
|
},
|
|
})
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
|
|
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
|
|
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
|
|
}
|
|
}
|
|
|
|
// blockingLister answers nothing until the probe's context is done, the way
|
|
// a black-holed endpoint does.
|
|
type blockingLister struct{}
|
|
|
|
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
func (blockingLister) StatObject(ctx context.Context, _, _ string) (int64, error) {
|
|
<-ctx.Done()
|
|
return 0, ctx.Err()
|
|
}
|
|
|
|
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
|
|
// is not back well before the default timeout would have expired.
|
|
func withShortProbeTimeout(t *testing.T, fn func()) {
|
|
t.Helper()
|
|
saved := artifactSizeProbeTimeout
|
|
artifactSizeProbeTimeout = 50 * time.Millisecond
|
|
defer func() { artifactSizeProbeTimeout = saved }()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
fn()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("the probe did not give up at its timeout")
|
|
}
|
|
}
|
|
|
|
// failingReads wraps a fake client whose every Get fails, like an apiserver
|
|
// that times out.
|
|
func failingReads(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
|
return errors.New("etcdserver: request timed out")
|
|
},
|
|
})
|
|
}
|
|
|
|
func TestFailBackupArtifact(t *testing.T) {
|
|
newJob := func() *backupsv1alpha1.BackupJob {
|
|
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
|
}
|
|
|
|
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
|
|
j := newJob()
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
|
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
|
|
if !errors.Is(err, errBackupNotReadBack) {
|
|
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
|
|
}
|
|
persisted := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Error("the BackupJob was failed although its Backup exists")
|
|
}
|
|
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
|
|
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
|
|
}
|
|
})
|
|
|
|
t.Run("any other error fails the BackupJob", func(t *testing.T) {
|
|
j := newJob()
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
|
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
|
|
t.Fatalf("failBackupArtifact: %v", err)
|
|
}
|
|
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("phase = %q, want Failed", j.Status.Phase)
|
|
}
|
|
})
|
|
}
|
|
|
|
// unrelatedCAPEM mints a CA that signed nothing an httptest server serves:
|
|
// every httptest server shares one certificate.
|
|
func unrelatedCAPEM(t *testing.T) []byte {
|
|
t.Helper()
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
|
|
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
|
}
|
|
|
|
// TestMinioObjectLister_EnforcesTheCA pins that a configured CA is the
|
|
// trust the client verifies with, whether alone or on top of the system
|
|
// roots: an endpoint whose certificate it did not sign is refused.
|
|
func TestMinioObjectLister_EnforcesTheCA(t *testing.T) {
|
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Length", "1")
|
|
w.Header().Set("ETag", `"abc"`)
|
|
w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT")
|
|
}))
|
|
defer srv.Close()
|
|
// minio-go retries a TLS failure with backoff; a refusal needs no more
|
|
// than the first attempt.
|
|
stat := func(ca []byte, caOnly bool) error {
|
|
l, err := newMinioObjectLister(s3Connection{
|
|
Endpoint: strings.TrimPrefix(srv.URL, "https://"), Secure: true,
|
|
AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1",
|
|
CAPEM: ca, CAOnly: caOnly,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("newMinioObjectLister: %v", err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
_, err = l.StatObject(ctx, "cozy-backups", "k")
|
|
return err
|
|
}
|
|
signer := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
|
|
for _, caOnly := range []bool{true, false} {
|
|
if err := stat(signer, caOnly); err != nil {
|
|
t.Errorf("CAOnly=%v: the CA that signed the endpoint: %v", caOnly, err)
|
|
}
|
|
if err := stat(unrelatedCAPEM(t), caOnly); err == nil {
|
|
t.Errorf("CAOnly=%v: StatObject succeeded against a certificate the CA did not sign, want a TLS error", caOnly)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMinioObjectLister_StatWithoutSize pins that a HEAD answer without
|
|
// Content-Length, which minio-go turns into size -1, is not a size.
|
|
func TestMinioObjectLister_StatWithoutSize(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("ETag", `"abc"`)
|
|
w.Header().Set("Last-Modified", "Wed, 21 Oct 2026 10:00:00 GMT")
|
|
}))
|
|
defer srv.Close()
|
|
l, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), AccessKey: "AKID", SecretKey: "secret", Region: "us-east-1"})
|
|
if err != nil {
|
|
t.Fatalf("newMinioObjectLister: %v", err)
|
|
}
|
|
if size, err := l.StatObject(context.Background(), "cozy-backups", "k"); err == nil {
|
|
t.Errorf("StatObject = %d, nil; want an error for a response without Content-Length", size)
|
|
}
|
|
}
|
|
|
|
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
|
|
// roots" and "the CA alone". The system pool holds the endpoint's
|
|
// certificate, and the configured CA is an unrelated one, so the endpoint
|
|
// must stop being trusted only when CAOnly is set.
|
|
func TestEndpointTrustPool_CAOnly(t *testing.T) {
|
|
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
|
|
defer endpoint.Close()
|
|
system := x509.NewCertPool()
|
|
system.AddCert(endpoint.Certificate())
|
|
|
|
unrelatedCA := unrelatedCAPEM(t)
|
|
|
|
verifies := func(caOnly bool) bool {
|
|
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
|
|
if err != nil {
|
|
t.Fatalf("endpointTrustPool: %v", err)
|
|
}
|
|
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
|
|
return err == nil
|
|
}
|
|
if !verifies(false) {
|
|
t.Error("CA on top of the system roots: the endpoint was not trusted")
|
|
}
|
|
if verifies(true) {
|
|
t.Error("CA alone: the endpoint was trusted through the system roots")
|
|
}
|
|
}
|
|
|
|
// forbidBackupCreate wraps a fake client whose Create of a Backup is
|
|
// rejected, the way an admission webhook rejects one.
|
|
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
|
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
|
|
}
|
|
return cl.Create(ctx, obj, opts...)
|
|
},
|
|
})
|
|
}
|