mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
With an endpoint CA the probe trusted the system roots plus that CA. barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the default trust store, so a CNPG probe could accept an endpoint the backup itself would have refused. Let the connection say whether the CA adds to the system roots or replaces them, and replace them for CNPG. mariadb-operator appends its CA to the system pool, so MariaDB keeps that. Pin in reconcile tests that a Create the apiserver rejects still fails the BackupJob, so only the read-back of an existing Backup is retried. Assisted-by: LLM Signed-off-by: Andrey Kolkov <androndo@gmail.com>
288 lines
10 KiB
Go
288 lines
10 KiB
Go
// SPDX-License-Identifier: Apache-2.0
|
|
package backupcontroller
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"math/big"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
|
|
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
|
)
|
|
|
|
type fakeS3Lister struct {
|
|
objects []s3ObjectInfo
|
|
err error
|
|
|
|
conn s3Connection
|
|
bucket, prefix string
|
|
recursive bool
|
|
calls int
|
|
}
|
|
|
|
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
|
|
f.calls++
|
|
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
|
|
return f.objects, f.err
|
|
}
|
|
|
|
func s3CredsSecret(namespace, name string) *corev1.Secret {
|
|
return &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
|
|
Data: map[string][]byte{
|
|
"AWS_ACCESS_KEY_ID": []byte("AKID"),
|
|
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
|
|
},
|
|
}
|
|
}
|
|
|
|
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
|
|
// by a fake client seeded with objs (typically the credentials Secret).
|
|
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
|
|
t.Helper()
|
|
c := newMariaDBStrategyTestClient(t, objs...)
|
|
return &BackupJobReconciler{
|
|
Client: c,
|
|
Scheme: c.Scheme(),
|
|
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
|
l.conn = conn
|
|
return l, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestCollectObjects(t *testing.T) {
|
|
listing := func() <-chan minio.ObjectInfo {
|
|
ch := make(chan minio.ObjectInfo, 1)
|
|
ch <- minio.ObjectInfo{Key: "a", Size: 1}
|
|
close(ch)
|
|
return ch
|
|
}
|
|
|
|
got, err := collectObjects(context.Background(), listing())
|
|
if err != nil || len(got) != 1 || got[0].Size != 1 {
|
|
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
|
|
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
|
|
}
|
|
|
|
errCh := make(chan minio.ObjectInfo, 1)
|
|
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
|
|
close(errCh)
|
|
if _, err := collectObjects(context.Background(), errCh); err == nil {
|
|
t.Error("listing error was not returned")
|
|
}
|
|
}
|
|
|
|
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
|
|
// Backup the cache does not show yet: the previous reconcile created it
|
|
// moments earlier.
|
|
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
|
|
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
|
r := &BackupJobReconciler{
|
|
Client: newMariaDBStrategyTestClient(t),
|
|
APIReader: newMariaDBStrategyTestClient(t, backup),
|
|
}
|
|
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
|
|
t.Fatal("existingBackup = nil; want the Backup the API server holds")
|
|
}
|
|
r.APIReader = newMariaDBStrategyTestClient(t)
|
|
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
|
|
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
|
|
}
|
|
}
|
|
|
|
// staleCache wraps a fake client so that Get reports NotFound for Backups,
|
|
// the way an informer cache does right after another writer created one.
|
|
func staleCache(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
|
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
}
|
|
|
|
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
|
|
// the retry check runs before the Backup exists, the refetch after Create
|
|
// collides with it.
|
|
func notFoundOnce(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
missed := false
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
|
|
missed = true
|
|
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
|
|
}
|
|
return cl.Get(ctx, key, obj, opts...)
|
|
},
|
|
})
|
|
}
|
|
|
|
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
|
|
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
|
|
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
|
return errors.New("etcdserver: request timed out")
|
|
},
|
|
})
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
|
|
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
|
|
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
|
|
}
|
|
}
|
|
|
|
// blockingLister answers nothing until the probe's context is done, the way
|
|
// a black-holed endpoint does.
|
|
type blockingLister struct{}
|
|
|
|
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
|
|
// is not back well before the default timeout would have expired.
|
|
func withShortProbeTimeout(t *testing.T, fn func()) {
|
|
t.Helper()
|
|
saved := artifactSizeProbeTimeout
|
|
artifactSizeProbeTimeout = 50 * time.Millisecond
|
|
defer func() { artifactSizeProbeTimeout = saved }()
|
|
done := make(chan struct{})
|
|
go func() {
|
|
fn()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("the probe did not give up at its timeout")
|
|
}
|
|
}
|
|
|
|
// failingReads wraps a fake client whose every Get fails, like an apiserver
|
|
// that times out.
|
|
func failingReads(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
|
|
return errors.New("etcdserver: request timed out")
|
|
},
|
|
})
|
|
}
|
|
|
|
func TestFailBackupArtifact(t *testing.T) {
|
|
newJob := func() *backupsv1alpha1.BackupJob {
|
|
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
|
|
}
|
|
|
|
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
|
|
j := newJob()
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
|
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
|
|
if !errors.Is(err, errBackupNotReadBack) {
|
|
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
|
|
}
|
|
persisted := &backupsv1alpha1.BackupJob{}
|
|
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
|
|
t.Fatalf("get BackupJob: %v", err)
|
|
}
|
|
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Error("the BackupJob was failed although its Backup exists")
|
|
}
|
|
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
|
|
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
|
|
}
|
|
})
|
|
|
|
t.Run("any other error fails the BackupJob", func(t *testing.T) {
|
|
j := newJob()
|
|
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
|
|
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
|
|
t.Fatalf("failBackupArtifact: %v", err)
|
|
}
|
|
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
|
|
t.Errorf("phase = %q, want Failed", j.Status.Phase)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
|
|
// roots" and "the CA alone". The system pool holds the endpoint's
|
|
// certificate, and the configured CA is an unrelated one, so the endpoint
|
|
// must stop being trusted only when CAOnly is set.
|
|
func TestEndpointTrustPool_CAOnly(t *testing.T) {
|
|
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
|
|
defer endpoint.Close()
|
|
system := x509.NewCertPool()
|
|
system.AddCert(endpoint.Certificate())
|
|
|
|
// Every httptest server shares one certificate, so the unrelated CA is
|
|
// minted here.
|
|
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
|
|
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
|
|
|
verifies := func(caOnly bool) bool {
|
|
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
|
|
if err != nil {
|
|
t.Fatalf("endpointTrustPool: %v", err)
|
|
}
|
|
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
|
|
return err == nil
|
|
}
|
|
if !verifies(false) {
|
|
t.Error("CA on top of the system roots: the endpoint was not trusted")
|
|
}
|
|
if verifies(true) {
|
|
t.Error("CA alone: the endpoint was trusted through the system roots")
|
|
}
|
|
}
|
|
|
|
// forbidBackupCreate wraps a fake client whose Create of a Backup is
|
|
// rejected, the way an admission webhook rejects one.
|
|
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
|
|
t.Helper()
|
|
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
|
|
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
|
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
|
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
|
|
}
|
|
return cl.Create(ctx, obj, opts...)
|
|
},
|
|
})
|
|
}
|