mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-21 20:22:41 +09:00
ipn/store/kubestore: cache TLS certs briefly on write replicas (#22103)
The write replica of an HA ProxyGroup (cert share "rw") doesn't keep TLS certs in memory, so that it notices a recreated, empty cert Secret and issues a new cert. As a result, it read the cert Secret from the API server on every TLS handshake, with a 30s timeout. When the API server was slow or unavailable, handshakes on that replica waited for the read, and feature/acme repeated the read under the per-domain lock. Clients with a 10s handshake timeout, such as kubectl, failed. This commit adds a small per-domain cache of the last-known-good cert and key for write replicas. The replica serves a cached cert for 30s before it reads the Secret again. When it has a cached cert, the read uses a 3s timeout, and on an error other than not found or forbidden the replica serves the cached cert and waits another 30s before the next read. A read that finds the Secret missing, forbidden or empty removes the cached cert and returns ipn.ErrStateNotExist as before, so a recreated Ingress still gets a new cert, at most 30s later. Concurrent reads of the same Secret share one GET, and a read that is in flight when a cert is written does not replace the written cert. A Secret watch remains the long-term fix. Fixes #22009 Change-Id: Id107bf0a7628a1a026f94e621173d7c5d02fb451 Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
1 parent
519a9f502c
commit
c8ba9da543
2 files changed
+541
-17
No files matched your search
@@ -14,6 +14,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"tailscale.com/envknob"
|
||||
@@ -24,9 +25,11 @@ import (
|
||||
"tailscale.com/kube/kubeapi"
|
||||
"tailscale.com/kube/kubeclient"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
"tailscale.com/tstime"
|
||||
"tailscale.com/types/logger"
|
||||
"tailscale.com/util/dnsname"
|
||||
"tailscale.com/util/mak"
|
||||
"tailscale.com/util/singleflight"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -44,6 +47,17 @@ const (
|
||||
// state Secret and emit an Event.
|
||||
timeout = 30 * time.Second
|
||||
|
||||
// rwCertTTL is how long a write replica serves a TLS cert from its
|
||||
// in-memory cache before it reads the cert Secret again. A short TTL
|
||||
// limits how long the replica can serve a stale cert after the cert
|
||||
// Secret is recreated, for example after Ingress recreation.
|
||||
rwCertTTL = 30 * time.Second
|
||||
// rwCertFallbackTimeout is the timeout for a write replica's cert
|
||||
// Secret read when it has a last-known-good cert to fall back to.
|
||||
// TLS handshakes wait for this read, so it must be much shorter than
|
||||
// typical client handshake timeouts (10s for kubectl).
|
||||
rwCertFallbackTimeout = 3 * time.Second
|
||||
|
||||
reasonTailscaleStateUpdated = "TailscaledStateUpdated"
|
||||
reasonTailscaleStateLoaded = "TailscaleStateLoaded"
|
||||
reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed"
|
||||
@@ -90,6 +104,32 @@ type Store struct {
|
||||
// memory holds the latest tailscale state. Writes write state to a kube
|
||||
// Secret and memory, Reads read from memory.
|
||||
memory mem.Store
|
||||
|
||||
// clock is used for rwCerts expiry.
|
||||
clock tstime.DefaultClock
|
||||
|
||||
// certReads deduplicates concurrent cert Secret reads per domain.
|
||||
certReads singleflight.Group[string, *rwCert]
|
||||
|
||||
// rwCertsMu guards rwCerts.
|
||||
rwCertsMu sync.Mutex
|
||||
// rwCerts holds the last-known-good TLS cert and key per domain for
|
||||
// write replicas (cert share "rw"). Write replicas don't keep certs in
|
||||
// memory, so that they notice a recreated, empty cert Secret and issue
|
||||
// a new cert. Instead, they use rwCerts to skip Secret reads for
|
||||
// rwCertTTL and to keep serving TLS when the API server is unavailable.
|
||||
// Entries are never modified, only replaced, so that a read can tell
|
||||
// whether a write replaced the entry while the read was in flight.
|
||||
rwCerts map[string]*rwCert
|
||||
}
|
||||
|
||||
// rwCert is a TLS cert and key cached by a write replica.
|
||||
type rwCert struct {
|
||||
// cert and key are returned to every reader of the cache entry, so
|
||||
// they must not be modified.
|
||||
cert, key []byte
|
||||
// recheckAt is when the replica reads the cert Secret again.
|
||||
recheckAt time.Time
|
||||
}
|
||||
|
||||
// New returns a new Store that persists state to Kubernets Secret(s).
|
||||
@@ -153,7 +193,8 @@ func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*S
|
||||
// Write replicas never load certs in-memory to avoid a situation where,
|
||||
// after Ingress recreation (and the associated cert Secret recreation), new
|
||||
// TLS certs don't get issued because the write replica still has certs
|
||||
// in-memory. Instead, write replicas fetch certs from Secret on each request.
|
||||
// in-memory. Instead, write replicas fetch certs from Secret, with a short
|
||||
// TTL cache (see rwCerts).
|
||||
if s.certShareMode == "ro" {
|
||||
sel := s.certSecretSelector()
|
||||
if err := s.loadCerts(context.Background(), sel); err != nil {
|
||||
@@ -365,12 +406,14 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
|
||||
// TODO(irbekrm): certs for write replicas are currently not
|
||||
// written to memory to avoid out of sync memory state after
|
||||
// Ingress resources have been recreated. This means that TLS
|
||||
// certs for write replicas are retrieved from the Secret on
|
||||
// each HTTPS request. This is a temporary solution till we
|
||||
// certs for write replicas are retrieved from the Secret once
|
||||
// rwCertTTL has passed. This is a temporary solution till we
|
||||
// implement a Secret watch.
|
||||
if s.certShareMode != "rw" {
|
||||
s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
|
||||
s.memory.WriteState(ipn.StateKey(domain+".key"), key)
|
||||
} else {
|
||||
s.setRWCert(domain, cert, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -378,7 +421,9 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
|
||||
// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
|
||||
// domain-specific Secret. It first checks the in-memory store, if not found in
|
||||
// memory and running cert store in read-only mode, looks up a Secret.
|
||||
// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets.
|
||||
// Write replicas of HA Ingress don't use the in-memory store. They read the
|
||||
// Secret at most once per rwCertTTL, and serve the last-known-good cert if the
|
||||
// read fails with an error other than not found or forbidden.
|
||||
func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
|
||||
if err := dnsname.ValidHostname(domain); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
|
||||
@@ -396,14 +441,40 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
|
||||
return nil, nil, ipn.ErrStateNotExist
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
cached := s.getRWCert(domain)
|
||||
if cached != nil && s.clock.Now().Before(cached.recheckAt) {
|
||||
return cached.cert, cached.key, nil
|
||||
}
|
||||
// Handshakes that arrive while a read is in flight wait for that read
|
||||
// instead of reading the Secret again.
|
||||
c, err, _ := s.certReads.Do(domain, func() (*rwCert, error) {
|
||||
return s.readCertSecret(domain, cached)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return c.cert, c.key, nil
|
||||
}
|
||||
|
||||
// readCertSecret reads the TLS cert and key for domain from its Secret.
|
||||
// cached is the write replica's cached cert for domain at the start of the
|
||||
// read, or nil. The read updates the cache only if cached is still the cache
|
||||
// entry for domain, so that a read which started before a WriteTLSCertAndKey
|
||||
// doesn't replace or delete the cert that it wrote.
|
||||
func (s *Store) readCertSecret(domain string, cached *rwCert) (*rwCert, error) {
|
||||
readTimeout := timeout
|
||||
if cached != nil {
|
||||
readTimeout = rwCertFallbackTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), readTimeout)
|
||||
defer cancel()
|
||||
secret, err := s.client.GetSecret(ctx, domain)
|
||||
if err != nil {
|
||||
if kubeclient.IsNotFoundErr(err) {
|
||||
s.replaceRWCert(domain, cached, nil)
|
||||
// TODO(irbekrm): we should return a more specific error
|
||||
// that wraps ipn.ErrStateNotExist here.
|
||||
return nil, nil, ipn.ErrStateNotExist
|
||||
return nil, ipn.ErrStateNotExist
|
||||
}
|
||||
st, ok := err.(*kubeapi.Status)
|
||||
if ok && st.Code == http.StatusForbidden && (s.certShareMode == "ro" || s.certShareMode == "rw") {
|
||||
@@ -420,30 +491,81 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
|
||||
// This code path gets triggered by the admin UI's machine page,
|
||||
// which queries for the node's own TLS cert existing via the
|
||||
// "tls-cert-status" c2n API.
|
||||
return nil, nil, ipn.ErrStateNotExist
|
||||
s.replaceRWCert(domain, cached, nil)
|
||||
return nil, ipn.ErrStateNotExist
|
||||
}
|
||||
return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
|
||||
if cached != nil {
|
||||
// The API server is slow or unavailable. Serve the
|
||||
// last-known-good cert, and don't read the Secret again
|
||||
// for rwCertTTL, so that handshakes don't wait on every
|
||||
// read and we don't add load to the API server.
|
||||
s.logf("kubestore: getting TLS Secret %q: %v; using last-known-good cert", domain, err)
|
||||
s.replaceRWCert(domain, cached, s.newRWCert(cached.cert, cached.key))
|
||||
return cached, nil
|
||||
}
|
||||
return nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
|
||||
}
|
||||
cert = secret.Data[keyTLSCert]
|
||||
key = secret.Data[keyTLSKey]
|
||||
if len(cert) == 0 || len(key) == 0 {
|
||||
return nil, nil, ipn.ErrStateNotExist
|
||||
c := s.newRWCert(secret.Data[keyTLSCert], secret.Data[keyTLSKey])
|
||||
if len(c.cert) == 0 || len(c.key) == 0 {
|
||||
s.replaceRWCert(domain, cached, nil)
|
||||
return nil, ipn.ErrStateNotExist
|
||||
}
|
||||
// TODO(irbekrm): a read between these two separate writes would
|
||||
// get a mismatched cert and key. Allow writing both cert and
|
||||
// key to the memory store in a single, lock-protected operation.
|
||||
//
|
||||
// TODO(irbekrm): currently certs for write replicas of HA Ingress get
|
||||
// retrieved from the cluster Secret on each HTTPS request to avoid a
|
||||
// situation when after Ingress recreation stale certs are read from
|
||||
// retrieved from the cluster Secret once rwCertTTL has passed to avoid
|
||||
// a situation when after Ingress recreation stale certs are read from
|
||||
// memory.
|
||||
// Fix this by watching Secrets to ensure that memory store gets updated
|
||||
// when Secrets are deleted.
|
||||
if s.certShareMode == "ro" {
|
||||
s.memory.WriteState(ipn.StateKey(certKey), cert)
|
||||
s.memory.WriteState(ipn.StateKey(keyKey), key)
|
||||
s.memory.WriteState(ipn.StateKey(domain+".crt"), c.cert)
|
||||
s.memory.WriteState(ipn.StateKey(domain+".key"), c.key)
|
||||
} else {
|
||||
s.replaceRWCert(domain, cached, c)
|
||||
}
|
||||
return cert, key, nil
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// newRWCert returns a write replica cert for cert and key, to read again
|
||||
// after rwCertTTL.
|
||||
func (s *Store) newRWCert(cert, key []byte) *rwCert {
|
||||
return &rwCert{cert: cert, key: key, recheckAt: s.clock.Now().Add(rwCertTTL)}
|
||||
}
|
||||
|
||||
// getRWCert returns the cached write replica cert for domain, or nil.
|
||||
func (s *Store) getRWCert(domain string) *rwCert {
|
||||
s.rwCertsMu.Lock()
|
||||
defer s.rwCertsMu.Unlock()
|
||||
return s.rwCerts[domain]
|
||||
}
|
||||
|
||||
// setRWCert caches copies of cert and key for domain and resets its TTL.
|
||||
// It copies them because the caller owns them.
|
||||
func (s *Store) setRWCert(domain string, cert, key []byte) {
|
||||
c := s.newRWCert(bytes.Clone(cert), bytes.Clone(key))
|
||||
s.rwCertsMu.Lock()
|
||||
defer s.rwCertsMu.Unlock()
|
||||
mak.Set(&s.rwCerts, domain, c)
|
||||
}
|
||||
|
||||
// replaceRWCert replaces the cached write replica cert for domain with next,
|
||||
// or removes it if next is nil. It does nothing if the cached cert is no
|
||||
// longer old, for example because WriteTLSCertAndKey wrote a new cert. A nil
|
||||
// old means that no cert was cached.
|
||||
func (s *Store) replaceRWCert(domain string, old, next *rwCert) {
|
||||
s.rwCertsMu.Lock()
|
||||
defer s.rwCertsMu.Unlock()
|
||||
if s.rwCerts[domain] != old {
|
||||
return
|
||||
}
|
||||
if next == nil {
|
||||
delete(s.rwCerts, domain)
|
||||
return
|
||||
}
|
||||
mak.Set(&s.rwCerts, domain, next)
|
||||
}
|
||||
|
||||
func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
|
||||
|
||||
@@ -8,9 +8,14 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"tailscale.com/envknob"
|
||||
@@ -19,6 +24,8 @@ import (
|
||||
"tailscale.com/kube/kubeapi"
|
||||
"tailscale.com/kube/kubeclient"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
"tailscale.com/tstest"
|
||||
"tailscale.com/tstime"
|
||||
)
|
||||
|
||||
func TestKubernetesPodMigrationWithTPMAttestationKey(t *testing.T) {
|
||||
@@ -635,6 +642,401 @@ func TestReadTLSCertAndKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadTLSCertAndKeyWriteReplicaCache(t *testing.T) {
|
||||
const testDomain = "my-app.tailnetxyz.ts.net"
|
||||
|
||||
var (
|
||||
secretData map[string][]byte
|
||||
secretGetErr error
|
||||
gets int
|
||||
lastTimeout time.Duration
|
||||
)
|
||||
client := &kubeclient.FakeClient{
|
||||
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
|
||||
gets++
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
lastTimeout = time.Until(deadline)
|
||||
}
|
||||
if secretGetErr != nil {
|
||||
return nil, secretGetErr
|
||||
}
|
||||
return &kubeapi.Secret{Data: secretData}, nil
|
||||
},
|
||||
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
clock := tstest.NewClock(tstest.ClockOpts{})
|
||||
s := &Store{
|
||||
client: client,
|
||||
secretName: "ts-state",
|
||||
certShareMode: "rw",
|
||||
clock: tstime.DefaultClock{Clock: clock},
|
||||
logf: t.Logf,
|
||||
}
|
||||
tlsData := func(cert, key string) map[string][]byte {
|
||||
return map[string][]byte{"tls.crt": []byte(cert), "tls.key": []byte(key)}
|
||||
}
|
||||
|
||||
steps := []struct {
|
||||
name string
|
||||
advance time.Duration
|
||||
secretData map[string][]byte
|
||||
secretGetErr error
|
||||
// write, if set, writes cert and key with WriteTLSCertAndKey
|
||||
// before the read.
|
||||
write map[string][]byte
|
||||
wantCert string
|
||||
wantKey string
|
||||
wantErr error
|
||||
wantGet bool
|
||||
wantTimeout time.Duration // max GetSecret timeout, if wantGet
|
||||
}{
|
||||
{
|
||||
name: "no_cache_api_error",
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantErr: errors.New("api error"),
|
||||
wantGet: true,
|
||||
wantTimeout: timeout,
|
||||
},
|
||||
{
|
||||
name: "first_read_from_secret",
|
||||
secretData: tlsData("cert-a", "key-a"),
|
||||
wantCert: "cert-a",
|
||||
wantKey: "key-a",
|
||||
wantGet: true,
|
||||
wantTimeout: timeout,
|
||||
},
|
||||
{
|
||||
name: "within_ttl_served_from_cache",
|
||||
advance: rwCertTTL / 2,
|
||||
secretData: tlsData("cert-b", "key-b"),
|
||||
wantCert: "cert-a",
|
||||
wantKey: "key-a",
|
||||
},
|
||||
{
|
||||
name: "after_ttl_read_from_secret",
|
||||
advance: rwCertTTL,
|
||||
secretData: tlsData("cert-b", "key-b"),
|
||||
wantCert: "cert-b",
|
||||
wantKey: "key-b",
|
||||
wantGet: true,
|
||||
wantTimeout: rwCertFallbackTimeout,
|
||||
},
|
||||
{
|
||||
name: "after_ttl_api_error_serves_last_known_good",
|
||||
advance: rwCertTTL,
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantCert: "cert-b",
|
||||
wantKey: "key-b",
|
||||
wantGet: true,
|
||||
wantTimeout: rwCertFallbackTimeout,
|
||||
},
|
||||
{
|
||||
name: "after_fallback_served_from_cache",
|
||||
advance: rwCertTTL / 2,
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantCert: "cert-b",
|
||||
wantKey: "key-b",
|
||||
},
|
||||
{
|
||||
name: "after_ttl_secret_emptied",
|
||||
advance: rwCertTTL,
|
||||
secretData: map[string][]byte{},
|
||||
wantErr: ipn.ErrStateNotExist,
|
||||
wantGet: true,
|
||||
},
|
||||
{
|
||||
name: "emptied_secret_clears_cache",
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantErr: errors.New("api error"),
|
||||
wantGet: true,
|
||||
wantTimeout: timeout,
|
||||
},
|
||||
{
|
||||
name: "written_cert_served_from_cache",
|
||||
write: tlsData("cert-c", "key-c"),
|
||||
secretData: tlsData("cert-c", "key-c"),
|
||||
wantCert: "cert-c",
|
||||
wantKey: "key-c",
|
||||
},
|
||||
{
|
||||
name: "after_ttl_secret_not_found",
|
||||
advance: rwCertTTL,
|
||||
secretGetErr: &kubeapi.Status{Code: 404},
|
||||
wantErr: ipn.ErrStateNotExist,
|
||||
wantGet: true,
|
||||
},
|
||||
{
|
||||
name: "not_found_clears_cache",
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantErr: errors.New("api error"),
|
||||
wantGet: true,
|
||||
wantTimeout: timeout,
|
||||
},
|
||||
{
|
||||
name: "written_cert_served_from_cache_again",
|
||||
write: tlsData("cert-d", "key-d"),
|
||||
secretData: tlsData("cert-d", "key-d"),
|
||||
wantCert: "cert-d",
|
||||
wantKey: "key-d",
|
||||
},
|
||||
{
|
||||
name: "after_ttl_secret_forbidden",
|
||||
advance: rwCertTTL,
|
||||
secretGetErr: &kubeapi.Status{Code: 403},
|
||||
wantErr: ipn.ErrStateNotExist,
|
||||
wantGet: true,
|
||||
},
|
||||
{
|
||||
name: "forbidden_clears_cache",
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantErr: errors.New("api error"),
|
||||
wantGet: true,
|
||||
wantTimeout: timeout,
|
||||
},
|
||||
}
|
||||
for _, st := range steps {
|
||||
clock.Advance(st.advance)
|
||||
secretData = st.secretData
|
||||
secretGetErr = st.secretGetErr
|
||||
if st.write != nil {
|
||||
if err := s.WriteTLSCertAndKey(testDomain, st.write["tls.crt"], st.write["tls.key"]); err != nil {
|
||||
t.Fatalf("%s: WriteTLSCertAndKey: %v", st.name, err)
|
||||
}
|
||||
}
|
||||
gets = 0
|
||||
|
||||
cert, key, err := s.ReadTLSCertAndKey(testDomain)
|
||||
switch {
|
||||
case st.wantErr != nil:
|
||||
if err == nil || !strings.Contains(err.Error(), st.wantErr.Error()) {
|
||||
t.Errorf("%s: got error %v, want error containing %v", st.name, err, st.wantErr)
|
||||
}
|
||||
case err != nil:
|
||||
t.Errorf("%s: unexpected error: %v", st.name, err)
|
||||
default:
|
||||
if string(cert) != st.wantCert || string(key) != st.wantKey {
|
||||
t.Errorf("%s: got cert %q key %q, want cert %q key %q", st.name, cert, key, st.wantCert, st.wantKey)
|
||||
}
|
||||
}
|
||||
if gotGet := gets > 0; gotGet != st.wantGet {
|
||||
t.Errorf("%s: read Secret = %v, want %v", st.name, gotGet, st.wantGet)
|
||||
}
|
||||
if st.wantGet && st.wantTimeout != 0 && (lastTimeout > st.wantTimeout || lastTimeout < st.wantTimeout-time.Second) {
|
||||
t.Errorf("%s: GetSecret timeout = %v, want %v", st.name, lastTimeout, st.wantTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadTLSCertAndKeyWriteReplicaWriteDuringRead tests that a cert Secret
|
||||
// read that is in flight when WriteTLSCertAndKey writes a new cert doesn't
|
||||
// replace or remove the new cert in the write replica's cache.
|
||||
func TestReadTLSCertAndKeyWriteReplicaWriteDuringRead(t *testing.T) {
|
||||
const testDomain = "my-app.tailnetxyz.ts.net"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
// cached, if set, is cached and expired before the read.
|
||||
cached string
|
||||
// secretData and secretGetErr are what the in-flight read gets.
|
||||
secretData map[string][]byte
|
||||
secretGetErr error
|
||||
wantCert string
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "read_returns_old_cert",
|
||||
secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")},
|
||||
wantCert: "cert-old",
|
||||
},
|
||||
{
|
||||
name: "read_returns_not_found",
|
||||
secretGetErr: &kubeapi.Status{Code: 404},
|
||||
wantErr: ipn.ErrStateNotExist,
|
||||
},
|
||||
{
|
||||
name: "read_returns_old_cert_with_cached_cert",
|
||||
cached: "cert-old",
|
||||
secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")},
|
||||
wantCert: "cert-old",
|
||||
},
|
||||
{
|
||||
name: "read_returns_not_found_with_cached_cert",
|
||||
cached: "cert-old",
|
||||
secretGetErr: &kubeapi.Status{Code: 404},
|
||||
wantErr: ipn.ErrStateNotExist,
|
||||
},
|
||||
{
|
||||
name: "read_returns_api_error_with_cached_cert",
|
||||
cached: "cert-old",
|
||||
secretGetErr: errors.New("api error"),
|
||||
wantCert: "cert-old",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var gets atomic.Int32
|
||||
getStarted := make(chan struct{})
|
||||
releaseGet := make(chan struct{})
|
||||
client := &kubeclient.FakeClient{
|
||||
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
|
||||
if gets.Add(1) > 1 {
|
||||
// WriteTLSCertAndKey reads the Secret
|
||||
// before it updates it.
|
||||
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil
|
||||
}
|
||||
close(getStarted)
|
||||
<-releaseGet
|
||||
if tt.secretGetErr != nil {
|
||||
return nil, tt.secretGetErr
|
||||
}
|
||||
return &kubeapi.Secret{Data: tt.secretData}, nil
|
||||
},
|
||||
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
|
||||
return nil
|
||||
},
|
||||
UpdateSecretImpl: func(context.Context, *kubeapi.Secret) error {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
clock := tstest.NewClock(tstest.ClockOpts{})
|
||||
s := &Store{
|
||||
client: client,
|
||||
secretName: "ts-state",
|
||||
certShareMode: "rw",
|
||||
clock: tstime.DefaultClock{Clock: clock},
|
||||
logf: t.Logf,
|
||||
}
|
||||
if tt.cached != "" {
|
||||
s.setRWCert(testDomain, []byte(tt.cached), []byte("key"))
|
||||
clock.Advance(rwCertTTL)
|
||||
}
|
||||
|
||||
type result struct {
|
||||
cert []byte
|
||||
err error
|
||||
}
|
||||
done := make(chan result)
|
||||
go func() {
|
||||
cert, _, err := s.ReadTLSCertAndKey(testDomain)
|
||||
done <- result{cert, err}
|
||||
}()
|
||||
<-getStarted
|
||||
if err := s.WriteTLSCertAndKey(testDomain, []byte("cert-new"), []byte("key-new")); err != nil {
|
||||
t.Fatalf("WriteTLSCertAndKey: %v", err)
|
||||
}
|
||||
close(releaseGet)
|
||||
|
||||
res := <-done
|
||||
if tt.wantErr != nil {
|
||||
if !errors.Is(res.err, tt.wantErr) {
|
||||
t.Errorf("in-flight read: got error %v, want %v", res.err, tt.wantErr)
|
||||
}
|
||||
} else if res.err != nil || string(res.cert) != tt.wantCert {
|
||||
t.Errorf("in-flight read: got cert %q, error %v, want cert %q", res.cert, res.err, tt.wantCert)
|
||||
}
|
||||
|
||||
// The next read must serve the new cert from the cache.
|
||||
getsBefore := gets.Load()
|
||||
cert, key, err := s.ReadTLSCertAndKey(testDomain)
|
||||
if err != nil {
|
||||
t.Fatalf("next read: %v", err)
|
||||
}
|
||||
if string(cert) != "cert-new" || string(key) != "key-new" {
|
||||
t.Errorf("next read: got cert %q key %q, want cert %q key %q", cert, key, "cert-new", "key-new")
|
||||
}
|
||||
if gets.Load() != getsBefore {
|
||||
t.Errorf("next read: read Secret, want cached cert")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteTLSCertAndKeyWriteReplicaCopiesCert tests that a write replica
|
||||
// caches copies of the cert and key that it writes, so that a caller that
|
||||
// modifies them afterwards doesn't change the cached cert.
|
||||
func TestWriteTLSCertAndKeyWriteReplicaCopiesCert(t *testing.T) {
|
||||
const testDomain = "my-app.tailnetxyz.ts.net"
|
||||
|
||||
client := &kubeclient.FakeClient{
|
||||
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
|
||||
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert"), "tls.key": []byte("key")}}, nil
|
||||
},
|
||||
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
s := &Store{
|
||||
client: client,
|
||||
secretName: "ts-state",
|
||||
certShareMode: "rw",
|
||||
logf: t.Logf,
|
||||
}
|
||||
cert, key := []byte("cert"), []byte("key")
|
||||
if err := s.WriteTLSCertAndKey(testDomain, cert, key); err != nil {
|
||||
t.Fatalf("WriteTLSCertAndKey: %v", err)
|
||||
}
|
||||
copy(cert, "XXXX")
|
||||
copy(key, "XXX")
|
||||
|
||||
gotCert, gotKey, err := s.ReadTLSCertAndKey(testDomain)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadTLSCertAndKey: %v", err)
|
||||
}
|
||||
if string(gotCert) != "cert" || string(gotKey) != "key" {
|
||||
t.Errorf("got cert %q key %q, want cert %q key %q", gotCert, gotKey, "cert", "key")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadTLSCertAndKeyWriteReplicaConcurrentReads tests that concurrent
|
||||
// reads of an expired cached cert read the cert Secret only once.
|
||||
func TestReadTLSCertAndKeyWriteReplicaConcurrentReads(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
const (
|
||||
testDomain = "my-app.tailnetxyz.ts.net"
|
||||
readers = 10
|
||||
)
|
||||
var gets atomic.Int32
|
||||
releaseGet := make(chan struct{})
|
||||
client := &kubeclient.FakeClient{
|
||||
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
|
||||
gets.Add(1)
|
||||
<-releaseGet
|
||||
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil
|
||||
},
|
||||
}
|
||||
clock := tstest.NewClock(tstest.ClockOpts{})
|
||||
s := &Store{
|
||||
client: client,
|
||||
secretName: "ts-state",
|
||||
certShareMode: "rw",
|
||||
clock: tstime.DefaultClock{Clock: clock},
|
||||
logf: t.Logf,
|
||||
}
|
||||
s.setRWCert(testDomain, []byte("cert-old"), []byte("key-old"))
|
||||
clock.Advance(rwCertTTL)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range readers {
|
||||
wg.Go(func() {
|
||||
cert, _, err := s.ReadTLSCertAndKey(testDomain)
|
||||
if err != nil || string(cert) != "cert-new" {
|
||||
t.Errorf("got cert %q, error %v, want cert %q", cert, err, "cert-new")
|
||||
}
|
||||
})
|
||||
}
|
||||
// Wait for all readers to block on the in-flight read.
|
||||
synctest.Wait()
|
||||
close(releaseGet)
|
||||
wg.Wait()
|
||||
|
||||
if got := gets.Load(); got != 1 {
|
||||
t.Errorf("GetSecret called %d times, want 1", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewWithClient(t *testing.T) {
|
||||
const (
|
||||
secretName = "ts-state"
|
||||
|
||||
Reference in new issue
Block a user