ipn/store/kubestore: cache TLS certs briefly on write replicas (#22103)

The write replica of an HA ProxyGroup (cert share "rw") doesn't keep TLS
certs in memory, so that it notices a recreated, empty cert Secret and
issues a new cert. As a result, it read the cert Secret from the API
server on every TLS handshake, with a 30s timeout. When the API server
was slow or unavailable, handshakes on that replica waited for the read,
and feature/acme repeated the read under the per-domain lock. Clients
with a 10s handshake timeout, such as kubectl, failed.

This commit adds a small per-domain cache of the last-known-good cert
and key for write replicas. The replica serves a cached cert for 30s
before it reads the Secret again. When it has a cached cert, the read
uses a 3s timeout, and on an error other than not found or forbidden
the replica serves the cached cert and waits another 30s before the
next read. A read that finds the Secret missing, forbidden or empty
removes the cached cert and returns ipn.ErrStateNotExist as before, so
a recreated Ingress still gets a new cert, at most 30s later.
Concurrent reads of the same Secret share one GET, and a read that is
in flight when a cert is written does not replace the written cert.

A Secret watch remains the long-term fix.

Fixes #22009

Change-Id: Id107bf0a7628a1a026f94e621173d7c5d02fb451

Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
David Bond authored and GitHub committed 2026-11-20 12:32:02 +00:00
1 parent 519a9f502c
commit c8ba9da543
2 files changed
+541 -17

No files matched your search

+139 -17
View File
@@ -14,6 +14,7 @@ import (
"net/http"
"os"
"strings"
"sync"
"time"
"tailscale.com/envknob"
@@ -24,9 +25,11 @@ import (
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstime"
"tailscale.com/types/logger"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
"tailscale.com/util/singleflight"
)
func init() {
@@ -44,6 +47,17 @@ const (
// state Secret and emit an Event.
timeout = 30 * time.Second
// rwCertTTL is how long a write replica serves a TLS cert from its
// in-memory cache before it reads the cert Secret again. A short TTL
// limits how long the replica can serve a stale cert after the cert
// Secret is recreated, for example after Ingress recreation.
rwCertTTL = 30 * time.Second
// rwCertFallbackTimeout is the timeout for a write replica's cert
// Secret read when it has a last-known-good cert to fall back to.
// TLS handshakes wait for this read, so it must be much shorter than
// typical client handshake timeouts (10s for kubectl).
rwCertFallbackTimeout = 3 * time.Second
reasonTailscaleStateUpdated = "TailscaledStateUpdated"
reasonTailscaleStateLoaded = "TailscaleStateLoaded"
reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed"
@@ -90,6 +104,32 @@ type Store struct {
// memory holds the latest tailscale state. Writes write state to a kube
// Secret and memory, Reads read from memory.
memory mem.Store
// clock is used for rwCerts expiry.
clock tstime.DefaultClock
// certReads deduplicates concurrent cert Secret reads per domain.
certReads singleflight.Group[string, *rwCert]
// rwCertsMu guards rwCerts.
rwCertsMu sync.Mutex
// rwCerts holds the last-known-good TLS cert and key per domain for
// write replicas (cert share "rw"). Write replicas don't keep certs in
// memory, so that they notice a recreated, empty cert Secret and issue
// a new cert. Instead, they use rwCerts to skip Secret reads for
// rwCertTTL and to keep serving TLS when the API server is unavailable.
// Entries are never modified, only replaced, so that a read can tell
// whether a write replaced the entry while the read was in flight.
rwCerts map[string]*rwCert
}
// rwCert is a TLS cert and key cached by a write replica.
type rwCert struct {
// cert and key are returned to every reader of the cache entry, so
// they must not be modified.
cert, key []byte
// recheckAt is when the replica reads the cert Secret again.
recheckAt time.Time
}
// New returns a new Store that persists state to Kubernets Secret(s).
@@ -153,7 +193,8 @@ func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*S
// Write replicas never load certs in-memory to avoid a situation where,
// after Ingress recreation (and the associated cert Secret recreation), new
// TLS certs don't get issued because the write replica still has certs
// in-memory. Instead, write replicas fetch certs from Secret on each request.
// in-memory. Instead, write replicas fetch certs from Secret, with a short
// TTL cache (see rwCerts).
if s.certShareMode == "ro" {
sel := s.certSecretSelector()
if err := s.loadCerts(context.Background(), sel); err != nil {
@@ -365,12 +406,14 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
// TODO(irbekrm): certs for write replicas are currently not
// written to memory to avoid out of sync memory state after
// Ingress resources have been recreated. This means that TLS
// certs for write replicas are retrieved from the Secret on
// each HTTPS request. This is a temporary solution till we
// certs for write replicas are retrieved from the Secret once
// rwCertTTL has passed. This is a temporary solution till we
// implement a Secret watch.
if s.certShareMode != "rw" {
s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
s.memory.WriteState(ipn.StateKey(domain+".key"), key)
} else {
s.setRWCert(domain, cert, key)
}
return nil
}
@@ -378,7 +421,9 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
// domain-specific Secret. It first checks the in-memory store, if not found in
// memory and running cert store in read-only mode, looks up a Secret.
// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets.
// Write replicas of HA Ingress don't use the in-memory store. They read the
// Secret at most once per rwCertTTL, and serve the last-known-good cert if the
// read fails with an error other than not found or forbidden.
func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
if err := dnsname.ValidHostname(domain); err != nil {
return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
@@ -396,14 +441,40 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
return nil, nil, ipn.ErrStateNotExist
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
cached := s.getRWCert(domain)
if cached != nil && s.clock.Now().Before(cached.recheckAt) {
return cached.cert, cached.key, nil
}
// Handshakes that arrive while a read is in flight wait for that read
// instead of reading the Secret again.
c, err, _ := s.certReads.Do(domain, func() (*rwCert, error) {
return s.readCertSecret(domain, cached)
})
if err != nil {
return nil, nil, err
}
return c.cert, c.key, nil
}
// readCertSecret reads the TLS cert and key for domain from its Secret.
// cached is the write replica's cached cert for domain at the start of the
// read, or nil. The read updates the cache only if cached is still the cache
// entry for domain, so that a read which started before a WriteTLSCertAndKey
// doesn't replace or delete the cert that it wrote.
func (s *Store) readCertSecret(domain string, cached *rwCert) (*rwCert, error) {
readTimeout := timeout
if cached != nil {
readTimeout = rwCertFallbackTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), readTimeout)
defer cancel()
secret, err := s.client.GetSecret(ctx, domain)
if err != nil {
if kubeclient.IsNotFoundErr(err) {
s.replaceRWCert(domain, cached, nil)
// TODO(irbekrm): we should return a more specific error
// that wraps ipn.ErrStateNotExist here.
return nil, nil, ipn.ErrStateNotExist
return nil, ipn.ErrStateNotExist
}
st, ok := err.(*kubeapi.Status)
if ok && st.Code == http.StatusForbidden && (s.certShareMode == "ro" || s.certShareMode == "rw") {
@@ -420,30 +491,81 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
// This code path gets triggered by the admin UI's machine page,
// which queries for the node's own TLS cert existing via the
// "tls-cert-status" c2n API.
return nil, nil, ipn.ErrStateNotExist
s.replaceRWCert(domain, cached, nil)
return nil, ipn.ErrStateNotExist
}
return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
if cached != nil {
// The API server is slow or unavailable. Serve the
// last-known-good cert, and don't read the Secret again
// for rwCertTTL, so that handshakes don't wait on every
// read and we don't add load to the API server.
s.logf("kubestore: getting TLS Secret %q: %v; using last-known-good cert", domain, err)
s.replaceRWCert(domain, cached, s.newRWCert(cached.cert, cached.key))
return cached, nil
}
return nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
}
cert = secret.Data[keyTLSCert]
key = secret.Data[keyTLSKey]
if len(cert) == 0 || len(key) == 0 {
return nil, nil, ipn.ErrStateNotExist
c := s.newRWCert(secret.Data[keyTLSCert], secret.Data[keyTLSKey])
if len(c.cert) == 0 || len(c.key) == 0 {
s.replaceRWCert(domain, cached, nil)
return nil, ipn.ErrStateNotExist
}
// TODO(irbekrm): a read between these two separate writes would
// get a mismatched cert and key. Allow writing both cert and
// key to the memory store in a single, lock-protected operation.
//
// TODO(irbekrm): currently certs for write replicas of HA Ingress get
// retrieved from the cluster Secret on each HTTPS request to avoid a
// situation when after Ingress recreation stale certs are read from
// retrieved from the cluster Secret once rwCertTTL has passed to avoid
// a situation when after Ingress recreation stale certs are read from
// memory.
// Fix this by watching Secrets to ensure that memory store gets updated
// when Secrets are deleted.
if s.certShareMode == "ro" {
s.memory.WriteState(ipn.StateKey(certKey), cert)
s.memory.WriteState(ipn.StateKey(keyKey), key)
s.memory.WriteState(ipn.StateKey(domain+".crt"), c.cert)
s.memory.WriteState(ipn.StateKey(domain+".key"), c.key)
} else {
s.replaceRWCert(domain, cached, c)
}
return cert, key, nil
return c, nil
}
// newRWCert returns a write replica cert for cert and key, to read again
// after rwCertTTL.
func (s *Store) newRWCert(cert, key []byte) *rwCert {
return &rwCert{cert: cert, key: key, recheckAt: s.clock.Now().Add(rwCertTTL)}
}
// getRWCert returns the cached write replica cert for domain, or nil.
func (s *Store) getRWCert(domain string) *rwCert {
s.rwCertsMu.Lock()
defer s.rwCertsMu.Unlock()
return s.rwCerts[domain]
}
// setRWCert caches copies of cert and key for domain and resets its TTL.
// It copies them because the caller owns them.
func (s *Store) setRWCert(domain string, cert, key []byte) {
c := s.newRWCert(bytes.Clone(cert), bytes.Clone(key))
s.rwCertsMu.Lock()
defer s.rwCertsMu.Unlock()
mak.Set(&s.rwCerts, domain, c)
}
// replaceRWCert replaces the cached write replica cert for domain with next,
// or removes it if next is nil. It does nothing if the cached cert is no
// longer old, for example because WriteTLSCertAndKey wrote a new cert. A nil
// old means that no cert was cached.
func (s *Store) replaceRWCert(domain string, old, next *rwCert) {
s.rwCertsMu.Lock()
defer s.rwCertsMu.Unlock()
if s.rwCerts[domain] != old {
return
}
if next == nil {
delete(s.rwCerts, domain)
return
}
mak.Set(&s.rwCerts, domain, next)
}
func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
+402
View File
@@ -8,9 +8,14 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/google/go-cmp/cmp"
"tailscale.com/envknob"
@@ -19,6 +24,8 @@ import (
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/tstime"
)
func TestKubernetesPodMigrationWithTPMAttestationKey(t *testing.T) {
@@ -635,6 +642,401 @@ func TestReadTLSCertAndKey(t *testing.T) {
}
}
func TestReadTLSCertAndKeyWriteReplicaCache(t *testing.T) {
const testDomain = "my-app.tailnetxyz.ts.net"
var (
secretData map[string][]byte
secretGetErr error
gets int
lastTimeout time.Duration
)
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
gets++
if deadline, ok := ctx.Deadline(); ok {
lastTimeout = time.Until(deadline)
}
if secretGetErr != nil {
return nil, secretGetErr
}
return &kubeapi.Secret{Data: secretData}, nil
},
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
return nil
},
}
clock := tstest.NewClock(tstest.ClockOpts{})
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: "rw",
clock: tstime.DefaultClock{Clock: clock},
logf: t.Logf,
}
tlsData := func(cert, key string) map[string][]byte {
return map[string][]byte{"tls.crt": []byte(cert), "tls.key": []byte(key)}
}
steps := []struct {
name string
advance time.Duration
secretData map[string][]byte
secretGetErr error
// write, if set, writes cert and key with WriteTLSCertAndKey
// before the read.
write map[string][]byte
wantCert string
wantKey string
wantErr error
wantGet bool
wantTimeout time.Duration // max GetSecret timeout, if wantGet
}{
{
name: "no_cache_api_error",
secretGetErr: errors.New("api error"),
wantErr: errors.New("api error"),
wantGet: true,
wantTimeout: timeout,
},
{
name: "first_read_from_secret",
secretData: tlsData("cert-a", "key-a"),
wantCert: "cert-a",
wantKey: "key-a",
wantGet: true,
wantTimeout: timeout,
},
{
name: "within_ttl_served_from_cache",
advance: rwCertTTL / 2,
secretData: tlsData("cert-b", "key-b"),
wantCert: "cert-a",
wantKey: "key-a",
},
{
name: "after_ttl_read_from_secret",
advance: rwCertTTL,
secretData: tlsData("cert-b", "key-b"),
wantCert: "cert-b",
wantKey: "key-b",
wantGet: true,
wantTimeout: rwCertFallbackTimeout,
},
{
name: "after_ttl_api_error_serves_last_known_good",
advance: rwCertTTL,
secretGetErr: errors.New("api error"),
wantCert: "cert-b",
wantKey: "key-b",
wantGet: true,
wantTimeout: rwCertFallbackTimeout,
},
{
name: "after_fallback_served_from_cache",
advance: rwCertTTL / 2,
secretGetErr: errors.New("api error"),
wantCert: "cert-b",
wantKey: "key-b",
},
{
name: "after_ttl_secret_emptied",
advance: rwCertTTL,
secretData: map[string][]byte{},
wantErr: ipn.ErrStateNotExist,
wantGet: true,
},
{
name: "emptied_secret_clears_cache",
secretGetErr: errors.New("api error"),
wantErr: errors.New("api error"),
wantGet: true,
wantTimeout: timeout,
},
{
name: "written_cert_served_from_cache",
write: tlsData("cert-c", "key-c"),
secretData: tlsData("cert-c", "key-c"),
wantCert: "cert-c",
wantKey: "key-c",
},
{
name: "after_ttl_secret_not_found",
advance: rwCertTTL,
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
wantGet: true,
},
{
name: "not_found_clears_cache",
secretGetErr: errors.New("api error"),
wantErr: errors.New("api error"),
wantGet: true,
wantTimeout: timeout,
},
{
name: "written_cert_served_from_cache_again",
write: tlsData("cert-d", "key-d"),
secretData: tlsData("cert-d", "key-d"),
wantCert: "cert-d",
wantKey: "key-d",
},
{
name: "after_ttl_secret_forbidden",
advance: rwCertTTL,
secretGetErr: &kubeapi.Status{Code: 403},
wantErr: ipn.ErrStateNotExist,
wantGet: true,
},
{
name: "forbidden_clears_cache",
secretGetErr: errors.New("api error"),
wantErr: errors.New("api error"),
wantGet: true,
wantTimeout: timeout,
},
}
for _, st := range steps {
clock.Advance(st.advance)
secretData = st.secretData
secretGetErr = st.secretGetErr
if st.write != nil {
if err := s.WriteTLSCertAndKey(testDomain, st.write["tls.crt"], st.write["tls.key"]); err != nil {
t.Fatalf("%s: WriteTLSCertAndKey: %v", st.name, err)
}
}
gets = 0
cert, key, err := s.ReadTLSCertAndKey(testDomain)
switch {
case st.wantErr != nil:
if err == nil || !strings.Contains(err.Error(), st.wantErr.Error()) {
t.Errorf("%s: got error %v, want error containing %v", st.name, err, st.wantErr)
}
case err != nil:
t.Errorf("%s: unexpected error: %v", st.name, err)
default:
if string(cert) != st.wantCert || string(key) != st.wantKey {
t.Errorf("%s: got cert %q key %q, want cert %q key %q", st.name, cert, key, st.wantCert, st.wantKey)
}
}
if gotGet := gets > 0; gotGet != st.wantGet {
t.Errorf("%s: read Secret = %v, want %v", st.name, gotGet, st.wantGet)
}
if st.wantGet && st.wantTimeout != 0 && (lastTimeout > st.wantTimeout || lastTimeout < st.wantTimeout-time.Second) {
t.Errorf("%s: GetSecret timeout = %v, want %v", st.name, lastTimeout, st.wantTimeout)
}
}
}
// TestReadTLSCertAndKeyWriteReplicaWriteDuringRead tests that a cert Secret
// read that is in flight when WriteTLSCertAndKey writes a new cert doesn't
// replace or remove the new cert in the write replica's cache.
func TestReadTLSCertAndKeyWriteReplicaWriteDuringRead(t *testing.T) {
const testDomain = "my-app.tailnetxyz.ts.net"
tests := []struct {
name string
// cached, if set, is cached and expired before the read.
cached string
// secretData and secretGetErr are what the in-flight read gets.
secretData map[string][]byte
secretGetErr error
wantCert string
wantErr error
}{
{
name: "read_returns_old_cert",
secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")},
wantCert: "cert-old",
},
{
name: "read_returns_not_found",
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
},
{
name: "read_returns_old_cert_with_cached_cert",
cached: "cert-old",
secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")},
wantCert: "cert-old",
},
{
name: "read_returns_not_found_with_cached_cert",
cached: "cert-old",
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
},
{
name: "read_returns_api_error_with_cached_cert",
cached: "cert-old",
secretGetErr: errors.New("api error"),
wantCert: "cert-old",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var gets atomic.Int32
getStarted := make(chan struct{})
releaseGet := make(chan struct{})
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if gets.Add(1) > 1 {
// WriteTLSCertAndKey reads the Secret
// before it updates it.
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil
}
close(getStarted)
<-releaseGet
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
return &kubeapi.Secret{Data: tt.secretData}, nil
},
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
return nil
},
UpdateSecretImpl: func(context.Context, *kubeapi.Secret) error {
return nil
},
}
clock := tstest.NewClock(tstest.ClockOpts{})
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: "rw",
clock: tstime.DefaultClock{Clock: clock},
logf: t.Logf,
}
if tt.cached != "" {
s.setRWCert(testDomain, []byte(tt.cached), []byte("key"))
clock.Advance(rwCertTTL)
}
type result struct {
cert []byte
err error
}
done := make(chan result)
go func() {
cert, _, err := s.ReadTLSCertAndKey(testDomain)
done <- result{cert, err}
}()
<-getStarted
if err := s.WriteTLSCertAndKey(testDomain, []byte("cert-new"), []byte("key-new")); err != nil {
t.Fatalf("WriteTLSCertAndKey: %v", err)
}
close(releaseGet)
res := <-done
if tt.wantErr != nil {
if !errors.Is(res.err, tt.wantErr) {
t.Errorf("in-flight read: got error %v, want %v", res.err, tt.wantErr)
}
} else if res.err != nil || string(res.cert) != tt.wantCert {
t.Errorf("in-flight read: got cert %q, error %v, want cert %q", res.cert, res.err, tt.wantCert)
}
// The next read must serve the new cert from the cache.
getsBefore := gets.Load()
cert, key, err := s.ReadTLSCertAndKey(testDomain)
if err != nil {
t.Fatalf("next read: %v", err)
}
if string(cert) != "cert-new" || string(key) != "key-new" {
t.Errorf("next read: got cert %q key %q, want cert %q key %q", cert, key, "cert-new", "key-new")
}
if gets.Load() != getsBefore {
t.Errorf("next read: read Secret, want cached cert")
}
})
}
}
// TestWriteTLSCertAndKeyWriteReplicaCopiesCert tests that a write replica
// caches copies of the cert and key that it writes, so that a caller that
// modifies them afterwards doesn't change the cached cert.
func TestWriteTLSCertAndKeyWriteReplicaCopiesCert(t *testing.T) {
const testDomain = "my-app.tailnetxyz.ts.net"
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert"), "tls.key": []byte("key")}}, nil
},
JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error {
return nil
},
}
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: "rw",
logf: t.Logf,
}
cert, key := []byte("cert"), []byte("key")
if err := s.WriteTLSCertAndKey(testDomain, cert, key); err != nil {
t.Fatalf("WriteTLSCertAndKey: %v", err)
}
copy(cert, "XXXX")
copy(key, "XXX")
gotCert, gotKey, err := s.ReadTLSCertAndKey(testDomain)
if err != nil {
t.Fatalf("ReadTLSCertAndKey: %v", err)
}
if string(gotCert) != "cert" || string(gotKey) != "key" {
t.Errorf("got cert %q key %q, want cert %q key %q", gotCert, gotKey, "cert", "key")
}
}
// TestReadTLSCertAndKeyWriteReplicaConcurrentReads tests that concurrent
// reads of an expired cached cert read the cert Secret only once.
func TestReadTLSCertAndKeyWriteReplicaConcurrentReads(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
readers = 10
)
var gets atomic.Int32
releaseGet := make(chan struct{})
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
gets.Add(1)
<-releaseGet
return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil
},
}
clock := tstest.NewClock(tstest.ClockOpts{})
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: "rw",
clock: tstime.DefaultClock{Clock: clock},
logf: t.Logf,
}
s.setRWCert(testDomain, []byte("cert-old"), []byte("key-old"))
clock.Advance(rwCertTTL)
var wg sync.WaitGroup
for range readers {
wg.Go(func() {
cert, _, err := s.ReadTLSCertAndKey(testDomain)
if err != nil || string(cert) != "cert-new" {
t.Errorf("got cert %q, error %v, want cert %q", cert, err, "cert-new")
}
})
}
// Wait for all readers to block on the in-flight read.
synctest.Wait()
close(releaseGet)
wg.Wait()
if got := gets.Load(); got != 1 {
t.Errorf("GetSecret called %d times, want 1", got)
}
})
}
func TestNewWithClient(t *testing.T) {
const (
secretName = "ts-state"