From c8ba9da543ff4fc1e28b341f2197dd2356b3c9fd Mon Sep 17 00:00:00 2001 From: David Bond Date: Fri, 20 Nov 2026 12:32:02 +0000 Subject: [PATCH] ipn/store/kubestore: cache TLS certs briefly on write replicas (#22103) The write replica of an HA ProxyGroup (cert share "rw") doesn't keep TLS certs in memory, so that it notices a recreated, empty cert Secret and issues a new cert. As a result, it read the cert Secret from the API server on every TLS handshake, with a 30s timeout. When the API server was slow or unavailable, handshakes on that replica waited for the read, and feature/acme repeated the read under the per-domain lock. Clients with a 10s handshake timeout, such as kubectl, failed. This commit adds a small per-domain cache of the last-known-good cert and key for write replicas. The replica serves a cached cert for 30s before it reads the Secret again. When it has a cached cert, the read uses a 3s timeout, and on an error other than not found or forbidden the replica serves the cached cert and waits another 30s before the next read. A read that finds the Secret missing, forbidden or empty removes the cached cert and returns ipn.ErrStateNotExist as before, so a recreated Ingress still gets a new cert, at most 30s later. Concurrent reads of the same Secret share one GET, and a read that is in flight when a cert is written does not replace the written cert. A Secret watch remains the long-term fix. Fixes #22009 Change-Id: Id107bf0a7628a1a026f94e621173d7c5d02fb451 Signed-off-by: David Bond --- ipn/store/kubestore/store_kube.go | 156 ++++++++-- ipn/store/kubestore/store_kube_test.go | 402 +++++++++++++++++++++++++ 2 files changed, 541 insertions(+), 17 deletions(-) diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 219243d3f..e3873654e 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "strings" + "sync" "time" "tailscale.com/envknob" @@ -24,9 +25,11 @@ import ( "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" + "tailscale.com/tstime" "tailscale.com/types/logger" "tailscale.com/util/dnsname" "tailscale.com/util/mak" + "tailscale.com/util/singleflight" ) func init() { @@ -44,6 +47,17 @@ const ( // state Secret and emit an Event. timeout = 30 * time.Second + // rwCertTTL is how long a write replica serves a TLS cert from its + // in-memory cache before it reads the cert Secret again. A short TTL + // limits how long the replica can serve a stale cert after the cert + // Secret is recreated, for example after Ingress recreation. + rwCertTTL = 30 * time.Second + // rwCertFallbackTimeout is the timeout for a write replica's cert + // Secret read when it has a last-known-good cert to fall back to. + // TLS handshakes wait for this read, so it must be much shorter than + // typical client handshake timeouts (10s for kubectl). + rwCertFallbackTimeout = 3 * time.Second + reasonTailscaleStateUpdated = "TailscaledStateUpdated" reasonTailscaleStateLoaded = "TailscaleStateLoaded" reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed" @@ -90,6 +104,32 @@ type Store struct { // memory holds the latest tailscale state. Writes write state to a kube // Secret and memory, Reads read from memory. memory mem.Store + + // clock is used for rwCerts expiry. + clock tstime.DefaultClock + + // certReads deduplicates concurrent cert Secret reads per domain. + certReads singleflight.Group[string, *rwCert] + + // rwCertsMu guards rwCerts. + rwCertsMu sync.Mutex + // rwCerts holds the last-known-good TLS cert and key per domain for + // write replicas (cert share "rw"). Write replicas don't keep certs in + // memory, so that they notice a recreated, empty cert Secret and issue + // a new cert. Instead, they use rwCerts to skip Secret reads for + // rwCertTTL and to keep serving TLS when the API server is unavailable. + // Entries are never modified, only replaced, so that a read can tell + // whether a write replaced the entry while the read was in flight. + rwCerts map[string]*rwCert +} + +// rwCert is a TLS cert and key cached by a write replica. +type rwCert struct { + // cert and key are returned to every reader of the cache entry, so + // they must not be modified. + cert, key []byte + // recheckAt is when the replica reads the cert Secret again. + recheckAt time.Time } // New returns a new Store that persists state to Kubernets Secret(s). @@ -153,7 +193,8 @@ func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*S // Write replicas never load certs in-memory to avoid a situation where, // after Ingress recreation (and the associated cert Secret recreation), new // TLS certs don't get issued because the write replica still has certs - // in-memory. Instead, write replicas fetch certs from Secret on each request. + // in-memory. Instead, write replicas fetch certs from Secret, with a short + // TTL cache (see rwCerts). if s.certShareMode == "ro" { sel := s.certSecretSelector() if err := s.loadCerts(context.Background(), sel); err != nil { @@ -365,12 +406,14 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) // TODO(irbekrm): certs for write replicas are currently not // written to memory to avoid out of sync memory state after // Ingress resources have been recreated. This means that TLS - // certs for write replicas are retrieved from the Secret on - // each HTTPS request. This is a temporary solution till we + // certs for write replicas are retrieved from the Secret once + // rwCertTTL has passed. This is a temporary solution till we // implement a Secret watch. if s.certShareMode != "rw" { s.memory.WriteState(ipn.StateKey(domain+".crt"), cert) s.memory.WriteState(ipn.StateKey(domain+".key"), key) + } else { + s.setRWCert(domain, cert, key) } return nil } @@ -378,7 +421,9 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) // ReadTLSCertAndKey reads a TLS cert and key from memory or from a // domain-specific Secret. It first checks the in-memory store, if not found in // memory and running cert store in read-only mode, looks up a Secret. -// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets. +// Write replicas of HA Ingress don't use the in-memory store. They read the +// Secret at most once per rwCertTTL, and serve the last-known-good cert if the +// read fails with an error other than not found or forbidden. func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { if err := dnsname.ValidHostname(domain); err != nil { return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err) @@ -396,14 +441,40 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { return nil, nil, ipn.ErrStateNotExist } - ctx, cancel := context.WithTimeout(context.Background(), timeout) + cached := s.getRWCert(domain) + if cached != nil && s.clock.Now().Before(cached.recheckAt) { + return cached.cert, cached.key, nil + } + // Handshakes that arrive while a read is in flight wait for that read + // instead of reading the Secret again. + c, err, _ := s.certReads.Do(domain, func() (*rwCert, error) { + return s.readCertSecret(domain, cached) + }) + if err != nil { + return nil, nil, err + } + return c.cert, c.key, nil +} + +// readCertSecret reads the TLS cert and key for domain from its Secret. +// cached is the write replica's cached cert for domain at the start of the +// read, or nil. The read updates the cache only if cached is still the cache +// entry for domain, so that a read which started before a WriteTLSCertAndKey +// doesn't replace or delete the cert that it wrote. +func (s *Store) readCertSecret(domain string, cached *rwCert) (*rwCert, error) { + readTimeout := timeout + if cached != nil { + readTimeout = rwCertFallbackTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), readTimeout) defer cancel() secret, err := s.client.GetSecret(ctx, domain) if err != nil { if kubeclient.IsNotFoundErr(err) { + s.replaceRWCert(domain, cached, nil) // TODO(irbekrm): we should return a more specific error // that wraps ipn.ErrStateNotExist here. - return nil, nil, ipn.ErrStateNotExist + return nil, ipn.ErrStateNotExist } st, ok := err.(*kubeapi.Status) if ok && st.Code == http.StatusForbidden && (s.certShareMode == "ro" || s.certShareMode == "rw") { @@ -420,30 +491,81 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) { // This code path gets triggered by the admin UI's machine page, // which queries for the node's own TLS cert existing via the // "tls-cert-status" c2n API. - return nil, nil, ipn.ErrStateNotExist + s.replaceRWCert(domain, cached, nil) + return nil, ipn.ErrStateNotExist } - return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err) + if cached != nil { + // The API server is slow or unavailable. Serve the + // last-known-good cert, and don't read the Secret again + // for rwCertTTL, so that handshakes don't wait on every + // read and we don't add load to the API server. + s.logf("kubestore: getting TLS Secret %q: %v; using last-known-good cert", domain, err) + s.replaceRWCert(domain, cached, s.newRWCert(cached.cert, cached.key)) + return cached, nil + } + return nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err) } - cert = secret.Data[keyTLSCert] - key = secret.Data[keyTLSKey] - if len(cert) == 0 || len(key) == 0 { - return nil, nil, ipn.ErrStateNotExist + c := s.newRWCert(secret.Data[keyTLSCert], secret.Data[keyTLSKey]) + if len(c.cert) == 0 || len(c.key) == 0 { + s.replaceRWCert(domain, cached, nil) + return nil, ipn.ErrStateNotExist } // TODO(irbekrm): a read between these two separate writes would // get a mismatched cert and key. Allow writing both cert and // key to the memory store in a single, lock-protected operation. // // TODO(irbekrm): currently certs for write replicas of HA Ingress get - // retrieved from the cluster Secret on each HTTPS request to avoid a - // situation when after Ingress recreation stale certs are read from + // retrieved from the cluster Secret once rwCertTTL has passed to avoid + // a situation when after Ingress recreation stale certs are read from // memory. // Fix this by watching Secrets to ensure that memory store gets updated // when Secrets are deleted. if s.certShareMode == "ro" { - s.memory.WriteState(ipn.StateKey(certKey), cert) - s.memory.WriteState(ipn.StateKey(keyKey), key) + s.memory.WriteState(ipn.StateKey(domain+".crt"), c.cert) + s.memory.WriteState(ipn.StateKey(domain+".key"), c.key) + } else { + s.replaceRWCert(domain, cached, c) } - return cert, key, nil + return c, nil +} + +// newRWCert returns a write replica cert for cert and key, to read again +// after rwCertTTL. +func (s *Store) newRWCert(cert, key []byte) *rwCert { + return &rwCert{cert: cert, key: key, recheckAt: s.clock.Now().Add(rwCertTTL)} +} + +// getRWCert returns the cached write replica cert for domain, or nil. +func (s *Store) getRWCert(domain string) *rwCert { + s.rwCertsMu.Lock() + defer s.rwCertsMu.Unlock() + return s.rwCerts[domain] +} + +// setRWCert caches copies of cert and key for domain and resets its TTL. +// It copies them because the caller owns them. +func (s *Store) setRWCert(domain string, cert, key []byte) { + c := s.newRWCert(bytes.Clone(cert), bytes.Clone(key)) + s.rwCertsMu.Lock() + defer s.rwCertsMu.Unlock() + mak.Set(&s.rwCerts, domain, c) +} + +// replaceRWCert replaces the cached write replica cert for domain with next, +// or removes it if next is nil. It does nothing if the cached cert is no +// longer old, for example because WriteTLSCertAndKey wrote a new cert. A nil +// old means that no cert was cached. +func (s *Store) replaceRWCert(domain string, old, next *rwCert) { + s.rwCertsMu.Lock() + defer s.rwCertsMu.Unlock() + if s.rwCerts[domain] != old { + return + } + if next == nil { + delete(s.rwCerts, domain) + return + } + mak.Set(&s.rwCerts, domain, next) } func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) { diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index 938d7f8f5..b0af86e5b 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -8,9 +8,14 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "strings" + "sync" + "sync/atomic" "testing" + "testing/synctest" + "time" "github.com/google/go-cmp/cmp" "tailscale.com/envknob" @@ -19,6 +24,8 @@ import ( "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" + "tailscale.com/tstest" + "tailscale.com/tstime" ) func TestKubernetesPodMigrationWithTPMAttestationKey(t *testing.T) { @@ -635,6 +642,401 @@ func TestReadTLSCertAndKey(t *testing.T) { } } +func TestReadTLSCertAndKeyWriteReplicaCache(t *testing.T) { + const testDomain = "my-app.tailnetxyz.ts.net" + + var ( + secretData map[string][]byte + secretGetErr error + gets int + lastTimeout time.Duration + ) + client := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + gets++ + if deadline, ok := ctx.Deadline(); ok { + lastTimeout = time.Until(deadline) + } + if secretGetErr != nil { + return nil, secretGetErr + } + return &kubeapi.Secret{Data: secretData}, nil + }, + JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error { + return nil + }, + } + clock := tstest.NewClock(tstest.ClockOpts{}) + s := &Store{ + client: client, + secretName: "ts-state", + certShareMode: "rw", + clock: tstime.DefaultClock{Clock: clock}, + logf: t.Logf, + } + tlsData := func(cert, key string) map[string][]byte { + return map[string][]byte{"tls.crt": []byte(cert), "tls.key": []byte(key)} + } + + steps := []struct { + name string + advance time.Duration + secretData map[string][]byte + secretGetErr error + // write, if set, writes cert and key with WriteTLSCertAndKey + // before the read. + write map[string][]byte + wantCert string + wantKey string + wantErr error + wantGet bool + wantTimeout time.Duration // max GetSecret timeout, if wantGet + }{ + { + name: "no_cache_api_error", + secretGetErr: errors.New("api error"), + wantErr: errors.New("api error"), + wantGet: true, + wantTimeout: timeout, + }, + { + name: "first_read_from_secret", + secretData: tlsData("cert-a", "key-a"), + wantCert: "cert-a", + wantKey: "key-a", + wantGet: true, + wantTimeout: timeout, + }, + { + name: "within_ttl_served_from_cache", + advance: rwCertTTL / 2, + secretData: tlsData("cert-b", "key-b"), + wantCert: "cert-a", + wantKey: "key-a", + }, + { + name: "after_ttl_read_from_secret", + advance: rwCertTTL, + secretData: tlsData("cert-b", "key-b"), + wantCert: "cert-b", + wantKey: "key-b", + wantGet: true, + wantTimeout: rwCertFallbackTimeout, + }, + { + name: "after_ttl_api_error_serves_last_known_good", + advance: rwCertTTL, + secretGetErr: errors.New("api error"), + wantCert: "cert-b", + wantKey: "key-b", + wantGet: true, + wantTimeout: rwCertFallbackTimeout, + }, + { + name: "after_fallback_served_from_cache", + advance: rwCertTTL / 2, + secretGetErr: errors.New("api error"), + wantCert: "cert-b", + wantKey: "key-b", + }, + { + name: "after_ttl_secret_emptied", + advance: rwCertTTL, + secretData: map[string][]byte{}, + wantErr: ipn.ErrStateNotExist, + wantGet: true, + }, + { + name: "emptied_secret_clears_cache", + secretGetErr: errors.New("api error"), + wantErr: errors.New("api error"), + wantGet: true, + wantTimeout: timeout, + }, + { + name: "written_cert_served_from_cache", + write: tlsData("cert-c", "key-c"), + secretData: tlsData("cert-c", "key-c"), + wantCert: "cert-c", + wantKey: "key-c", + }, + { + name: "after_ttl_secret_not_found", + advance: rwCertTTL, + secretGetErr: &kubeapi.Status{Code: 404}, + wantErr: ipn.ErrStateNotExist, + wantGet: true, + }, + { + name: "not_found_clears_cache", + secretGetErr: errors.New("api error"), + wantErr: errors.New("api error"), + wantGet: true, + wantTimeout: timeout, + }, + { + name: "written_cert_served_from_cache_again", + write: tlsData("cert-d", "key-d"), + secretData: tlsData("cert-d", "key-d"), + wantCert: "cert-d", + wantKey: "key-d", + }, + { + name: "after_ttl_secret_forbidden", + advance: rwCertTTL, + secretGetErr: &kubeapi.Status{Code: 403}, + wantErr: ipn.ErrStateNotExist, + wantGet: true, + }, + { + name: "forbidden_clears_cache", + secretGetErr: errors.New("api error"), + wantErr: errors.New("api error"), + wantGet: true, + wantTimeout: timeout, + }, + } + for _, st := range steps { + clock.Advance(st.advance) + secretData = st.secretData + secretGetErr = st.secretGetErr + if st.write != nil { + if err := s.WriteTLSCertAndKey(testDomain, st.write["tls.crt"], st.write["tls.key"]); err != nil { + t.Fatalf("%s: WriteTLSCertAndKey: %v", st.name, err) + } + } + gets = 0 + + cert, key, err := s.ReadTLSCertAndKey(testDomain) + switch { + case st.wantErr != nil: + if err == nil || !strings.Contains(err.Error(), st.wantErr.Error()) { + t.Errorf("%s: got error %v, want error containing %v", st.name, err, st.wantErr) + } + case err != nil: + t.Errorf("%s: unexpected error: %v", st.name, err) + default: + if string(cert) != st.wantCert || string(key) != st.wantKey { + t.Errorf("%s: got cert %q key %q, want cert %q key %q", st.name, cert, key, st.wantCert, st.wantKey) + } + } + if gotGet := gets > 0; gotGet != st.wantGet { + t.Errorf("%s: read Secret = %v, want %v", st.name, gotGet, st.wantGet) + } + if st.wantGet && st.wantTimeout != 0 && (lastTimeout > st.wantTimeout || lastTimeout < st.wantTimeout-time.Second) { + t.Errorf("%s: GetSecret timeout = %v, want %v", st.name, lastTimeout, st.wantTimeout) + } + } +} + +// TestReadTLSCertAndKeyWriteReplicaWriteDuringRead tests that a cert Secret +// read that is in flight when WriteTLSCertAndKey writes a new cert doesn't +// replace or remove the new cert in the write replica's cache. +func TestReadTLSCertAndKeyWriteReplicaWriteDuringRead(t *testing.T) { + const testDomain = "my-app.tailnetxyz.ts.net" + + tests := []struct { + name string + // cached, if set, is cached and expired before the read. + cached string + // secretData and secretGetErr are what the in-flight read gets. + secretData map[string][]byte + secretGetErr error + wantCert string + wantErr error + }{ + { + name: "read_returns_old_cert", + secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")}, + wantCert: "cert-old", + }, + { + name: "read_returns_not_found", + secretGetErr: &kubeapi.Status{Code: 404}, + wantErr: ipn.ErrStateNotExist, + }, + { + name: "read_returns_old_cert_with_cached_cert", + cached: "cert-old", + secretData: map[string][]byte{"tls.crt": []byte("cert-old"), "tls.key": []byte("key-old")}, + wantCert: "cert-old", + }, + { + name: "read_returns_not_found_with_cached_cert", + cached: "cert-old", + secretGetErr: &kubeapi.Status{Code: 404}, + wantErr: ipn.ErrStateNotExist, + }, + { + name: "read_returns_api_error_with_cached_cert", + cached: "cert-old", + secretGetErr: errors.New("api error"), + wantCert: "cert-old", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gets atomic.Int32 + getStarted := make(chan struct{}) + releaseGet := make(chan struct{}) + client := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + if gets.Add(1) > 1 { + // WriteTLSCertAndKey reads the Secret + // before it updates it. + return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil + } + close(getStarted) + <-releaseGet + if tt.secretGetErr != nil { + return nil, tt.secretGetErr + } + return &kubeapi.Secret{Data: tt.secretData}, nil + }, + JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error { + return nil + }, + UpdateSecretImpl: func(context.Context, *kubeapi.Secret) error { + return nil + }, + } + clock := tstest.NewClock(tstest.ClockOpts{}) + s := &Store{ + client: client, + secretName: "ts-state", + certShareMode: "rw", + clock: tstime.DefaultClock{Clock: clock}, + logf: t.Logf, + } + if tt.cached != "" { + s.setRWCert(testDomain, []byte(tt.cached), []byte("key")) + clock.Advance(rwCertTTL) + } + + type result struct { + cert []byte + err error + } + done := make(chan result) + go func() { + cert, _, err := s.ReadTLSCertAndKey(testDomain) + done <- result{cert, err} + }() + <-getStarted + if err := s.WriteTLSCertAndKey(testDomain, []byte("cert-new"), []byte("key-new")); err != nil { + t.Fatalf("WriteTLSCertAndKey: %v", err) + } + close(releaseGet) + + res := <-done + if tt.wantErr != nil { + if !errors.Is(res.err, tt.wantErr) { + t.Errorf("in-flight read: got error %v, want %v", res.err, tt.wantErr) + } + } else if res.err != nil || string(res.cert) != tt.wantCert { + t.Errorf("in-flight read: got cert %q, error %v, want cert %q", res.cert, res.err, tt.wantCert) + } + + // The next read must serve the new cert from the cache. + getsBefore := gets.Load() + cert, key, err := s.ReadTLSCertAndKey(testDomain) + if err != nil { + t.Fatalf("next read: %v", err) + } + if string(cert) != "cert-new" || string(key) != "key-new" { + t.Errorf("next read: got cert %q key %q, want cert %q key %q", cert, key, "cert-new", "key-new") + } + if gets.Load() != getsBefore { + t.Errorf("next read: read Secret, want cached cert") + } + }) + } +} + +// TestWriteTLSCertAndKeyWriteReplicaCopiesCert tests that a write replica +// caches copies of the cert and key that it writes, so that a caller that +// modifies them afterwards doesn't change the cached cert. +func TestWriteTLSCertAndKeyWriteReplicaCopiesCert(t *testing.T) { + const testDomain = "my-app.tailnetxyz.ts.net" + + client := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert"), "tls.key": []byte("key")}}, nil + }, + JSONPatchResourceImpl: func(context.Context, string, string, []kubeclient.JSONPatch) error { + return nil + }, + } + s := &Store{ + client: client, + secretName: "ts-state", + certShareMode: "rw", + logf: t.Logf, + } + cert, key := []byte("cert"), []byte("key") + if err := s.WriteTLSCertAndKey(testDomain, cert, key); err != nil { + t.Fatalf("WriteTLSCertAndKey: %v", err) + } + copy(cert, "XXXX") + copy(key, "XXX") + + gotCert, gotKey, err := s.ReadTLSCertAndKey(testDomain) + if err != nil { + t.Fatalf("ReadTLSCertAndKey: %v", err) + } + if string(gotCert) != "cert" || string(gotKey) != "key" { + t.Errorf("got cert %q key %q, want cert %q key %q", gotCert, gotKey, "cert", "key") + } +} + +// TestReadTLSCertAndKeyWriteReplicaConcurrentReads tests that concurrent +// reads of an expired cached cert read the cert Secret only once. +func TestReadTLSCertAndKeyWriteReplicaConcurrentReads(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + const ( + testDomain = "my-app.tailnetxyz.ts.net" + readers = 10 + ) + var gets atomic.Int32 + releaseGet := make(chan struct{}) + client := &kubeclient.FakeClient{ + GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { + gets.Add(1) + <-releaseGet + return &kubeapi.Secret{Data: map[string][]byte{"tls.crt": []byte("cert-new"), "tls.key": []byte("key-new")}}, nil + }, + } + clock := tstest.NewClock(tstest.ClockOpts{}) + s := &Store{ + client: client, + secretName: "ts-state", + certShareMode: "rw", + clock: tstime.DefaultClock{Clock: clock}, + logf: t.Logf, + } + s.setRWCert(testDomain, []byte("cert-old"), []byte("key-old")) + clock.Advance(rwCertTTL) + + var wg sync.WaitGroup + for range readers { + wg.Go(func() { + cert, _, err := s.ReadTLSCertAndKey(testDomain) + if err != nil || string(cert) != "cert-new" { + t.Errorf("got cert %q, error %v, want cert %q", cert, err, "cert-new") + } + }) + } + // Wait for all readers to block on the in-flight read. + synctest.Wait() + close(releaseGet) + wg.Wait() + + if got := gets.Load(); got != 1 { + t.Errorf("GetSecret called %d times, want 1", got) + } + }) +} + func TestNewWithClient(t *testing.T) { const ( secretName = "ts-state"