From 68ec83c60bf0922d3d560ee14be4b86c4d8190b1 Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Wed, 14 Oct 2026 15:00:40 +0000 Subject: [PATCH] cmd/k8s-operator: test requeue on optimistic lock errors Adds tests that a conflict while provisioning a Service, a Service on a ProxyGroup, a ProxyGroup or a Connector proxy results in a timed requeue, and that the Connector and ProxyGroup status is set to creating rather than failed. The test for a Service on a ProxyGroup fails the update to the shared ingress config ConfigMap, as no watch maps that ConfigMap back to the Service, so without the requeue the conflict was never retried. Updates #22074 Updates #18922 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/connector_test.go | 54 +++++++++++++++++++++++++++ cmd/k8s-operator/operator_test.go | 46 +++++++++++++++++++++++ cmd/k8s-operator/proxygroup_test.go | 57 +++++++++++++++++++++++++++++ cmd/k8s-operator/svc-for-pg_test.go | 22 +++++++++++ cmd/k8s-operator/testutils_test.go | 2 + 5 files changed, 181 insertions(+) diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 29b716a78..ff95408d1 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "maps" "net/netip" @@ -21,10 +22,14 @@ import ( "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" @@ -822,3 +827,52 @@ func TestConnectorWithStaticEndpoints(t *testing.T) { t.Errorf("expected StatefulSet %q to no longer have a PORT env, got %v", shortName, env) } } + +func TestConnectorOptimisticLockErrorRequeues(t *testing.T) { + cn := &tsapi.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("1234-UID"), + }, + Spec: tsapi.ConnectorSpec{ + Replicas: new(int32(1)), + ExitNode: true, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(cn). + WithStatusSubresource(cn). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg)) + } + return c.Create(ctx, obj, opts...) + }, + }). + Build() + cr := &ConnectorReconciler{ + Client: fc, + recorder: record.NewFakeRecorder(10), + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + clock: tstest.NewClock(tstest.ClockOpts{}), + logger: zap.Must(zap.NewDevelopment()).Sugar(), + } + + expectRequeue(t, cr, "", "test") + + got := new(tsapi.Connector) + if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil { + t.Fatal(err) + } + if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != reasonConnectorCreating { + t.Errorf("want condition reason %q, got %+v", reasonConnectorCreating, got.Status.Conditions) + } +} diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 8646bac25..d75d8b744 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "testing" "time" @@ -18,11 +19,14 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/k8s-operator/apis/v1alpha1" @@ -2278,3 +2282,45 @@ func TestStateSecretHandlerForProxyGroup(t *testing.T) { }) } } + +func TestServiceOptimisticLockErrorRequeues(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: new("tailscale"), + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(svc). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if _, ok := obj.(*corev1.Service); ok { + return apierrors.NewConflict(schema.GroupResource{Resource: "services"}, obj.GetName(), errors.New(optimisticLockErrorMsg)) + } + return c.Update(ctx, obj, opts...) + }, + }). + Build() + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zap.Must(zap.NewDevelopment()).Sugar(), + clock: tstest.NewClock(tstest.ClockOpts{}), + recorder: record.NewFakeRecorder(100), + } + + expectRequeue(t, sr, "default", "test") +} diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 21a50c36b..2baf1c38e 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -6,7 +6,9 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "net/netip" "reflect" @@ -21,12 +23,15 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "tailscale.com/client/tailscale/v2" "tailscale.com/ipn" @@ -2450,3 +2455,55 @@ func testCasesForLEStagingTests() []leStagingTestCase { }, } } + +func TestProxyGroupOptimisticLockErrorRequeues(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg). + WithStatusSubresource(pg). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg)) + } + return c.Create(ctx, obj, opts...) + }, + }). + Build() + fr := record.NewFakeRecorder(1) + r := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + Client: fc, + clients: tsclient.NewProvider(&fakeTSClient{}), + recorder: fr, + log: zap.NewNop().Sugar(), + clock: tstest.NewClock(tstest.ClockOpts{}), + reissuer: tailscaled.NewReissuer(), + } + + expectRequeue(t, r, "", pg.Name) + + if err := fc.Get(t.Context(), client.ObjectKeyFromObject(pg), pg); err != nil { + t.Fatalf("getting ProxyGroup: %v", err) + } + cond := apimeta.FindStatusCondition(pg.Status.Conditions, string(tsapi.ProxyGroupReady)) + if cond == nil || cond.Reason != reasonProxyGroupCreating || cond.Message != "ProxyGroup resources are being updated, retrying" { + t.Errorf("ProxyGroupReady condition = %+v, want reason %q and the retrying message", cond, reasonProxyGroupCreating) + } + if len(fr.Events) != 0 { + t.Errorf("got %d events, want none", len(fr.Events)) + } +} diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index cfaf089af..4259604f7 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -8,6 +8,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "math/rand/v2" "net/netip" @@ -18,11 +19,14 @@ import ( "go.uber.org/zap" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "tailscale.com/client/tailscale/v2" tsoperator "tailscale.com/k8s-operator" @@ -649,3 +653,21 @@ func TestProxyGroupFromServiceFinalizers(t *testing.T) { }) } } + +// TestServicePGReconciler_OptimisticLockErrorRequeues fails the update to the +// ProxyGroup's ingress config ConfigMap, which the reconciler doesn't watch, +// so a dropped request is only retried by the requeue. +func TestServicePGReconciler_OptimisticLockErrorRequeues(t *testing.T) { + svcPGR, stateSecret, fc, _, _ := setupServiceTest(t) + svc, _ := setupTestService(t, "test-svc", "", "1.2.3.4", fc, stateSecret) + + svcPGR.Client = interceptor.NewClient(fc.(client.WithWatch), interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if cm, ok := obj.(*corev1.ConfigMap); ok { + return apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, cm.Name, errors.New(optimisticLockErrorMsg)) + } + return c.Update(ctx, obj, opts...) + }, + }) + expectRequeue(t, svcPGR, "default", svc.Name) +} diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 11c4565a2..5e22a9457 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -48,6 +48,8 @@ const ( // tsNamespace is the namespace the operator is installed in for tests that don't override it. tsNamespace = "tailscale" + + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" ) // confgOpts contains configuration options for creating cluster resources for