cmd/k8s-operator: test requeue on optimistic lock errors

Adds tests that a conflict while provisioning a Service, a Service on a
ProxyGroup, a ProxyGroup or a Connector proxy results in a timed
requeue, and that the Connector and ProxyGroup status is set to
creating rather than failed. The test for a Service on a ProxyGroup
fails the update to the shared ingress config ConfigMap, as no watch
maps that ConfigMap back to the Service, so without the requeue the
conflict was never retried.

Updates #22074
Updates #18922

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
chaosinthecrd authored and Tom Meadows committed 2026-11-05 15:27:01 +00:00
1 parent 9a44f4f69c
commit 68ec83c60b
5 files changed
+181

No files matched your search

+54
View File
@@ -8,6 +8,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"net/netip"
@@ -21,10 +22,14 @@ import (
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"tailscale.com/ipn"
tsoperator "tailscale.com/k8s-operator"
@@ -822,3 +827,52 @@ func TestConnectorWithStaticEndpoints(t *testing.T) {
t.Errorf("expected StatefulSet %q to no longer have a PORT env, got %v", shortName, env)
}
}
func TestConnectorOptimisticLockErrorRequeues(t *testing.T) {
cn := &tsapi.Connector{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: types.UID("1234-UID"),
},
Spec: tsapi.ConnectorSpec{
Replicas: new(int32(1)),
ExitNode: true,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(cn).
WithStatusSubresource(cn).
WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*appsv1.StatefulSet); ok {
return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
}
return c.Create(ctx, obj, opts...)
},
}).
Build()
cr := &ConnectorReconciler{
Client: fc,
recorder: record.NewFakeRecorder(10),
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(&fakeTSClient{}),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
clock: tstest.NewClock(tstest.ClockOpts{}),
logger: zap.Must(zap.NewDevelopment()).Sugar(),
}
expectRequeue(t, cr, "", "test")
got := new(tsapi.Connector)
if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil {
t.Fatal(err)
}
if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != reasonConnectorCreating {
t.Errorf("want condition reason %q, got %+v", reasonConnectorCreating, got.Status.Conditions)
}
}
+46
View File
@@ -8,6 +8,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"testing"
"time"
@@ -18,11 +19,14 @@ import (
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/k8s-operator/apis/v1alpha1"
@@ -2278,3 +2282,45 @@ func TestStateSecretHandlerForProxyGroup(t *testing.T) {
})
}
}
func TestServiceOptimisticLockErrorRequeues(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: new("tailscale"),
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(svc).
WithInterceptorFuncs(interceptor.Funcs{
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if _, ok := obj.(*corev1.Service); ok {
return apierrors.NewConflict(schema.GroupResource{Resource: "services"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
}
return c.Update(ctx, obj, opts...)
},
}).
Build()
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
clients: tsclient.NewProvider(&fakeTSClient{}),
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zap.Must(zap.NewDevelopment()).Sugar(),
clock: tstest.NewClock(tstest.ClockOpts{}),
recorder: record.NewFakeRecorder(100),
}
expectRequeue(t, sr, "default", "test")
}
+57
View File
@@ -6,7 +6,9 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/netip"
"reflect"
@@ -21,12 +23,15 @@ import (
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"tailscale.com/client/tailscale/v2"
"tailscale.com/ipn"
@@ -2450,3 +2455,55 @@ func testCasesForLEStagingTests() []leStagingTestCase {
},
}
}
func TestProxyGroupOptimisticLockErrorRequeues(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
Generation: 1,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeEgress,
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg).
WithStatusSubresource(pg).
WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*appsv1.StatefulSet); ok {
return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
}
return c.Create(ctx, obj, opts...)
},
}).
Build()
fr := record.NewFakeRecorder(1)
r := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
tsFirewallMode: "auto",
Client: fc,
clients: tsclient.NewProvider(&fakeTSClient{}),
recorder: fr,
log: zap.NewNop().Sugar(),
clock: tstest.NewClock(tstest.ClockOpts{}),
reissuer: tailscaled.NewReissuer(),
}
expectRequeue(t, r, "", pg.Name)
if err := fc.Get(t.Context(), client.ObjectKeyFromObject(pg), pg); err != nil {
t.Fatalf("getting ProxyGroup: %v", err)
}
cond := apimeta.FindStatusCondition(pg.Status.Conditions, string(tsapi.ProxyGroupReady))
if cond == nil || cond.Reason != reasonProxyGroupCreating || cond.Message != "ProxyGroup resources are being updated, retrying" {
t.Errorf("ProxyGroupReady condition = %+v, want reason %q and the retrying message", cond, reasonProxyGroupCreating)
}
if len(fr.Events) != 0 {
t.Errorf("got %d events, want none", len(fr.Events))
}
}
+22
View File
@@ -8,6 +8,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"net/netip"
@@ -18,11 +19,14 @@ import (
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"tailscale.com/client/tailscale/v2"
tsoperator "tailscale.com/k8s-operator"
@@ -649,3 +653,21 @@ func TestProxyGroupFromServiceFinalizers(t *testing.T) {
})
}
}
// TestServicePGReconciler_OptimisticLockErrorRequeues fails the update to the
// ProxyGroup's ingress config ConfigMap, which the reconciler doesn't watch,
// so a dropped request is only retried by the requeue.
func TestServicePGReconciler_OptimisticLockErrorRequeues(t *testing.T) {
svcPGR, stateSecret, fc, _, _ := setupServiceTest(t)
svc, _ := setupTestService(t, "test-svc", "", "1.2.3.4", fc, stateSecret)
svcPGR.Client = interceptor.NewClient(fc.(client.WithWatch), interceptor.Funcs{
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if cm, ok := obj.(*corev1.ConfigMap); ok {
return apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, cm.Name, errors.New(optimisticLockErrorMsg))
}
return c.Update(ctx, obj, opts...)
},
})
expectRequeue(t, svcPGR, "default", svc.Name)
}
+2
View File
@@ -48,6 +48,8 @@ const (
// tsNamespace is the namespace the operator is installed in for tests that don't override it.
tsNamespace = "tailscale"
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)
// confgOpts contains configuration options for creating cluster resources for