mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-22 12:42:42 +09:00
cmd/k8s-operator: test requeue on optimistic lock errors
Adds tests that a conflict while provisioning a Service, a Service on a ProxyGroup, a ProxyGroup or a Connector proxy results in a timed requeue, and that the Connector and ProxyGroup status is set to creating rather than failed. The test for a Service on a ProxyGroup fails the update to the shared ingress config ConfigMap, as no watch maps that ConfigMap back to the Service, so without the requeue the conflict was never retried. Updates #22074 Updates #18922 Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
1 parent
9a44f4f69c
commit
68ec83c60b
5 files changed
+181
No files matched your search
@@ -8,6 +8,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/netip"
|
||||
@@ -21,10 +22,14 @@ import (
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
|
||||
"tailscale.com/ipn"
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
@@ -822,3 +827,52 @@ func TestConnectorWithStaticEndpoints(t *testing.T) {
|
||||
t.Errorf("expected StatefulSet %q to no longer have a PORT env, got %v", shortName, env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectorOptimisticLockErrorRequeues(t *testing.T) {
|
||||
cn := &tsapi.Connector{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
UID: types.UID("1234-UID"),
|
||||
},
|
||||
Spec: tsapi.ConnectorSpec{
|
||||
Replicas: new(int32(1)),
|
||||
ExitNode: true,
|
||||
},
|
||||
}
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
WithObjects(cn).
|
||||
WithStatusSubresource(cn).
|
||||
WithInterceptorFuncs(interceptor.Funcs{
|
||||
Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
||||
if _, ok := obj.(*appsv1.StatefulSet); ok {
|
||||
return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
|
||||
}
|
||||
return c.Create(ctx, obj, opts...)
|
||||
},
|
||||
}).
|
||||
Build()
|
||||
cr := &ConnectorReconciler{
|
||||
Client: fc,
|
||||
recorder: record.NewFakeRecorder(10),
|
||||
ssr: &tailscaleSTSReconciler{
|
||||
Client: fc,
|
||||
clients: tsclient.NewProvider(&fakeTSClient{}),
|
||||
defaultTags: []string{"tag:k8s"},
|
||||
operatorNamespace: "operator-ns",
|
||||
proxyImage: "tailscale/tailscale",
|
||||
},
|
||||
clock: tstest.NewClock(tstest.ClockOpts{}),
|
||||
logger: zap.Must(zap.NewDevelopment()).Sugar(),
|
||||
}
|
||||
|
||||
expectRequeue(t, cr, "", "test")
|
||||
|
||||
got := new(tsapi.Connector)
|
||||
if err := fc.Get(context.Background(), types.NamespacedName{Name: "test"}, got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Status.Conditions) != 1 || got.Status.Conditions[0].Reason != reasonConnectorCreating {
|
||||
t.Errorf("want condition reason %q, got %+v", reasonConnectorCreating, got.Status.Conditions)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -18,11 +19,14 @@ import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
@@ -2278,3 +2282,45 @@ func TestStateSecretHandlerForProxyGroup(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceOptimisticLockErrorRequeues(t *testing.T) {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "default",
|
||||
UID: types.UID("1234-UID"),
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
ClusterIP: "10.20.30.40",
|
||||
Type: corev1.ServiceTypeLoadBalancer,
|
||||
LoadBalancerClass: new("tailscale"),
|
||||
},
|
||||
}
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
WithObjects(svc).
|
||||
WithInterceptorFuncs(interceptor.Funcs{
|
||||
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
|
||||
if _, ok := obj.(*corev1.Service); ok {
|
||||
return apierrors.NewConflict(schema.GroupResource{Resource: "services"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
|
||||
}
|
||||
return c.Update(ctx, obj, opts...)
|
||||
},
|
||||
}).
|
||||
Build()
|
||||
sr := &ServiceReconciler{
|
||||
Client: fc,
|
||||
ssr: &tailscaleSTSReconciler{
|
||||
Client: fc,
|
||||
clients: tsclient.NewProvider(&fakeTSClient{}),
|
||||
defaultTags: []string{"tag:k8s"},
|
||||
operatorNamespace: "operator-ns",
|
||||
proxyImage: "tailscale/tailscale",
|
||||
},
|
||||
logger: zap.Must(zap.NewDevelopment()).Sugar(),
|
||||
clock: tstest.NewClock(tstest.ClockOpts{}),
|
||||
recorder: record.NewFakeRecorder(100),
|
||||
}
|
||||
|
||||
expectRequeue(t, sr, "default", "test")
|
||||
}
|
||||
@@ -6,7 +6,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"reflect"
|
||||
@@ -21,12 +23,15 @@ import (
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
"tailscale.com/client/tailscale/v2"
|
||||
|
||||
"tailscale.com/ipn"
|
||||
@@ -2450,3 +2455,55 @@ func testCasesForLEStagingTests() []leStagingTestCase {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyGroupOptimisticLockErrorRequeues(t *testing.T) {
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Finalizers: []string{"tailscale.com/finalizer"},
|
||||
Generation: 1,
|
||||
},
|
||||
Spec: tsapi.ProxyGroupSpec{
|
||||
Type: tsapi.ProxyGroupTypeEgress,
|
||||
},
|
||||
}
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
WithObjects(pg).
|
||||
WithStatusSubresource(pg).
|
||||
WithInterceptorFuncs(interceptor.Funcs{
|
||||
Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
||||
if _, ok := obj.(*appsv1.StatefulSet); ok {
|
||||
return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, obj.GetName(), errors.New(optimisticLockErrorMsg))
|
||||
}
|
||||
return c.Create(ctx, obj, opts...)
|
||||
},
|
||||
}).
|
||||
Build()
|
||||
fr := record.NewFakeRecorder(1)
|
||||
r := &ProxyGroupReconciler{
|
||||
tsNamespace: tsNamespace,
|
||||
tsProxyImage: testProxyImage,
|
||||
defaultTags: []string{"tag:test-tag"},
|
||||
tsFirewallMode: "auto",
|
||||
Client: fc,
|
||||
clients: tsclient.NewProvider(&fakeTSClient{}),
|
||||
recorder: fr,
|
||||
log: zap.NewNop().Sugar(),
|
||||
clock: tstest.NewClock(tstest.ClockOpts{}),
|
||||
reissuer: tailscaled.NewReissuer(),
|
||||
}
|
||||
|
||||
expectRequeue(t, r, "", pg.Name)
|
||||
|
||||
if err := fc.Get(t.Context(), client.ObjectKeyFromObject(pg), pg); err != nil {
|
||||
t.Fatalf("getting ProxyGroup: %v", err)
|
||||
}
|
||||
cond := apimeta.FindStatusCondition(pg.Status.Conditions, string(tsapi.ProxyGroupReady))
|
||||
if cond == nil || cond.Reason != reasonProxyGroupCreating || cond.Message != "ProxyGroup resources are being updated, retrying" {
|
||||
t.Errorf("ProxyGroupReady condition = %+v, want reason %q and the retrying message", cond, reasonProxyGroupCreating)
|
||||
}
|
||||
if len(fr.Events) != 0 {
|
||||
t.Errorf("got %d events, want none", len(fr.Events))
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net/netip"
|
||||
@@ -18,11 +19,14 @@ import (
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
"tailscale.com/client/tailscale/v2"
|
||||
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
@@ -649,3 +653,21 @@ func TestProxyGroupFromServiceFinalizers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestServicePGReconciler_OptimisticLockErrorRequeues fails the update to the
|
||||
// ProxyGroup's ingress config ConfigMap, which the reconciler doesn't watch,
|
||||
// so a dropped request is only retried by the requeue.
|
||||
func TestServicePGReconciler_OptimisticLockErrorRequeues(t *testing.T) {
|
||||
svcPGR, stateSecret, fc, _, _ := setupServiceTest(t)
|
||||
svc, _ := setupTestService(t, "test-svc", "", "1.2.3.4", fc, stateSecret)
|
||||
|
||||
svcPGR.Client = interceptor.NewClient(fc.(client.WithWatch), interceptor.Funcs{
|
||||
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
|
||||
if cm, ok := obj.(*corev1.ConfigMap); ok {
|
||||
return apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, cm.Name, errors.New(optimisticLockErrorMsg))
|
||||
}
|
||||
return c.Update(ctx, obj, opts...)
|
||||
},
|
||||
})
|
||||
expectRequeue(t, svcPGR, "default", svc.Name)
|
||||
}
|
||||
@@ -48,6 +48,8 @@ const (
|
||||
|
||||
// tsNamespace is the namespace the operator is installed in for tests that don't override it.
|
||||
tsNamespace = "tailscale"
|
||||
|
||||
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
|
||||
)
|
||||
|
||||
// confgOpts contains configuration options for creating cluster resources for
|
||||
|
||||
Reference in new issue
Block a user