From 9a44f4f69ce0e0da82d23e5bd181ce5aac0a04ae Mon Sep 17 00:00:00 2001 From: chaosinthecrd Date: Wed, 14 Oct 2026 15:00:40 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator/reconciler: requeue on optimistic lock errors Reconcilers logged "optimistic lock error, retrying" with the full conflict error at Info, and Connector, Recorder and ProxyGroup also put it in their status. That reads like a failure, but a conflict between a cached read and an update is expected and is retried. The reconcilers also returned an empty result, so the retry only happened if a watch event arrived. This fixes this by logging the conflict at Debug as an update conflict, using a neutral status message and requeueing after a short delay. Updates #22074 Updates #18922 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/api-server-proxy-pg.go | 8 +++---- cmd/k8s-operator/connector.go | 5 ++-- cmd/k8s-operator/egress-services.go | 6 ++--- cmd/k8s-operator/ingress.go | 6 ++--- cmd/k8s-operator/proxygroup.go | 24 +++++++++++++------ cmd/k8s-operator/svc-for-pg.go | 6 ++--- cmd/k8s-operator/svc.go | 6 ++--- .../reconciler/dnsrecords/dnsrecords.go | 2 +- .../reconciler/nameserver/nameserver.go | 7 ++++-- k8s-operator/reconciler/recorder/recorder.go | 11 +++++---- 10 files changed, 49 insertions(+), 32 deletions(-) diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 94ecac7be..067424f46 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -86,8 +86,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return res, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return res, err @@ -96,8 +96,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient) if err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return reconcile.Result{}, err } diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 41e21dad0..e9b95c3ea 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -148,9 +148,10 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque message := fmt.Sprintf(messageConnectorCreationFailed, err) if reconciler.IsOptimisticLockError(err) { reason = reasonConnectorCreating - message = fmt.Sprintf("optimistic lock error, retrying: %s", err) + message = "Connector resources are being updated, retrying" + logger.Debugf("update conflict, retrying: %v", err) err = nil - logger.Info(message) + res = reconcile.Result{RequeueAfter: shortRequeue} } else { a.recorder.Event(cn, corev1.EventTypeWarning, reason, message) } diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7ec758fd7..8bfccabae 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -163,10 +163,10 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re if err := esr.maybeProvision(ctx, svc, lg); err != nil { if reconciler.IsOptimisticLockError(err) { - lg.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + lg.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return res, nil diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index a09524d59..b6cc8d4e0 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -84,10 +84,10 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request if err := a.maybeProvision(ctx, logger, ing); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return reconcile.Result{}, nil diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index cf1c25e9f..bea9fc21b 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -149,8 +149,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return reconcile.Result{}, err } else if !done { @@ -167,7 +167,13 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ oldPGStatus := pg.Status.DeepCopy() staticEndpoints, nrr, err := r.reconcilePG(ctx, tsClient, pg, logger) - return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) + if err = errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)); err != nil { + return reconcile.Result{}, err + } + if nrr != nil && nrr.requeue { + return reconcile.Result{RequeueAfter: shortRequeue}, nil + } + return reconcile.Result{}, nil } // reconcilePG handles all reconciliation of a ProxyGroup that is not marked @@ -474,7 +480,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { if reconciler.IsOptimisticLockError(updateErr) { - logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) + logger.Debugf("status update conflict, will reconcile on the next event: %v", updateErr) updateErr = nil } err = errors.Join(err, updateErr) @@ -1100,9 +1106,12 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool { func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) if reconciler.IsOptimisticLockError(err) { - msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) - logger.Info(msg) - return notReady(reasonProxyGroupCreating, msg) + logger.Debugf("update conflict, retrying: %v", err) + return nil, ¬ReadyReason{ + reason: reasonProxyGroupCreating, + message: "ProxyGroup resources are being updated, retrying", + requeue: true, + }, nil } r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) @@ -1115,4 +1124,5 @@ func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.Su type notReadyReason struct { reason string message string + requeue bool // set for optimistic lock errors, which are retried after shortRequeue } diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 61ff57cf8..be362dae5 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -191,10 +191,10 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient) if err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } if needsRequeue { res = reconcile.Result{RequeueAfter: requeueInterval()} diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index daa5edf9c..26c6d4f64 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -125,10 +125,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request if err := a.maybeProvision(ctx, logger, svc); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return reconcile.Result{}, nil diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index 30daf87b9..e11eeb5cf 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -157,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res if err := r.maybeProvision(ctx, proxySvc, logger); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) + logger.Debugf("update conflict, retrying: %v", err) return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { return reconcile.Result{}, err diff --git a/k8s-operator/reconciler/nameserver/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 705b7e177..ae44e0e30 100644 --- a/k8s-operator/reconciler/nameserver/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -12,6 +12,7 @@ import ( _ "embed" "errors" "fmt" + "time" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" @@ -49,6 +50,8 @@ const ( defaultNameserverImageRepo = "tailscale/k8s-nameserver" defaultNameserverImageTag = "stable" + + shortRequeue = 5 * time.Second ) var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) @@ -159,8 +162,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res } if err = r.maybeProvision(ctx, &dnsCfg); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err) } diff --git a/k8s-operator/reconciler/recorder/recorder.go b/k8s-operator/reconciler/recorder/recorder.go index 60497b578..826ed7371 100644 --- a/k8s-operator/reconciler/recorder/recorder.go +++ b/k8s-operator/reconciler/recorder/recorder.go @@ -214,12 +214,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco message := fmt.Sprintf("failed creating Recorder: %s", err) if reconciler.IsOptimisticLockError(err) { reason = reasonRecorderCreating - message = fmt.Sprintf("optimistic lock error, retrying: %s", err) + message = "Recorder resources are being updated, retrying" + logger.Debugf("update conflict, retrying: %v", err) err = nil - logger.Info(message) - } else { - r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message) + if _, err = setStatusReady(tsr, metav1.ConditionFalse, reason, message); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message) return setStatusReady(tsr, metav1.ConditionFalse, reason, message) }