cmd/k8s-operator,k8s-operator/reconciler: requeue on optimistic lock errors

Reconcilers logged "optimistic lock error, retrying" with the full
conflict error at Info, and Connector, Recorder and ProxyGroup also put
it in their status. That reads like a failure, but a conflict between a
cached read and an update is expected and is retried. The reconcilers
also returned an empty result, so the retry only happened if a watch
event arrived. This fixes this by logging the conflict at Debug as an
update conflict, using a neutral status message and requeueing after a
short delay.

Updates #22074
Updates #18922

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
chaosinthecrd authored and Tom Meadows committed 2026-11-05 15:27:01 +00:00
1 parent 2993b5e6de
commit 9a44f4f69c
10 files changed
+49 -32

No files matched your search

+4 -4
View File
@@ -86,8 +86,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
if markedForDeletion(pg) {
logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up")
if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return res, nil
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return res, err
@@ -96,8 +96,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient)
if err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
}
+3 -2
View File
@@ -148,9 +148,10 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
message := fmt.Sprintf(messageConnectorCreationFailed, err)
if reconciler.IsOptimisticLockError(err) {
reason = reasonConnectorCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
message = "Connector resources are being updated, retrying"
logger.Debugf("update conflict, retrying: %v", err)
err = nil
logger.Info(message)
res = reconcile.Result{RequeueAfter: shortRequeue}
} else {
a.recorder.Event(cn, corev1.EventTypeWarning, reason, message)
}
+3 -3
View File
@@ -163,10 +163,10 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
if err := esr.maybeProvision(ctx, svc, lg); err != nil {
if reconciler.IsOptimisticLockError(err) {
lg.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
lg.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
}
return res, nil
+3 -3
View File
@@ -84,10 +84,10 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
if err := a.maybeProvision(ctx, logger, ing); err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
+17 -7
View File
@@ -149,8 +149,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
} else if !done {
@@ -167,7 +167,13 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
oldPGStatus := pg.Status.DeepCopy()
staticEndpoints, nrr, err := r.reconcilePG(ctx, tsClient, pg, logger)
return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints))
if err = errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)); err != nil {
return reconcile.Result{}, err
}
if nrr != nil && nrr.requeue {
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, nil
}
// reconcilePG handles all reconciliation of a ProxyGroup that is not marked
@@ -474,7 +480,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) {
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
if reconciler.IsOptimisticLockError(updateErr) {
logger.Infof("optimistic lock error updating status, retrying: %s", updateErr)
logger.Debugf("status update conflict, will reconcile on the next event: %v", updateErr)
updateErr = nil
}
err = errors.Join(err, updateErr)
@@ -1100,9 +1106,12 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool {
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
err := fmt.Errorf(format, a...)
if reconciler.IsOptimisticLockError(err) {
msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error())
logger.Info(msg)
return notReady(reasonProxyGroupCreating, msg)
logger.Debugf("update conflict, retrying: %v", err)
return nil, &notReadyReason{
reason: reasonProxyGroupCreating,
message: "ProxyGroup resources are being updated, retrying",
requeue: true,
}, nil
}
r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
@@ -1115,4 +1124,5 @@ func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.Su
type notReadyReason struct {
reason string
message string
requeue bool // set for optimistic lock errors, which are retried after shortRequeue
}
+3 -3
View File
@@ -191,10 +191,10 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque
needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient)
if err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
}
if needsRequeue {
res = reconcile.Result{RequeueAfter: requeueInterval()}
+3 -3
View File
@@ -125,10 +125,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
if err := a.maybeProvision(ctx, logger, svc); err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
@@ -157,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
if err := r.maybeProvision(ctx, proxySvc, logger); err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
} else {
return reconcile.Result{}, err
@@ -12,6 +12,7 @@ import (
_ "embed"
"errors"
"fmt"
"time"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
@@ -49,6 +50,8 @@ const (
defaultNameserverImageRepo = "tailscale/k8s-nameserver"
defaultNameserverImageTag = "stable"
shortRequeue = 5 * time.Second
)
var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
@@ -159,8 +162,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
}
if err = r.maybeProvision(ctx, &dnsCfg); err != nil {
if reconciler.IsOptimisticLockError(err) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
logger.Debugf("update conflict, retrying: %v", err)
return reconcile.Result{RequeueAfter: shortRequeue}, nil
} else {
return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
}
+7 -4
View File
@@ -214,12 +214,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
message := fmt.Sprintf("failed creating Recorder: %s", err)
if reconciler.IsOptimisticLockError(err) {
reason = reasonRecorderCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
message = "Recorder resources are being updated, retrying"
logger.Debugf("update conflict, retrying: %v", err)
err = nil
logger.Info(message)
} else {
r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
if _, err = setStatusReady(tsr, metav1.ConditionFalse, reason, message); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{RequeueAfter: shortRequeue}, nil
}
r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
return setStatusReady(tsr, metav1.ConditionFalse, reason, message)
}