mirror of
https://github.com/tailscale/tailscale.git
synced 2026-11-22 04:32:40 +09:00
cmd/k8s-operator,k8s-operator/reconciler: requeue on optimistic lock errors
Reconcilers logged "optimistic lock error, retrying" with the full conflict error at Info, and Connector, Recorder and ProxyGroup also put it in their status. That reads like a failure, but a conflict between a cached read and an update is expected and is retried. The reconcilers also returned an empty result, so the retry only happened if a watch event arrived. This fixes this by logging the conflict at Debug as an update conflict, using a neutral status message and requeueing after a short delay. Updates #22074 Updates #18922 Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
1 parent
2993b5e6de
commit
9a44f4f69c
10 files changed
+49
-32
No files matched your search
@@ -86,8 +86,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
|
||||
if markedForDeletion(pg) {
|
||||
logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up")
|
||||
if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
return res, nil
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
|
||||
return res, err
|
||||
@@ -96,8 +96,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re
|
||||
err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient)
|
||||
if err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
return reconcile.Result{}, nil
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
@@ -148,9 +148,10 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
|
||||
message := fmt.Sprintf(messageConnectorCreationFailed, err)
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
reason = reasonConnectorCreating
|
||||
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
|
||||
message = "Connector resources are being updated, retrying"
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
err = nil
|
||||
logger.Info(message)
|
||||
res = reconcile.Result{RequeueAfter: shortRequeue}
|
||||
} else {
|
||||
a.recorder.Event(cn, corev1.EventTypeWarning, reason, message)
|
||||
}
|
||||
|
||||
@@ -163,10 +163,10 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
|
||||
|
||||
if err := esr.maybeProvision(ctx, svc, lg); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
lg.Infof("optimistic lock error, retrying: %s", err)
|
||||
} else {
|
||||
return reconcile.Result{}, err
|
||||
lg.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
|
||||
@@ -84,10 +84,10 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
|
||||
|
||||
if err := a.maybeProvision(ctx, logger, ing); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
} else {
|
||||
return reconcile.Result{}, err
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
|
||||
@@ -149,8 +149,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
|
||||
|
||||
if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
return reconcile.Result{}, nil
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
} else if !done {
|
||||
@@ -167,7 +167,13 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
|
||||
|
||||
oldPGStatus := pg.Status.DeepCopy()
|
||||
staticEndpoints, nrr, err := r.reconcilePG(ctx, tsClient, pg, logger)
|
||||
return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints))
|
||||
if err = errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)); err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
if nrr != nil && nrr.requeue {
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
// reconcilePG handles all reconciliation of a ProxyGroup that is not marked
|
||||
@@ -474,7 +480,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za
|
||||
if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) {
|
||||
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
|
||||
if reconciler.IsOptimisticLockError(updateErr) {
|
||||
logger.Infof("optimistic lock error updating status, retrying: %s", updateErr)
|
||||
logger.Debugf("status update conflict, will reconcile on the next event: %v", updateErr)
|
||||
updateErr = nil
|
||||
}
|
||||
err = errors.Join(err, updateErr)
|
||||
@@ -1100,9 +1106,12 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool {
|
||||
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
|
||||
err := fmt.Errorf(format, a...)
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error())
|
||||
logger.Info(msg)
|
||||
return notReady(reasonProxyGroupCreating, msg)
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return nil, ¬ReadyReason{
|
||||
reason: reasonProxyGroupCreating,
|
||||
message: "ProxyGroup resources are being updated, retrying",
|
||||
requeue: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
|
||||
@@ -1115,4 +1124,5 @@ func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.Su
|
||||
type notReadyReason struct {
|
||||
reason string
|
||||
message string
|
||||
requeue bool // set for optimistic lock errors, which are retried after shortRequeue
|
||||
}
|
||||
@@ -191,10 +191,10 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque
|
||||
needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient)
|
||||
if err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
} else {
|
||||
return reconcile.Result{}, err
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
if needsRequeue {
|
||||
res = reconcile.Result{RequeueAfter: requeueInterval()}
|
||||
|
||||
@@ -125,10 +125,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
|
||||
|
||||
if err := a.maybeProvision(ctx, logger, svc); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
} else {
|
||||
return reconcile.Result{}, err
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
|
||||
@@ -157,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
|
||||
|
||||
if err := r.maybeProvision(ctx, proxySvc, logger); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
} else {
|
||||
return reconcile.Result{}, err
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
@@ -49,6 +50,8 @@ const (
|
||||
|
||||
defaultNameserverImageRepo = "tailscale/k8s-nameserver"
|
||||
defaultNameserverImageTag = "stable"
|
||||
|
||||
shortRequeue = 5 * time.Second
|
||||
)
|
||||
|
||||
var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount)
|
||||
@@ -159,8 +162,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res
|
||||
}
|
||||
if err = r.maybeProvision(ctx, &dnsCfg); err != nil {
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
logger.Infof("optimistic lock error, retrying: %s", err)
|
||||
return reconcile.Result{}, nil
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
} else {
|
||||
return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
|
||||
}
|
||||
|
||||
@@ -214,12 +214,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
|
||||
message := fmt.Sprintf("failed creating Recorder: %s", err)
|
||||
if reconciler.IsOptimisticLockError(err) {
|
||||
reason = reasonRecorderCreating
|
||||
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
|
||||
message = "Recorder resources are being updated, retrying"
|
||||
logger.Debugf("update conflict, retrying: %v", err)
|
||||
err = nil
|
||||
logger.Info(message)
|
||||
} else {
|
||||
r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
|
||||
if _, err = setStatusReady(tsr, metav1.ConditionFalse, reason, message); err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
return reconcile.Result{RequeueAfter: shortRequeue}, nil
|
||||
}
|
||||
r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
|
||||
return setStatusReady(tsr, metav1.ConditionFalse, reason, message)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user