diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go index 94ecac7be..067424f46 100644 --- a/cmd/k8s-operator/api-server-proxy-pg.go +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -86,8 +86,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") if err = r.maybeCleanup(ctx, serviceName, pg, logger, tsClient); reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return res, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return res, err @@ -96,8 +96,8 @@ func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req re err = r.maybeProvision(ctx, serviceName, pg, logger, tsClient) if err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return reconcile.Result{}, err } diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 41e21dad0..e9b95c3ea 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -148,9 +148,10 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque message := fmt.Sprintf(messageConnectorCreationFailed, err) if reconciler.IsOptimisticLockError(err) { reason = reasonConnectorCreating - message = fmt.Sprintf("optimistic lock error, retrying: %s", err) + message = "Connector resources are being updated, retrying" + logger.Debugf("update conflict, retrying: %v", err) err = nil - logger.Info(message) + res = reconcile.Result{RequeueAfter: shortRequeue} } else { a.recorder.Event(cn, corev1.EventTypeWarning, reason, message) } diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7ec758fd7..8bfccabae 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -163,10 +163,10 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re if err := esr.maybeProvision(ctx, svc, lg); err != nil { if reconciler.IsOptimisticLockError(err) { - lg.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + lg.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return res, nil diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index a09524d59..b6cc8d4e0 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -84,10 +84,10 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request if err := a.maybeProvision(ctx, logger, ing); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return reconcile.Result{}, nil diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index cf1c25e9f..bea9fc21b 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -149,8 +149,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ if done, err := r.maybeCleanup(ctx, tsClient, pg); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } return reconcile.Result{}, err } else if !done { @@ -167,7 +167,13 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ oldPGStatus := pg.Status.DeepCopy() staticEndpoints, nrr, err := r.reconcilePG(ctx, tsClient, pg, logger) - return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) + if err = errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)); err != nil { + return reconcile.Result{}, err + } + if nrr != nil && nrr.requeue { + return reconcile.Result{RequeueAfter: shortRequeue}, nil + } + return reconcile.Result{}, nil } // reconcilePG handles all reconciliation of a ProxyGroup that is not marked @@ -474,7 +480,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { if reconciler.IsOptimisticLockError(updateErr) { - logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) + logger.Debugf("status update conflict, will reconcile on the next event: %v", updateErr) updateErr = nil } err = errors.Join(err, updateErr) @@ -1100,9 +1106,12 @@ func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool { func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) if reconciler.IsOptimisticLockError(err) { - msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) - logger.Info(msg) - return notReady(reasonProxyGroupCreating, msg) + logger.Debugf("update conflict, retrying: %v", err) + return nil, ¬ReadyReason{ + reason: reasonProxyGroupCreating, + message: "ProxyGroup resources are being updated, retrying", + requeue: true, + }, nil } r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) @@ -1115,4 +1124,5 @@ func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.Su type notReadyReason struct { reason string message string + requeue bool // set for optimistic lock errors, which are retried after shortRequeue } diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 61ff57cf8..be362dae5 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -191,10 +191,10 @@ func (r *HAServiceReconciler) Reconcile(ctx context.Context, req reconcile.Reque needsRequeue, err = r.maybeProvision(ctx, hostname, svc, pg, logger, tsClient) if err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } if needsRequeue { res = reconcile.Result{RequeueAfter: requeueInterval()} diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index daa5edf9c..26c6d4f64 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -125,10 +125,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request if err := a.maybeProvision(ctx, logger, svc); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - } else { - return reconcile.Result{}, err + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + return reconcile.Result{}, err } return reconcile.Result{}, nil diff --git a/k8s-operator/reconciler/dnsrecords/dnsrecords.go b/k8s-operator/reconciler/dnsrecords/dnsrecords.go index 30daf87b9..e11eeb5cf 100644 --- a/k8s-operator/reconciler/dnsrecords/dnsrecords.go +++ b/k8s-operator/reconciler/dnsrecords/dnsrecords.go @@ -157,7 +157,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res if err := r.maybeProvision(ctx, proxySvc, logger); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) + logger.Debugf("update conflict, retrying: %v", err) return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { return reconcile.Result{}, err diff --git a/k8s-operator/reconciler/nameserver/nameserver.go b/k8s-operator/reconciler/nameserver/nameserver.go index 705b7e177..ae44e0e30 100644 --- a/k8s-operator/reconciler/nameserver/nameserver.go +++ b/k8s-operator/reconciler/nameserver/nameserver.go @@ -12,6 +12,7 @@ import ( _ "embed" "errors" "fmt" + "time" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" @@ -49,6 +50,8 @@ const ( defaultNameserverImageRepo = "tailscale/k8s-nameserver" defaultNameserverImageTag = "stable" + + shortRequeue = 5 * time.Second ) var gaugeNameserverResources = clientmetric.NewGauge(kubetypes.MetricNameserverCount) @@ -159,8 +162,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (res } if err = r.maybeProvision(ctx, &dnsCfg); err != nil { if reconciler.IsOptimisticLockError(err) { - logger.Infof("optimistic lock error, retrying: %s", err) - return reconcile.Result{}, nil + logger.Debugf("update conflict, retrying: %v", err) + return reconcile.Result{RequeueAfter: shortRequeue}, nil } else { return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err) } diff --git a/k8s-operator/reconciler/recorder/recorder.go b/k8s-operator/reconciler/recorder/recorder.go index 60497b578..826ed7371 100644 --- a/k8s-operator/reconciler/recorder/recorder.go +++ b/k8s-operator/reconciler/recorder/recorder.go @@ -214,12 +214,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco message := fmt.Sprintf("failed creating Recorder: %s", err) if reconciler.IsOptimisticLockError(err) { reason = reasonRecorderCreating - message = fmt.Sprintf("optimistic lock error, retrying: %s", err) + message = "Recorder resources are being updated, retrying" + logger.Debugf("update conflict, retrying: %v", err) err = nil - logger.Info(message) - } else { - r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message) + if _, err = setStatusReady(tsr, metav1.ConditionFalse, reason, message); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{RequeueAfter: shortRequeue}, nil } + r.recorder.Event(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message) return setStatusReady(tsr, metav1.ConditionFalse, reason, message) }