From 1ffbb30d25440db2ec2872aa1bcc48f8a37b8e05 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 1 Jul 2026 03:53:52 +0300 Subject: [PATCH 1/2] fix(e2e): widen VictoriaMetrics operational waits to 15m under load vlclusters/generic and vmcluster wait for updateStatus=operational with a 10m budget, while their vmalert/vmalertmanager sibling in the same block already uses 15m. All three are vm-operator-managed monitoring resources that only become operational once their workloads schedule and turn Ready, and during platform bring-up they contend for node resources with the rest of the install. On a calm sandbox each converges in under a second, but under install-time load (concurrent e2e sandboxes on one runner) monitoring bring-up is slow, and a 10m/15m split inside a block whose resources all contend for the same node capacity risks failing the whole install on a PR that never touched monitoring. Unify vlclusters and vmcluster on the sibling's 15m budget so the whole block shares one headroom, and dump resource YAML plus pod state on timeout across all three waits so a genuine stuck-not-slow regression stays legible instead of a bare "timed out" line. The extra budget costs near-zero wall-time on the happy path and stays inside the E2E job budget. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-install-cozystack.bats | 39 +++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index bf394ea7c..fffb933f0 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -213,14 +213,45 @@ EOF timeout 60 sh -ec 'until kubectl get sts/etcd -n tenant-root >/dev/null 2>&1; do sleep 2; done' kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=10m - # VictoriaMetrics components + # VictoriaMetrics components. vmalert/vmalertmanager, vlclusters/generic and + # vmcluster/shortterm+longterm are all vm-operator-managed resources that flip + # updateStatus=operational only once their workloads are scheduled and Ready. + # During platform bring-up they contend for node resources with the rest of + # the install, so convergence is load-sensitive: on a calm sandbox each reaches + # operational in under a second, but under install-time load (concurrent e2e + # sandboxes on one runner) monitoring bring-up is slow. vmalert already uses a + # 15m budget; vlclusters and vmcluster used 10m, so this block carried a + # 10m/15m split even though all three contend for the same node capacity and a + # slow VictoriaLogs bring-up can fail the install on a PR that never touched + # monitoring. Unify the block on one 15m budget (near-zero cost in the happy + # path, comfortably inside the E2E job budget) and dump live status on timeout + # so a genuine stuck-not-slow regression stays legible instead of surfacing as + # a bare "timed out" line. timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done' - kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m + kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { + echo "=== vmalert/vmalert-shortterm, vmalertmanager/alertmanager did not reach updateStatus=operational ===" + kubectl get vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root -o yaml 2>&1 || true + echo "=== tenant-root pods ===" + kubectl get pods -n tenant-root -o wide 2>&1 || true + false + } timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done' - kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=10m + kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { + echo "=== vlclusters/generic did not reach updateStatus=operational ===" + kubectl get vlclusters/generic -n tenant-root -o yaml 2>&1 || true + echo "=== tenant-root pods ===" + kubectl get pods -n tenant-root -o wide 2>&1 || true + false + } timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' - kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=10m + kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { + echo "=== vmcluster/shortterm,longterm did not reach updateStatus=operational ===" + kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root -o yaml 2>&1 || true + echo "=== tenant-root pods ===" + kubectl get pods -n tenant-root -o wide 2>&1 || true + false + } # Grafana timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done' From 09f9d0bdec855d25d25b8cbe067c556a1c67f2e2 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 1 Jul 2026 05:01:31 +0300 Subject: [PATCH 2/2] fix(e2e): widen Grafana bring-up waits to 15m under load The Configure-Tenant install test completes the tenant-root monitoring bring-up by waiting for the grafana-db CNPG cluster and the grafana-deployment Deployment. Both used a 10m budget while the VictoriaMetrics waits above them use 15m. They complete the same monitoring bring-up and contend for the same node resources during install, so under install-time load (concurrent e2e sandboxes on one runner) either can be slow, and the 10m/15m split within the same contend-for-resources bring-up risks failing the whole install on a PR that never touched monitoring. Raise both Grafana waits to the same uniform 15m budget as the vm-operator waits and wrap them in an on-timeout diagnostic block that dumps the resource YAML plus tenant-root pod state, so a genuinely stuck resource still fails at 15m with legible diagnostics instead of a bare "timed out" line. The extra budget costs near-zero wall-time on the happy path and stays inside the E2E job budget. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- hack/e2e-install-cozystack.bats | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index fffb933f0..e7f501c1b 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -253,11 +253,30 @@ EOF false } - # Grafana + # Grafana. The grafana-db CNPG cluster and the grafana-deployment Deployment + # complete the tenant-root monitoring bring-up and contend for the same node + # resources as the VictoriaMetrics stack above during install. Under + # install-time load (concurrent e2e sandboxes on one runner) either can be slow + # and fail the install on a PR that never touched monitoring, so both move from + # their 10m budget to the same uniform 15m as the vm-operator waits above and + # dump live status on timeout to keep a genuine stuck-not-slow regression + # legible instead of surfacing as a bare "timed out" line. timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done' - kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=10m + kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=15m || { + echo "=== clusters.postgresql.cnpg.io/grafana-db did not reach condition=ready ===" + kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root -o yaml 2>&1 || true + echo "=== tenant-root pods ===" + kubectl get pods -n tenant-root -o wide 2>&1 || true + false + } timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done' - kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=10m + kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=15m || { + echo "=== deploy/grafana-deployment did not reach condition=available ===" + kubectl get deploy/grafana-deployment -n tenant-root -o yaml 2>&1 || true + echo "=== tenant-root pods ===" + kubectl get pods -n tenant-root -o wide 2>&1 || true + false + } # Verify Grafana via ingress ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}')