diff --git a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml index 7c583dce6..aaf2daa74 100644 --- a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml @@ -19,18 +19,31 @@ spec: try: - script: # ~25m Kamaji bringup + worker join + LB/NFS/ouroboros assertions. - # The failure path can spend ~5m10s in bounded guest Talos capture, - # 35s per virt-launcher Pod reading the guest serial console, one 35s - # list read, one 35s read of the wedge check that runs before any - # other diagnostic, and when any console read comes back silent two - # more 35s reads for the Pod state that says whether the container - # started (30s each plus the 5s kill grace, so ~3m30s at the pool's - # minReplicas of 2 once a silent read pulls in the Pod-state pair, - # and ~5m50s at worst -- that walk is capped at 6 - # Pods and the Pod-state reads are taken once, so neither follows - # maxReplicas), and another 6m30s in its in-process tenant snapshot. - # Keep the outer op above those inner budgets with useful - # scheduling/API-call slack. + # The on-failure diagnostics phase carries its own 8m wall-clock budget + # (COZY_DIAG_PHASE_BUDGET in _lib/run-kubernetes.sh): once it is spent, the + # collectors that have not started are declined out loud rather than + # attempted, so the in-process tenant crust-gather snapshot, the one the + # failing exit triggers and the largest artifact this path produces, still + # has its time. Why that budget and not another is derived beside it, from + # the op ceiling here minus the bringup and the snapshot; it is not repeated + # here, because a number kept in two places is a number that drifts. + # + # What this ceiling covers, in the order the phase spends it, cheapest first + # so that a budget running out drops the collectors that cost minutes rather + # than the reads that discriminate between failure modes: + # node-join diagnostics reads + # serial-console family (~5m50s at worst, ~3m30s at the pool's minimum) + # guest Talos capture + # talos-image-cache diagnosis + # and then, after the phase rather than inside it, the in-process tenant + # crust-gather snapshot. + # + # Those do not fit what is left of this op once the bringup ahead of them and + # the snapshot behind them are counted, which is why the phase is bounded as a + # phase rather than this ceiling raised. None of it is a worst case and none is + # claimed: several of those collectors carry no wall-clock bound at all, and + # the bringup's own waits already exceed this op on ceilings before a single + # diagnostic runs. Those residuals are tracked in cozystack/cozystack#3666. timeout: 50m # Co-locate the tenant crust-gather snapshot (captured by # run-kubernetes.sh on failure) under the same snapshots// dir as diff --git a/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml index 085eb9c94..70ec9cfa3 100644 --- a/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml @@ -13,18 +13,31 @@ spec: - name: tenant-kubernetes-previous-version try: - script: - # The failure path can spend ~5m10s in bounded guest Talos capture, - # 35s per virt-launcher Pod reading the guest serial console, one 35s - # list read, one 35s read of the wedge check that runs before any - # other diagnostic, and when any console read comes back silent two - # more 35s reads for the Pod state that says whether the container - # started (30s each plus the 5s kill grace, so ~3m30s at the pool's - # minReplicas of 2 once a silent read pulls in the Pod-state pair, - # and ~5m50s at worst -- that walk is capped at 6 - # Pods and the Pod-state reads are taken once, so neither follows - # maxReplicas), and another 6m30s in its in-process tenant snapshot. - # Keep the outer op above those inner budgets with useful - # scheduling/API-call slack. + # The on-failure diagnostics phase carries its own 8m wall-clock budget + # (COZY_DIAG_PHASE_BUDGET in _lib/run-kubernetes.sh): once it is spent, the + # collectors that have not started are declined out loud rather than + # attempted, so the in-process tenant crust-gather snapshot, the one the + # failing exit triggers and the largest artifact this path produces, still + # has its time. Why that budget and not another is derived beside it, from + # the op ceiling here minus the bringup and the snapshot; it is not repeated + # here, because a number kept in two places is a number that drifts. + # + # What this ceiling covers, in the order the phase spends it, cheapest first + # so that a budget running out drops the collectors that cost minutes rather + # than the reads that discriminate between failure modes: + # node-join diagnostics reads + # serial-console family (~5m50s at worst, ~3m30s at the pool's minimum) + # guest Talos capture + # talos-image-cache diagnosis + # and then, after the phase rather than inside it, the in-process tenant + # crust-gather snapshot. + # + # Those do not fit what is left of this op once the bringup ahead of them and + # the snapshot behind them are counted, which is why the phase is bounded as a + # phase rather than this ceiling raised. None of it is a worst case and none is + # claimed: several of those collectors carry no wall-clock bound at all, and + # the bringup's own waits already exceed this op on ceilings before a single + # diagnostic runs. Those residuals are tracked in cozystack/cozystack#3666. timeout: 50m # Co-locate the tenant crust-gather snapshot (captured by # run-kubernetes.sh on failure) under the same snapshots// dir as