From a057d738d1dfa579a30b6848c426f067b5aba750 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 1 Oct 2026 13:54:23 +0300 Subject: [PATCH 1/2] test(e2e): give the gateway listener-set op room for its own waits The step's polls can wait 660s under a 6m op, and its cleanup trap deletes two tenants with no bound on the wait. When the op ceiling fires first, Chainsaw kills the shell: the poll that was running never reports, the trap never runs, and both tenants stay behind for every later suite. The deletes keep waiting, child before parent, since a parent deleted while its child is still uninstalling wedges its own cleanup. Each delete runs under an outer timeout with a kill grace, because kubectl's --timeout bounds only the wait after the request and not discovery or a stalled request. The parent is deleted only once the child is gone. A failed tenant delete fails the step with kubectl's error, its exit status and the tenant left behind, rather than passing with the leak. The op is sized above the polls plus those bounds, and a unit guard holds it to that sum and the cleanup to that order. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- hack/chainsaw-op-budgets.bats | 131 +++++++++++++++++++ hack/e2e-chainsaw/gateway/chainsaw-test.yaml | 32 ++++- 2 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 hack/chainsaw-op-budgets.bats diff --git a/hack/chainsaw-op-budgets.bats b/hack/chainsaw-op-budgets.bats new file mode 100644 index 000000000..126ffe477 --- /dev/null +++ b/hack/chainsaw-op-budgets.bats @@ -0,0 +1,131 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Contract: a Chainsaw script op whose script carries its own waits gives those +# waits room to fire first. When the op ceiling fires instead, Chainsaw kills +# the shell, the script's own failure message never prints, and a cleanup trap +# never runs (.chainsaw.yaml states the rule above its catch scripts). +# +# The gateway step is held by arithmetic: its polls are `timeout N` prefixes +# and its cleanup deletes run under `timeout -k K N`, so their sum, kill +# grace included, is readable from the text. kubectl's own `--timeout` is not +# counted: it bounds only the wait after the delete request, not discovery or +# the request itself. Only those two spellings are summed: a wait written any +# other way, a `kubectl wait` or a sleep loop, is invisible to the sum. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# own line; a `}` at column zero gets `return 0` inserted ahead of it, so the +# helpers below report through stdout only. +# +# Run with: hack/cozytest.sh hack/chainsaw-op-budgets.bats +# ----------------------------------------------------------------------------- + +GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml" +GATEWAY_STEP="parent-child-and-route-drive-listener-set" + +# Seconds the gateway op keeps above its summed waits for what carries no cap +# of its own: three applies and one label read. +GATEWAY_UNCAPPED=60 + +# Print the lines of the first `- script:` op inside step $2 of file $1, with +# the op's own indentation, from `- script:` up to the next op or step. +op_lines() { + awk -v step="$2" ' + $0 ~ "^[[:space:]]*- name: " step "[[:space:]]*$" { instep = 1; next } + instep && !inop && /^[[:space:]]*- script:[[:space:]]*$/ { + match($0, /^[[:space:]]*/); opindent = RLENGTH; inop = 1; print; next + } + inop && /[^[:space:]]/ { + match($0, /^[[:space:]]*/) + if (RLENGTH <= opindent) exit + } + inop { print } + instep && /^---/ { exit } + ' "$1" +} + +# Print the op timeout in seconds of the op in $1 (op_lines output), or +# nothing when it states none in minutes or seconds. +op_timeout_seconds() { + printf '%s\n' "$1" | awk ' + /^[[:space:]]*timeout:[[:space:]]*[0-9]+m[[:space:]]*$/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH) * 60; exit } + /^[[:space:]]*timeout:[[:space:]]*[0-9]+s[[:space:]]*$/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH); exit } + /^[[:space:]]*content:/ { exit } + ' +} + +# Print the op's script, dedented, without the content key. +op_content() { + printf '%s\n' "$1" | awk ' + !incontent && /^[[:space:]]*content:[[:space:]]*\|[[:space:]]*$/ { match($0, /^[[:space:]]*/); keyindent = RLENGTH; incontent = 1; next } + incontent && /[^[:space:]]/ { + match($0, /^[[:space:]]*/) + if (RLENGTH <= keyindent) exit + if (!bodyindent) bodyindent = RLENGTH + } + incontent { print substr($0, bodyindent + 1) } + ' +} + +@test "the gateway listener-set op outlasts the waits inside it" { + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + [ -n "$gw_op" ] || { echo "no script op found in step $GATEWAY_STEP of $GATEWAY_SUITE" >&2; exit 1; } + gw_timeout=$(op_timeout_seconds "$gw_op") + case "$gw_timeout" in ''|*[!0-9]*) echo "the $GATEWAY_STEP op states no timeout in m or s (got '${gw_timeout}')" >&2; exit 1 ;; esac + gw_body=$(op_content "$gw_op") + gw_polls=$(printf '%s\n' "$gw_body" | grep -oE '^[[:space:]]*timeout +[0-9]+' | grep -oE '[0-9]+$' || true) + gw_n=$(printf '%s\n' "$gw_polls" | grep -c . || true) + [ "$gw_n" -ge 5 ] || { echo "found ${gw_n} capped polls in $GATEWAY_STEP, expected at least 5; the extractor is reading the wrong lines" >&2; exit 1; } + gw_deletes=$(printf '%s\n' "$gw_body" | grep -E '(^|[[:space:]])kubectl .* delete ' | grep -oE '(^|[[:space:]])timeout -k [0-9]+ [0-9]+ kubectl ' | awk '{ print $3 + $4 }' || true) + gw_sum=0 + for gw_w in $gw_polls $gw_deletes; do gw_sum=$(( gw_sum + gw_w )); done + [ $(( gw_timeout - gw_sum )) -ge "$GATEWAY_UNCAPPED" ] || { + echo "the $GATEWAY_STEP op allows ${gw_timeout}s; its polls ($(printf '%s ' $gw_polls)) and cleanup deletes ($(printf '%s ' $gw_deletes)) may wait ${gw_sum}s" >&2 + echo "it must keep ${GATEWAY_UNCAPPED}s above them for the uncapped applies and reads" >&2 + exit 1 + } +} + +@test "every delete in the gateway listener-set cleanup is bounded as a whole" { + # A tenant delete blocks until its child namespaces drain, and a stalled + # API request blocks with no bound at all, so a delete not wrapped in an + # outer timeout is a wait the op sum above cannot see. + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + gw_deletes=$(op_content "$gw_op" | grep -E '(^|[[:space:]])kubectl .* delete ' || true) + [ -n "$gw_deletes" ] || { echo "no kubectl delete found in $GATEWAY_STEP; the cleanup this test reasons about is gone" >&2; exit 1; } + gw_bad=$(printf '%s\n' "$gw_deletes" | grep -vE '(^|[[:space:]])timeout -k [0-9]+ [0-9]+ kubectl ' || true) + [ -z "$gw_bad" ] || { echo "deletes in $GATEWAY_STEP whose wait has no bound:" >&2; printf '%s\n' "$gw_bad" >&2; exit 1; } +} + +@test "the gateway listener-set cleanup deletes the parent only once the child is gone" { + # A parent deleted while its child is still uninstalling wedges the + # parent's cleanup, so a child delete that ran into its bound must leave + # the parent alone. + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + gw_order=$(op_content "$gw_op" | awk ' + /^[[:space:]]*if timeout -k [0-9]+ [0-9]+ kubectl .* delete tenant rchild .*; then[[:space:]]*$/ { guarded = 1; next } + guarded && /[^[:space:]]/ { print (/delete tenant rparent/ ? "ok" : "unguarded"); exit } + /delete tenant rparent/ { print "unguarded"; exit } + ') + [ "$gw_order" = ok ] || { echo "the rparent delete in $GATEWAY_STEP does not sit directly under a successful rchild delete (got '${gw_order}')" >&2; exit 1; } +} + +@test "a failed tenant delete in the gateway cleanup fails the step with its own error" { + # A tenant left in place is one every later suite inherits, so + # the step must say so and fail rather than pass with the leak. + gw_dir=$(mktemp -d) + op_content "$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP")" | awk '/^cleanup\(\) \{/ { on = 1 } on { print } on && /^\}/ { exit }' > "$gw_dir/cleanup.sh" + grep -q 'delete tenant rchild' "$gw_dir/cleanup.sh" || { echo "no cleanup() with the rchild delete found in $GATEWAY_STEP" >&2; exit 1; } + # The stub fails the delete of the tenant named in STUB_FAIL_TENANT with a + # Forbidden error, which is not the bound firing, and logs every delete. + printf '%s\n' '#!/bin/sh' 'echo "$*" >> "$(dirname "$0")/calls"' 'case "$*" in *"delete tenant $STUB_FAIL_TENANT "*) echo "Error from server (Forbidden): stubbed" >&2; exit 3 ;; esac' > "$gw_dir/kubectl" + chmod +x "$gw_dir/kubectl" + for gw_tenant in rchild rparent; do + : > "$gw_dir/calls" + gw_rc=0 + gw_out=$(PATH="$gw_dir:$PATH" STUB_FAIL_TENANT="$gw_tenant" sh -c '. "$1/cleanup.sh"; ns=tenant-test; cleanup' sh "$gw_dir" 2>&1) || gw_rc=$? + [ "$gw_rc" -ne 0 ] || { printf '%s\n' "$gw_out" >&2; echo "cleanup exited 0 with the $gw_tenant delete failed" >&2; exit 1; } + case "$gw_out" in *"Forbidden"*"$gw_tenant delete exited 3"*"rparent is left in tenant-test"*) ;; *) printf '%s\n' "$gw_out" >&2; echo "with the $gw_tenant delete failed, the output does not carry kubectl's error, its exit status and the tenant left behind" >&2; exit 1 ;; esac + if [ "$gw_tenant" = rchild ] && grep -q 'delete tenant rparent' "$gw_dir/calls"; then echo "the parent was deleted although the child delete failed" >&2; exit 1; fi + done + rm -rf "$gw_dir" +} diff --git a/hack/e2e-chainsaw/gateway/chainsaw-test.yaml b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml index 5e375731b..254c8629d 100644 --- a/hack/e2e-chainsaw/gateway/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml @@ -977,16 +977,40 @@ spec: steps: - name: parent-child-and-route-drive-listener-set try: + # Above the 660s of polls plus the 675s the trap's deletes may take with + # their kill grace, so a poll or a delete is what fires and the trap still + # runs: a kill at the op ceiling would skip cleanup and leave both tenants + # behind for every later suite. hack/chainsaw-op-budgets.bats holds the op + # to that sum. - script: - timeout: 6m + timeout: 24m content: | set -eu ns="$NAMESPACE" + # Each delete waits, child before parent, because a parent deleted + # while its child is still uninstalling wedges the parent's cleanup. + # Each delete is bounded as a whole by an outer timeout, because the + # trap runs inside the op ceiling and kubectl's --timeout covers only + # the wait after the request, not discovery or a stalled request. A + # tenant delete gets the 5m that .chainsaw.yaml gives a delete op, + # because its child namespaces take minutes to drain. A child still + # uninstalling at its bound leaves the parent in place: a leaked + # tenant rather than a wedged one. A failed tenant delete fails the + # step with kubectl's own error and exit status (124 is the bound), + # so the leak is reported here rather than in whichever suite trips + # on it. cleanup() { - kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true - kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null 2>&1 || true - kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null 2>&1 || true + timeout -k 5 60 kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true + if timeout -k 5 300 kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null; then + timeout -k 5 300 kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null || { + echo "the tenant rparent delete exited $?, so tenant rparent is left in $ns" >&2 + exit 1 + } + else + echo "the tenant rchild delete exited $?, so tenant rparent is left in $ns" >&2 + exit 1 + fi } trap cleanup EXIT From cc731e3105ad33c0a678aa2ebe66a156b162a2eb Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 1 Oct 2026 13:54:33 +0300 Subject: [PATCH 2/2] test(e2e): keep redis-tls failures legible under the op ceiling The CA-projection step caps each kubectl at 30s, but more capped reads run than its 5m op can cover, so a degraded apiserver could spend the ceiling and leave nothing but "signal: killed". The reads now share a step budget held under the op, and a read that hits its bound or finds the budget spent says which read it was. Two exec checks reported the 15s redis-cli cap for any failure, sending a reader after a slow redis when the exec was refused or the container was missing. They now report the exit status and what the exec printed. The step also runs under bash with pipefail: under dash a failed Secret read took base64's zero status and surfaced as a wrong source name. The private-key check reads the projection already fetched, since a second read that failed made it pass. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- hack/chainsaw-op-budgets.bats | 208 ++++++++++++++++++++- hack/e2e-chainsaw/redis/chainsaw-test.yaml | 72 ++++--- 2 files changed, 258 insertions(+), 22 deletions(-) diff --git a/hack/chainsaw-op-budgets.bats b/hack/chainsaw-op-budgets.bats index 126ffe477..c5455e79b 100644 --- a/hack/chainsaw-op-budgets.bats +++ b/hack/chainsaw-op-budgets.bats @@ -10,7 +10,12 @@ # grace included, is readable from the text. kubectl's own `--timeout` is not # counted: it bounds only the wait after the delete request, not discovery or # the request itself. Only those two spellings are summed: a wait written any -# other way, a `kubectl wait` or a sleep loop, is invisible to the sum. +# other way, a `kubectl wait` or a sleep loop, is invisible to the sum. The +# redis-tls step cannot be summed that way, because some of its +# reads sit in loops and one loop polls, so it carries a step budget that +# every kubectl call is clamped to, and the guard holds that budget under the +# op. Its failure messages are checked by running the step's script against a +# stubbed kubectl. # # cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its # own line; a `}` at column zero gets `return 0` inserted ahead of it, so the @@ -21,10 +26,15 @@ GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml" GATEWAY_STEP="parent-child-and-route-drive-listener-set" +REDIS_SUITE="hack/e2e-chainsaw/redis/chainsaw-test.yaml" +REDIS_STEP="verify-tenant-ca-projection" # Seconds the gateway op keeps above its summed waits for what carries no cap # of its own: three applies and one label read. GATEWAY_UNCAPPED=60 +# Seconds the redis op keeps above its step budget: the kill grace on the last +# clamped read, one 5s poll sleep, and the openssl and shell work between reads. +REDIS_AFTER_BUDGET=30 # Print the lines of the first `- script:` op inside step $2 of file $1, with # the op's own indentation, from `- script:` up to the next op or step. @@ -53,6 +63,15 @@ op_timeout_seconds() { ' } +# Print the op's shell, or `sh`, the Chainsaw default, when it names none. +op_shell() { + sh_name=$(printf '%s\n' "$1" | awk ' + /^[[:space:]]*shell:[[:space:]]*[^[:space:]]+[[:space:]]*$/ { sub(/^[[:space:]]*shell:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print; exit } + /^[[:space:]]*content:/ { exit } + ') + printf '%s\n' "${sh_name:-sh}" +} + # Print the op's script, dedented, without the content key. op_content() { printf '%s\n' "$1" | awk ' @@ -66,6 +85,81 @@ op_content() { ' } +# Write a kubectl stub into directory $1 that answers the redis-tls step's +# reads as a healthy cluster would. STUB_FAIL, when set, is a pattern matched +# against the arguments: a matching call prints STUB_ERR to stderr and exits +# STUB_RC instead, after printing STUB_OUT to stdout when that is set, from the +# STUB_FAIL_FROM-th matching call on (default the first). STUB_PROJ_KEY=1 puts +# PEM private key material into the projection. STUB_SLEEP_ON is a pattern +# whose matching calls sleep STUB_SLEEP seconds first; the stub execs sleep, +# so a timeout that kills it leaves no child holding the output pipe. +write_redis_stubs() { + cat > "$1/kubectl" <<'STUB' +#!/bin/sh +args="$*" +if [ -n "${STUB_SLEEP_ON:-}" ]; then + case "$args" in + $STUB_SLEEP_ON) exec sleep "${STUB_SLEEP:-60}" ;; + esac +fi +if [ -n "${STUB_FAIL:-}" ]; then + case "$args" in + $STUB_FAIL) + echo x >> "$(dirname "$0")/fail-matches" + if [ "$(wc -l < "$(dirname "$0")/fail-matches")" -ge "${STUB_FAIL_FROM:-1}" ]; then + [ -z "${STUB_OUT:-}" ] || echo "$STUB_OUT" + echo "${STUB_ERR:-error: stubbed failure}" >&2; exit "${STUB_RC:-1}" + fi ;; + esac +fi +cert_b64=$(printf 'CERT' | base64) +proj_b64=$cert_b64 +if [ "${STUB_PROJ_KEY:-}" = 1 ]; then + proj_b64=$(printf 'CERT\n-----BEGIN PRIVATE KEY-----\nKEY\n' | base64 | tr -d '\n') +fi +case "$args" in + *"get secret redis-secure.tenant-ca -o go-template"*) printf 'ca.crt\n' ;; + *"get secret redis-secure.ca-cert -o go-template"*) printf 'ca.crt\n' ;; + *"get secret redis-secure.ca-tls -o go-template"*) printf 'ca.crt\ntls.crt\ntls.key\n' ;; + *"get secret redis-secure.ca-tls -o jsonpath"*) printf '%s' "$cert_b64" ;; + *"get secret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$proj_b64" ;; + *"get tenantsecret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$(printf -- '-----BEGIN CERTIFICATE-----' | base64)" ;; + *"get tenantsecret "*) echo "Error from server (NotFound): tenantsecrets \"x\" not found" >&2; exit 1 ;; + *"exec rfr-redis-secure-0 -c redis -- cat /tls/ca.crt"*) printf 'CERT\n' ;; + *"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli --tls"*) printf 'PONG\n' ;; + *"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli -h"*) printf 'I/O error\n'; exit 1 ;; + *"exec rfr-redis-secure-0 -c redis -- sh -c"*) printf 'role:master\n' ;; + *"exec rfr-redis-secure-1 -c redis -- sh -c"*) printf 'role:slave\nmaster_link_status:up\n' ;; + *"get pod -l app.kubernetes.io/component=sentinel"*) printf 'rfs-redis-secure-0' ;; + *"exec rfs-redis-secure-0 -c sentinel"*) printf '10.0.0.1\n6379\n' ;; + *) echo "kubectl stub: unexpected call: $args" >&2; exit 99 ;; +esac +STUB + cat > "$1/openssl" <<'STUB' +#!/bin/sh +cat >/dev/null +echo "sha256 Fingerprint=AA" +STUB + chmod +x "$1/kubectl" "$1/openssl" +} + +# Run the redis-tls step's script, as its op's shell would, against the stubs +# in $1. Prints the script's combined output and then `rc=`. RS_BUDGET, +# when set, replaces the step's budget so the clamp can be driven in seconds. +run_redis_step() { + rs_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP") + rs_shell=$(op_shell "$rs_op") + op_content "$rs_op" > "$1/step.sh" + if [ -n "${RS_BUDGET:-}" ]; then + sed "s/^step_budget=[0-9]*\$/step_budget=${RS_BUDGET}/" "$1/step.sh" > "$1/step.budget" + if cmp -s "$1/step.sh" "$1/step.budget"; then echo "no step_budget line to override"; echo "rc=97"; return; fi + mv "$1/step.budget" "$1/step.sh" + fi + rs_rc=0 + rs_out=$(cd "$1" && PATH="$1:$PATH" NAMESPACE=tenant-test STUB_FAIL="${STUB_FAIL:-}" STUB_FAIL_FROM="${STUB_FAIL_FROM:-1}" STUB_RC="${STUB_RC:-1}" STUB_ERR="${STUB_ERR:-}" STUB_OUT="${STUB_OUT:-}" STUB_SLEEP_ON="${STUB_SLEEP_ON:-}" STUB_SLEEP="${STUB_SLEEP:-}" STUB_PROJ_KEY="${STUB_PROJ_KEY:-}" "$rs_shell" "$1/step.sh" 2>&1) || rs_rc=$? + printf '%s\nrc=%s\n' "$rs_out" "$rs_rc" +} + @test "the gateway listener-set op outlasts the waits inside it" { gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") [ -n "$gw_op" ] || { echo "no script op found in step $GATEWAY_STEP of $GATEWAY_SUITE" >&2; exit 1; } @@ -129,3 +223,115 @@ op_content() { done rm -rf "$gw_dir" } + +@test "the redis-tls projection op outlasts the step budget its reads are clamped to" { + rt_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP") + [ -n "$rt_op" ] || { echo "no script op found in step $REDIS_STEP of $REDIS_SUITE" >&2; exit 1; } + rt_timeout=$(op_timeout_seconds "$rt_op") + case "$rt_timeout" in ''|*[!0-9]*) echo "the $REDIS_STEP op states no timeout in m or s (got '${rt_timeout}')" >&2; exit 1 ;; esac + rt_body=$(op_content "$rt_op") + rt_budget=$(printf '%s\n' "$rt_body" | sed -n 's/^step_budget=\([0-9][0-9]*\)$/\1/p') + [ -n "$rt_budget" ] || { + echo "$REDIS_STEP sets no step_budget, so nothing bounds the sum of its reads below the ${rt_timeout}s op" >&2 + echo "its kubectl call sites alone come to $(printf '%s\n' "$rt_body" | grep -cE '(^|[^[:alnum:]_])k -n ')" >&2 + exit 1 + } + [ $(( rt_timeout - rt_budget )) -ge "$REDIS_AFTER_BUDGET" ] || { + echo "the $REDIS_STEP op allows ${rt_timeout}s and the step budget is ${rt_budget}s" >&2 + echo "it must keep ${REDIS_AFTER_BUDGET}s above the budget for the last read's kill grace and the work between reads" >&2 + exit 1 + } + # The budget bounds only the reads that go through k(). A bare kubectl + # outside it would add an uncapped read the arithmetic above never sees. + rt_bare=$(printf '%s\n' "$rt_body" | grep -vE '^[[:space:]]*#' | grep -nE '(^|[^[:alnum:]_-])kubectl([[:space:]]|$)' | grep -v '"\$k_left" kubectl' || true) + [ -z "$rt_bare" ] || { echo "kubectl calls in $REDIS_STEP that bypass the clamped k():" >&2; printf '%s\n' "$rt_bare" >&2; exit 1; } +} + +@test "the redis-tls projection script passes against a healthy stub" { + # The positive control for the two failure tests below: without it, a + # stub that broke the script somewhere else would make them pass. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") ;; *) printf '%s\n' "$rt_out" >&2; echo "the step failed against a healthy stub" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed replication exec reports its own error, not the 15s cap" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*exec rfr-redis-secure-1 -c redis -- sh -c*' STUB_RC=1 \ + STUB_ERR='Error from server (Forbidden): pods "rfr-redis-secure-1" is forbidden' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a forbidden exec" >&2; exit 1 ;; esac + case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a forbidden exec was reported as the 15s cap firing" >&2; exit 1 ;; esac + case "$rt_out" in *"rfr-redis-secure-1"*"exit 1"*"Forbidden"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the message does not carry the exit status and what the exec printed" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed sentinel exec reports its own error, not the 15s cap" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*exec rfs-redis-secure-0 -c sentinel*' STUB_RC=1 \ + STUB_ERR='command terminated with exit code 1' STUB_OUT='NOAUTH Authentication required.' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed sentinel exec" >&2; exit 1 ;; esac + case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a refused sentinel command was reported as the 15s cap firing" >&2; exit 1 ;; esac + case "$rt_out" in *"command terminated with exit code 1"*"rfs-redis-secure-0 failed (exit 1): NOAUTH Authentication required."*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the output does not carry what the exec wrote to both streams and its exit status" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed Secret read is not reported as a wrong source name" { + # base64 -d exits 0 on empty input, so without pipefail the pipeline takes + # its status and the emptiness check below names the wrong cause. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*get secret redis-secure.ca-tls -o jsonpath*' STUB_RC=1 \ + STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed Secret read" >&2; exit 1 ;; esac + case "$rt_out" in *"source name or key is wrong"*) printf '%s\n' "$rt_out" >&2; echo "an API error was reported as a wrong source name" >&2; exit 1 ;; esac + case "$rt_out" in *"ServiceUnavailable"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the API error is missing from the output" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "the private-key check fails when the projection it reads carries a key" { + # The projection is read once above this check. A check that reads it + # again passes when that second read fails, because an empty read holds + # no key material: here the first read returns a key and the second one + # is refused. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_PROJ_KEY=1 STUB_FAIL='*get secret redis-secure.tenant-ca -o jsonpath*' STUB_FAIL_FROM=2 STUB_RC=1 \ + STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with key material in the projection" >&2; exit 1 ;; esac + case "$rt_out" in *"projection carries PEM private key material"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the step did not fail on the key material" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a read that finds the step budget spent does not run and says so" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(RS_BUDGET=0 run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with its budget spent before the first read" >&2; exit 1 ;; esac + case "$rt_out" in *"step budget is spent, so this read did not run: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the first read did not report the spent budget" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a read is clamped to what the step budget has left and names itself" { + # A 2s budget against a read that would hang for 20s: the clamp has to + # end the read at the budget, not at the 30s per-read cap. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_start=$(date +%s) + rt_out=$(RS_BUDGET=2 STUB_SLEEP_ON='*get secret redis-secure.tenant-ca -o go-template*' STUB_SLEEP=20 run_redis_step "$rt_dir") + rt_took=$(( $(date +%s) - rt_start )) + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a hung read" >&2; exit 1 ;; esac + case "$rt_out" in *"ran into its "[12]"s bound: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the hung read did not name itself at the budget's bound" >&2; exit 1 ;; esac + [ "$rt_took" -lt 15 ] || { echo "the step took ${rt_took}s against a 2s budget; the read was not clamped" >&2; exit 1; } + rm -rf "$rt_dir" +} diff --git a/hack/e2e-chainsaw/redis/chainsaw-test.yaml b/hack/e2e-chainsaw/redis/chainsaw-test.yaml index 0b2ad88fa..4b06d6cc8 100644 --- a/hack/e2e-chainsaw/redis/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/redis/chainsaw-test.yaml @@ -270,14 +270,13 @@ spec: # No retry around this script. The replica link it reads is settled by # none of the asserts above and carries its own bounded wait below. The # whole step measured 3.0s on run 34711755303, so link_wait is wide - # against what it costs today. That wait draws link_wait plus one poll - # interval plus a read pair at the k() cap worst case, since the deadline - # is tested before the sleep and never after, and the op ceiling covers - # that on top of reads ahead of it at the cost that measurement shows, - # not at their caps, whose sum exceeds any ceiling this step could carry. - # Sized generously on purpose: unused margin costs a passing run nothing, - # while a ceiling that expires first kills the shell and takes the wait's - # own failure message with it. + # against what it costs today. The reads' 30s caps sum past the 5m op, + # so they share step_budget instead: no read starts after it, one + # already running ends within its 5s kill grace of it, and the op sits + # above both, so a read or the budget is what fails and names itself + # rather than a ceiling that kills the shell and takes the script's own + # message with it. hack/chainsaw-op-budgets.bats holds the budget under + # the op. # # Every redis-cli below runs under the container's own `timeout`. That # inner cap, not the op timeout, is what keeps a dial that never answers @@ -288,17 +287,41 @@ spec: # failing. Both engine images carry /usr/bin/timeout; expiry reports 143 # on the sentinel's busybox and 124 on the data pods' coreutils, and # neither is zero, which is all these checks read. + # + # bash for pipefail: base64 -d exits 0 on empty input, so without it a + # failed Secret read below takes the decoder's status and is reported as + # the emptiness it left behind. The sandbox's /bin/sh is dash, which + # rejects `set -o pipefail`; `shell:` is a Script field in the Chainsaw + # v0.2.15 the sandbox image installs. - script: timeout: 5m + shell: bash content: | - set -eu + set -euo pipefail # Every kubectl below runs under a wall-clock bound. The inner # `timeout` on a redis-cli cannot fire if the exec stream itself # never establishes, and a kubectl still running holds this script's # stdout open, which is what turns a stuck call into a silent run to - # the job ceiling instead of a failure. - k() { timeout 30 kubectl "$@"; } + # the job ceiling instead of a failure. A call gets 30s at most and + # never more than step_budget has left. + step_budget=240 + step_deadline=$(( $(date +%s) + step_budget )) + k() { + k_left=$(( step_deadline - $(date +%s) )) + if [ "$k_left" -le 0 ]; then + echo "the ${step_budget}s step budget is spent, so this read did not run: $*" >&2 + return 124 + fi + [ "$k_left" -le 30 ] || k_left=30 + k_start=$(date +%s) + k_rc=0 + timeout -k 5 "$k_left" kubectl "$@" || k_rc=$? + if [ "$k_rc" -ne 0 ] && [ $(( $(date +%s) - k_start )) -ge "$k_left" ]; then + echo "this read ran into its ${k_left}s bound: $*" >&2 + fi + return "$k_rc" + } # Exactly one key, and it is ca.crt. The source carries the CA # private key, so any second key is a leak in the making. @@ -332,14 +355,15 @@ spec: fi # The source really does hold the private key, or the leak check - # below would pass while proving nothing. + # below would pass while proving nothing. grep reads to EOF here + # rather than quitting at the match under -q: with pipefail, a + # kubectl still writing would take SIGPIPE and fail the pipeline. if ! k -n "$NAMESPACE" get secret redis-secure.ca-tls \ - -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'tls.key'; then + -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -x 'tls.key' >/dev/null; then echo "redis-secure.ca-tls no longer carries tls.key: this check has stopped proving anything" >&2 exit 1 fi - if k -n "$NAMESPACE" get secret redis-secure.tenant-ca \ - -o jsonpath='{.data.ca\.crt}' | base64 -d | grep -qi 'BEGIN .*PRIVATE KEY'; then + if grep -qi 'BEGIN .*PRIVATE KEY' <<<"$proj"; then echo "projection carries PEM private key material" >&2 exit 1 fi @@ -455,9 +479,11 @@ spec: linked=0 seen= for i in 0 1; do - if ! info=$(k -n "$NAMESPACE" exec "rfr-redis-secure-$i" -c redis -- \ - sh -c 'REDISCLI_AUTH="$REDIS_PASSWORD" timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 6379 info replication'); then - echo "rfr-redis-secure-$i did not answer INFO replication within the 15s cap" >&2 + rc=0 + info=$(k -n "$NAMESPACE" exec "rfr-redis-secure-$i" -c redis -- \ + sh -c 'REDISCLI_AUTH="$REDIS_PASSWORD" timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 6379 info replication' 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + echo "INFO replication on rfr-redis-secure-$i failed (exit $rc): $info" >&2 exit 1 fi case "$info" in @@ -487,10 +513,14 @@ spec: sentinel=$(k -n "$NAMESPACE" get pod \ -l app.kubernetes.io/component=sentinel,app.kubernetes.io/name=redis-secure \ -o jsonpath='{.items[0].metadata.name}') - if ! addr=$(k -n "$NAMESPACE" exec "$sentinel" -c sentinel -- \ + rc=0 + addr=$(k -n "$NAMESPACE" exec "$sentinel" -c sentinel -- \ timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 26379 \ - sentinel get-master-addr-by-name mymaster); then - echo "the sentinel did not answer get-master-addr-by-name within the 15s cap" >&2 + sentinel get-master-addr-by-name mymaster) || rc=$? + # stderr stays out of $addr, whose first line is parsed below; what + # the exec wrote there is already in the log above this line. + if [ "$rc" -ne 0 ]; then + echo "get-master-addr-by-name on $sentinel failed (exit $rc): $addr" >&2 exit 1 fi addr=$(echo "$addr" | head -n 1 | tr -d '\r')