diff --git a/hack/chainsaw-op-budgets.bats b/hack/chainsaw-op-budgets.bats new file mode 100644 index 000000000..c5455e79b --- /dev/null +++ b/hack/chainsaw-op-budgets.bats @@ -0,0 +1,337 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Contract: a Chainsaw script op whose script carries its own waits gives those +# waits room to fire first. When the op ceiling fires instead, Chainsaw kills +# the shell, the script's own failure message never prints, and a cleanup trap +# never runs (.chainsaw.yaml states the rule above its catch scripts). +# +# The gateway step is held by arithmetic: its polls are `timeout N` prefixes +# and its cleanup deletes run under `timeout -k K N`, so their sum, kill +# grace included, is readable from the text. kubectl's own `--timeout` is not +# counted: it bounds only the wait after the delete request, not discovery or +# the request itself. Only those two spellings are summed: a wait written any +# other way, a `kubectl wait` or a sleep loop, is invisible to the sum. The +# redis-tls step cannot be summed that way, because some of its +# reads sit in loops and one loop polls, so it carries a step budget that +# every kubectl call is clamped to, and the guard holds that budget under the +# op. Its failure messages are checked by running the step's script against a +# stubbed kubectl. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# own line; a `}` at column zero gets `return 0` inserted ahead of it, so the +# helpers below report through stdout only. +# +# Run with: hack/cozytest.sh hack/chainsaw-op-budgets.bats +# ----------------------------------------------------------------------------- + +GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml" +GATEWAY_STEP="parent-child-and-route-drive-listener-set" +REDIS_SUITE="hack/e2e-chainsaw/redis/chainsaw-test.yaml" +REDIS_STEP="verify-tenant-ca-projection" + +# Seconds the gateway op keeps above its summed waits for what carries no cap +# of its own: three applies and one label read. +GATEWAY_UNCAPPED=60 +# Seconds the redis op keeps above its step budget: the kill grace on the last +# clamped read, one 5s poll sleep, and the openssl and shell work between reads. +REDIS_AFTER_BUDGET=30 + +# Print the lines of the first `- script:` op inside step $2 of file $1, with +# the op's own indentation, from `- script:` up to the next op or step. +op_lines() { + awk -v step="$2" ' + $0 ~ "^[[:space:]]*- name: " step "[[:space:]]*$" { instep = 1; next } + instep && !inop && /^[[:space:]]*- script:[[:space:]]*$/ { + match($0, /^[[:space:]]*/); opindent = RLENGTH; inop = 1; print; next + } + inop && /[^[:space:]]/ { + match($0, /^[[:space:]]*/) + if (RLENGTH <= opindent) exit + } + inop { print } + instep && /^---/ { exit } + ' "$1" +} + +# Print the op timeout in seconds of the op in $1 (op_lines output), or +# nothing when it states none in minutes or seconds. +op_timeout_seconds() { + printf '%s\n' "$1" | awk ' + /^[[:space:]]*timeout:[[:space:]]*[0-9]+m[[:space:]]*$/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH) * 60; exit } + /^[[:space:]]*timeout:[[:space:]]*[0-9]+s[[:space:]]*$/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH); exit } + /^[[:space:]]*content:/ { exit } + ' +} + +# Print the op's shell, or `sh`, the Chainsaw default, when it names none. +op_shell() { + sh_name=$(printf '%s\n' "$1" | awk ' + /^[[:space:]]*shell:[[:space:]]*[^[:space:]]+[[:space:]]*$/ { sub(/^[[:space:]]*shell:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print; exit } + /^[[:space:]]*content:/ { exit } + ') + printf '%s\n' "${sh_name:-sh}" +} + +# Print the op's script, dedented, without the content key. +op_content() { + printf '%s\n' "$1" | awk ' + !incontent && /^[[:space:]]*content:[[:space:]]*\|[[:space:]]*$/ { match($0, /^[[:space:]]*/); keyindent = RLENGTH; incontent = 1; next } + incontent && /[^[:space:]]/ { + match($0, /^[[:space:]]*/) + if (RLENGTH <= keyindent) exit + if (!bodyindent) bodyindent = RLENGTH + } + incontent { print substr($0, bodyindent + 1) } + ' +} + +# Write a kubectl stub into directory $1 that answers the redis-tls step's +# reads as a healthy cluster would. STUB_FAIL, when set, is a pattern matched +# against the arguments: a matching call prints STUB_ERR to stderr and exits +# STUB_RC instead, after printing STUB_OUT to stdout when that is set, from the +# STUB_FAIL_FROM-th matching call on (default the first). STUB_PROJ_KEY=1 puts +# PEM private key material into the projection. STUB_SLEEP_ON is a pattern +# whose matching calls sleep STUB_SLEEP seconds first; the stub execs sleep, +# so a timeout that kills it leaves no child holding the output pipe. +write_redis_stubs() { + cat > "$1/kubectl" <<'STUB' +#!/bin/sh +args="$*" +if [ -n "${STUB_SLEEP_ON:-}" ]; then + case "$args" in + $STUB_SLEEP_ON) exec sleep "${STUB_SLEEP:-60}" ;; + esac +fi +if [ -n "${STUB_FAIL:-}" ]; then + case "$args" in + $STUB_FAIL) + echo x >> "$(dirname "$0")/fail-matches" + if [ "$(wc -l < "$(dirname "$0")/fail-matches")" -ge "${STUB_FAIL_FROM:-1}" ]; then + [ -z "${STUB_OUT:-}" ] || echo "$STUB_OUT" + echo "${STUB_ERR:-error: stubbed failure}" >&2; exit "${STUB_RC:-1}" + fi ;; + esac +fi +cert_b64=$(printf 'CERT' | base64) +proj_b64=$cert_b64 +if [ "${STUB_PROJ_KEY:-}" = 1 ]; then + proj_b64=$(printf 'CERT\n-----BEGIN PRIVATE KEY-----\nKEY\n' | base64 | tr -d '\n') +fi +case "$args" in + *"get secret redis-secure.tenant-ca -o go-template"*) printf 'ca.crt\n' ;; + *"get secret redis-secure.ca-cert -o go-template"*) printf 'ca.crt\n' ;; + *"get secret redis-secure.ca-tls -o go-template"*) printf 'ca.crt\ntls.crt\ntls.key\n' ;; + *"get secret redis-secure.ca-tls -o jsonpath"*) printf '%s' "$cert_b64" ;; + *"get secret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$proj_b64" ;; + *"get tenantsecret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$(printf -- '-----BEGIN CERTIFICATE-----' | base64)" ;; + *"get tenantsecret "*) echo "Error from server (NotFound): tenantsecrets \"x\" not found" >&2; exit 1 ;; + *"exec rfr-redis-secure-0 -c redis -- cat /tls/ca.crt"*) printf 'CERT\n' ;; + *"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli --tls"*) printf 'PONG\n' ;; + *"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli -h"*) printf 'I/O error\n'; exit 1 ;; + *"exec rfr-redis-secure-0 -c redis -- sh -c"*) printf 'role:master\n' ;; + *"exec rfr-redis-secure-1 -c redis -- sh -c"*) printf 'role:slave\nmaster_link_status:up\n' ;; + *"get pod -l app.kubernetes.io/component=sentinel"*) printf 'rfs-redis-secure-0' ;; + *"exec rfs-redis-secure-0 -c sentinel"*) printf '10.0.0.1\n6379\n' ;; + *) echo "kubectl stub: unexpected call: $args" >&2; exit 99 ;; +esac +STUB + cat > "$1/openssl" <<'STUB' +#!/bin/sh +cat >/dev/null +echo "sha256 Fingerprint=AA" +STUB + chmod +x "$1/kubectl" "$1/openssl" +} + +# Run the redis-tls step's script, as its op's shell would, against the stubs +# in $1. Prints the script's combined output and then `rc=`. RS_BUDGET, +# when set, replaces the step's budget so the clamp can be driven in seconds. +run_redis_step() { + rs_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP") + rs_shell=$(op_shell "$rs_op") + op_content "$rs_op" > "$1/step.sh" + if [ -n "${RS_BUDGET:-}" ]; then + sed "s/^step_budget=[0-9]*\$/step_budget=${RS_BUDGET}/" "$1/step.sh" > "$1/step.budget" + if cmp -s "$1/step.sh" "$1/step.budget"; then echo "no step_budget line to override"; echo "rc=97"; return; fi + mv "$1/step.budget" "$1/step.sh" + fi + rs_rc=0 + rs_out=$(cd "$1" && PATH="$1:$PATH" NAMESPACE=tenant-test STUB_FAIL="${STUB_FAIL:-}" STUB_FAIL_FROM="${STUB_FAIL_FROM:-1}" STUB_RC="${STUB_RC:-1}" STUB_ERR="${STUB_ERR:-}" STUB_OUT="${STUB_OUT:-}" STUB_SLEEP_ON="${STUB_SLEEP_ON:-}" STUB_SLEEP="${STUB_SLEEP:-}" STUB_PROJ_KEY="${STUB_PROJ_KEY:-}" "$rs_shell" "$1/step.sh" 2>&1) || rs_rc=$? + printf '%s\nrc=%s\n' "$rs_out" "$rs_rc" +} + +@test "the gateway listener-set op outlasts the waits inside it" { + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + [ -n "$gw_op" ] || { echo "no script op found in step $GATEWAY_STEP of $GATEWAY_SUITE" >&2; exit 1; } + gw_timeout=$(op_timeout_seconds "$gw_op") + case "$gw_timeout" in ''|*[!0-9]*) echo "the $GATEWAY_STEP op states no timeout in m or s (got '${gw_timeout}')" >&2; exit 1 ;; esac + gw_body=$(op_content "$gw_op") + gw_polls=$(printf '%s\n' "$gw_body" | grep -oE '^[[:space:]]*timeout +[0-9]+' | grep -oE '[0-9]+$' || true) + gw_n=$(printf '%s\n' "$gw_polls" | grep -c . || true) + [ "$gw_n" -ge 5 ] || { echo "found ${gw_n} capped polls in $GATEWAY_STEP, expected at least 5; the extractor is reading the wrong lines" >&2; exit 1; } + gw_deletes=$(printf '%s\n' "$gw_body" | grep -E '(^|[[:space:]])kubectl .* delete ' | grep -oE '(^|[[:space:]])timeout -k [0-9]+ [0-9]+ kubectl ' | awk '{ print $3 + $4 }' || true) + gw_sum=0 + for gw_w in $gw_polls $gw_deletes; do gw_sum=$(( gw_sum + gw_w )); done + [ $(( gw_timeout - gw_sum )) -ge "$GATEWAY_UNCAPPED" ] || { + echo "the $GATEWAY_STEP op allows ${gw_timeout}s; its polls ($(printf '%s ' $gw_polls)) and cleanup deletes ($(printf '%s ' $gw_deletes)) may wait ${gw_sum}s" >&2 + echo "it must keep ${GATEWAY_UNCAPPED}s above them for the uncapped applies and reads" >&2 + exit 1 + } +} + +@test "every delete in the gateway listener-set cleanup is bounded as a whole" { + # A tenant delete blocks until its child namespaces drain, and a stalled + # API request blocks with no bound at all, so a delete not wrapped in an + # outer timeout is a wait the op sum above cannot see. + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + gw_deletes=$(op_content "$gw_op" | grep -E '(^|[[:space:]])kubectl .* delete ' || true) + [ -n "$gw_deletes" ] || { echo "no kubectl delete found in $GATEWAY_STEP; the cleanup this test reasons about is gone" >&2; exit 1; } + gw_bad=$(printf '%s\n' "$gw_deletes" | grep -vE '(^|[[:space:]])timeout -k [0-9]+ [0-9]+ kubectl ' || true) + [ -z "$gw_bad" ] || { echo "deletes in $GATEWAY_STEP whose wait has no bound:" >&2; printf '%s\n' "$gw_bad" >&2; exit 1; } +} + +@test "the gateway listener-set cleanup deletes the parent only once the child is gone" { + # A parent deleted while its child is still uninstalling wedges the + # parent's cleanup, so a child delete that ran into its bound must leave + # the parent alone. + gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP") + gw_order=$(op_content "$gw_op" | awk ' + /^[[:space:]]*if timeout -k [0-9]+ [0-9]+ kubectl .* delete tenant rchild .*; then[[:space:]]*$/ { guarded = 1; next } + guarded && /[^[:space:]]/ { print (/delete tenant rparent/ ? "ok" : "unguarded"); exit } + /delete tenant rparent/ { print "unguarded"; exit } + ') + [ "$gw_order" = ok ] || { echo "the rparent delete in $GATEWAY_STEP does not sit directly under a successful rchild delete (got '${gw_order}')" >&2; exit 1; } +} + +@test "a failed tenant delete in the gateway cleanup fails the step with its own error" { + # A tenant left in place is one every later suite inherits, so + # the step must say so and fail rather than pass with the leak. + gw_dir=$(mktemp -d) + op_content "$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP")" | awk '/^cleanup\(\) \{/ { on = 1 } on { print } on && /^\}/ { exit }' > "$gw_dir/cleanup.sh" + grep -q 'delete tenant rchild' "$gw_dir/cleanup.sh" || { echo "no cleanup() with the rchild delete found in $GATEWAY_STEP" >&2; exit 1; } + # The stub fails the delete of the tenant named in STUB_FAIL_TENANT with a + # Forbidden error, which is not the bound firing, and logs every delete. + printf '%s\n' '#!/bin/sh' 'echo "$*" >> "$(dirname "$0")/calls"' 'case "$*" in *"delete tenant $STUB_FAIL_TENANT "*) echo "Error from server (Forbidden): stubbed" >&2; exit 3 ;; esac' > "$gw_dir/kubectl" + chmod +x "$gw_dir/kubectl" + for gw_tenant in rchild rparent; do + : > "$gw_dir/calls" + gw_rc=0 + gw_out=$(PATH="$gw_dir:$PATH" STUB_FAIL_TENANT="$gw_tenant" sh -c '. "$1/cleanup.sh"; ns=tenant-test; cleanup' sh "$gw_dir" 2>&1) || gw_rc=$? + [ "$gw_rc" -ne 0 ] || { printf '%s\n' "$gw_out" >&2; echo "cleanup exited 0 with the $gw_tenant delete failed" >&2; exit 1; } + case "$gw_out" in *"Forbidden"*"$gw_tenant delete exited 3"*"rparent is left in tenant-test"*) ;; *) printf '%s\n' "$gw_out" >&2; echo "with the $gw_tenant delete failed, the output does not carry kubectl's error, its exit status and the tenant left behind" >&2; exit 1 ;; esac + if [ "$gw_tenant" = rchild ] && grep -q 'delete tenant rparent' "$gw_dir/calls"; then echo "the parent was deleted although the child delete failed" >&2; exit 1; fi + done + rm -rf "$gw_dir" +} + +@test "the redis-tls projection op outlasts the step budget its reads are clamped to" { + rt_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP") + [ -n "$rt_op" ] || { echo "no script op found in step $REDIS_STEP of $REDIS_SUITE" >&2; exit 1; } + rt_timeout=$(op_timeout_seconds "$rt_op") + case "$rt_timeout" in ''|*[!0-9]*) echo "the $REDIS_STEP op states no timeout in m or s (got '${rt_timeout}')" >&2; exit 1 ;; esac + rt_body=$(op_content "$rt_op") + rt_budget=$(printf '%s\n' "$rt_body" | sed -n 's/^step_budget=\([0-9][0-9]*\)$/\1/p') + [ -n "$rt_budget" ] || { + echo "$REDIS_STEP sets no step_budget, so nothing bounds the sum of its reads below the ${rt_timeout}s op" >&2 + echo "its kubectl call sites alone come to $(printf '%s\n' "$rt_body" | grep -cE '(^|[^[:alnum:]_])k -n ')" >&2 + exit 1 + } + [ $(( rt_timeout - rt_budget )) -ge "$REDIS_AFTER_BUDGET" ] || { + echo "the $REDIS_STEP op allows ${rt_timeout}s and the step budget is ${rt_budget}s" >&2 + echo "it must keep ${REDIS_AFTER_BUDGET}s above the budget for the last read's kill grace and the work between reads" >&2 + exit 1 + } + # The budget bounds only the reads that go through k(). A bare kubectl + # outside it would add an uncapped read the arithmetic above never sees. + rt_bare=$(printf '%s\n' "$rt_body" | grep -vE '^[[:space:]]*#' | grep -nE '(^|[^[:alnum:]_-])kubectl([[:space:]]|$)' | grep -v '"\$k_left" kubectl' || true) + [ -z "$rt_bare" ] || { echo "kubectl calls in $REDIS_STEP that bypass the clamped k():" >&2; printf '%s\n' "$rt_bare" >&2; exit 1; } +} + +@test "the redis-tls projection script passes against a healthy stub" { + # The positive control for the two failure tests below: without it, a + # stub that broke the script somewhere else would make them pass. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") ;; *) printf '%s\n' "$rt_out" >&2; echo "the step failed against a healthy stub" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed replication exec reports its own error, not the 15s cap" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*exec rfr-redis-secure-1 -c redis -- sh -c*' STUB_RC=1 \ + STUB_ERR='Error from server (Forbidden): pods "rfr-redis-secure-1" is forbidden' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a forbidden exec" >&2; exit 1 ;; esac + case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a forbidden exec was reported as the 15s cap firing" >&2; exit 1 ;; esac + case "$rt_out" in *"rfr-redis-secure-1"*"exit 1"*"Forbidden"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the message does not carry the exit status and what the exec printed" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed sentinel exec reports its own error, not the 15s cap" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*exec rfs-redis-secure-0 -c sentinel*' STUB_RC=1 \ + STUB_ERR='command terminated with exit code 1' STUB_OUT='NOAUTH Authentication required.' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed sentinel exec" >&2; exit 1 ;; esac + case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a refused sentinel command was reported as the 15s cap firing" >&2; exit 1 ;; esac + case "$rt_out" in *"command terminated with exit code 1"*"rfs-redis-secure-0 failed (exit 1): NOAUTH Authentication required."*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the output does not carry what the exec wrote to both streams and its exit status" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a failed Secret read is not reported as a wrong source name" { + # base64 -d exits 0 on empty input, so without pipefail the pipeline takes + # its status and the emptiness check below names the wrong cause. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_FAIL='*get secret redis-secure.ca-tls -o jsonpath*' STUB_RC=1 \ + STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed Secret read" >&2; exit 1 ;; esac + case "$rt_out" in *"source name or key is wrong"*) printf '%s\n' "$rt_out" >&2; echo "an API error was reported as a wrong source name" >&2; exit 1 ;; esac + case "$rt_out" in *"ServiceUnavailable"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the API error is missing from the output" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "the private-key check fails when the projection it reads carries a key" { + # The projection is read once above this check. A check that reads it + # again passes when that second read fails, because an empty read holds + # no key material: here the first read returns a key and the second one + # is refused. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(STUB_PROJ_KEY=1 STUB_FAIL='*get secret redis-secure.tenant-ca -o jsonpath*' STUB_FAIL_FROM=2 STUB_RC=1 \ + STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with key material in the projection" >&2; exit 1 ;; esac + case "$rt_out" in *"projection carries PEM private key material"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the step did not fail on the key material" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a read that finds the step budget spent does not run and says so" { + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_out=$(RS_BUDGET=0 run_redis_step "$rt_dir") + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with its budget spent before the first read" >&2; exit 1 ;; esac + case "$rt_out" in *"step budget is spent, so this read did not run: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the first read did not report the spent budget" >&2; exit 1 ;; esac + rm -rf "$rt_dir" +} + +@test "a read is clamped to what the step budget has left and names itself" { + # A 2s budget against a read that would hang for 20s: the clamp has to + # end the read at the budget, not at the 30s per-read cap. + rt_dir=$(mktemp -d) + write_redis_stubs "$rt_dir" + rt_start=$(date +%s) + rt_out=$(RS_BUDGET=2 STUB_SLEEP_ON='*get secret redis-secure.tenant-ca -o go-template*' STUB_SLEEP=20 run_redis_step "$rt_dir") + rt_took=$(( $(date +%s) - rt_start )) + case "$rt_out" in *" +rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a hung read" >&2; exit 1 ;; esac + case "$rt_out" in *"ran into its "[12]"s bound: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the hung read did not name itself at the budget's bound" >&2; exit 1 ;; esac + [ "$rt_took" -lt 15 ] || { echo "the step took ${rt_took}s against a 2s budget; the read was not clamped" >&2; exit 1; } + rm -rf "$rt_dir" +} diff --git a/hack/e2e-chainsaw/gateway/chainsaw-test.yaml b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml index 5e375731b..254c8629d 100644 --- a/hack/e2e-chainsaw/gateway/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/gateway/chainsaw-test.yaml @@ -977,16 +977,40 @@ spec: steps: - name: parent-child-and-route-drive-listener-set try: + # Above the 660s of polls plus the 675s the trap's deletes may take with + # their kill grace, so a poll or a delete is what fires and the trap still + # runs: a kill at the op ceiling would skip cleanup and leave both tenants + # behind for every later suite. hack/chainsaw-op-budgets.bats holds the op + # to that sum. - script: - timeout: 6m + timeout: 24m content: | set -eu ns="$NAMESPACE" + # Each delete waits, child before parent, because a parent deleted + # while its child is still uninstalling wedges the parent's cleanup. + # Each delete is bounded as a whole by an outer timeout, because the + # trap runs inside the op ceiling and kubectl's --timeout covers only + # the wait after the request, not discovery or a stalled request. A + # tenant delete gets the 5m that .chainsaw.yaml gives a delete op, + # because its child namespaces take minutes to drain. A child still + # uninstalling at its bound leaves the parent in place: a leaked + # tenant rather than a wedged one. A failed tenant delete fails the + # step with kubectl's own error and exit status (124 is the bound), + # so the leak is reported here rather than in whichever suite trips + # on it. cleanup() { - kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true - kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null 2>&1 || true - kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null 2>&1 || true + timeout -k 5 60 kubectl -n tenant-test-rparent-rchild delete httproute inherit-probe --ignore-not-found >/dev/null 2>&1 || true + if timeout -k 5 300 kubectl -n tenant-test-rparent delete tenant rchild --ignore-not-found >/dev/null; then + timeout -k 5 300 kubectl -n "$ns" delete tenant rparent --ignore-not-found >/dev/null || { + echo "the tenant rparent delete exited $?, so tenant rparent is left in $ns" >&2 + exit 1 + } + else + echo "the tenant rchild delete exited $?, so tenant rparent is left in $ns" >&2 + exit 1 + fi } trap cleanup EXIT diff --git a/hack/e2e-chainsaw/redis/chainsaw-test.yaml b/hack/e2e-chainsaw/redis/chainsaw-test.yaml index 0b2ad88fa..4b06d6cc8 100644 --- a/hack/e2e-chainsaw/redis/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/redis/chainsaw-test.yaml @@ -270,14 +270,13 @@ spec: # No retry around this script. The replica link it reads is settled by # none of the asserts above and carries its own bounded wait below. The # whole step measured 3.0s on run 34711755303, so link_wait is wide - # against what it costs today. That wait draws link_wait plus one poll - # interval plus a read pair at the k() cap worst case, since the deadline - # is tested before the sleep and never after, and the op ceiling covers - # that on top of reads ahead of it at the cost that measurement shows, - # not at their caps, whose sum exceeds any ceiling this step could carry. - # Sized generously on purpose: unused margin costs a passing run nothing, - # while a ceiling that expires first kills the shell and takes the wait's - # own failure message with it. + # against what it costs today. The reads' 30s caps sum past the 5m op, + # so they share step_budget instead: no read starts after it, one + # already running ends within its 5s kill grace of it, and the op sits + # above both, so a read or the budget is what fails and names itself + # rather than a ceiling that kills the shell and takes the script's own + # message with it. hack/chainsaw-op-budgets.bats holds the budget under + # the op. # # Every redis-cli below runs under the container's own `timeout`. That # inner cap, not the op timeout, is what keeps a dial that never answers @@ -288,17 +287,41 @@ spec: # failing. Both engine images carry /usr/bin/timeout; expiry reports 143 # on the sentinel's busybox and 124 on the data pods' coreutils, and # neither is zero, which is all these checks read. + # + # bash for pipefail: base64 -d exits 0 on empty input, so without it a + # failed Secret read below takes the decoder's status and is reported as + # the emptiness it left behind. The sandbox's /bin/sh is dash, which + # rejects `set -o pipefail`; `shell:` is a Script field in the Chainsaw + # v0.2.15 the sandbox image installs. - script: timeout: 5m + shell: bash content: | - set -eu + set -euo pipefail # Every kubectl below runs under a wall-clock bound. The inner # `timeout` on a redis-cli cannot fire if the exec stream itself # never establishes, and a kubectl still running holds this script's # stdout open, which is what turns a stuck call into a silent run to - # the job ceiling instead of a failure. - k() { timeout 30 kubectl "$@"; } + # the job ceiling instead of a failure. A call gets 30s at most and + # never more than step_budget has left. + step_budget=240 + step_deadline=$(( $(date +%s) + step_budget )) + k() { + k_left=$(( step_deadline - $(date +%s) )) + if [ "$k_left" -le 0 ]; then + echo "the ${step_budget}s step budget is spent, so this read did not run: $*" >&2 + return 124 + fi + [ "$k_left" -le 30 ] || k_left=30 + k_start=$(date +%s) + k_rc=0 + timeout -k 5 "$k_left" kubectl "$@" || k_rc=$? + if [ "$k_rc" -ne 0 ] && [ $(( $(date +%s) - k_start )) -ge "$k_left" ]; then + echo "this read ran into its ${k_left}s bound: $*" >&2 + fi + return "$k_rc" + } # Exactly one key, and it is ca.crt. The source carries the CA # private key, so any second key is a leak in the making. @@ -332,14 +355,15 @@ spec: fi # The source really does hold the private key, or the leak check - # below would pass while proving nothing. + # below would pass while proving nothing. grep reads to EOF here + # rather than quitting at the match under -q: with pipefail, a + # kubectl still writing would take SIGPIPE and fail the pipeline. if ! k -n "$NAMESPACE" get secret redis-secure.ca-tls \ - -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -qx 'tls.key'; then + -o go-template='{{if .data}}{{range $k, $v := .data}}{{$k}}{{"\n"}}{{end}}{{end}}' | grep -x 'tls.key' >/dev/null; then echo "redis-secure.ca-tls no longer carries tls.key: this check has stopped proving anything" >&2 exit 1 fi - if k -n "$NAMESPACE" get secret redis-secure.tenant-ca \ - -o jsonpath='{.data.ca\.crt}' | base64 -d | grep -qi 'BEGIN .*PRIVATE KEY'; then + if grep -qi 'BEGIN .*PRIVATE KEY' <<<"$proj"; then echo "projection carries PEM private key material" >&2 exit 1 fi @@ -455,9 +479,11 @@ spec: linked=0 seen= for i in 0 1; do - if ! info=$(k -n "$NAMESPACE" exec "rfr-redis-secure-$i" -c redis -- \ - sh -c 'REDISCLI_AUTH="$REDIS_PASSWORD" timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 6379 info replication'); then - echo "rfr-redis-secure-$i did not answer INFO replication within the 15s cap" >&2 + rc=0 + info=$(k -n "$NAMESPACE" exec "rfr-redis-secure-$i" -c redis -- \ + sh -c 'REDISCLI_AUTH="$REDIS_PASSWORD" timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 6379 info replication' 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + echo "INFO replication on rfr-redis-secure-$i failed (exit $rc): $info" >&2 exit 1 fi case "$info" in @@ -487,10 +513,14 @@ spec: sentinel=$(k -n "$NAMESPACE" get pod \ -l app.kubernetes.io/component=sentinel,app.kubernetes.io/name=redis-secure \ -o jsonpath='{.items[0].metadata.name}') - if ! addr=$(k -n "$NAMESPACE" exec "$sentinel" -c sentinel -- \ + rc=0 + addr=$(k -n "$NAMESPACE" exec "$sentinel" -c sentinel -- \ timeout 15 redis-cli --tls --cacert /tls/ca.crt -h localhost -p 26379 \ - sentinel get-master-addr-by-name mymaster); then - echo "the sentinel did not answer get-master-addr-by-name within the 15s cap" >&2 + sentinel get-master-addr-by-name mymaster) || rc=$? + # stderr stays out of $addr, whose first line is parsed below; what + # the exec wrote there is already in the log above this line. + if [ "$rc" -ne 0 ]; then + echo "get-master-addr-by-name on $sentinel failed (exit $rc): $addr" >&2 exit 1 fi addr=$(echo "$addr" | head -n 1 | tr -d '\r')