#!/usr/bin/env bats # Regression coverage for tenant-worker guest serial console capture in # hack/e2e-chainsaw/_lib/run-kubernetes.sh. cozytest.sh ends an @test block at # the first bare closing brace, so command mocks stay at top level. # # The capture exists for the one failure class no other collector reaches: a # worker that stalls in the guest before Talos apid answers. Its defining # property is therefore what it does NOT need — no talosctl, no reader # certificate, no helper Pod, nothing inside the guest. These tests pin that # property, because a capture that grew an in-guest dependency would still look # healthy while being blind to exactly the runs it was written for. load test_helper kubectl_calls=/dev/null kubectl_pod_names= kubectl_list_rc=0 kubectl_list_stderr= kubectl_init_names= kubectl_init_rc=0 kubectl_wedge_rows= kubectl_wedge_rc=0 kubectl_wedge_stderr= mktemp_fail=0 kubectl_logs_output='mock console output' kubectl_logs_rc=0 kubectl_logs_stderr= kubectl_timeline_rows='name=virt-launcher-worker-a-11111 phase=Running podStartTime=2026-08-15T14:35:26Z computeStartedAt=2026-08-15T14:35:47Z consoleStartedAt=2026-08-15T14:35:47Z' kubectl_timeline_rc=0 kubectl_timeline_stderr= talosctl_calls=/dev/null timeout_calls=/dev/null timeout_fail_pod= date_now= date() { # Same shape the node-join suite uses: a fixed clock so the passing path's # remaining-time arithmetic is decided by the test rather than by when it ran. if [ -n "${date_now}" ] && [ "${1:-}" = +%s ]; then printf '%s\n' "${date_now}" return 0 fi command date "$@" } kubectl() { printf '%s\n' "$*" >>"${kubectl_calls}" if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then case "$*" in *computeStartedAt*) # Rows first, status after, so a cut-off read can be modelled as what it # is: a prefix that arrived plus a non-zero status. Returning early on # the status instead would make "failed" and "returned nothing" the same # scenario, and the branch that tells them apart untestable. # # stderr is staged separately for the same reason every other arm here # stages it: kubectl writes warnings beside a healthy read as well as # errors beside a failed one, and which file those land in is decided by # the status rather than by anything in the text. [ -z "${kubectl_timeline_stderr}" ] || printf '%s\n' "${kubectl_timeline_stderr}" >&2 [ -z "${kubectl_timeline_rows}" ] || printf '%s\n' "${kubectl_timeline_rows}" return "${kubectl_timeline_rc}" ;; *initContainerStatuses*) [ -z "${kubectl_wedge_stderr}" ] || printf '%s\n' "${kubectl_wedge_stderr}" >&2 [ "${kubectl_wedge_rc}" -eq 0 ] || return "${kubectl_wedge_rc}" [ -z "${kubectl_wedge_rows}" ] || printf '%s\n' "${kubectl_wedge_rows}" return 0 ;; *initContainers*) [ "${kubectl_init_rc}" -eq 0 ] || return "${kubectl_init_rc}" [ -z "${kubectl_init_names}" ] || printf '%s\n' "${kubectl_init_names}" return 0 ;; esac [ -z "${kubectl_list_stderr}" ] || printf '%s\n' "${kubectl_list_stderr}" >&2 [ "${kubectl_list_rc}" -eq 0 ] || return "${kubectl_list_rc}" [ -z "${kubectl_pod_names}" ] || printf '%s\n' ${kubectl_pod_names} return 0 fi if [ "${3:-}" = logs ]; then [ -z "${kubectl_logs_stderr}" ] || printf '%s\n' "${kubectl_logs_stderr}" >&2 [ "${kubectl_logs_rc}" -eq 0 ] || return "${kubectl_logs_rc}" [ -z "${kubectl_logs_output}" ] || printf '%s for %s\n' "${kubectl_logs_output}" "${4:-}" return 0 fi return 0 } talosctl() { printf '%s\n' "$*" >>"${talosctl_calls}" } timeout() { local command_rc=0 printf '%s\n' "$*" >>"${timeout_calls}" [ "${1:-}" = -k ] || return 97 shift 3 "$@" || command_rc=$? if [ -n "${timeout_fail_pod}" ]; then case " $* " in *" ${timeout_fail_pod} "*) return 124 ;; esac fi return "${command_rc}" } mktemp() { # The scratch file the wedge check allocates is the one failure the function # cannot report through kubectl, so it needs its own knob. `-d` is left # working because the tests themselves use it for their own temp dirs. if [ "${mktemp_fail}" = 1 ] && [ "${1:-}" != -d ]; then return 1 fi command mktemp "$@" } assert_file_contains() { local needle="$1" local file="$2" case "$(cat "${file}")" in *"${needle}"*) return 0 ;; esac printf 'expected %s to contain: %s\n' "${file}" "${needle}" >&2 return 1 } assert_file_lacks_pattern() { local pattern="$1" local file="$2" # A missing file must fail rather than vacuously pass: awk exits 2 on an # unreadable path, which is indistinguishable from "no line matched" once the # status is folded into an if. if [ ! -f "${file}" ]; then printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 return 1 fi if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 return 1 fi } # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command is emitted on the very stderr being # redirected, and every assertion about those files would be reading the # tracer. Call through this wrapper so the sinks hold what production would # put in them. run_capture() { local _rc=0 ( set +x; cozy_capture_tenant_serial_console "${1:-capture staged by a unit test}" "${2:-6}" ) || _rc=$? return "${_rc}" } @test "a wedged console container is named in the headline, not buried" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false PodInitializing virt-launcher-worker-a-11111\nfalse PodInitializing virt-launcher-worker-b-22222')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # If kubevirt/kubevirt#15989 still bites, the suite's own headline reads # "fewer than 2 tenant nodes Ready within 29m" -- byte-identical to the # failure this instrumentation exists to study. A triager would file the # experiment's answer as the known flake. The diagnostic has to name its # own worst case before the noise starts. assert_file_contains 'guest-console-log' "$tmp/out" assert_file_contains '15989' "$tmp/out" assert_file_contains 'virt-launcher-worker-a-11111' "$tmp/out" rm -rf "$tmp" } @test "a read that never answered is not reported as a healthy console" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rc=1 kubectl_wedge_rows="$(printf 'false PodInitializing virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # The guard against a false headline must not become a false all-clear. # An apiserver blip here would otherwise leave the reader with no line at # all, which reads as "the console container is fine" -- nothing was # checked. Same rule the sibling attach check follows. assert_file_lacks_pattern '15989' "$tmp/out" assert_file_contains 'unknown, not fine' "$tmp/out" rm -rf "$tmp" } @test "the wedge row carries the waiting reason, not just the ready bit" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false PodInitializing virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # ready=false on a sidecar means "not running", which a terminated Pod # satisfies as well as a wedged one. The reason keeps the two apart. assert_file_contains 'PodInitializing' "$tmp/out" assert_file_contains 'state.waiting.reason' "$kubectl_calls" rm -rf "$tmp" } @test "the taxonomy does not claim to be exhaustive" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false OOMKilled virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # Two named reasons do not cover the space -- OOMKilled and ContainerCreating # reach this line too, and a superseded Pod killed past its grace period # reads Error. A headline that enumerates two and stops implies the rest # cannot happen, which is the over-claim this whole line was rewritten to # stop making. assert_file_contains 'any other reason, or none, is not covered' "$tmp/out" rm -rf "$tmp" } @test "the wedge taxonomy names CrashLoopBackOff as the bug, not as a superseded Pod" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false CrashLoopBackOff virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # The sidecar restarts on failure, so the hang settles into CrashLoopBackOff # and passes through Error. Filing those under "superseded Pod" would hand a # triager the experiment's own failure labelled as noise -- the precise # inversion this function exists to prevent. assert_file_contains 'CrashLoopBackOff or Error below is the wedge shape' "$tmp/out" assert_file_lacks_pattern 'Completed or Error below means a superseded' "$tmp/out" rm -rf "$tmp" } @test "a superseded Pod is labelled, not called a wedge" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false Completed virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # ContainerState is a union, so a waiting reason alone is blank for the # terminated case and cannot separate it from anything else at ready=false. # The terminated reason is what labels the superseded Pod, and Completed is # the shape that means superseded rather than wedged. assert_file_contains 'Completed' "$tmp/out" assert_file_contains 'state.terminated.reason' "$kubectl_calls" rm -rf "$tmp" } @test "the headline points at the describe rather than concluding" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # The bit cannot separate every shape, so the line must not assert one. A # headline that concludes more than it established is the same mislabel this # function exists to prevent, pointing the other way. assert_file_contains 'if these are current Pods' "$tmp/out" assert_file_contains 'describe that follows settles it' "$tmp/out" rm -rf "$tmp" } @test "a scratch file that could not be allocated is not silence either" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'false CrashLoopBackOff virt-launcher-worker-a-11111')" mktemp_fail=1 ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 mktemp_fail=0 # The function argues twelve lines further down that silence would read as # "the console container is fine" when nothing was checked. That argument # does not stop applying one branch earlier. assert_file_contains 'unknown, not fine' "$tmp/out" assert_file_lacks_pattern '15989' "$tmp/out" rm -rf "$tmp" } @test "a failed read keeps the reason it failed" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rc=1 kubectl_wedge_stderr="Error from server: etcdserver: leader changed" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # Discarding stderr here would report a bare exit code for a failure whose # cause kubectl already named, on the one path that exists to stop a reader # guessing. Same rule the sibling attach check follows. assert_file_contains 'leader changed' "$tmp/out" rm -rf "$tmp" } @test "a healthy console container says nothing in the headline" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_wedge_rows="$(printf 'true virt-launcher-worker-a-11111')" ( set +x; cozy_report_guest_console_wedge ) >"$tmp/out" 2>&1 # A marker that fires on healthy runs is one a reader learns to skip. assert_file_lacks_pattern '15989' "$tmp/out" rm -rf "$tmp" } @test "the wedge check runs before the rest of the node-join diagnostics" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh head=$(grep -n 'node-join failed: fewer than 2 tenant nodes Ready' "$lib" | head -n 1 | cut -d: -f1) wedge=$(grep -n '^ *cozy_report_guest_console_wedge || true$' "$lib" | head -n 1 | cut -d: -f1) first=$(grep -n 'describe nodes' "$lib" | head -n 1 | cut -d: -f1) if [ -z "$wedge" ]; then echo "expected the node-join failure path to check for a wedged console container" >&2 return 1 fi if [ "$wedge" -le "$head" ] || [ "$wedge" -ge "$first" ]; then echo "the wedge check (line $wedge) must sit between the headline ($head) and the first diagnostic ($first)" >&2 return 1 fi } @test "console capture needs nothing from inside the guest" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" talosctl_calls="$tmp/talosctl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-smoke run_capture assert_file_contains 'logs virt-launcher-worker-a-11111 -c guest-console-log' "$kubectl_calls" assert_file_lacks_pattern 'certificate|exec|apply|talosconfig' "$kubectl_calls" [ ! -s "$talosctl_calls" ] assert_file_contains 'mock console output' "$COZY_REPORT_DIR/snapshots/console-smoke/tenant-serial-console/virt-launcher-worker-a-11111.log" rm -rf "$tmp" } @test "each read is bounded and keeps its exit code" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-bounded run_capture assert_file_contains '-k 5 30 kubectl -n tenant-test logs virt-launcher-worker-a-11111 -c guest-console-log' "$timeout_calls" assert_file_contains '[capture exit code: 0]' "$COZY_REPORT_DIR/snapshots/console-bounded/tenant-serial-console/virt-launcher-worker-a-11111.log" rm -rf "$tmp" } @test "the read keeps the boot output instead of the tail" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-head run_capture # A guest that stalls before apid is diagnosed from what it printed while # booting, and a guest that instead repaints the console indefinitely would # push that boot output out of any --tail window. --limit-bytes truncates # from the far end instead, which keeps whatever the kubelet still holds. # It cannot recover what container-log rotation already dropped. assert_file_contains '--limit-bytes=' "$kubectl_calls" assert_file_lacks_pattern '--tail' "$kubectl_calls" rm -rf "$tmp" } @test "a pod whose read times out does not stop the next pod" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111 virt-launcher-worker-b-22222" timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-partial run_capture assert_file_contains '[capture exit code: 124]' "$COZY_REPORT_DIR/snapshots/console-partial/tenant-serial-console/virt-launcher-worker-a-11111.log" assert_file_contains '[capture exit code: 0]' "$COZY_REPORT_DIR/snapshots/console-partial/tenant-serial-console/virt-launcher-worker-b-22222.log" rm -rf "$tmp" } @test "an empty virt-launcher list is recorded rather than passed over" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-empty capture_rc=0 run_capture || capture_rc=$? [ "$capture_rc" -eq 1 ] assert_file_contains 'no virt-launcher Pod found' "$COZY_REPORT_DIR/snapshots/console-empty/tenant-serial-console/setup-error.log" assert_file_lacks_pattern 'guest-console-log' "$timeout_calls" rm -rf "$tmp" } @test "a failed Pod listing is recorded rather than passed over" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_list_rc=1 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-list-error capture_rc=0 run_capture || capture_rc=$? [ "$capture_rc" -eq 1 ] assert_file_contains 'failed to list tenant virt-launcher Pods' "$COZY_REPORT_DIR/snapshots/console-list-error/tenant-serial-console/setup-error.log" assert_file_lacks_pattern 'guest-console-log' "$timeout_calls" rm -rf "$tmp" } @test "a warning on a successful listing is kept apart from a setup failure" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_list_stderr="W0807 partial results warning" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-warn run_capture # kubectl writes to stderr and exits 0 for warnings and for "No resources # found" alike, so the exit status decides the file. A warning filed as a # setup error reads as a broken collector on an otherwise healthy run. assert_file_contains 'partial results warning' "$COZY_REPORT_DIR/snapshots/console-warn/tenant-serial-console/READ-WARNINGS.txt" [ ! -f "$COZY_REPORT_DIR/snapshots/console-warn/tenant-serial-console/setup-error.log" ] rm -rf "$tmp" } @test "a healthy listing leaves no warnings file behind" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-quiet run_capture [ ! -f "$COZY_REPORT_DIR/snapshots/console-quiet/tenant-serial-console/READ-WARNINGS.txt" ] [ ! -f "$COZY_REPORT_DIR/snapshots/console-quiet/tenant-serial-console/setup-error.log" ] rm -rf "$tmp" } @test "a failed listing keeps the stderr that explains it" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_list_rc=1 kubectl_list_stderr="Error from server (Forbidden): pods is forbidden" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-forbidden capture_rc=0 run_capture || capture_rc=$? [ "$capture_rc" -eq 1 ] assert_file_contains 'Forbidden' "$COZY_REPORT_DIR/snapshots/console-forbidden/tenant-serial-console/setup-error.log" rm -rf "$tmp" } @test "the attach check passes when the console container is present" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=guest-console-log" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? [ "$attach_rc" -eq 0 ] rm -rf "$tmp" } @test "a matched Pod with no init containers at all is the finding, not a miss" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? # This is the shape an inert setting actually produces here: guest-console-log # is the only init container these Pods ever get, so "the field did not take # effect" means initContainers is absent, not that some other name appears. # Reading a bare list of container names would render it byte-identical to # "no Pod matched" and let the suite pass over the one thing this checks. [ "$attach_rc" -eq 2 ] rm -rf "$tmp" } @test "a read that never answered is not reported as an inert setting" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_rc=1 attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) 2>"$tmp/err" || attach_rc=$? # Piping the read into grep would collapse this onto the absent-container # branch: no output, grep fails, and an API blip gets published as "the # override did not take effect" -- a cause nothing established. The message # is asserted, not just the code: a failed read and a read that matched # nothing share an exit code but are different facts, and only the status # tells them apart. [ "$attach_rc" -eq 1 ] assert_file_contains 'could not read tenant virt-launcher Pods' "$tmp/err" rm -rf "$tmp" } @test "a read that matched no Pod is not reported as an inert setting either" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names= attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) 2>"$tmp/err" || attach_rc=$? # A selector that matched nothing says nothing about any Pod's containers, # so it belongs with the failed read rather than with the absent container. [ "$attach_rc" -eq 1 ] assert_file_contains 'no tenant virt-launcher Pod matched' "$tmp/err" rm -rf "$tmp" } @test "the absent-container diagnostics are bounded like the check itself" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? [ "$attach_rc" -eq 2 ] # These run on a path that ends in exit 1; an unbounded hang here would take # the step's deadline and the tenant snapshot the exit is meant to reach. [ "$(grep -c 'get virtualmachineinstances' "$timeout_calls")" -eq 1 ] [ "$(grep -c 'get pods' "$timeout_calls")" -eq 2 ] rm -rf "$tmp" } @test "a silent read records whether the container ever started" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_output= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-silent run_capture # An empty console log has two causes that look identical in the artifact: # the guest printed nothing, or the container never started so there was no # stream at all. The second is what kubevirt/kubevirt#15989 produces, and it # is the reading that would otherwise be filed as "the guest was silent". # The Pod's own state is the only thing that separates them. d="$COZY_REPORT_DIR/snapshots/console-silent/tenant-serial-console" assert_file_contains 'describe pod' "$d/POD-STATE.txt" assert_file_contains 'events' "$d/POD-STATE.txt" assert_file_contains 'returned nothing' "$d/virt-launcher-worker-a-11111.log" rm -rf "$tmp" } @test "a container that never started is not called truncated or silent" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_output= kubectl_logs_rc=1 kubectl_logs_stderr="Error from server (BadRequest): container guest-console-log is not valid for pod virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-absent run_capture # This is the shape kubevirt/kubevirt#15989 produces and the headline case # for the whole collector. Folding stderr into the capture would make the # file non-empty and the console would read as having said something; # nothing was truncated, because nothing was ever read. d="$COZY_REPORT_DIR/snapshots/console-absent/tenant-serial-console" assert_file_contains 'no console at all' "$d/virt-launcher-worker-a-11111.log" assert_file_lacks_pattern 'truncated' "$d/virt-launcher-worker-a-11111.log" assert_file_contains 'is not valid for pod' "$d/virt-launcher-worker-a-11111.read-error.log" [ -f "$d/POD-STATE.txt" ] rm -rf "$tmp" } @test "a read killed without a word does not point at a missing file" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_output= timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-killed run_capture # timeout kills the child without a word, so both streams can be empty. # Naming read-error.log then sends the reader after evidence that was # never written. d="$COZY_REPORT_DIR/snapshots/console-killed/tenant-serial-console" [ ! -f "$d/virt-launcher-worker-a-11111.read-error.log" ] assert_file_contains 'failed silently' "$d/virt-launcher-worker-a-11111.log" assert_file_lacks_pattern 'see read-error.log' "$d/virt-launcher-worker-a-11111.log" rm -rf "$tmp" } @test "a warning beside a healthy capture is not filed as a read error" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_stderr="Warning: deprecated flag" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-podwarn run_capture # Same rule the Pod-list read follows: kubectl writes warnings on stderr # with a zero status, so presence of stderr does not make it an error. d="$COZY_REPORT_DIR/snapshots/console-podwarn/tenant-serial-console" assert_file_contains 'deprecated flag' "$d/virt-launcher-worker-a-11111.READ-WARNINGS.txt" [ ! -f "$d/virt-launcher-worker-a-11111.read-error.log" ] [ ! -f "$d/POD-STATE.txt" ] rm -rf "$tmp" } @test "a healthy read leaves no read-error file" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-noerr run_capture [ ! -f "$COZY_REPORT_DIR/snapshots/console-noerr/tenant-serial-console/virt-launcher-worker-a-11111.read-error.log" ] rm -rf "$tmp" } @test "a truncated read is not reported as silence" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-truncated out=$( ( set +x; cozy_capture_tenant_serial_console 'a unit test' 6 ) 2>/dev/null ) printf '%s\n' "$out" >"$tmp/out" # The read broke off mid-stream, so the file holds a prefix. Calling that # "no console output" is the same unestablished claim about silence the # Pod-state capture exists to prevent. f="$COZY_REPORT_DIR/snapshots/console-truncated/tenant-serial-console/virt-launcher-worker-a-11111.log" assert_file_contains 'mock console output' "$f" assert_file_contains 'truncated' "$f" assert_file_lacks_pattern 'no console output' "$f" # And the summary line the passing path reads must not undo that. It is fed # by the same counter that gates the Pod-state read, which a cut-short console # needs as much as an empty one -- so counting them together is right and # naming them together is not. On the green path this line is the only thing # anyone sees, and a partial baseline reported as no baseline sends the reader # past evidence that is sitting in the artifact. assert_file_contains 'of the 1 guest consoles read, 0 came back empty and 1 were cut short' "$tmp/out" rm -rf "$tmp" } @test "a failed read records the Pod state too" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-failed-read run_capture assert_file_contains 'describe pod' "$COZY_REPORT_DIR/snapshots/console-failed-read/tenant-serial-console/POD-STATE.txt" rm -rf "$tmp" } @test "a healthy capture leaves no Pod-state file" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-healthy run_capture [ ! -f "$COZY_REPORT_DIR/snapshots/console-healthy/tenant-serial-console/POD-STATE.txt" ] rm -rf "$tmp" } @test "the Pod-state reads are bounded and taken once for the whole selector" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="p1 p2 p3" kubectl_logs_output= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-once run_capture # One describe and one events read for the selector, not per Pod: three # silent workers is one finding, and paying for it per Pod would put an # unbounded term back into the failure path the cap exists to bound. [ "$(grep -c 'describe pods' "$timeout_calls")" -eq 1 ] [ "$(grep -c 'get events' "$timeout_calls")" -eq 1 ] rm -rf "$tmp" } @test "the attach check accepts the container wherever KubeVirt puts it" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" # It is an init container today. The question the check asks is whether the # container exists at all, so reading both lists costs nothing and removes # the one way a KubeVirt-side move would red a green run over a diagnostic. kubectl_init_names="virt-launcher-worker-a-11111= guest-console-log" accept_rc=0 ( set +x; cozy_assert_guest_console_attached ) || accept_rc=$? [ "$accept_rc" -eq 0 ] kubectl_init_names="virt-launcher-worker-a-11111=" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? [ "$attach_rc" -eq 2 ] assert_file_contains '{.spec.containers[*].name}' "$kubectl_calls" rm -rf "$tmp" } @test "a passing attach check records the positive outcome, not just the negative" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="$(printf 'p1=\np2=guest-console-log')" ( set +x; cozy_assert_guest_console_attached ) >"$tmp/out" 2>&1 # This run doubles as the revalidation of a platform-wide workaround, so the # positive outcome has to leave an artifact too. Returning 0 in silence # leaves "the override worked" to be inferred from the absence of a # complaint -- the exact inference this collector refuses to make elsewhere. assert_file_contains 'attached on 1 of 2 virt-launcher Pods' "$tmp/out" rm -rf "$tmp" } @test "a truncated read names the error file when there is one" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_stderr="Warning: stream reset" timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-trunc-err run_capture # Same discipline as the silent branch: name a file only when it is there. d="$COZY_REPORT_DIR/snapshots/console-trunc-err/tenant-serial-console" assert_file_contains 'truncated' "$d/virt-launcher-worker-a-11111.log" assert_file_contains 'read-error.log' "$d/virt-launcher-worker-a-11111.log" assert_file_contains 'stream reset' "$d/virt-launcher-worker-a-11111.read-error.log" rm -rf "$tmp" } @test "a truncated read with no error file does not name one" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" timeout_fail_pod=virt-launcher-worker-a-11111 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-trunc-noerr run_capture d="$COZY_REPORT_DIR/snapshots/console-trunc-noerr/tenant-serial-console" assert_file_contains 'truncated' "$d/virt-launcher-worker-a-11111.log" assert_file_lacks_pattern 'read-error.log' "$d/virt-launcher-worker-a-11111.log" rm -rf "$tmp" } @test "the typical-cost figure in both chainsaw comments counts the Pod-state reads" { # A comment carrying a number that the code contradicts is worse than no # number: it outlives the session and is trusted by the next reader. for f in hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml \ hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml; do grep -q '4m05s' "$f" || { echo "expected $f to state the minReplicas-2 cost including the wedge and Pod-state reads" >&2 return 1 } grep -q '6m25s' "$f" || { echo "expected $f to state the worst-case cost including the wedge read" >&2 return 1 } done } @test "the attach read gives every matched Pod a non-empty line" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=guest-console-log" ( set +x; cozy_assert_guest_console_attached ) # Asserted on the query rather than the answer, because no mock can stand in # for kubectl's jsonpath engine and this is the property the answer depends # on. Without the Pod name, a Pod whose initContainers list is absent emits a # bare newline that command substitution strips, so "Pods matched, none # carries the container" arrives byte-identical to "no Pod matched" -- and # the check silently stops being able to report the one thing it is for. assert_file_contains '{.metadata.name}{"="}{.spec.initContainers[*].name}' "$kubectl_calls" rm -rf "$tmp" } @test "the failure-path diagnostics carry a client deadline too" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? [ "$attach_rc" -eq 2 ] # timeout bounds the process; --request-timeout bounds the call the process # makes. The wrapper alone leaves kubectl free to sit in its own retry loop # for the whole 30s and return nothing. [ "$(grep -c -- '--request-timeout=30s' "$kubectl_calls")" -eq 3 ] rm -rf "$tmp" } @test "one Pod carrying the container is enough to pass" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="$(printf 'virt-launcher-worker-a-11111=\nvirt-launcher-worker-b-22222=guest-console-log')" attach_rc=0 ( set +x; cozy_assert_guest_console_attached ) || attach_rc=$? # Deliberately "any", not "every": the namespace is not scoped to this # release, so an every-Pod rule would fail a run over a Pod that is not # this test's. It can miss; it cannot false-fail. [ "$attach_rc" -eq 0 ] rm -rf "$tmp" } @test "the attach read is bounded like every other read here" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_init_names="virt-launcher-worker-a-11111=guest-console-log" ( set +x; cozy_assert_guest_console_attached ) assert_file_contains '-k 5 30 kubectl -n tenant-test get pods' "$timeout_calls" rm -rf "$tmp" } @test "only an answered read that lacks the container fails the suite" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # The two non-zero outcomes must not be treated alike: exit 1 says the read # did not answer and is not evidence, exit 2 says it did and the container # is absent. grep -q 'if \[ "${attach_rc}" -eq 2 \]; then' "$lib" } @test "the console capture stops at a cap and records what it dropped" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="p1 p2 p3 p4 p5 p6 p7 p8" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-capped run_capture d="$COZY_REPORT_DIR/snapshots/console-capped/tenant-serial-console" [ -f "$d/p6.log" ] [ ! -f "$d/p7.log" ] assert_file_contains 'stopped after 6 Pods; 8 matched in total' "$d/COLLECTION-TRUNCATED.txt" rm -rf "$tmp" } @test "an uncapped pool leaves no truncation marker" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="p1 p2" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-uncapped run_capture [ ! -f "$COZY_REPORT_DIR/snapshots/console-uncapped/tenant-serial-console/COLLECTION-TRUNCATED.txt" ] rm -rf "$tmp" } @test "the attach check sits on the passing path, after the functional block" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # Two placements are wrong and this pins against both. In the failure block # it would answer only in the run where the console can no longer be # recovered. Ahead of the functional assertions it would let a diagnostic # preempt the checks the suite exists for. gate=$(grep -n '^ *cozy_assert_guest_console_attached || attach_rc=$?$' "$lib" | head -n 1 | cut -d: -f1) capture=$(grep -n "^ *cozy_capture_tenant_serial_console 'node-join failed" "$lib" | head -n 1 | cut -d: -f1) if [ -z "$gate" ]; then echo "expected the suite to verify guest-console-log attached on the passing path" >&2 return 1 fi if [ -z "$capture" ]; then echo "expected to find the failure-path console capture in $lib" >&2 return 1 fi if [ "$gate" -le "$capture" ]; then echo "the attach check (line $gate) must sit after node-join succeeds, not in the failure block (line $capture)" >&2 return 1 fi # Anchored on the LAST functional assertion in the function, not a # mid-file one, or everything between it and the gate is unprotected. # # On the call rather than on the text of its arguments. The arguments name # local variables of the library, which a refactor there is free to rename # without touching what this test asserts; an anchor that spells them out # stops matching on that rename, and the failure lands on whoever rebases # second rather than on either author. The call is what the ordering is # about, so it is what the anchor reads. last=$(grep -n '^ *cozy_guard_all_helmreleases ' "$lib" | tail -n 1 | cut -d: -f1) if [ -z "$last" ]; then echo "expected to find the HelmRelease remediation guard in $lib" >&2 return 1 fi if [ "$gate" -le "$last" ]; then echo "the attach check (line $gate) must sit after every functional assertion (last at line $last)" >&2 return 1 fi } @test "the node-join failure path captures the console before the talosctl capture" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # Ordering is the whole point: the talosctl capture needs an apid that the # failure class this collects never reached, so running it first spends the # failure path's budget on the collector that cannot answer. console=$(grep -n "^ *cozy_capture_tenant_serial_console 'node-join failed" "$lib" | head -n 1 | cut -d: -f1) talos=$(grep -n '^ *cozy_capture_tenant_talos "${test_name}" || true$' "$lib" | head -n 1 | cut -d: -f1) if [ -z "$console" ]; then echo "expected the node-join failure path to capture the guest serial console" >&2 return 1 fi if [ -z "$talos" ]; then echo "expected to find the talosctl capture call in $lib" >&2 return 1 fi if [ "$console" -ge "$talos" ]; then echo "serial console capture (line $console) must run before the talosctl capture (line $talos)" >&2 return 1 fi } @test "the passing path captures the console and leaves teardown to Chainsaw finally" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # An instrument that only runs on the failure path answers in one direction. # Every console this tree has ever captured came from a run that failed, so # the boot stages a healthy worker goes through, and how long each takes, have # no measured value to compare a failing run against -- and "slower than # usual" is the claim the failure keeps resting on. The passing run is where # that value comes from, and it exists only until the tenant is deleted. gate=$(grep -n '^ *cozy_assert_guest_console_attached || attach_rc=$?$' "$lib" | head -n 1 | cut -d: -f1) green=$(grep -n "^ *if ! cozy_capture_tenant_serial_console 'the suite passed" "$lib" | head -n 1 | cut -d: -f1) function_end=$(awk -v start="$green" 'NR > start && /^}$/ { print NR; exit }' "$lib") for v in gate green function_end; do eval "n=\$$v" if [ -z "$n" ]; then echo "expected to locate $v in $lib" >&2 return 1 fi done # After the attach check, because that check is what says there is a container # to read. The function must not start an asynchronous tenant teardown after # the capture: each suite's finally block owns the single authoritative delete. if [ "$green" -le "$gate" ]; then echo "the passing-path capture (line $green) must sit after the attach check ($gate)" >&2 return 1 fi if sed -n "${green},${function_end}p" "$lib" | grep -Eq 'delete kubernetes(nodeses|es)[.]apps[.]cozystack[.]io'; then echo "run_kubernetes_test still starts tenant teardown on its success path" >&2 return 1 fi for suite in latest previous; do grep -Fq "cozy_cleanup test-${suite}-version" "hack/e2e-chainsaw/kubernetes-${suite}/chainsaw-test.yaml" || { echo "kubernetes-${suite} finally does not own authoritative tenant teardown" >&2 return 1 } done } @test "the passing path asks for the pool minimum, not the pool" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # The failure path runs inside a phase whose budget is derived against the # whole operation, so it can afford the pool at its maximum. The passing path # has neither a phase nor a budget: it runs inside the same 67m operation the # suite has already spent most of, and every read it issues is bounded # individually but not as a group. A walk over the maximum pool could # therefore turn a suite that proved everything it exists to prove into a red # run for collecting a debugging aid slowly. Two workers booting is the same # baseline as ten, so that is what it asks for. green=$(grep -n "cozy_capture_tenant_serial_console 'the suite passed[^']*' 2;" "$lib" | head -n 1 | cut -d: -f1) red=$(grep -n "cozy_capture_tenant_serial_console 'node-join failed[^']*' 6 || true" "$lib" | head -n 1 | cut -d: -f1) if [ -z "$green" ]; then echo "expected the passing path to cap its console walk at the pool minimum" >&2 return 1 fi if [ -z "$red" ]; then echo "expected the failure path to keep covering the pool at its maximum" >&2 return 1 fi } @test "the walk covers exactly the number of Pods the caller asked for" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-a virt-launcher-b virt-launcher-c" kubectl_logs_output= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-cap-arg out=$( ( set +x; cozy_capture_tenant_serial_console 'a unit test' 2 ) 2>/dev/null ) printf '%s\n' "$out" >"$tmp/out" # The cap is the caller's, not a literal in the walk, or the passing path # cannot be cheaper than the failure path and the argument above is decorative. dir="$COZY_REPORT_DIR/snapshots/console-cap-arg/tenant-serial-console" [ -f "$dir/virt-launcher-a.log" ] [ -f "$dir/virt-launcher-b.log" ] [ ! -f "$dir/virt-launcher-c.log" ] assert_file_contains 'capture stopped after 2 Pods; 3 matched in total' \ "$dir/COLLECTION-TRUNCATED.txt" # And the count on stdout is over the Pods actually read. The cap probe runs # on the Pod past the cap, so the counter driving it is one higher than the # walk; reported as the denominator it would name a number that is neither # what was read nor what matched, and disagree with the truncation marker # written beside it -- on the path where nobody opens that marker. assert_file_contains 'of the 2 guest consoles read, 2 came back empty and 0 were cut short' "$tmp/out" rm -rf "$tmp" } @test "a passing-path capture that failed is said out loud instead of failing the suite" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # Two failure modes, and this pins against both. Left bare, a non-zero return # would fail a suite that proved everything it exists to prove, on a # debugging aid. Silenced with `|| true`, a capture that collected nothing # would leave a green run whose only trace of the gap is an artifact nobody # opens on a green run. green=$(grep -n "^ *if ! cozy_capture_tenant_serial_console 'the suite passed" "$lib" | head -n 1 | cut -d: -f1) if [ -z "$green" ]; then echo "expected the passing path to capture the console through a tested call" >&2 return 1 fi # Bounded by the branch rather than by a line count: the warning is what the # branch is for, and a fixed offset would break on a comment being added to it # while a whole-file search would accept a warning belonging to something # else. warn=$(awk -v g="$green" ' NR <= g { next } /^ fi$/ { exit } /WARNING: the guest console capture/ { print NR; exit } ' "$lib") if [ -z "$warn" ]; then echo "expected the failed passing-path capture to warn in the job log" >&2 return 1 fi } @test "the capture says which run produced it" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-context run_capture 'the suite passed: captured after the last assertion' # Both paths write the same directory, and the two readings mean opposite # things: one is the failure being studied, the other is the baseline it is # compared against. Which one a tarball holds is otherwise recoverable only # from the run's verdict somewhere else entirely. assert_file_contains 'the suite passed: captured after the last assertion' \ "$COZY_REPORT_DIR/snapshots/console-context/tenant-serial-console/CAPTURE-CONTEXT.txt" rm -rf "$tmp" } @test "a walk that came back with empty consoles says so on stdout" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_logs_output= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-silent-count out=$( ( set +x; cozy_capture_tenant_serial_console 'a unit test' 6 ) 2>/dev/null ) printf '%s\n' "$out" >"$tmp/out" # The per-pod notes and POD-STATE.txt already record this in the report, and # on the failure path that is enough because the report gets opened. The # passing path is the one that does not: a baseline that came back empty # there would otherwise be visible only to whoever downloads a green run's # artifact. The count is not a failure and the capture still returns zero, so # stdout is the only place the two callers share. assert_file_contains 'of the 1 guest consoles read, 1 came back empty and 0 were cut short' "$tmp/out" rm -rf "$tmp" } @test "a healthy walk says nothing about silent consoles" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-silent-none out=$( ( set +x; cozy_capture_tenant_serial_console 'a unit test' 6 ) 2>/dev/null ) printf '%s\n' "$out" >"$tmp/out" # The other half of the pair. A line printed on every run is a line nobody # reads, and this one is meant to be read on the green path. assert_file_lacks_pattern 'guest consoles read, ' "$tmp/out" rm -rf "$tmp" } @test "the context is written even when there is nothing to capture" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-context-empty rc=0 run_capture 'the suite passed: captured after the last assertion' || rc=$? # The empty case is the one where the context matters most: a directory # holding only a failure note is where a reader most needs to know which run # left it, and the early return is the path most likely to skip writing it. [ "$rc" -ne 0 ] assert_file_contains 'the suite passed' \ "$COZY_REPORT_DIR/snapshots/console-context-empty/tenant-serial-console/CAPTURE-CONTEXT.txt" rm -rf "$tmp" } @test "each console says when it was read, so its last kernel stamp has a wall clock to sit against" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-read-instant run_capture # A console log is stamped in guest seconds since the guest's own kernel # started, and it ends wherever the guest had got to when the read ran. Those # two facts alone cannot say whether a short log means a guest that started # late or a guest that went quiet, because the instant of the read is not in # the file. Without this line the only trace of it is the artifact's mtime, # which is metadata rather than content: nothing in the report says it, and a # reader has to know to go looking outside the file for it. f="$COZY_REPORT_DIR/snapshots/console-read-instant/tenant-serial-console/virt-launcher-worker-a-11111.log" assert_file_contains 'console read started at ' "$f" assert_file_contains ' epoch seconds' "$f" rm -rf "$tmp" } @test "the capture records when each guest started, beside the console it captured" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline run_capture # The other end of the subtraction. With the read instant above and the # compute container's start here, how long the guest had been alive when the # console was read is one subtraction away, and comparing that against the # last stamp in the log says which of the two happened: a guest that had not # lived long enough to reach a milestone, or one that lived and stopped # printing. Without it both look like the same short file. f="$COZY_REPORT_DIR/snapshots/console-timeline/tenant-serial-console/CAPTURE-TIMELINE.txt" assert_file_contains 'computeStartedAt=2026-08-15T14:35:47Z' "$f" # The subtraction is spelled out rather than left to be reconstructed. This # file is read by whoever opens a failed run's tarball months later, and the # relation between a guest-relative kernel stamp and two absolute instants is # exactly the step a reader gets wrong. assert_file_contains 'epoch seconds' "$f" rm -rf "$tmp" } @test "a start-time read that failed is named rather than left out" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" # Non-zero AND nothing written: the read that never got a row across, which is # a different fact from the one cut off part way through and takes a different # note. Both have to be staged explicitly or the mock decides which arm runs. kubectl_timeline_rc=9 kubectl_timeline_rows= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-failed run_capture # An absent file reads as a capture that had no reason to write one, which is # the silence every collector here is built to refuse. The read failing is a # statement about this machine's view of the cluster, not about when the # guests started, and the file has to say which of those it is holding. f="$COZY_REPORT_DIR/snapshots/console-timeline-failed/tenant-serial-console/CAPTURE-TIMELINE.txt" assert_file_contains 'exit 9' "$f" assert_file_lacks_pattern 'computeStartedAt=2026' "$f" rm -rf "$tmp" } @test "a start-time read cut off part way is not reported as having observed nothing" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_timeline_rc=124 COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-cut run_capture # A read killed at its bound can leave rows behind and still report failure. # Saying "when these guests started was not observed" under start times that # are plainly sitting there makes the file contradict itself, and a reader who # believes the note discards the rows -- which for a capture that exists to be # subtracted is the whole value of the file. f="$COZY_REPORT_DIR/snapshots/console-timeline-cut/tenant-serial-console/CAPTURE-TIMELINE.txt" assert_file_contains 'computeStartedAt=2026-08-15T14:35:47Z' "$f" assert_file_contains 'cut off (exit 124)' "$f" assert_file_lacks_pattern 'was not observed, which is not the same' "$f" rm -rf "$tmp" } @test "a start-time read that answered with nothing is not reported as a start time" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_timeline_rows= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-empty run_capture # The third outcome, and the one that looks most like success: the read # answered, and answered with nothing. A zero-length file here would be read # as a capture that ran and found the guests had no start times, which is not # a state a Pod can be in -- the walk above just read consoles from these # same Pods. f="$COZY_REPORT_DIR/snapshots/console-timeline-empty/tenant-serial-console/CAPTURE-TIMELINE.txt" assert_file_contains 'returned no rows' "$f" rm -rf "$tmp" } @test "the start-time read is bounded like every other read in this capture" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-bound run_capture # This read runs on the same failure path as the rest, where the apiserver is # the component least likely to answer. An unbounded one does not lose only # itself: it holds the op until the op is killed, and the tenant snapshot # queued behind it is lost rather than truncated. assert_file_contains 'computeStartedAt' "$timeout_calls" assert_file_contains 'request-timeout' "$kubectl_calls" rm -rf "$tmp" } @test "a warning beside a healthy start-time read is not filed as an error" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_timeline_stderr='Warning: v1 Pod is deprecated in this build' COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-warn run_capture # kubectl writes deprecation and partial-result warnings on stderr with a zero # exit, so what a message means is decided by the status and not by the fact # that something was written. Filed as an error, a healthy read sends the next # reader after a failure that did not happen; folded into the rows, it sits # inside the data the success probe reads. dir="$COZY_REPORT_DIR/snapshots/console-timeline-warn/tenant-serial-console" assert_file_contains 'v1 Pod is deprecated' "$dir/CAPTURE-TIMELINE-warnings.txt" [ ! -f "$dir/CAPTURE-TIMELINE-error.log" ] assert_file_lacks_pattern 'deprecated' "$dir/CAPTURE-TIMELINE.txt" rm -rf "$tmp" } @test "the stderr of a start-time read that failed is kept as the diagnosis" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_timeline_rc=1 kubectl_timeline_rows= kubectl_timeline_stderr='error: the server could not find the requested resource' COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-err run_capture # The other direction: on a failed read that message IS the diagnosis, and # dropping it leaves a note saying the read failed with nothing saying why. # The note has to name the file, or the evidence sits in a directory the # reader has no reason to open. dir="$COZY_REPORT_DIR/snapshots/console-timeline-err/tenant-serial-console" assert_file_contains 'could not find the requested resource' "$dir/CAPTURE-TIMELINE-error.log" [ ! -f "$dir/CAPTURE-TIMELINE-warnings.txt" ] assert_file_contains 'CAPTURE-TIMELINE-error.log' "$dir/CAPTURE-TIMELINE.txt" rm -rf "$tmp" } @test "a start-time read that failed without a word does not point at a missing file" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" kubectl_timeline_rc=124 kubectl_timeline_rows= kubectl_timeline_stderr= COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-silent run_capture # timeout kills the child without a word, so this read can fail having written # to neither stream. Naming the error log unconditionally would then send the # reader after a file that was never created, which is the same trap the # per-Pod notes above are written to avoid. dir="$COZY_REPORT_DIR/snapshots/console-timeline-silent/tenant-serial-console" [ ! -f "$dir/CAPTURE-TIMELINE-error.log" ] assert_file_contains 'gave no reason on either stream' "$dir/CAPTURE-TIMELINE.txt" assert_file_lacks_pattern 'reason it gave is in' "$dir/CAPTURE-TIMELINE.txt" rm -rf "$tmp" } @test "the console container's start is asked for in both status lists" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_pod_names="virt-launcher-worker-a-11111" COZY_REPORT_DIR="$tmp/report" COZY_SNAPSHOT_NAME=console-timeline-lists run_capture # KubeVirt runs guest-console-log as an init container, so its runtime state # lives under initContainerStatuses. Asking only containerStatuses returns # empty for a container that started perfectly well, and an empty start time # here is supposed to mean the container never ran -- the one reading this # field exists to support. The attach check reads both spec lists for the # same reason; a missing key costs an empty string, so naming both is free. assert_file_contains 'initContainerStatuses[?(@.name=="guest-console-log")]' "$kubectl_calls" assert_file_contains 'containerStatuses[?(@.name=="compute")]' "$kubectl_calls" rm -rf "$tmp" } @test "a run near its operation ceiling declines the baseline instead of being killed collecting it" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) # The capture costs real wall clock and runs after everything the suite # proves. Inside an operation with a fixed ceiling that makes it a way for a # green run to end red, which is the one thing a passing-path collector may # not do. Driven rather than grepped: the direction of the comparison IS the # safety property, and an inverted one runs the capture on exactly the runs # that cannot afford it while every structural check still passes. date_now=1000000000 _COZY_RUN_STARTED_AT=$(( date_now - (COZY_OP_CEILING - COZY_GREEN_CAPTURE_RESERVE) - 1 )) rc=0 ( set +x; cozy_green_capture_has_room 'the baseline' ) >"$tmp/out" 2>&1 || rc=$? [ "$rc" -eq 1 ] # And the decline is spoken. A collector that stops running silently leaves a # green run that looks exactly like one where the baseline was collected and # came back empty, which is the reading every note in this file exists to # prevent. assert_file_contains 'not collected' "$tmp/out" assert_file_contains 'the baseline' "$tmp/out" rm -rf "$tmp" } @test "a run with the operation's room to spare collects the baseline" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) # The other direction, and the one an inverted comparison breaks silently: a # gate that declines a run with forty minutes left costs the baseline on every # healthy run, and the only symptom is a diagnostic that stopped appearing. date_now=1000000000 _COZY_RUN_STARTED_AT=$(( date_now - 60 )) rc=0 ( set +x; cozy_green_capture_has_room 'the baseline' ) >"$tmp/out" 2>&1 || rc=$? [ "$rc" -eq 0 ] assert_file_lacks_pattern 'not collected' "$tmp/out" rm -rf "$tmp" } @test "a run whose start was never stamped declines rather than dying on the arithmetic" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) # The stamp is set by run_kubernetes_test, which no unit test invokes, so # nothing else here would notice it being deleted. Under the `set -eu` the # chainsaw script runs with, an unbound name inside the arithmetic aborts the # whole function -- ending a passing run red, which is the outcome this gate # exists to prevent. Declared zero at file scope, it fails closed instead. date_now=1000000000 rc=0 ( set -u; set +x; cozy_green_capture_has_room 'the baseline' ) >"$tmp/out" 2>&1 || rc=$? [ "$rc" -eq 1 ] assert_file_contains 'not collected' "$tmp/out" rm -rf "$tmp" } @test "the passing path decides through the gate rather than around it" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # The behavioural tests above cover the predicate. Its wiring is what they # cannot reach: run_kubernetes_test is not callable from a unit test, so the # stamp it takes and the gate it calls are only assertable as source. Both # halves fail the same silent way -- a correct predicate nothing calls # protects nothing, and a predicate called without the stamp reads the whole # epoch as elapsed and declines the baseline on every run forever. stamp=$(grep -n '^ _COZY_RUN_STARTED_AT=\$(date +%s)$' "$lib" | head -n 1 | cut -d: -f1) gate=$(grep -n 'cozy_green_capture_has_room ' "$lib" | tail -n 1 | cut -d: -f1) green=$(grep -n "cozy_capture_tenant_serial_console 'the suite passed" "$lib" | head -n 1 | cut -d: -f1) for v in stamp gate green; do eval "n=\$$v" if [ -z "$n" ]; then echo "expected to locate $v in $lib" >&2 return 1 fi done # Inside the function whose elapsed time it measures, not at file scope: taken # once when the library is sourced, it would time the sourcing rather than the # run, and every suite after the first would read as already over its ceiling. fn=$(grep -n '^run_kubernetes_test() {$' "$lib" | head -n 1 | cut -d: -f1) if [ -z "$fn" ] || [ "$stamp" -le "$fn" ]; then echo "the run's start stamp (line $stamp) must be taken inside run_kubernetes_test (line $fn)" >&2 return 1 fi if [ "$stamp" -ge "$gate" ] || [ "$gate" -ge "$green" ]; then echo "expected the stamp ($stamp) before the gate ($gate) before the capture ($green)" >&2 return 1 fi } @test "the operation ceiling the reserve is measured against is the one the suites give" { lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh # A reserve is only meaningful against the right ceiling. This number is a # restatement of one that lives in the suite files, so it is read from both # sides rather than trusted: a suite that raises or lowers its `timeout:` # otherwise leaves the reserve protecting a ceiling that moved, and the # decline either stops firing when it should or starts firing when it should # not. ceiling=$(grep -oE '^COZY_OP_CEILING=[0-9]+' "$lib" | head -n 1 | sed -E 's/.*=//') reserve=$(grep -oE '^COZY_GREEN_CAPTURE_RESERVE=[0-9]+' "$lib" | head -n 1 | sed -E 's/.*=//') for v in ceiling reserve; do eval "n=\$$v" if [ -z "$n" ]; then echo "expected to read $v from $lib; without it this guard reports success for having lost its input" >&2 return 1 fi done for f in hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml \ hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml; do # The first `timeout:` in each file is the bringup op's, which is the one # this whole helper runs inside; the teardown step below it has its own. minutes=$(grep -oE '^ *timeout: [0-9]+m$' "$f" | head -n 1 | grep -oE '[0-9]+') if [ -z "$minutes" ]; then echo "expected to read the operation timeout from $f" >&2 return 1 fi if [ "$((minutes * 60))" -ne "$ceiling" ]; then echo "$f gives the operation ${minutes}m but the reserve is measured against ${ceiling}s" >&2 return 1 fi done # And the reserve has to leave room for the capture it is reserving for, or it # protects nothing: the walk it admits would still run past the ceiling. if [ "$reserve" -lt 210 ]; then echo "the reserve ${reserve}s is under the 210s the capture can cost at this caller's cap" >&2 return 1 fi } @test "the e2e tenant node group asks KubeVirt for the guest console log" { # Anchored to a bare two-space-indented key so a comment or a prose mention # of the field cannot stand in for the setting itself. Since the Phase 2 split # the worker pool is a separate KubernetesNodes CR whose fields sit two spaces # under spec, not six spaces under a nodeGroups. block. It does not prove # the key sits in that CR specifically -- the render is what would catch that, # and the chart's own suites cover it. grep -q '^ logSerialConsole: true$' hack/e2e-chainsaw/_lib/run-kubernetes.sh }