Files
cozystack/hack/run-kubernetes-cpu-throttle_test.bats
T
Myasnikov Daniil cc942fed68 test(tests): run unit suites with Bats and retain POSIX coverage
The shell translator rewrites test bodies and loses Bats reporting.
The unit lane needs complete TAP results and JUnit reports without
changing how live-cluster suites run.

Sourced production helpers still need a dash pass, and unit test bodies
need nounset without exporting shell options into subprocesses.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
2026-10-07 11:04:00 +03:00

1638 lines
72 KiB
Bash

#!/usr/bin/env bats
# Regression coverage for the tenant-worker CPU throttling capture in
# hack/e2e-chainsaw/_lib/run-kubernetes.sh. cozytest.sh ends an @test block at
# the first bare closing brace, so command mocks stay at top level.
#
# What this capture answers, and why nothing else in the tree answers it: when
# a worker misses the node-Ready deadline, the sandbox node it runs on is
# routinely half idle while the guest makes no progress. Node-level utilisation
# and the node's dmesg both look healthy in that state, so "the host was
# starved" and "the VM was held at its own CFS ceiling" produce the same
# evidence from outside. Only the container's own throttling counters separate
# them, and a capture that reports an unread kubelet as a quiet one would hand
# the next reader the same ambiguity while looking like it had resolved it.
# These tests pin the separation, not the reading.
load test_helper
kubectl_calls=/dev/null
kubectl_node_names=
kubectl_list_rc=0
kubectl_list_stderr=
kubectl_raw_rc=0
kubectl_raw_stderr=
# Two of the fixtures in this file claim to be wire shapes, and the rest do not.
# This one and the uncapped test's single line are the two cAdvisor can actually
# produce -- every family for a capped container, the period alone for one
# with no limit, the two sides of the same Quota != 0 gate. Usage sits outside
# that gate and is published for any container at all, so it belongs in both
# shapes; the uncapped fixture stages only the line its own assertion needs, as
# the rest of this file does. Every other fixture
# below stages only the lines its own assertion needs, so a missing sibling
# series there means "not relevant here" rather than "absent on the wire".
kubectl_raw_output='container_cpu_usage_seconds_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 118.4
container_cpu_cfs_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 51200
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211
container_cpu_cfs_throttled_seconds_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 12.5
container_spec_cpu_quota{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000
container_spec_cpu_period{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000'
timeout_calls=/dev/null
timeout_fail_node=
kubectl() {
printf '%s\n' "$*" >>"${kubectl_calls}"
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then
[ -z "${kubectl_list_stderr}" ] || printf '%s\n' "${kubectl_list_stderr}" >&2
[ "${kubectl_list_rc}" -eq 0 ] || return "${kubectl_list_rc}"
[ -z "${kubectl_node_names}" ] || printf '%s\n' ${kubectl_node_names}
return 0
fi
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then
[ -z "${kubectl_raw_stderr}" ] || printf '%s\n' "${kubectl_raw_stderr}" >&2
# Output is emitted before the status is honoured, because a read that is
# cut off part way through has already written what it managed to read. A
# stub that returns first can only ever produce empty-plus-failure, and
# the branch that labels a partial capture would be unreachable in tests
# while being the likeliest one in production.
[ -z "${kubectl_raw_output}" ] || printf '%s\n' "${kubectl_raw_output}"
return "${kubectl_raw_rc}"
fi
return 0
}
timeout() {
local command_rc=0
printf '%s\n' "$*" >>"${timeout_calls}"
# Return 97 rather than running the command when the wrapper is not the
# bounded form: a read that lost its ceiling must not pass as a read.
[ "${1:-}" = -k ] || return 97
shift 3
"$@" || command_rc=$?
if [ -n "${timeout_fail_node}" ]; then
case " $* " in
*"${timeout_fail_node}"*) return 124 ;;
esac
fi
return "${command_rc}"
}
assert_file_contains() {
local needle="$1"
local file="$2"
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${needle}" >&2
return 1
fi
case "$(cat "${file}")" in
*"${needle}"*) return 0 ;;
esac
printf 'expected %s to contain: %s\n' "${file}" "${needle}" >&2
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label". awk exits 2 on an unreadable path,
# which folded into an if reads the same way, so the file is checked first.
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status, not on truthiness. awk exits 1 for "no line
# matched" and 2 for "I could not evaluate this" -- an unparseable regex, most
# of all. Folded into `if ... then fail; fi`, both land in the passing branch,
# so the assertion "this pattern is absent" is satisfied by the matcher giving
# up. That is the wrong direction for a negative assertion: a positive one
# fails loudly when its matcher breaks and gets fixed, this one goes green and
# stays green. Every claim in this file about something NOT appearing rests on
# the three-way split below.
# `|| _rc=$?` rather than a bare call: both runners set -e, and awk exiting 1
# for "no match" -- the ordinary success case here -- would otherwise abort the
# test before the branch below ever runs. The `if` form this replaces was
# exempt only because conditions are exempt.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# COZY_REPORT_DIR is set inline in each test below rather than through a helper,
# which is the opposite of what the node-join suite does. The difference is
# deliberate and shrinks to one line: there the assignment carries a ten-line
# explanation that would otherwise stand in twenty copies, so the helper exists
# to hold the explanation, not the assignment. Here it is a bare one-liner with
# nothing to deduplicate, and a helper would buy indirection at no saving. The
# reason it must be set at all is the same in both files and lives there.
#
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
# redirected, and every assertion about those files would be reading the
# tracer. Call through this wrapper so the sinks hold what production writes.
run_capture() {
local _rc=0
( set +x; cozy_capture_tenant_worker_cpu_throttle 1 ) || _rc=$?
return "${_rc}"
}
@test "the phase-start warning names this collector as the exception it is" {
# That warning is what an operator reads in the CI log when `timeout` is
# missing, and the sibling suite treats its collector list as load-bearing
# rather than decorative -- it says which collectors do still fail that way.
# This one calls `timeout` directly, which puts it in the named class, and
# then does NOT fail that way because it carries a fallback. Unnamed, the
# sentence sends the reader to the opposite conclusion about the one collector
# whose behaviour differs, and a comment in the source does not reach them.
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
# Anchored on the phrase unique to the phase warning itself: the bracket-pair
# captures write their own [bounds] notes carrying "timeout is not on PATH",
# so the first match of that phrase is no longer this sentence.
warn=$(grep -n 'run UNBOUNDED' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$warn" ]; then
echo "expected the phase to still warn when timeout is missing" >&2
return 1
fi
line=$(sed -n "${warn}p" "$lib")
case "$line" in
*'exit 127 and collect nothing'*) ;;
*) echo "expected the warning to still name the 127 class it is contrasting against" >&2; return 1 ;;
esac
# The sentence went through false shapes before this one, including "except
# the worker CPU throttling capture", which read as a sole exception even
# though talos-image-cache has the same shape. An enumeration of a growing set
# is wrong the next time someone adds to it.
# What is pinned now is the DISCRIMINATOR: guarding the call with command -v
# is what separates the two outcomes, and every collector that guards is
# named. A fourth one added without a mention fails this.
case "$line" in
*'guard the call with command -v'*) ;;
*) echo "expected the warning to state what separates the two outcomes" >&2; return 1 ;;
esac
for c in 'CPU throttling' 'talos-image-cache'; do
case "$line" in
*"$c"*) ;;
*) echo "expected the warning to name the guarded collector: $c" >&2; return 1 ;;
esac
done
}
@test "a negative assertion fails when its own matcher cannot evaluate" {
# The helper below every "must not appear" claim in this file. Folded into an
# `if`, awk exiting 2 for an unparseable regex lands in the same branch as
# exiting 1 for "no match", so the assertion passes without having looked.
# What rides on it here are the claims that the artifact does NOT say
# something -- that a wedged kubelet was not written up as one that answered,
# that a capture was not labelled uncapped, that a log which was never
# written is not named. Those are the collector's own contract stated
# negatively, so a matcher that fails open turns each of them into a sentence
# nobody checked.
tmp=$(mktemp -d)
printf 'anything\n' >"$tmp/subject"
# A regex awk cannot parse. If the helper still reports "absent", it is
# reporting its own failure as a finding about the file.
if assert_file_lacks_pattern '*(' "$tmp/subject" 2>/dev/null; then
echo "FAIL: an unparseable pattern was reported as absent" >&2
rm -rf "$tmp"
return 1
fi
# And the two honest outcomes still work, or the guard above is bought by
# breaking the helper for everything.
assert_file_lacks_pattern 'nothinglikethis' "$tmp/subject"
if assert_file_lacks_pattern 'anything' "$tmp/subject" 2>/dev/null; then
echo "FAIL: a present pattern was reported as absent" >&2
rm -rf "$tmp"
return 1
fi
rm -rf "$tmp"
}
@test "the counters are read from the kubelet of the node the worker runs on" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# The endpoint is per node and the counters are per container, so the node
# name has to come from the Pod. Reading a fixed node, or the wrong one,
# returns a healthy stranger's counters.
assert_file_contains 'get --raw /api/v1/nodes/srv1/proxy/metrics/cadvisor' "$kubectl_calls"
# And the node name comes from the field this caller asks the listing for.
# That projection is a parameter of the shared listing, and the other caller
# of it passes the Pod name instead; swapped, this capture would ask kubelets
# named after Pods and get nothing, with no error to say why. The mock answers
# any listing regardless of what was asked for, so nothing else here sees it.
assert_file_contains 'jsonpath={range .items[*]}{.spec.nodeName}{"\n"}{end}' "$kubectl_calls"
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$capture"
# One assertion per alternative in the filter's regex, because a typo in an
# alternative no test drives through drops that family silently while every
# other assertion here stays green. The scheduled-periods count is the
# denominator the throttled count is read against, and the duration is what
# says whether a small ratio was a long stall.
assert_file_contains 'container_cpu_cfs_periods_total' "$capture"
assert_file_contains 'container_cpu_cfs_throttled_seconds_total' "$capture"
# The negative half of the labels. A complete capture that tells the reader
# its answer is missing, while the answer sits in the same file, is the same
# "capture misreads itself" harm as silence reported as an answer -- just
# inverted, and it fires on every healthy run rather than on the rare one.
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'unknown' "$capture"
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
[ ! -f "$dir/srv1.read-error.log" ]
[ ! -f "$dir/READ-WARNINGS.txt" ]
[ ! -f "$dir/COLLECTION-FAILED.txt" ]
# The in-tree name the stream falls back to when mktemp fails. The live
# cleanup pin is "the raw stream is removed after each node", which follows
# the stream to TMPDIR; this one only says the fallback leaves nothing behind
# either, and it would go green on a run that never took the fallback.
[ ! -f "$dir/srv1.stream.tmp" ]
rm -rf "$tmp"
}
@test "what the workers consumed is captured beside what they were denied" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# The throttled counters say the container met its ceiling. They cannot say
# how much CPU it got, and the two are different quantities: a guest held at a
# one-core ceiling and a guest scheduled onto a physical CPU for a tenth of
# that ceiling both report throttled periods, and the ratio alone does not
# separate them. The consumed seconds are what does, and cAdvisor publishes
# them on the same stream this capture already reads, so the answer costs a
# wider filter rather than another read.
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'container_cpu_usage_seconds_total' "$capture"
rm -rf "$tmp"
}
@test "each read is bounded and keeps its exit code" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# An unbounded read here would hold the whole diagnostics block against a
# wedged kubelet, and the phase budget bounds when a read may START, never
# how long one already running may take.
# Named by their verbs, not by the bound: both reads carry the same one, so a
# bare '-k 5 20 kubectl' would be satisfied twice by whichever one survived
# and would not notice the other losing its ceiling.
# The selector travels with the listing. Without it the walk covers every Pod
# in the namespace, and the node set becomes every node running anything.
assert_file_contains '-k 5 20 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher' "$timeout_calls"
assert_file_contains '-k 5 20 kubectl get --raw' "$timeout_calls"
# Both bounds, not one. The wall-clock wrapper bounds the process and
# --request-timeout bounds the HTTP request underneath it; the second is the
# only one left when `timeout` is not on PATH, which is the case the fallback
# below exists for.
assert_file_contains '--request-timeout=20s' "$kubectl_calls"
assert_file_contains '[capture exit code: 0]' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
rm -rf "$tmp"
}
@test "the sample number decides which reading a file belongs to" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
( set +x; cozy_capture_tenant_worker_cpu_throttle 2 )
# Every counter here is cumulative since the container started, so one reading
# is an average over an uptime and the pair is what gives a rate over the
# window the deadline covered. A second reading written over the first leaves
# one file and no interval -- collected, and unable to answer the question it
# was collected for.
assert_file_contains 'container_cpu_usage_seconds_total' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-2/srv1.txt"
[ ! -d "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1" ]
rm -rf "$tmp"
}
@test "each reading records when it was taken" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# The knob names how long the block WAITS between its two passes, which is not
# the interval a subject's two readings span: the other subject's pass falls
# between them, and on the run this collector exists for that pass is the slow
# part. A reader who divides by the advertised number is then wrong by
# whatever that pass cost, in the direction that understates a rate. The stamp
# is what makes the real interval readable off the pair.
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'read attempted from' "$capture"
# Both ends, not one. A single stamp leaves the sampling instant somewhere
# inside a read whose duration is what goes wrong on the run this collector
# exists for, so the interval a reader divides by would be off by up to the
# read bound with nothing in the artifact saying so.
stamp=$(sed -n 's/.*read attempted from \([0-9][0-9]*\) to \([0-9][0-9]*\) epoch seconds.*/\1 \2/p' "$capture")
if [ -z "$stamp" ]; then
echo "expected a bare epoch stamp in $capture" >&2
cat "$capture" >&2
return 1
fi
rm -rf "$tmp"
}
@test "the raw stream is removed after each node" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1 srv2"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# The stream is staged outside the report tree, so the assertion that used to
# catch a missing cleanup -- "no .stream.tmp in the report dir" -- cannot see
# it any more. Point TMPDIR at a directory of our own and check it is empty
# afterwards: mktemp honours TMPDIR, so this pins the cleanup wherever the
# file is staged, rather than pinning the path it used to have.
scratch="$tmp/scratch"
mkdir -p "$scratch"
TMPDIR="$scratch" run_capture
left=$(find "$scratch" -type f | wc -l | tr -d ' ')
if [ "$left" -ne 0 ]; then
echo "expected the raw metric stream to be removed; $left file(s) left in TMPDIR" >&2
find "$scratch" -type f >&2
return 1
fi
rm -rf "$tmp"
}
@test "a lowered read budget moves both of this collector's bounds" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# Assigned after sourcing, which is how both a caller and a test set it. A
# collector that took the knob's value at source time would keep 20 here.
#
# Not restored afterwards, and deliberately: run_capture calls the collector
# in an explicit subshell, and both runners give every @test a subshell of its
# own, so the assignment cannot outlive this body in either direction. A reset
# on the last line would be dead code that reads as load-bearing cleanup and
# teaches the next reader that state crosses tests here.
COZY_DIAG_READ_TIMEOUT=7
run_capture
assert_file_contains '-k 5 7 kubectl -n tenant-test get pods' "$timeout_calls"
assert_file_contains '-k 5 7 kubectl get --raw' "$timeout_calls"
assert_file_contains '--request-timeout=7s' "$kubectl_calls"
rm -rf "$tmp"
}
@test "a zero read budget is corrected rather than disabling the bound" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# Zero is the value that makes revalidation load-bearing rather than tidy:
# `timeout -k 5 0` disables the timeout outright and `--request-timeout=0s`
# means no timeout to kubectl, so a zero assigned after sourcing restores the
# unbounded read on exactly the wedged-kubelet path the bound exists for.
COZY_DIAG_READ_TIMEOUT=0
run_capture
assert_file_contains '-k 5 20 kubectl get --raw' "$timeout_calls"
# Bracketed rather than backslash-escaped: awk warns on `\-`, and a warning
# nobody reads is how an unparseable pattern would arrive here.
assert_file_lacks_pattern '[-]k 5 0 ' "$timeout_calls"
assert_file_lacks_pattern 'request-timeout=0s' "$kubectl_calls"
rm -rf "$tmp"
}
@test "the read's own status survives the filter that follows it" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# A kubelet that answered in full, whose body carries nothing for this
# namespace. Piped straight into grep the collector would report the read's
# status as grep's and call a healthy read a failure -- and, the other way
# round, would report a dead kubelet as a node with no tenant container.
kubectl_raw_output='container_cpu_cfs_periods_total{container="x",namespace="kube-system",pod="p"} 7'
kubectl_raw_rc=0
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains '[capture exit code: 0]' "$capture"
assert_file_contains 'the kubelet answered' "$capture"
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
rm -rf "$tmp"
}
@test "a node whose read times out does not stop the next node" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1 srv2"
timeout_fail_node=srv1
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-partial
run_capture
# One throttled worker answers the question, but only if the walk survives
# the node that did not answer -- and the wedged one is as likely as not to
# be first, since the node list is sorted.
dir="$COZY_REPORT_DIR/snapshots/throttle-partial/tenant-cpu-throttle/sample-1"
assert_file_contains '[capture exit code: 124]' "$dir/srv1.txt"
assert_file_contains '[capture exit code: 0]' "$dir/srv2.txt"
rm -rf "$tmp"
}
@test "a warning on a successful listing is kept apart from a setup failure" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_list_stderr='Warning: v1 ComponentStatus is deprecated'
kubectl_list_rc=0
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# kubectl writes warnings on stderr with a zero exit, so the filename is
# decided by the status rather than by something having been written. Filed
# as a collection failure, a healthy listing reads as a collector that never
# ran at all.
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
assert_file_contains 'ComponentStatus is deprecated' "$dir/READ-WARNINGS.txt"
[ ! -f "$dir/COLLECTION-FAILED.txt" ]
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$dir/srv1.txt"
rm -rf "$tmp"
}
@test "a pool below the cap leaves no truncation marker" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1 srv2"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-uncapped
run_capture
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-uncapped/tenant-cpu-throttle/sample-1/COLLECTION-TRUNCATED.txt" ]
rm -rf "$tmp"
}
@test "repeated nodes are asked once" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
# Two workers co-scheduled on one node is the ordinary case, not a corner:
# the endpoint is per node and carries both, so asking twice spends a second
# read out of a budget that declines whatever has not started, and buys a
# byte-for-byte duplicate.
kubectl_node_names="srv1 srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
count=$(grep -c 'get --raw' "$kubectl_calls" || true)
if [ "$count" -ne 1 ]; then
echo "expected one kubelet read for one node, got $count" >&2
return 1
fi
rm -rf "$tmp"
}
@test "the collector's own caps stay what its comments claim" {
# Values, not sums. A comment stating a total has to be kept true in every
# copy of it and falsifies silently; a cap the code carries as a literal can
# be pinned against the code itself, which is what this does.
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
# The walk and the read moved into the body this capture now shares with the
# network counters, so the subject of these assertions is the code the
# collector runs rather than the function that carries its name -- which is
# now a six-line call. Reading only the named function would leave every
# assertion below vacuously true against a body it no longer contains.
fn=$(awk '/^cozy_capture_tenant_worker_cpu_throttle\(\)/,/^}/' "$lib"
awk '/^_cozy_capture_worker_cadvisor\(\)/,/^}/' "$lib"
awk '/^_cozy_cadvisor_node_stream\(\)/,/^}/' "$lib"
awk '/^_cozy_virt_launcher_listing\(\)/,/^}/' "$lib")
case "$fn" in
*'max_nodes=3'*) ;;
*) echo "expected the collector to cap the walk at the 3 nodes the comments state" >&2; return 1 ;;
esac
# The knob, not a literal that equals it today. A hardcoded 20 keeps its
# value when the block's budget is lowered, so the one read that ignores the
# knob is the one that outlives it -- and the sibling suite that lowers the
# knob is what caught exactly that here.
case "$fn" in
*'timeout -k "${COZY_DIAG_READ_GRACE}" "${COZY_DIAG_READ_TIMEOUT}"'*) ;;
*) echo "expected each kubelet read to take the block's read-budget knob" >&2; return 1 ;;
esac
case "$fn" in
*'timeout -k 5 20'*) echo "expected no hardcoded bound to survive beside the knob" >&2; return 1 ;;
esac
# Checked in the source as well as exercised, not instead of it. `command -v`
# finds the shell function this file defines for `timeout`, so the fallback arm
# cannot be reached from a test body -- but it can be reached from a subprocess
# with a stripped PATH, and a test further down does exactly that.
# The scratch stream must not live in the report tree. This is a source-level
# claim on purpose: the case it guards against is a hard kill between writing
# the stream and removing it, which no unit test can stage, so the only thing
# that can be pinned is where the path points.
#
# Matched against CODE, with comment lines stripped first. The prose here
# names both `mktemp` and the fallback path, so a pattern run over the whole
# function is satisfied by the explanation of the rule instead of by the rule
# -- deleting the assignment outright would leave such a check green.
code=$(printf '%s\n' "$fn" | grep -v '^[[:space:]]*#')
case "$code" in
*'stream=$(mktemp "'*) ;;
*) echo "expected the raw metric stream to be staged via mktemp with an explicit template" >&2; return 1 ;;
esac
# The template must be explicit: BSD mktemp with no template ignores TMPDIR
# and always lands in the system directory, so a bare call puts the file
# somewhere neither this suite nor an operator chose.
case "$code" in
*'mktemp "${TMPDIR:-/tmp}/'*) ;;
*) echo "expected the scratch template to honour TMPDIR explicitly" >&2; return 1 ;;
esac
# report_dir may appear only as the fallback, never as the primary target.
primary=$(printf '%s\n' "$code" | grep 'stream=' | grep -v '^[[:space:]]*||' | head -n 1)
case "$primary" in
*'${report_dir}'*) echo "the raw stream must not be staged inside the report tree" >&2; return 1 ;;
esac
# Both reads, not just the kubelet one: the listing has the same two arms and
# the same fallback, so counting only `get --raw` leaves the arm that matters
# for the listing unwatched for exactly the same reason.
# This loop is the cheap half of that pair: it holds every read that EXISTS to
# the flag. It cannot see a branch that was deleted, because a count over the
# reads still present balances either way -- that is what the stripped-PATH
# test buys, and why both are here.
for verb in 'kubectl get --raw' 'kubectl -n tenant-test get pods'; do
reads=$(printf '%s\n' "$fn" | grep -c "$verb" || true)
bounded=$(printf '%s\n' "$fn" | grep -A3 "$verb" | grep -c -- '--request-timeout=' || true)
if [ "$reads" -eq 0 ] || [ "$reads" -ne "$bounded" ]; then
echo "every '$verb' must carry --request-timeout; $bounded of $reads do" >&2
return 1
fi
done
}
@test "both chainsaw comments list the collector where the phase actually spends it" {
# The ORDER is the claim, not any duration. Those comments describe the order
# the phase budget is spent in, and a budget that runs out declines whatever
# has not started -- so an entry in the wrong place misdescribes which
# collector survives a tight run. Pinning a minute figure instead would pin
# the one thing in that comment nobody can keep true.
for f in hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml \
hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml; do
reads=$(grep -n 'node-join diagnostics reads' "$f" | head -n 1 | cut -d: -f1)
cgroup=$(grep -n 'worker CPU usage and throttling counters' "$f" | head -n 1 | cut -d: -f1)
console=$(grep -n 'serial-console family' "$f" | head -n 1 | cut -d: -f1)
talos=$(grep -n 'guest Talos capture' "$f" | head -n 1 | cut -d: -f1)
if [ -z "$reads" ] || [ -z "$cgroup" ] || [ -z "$console" ] || [ -z "$talos" ]; then
echo "expected $f to list the reads, the counter capture, the console family and the guest Talos capture" >&2
return 1
fi
# Below the console, not above it. This capture is read twice and can spend
# most of the phase budget, and admission gates only when a collector may
# start, so above the console it would bound what the console gets -- and
# the console is the only surface that survives a worker which never reached
# apid. Above the guest Talos capture, which needs the apid that failure
# never reached and is the more expensive of the two.
if [ "$cgroup" -le "$console" ] || [ "$cgroup" -ge "$talos" ]; then
echo "in $f the counter capture must sit between the console family and the guest Talos capture" >&2
return 1
fi
if [ "$console" -le "$reads" ]; then
echo "in $f the console family must sit after the node-join reads" >&2
return 1
fi
done
}
@test "the capture is declined by the phase budget rather than running unconditionally" {
# Ungated, this collector would spend its budget after the phase had already
# declined the cheaper reads around it -- which is the failure the phase
# budget exists to prevent, reintroduced by the newest caller.
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
gate=$(grep -n "^ *if cozy_diag_phase_has_time '(d) tenant worker CPU counters, sandbox node CPU time and worker per-thread CPU time'; then$" \
"$lib" | head -n 1 | cut -d: -f1)
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$gate" ] || [ -z "$call" ]; then
echo "expected the capture to sit behind a cozy_diag_phase_has_time gate" >&2
return 1
fi
# THIS gate, not just some gate above the call. A line comparison alone is
# satisfied by any earlier gate in the block, which would leave the capture
# guarded by a decision taken about something else. The call now sits inside
# the sampling loop rather than on the line after the gate, so adjacency is no
# longer the way to say it: what says it is that no other gate opens in
# between.
next=$(awk -v g="$gate" 'NR > g && /cozy_diag_phase_has_time/ { print NR; exit }' "$lib")
if [ -z "$next" ]; then
echo "expected another phase gate after line $gate, so this test could bound the section" >&2
return 1
fi
if [ "$call" -le "$gate" ] || [ "$call" -ge "$next" ]; then
echo "the capture (line $call) must sit between its gate ($gate) and the next one ($next)" >&2
return 1
fi
}
@test "the capture runs ahead of the collectors that cost more, and behind the one it could starve" {
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$call" ]; then
echo "expected the failure path to call the throttling capture" >&2
return 1
fi
# What runs last is what the budget declines, so position is what decides
# whether this collector is ever taken on a tight run. Its question has no
# other answer in this tree, so it precedes the legs below. Stated without a
# count on purpose: the loop below is the enumeration, and a sentence that
# also counted would go stale the first time a leg is added to it.
for leg in 'cozy_capture_tenant_talos "${test_name}" || true' \
'talos_image_cache_diagnose || true'; do
line=$(grep -n -F -x " ${leg}" "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$line" ]; then
echo "expected the failure block to still call: ${leg}" >&2
return 1
fi
if [ "$call" -ge "$line" ]; then
echo "the throttling capture (line $call) must run before ${leg} (line $line)" >&2
return 1
fi
done
# And behind the console. Reading the counters twice costs the walk twice
# plus the interval, which is most of the phase budget at the read bound and
# the node cap, and admission gates only when a collector may START. Ahead of
# the console, a run that hits those bounds would leave the console never
# started rather than cut short -- and the console is the only capture that
# survives a worker which never reached apid, the shape this failure usually
# takes. Cost, not worth, is what puts it here.
console=$(grep -n "cozy_capture_tenant_serial_console 'node-join failed" "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$console" ]; then
echo "expected the failure block to still capture the guest serial console" >&2
return 1
fi
if [ "$call" -le "$console" ]; then
echo "the throttling capture (line $call) must run after the console capture (line $console)" >&2
return 1
fi
}
@test "the ceiling is captured beside the counters" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
# The throttled counters on their own say a container hit some ceiling, not
# which one. Without the quota beside them the reader cannot tell a VM capped
# at one core from one capped at eight, which is the whole question.
assert_file_contains 'container_spec_cpu_quota' "$capture"
# The period travels with the quota because the quota alone is meaningless:
# 100000us of quota is one core against a 100000us period and a tenth of one
# against a 1000000us period.
assert_file_contains 'container_spec_cpu_period' "$capture"
rm -rf "$tmp"
}
@test "the capture tells the reader how to pair it with its sibling sample" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# The instruction lives with the subject that has a sibling rather than in the
# walk both subjects share, because the other caller of that walk takes one
# sample and would otherwise carry an instruction pointing at a directory that
# is never created.
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'subtract this node file from its sibling under the other sample directory' "$capture"
rm -rf "$tmp"
}
@test "a read that did not finish is not told to pair with a sibling" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# Series arrived and the read still failed, which is a stream cut off part way
# through.
kubectl_raw_rc=124
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'these counters are incomplete' "$capture"
# Telling a reader to subtract two files asserts that both hold a whole
# reading. This one is already marked incomplete, and a difference taken
# against it understates the counter by whatever the read did not return --
# in the direction that makes a starved worker look busier than it was. The
# sandbox capture withholds its column legend from a short read for the same
# reason, so the pair answers this the same way on both sides.
assert_file_lacks_pattern 'subtract this node file from its sibling' "$capture"
rm -rf "$tmp"
}
@test "an unquota'd container is reported as uncapped rather than as a gap" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# One line, and it is the whole shape: cAdvisor gates the quota series and all
# three CFS counters on the same non-zero quota, so an uncapped container puts
# the period on the wire and nothing else. Staging a CFS counter beside it
# would be a shape the endpoint cannot produce, and the test would then pass
# on a fixture rather than on the contract. It is an answer, not a short read:
# filed as "nothing was collected" it would send the reader looking for a
# ceiling that does not exist.
kubectl_raw_output='container_spec_cpu_period{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'container_spec_cpu_period' "$capture"
assert_file_lacks_pattern 'unknown' "$capture"
assert_file_lacks_pattern 'incomplete' "$capture"
# Said in words, not left to inference. One line at exit 0 is also what a
# truncated read looks like, and every other outcome in this collector gets a
# sentence -- so the one answer it was built to deliver cannot be the one that
# gets none.
assert_file_contains 'running uncapped' "$capture"
rm -rf "$tmp"
}
@test "a quota probe that could not read says nothing about the ceiling" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# The last grep in the function is the one that decides whether the artifact
# makes a positive claim about the ceiling. grep exits 2 when it cannot read,
# and a bare `!` turns that into "no quota found", which is the direction of
# error this whole collector exists to refuse: a statement about the cluster
# derived from a read that failed on this runner.
grep() {
case "$*" in
*'^container_spec_cpu_quota{'*) return 2 ;;
esac
command grep "$@"
}
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_lacks_pattern 'running uncapped' "$capture"
unset -f grep
rm -rf "$tmp"
}
@test "a capped container is not labelled uncapped" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# The default fixture is the capped shape: every family, quota included.
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'container_spec_cpu_quota' "$capture"
assert_file_lacks_pattern 'running uncapped' "$capture"
rm -rf "$tmp"
}
@test "a read killed by the grace signal says so rather than leaving the code alone" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=137
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# 137 is what `timeout -k` leaves behind when the child ignored SIGTERM and
# had to be killed, by whatever did it. Naming every way a read comes up
# short is this collector's whole argument,
# so the one status that means "killed, not finished" cannot be the exception.
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'exit 137' "$capture"
assert_file_lacks_pattern 'exit 124' "$capture"
rm -rf "$tmp"
}
@test "a kubelet that was not read is recorded as unknown, not as an absence of throttling" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=1
kubectl_raw_output=
kubectl_raw_stderr='error: unable to connect to the server'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
# This is the failure this collector exists to prevent, inverted: an empty
# file here reads as a container that never hit its ceiling, which is the
# exact conclusion the capture was added to stop people reaching by default.
assert_file_contains 'unknown' "$capture"
# `unknown` alone is shared by both labels, so it pins the word and not the
# branch: disable this arm and an unread kubelet falls through to the next
# one, where the artifact tells the reader the kubelet ANSWERED and merely
# carried nothing. Pin the sentence that only this arm writes.
assert_file_contains 'the kubelet was not read' "$capture"
# And that it points at the log rather than leaving the reader to find it.
assert_file_contains 'see read-error.log' "$capture"
# The stamp claims only when the read was attempted. This arm has just said
# nothing was read, so a line asserting the counters were sampled inside the
# bracket contradicts the line above it -- the same two-lines-disagreeing
# failure the pairing note and the legend are both gated against.
assert_file_lacks_pattern 'counters were sampled' "$capture"
assert_file_contains 'read attempted from' "$capture"
assert_file_contains 'error: unable to connect to the server' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.read-error.log"
rm -rf "$tmp"
}
@test "a forbidden nodes-proxy read is preserved rather than folded into silence" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=1
kubectl_raw_output=
# RBAC on nodes/proxy is a failure mode this read has and an in-container
# read did not. It is also the one an operator can fix, so the 403 has to
# reach the artifact verbatim instead of becoming a generic "not read".
kubectl_raw_stderr='Error from server (Forbidden): nodes "srv1" is forbidden: User "system:serviceaccount:x:y" cannot get resource "nodes/proxy"'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
assert_file_contains 'the kubelet was not read' "$dir/srv1.txt"
assert_file_contains 'nodes/proxy' "$dir/srv1.read-error.log"
assert_file_contains 'Forbidden' "$dir/srv1.read-error.log"
rm -rf "$tmp"
}
@test "a read killed without a word is not reported as a kubelet that answered" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# `timeout` kills its child and writes nothing, and kubectl dies on SIGTERM
# without a word, so this -- not the tidy error above -- is the shape a
# wedged kubelet actually produces. Keyed on whether stderr exists rather
# than on the status, the collector calls it an answer.
kubectl_raw_rc=124
kubectl_raw_output=
kubectl_raw_stderr=
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'the kubelet was not read' "$capture"
# The half that makes it a defect rather than a wording preference: the
# artifact must not state the opposite of what happened.
assert_file_lacks_pattern 'the kubelet answered' "$capture"
# No error log exists for a silent kill, so pointing at one sends the reader
# after evidence that was never written.
assert_file_lacks_pattern 'see read-error.log' "$capture"
assert_file_contains 'without a word' "$capture"
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.read-error.log" ]
rm -rf "$tmp"
}
@test "a stream cut short without a word is still marked incomplete" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# Counters arrived, the ceiling did not, and the read died silently. This is
# the worst shape in the set: the header states that counters without a quota
# mean a container running uncapped, so an unlabelled truncation does not
# merely lose information -- it reads as a positive finding about the ceiling.
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211'
kubectl_raw_rc=137
kubectl_raw_stderr=
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'incomplete' "$capture"
assert_file_lacks_pattern 'see read-error.log' "$capture"
rm -rf "$tmp"
}
@test "a filter that cannot read the stream blames neither the kubelet nor the namespace" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# grep exits 2 when it cannot read its input -- a full TMPDIR on the runner
# reaches this. The kubelet read itself succeeded, so the artifact must not
# say the kubelet answered and carried nothing (that is the conflation this
# collector exists to prevent), and must not say the kubelet was not read
# (it was). It has to name the local failure and nothing else.
grep() { case "$*" in *"-E "*) command grep "$@" >/dev/null 2>&1; return 2 ;; esac; command grep "$@"; }
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'could not be read back on this runner' "$capture"
assert_file_lacks_pattern 'the kubelet answered' "$capture"
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
unset -f grep
rm -rf "$tmp"
}
@test "a filter failure that said why points at the log it wrote" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# The half of this branch the sibling tests do not reach: they all silence the
# filter's stderr, so all of them land in the wordless variant and none drives
# the one that names the log. Both variants open with the same sentence, so a
# test asserting only that sentence passes on either.
grep() {
case "$*" in
*"-E "*) echo 'grep: input: Input/output error' >&2; return 2 ;;
esac
command grep "$@"
}
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
assert_file_contains 'could not be read back on this runner' "$dir/srv1.txt"
assert_file_contains 'see filter-error.log' "$dir/srv1.txt"
assert_file_lacks_pattern 'the filter said nothing about why' "$dir/srv1.txt"
assert_file_contains 'Input/output error' "$dir/srv1.filter-error.log"
unset -f grep
rm -rf "$tmp"
}
@test "a filter failure never names a log that was not written" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# Fails silently: exit 2 with nothing on stderr. Naming a file that was never
# created sends the reader after evidence that does not exist -- the same rule
# every other message in this function follows.
grep() { case "$*" in *"-E "*) return 2 ;; esac; command grep "$@"; }
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
assert_file_contains 'the filter said nothing about why' "$dir/srv1.txt"
assert_file_lacks_pattern 'see filter-error.log' "$dir/srv1.txt"
[ ! -f "$dir/srv1.filter-error.log" ]
unset -f grep
rm -rf "$tmp"
}
@test "a namespace filter that cannot read its input is not reported as an answered kubelet" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# The same unreadable-input failure as the metric filter above, one stage
# later. Every stage of the filter narrows the same stream, so an exit 2 from
# any of them leaves the capture empty for a reason that has nothing to do
# with what the kubelet said.
grep() { case "$*" in *'namespace="tenant-test"'*) return 2 ;; esac; command grep "$@"; }
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'could not be read back on this runner' "$capture"
assert_file_lacks_pattern 'the kubelet answered' "$capture"
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
unset -f grep
rm -rf "$tmp"
}
@test "a worker filter that cannot read its input is not reported as an answered kubelet" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# A chain that pipes the stage above into this one drops that stage's status
# at the pipe and this one's at the trailing `|| true`, so neither reaches the
# artifact and both need their own test: one of them passing says nothing
# about the other.
grep() { case "$*" in *'pod="virt-launcher-'*) return 2 ;; esac; command grep "$@"; }
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'could not be read back on this runner' "$capture"
assert_file_lacks_pattern 'the kubelet answered' "$capture"
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
unset -f grep
rm -rf "$tmp"
}
@test "a filter that fails after writing leaves the capture marked incomplete" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
# A write error is where grep emits what it had flushed and then exits 2 --
# ENOSPC on a full disk reaches it, and so does the full TMPDIR this split was
# written for. The counters that arrive are real, so the capture is not
# unknown; it is short, and the family that goes missing first is the quota,
# whose absence reads as a container running uncapped.
grep() {
case "$*" in
*'pod="virt-launcher-'*)
command grep "$@" | head -n 1
echo 'grep: stdout: File too large' >&2
return 2
;;
esac
command grep "$@"
}
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'incomplete' "$capture"
assert_file_contains 'see filter-error.log' "$capture"
# The read itself was fine, so nothing here may blame it.
assert_file_lacks_pattern 'the read was cut short' "$capture"
assert_file_lacks_pattern 'see read-error.log' "$capture"
unset -f grep
rm -rf "$tmp"
}
@test "a filter that fails after writing never names a log that was not written" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
grep() {
case "$*" in
*'pod="virt-launcher-'*) command grep "$@" | head -n 1; return 2 ;;
esac
command grep "$@"
}
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
assert_file_contains 'incomplete' "$dir/srv1.txt"
assert_file_contains 'the filter said nothing about why' "$dir/srv1.txt"
assert_file_lacks_pattern 'see filter-error.log' "$dir/srv1.txt"
[ ! -f "$dir/srv1.filter-error.log" ]
unset -f grep
rm -rf "$tmp"
}
@test "a namespace whose containers are none of them workers names the worker filter" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=0
# Series for this namespace, none of them from a worker Pod -- what an
# upstream rename of the virt-launcher prefix looks like from here. Saying the
# kubelet reported nothing for the namespace would be false in this state and
# would send a reader after scheduling while the filter is what emptied the
# capture.
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="kube-apiserver",namespace="tenant-test",pod="kube-apiserver-a"} 7
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="RENAMED-launcher-worker-a"} 4211'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'unknown' "$capture"
assert_file_contains 'none of them from a worker Pod' "$capture"
assert_file_lacks_pattern 'reported no CPU series' "$capture"
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
# The rejected series must not ride along either: naming the filter is not a
# licence to print what it rejected.
assert_file_lacks_pattern 'kube-apiserver' "$capture"
rm -rf "$tmp"
}
@test "a kubelet that answered without this namespace is not reported as unread" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=0
kubectl_raw_output=
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
# Distinct from the kubelet that never answered: this one did, and an
# artifact that blames the read would send an operator after RBAC and
# networking when the actual finding is that no worker container was there.
assert_file_contains 'unknown' "$capture"
assert_file_contains 'the kubelet answered' "$capture"
assert_file_lacks_pattern 'see read-error.log' "$capture"
# Both empty-capture arms open with "the kubelet answered", so the line above
# cannot tell them apart on its own. This one is about a node carrying nothing
# of the namespace, and the sibling arm's wording is what it must not be.
assert_file_contains 'no CPU series for a tenant-test worker' "$capture"
assert_file_lacks_pattern 'none of them from a worker Pod' "$capture"
rm -rf "$tmp"
}
@test "with timeout off PATH the kubelet read still runs and keeps its request bound" {
# The fallback arm is what the phase-start warning promises for this collector:
# on a runner without the binary it runs UNBOUNDED rather than exiting 127, and
# `--request-timeout` is then the only ceiling left. A promise the code does not
# keep is what this whole collector is against, so it is entered here rather
# than inspected.
#
# A subprocess with a stripped PATH, because this file mocks `timeout` as a
# shell function and `command -v` finds functions -- the condition cannot be
# reached from inside a test body. Same device as the sibling suite uses for
# the block's own reads.
tmp=$(mktemp -d)
mkdir -p "$tmp/bin"
# Everything the collector shells out to, minus `timeout` itself. Missing one
# would fail this test for the wrong reason and read as the arm being broken.
for c in mkdir sort grep mv rm mktemp wc tr cat date; do
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin; do
if [ -x "$d/$c" ]; then
ln -sf "$d/$c" "$tmp/bin/$c"
break
fi
done
done
for c in mkdir sort grep mv rm mktemp wc tr date; do
if [ ! -x "$tmp/bin/$c" ]; then
echo "FAIL: could not stage $c in the stripped PATH; the check below would be vacuous" >&2
return 1
fi
done
# Only the staged directory is checked. `command -v timeout` is always true
# here -- this file defines it as a shell function, which is the very fact the
# subprocess exists to escape -- so folding it in would read as if both halves
# mattered while only this one does.
if [ -e "$tmp/bin/timeout" ]; then
echo "FAIL: timeout leaked into the stripped PATH; this test would prove nothing" >&2
return 1
fi
out=$(bash -c '
set -eu
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
kubectl() {
# To a file, not to stderr. The collector redirects the stderr of every
# read into its own artifact, so a stub that announces itself there is
# captured by the code under test and never reaches this output.
printf "KUBECTL %s\n" "$*" >>"$CALLS"
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then printf "srv1\n"; return 0; fi
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then
printf "container_cpu_cfs_throttled_periods_total{container=\"compute\",namespace=\"tenant-test\",pod=\"virt-launcher-a\"} 7\n"
return 0
fi
return 0
}
CALLS='"$tmp"'/calls
COZY_REPORT_DIR='"$tmp"'/report
COZY_SNAPSHOT_NAME=fallback
PATH='"$tmp"'/bin
cozy_capture_tenant_worker_cpu_throttle 1
' 2>&1) || true
printf '%s\n' "$out" >"$tmp/out"
# The read ran at all -- bounded into exit 127 would leave nothing here.
assert_file_contains 'KUBECTL get --raw' "$tmp/calls"
# And carried the only bound it still has. This is the assertion the source
# check cannot make: it proves the flag reaches the arm that is actually taken.
assert_file_contains '--request-timeout=20s' "$tmp/calls"
# And produced counters rather than an "unknown" label.
assert_file_contains 'container_cpu_cfs_throttled_periods_total' \
"$tmp/report/snapshots/fallback/tenant-cpu-throttle/sample-1/srv1.txt"
rm -rf "$tmp"
}
@test "a killed read is not written up as a grace period that never ran" {
tmp=$(mktemp -d)
mkdir -p "$tmp/bin"
for c in mkdir sort grep mv rm mktemp wc tr cat date; do
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin; do
if [ -x "$d/$c" ]; then
ln -sf "$d/$c" "$tmp/bin/$c"
break
fi
done
done
if [ ! -x "$tmp/bin/grep" ]; then
echo "could not stage a PATH without timeout" >&2
return 1
fi
# 137 on the arm that has no `timeout` at all. The status is 128+SIGKILL and
# says only that something killed the read -- an OOM killer on a loaded
# runner, a teardown signalling the process group. On this arm the grace
# period is the one cause it cannot have been, because there is no `timeout`
# in the call. cozy_diag_read refuses the same wording for the same reason.
out=$(bash -c '
set -eu
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
kubectl() {
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then printf "srv1\n"; return 0; fi
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then return 137; fi
return 0
}
COZY_REPORT_DIR='"$tmp"'/report
COZY_SNAPSHOT_NAME=killed
PATH='"$tmp"'/bin
cozy_capture_tenant_worker_cpu_throttle 1
' 2>&1) || true
printf '%s\n' "$out" >"$tmp/out"
capture="$tmp/report/snapshots/killed/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'exit 137' "$capture"
assert_file_lacks_pattern 'grace period' "$capture"
rm -rf "$tmp"
}
@test "the tenant control plane is not reported as a worker" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# tenant-test is not a worker-only namespace. It also carries the Kamaji
# control plane -- the same file reads the talos-csr-signer sidecar out of it
# -- and the CDI importer Pods. Those containers carry their own CPU limits,
# so they have a real CFS quota and can show genuine throttling. Reported
# under a heading that says "worker", a throttled apiserver answers the
# question with the wrong subject, which is the harm the neighbouring test
# guards against from a different namespace -- and this direction is likelier.
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="kube-apiserver",namespace="tenant-test",pod="kubernetes-test-latest-version-abc"} 9999
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 4211'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'virt-launcher-worker-a-11111' "$capture"
assert_file_lacks_pattern 'kube-apiserver' "$capture"
rm -rf "$tmp"
}
@test "another tenant's worker on the same node is not reported as ours" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# The node filter is per node, and a management node hosts whatever is
# scheduled on it. A virt-launcher belonging to a different tenant carries the
# same Pod-name prefix and the same container, so the Pod filter alone cannot
# separate them -- only the namespace can. Reported here it would be a
# stranger's throttling presented as this suite's worker, which is the same
# wrong-subject harm as the control plane, one namespace over.
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-other",pod="virt-launcher-elsewhere-99999"} 8888
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 4211'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_contains 'virt-launcher-worker-a-11111' "$capture"
assert_file_lacks_pattern 'tenant-other' "$capture"
assert_file_lacks_pattern 'virt-launcher-elsewhere' "$capture"
rm -rf "$tmp"
}
@test "a series that merely contains a metric name does not pass the filter" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# An unanchored match would take any series whose name embeds one of the
# five, and the artifact would carry numbers that answer a different
# question while looking like the ones that answer this one.
# The Pod label has to match the worker filter, or the anchor is not what is
# under test: without it these lines are dropped for the wrong reason and the
# assertion holds whether the regex is anchored or not.
kubectl_raw_output='xcontainer_cpu_cfs_periods_total{namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 1
recorded_container_spec_cpu_quota{namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 2'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
assert_file_lacks_pattern 'xcontainer' "$capture"
assert_file_lacks_pattern 'recorded_container' "$capture"
assert_file_contains 'the kubelet answered' "$capture"
rm -rf "$tmp"
}
@test "a read status cannot be read as the guest's own answer" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_rc=124
kubectl_raw_output=
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
# `timeout` reports 124 on expiry and passes the command's own status through
# otherwise, so a 124 here is either this collector's deadline or the read
# exiting 124 by itself, and nothing separates them. The file states that
# rather than picking one.
assert_file_contains '124' "$capture"
assert_file_contains 'cannot be told apart' "$capture"
assert_file_contains '[capture exit code: 124]' "$capture"
rm -rf "$tmp"
}
@test "the walk is capped and names both counts when it stops" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="n1 n2 n3 n4 n5 n6"
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
# This runs inside a failure path that already spends minutes on other
# collectors under one chainsaw op timeout. A short listing otherwise reads
# as a small cluster rather than a cut walk.
truncated="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-TRUNCATED.txt"
assert_file_contains '6 carried a worker in total' "$truncated"
# n4, not n6: the cap's VALUE is the claim, and the two chainsaw Test files
# order the failure path on it. Asserting some later node is absent leaves a
# larger cap green, which would silently overspend the budget.
[ -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/n3.txt" ]
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/n4.txt" ]
rm -rf "$tmp"
}
@test "a read cut off part way through is not presented as a complete answer" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
# The counters arrived; the stream died before the spec families that carry
# the ceiling. This is the one partial result that reads like a complete
# answer, because a missing quota otherwise means "uncapped".
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211'
kubectl_raw_rc=1
kubectl_raw_stderr='error: unexpected EOF'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
capture="$dir/srv1.txt"
assert_file_contains 'incomplete' "$capture"
assert_file_contains 'see read-error.log' "$capture"
assert_file_contains 'unexpected EOF' "$dir/srv1.read-error.log"
rm -rf "$tmp"
}
@test "a namespace with no scheduled worker says so instead of leaving an empty tree" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names=
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture || true
# Distinct from the listing that never answered: this one got a real answer.
# An unscheduled virt-launcher has no node, so there is no kubelet to ask --
# which is itself a finding about the failure, not a gap in the collector.
assert_file_contains 'no virt-launcher Pod with a node assigned' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
rm -rf "$tmp"
}
@test "a warning beside a healthy read is not filed as a failure" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_node_names="srv1"
kubectl_raw_stderr='Warning: metrics endpoint is deprecated'
kubectl_raw_rc=0
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
# kubectl writes deprecation and partial-result warnings on stderr with a
# zero exit, so the exit status decides the filename, not the fact that
# something was written. Filed as an error, a healthy read would look failed.
assert_file_contains 'deprecated' "$dir/srv1.READ-WARNINGS.txt"
[ ! -f "$dir/srv1.read-error.log" ]
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$dir/srv1.txt"
rm -rf "$tmp"
}
@test "a Pod list that never returned is not reported as no workers" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
kubectl_calls="$tmp/kubectl.calls"
timeout_calls="$tmp/timeout.calls"
kubectl_list_rc=1
kubectl_list_stderr='Unable to connect to the server'
COZY_REPORT_DIR="$tmp/report"
COZY_SNAPSHOT_NAME=throttle-smoke
run_capture || true
# An empty listing is what a cluster with no virt-launcher Pods produces. A
# listing that never answered has to leave something else behind, and the
# status it is judged on has to be kubectl's rather than that of whatever
# post-processing follows it.
assert_file_contains 'Unable to connect to the server' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
assert_file_contains 'failed to list' \
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
rm -rf "$tmp"
}
@test "the capture is wired into the node-join failure path" {
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
# The reporter is its own function now, so its declaration and its closing
# brace bound the region. Anchoring on the `exit 1` that follows would measure
# the caller instead: the exit sits outside this function, and every line of
# the reporter would count as inside any span reaching it.
head=$(grep -n '^cozy_report_node_join_failure() {$' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$head" ]; then
echo "expected the node-join failure reporter to still be a function" >&2
return 1
fi
tail=$(awk -v h="$head" 'NR > h && $0 == "}" { print NR; exit }' "$lib")
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
if [ -z "$call" ] || [ -z "$tail" ]; then
echo "expected the node-join failure path to capture worker CPU throttling" >&2
return 1
fi
# A collector nothing calls is the shape this tree has shipped before: it
# reads as covered in review and produces nothing in the artifact. Pin the
# call site, not just the function.
if [ "$call" -le "$head" ] || [ "$call" -ge "$tail" ]; then
echo "the throttling capture (line $call) must sit inside the reporter ($head..$tail)" >&2
return 1
fi
# And that the reporter is still reached: a function nobody calls has the
# same artifact as a collector nobody calls, one level up.
grep -q '^ *cozy_report_node_join_failure "${test_name}"$' "$lib" || {
echo "expected run_kubernetes_test to still call the node-join reporter" >&2
return 1
}
}