mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
The shell translator rewrites test bodies and loses Bats reporting. The unit lane needs complete TAP results and JUnit reports without changing how live-cluster suites run. Sourced production helpers still need a dash pass, and unit test bodies need nounset without exporting shell options into subprocesses. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
1638 lines
72 KiB
Bash
1638 lines
72 KiB
Bash
#!/usr/bin/env bats
|
|
# Regression coverage for the tenant-worker CPU throttling capture in
|
|
# hack/e2e-chainsaw/_lib/run-kubernetes.sh. cozytest.sh ends an @test block at
|
|
# the first bare closing brace, so command mocks stay at top level.
|
|
#
|
|
# What this capture answers, and why nothing else in the tree answers it: when
|
|
# a worker misses the node-Ready deadline, the sandbox node it runs on is
|
|
# routinely half idle while the guest makes no progress. Node-level utilisation
|
|
# and the node's dmesg both look healthy in that state, so "the host was
|
|
# starved" and "the VM was held at its own CFS ceiling" produce the same
|
|
# evidence from outside. Only the container's own throttling counters separate
|
|
# them, and a capture that reports an unread kubelet as a quiet one would hand
|
|
# the next reader the same ambiguity while looking like it had resolved it.
|
|
# These tests pin the separation, not the reading.
|
|
|
|
load test_helper
|
|
|
|
kubectl_calls=/dev/null
|
|
kubectl_node_names=
|
|
kubectl_list_rc=0
|
|
kubectl_list_stderr=
|
|
kubectl_raw_rc=0
|
|
kubectl_raw_stderr=
|
|
# Two of the fixtures in this file claim to be wire shapes, and the rest do not.
|
|
# This one and the uncapped test's single line are the two cAdvisor can actually
|
|
# produce -- every family for a capped container, the period alone for one
|
|
# with no limit, the two sides of the same Quota != 0 gate. Usage sits outside
|
|
# that gate and is published for any container at all, so it belongs in both
|
|
# shapes; the uncapped fixture stages only the line its own assertion needs, as
|
|
# the rest of this file does. Every other fixture
|
|
# below stages only the lines its own assertion needs, so a missing sibling
|
|
# series there means "not relevant here" rather than "absent on the wire".
|
|
kubectl_raw_output='container_cpu_usage_seconds_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 118.4
|
|
container_cpu_cfs_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 51200
|
|
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211
|
|
container_cpu_cfs_throttled_seconds_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 12.5
|
|
container_spec_cpu_quota{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000
|
|
container_spec_cpu_period{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000'
|
|
timeout_calls=/dev/null
|
|
timeout_fail_node=
|
|
|
|
kubectl() {
|
|
printf '%s\n' "$*" >>"${kubectl_calls}"
|
|
|
|
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then
|
|
[ -z "${kubectl_list_stderr}" ] || printf '%s\n' "${kubectl_list_stderr}" >&2
|
|
[ "${kubectl_list_rc}" -eq 0 ] || return "${kubectl_list_rc}"
|
|
[ -z "${kubectl_node_names}" ] || printf '%s\n' ${kubectl_node_names}
|
|
return 0
|
|
fi
|
|
|
|
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then
|
|
[ -z "${kubectl_raw_stderr}" ] || printf '%s\n' "${kubectl_raw_stderr}" >&2
|
|
# Output is emitted before the status is honoured, because a read that is
|
|
# cut off part way through has already written what it managed to read. A
|
|
# stub that returns first can only ever produce empty-plus-failure, and
|
|
# the branch that labels a partial capture would be unreachable in tests
|
|
# while being the likeliest one in production.
|
|
[ -z "${kubectl_raw_output}" ] || printf '%s\n' "${kubectl_raw_output}"
|
|
return "${kubectl_raw_rc}"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
timeout() {
|
|
local command_rc=0
|
|
printf '%s\n' "$*" >>"${timeout_calls}"
|
|
# Return 97 rather than running the command when the wrapper is not the
|
|
# bounded form: a read that lost its ceiling must not pass as a read.
|
|
[ "${1:-}" = -k ] || return 97
|
|
shift 3
|
|
"$@" || command_rc=$?
|
|
if [ -n "${timeout_fail_node}" ]; then
|
|
case " $* " in
|
|
*"${timeout_fail_node}"*) return 124 ;;
|
|
esac
|
|
fi
|
|
return "${command_rc}"
|
|
}
|
|
|
|
assert_file_contains() {
|
|
local needle="$1"
|
|
local file="$2"
|
|
|
|
if [ ! -f "${file}" ]; then
|
|
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${needle}" >&2
|
|
return 1
|
|
fi
|
|
case "$(cat "${file}")" in
|
|
*"${needle}"*) return 0 ;;
|
|
esac
|
|
printf 'expected %s to contain: %s\n' "${file}" "${needle}" >&2
|
|
return 1
|
|
}
|
|
|
|
assert_file_lacks_pattern() {
|
|
local pattern="$1"
|
|
local file="$2"
|
|
|
|
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
|
|
# succeeds on an unreadable path, which is indistinguishable from "the file
|
|
# exists and does not carry the label". awk exits 2 on an unreadable path,
|
|
# which folded into an if reads the same way, so the file is checked first.
|
|
if [ ! -f "${file}" ]; then
|
|
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
|
|
return 1
|
|
fi
|
|
# Branched on awk's exact status, not on truthiness. awk exits 1 for "no line
|
|
# matched" and 2 for "I could not evaluate this" -- an unparseable regex, most
|
|
# of all. Folded into `if ... then fail; fi`, both land in the passing branch,
|
|
# so the assertion "this pattern is absent" is satisfied by the matcher giving
|
|
# up. That is the wrong direction for a negative assertion: a positive one
|
|
# fails loudly when its matcher breaks and gets fixed, this one goes green and
|
|
# stays green. Every claim in this file about something NOT appearing rests on
|
|
# the three-way split below.
|
|
# `|| _rc=$?` rather than a bare call: both runners set -e, and awk exiting 1
|
|
# for "no match" -- the ordinary success case here -- would otherwise abort the
|
|
# test before the branch below ever runs. The `if` form this replaces was
|
|
# exempt only because conditions are exempt.
|
|
local _rc=0
|
|
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
|
|
case "${_rc}" in
|
|
0)
|
|
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
|
|
return 1
|
|
;;
|
|
1) return 0 ;;
|
|
*)
|
|
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# COZY_REPORT_DIR is set inline in each test below rather than through a helper,
|
|
# which is the opposite of what the node-join suite does. The difference is
|
|
# deliberate and shrinks to one line: there the assignment carries a ten-line
|
|
# explanation that would otherwise stand in twenty copies, so the helper exists
|
|
# to hold the explanation, not the assignment. Here it is a bare one-liner with
|
|
# nothing to deduplicate, and a helper would buy indirection at no saving. The
|
|
# reason it must be set at all is the same in both files and lives there.
|
|
#
|
|
# The function under test redirects a command's stderr into a report file and
|
|
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
|
|
# the tracer's own line for that command lands on the very stderr being
|
|
# redirected, and every assertion about those files would be reading the
|
|
# tracer. Call through this wrapper so the sinks hold what production writes.
|
|
run_capture() {
|
|
local _rc=0
|
|
( set +x; cozy_capture_tenant_worker_cpu_throttle 1 ) || _rc=$?
|
|
return "${_rc}"
|
|
}
|
|
|
|
@test "the phase-start warning names this collector as the exception it is" {
|
|
# That warning is what an operator reads in the CI log when `timeout` is
|
|
# missing, and the sibling suite treats its collector list as load-bearing
|
|
# rather than decorative -- it says which collectors do still fail that way.
|
|
# This one calls `timeout` directly, which puts it in the named class, and
|
|
# then does NOT fail that way because it carries a fallback. Unnamed, the
|
|
# sentence sends the reader to the opposite conclusion about the one collector
|
|
# whose behaviour differs, and a comment in the source does not reach them.
|
|
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
# Anchored on the phrase unique to the phase warning itself: the bracket-pair
|
|
# captures write their own [bounds] notes carrying "timeout is not on PATH",
|
|
# so the first match of that phrase is no longer this sentence.
|
|
warn=$(grep -n 'run UNBOUNDED' "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$warn" ]; then
|
|
echo "expected the phase to still warn when timeout is missing" >&2
|
|
return 1
|
|
fi
|
|
line=$(sed -n "${warn}p" "$lib")
|
|
case "$line" in
|
|
*'exit 127 and collect nothing'*) ;;
|
|
*) echo "expected the warning to still name the 127 class it is contrasting against" >&2; return 1 ;;
|
|
esac
|
|
# The sentence went through false shapes before this one, including "except
|
|
# the worker CPU throttling capture", which read as a sole exception even
|
|
# though talos-image-cache has the same shape. An enumeration of a growing set
|
|
# is wrong the next time someone adds to it.
|
|
# What is pinned now is the DISCRIMINATOR: guarding the call with command -v
|
|
# is what separates the two outcomes, and every collector that guards is
|
|
# named. A fourth one added without a mention fails this.
|
|
case "$line" in
|
|
*'guard the call with command -v'*) ;;
|
|
*) echo "expected the warning to state what separates the two outcomes" >&2; return 1 ;;
|
|
esac
|
|
for c in 'CPU throttling' 'talos-image-cache'; do
|
|
case "$line" in
|
|
*"$c"*) ;;
|
|
*) echo "expected the warning to name the guarded collector: $c" >&2; return 1 ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
@test "a negative assertion fails when its own matcher cannot evaluate" {
|
|
# The helper below every "must not appear" claim in this file. Folded into an
|
|
# `if`, awk exiting 2 for an unparseable regex lands in the same branch as
|
|
# exiting 1 for "no match", so the assertion passes without having looked.
|
|
# What rides on it here are the claims that the artifact does NOT say
|
|
# something -- that a wedged kubelet was not written up as one that answered,
|
|
# that a capture was not labelled uncapped, that a log which was never
|
|
# written is not named. Those are the collector's own contract stated
|
|
# negatively, so a matcher that fails open turns each of them into a sentence
|
|
# nobody checked.
|
|
tmp=$(mktemp -d)
|
|
printf 'anything\n' >"$tmp/subject"
|
|
|
|
# A regex awk cannot parse. If the helper still reports "absent", it is
|
|
# reporting its own failure as a finding about the file.
|
|
if assert_file_lacks_pattern '*(' "$tmp/subject" 2>/dev/null; then
|
|
echo "FAIL: an unparseable pattern was reported as absent" >&2
|
|
rm -rf "$tmp"
|
|
return 1
|
|
fi
|
|
# And the two honest outcomes still work, or the guard above is bought by
|
|
# breaking the helper for everything.
|
|
assert_file_lacks_pattern 'nothinglikethis' "$tmp/subject"
|
|
if assert_file_lacks_pattern 'anything' "$tmp/subject" 2>/dev/null; then
|
|
echo "FAIL: a present pattern was reported as absent" >&2
|
|
rm -rf "$tmp"
|
|
return 1
|
|
fi
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the counters are read from the kubelet of the node the worker runs on" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# The endpoint is per node and the counters are per container, so the node
|
|
# name has to come from the Pod. Reading a fixed node, or the wrong one,
|
|
# returns a healthy stranger's counters.
|
|
assert_file_contains 'get --raw /api/v1/nodes/srv1/proxy/metrics/cadvisor' "$kubectl_calls"
|
|
# And the node name comes from the field this caller asks the listing for.
|
|
# That projection is a parameter of the shared listing, and the other caller
|
|
# of it passes the Pod name instead; swapped, this capture would ask kubelets
|
|
# named after Pods and get nothing, with no error to say why. The mock answers
|
|
# any listing regardless of what was asked for, so nothing else here sees it.
|
|
assert_file_contains 'jsonpath={range .items[*]}{.spec.nodeName}{"\n"}{end}' "$kubectl_calls"
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$capture"
|
|
# One assertion per alternative in the filter's regex, because a typo in an
|
|
# alternative no test drives through drops that family silently while every
|
|
# other assertion here stays green. The scheduled-periods count is the
|
|
# denominator the throttled count is read against, and the duration is what
|
|
# says whether a small ratio was a long stall.
|
|
assert_file_contains 'container_cpu_cfs_periods_total' "$capture"
|
|
assert_file_contains 'container_cpu_cfs_throttled_seconds_total' "$capture"
|
|
# The negative half of the labels. A complete capture that tells the reader
|
|
# its answer is missing, while the answer sits in the same file, is the same
|
|
# "capture misreads itself" harm as silence reported as an answer -- just
|
|
# inverted, and it fires on every healthy run rather than on the rare one.
|
|
assert_file_lacks_pattern 'incomplete' "$capture"
|
|
assert_file_lacks_pattern 'unknown' "$capture"
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
[ ! -f "$dir/srv1.read-error.log" ]
|
|
[ ! -f "$dir/READ-WARNINGS.txt" ]
|
|
[ ! -f "$dir/COLLECTION-FAILED.txt" ]
|
|
# The in-tree name the stream falls back to when mktemp fails. The live
|
|
# cleanup pin is "the raw stream is removed after each node", which follows
|
|
# the stream to TMPDIR; this one only says the fallback leaves nothing behind
|
|
# either, and it would go green on a run that never took the fallback.
|
|
[ ! -f "$dir/srv1.stream.tmp" ]
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "what the workers consumed is captured beside what they were denied" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# The throttled counters say the container met its ceiling. They cannot say
|
|
# how much CPU it got, and the two are different quantities: a guest held at a
|
|
# one-core ceiling and a guest scheduled onto a physical CPU for a tenth of
|
|
# that ceiling both report throttled periods, and the ratio alone does not
|
|
# separate them. The consumed seconds are what does, and cAdvisor publishes
|
|
# them on the same stream this capture already reads, so the answer costs a
|
|
# wider filter rather than another read.
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'container_cpu_usage_seconds_total' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "each read is bounded and keeps its exit code" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# An unbounded read here would hold the whole diagnostics block against a
|
|
# wedged kubelet, and the phase budget bounds when a read may START, never
|
|
# how long one already running may take.
|
|
# Named by their verbs, not by the bound: both reads carry the same one, so a
|
|
# bare '-k 5 20 kubectl' would be satisfied twice by whichever one survived
|
|
# and would not notice the other losing its ceiling.
|
|
# The selector travels with the listing. Without it the walk covers every Pod
|
|
# in the namespace, and the node set becomes every node running anything.
|
|
assert_file_contains '-k 5 20 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher' "$timeout_calls"
|
|
assert_file_contains '-k 5 20 kubectl get --raw' "$timeout_calls"
|
|
# Both bounds, not one. The wall-clock wrapper bounds the process and
|
|
# --request-timeout bounds the HTTP request underneath it; the second is the
|
|
# only one left when `timeout` is not on PATH, which is the case the fallback
|
|
# below exists for.
|
|
assert_file_contains '--request-timeout=20s' "$kubectl_calls"
|
|
assert_file_contains '[capture exit code: 0]' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the sample number decides which reading a file belongs to" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
( set +x; cozy_capture_tenant_worker_cpu_throttle 2 )
|
|
|
|
# Every counter here is cumulative since the container started, so one reading
|
|
# is an average over an uptime and the pair is what gives a rate over the
|
|
# window the deadline covered. A second reading written over the first leaves
|
|
# one file and no interval -- collected, and unable to answer the question it
|
|
# was collected for.
|
|
assert_file_contains 'container_cpu_usage_seconds_total' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-2/srv1.txt"
|
|
[ ! -d "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1" ]
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "each reading records when it was taken" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# The knob names how long the block WAITS between its two passes, which is not
|
|
# the interval a subject's two readings span: the other subject's pass falls
|
|
# between them, and on the run this collector exists for that pass is the slow
|
|
# part. A reader who divides by the advertised number is then wrong by
|
|
# whatever that pass cost, in the direction that understates a rate. The stamp
|
|
# is what makes the real interval readable off the pair.
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'read attempted from' "$capture"
|
|
# Both ends, not one. A single stamp leaves the sampling instant somewhere
|
|
# inside a read whose duration is what goes wrong on the run this collector
|
|
# exists for, so the interval a reader divides by would be off by up to the
|
|
# read bound with nothing in the artifact saying so.
|
|
stamp=$(sed -n 's/.*read attempted from \([0-9][0-9]*\) to \([0-9][0-9]*\) epoch seconds.*/\1 \2/p' "$capture")
|
|
if [ -z "$stamp" ]; then
|
|
echo "expected a bare epoch stamp in $capture" >&2
|
|
cat "$capture" >&2
|
|
return 1
|
|
fi
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the raw stream is removed after each node" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1 srv2"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# The stream is staged outside the report tree, so the assertion that used to
|
|
# catch a missing cleanup -- "no .stream.tmp in the report dir" -- cannot see
|
|
# it any more. Point TMPDIR at a directory of our own and check it is empty
|
|
# afterwards: mktemp honours TMPDIR, so this pins the cleanup wherever the
|
|
# file is staged, rather than pinning the path it used to have.
|
|
scratch="$tmp/scratch"
|
|
mkdir -p "$scratch"
|
|
|
|
TMPDIR="$scratch" run_capture
|
|
|
|
left=$(find "$scratch" -type f | wc -l | tr -d ' ')
|
|
if [ "$left" -ne 0 ]; then
|
|
echo "expected the raw metric stream to be removed; $left file(s) left in TMPDIR" >&2
|
|
find "$scratch" -type f >&2
|
|
return 1
|
|
fi
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a lowered read budget moves both of this collector's bounds" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# Assigned after sourcing, which is how both a caller and a test set it. A
|
|
# collector that took the knob's value at source time would keep 20 here.
|
|
#
|
|
# Not restored afterwards, and deliberately: run_capture calls the collector
|
|
# in an explicit subshell, and both runners give every @test a subshell of its
|
|
# own, so the assignment cannot outlive this body in either direction. A reset
|
|
# on the last line would be dead code that reads as load-bearing cleanup and
|
|
# teaches the next reader that state crosses tests here.
|
|
COZY_DIAG_READ_TIMEOUT=7
|
|
|
|
run_capture
|
|
|
|
assert_file_contains '-k 5 7 kubectl -n tenant-test get pods' "$timeout_calls"
|
|
assert_file_contains '-k 5 7 kubectl get --raw' "$timeout_calls"
|
|
assert_file_contains '--request-timeout=7s' "$kubectl_calls"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a zero read budget is corrected rather than disabling the bound" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# Zero is the value that makes revalidation load-bearing rather than tidy:
|
|
# `timeout -k 5 0` disables the timeout outright and `--request-timeout=0s`
|
|
# means no timeout to kubectl, so a zero assigned after sourcing restores the
|
|
# unbounded read on exactly the wedged-kubelet path the bound exists for.
|
|
COZY_DIAG_READ_TIMEOUT=0
|
|
|
|
run_capture
|
|
|
|
assert_file_contains '-k 5 20 kubectl get --raw' "$timeout_calls"
|
|
# Bracketed rather than backslash-escaped: awk warns on `\-`, and a warning
|
|
# nobody reads is how an unparseable pattern would arrive here.
|
|
assert_file_lacks_pattern '[-]k 5 0 ' "$timeout_calls"
|
|
assert_file_lacks_pattern 'request-timeout=0s' "$kubectl_calls"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the read's own status survives the filter that follows it" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# A kubelet that answered in full, whose body carries nothing for this
|
|
# namespace. Piped straight into grep the collector would report the read's
|
|
# status as grep's and call a healthy read a failure -- and, the other way
|
|
# round, would report a dead kubelet as a node with no tenant container.
|
|
kubectl_raw_output='container_cpu_cfs_periods_total{container="x",namespace="kube-system",pod="p"} 7'
|
|
kubectl_raw_rc=0
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains '[capture exit code: 0]' "$capture"
|
|
assert_file_contains 'the kubelet answered' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a node whose read times out does not stop the next node" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1 srv2"
|
|
timeout_fail_node=srv1
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-partial
|
|
|
|
run_capture
|
|
|
|
# One throttled worker answers the question, but only if the walk survives
|
|
# the node that did not answer -- and the wedged one is as likely as not to
|
|
# be first, since the node list is sorted.
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-partial/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains '[capture exit code: 124]' "$dir/srv1.txt"
|
|
assert_file_contains '[capture exit code: 0]' "$dir/srv2.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a warning on a successful listing is kept apart from a setup failure" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_list_stderr='Warning: v1 ComponentStatus is deprecated'
|
|
kubectl_list_rc=0
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# kubectl writes warnings on stderr with a zero exit, so the filename is
|
|
# decided by the status rather than by something having been written. Filed
|
|
# as a collection failure, a healthy listing reads as a collector that never
|
|
# ran at all.
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains 'ComponentStatus is deprecated' "$dir/READ-WARNINGS.txt"
|
|
[ ! -f "$dir/COLLECTION-FAILED.txt" ]
|
|
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$dir/srv1.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a pool below the cap leaves no truncation marker" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1 srv2"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-uncapped
|
|
|
|
run_capture
|
|
|
|
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-uncapped/tenant-cpu-throttle/sample-1/COLLECTION-TRUNCATED.txt" ]
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "repeated nodes are asked once" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
# Two workers co-scheduled on one node is the ordinary case, not a corner:
|
|
# the endpoint is per node and carries both, so asking twice spends a second
|
|
# read out of a budget that declines whatever has not started, and buys a
|
|
# byte-for-byte duplicate.
|
|
kubectl_node_names="srv1 srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
count=$(grep -c 'get --raw' "$kubectl_calls" || true)
|
|
if [ "$count" -ne 1 ]; then
|
|
echo "expected one kubelet read for one node, got $count" >&2
|
|
return 1
|
|
fi
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the collector's own caps stay what its comments claim" {
|
|
# Values, not sums. A comment stating a total has to be kept true in every
|
|
# copy of it and falsifies silently; a cap the code carries as a literal can
|
|
# be pinned against the code itself, which is what this does.
|
|
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
# The walk and the read moved into the body this capture now shares with the
|
|
# network counters, so the subject of these assertions is the code the
|
|
# collector runs rather than the function that carries its name -- which is
|
|
# now a six-line call. Reading only the named function would leave every
|
|
# assertion below vacuously true against a body it no longer contains.
|
|
fn=$(awk '/^cozy_capture_tenant_worker_cpu_throttle\(\)/,/^}/' "$lib"
|
|
awk '/^_cozy_capture_worker_cadvisor\(\)/,/^}/' "$lib"
|
|
awk '/^_cozy_cadvisor_node_stream\(\)/,/^}/' "$lib"
|
|
awk '/^_cozy_virt_launcher_listing\(\)/,/^}/' "$lib")
|
|
case "$fn" in
|
|
*'max_nodes=3'*) ;;
|
|
*) echo "expected the collector to cap the walk at the 3 nodes the comments state" >&2; return 1 ;;
|
|
esac
|
|
# The knob, not a literal that equals it today. A hardcoded 20 keeps its
|
|
# value when the block's budget is lowered, so the one read that ignores the
|
|
# knob is the one that outlives it -- and the sibling suite that lowers the
|
|
# knob is what caught exactly that here.
|
|
case "$fn" in
|
|
*'timeout -k "${COZY_DIAG_READ_GRACE}" "${COZY_DIAG_READ_TIMEOUT}"'*) ;;
|
|
*) echo "expected each kubelet read to take the block's read-budget knob" >&2; return 1 ;;
|
|
esac
|
|
case "$fn" in
|
|
*'timeout -k 5 20'*) echo "expected no hardcoded bound to survive beside the knob" >&2; return 1 ;;
|
|
esac
|
|
# Checked in the source as well as exercised, not instead of it. `command -v`
|
|
# finds the shell function this file defines for `timeout`, so the fallback arm
|
|
# cannot be reached from a test body -- but it can be reached from a subprocess
|
|
# with a stripped PATH, and a test further down does exactly that.
|
|
# The scratch stream must not live in the report tree. This is a source-level
|
|
# claim on purpose: the case it guards against is a hard kill between writing
|
|
# the stream and removing it, which no unit test can stage, so the only thing
|
|
# that can be pinned is where the path points.
|
|
#
|
|
# Matched against CODE, with comment lines stripped first. The prose here
|
|
# names both `mktemp` and the fallback path, so a pattern run over the whole
|
|
# function is satisfied by the explanation of the rule instead of by the rule
|
|
# -- deleting the assignment outright would leave such a check green.
|
|
code=$(printf '%s\n' "$fn" | grep -v '^[[:space:]]*#')
|
|
case "$code" in
|
|
*'stream=$(mktemp "'*) ;;
|
|
*) echo "expected the raw metric stream to be staged via mktemp with an explicit template" >&2; return 1 ;;
|
|
esac
|
|
# The template must be explicit: BSD mktemp with no template ignores TMPDIR
|
|
# and always lands in the system directory, so a bare call puts the file
|
|
# somewhere neither this suite nor an operator chose.
|
|
case "$code" in
|
|
*'mktemp "${TMPDIR:-/tmp}/'*) ;;
|
|
*) echo "expected the scratch template to honour TMPDIR explicitly" >&2; return 1 ;;
|
|
esac
|
|
# report_dir may appear only as the fallback, never as the primary target.
|
|
primary=$(printf '%s\n' "$code" | grep 'stream=' | grep -v '^[[:space:]]*||' | head -n 1)
|
|
case "$primary" in
|
|
*'${report_dir}'*) echo "the raw stream must not be staged inside the report tree" >&2; return 1 ;;
|
|
esac
|
|
# Both reads, not just the kubelet one: the listing has the same two arms and
|
|
# the same fallback, so counting only `get --raw` leaves the arm that matters
|
|
# for the listing unwatched for exactly the same reason.
|
|
# This loop is the cheap half of that pair: it holds every read that EXISTS to
|
|
# the flag. It cannot see a branch that was deleted, because a count over the
|
|
# reads still present balances either way -- that is what the stripped-PATH
|
|
# test buys, and why both are here.
|
|
for verb in 'kubectl get --raw' 'kubectl -n tenant-test get pods'; do
|
|
reads=$(printf '%s\n' "$fn" | grep -c "$verb" || true)
|
|
bounded=$(printf '%s\n' "$fn" | grep -A3 "$verb" | grep -c -- '--request-timeout=' || true)
|
|
if [ "$reads" -eq 0 ] || [ "$reads" -ne "$bounded" ]; then
|
|
echo "every '$verb' must carry --request-timeout; $bounded of $reads do" >&2
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
@test "both chainsaw comments list the collector where the phase actually spends it" {
|
|
# The ORDER is the claim, not any duration. Those comments describe the order
|
|
# the phase budget is spent in, and a budget that runs out declines whatever
|
|
# has not started -- so an entry in the wrong place misdescribes which
|
|
# collector survives a tight run. Pinning a minute figure instead would pin
|
|
# the one thing in that comment nobody can keep true.
|
|
for f in hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml \
|
|
hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml; do
|
|
reads=$(grep -n 'node-join diagnostics reads' "$f" | head -n 1 | cut -d: -f1)
|
|
cgroup=$(grep -n 'worker CPU usage and throttling counters' "$f" | head -n 1 | cut -d: -f1)
|
|
console=$(grep -n 'serial-console family' "$f" | head -n 1 | cut -d: -f1)
|
|
talos=$(grep -n 'guest Talos capture' "$f" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$reads" ] || [ -z "$cgroup" ] || [ -z "$console" ] || [ -z "$talos" ]; then
|
|
echo "expected $f to list the reads, the counter capture, the console family and the guest Talos capture" >&2
|
|
return 1
|
|
fi
|
|
# Below the console, not above it. This capture is read twice and can spend
|
|
# most of the phase budget, and admission gates only when a collector may
|
|
# start, so above the console it would bound what the console gets -- and
|
|
# the console is the only surface that survives a worker which never reached
|
|
# apid. Above the guest Talos capture, which needs the apid that failure
|
|
# never reached and is the more expensive of the two.
|
|
if [ "$cgroup" -le "$console" ] || [ "$cgroup" -ge "$talos" ]; then
|
|
echo "in $f the counter capture must sit between the console family and the guest Talos capture" >&2
|
|
return 1
|
|
fi
|
|
if [ "$console" -le "$reads" ]; then
|
|
echo "in $f the console family must sit after the node-join reads" >&2
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
@test "the capture is declined by the phase budget rather than running unconditionally" {
|
|
# Ungated, this collector would spend its budget after the phase had already
|
|
# declined the cheaper reads around it -- which is the failure the phase
|
|
# budget exists to prevent, reintroduced by the newest caller.
|
|
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
gate=$(grep -n "^ *if cozy_diag_phase_has_time '(d) tenant worker CPU counters, sandbox node CPU time and worker per-thread CPU time'; then$" \
|
|
"$lib" | head -n 1 | cut -d: -f1)
|
|
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$gate" ] || [ -z "$call" ]; then
|
|
echo "expected the capture to sit behind a cozy_diag_phase_has_time gate" >&2
|
|
return 1
|
|
fi
|
|
# THIS gate, not just some gate above the call. A line comparison alone is
|
|
# satisfied by any earlier gate in the block, which would leave the capture
|
|
# guarded by a decision taken about something else. The call now sits inside
|
|
# the sampling loop rather than on the line after the gate, so adjacency is no
|
|
# longer the way to say it: what says it is that no other gate opens in
|
|
# between.
|
|
next=$(awk -v g="$gate" 'NR > g && /cozy_diag_phase_has_time/ { print NR; exit }' "$lib")
|
|
if [ -z "$next" ]; then
|
|
echo "expected another phase gate after line $gate, so this test could bound the section" >&2
|
|
return 1
|
|
fi
|
|
if [ "$call" -le "$gate" ] || [ "$call" -ge "$next" ]; then
|
|
echo "the capture (line $call) must sit between its gate ($gate) and the next one ($next)" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
@test "the capture runs ahead of the collectors that cost more, and behind the one it could starve" {
|
|
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$call" ]; then
|
|
echo "expected the failure path to call the throttling capture" >&2
|
|
return 1
|
|
fi
|
|
# What runs last is what the budget declines, so position is what decides
|
|
# whether this collector is ever taken on a tight run. Its question has no
|
|
# other answer in this tree, so it precedes the legs below. Stated without a
|
|
# count on purpose: the loop below is the enumeration, and a sentence that
|
|
# also counted would go stale the first time a leg is added to it.
|
|
for leg in 'cozy_capture_tenant_talos "${test_name}" || true' \
|
|
'talos_image_cache_diagnose || true'; do
|
|
line=$(grep -n -F -x " ${leg}" "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$line" ]; then
|
|
echo "expected the failure block to still call: ${leg}" >&2
|
|
return 1
|
|
fi
|
|
if [ "$call" -ge "$line" ]; then
|
|
echo "the throttling capture (line $call) must run before ${leg} (line $line)" >&2
|
|
return 1
|
|
fi
|
|
done
|
|
# And behind the console. Reading the counters twice costs the walk twice
|
|
# plus the interval, which is most of the phase budget at the read bound and
|
|
# the node cap, and admission gates only when a collector may START. Ahead of
|
|
# the console, a run that hits those bounds would leave the console never
|
|
# started rather than cut short -- and the console is the only capture that
|
|
# survives a worker which never reached apid, the shape this failure usually
|
|
# takes. Cost, not worth, is what puts it here.
|
|
console=$(grep -n "cozy_capture_tenant_serial_console 'node-join failed" "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$console" ]; then
|
|
echo "expected the failure block to still capture the guest serial console" >&2
|
|
return 1
|
|
fi
|
|
if [ "$call" -le "$console" ]; then
|
|
echo "the throttling capture (line $call) must run after the console capture (line $console)" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
@test "the ceiling is captured beside the counters" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
# The throttled counters on their own say a container hit some ceiling, not
|
|
# which one. Without the quota beside them the reader cannot tell a VM capped
|
|
# at one core from one capped at eight, which is the whole question.
|
|
assert_file_contains 'container_spec_cpu_quota' "$capture"
|
|
# The period travels with the quota because the quota alone is meaningless:
|
|
# 100000us of quota is one core against a 100000us period and a tenth of one
|
|
# against a 1000000us period.
|
|
assert_file_contains 'container_spec_cpu_period' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the capture tells the reader how to pair it with its sibling sample" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# The instruction lives with the subject that has a sibling rather than in the
|
|
# walk both subjects share, because the other caller of that walk takes one
|
|
# sample and would otherwise carry an instruction pointing at a directory that
|
|
# is never created.
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'subtract this node file from its sibling under the other sample directory' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a read that did not finish is not told to pair with a sibling" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# Series arrived and the read still failed, which is a stream cut off part way
|
|
# through.
|
|
kubectl_raw_rc=124
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'these counters are incomplete' "$capture"
|
|
# Telling a reader to subtract two files asserts that both hold a whole
|
|
# reading. This one is already marked incomplete, and a difference taken
|
|
# against it understates the counter by whatever the read did not return --
|
|
# in the direction that makes a starved worker look busier than it was. The
|
|
# sandbox capture withholds its column legend from a short read for the same
|
|
# reason, so the pair answers this the same way on both sides.
|
|
assert_file_lacks_pattern 'subtract this node file from its sibling' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "an unquota'd container is reported as uncapped rather than as a gap" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# One line, and it is the whole shape: cAdvisor gates the quota series and all
|
|
# three CFS counters on the same non-zero quota, so an uncapped container puts
|
|
# the period on the wire and nothing else. Staging a CFS counter beside it
|
|
# would be a shape the endpoint cannot produce, and the test would then pass
|
|
# on a fixture rather than on the contract. It is an answer, not a short read:
|
|
# filed as "nothing was collected" it would send the reader looking for a
|
|
# ceiling that does not exist.
|
|
kubectl_raw_output='container_spec_cpu_period{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 100000'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'container_spec_cpu_period' "$capture"
|
|
assert_file_lacks_pattern 'unknown' "$capture"
|
|
assert_file_lacks_pattern 'incomplete' "$capture"
|
|
# Said in words, not left to inference. One line at exit 0 is also what a
|
|
# truncated read looks like, and every other outcome in this collector gets a
|
|
# sentence -- so the one answer it was built to deliver cannot be the one that
|
|
# gets none.
|
|
assert_file_contains 'running uncapped' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a quota probe that could not read says nothing about the ceiling" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# The last grep in the function is the one that decides whether the artifact
|
|
# makes a positive claim about the ceiling. grep exits 2 when it cannot read,
|
|
# and a bare `!` turns that into "no quota found", which is the direction of
|
|
# error this whole collector exists to refuse: a statement about the cluster
|
|
# derived from a read that failed on this runner.
|
|
grep() {
|
|
case "$*" in
|
|
*'^container_spec_cpu_quota{'*) return 2 ;;
|
|
esac
|
|
command grep "$@"
|
|
}
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_lacks_pattern 'running uncapped' "$capture"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a capped container is not labelled uncapped" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# The default fixture is the capped shape: every family, quota included.
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'container_spec_cpu_quota' "$capture"
|
|
assert_file_lacks_pattern 'running uncapped' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a read killed by the grace signal says so rather than leaving the code alone" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=137
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# 137 is what `timeout -k` leaves behind when the child ignored SIGTERM and
|
|
# had to be killed, by whatever did it. Naming every way a read comes up
|
|
# short is this collector's whole argument,
|
|
# so the one status that means "killed, not finished" cannot be the exception.
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'exit 137' "$capture"
|
|
assert_file_lacks_pattern 'exit 124' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a kubelet that was not read is recorded as unknown, not as an absence of throttling" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=1
|
|
kubectl_raw_output=
|
|
kubectl_raw_stderr='error: unable to connect to the server'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
# This is the failure this collector exists to prevent, inverted: an empty
|
|
# file here reads as a container that never hit its ceiling, which is the
|
|
# exact conclusion the capture was added to stop people reaching by default.
|
|
assert_file_contains 'unknown' "$capture"
|
|
# `unknown` alone is shared by both labels, so it pins the word and not the
|
|
# branch: disable this arm and an unread kubelet falls through to the next
|
|
# one, where the artifact tells the reader the kubelet ANSWERED and merely
|
|
# carried nothing. Pin the sentence that only this arm writes.
|
|
assert_file_contains 'the kubelet was not read' "$capture"
|
|
# And that it points at the log rather than leaving the reader to find it.
|
|
assert_file_contains 'see read-error.log' "$capture"
|
|
# The stamp claims only when the read was attempted. This arm has just said
|
|
# nothing was read, so a line asserting the counters were sampled inside the
|
|
# bracket contradicts the line above it -- the same two-lines-disagreeing
|
|
# failure the pairing note and the legend are both gated against.
|
|
assert_file_lacks_pattern 'counters were sampled' "$capture"
|
|
assert_file_contains 'read attempted from' "$capture"
|
|
assert_file_contains 'error: unable to connect to the server' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.read-error.log"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a forbidden nodes-proxy read is preserved rather than folded into silence" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=1
|
|
kubectl_raw_output=
|
|
# RBAC on nodes/proxy is a failure mode this read has and an in-container
|
|
# read did not. It is also the one an operator can fix, so the 403 has to
|
|
# reach the artifact verbatim instead of becoming a generic "not read".
|
|
kubectl_raw_stderr='Error from server (Forbidden): nodes "srv1" is forbidden: User "system:serviceaccount:x:y" cannot get resource "nodes/proxy"'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains 'the kubelet was not read' "$dir/srv1.txt"
|
|
assert_file_contains 'nodes/proxy' "$dir/srv1.read-error.log"
|
|
assert_file_contains 'Forbidden' "$dir/srv1.read-error.log"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a read killed without a word is not reported as a kubelet that answered" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# `timeout` kills its child and writes nothing, and kubectl dies on SIGTERM
|
|
# without a word, so this -- not the tidy error above -- is the shape a
|
|
# wedged kubelet actually produces. Keyed on whether stderr exists rather
|
|
# than on the status, the collector calls it an answer.
|
|
kubectl_raw_rc=124
|
|
kubectl_raw_output=
|
|
kubectl_raw_stderr=
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'the kubelet was not read' "$capture"
|
|
# The half that makes it a defect rather than a wording preference: the
|
|
# artifact must not state the opposite of what happened.
|
|
assert_file_lacks_pattern 'the kubelet answered' "$capture"
|
|
# No error log exists for a silent kill, so pointing at one sends the reader
|
|
# after evidence that was never written.
|
|
assert_file_lacks_pattern 'see read-error.log' "$capture"
|
|
assert_file_contains 'without a word' "$capture"
|
|
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.read-error.log" ]
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a stream cut short without a word is still marked incomplete" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# Counters arrived, the ceiling did not, and the read died silently. This is
|
|
# the worst shape in the set: the header states that counters without a quota
|
|
# mean a container running uncapped, so an unlabelled truncation does not
|
|
# merely lose information -- it reads as a positive finding about the ceiling.
|
|
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211'
|
|
kubectl_raw_rc=137
|
|
kubectl_raw_stderr=
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'incomplete' "$capture"
|
|
assert_file_lacks_pattern 'see read-error.log' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a filter that cannot read the stream blames neither the kubelet nor the namespace" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# grep exits 2 when it cannot read its input -- a full TMPDIR on the runner
|
|
# reaches this. The kubelet read itself succeeded, so the artifact must not
|
|
# say the kubelet answered and carried nothing (that is the conflation this
|
|
# collector exists to prevent), and must not say the kubelet was not read
|
|
# (it was). It has to name the local failure and nothing else.
|
|
grep() { case "$*" in *"-E "*) command grep "$@" >/dev/null 2>&1; return 2 ;; esac; command grep "$@"; }
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'could not be read back on this runner' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet answered' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a filter failure that said why points at the log it wrote" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# The half of this branch the sibling tests do not reach: they all silence the
|
|
# filter's stderr, so all of them land in the wordless variant and none drives
|
|
# the one that names the log. Both variants open with the same sentence, so a
|
|
# test asserting only that sentence passes on either.
|
|
grep() {
|
|
case "$*" in
|
|
*"-E "*) echo 'grep: input: Input/output error' >&2; return 2 ;;
|
|
esac
|
|
command grep "$@"
|
|
}
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains 'could not be read back on this runner' "$dir/srv1.txt"
|
|
assert_file_contains 'see filter-error.log' "$dir/srv1.txt"
|
|
assert_file_lacks_pattern 'the filter said nothing about why' "$dir/srv1.txt"
|
|
assert_file_contains 'Input/output error' "$dir/srv1.filter-error.log"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a filter failure never names a log that was not written" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# Fails silently: exit 2 with nothing on stderr. Naming a file that was never
|
|
# created sends the reader after evidence that does not exist -- the same rule
|
|
# every other message in this function follows.
|
|
grep() { case "$*" in *"-E "*) return 2 ;; esac; command grep "$@"; }
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains 'the filter said nothing about why' "$dir/srv1.txt"
|
|
assert_file_lacks_pattern 'see filter-error.log' "$dir/srv1.txt"
|
|
[ ! -f "$dir/srv1.filter-error.log" ]
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a namespace filter that cannot read its input is not reported as an answered kubelet" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# The same unreadable-input failure as the metric filter above, one stage
|
|
# later. Every stage of the filter narrows the same stream, so an exit 2 from
|
|
# any of them leaves the capture empty for a reason that has nothing to do
|
|
# with what the kubelet said.
|
|
grep() { case "$*" in *'namespace="tenant-test"'*) return 2 ;; esac; command grep "$@"; }
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'could not be read back on this runner' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet answered' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a worker filter that cannot read its input is not reported as an answered kubelet" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# A chain that pipes the stage above into this one drops that stage's status
|
|
# at the pipe and this one's at the trailing `|| true`, so neither reaches the
|
|
# artifact and both need their own test: one of them passing says nothing
|
|
# about the other.
|
|
grep() { case "$*" in *'pod="virt-launcher-'*) return 2 ;; esac; command grep "$@"; }
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'could not be read back on this runner' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet answered' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a filter that fails after writing leaves the capture marked incomplete" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
# A write error is where grep emits what it had flushed and then exits 2 --
|
|
# ENOSPC on a full disk reaches it, and so does the full TMPDIR this split was
|
|
# written for. The counters that arrive are real, so the capture is not
|
|
# unknown; it is short, and the family that goes missing first is the quota,
|
|
# whose absence reads as a container running uncapped.
|
|
grep() {
|
|
case "$*" in
|
|
*'pod="virt-launcher-'*)
|
|
command grep "$@" | head -n 1
|
|
echo 'grep: stdout: File too large' >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
command grep "$@"
|
|
}
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'incomplete' "$capture"
|
|
assert_file_contains 'see filter-error.log' "$capture"
|
|
# The read itself was fine, so nothing here may blame it.
|
|
assert_file_lacks_pattern 'the read was cut short' "$capture"
|
|
assert_file_lacks_pattern 'see read-error.log' "$capture"
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a filter that fails after writing never names a log that was not written" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
grep() {
|
|
case "$*" in
|
|
*'pod="virt-launcher-'*) command grep "$@" | head -n 1; return 2 ;;
|
|
esac
|
|
command grep "$@"
|
|
}
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
assert_file_contains 'incomplete' "$dir/srv1.txt"
|
|
assert_file_contains 'the filter said nothing about why' "$dir/srv1.txt"
|
|
assert_file_lacks_pattern 'see filter-error.log' "$dir/srv1.txt"
|
|
[ ! -f "$dir/srv1.filter-error.log" ]
|
|
unset -f grep
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a namespace whose containers are none of them workers names the worker filter" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=0
|
|
# Series for this namespace, none of them from a worker Pod -- what an
|
|
# upstream rename of the virt-launcher prefix looks like from here. Saying the
|
|
# kubelet reported nothing for the namespace would be false in this state and
|
|
# would send a reader after scheduling while the filter is what emptied the
|
|
# capture.
|
|
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="kube-apiserver",namespace="tenant-test",pod="kube-apiserver-a"} 7
|
|
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="RENAMED-launcher-worker-a"} 4211'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'unknown' "$capture"
|
|
assert_file_contains 'none of them from a worker Pod' "$capture"
|
|
assert_file_lacks_pattern 'reported no CPU series' "$capture"
|
|
assert_file_lacks_pattern 'the kubelet was not read' "$capture"
|
|
# The rejected series must not ride along either: naming the filter is not a
|
|
# licence to print what it rejected.
|
|
assert_file_lacks_pattern 'kube-apiserver' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a kubelet that answered without this namespace is not reported as unread" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=0
|
|
kubectl_raw_output=
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
# Distinct from the kubelet that never answered: this one did, and an
|
|
# artifact that blames the read would send an operator after RBAC and
|
|
# networking when the actual finding is that no worker container was there.
|
|
assert_file_contains 'unknown' "$capture"
|
|
assert_file_contains 'the kubelet answered' "$capture"
|
|
assert_file_lacks_pattern 'see read-error.log' "$capture"
|
|
# Both empty-capture arms open with "the kubelet answered", so the line above
|
|
# cannot tell them apart on its own. This one is about a node carrying nothing
|
|
# of the namespace, and the sibling arm's wording is what it must not be.
|
|
assert_file_contains 'no CPU series for a tenant-test worker' "$capture"
|
|
assert_file_lacks_pattern 'none of them from a worker Pod' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "with timeout off PATH the kubelet read still runs and keeps its request bound" {
|
|
# The fallback arm is what the phase-start warning promises for this collector:
|
|
# on a runner without the binary it runs UNBOUNDED rather than exiting 127, and
|
|
# `--request-timeout` is then the only ceiling left. A promise the code does not
|
|
# keep is what this whole collector is against, so it is entered here rather
|
|
# than inspected.
|
|
#
|
|
# A subprocess with a stripped PATH, because this file mocks `timeout` as a
|
|
# shell function and `command -v` finds functions -- the condition cannot be
|
|
# reached from inside a test body. Same device as the sibling suite uses for
|
|
# the block's own reads.
|
|
tmp=$(mktemp -d)
|
|
mkdir -p "$tmp/bin"
|
|
# Everything the collector shells out to, minus `timeout` itself. Missing one
|
|
# would fail this test for the wrong reason and read as the arm being broken.
|
|
for c in mkdir sort grep mv rm mktemp wc tr cat date; do
|
|
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin; do
|
|
if [ -x "$d/$c" ]; then
|
|
ln -sf "$d/$c" "$tmp/bin/$c"
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
for c in mkdir sort grep mv rm mktemp wc tr date; do
|
|
if [ ! -x "$tmp/bin/$c" ]; then
|
|
echo "FAIL: could not stage $c in the stripped PATH; the check below would be vacuous" >&2
|
|
return 1
|
|
fi
|
|
done
|
|
# Only the staged directory is checked. `command -v timeout` is always true
|
|
# here -- this file defines it as a shell function, which is the very fact the
|
|
# subprocess exists to escape -- so folding it in would read as if both halves
|
|
# mattered while only this one does.
|
|
if [ -e "$tmp/bin/timeout" ]; then
|
|
echo "FAIL: timeout leaked into the stripped PATH; this test would prove nothing" >&2
|
|
return 1
|
|
fi
|
|
|
|
out=$(bash -c '
|
|
set -eu
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
kubectl() {
|
|
# To a file, not to stderr. The collector redirects the stderr of every
|
|
# read into its own artifact, so a stub that announces itself there is
|
|
# captured by the code under test and never reaches this output.
|
|
printf "KUBECTL %s\n" "$*" >>"$CALLS"
|
|
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then printf "srv1\n"; return 0; fi
|
|
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then
|
|
printf "container_cpu_cfs_throttled_periods_total{container=\"compute\",namespace=\"tenant-test\",pod=\"virt-launcher-a\"} 7\n"
|
|
return 0
|
|
fi
|
|
return 0
|
|
}
|
|
CALLS='"$tmp"'/calls
|
|
COZY_REPORT_DIR='"$tmp"'/report
|
|
COZY_SNAPSHOT_NAME=fallback
|
|
PATH='"$tmp"'/bin
|
|
cozy_capture_tenant_worker_cpu_throttle 1
|
|
' 2>&1) || true
|
|
printf '%s\n' "$out" >"$tmp/out"
|
|
|
|
# The read ran at all -- bounded into exit 127 would leave nothing here.
|
|
assert_file_contains 'KUBECTL get --raw' "$tmp/calls"
|
|
# And carried the only bound it still has. This is the assertion the source
|
|
# check cannot make: it proves the flag reaches the arm that is actually taken.
|
|
assert_file_contains '--request-timeout=20s' "$tmp/calls"
|
|
# And produced counters rather than an "unknown" label.
|
|
assert_file_contains 'container_cpu_cfs_throttled_periods_total' \
|
|
"$tmp/report/snapshots/fallback/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a killed read is not written up as a grace period that never ran" {
|
|
tmp=$(mktemp -d)
|
|
mkdir -p "$tmp/bin"
|
|
for c in mkdir sort grep mv rm mktemp wc tr cat date; do
|
|
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin; do
|
|
if [ -x "$d/$c" ]; then
|
|
ln -sf "$d/$c" "$tmp/bin/$c"
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
if [ ! -x "$tmp/bin/grep" ]; then
|
|
echo "could not stage a PATH without timeout" >&2
|
|
return 1
|
|
fi
|
|
|
|
# 137 on the arm that has no `timeout` at all. The status is 128+SIGKILL and
|
|
# says only that something killed the read -- an OOM killer on a loaded
|
|
# runner, a teardown signalling the process group. On this arm the grace
|
|
# period is the one cause it cannot have been, because there is no `timeout`
|
|
# in the call. cozy_diag_read refuses the same wording for the same reason.
|
|
out=$(bash -c '
|
|
set -eu
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
kubectl() {
|
|
if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then printf "srv1\n"; return 0; fi
|
|
if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then return 137; fi
|
|
return 0
|
|
}
|
|
COZY_REPORT_DIR='"$tmp"'/report
|
|
COZY_SNAPSHOT_NAME=killed
|
|
PATH='"$tmp"'/bin
|
|
cozy_capture_tenant_worker_cpu_throttle 1
|
|
' 2>&1) || true
|
|
printf '%s\n' "$out" >"$tmp/out"
|
|
|
|
capture="$tmp/report/snapshots/killed/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'exit 137' "$capture"
|
|
assert_file_lacks_pattern 'grace period' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the tenant control plane is not reported as a worker" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# tenant-test is not a worker-only namespace. It also carries the Kamaji
|
|
# control plane -- the same file reads the talos-csr-signer sidecar out of it
|
|
# -- and the CDI importer Pods. Those containers carry their own CPU limits,
|
|
# so they have a real CFS quota and can show genuine throttling. Reported
|
|
# under a heading that says "worker", a throttled apiserver answers the
|
|
# question with the wrong subject, which is the harm the neighbouring test
|
|
# guards against from a different namespace -- and this direction is likelier.
|
|
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="kube-apiserver",namespace="tenant-test",pod="kubernetes-test-latest-version-abc"} 9999
|
|
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 4211'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'virt-launcher-worker-a-11111' "$capture"
|
|
assert_file_lacks_pattern 'kube-apiserver' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "another tenant's worker on the same node is not reported as ours" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# The node filter is per node, and a management node hosts whatever is
|
|
# scheduled on it. A virt-launcher belonging to a different tenant carries the
|
|
# same Pod-name prefix and the same container, so the Pod filter alone cannot
|
|
# separate them -- only the namespace can. Reported here it would be a
|
|
# stranger's throttling presented as this suite's worker, which is the same
|
|
# wrong-subject harm as the control plane, one namespace over.
|
|
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-other",pod="virt-launcher-elsewhere-99999"} 8888
|
|
container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 4211'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_contains 'virt-launcher-worker-a-11111' "$capture"
|
|
assert_file_lacks_pattern 'tenant-other' "$capture"
|
|
assert_file_lacks_pattern 'virt-launcher-elsewhere' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a series that merely contains a metric name does not pass the filter" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# An unanchored match would take any series whose name embeds one of the
|
|
# five, and the artifact would carry numbers that answer a different
|
|
# question while looking like the ones that answer this one.
|
|
# The Pod label has to match the worker filter, or the anchor is not what is
|
|
# under test: without it these lines are dropped for the wrong reason and the
|
|
# assertion holds whether the regex is anchored or not.
|
|
kubectl_raw_output='xcontainer_cpu_cfs_periods_total{namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 1
|
|
recorded_container_spec_cpu_quota{namespace="tenant-test",pod="virt-launcher-worker-a-11111"} 2'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
assert_file_lacks_pattern 'xcontainer' "$capture"
|
|
assert_file_lacks_pattern 'recorded_container' "$capture"
|
|
assert_file_contains 'the kubelet answered' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a read status cannot be read as the guest's own answer" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_rc=124
|
|
kubectl_raw_output=
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
capture="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/srv1.txt"
|
|
# `timeout` reports 124 on expiry and passes the command's own status through
|
|
# otherwise, so a 124 here is either this collector's deadline or the read
|
|
# exiting 124 by itself, and nothing separates them. The file states that
|
|
# rather than picking one.
|
|
assert_file_contains '124' "$capture"
|
|
assert_file_contains 'cannot be told apart' "$capture"
|
|
assert_file_contains '[capture exit code: 124]' "$capture"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the walk is capped and names both counts when it stops" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="n1 n2 n3 n4 n5 n6"
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
# This runs inside a failure path that already spends minutes on other
|
|
# collectors under one chainsaw op timeout. A short listing otherwise reads
|
|
# as a small cluster rather than a cut walk.
|
|
truncated="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-TRUNCATED.txt"
|
|
assert_file_contains '6 carried a worker in total' "$truncated"
|
|
# n4, not n6: the cap's VALUE is the claim, and the two chainsaw Test files
|
|
# order the failure path on it. Asserting some later node is absent leaves a
|
|
# larger cap green, which would silently overspend the budget.
|
|
[ -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/n3.txt" ]
|
|
[ ! -f "$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/n4.txt" ]
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a read cut off part way through is not presented as a complete answer" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
# The counters arrived; the stream died before the spec families that carry
|
|
# the ceiling. This is the one partial result that reads like a complete
|
|
# answer, because a missing quota otherwise means "uncapped".
|
|
kubectl_raw_output='container_cpu_cfs_throttled_periods_total{container="compute",namespace="tenant-test",pod="virt-launcher-a"} 4211'
|
|
kubectl_raw_rc=1
|
|
kubectl_raw_stderr='error: unexpected EOF'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
capture="$dir/srv1.txt"
|
|
assert_file_contains 'incomplete' "$capture"
|
|
assert_file_contains 'see read-error.log' "$capture"
|
|
assert_file_contains 'unexpected EOF' "$dir/srv1.read-error.log"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a namespace with no scheduled worker says so instead of leaving an empty tree" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names=
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture || true
|
|
|
|
# Distinct from the listing that never answered: this one got a real answer.
|
|
# An unscheduled virt-launcher has no node, so there is no kubelet to ask --
|
|
# which is itself a finding about the failure, not a gap in the collector.
|
|
assert_file_contains 'no virt-launcher Pod with a node assigned' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a warning beside a healthy read is not filed as a failure" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_node_names="srv1"
|
|
kubectl_raw_stderr='Warning: metrics endpoint is deprecated'
|
|
kubectl_raw_rc=0
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture
|
|
|
|
dir="$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1"
|
|
# kubectl writes deprecation and partial-result warnings on stderr with a
|
|
# zero exit, so the exit status decides the filename, not the fact that
|
|
# something was written. Filed as an error, a healthy read would look failed.
|
|
assert_file_contains 'deprecated' "$dir/srv1.READ-WARNINGS.txt"
|
|
[ ! -f "$dir/srv1.read-error.log" ]
|
|
assert_file_contains 'container_cpu_cfs_throttled_periods_total' "$dir/srv1.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "a Pod list that never returned is not reported as no workers" {
|
|
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
tmp=$(mktemp -d)
|
|
kubectl_calls="$tmp/kubectl.calls"
|
|
timeout_calls="$tmp/timeout.calls"
|
|
kubectl_list_rc=1
|
|
kubectl_list_stderr='Unable to connect to the server'
|
|
COZY_REPORT_DIR="$tmp/report"
|
|
COZY_SNAPSHOT_NAME=throttle-smoke
|
|
|
|
run_capture || true
|
|
|
|
# An empty listing is what a cluster with no virt-launcher Pods produces. A
|
|
# listing that never answered has to leave something else behind, and the
|
|
# status it is judged on has to be kubectl's rather than that of whatever
|
|
# post-processing follows it.
|
|
assert_file_contains 'Unable to connect to the server' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
|
|
assert_file_contains 'failed to list' \
|
|
"$COZY_REPORT_DIR/snapshots/throttle-smoke/tenant-cpu-throttle/sample-1/COLLECTION-FAILED.txt"
|
|
rm -rf "$tmp"
|
|
}
|
|
|
|
@test "the capture is wired into the node-join failure path" {
|
|
lib=hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
|
# The reporter is its own function now, so its declaration and its closing
|
|
# brace bound the region. Anchoring on the `exit 1` that follows would measure
|
|
# the caller instead: the exit sits outside this function, and every line of
|
|
# the reporter would count as inside any span reaching it.
|
|
head=$(grep -n '^cozy_report_node_join_failure() {$' "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$head" ]; then
|
|
echo "expected the node-join failure reporter to still be a function" >&2
|
|
return 1
|
|
fi
|
|
tail=$(awk -v h="$head" 'NR > h && $0 == "}" { print NR; exit }' "$lib")
|
|
call=$(grep -n '^ *cozy_capture_tenant_worker_cpu_throttle "${_sample}" || true$' "$lib" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$call" ] || [ -z "$tail" ]; then
|
|
echo "expected the node-join failure path to capture worker CPU throttling" >&2
|
|
return 1
|
|
fi
|
|
# A collector nothing calls is the shape this tree has shipped before: it
|
|
# reads as covered in review and produces nothing in the artifact. Pin the
|
|
# call site, not just the function.
|
|
if [ "$call" -le "$head" ] || [ "$call" -ge "$tail" ]; then
|
|
echo "the throttling capture (line $call) must sit inside the reporter ($head..$tail)" >&2
|
|
return 1
|
|
fi
|
|
# And that the reporter is still reached: a function nobody calls has the
|
|
# same artifact as a collector nobody calls, one level up.
|
|
grep -q '^ *cozy_report_node_join_failure "${test_name}"$' "$lib" || {
|
|
echo "expected run_kubernetes_test to still call the node-join reporter" >&2
|
|
return 1
|
|
}
|
|
}
|