Files
cozystack/hack/run-kubernetes-kvm-exits_test.bats
T
Myasnikov Daniil cc942fed68 test(tests): run unit suites with Bats and retain POSIX coverage
The shell translator rewrites test bodies and loses Bats reporting.
The unit lane needs complete TAP results and JUnit reports without
changing how live-cluster suites run.

Sourced production helpers still need a dash pass, and unit test bodies
need nounset without exporting shell options into subprocesses.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
2026-10-07 11:04:00 +03:00

1019 lines
40 KiB
Bash

#!/usr/bin/env bats
# Regression coverage for the sandbox kernel's KVM exit counters in
# hack/e2e-chainsaw/_lib/run-kubernetes.sh. cozytest.sh ends an @test block at
# the first bare closing brace, so command mocks stay at top level.
#
# What this capture answers, and why nothing beside it answers the same thing:
# every other CPU reading in this tree describes a subject inside the sandbox --
# a cgroup, a thread, a node's own /proc/stat -- and each of them can say a guest
# was given its ticks while the guest gets no work done. None of them can say
# what a tick cost. The tenant worker is a guest of a sandbox node, which is
# itself a guest of the runner, and the price of nesting is paid where the
# runner's kernel serves the sandbox VMs: exits, nested page faults and TLB
# flushes that no counter inside the sandbox can see. That kernel publishes them
# in debugfs, and the sandbox container shares it, so this is a local read.
#
# So the failure this suite guards against is not a wrong number; it is an empty
# directory, which reads as a kernel that paid nothing while nothing was ever
# asked.
load test_helper
timeout_calls=/dev/null
timeout_rc_override=
unshare_calls=/dev/null
unshare_rc_override=
mount_calls=/dev/null
mount_rc=0
mount_stderr=
timeout() {
local command_rc=0
printf '%s\n' "$*" >>"${timeout_calls}"
# Return 97 rather than running the command when the wrapper is not the
# bounded form: a read that lost its ceiling must not pass as a read.
[ "${1:-}" = -k ] || return 97
shift 3
"$@" || command_rc=$?
[ -z "${timeout_rc_override}" ] || return "${timeout_rc_override}"
return "${command_rc}"
}
# Mocked as a function rather than staged as a binary because the collector
# calls it in this shell. Its own flags are stripped and the rest is run, so the
# program the collector composed is the program this suite exercises -- the
# alternative, matching the program text here, would pass against a collector
# that had stopped composing it.
unshare() {
local command_rc=0
printf '%s\n' "$*" >>"${unshare_calls}"
while [ "$#" -gt 0 ]; do
case "$1" in
--mount | --propagation) shift ;;
private) shift ;;
*) break ;;
esac
done
[ -z "${unshare_rc_override}" ] || return "${unshare_rc_override}"
# The status is captured and returned rather than left as the last command's,
# which is the same shape the wrapper above uses and not a matter of taste
# here: cozytest.sh rewrites every closing brace in the first column into
# `return 0` followed by the brace, so a mock ending on a bare command has its
# status replaced by zero under that runner and kept under bats. This mock
# exists to carry a non-zero mount refusal outward, so losing it turns the two
# tests about a refused mount green against a collector that never saw one.
"$@" || command_rc=$?
return "${command_rc}"
}
assert_file_contains() {
local needle="$1"
local file="$2"
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${needle}" >&2
return 1
fi
case "$(cat "${file}")" in
*"${needle}"*) return 0 ;;
esac
printf 'expected %s to contain: %s\n' "${file}" "${needle}" >&2
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
# redirected, and every assertion about those files would be reading the
# tracer. Call through this wrapper so the sinks hold what production writes.
run_capture() {
local _rc=0
( set +x; cozy_capture_sandbox_kvm_exits "${1:-1}" ) || _rc=$?
return "${_rc}"
}
# `mount` is staged as an executable rather than mocked as a function: the
# collector runs it inside `sh -c`, a subprocess this shell's functions do not
# reach. A function here would be invisible exactly where the assertion needs
# it, and the walk would run against a mount that never happened without the
# suite noticing.
stage_mount_stub() {
local dir="$1"
mkdir -p "${dir}/bin"
cat >"${dir}/bin/mount" <<STUB
#!/bin/sh
printf '%s\n' "\$*" >>"${dir}/mount.calls"
[ -z "${mount_stderr}" ] || printf '%s\n' "${mount_stderr}" >&2
exit ${mount_rc}
STUB
chmod +x "${dir}/bin/mount"
mount_calls="${dir}/mount.calls"
PATH="${dir}/bin:${PATH}"
}
# A tree of files shaped like the kernel's, because staging a subject worth
# reading needs a hypervisor. Two global counters and one per-VM directory is
# enough to tell the two levels apart; the walk does not count them.
stage_debugfs() {
local root="$1"
mkdir -p "${root}/kvm/4242-13"
printf '1200\n' >"${root}/kvm/exits"
printf '640\n' >"${root}/kvm/pf_taken"
printf '31\n' >"${root}/kvm/tlb_flush"
printf '800\n' >"${root}/kvm/4242-13/exits"
printf '17\n' >"${root}/kvm/4242-13/halt_exits"
# The shape the kernel actually puts in every per-VM directory beside the
# counters, reproduced rather than imagined: mmu_rmaps_stat is a seq_file
# printing a tab-separated histogram table. Without it in the fixture nothing
# here is shaped like the directory the walk meets in production, and both
# skip paths are unreachable from this suite.
{
printf 'Rmap_Count:\t0\t1\t2-3\t4-7\n'
printf 'Level=4K:\t6291000\t400\t50\t6\n'
printf 'Level=2M:\t12200\t88\t0\t0\n'
printf 'Level=1G:\t24\t0\t0\t0\n'
} >"${root}/kvm/4242-13/mmu_rmaps_stat"
}
stage() {
COZY_REPORT_DIR="$1/report"
COZY_SNAPSHOT_NAME=kvm-smoke
COZY_DIAG_KVM_DEBUGFS="$1/debugfs"
stage_debugfs "$1/debugfs"
stage_mount_stub "$1"
}
capture_path() {
printf '%s/snapshots/kvm-smoke/sandbox-kvm-exits/sample-%s/kvm-stats.txt' \
"${COZY_REPORT_DIR}" "${1:-1}"
}
marker_path() {
printf '%s/snapshots/kvm-smoke/sandbox-kvm-exits/sample-%s/COLLECTION-FAILED.txt' \
"${COZY_REPORT_DIR}" "${1:-1}"
}
@test "the kernel's own KVM counters are read through a debugfs of this capture's own" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
# The mount is what makes the counters reachable at all, and the namespace is
# what keeps it from being a change to the sandbox: the suite runs on for
# another twenty minutes after this, and a mount left behind would be this
# collector editing the environment the rest of the run measures.
assert_file_contains '--mount --propagation private' "$unshare_calls"
assert_file_contains '-t debugfs' "$mount_calls"
capture=$(capture_path)
assert_file_contains 'exits 1200' "$capture"
assert_file_contains 'tlb_flush 31' "$capture"
# What the walk found, counted at both levels and over the directories it
# opened. Three numbers rather than one because they answer different
# questions -- a kernel hosting no VM has counters and no directories, a VM
# that went away under the walk leaves a directory and no counters -- and
# without this the directory count and the guard that produces it are carried
# by nothing.
assert_file_contains 'counters read: 3 kernel-wide and 2 per-VM, across 1 per-VM director' "$capture"
assert_file_lacks_pattern 'unknown' "$capture"
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'unavailable' "$capture"
rm -rf "$tmp"
}
@test "a file the kernel computes on read is skipped by name and never opened" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
# mmu_rmaps_stat is not a counter and, unlike every other file here, reading
# it is not free: it is registered with single_open and seq_read, so its
# contents are computed on the first read, under the VM slots_lock and MMU
# write lock, walking every rmap head. A shape test cannot save that -- to see
# the shape you must read it, and the read is the cost -- so it is skipped by
# name before anything opens it. The two skip reasons are worded differently
# on purpose: this assertion fails, rather than silently passing on the shape
# path, if the name check is ever dropped.
capture=$(capture_path)
assert_file_contains '[skipped vm 4242-13] mmu_rmaps_stat: not opened' "$capture"
assert_file_lacks_pattern 'Rmap_Count' "$capture"
assert_file_lacks_pattern 'Level=4K' "$capture"
# And it is not counted as a per-VM counter either.
assert_file_contains 'counters read: 3 kernel-wide and 2 per-VM' "$capture"
rm -rf "$tmp"
}
@test "a file that is not a counter at all is skipped by its shape" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The general net behind the name list. A counter is one integer and nothing
# else; anything else in these directories would otherwise be emitted under a
# heading promising `level name value`, counted as a counter, and then offered
# to the reader as something to difference against its sibling.
{
printf 'header one two\n'
printf 'row 1 2\n'
} >"$tmp/debugfs/kvm/some_table"
printf 'not-a-number\n' >"$tmp/debugfs/kvm/some_text"
# A number followed by more of anything: the leading integer alone would
# pass, so this pins that the shape check reads the whole content rather than
# the first line. Without this fixture a check narrowed to the first line
# stays green, and a table whose first row happens to be a bare number would
# be filed as a counter.
{
printf '42\n'
printf 'and then some\n'
} >"$tmp/debugfs/kvm/some_prefixed_table"
run_capture
capture=$(capture_path)
assert_file_contains '[skipped global] some_table: not a single integer' "$capture"
assert_file_contains '[skipped global] some_text: not a single integer' "$capture"
assert_file_contains '[skipped global] some_prefixed_table: not a single integer' "$capture"
assert_file_lacks_pattern 'header one two' "$capture"
assert_file_lacks_pattern 'not-a-number' "$capture"
assert_file_lacks_pattern 'some_prefixed_table 42' "$capture"
# Still three kernel-wide counters: neither of the two joined them.
assert_file_contains 'counters read: 3 kernel-wide' "$capture"
rm -rf "$tmp"
}
@test "a file that cannot be read is skipped as unread, not called a non-counter" {
[ "$(id -u)" -ne 0 ] || skip "mode bits do not stop root from opening the file"
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# In production the failure is a live counter whose VM is being torn down:
# its entry is still listed while the kernel refuses the open, or refuses
# the read once the entry is detached. Neither state can be staged with a
# plain file, so an unreadable mode stands in; what the walk sees is the
# same, a listed file that yields no content.
printf '99\n' >"$tmp/debugfs/kvm/locked_out"
chmod 000 "$tmp/debugfs/kvm/locked_out"
run_capture
capture=$(capture_path)
assert_file_contains '[skipped global] locked_out: could not be read' "$capture"
assert_file_lacks_pattern 'locked_out: not a single integer' "$capture"
# Not read means not counted: still the three counters the fixture stages.
assert_file_contains 'counters read: 3 kernel-wide' "$capture"
chmod 700 "$tmp/debugfs/kvm/locked_out"
rm -rf "$tmp"
}
@test "the per-VM split is kept apart from the kernel-wide totals" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
# Both levels exist in debugfs and they answer different questions: the totals
# say what this kernel paid, the per-VM directories say whether the three
# sandbox VMs paid it evenly. Folded together, a single VM burning exits looks
# like a kernel under uniform load, which is the reading this split exists to
# refuse.
capture=$(capture_path)
assert_file_contains '[vm 4242-13] exits 800' "$capture"
assert_file_contains '[global] exits 1200' "$capture"
rm -rf "$tmp"
}
@test "the counter legend travels with the numbers" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
capture=$(capture_path)
# Which layer these numbers describe. The counters belong to the kernel
# hosting the sandbox VMs, so they price the sandbox's own execution and say
# nothing about what a tenant worker's guest did to its sandbox node -- and a
# reader who takes them for the second draws the opposite conclusion.
assert_file_contains 'the sandbox VMs and not what runs inside them' "$capture"
# Which of them a difference may be taken of. Subtracting an instantaneous
# gauge from its sibling reading yields a number with no meaning, and the
# names sit next to the cumulative ones in the same directory.
# Both of the vCPU-state gauges, not just the first. They sit in the same
# directory as everything else and under the same kind of name, and a list
# that names one of them reads as the closed set of things not to difference.
assert_file_contains 'guest_mode' "$capture"
assert_file_contains 'blocking' "$capture"
assert_file_contains 'cumulative' "$capture"
# The counter a reader will look for and not find, named where they look.
assert_file_contains 'no ept_violation counter' "$capture"
# Why the per-VM directories cannot be attributed to a sandbox node.
assert_file_contains 'cannot map' "$capture"
# The pairing instruction rides with the numbers on the one arm that holds a
# whole reading, as it does for the node CPU time capture beside it.
assert_file_contains 'subtracting this file from its sibling' "$capture"
rm -rf "$tmp"
}
@test "each reading records when it was taken" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
# This pair is not taken inside one diagnostics block: the first reading is
# taken before the node-join wait and the second only when that wait has run
# out, so the interval between them is the whole join window rather than a
# knob. Nothing else in the artifact records it.
capture=$(capture_path)
assert_file_contains 'read attempted from' "$capture"
stamp=$(sed -n 's/.*read attempted from \([0-9][0-9]*\) to \([0-9][0-9]*\) epoch seconds.*/\1 \2/p' "$capture")
if [ -z "$stamp" ]; then
echo "expected a bare epoch stamp in $capture" >&2
cat "$capture" >&2
return 1
fi
rm -rf "$tmp"
}
@test "the read is bounded and keeps its exit code" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture
# Local does not mean instant: a mount and a walk over debugfs both enter the
# kernel, and this one runs on the failure path of a host already wedged
# enough to lose a node join.
assert_file_contains '-k 5 20 unshare' "$timeout_calls"
assert_file_contains '[capture exit code: 0]' "$(capture_path)"
rm -rf "$tmp"
}
@test "a lowered read budget moves this collector's bound" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# Set after sourcing, which is how a caller and a test both set it. A value
# read only at assignment time would reach `timeout` unchecked here.
COZY_DIAG_READ_TIMEOUT=4
run_capture
assert_file_contains '-k 5 4 unshare' "$timeout_calls"
rm -rf "$tmp"
}
@test "a zero read budget is corrected rather than disabling the bound" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# `timeout -k 5 0` disables the timeout outright, so a zero assigned after
# sourcing would restore the unbounded read this bound exists to remove.
COZY_DIAG_READ_TIMEOUT=0
run_capture
assert_file_lacks_pattern '^-k 5 0 ' "$timeout_calls"
assert_file_contains "-k 5 $COZY_DIAG_READ_TIMEOUT_DEFAULT unshare" "$timeout_calls"
rm -rf "$tmp"
}
@test "no unshare on PATH is recorded rather than left as an empty directory" {
# A subprocess with a stripped PATH, because this file mocks `unshare` as a
# shell function and `command -v` finds functions -- the arm cannot be reached
# from inside a test body. The PATH is staged rather than emptied: the
# collector shells out before it gets to the check, so an empty PATH would
# fail this for the wrong reason and read as the arm being broken.
tmp=$(mktemp -d)
mkdir -p "$tmp/bin"
# /sbin and /usr/sbin are in the list because `mount` lives there on the host
# this suite is also run from, and a staging loop that cannot find it fails
# the test rather than quietly leaving the check to run against a PATH that
# is missing more than the one command it is about.
for c in mkdir grep rm mount date; do
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin /sbin /usr/sbin; do
if [ -x "$d/$c" ]; then
ln -sf "$d/$c" "$tmp/bin/$c"
break
fi
done
if [ ! -x "$tmp/bin/$c" ]; then
echo "FAIL: could not stage $c in the stripped PATH; the check below would be vacuous" >&2
return 1
fi
done
if [ -e "$tmp/bin/unshare" ]; then
echo "FAIL: unshare leaked into the stripped PATH; this test would prove nothing" >&2
return 1
fi
rc=0
# Assigned inside the subprocess rather than in front of it: bash resolves the
# command name with the PATH the assignment sets, so `PATH=... bash` cannot
# find bash itself.
out=$(bash -c '
set -eu
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
COZY_REPORT_DIR='"$tmp"'/report
COZY_SNAPSHOT_NAME=kvm-smoke
PATH='"$tmp"'/bin
cozy_capture_sandbox_kvm_exits 1
' 2>&1) || rc=$?
printf '%s\n' "$out" >"$tmp/out"
[ "$rc" -ne 0 ]
marker="$tmp/report/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1/COLLECTION-FAILED.txt"
assert_file_contains 'unshare is not on PATH' "$marker"
# The distinction the marker exists for, spelled out in the artifact: a tool
# that was never run says nothing about the kernel, and the reading a missing
# file invites is that nesting cost it nothing.
assert_file_contains 'not a reading that nesting cost it nothing' "$marker"
rm -rf "$tmp"
}
@test "no mount on PATH is recorded rather than left as an empty directory" {
# The sibling of the unshare check above. Staged the same way and for the same
# reason, with one difference: `unshare` has to be present for the collector
# to reach the check this test is about, and the host running this suite may
# have no such binary, so a stub stands in. It is never executed -- the
# collector returns at the check below it.
tmp=$(mktemp -d)
mkdir -p "$tmp/bin"
for c in mkdir grep rm mv date; do
for d in /bin /usr/bin /usr/local/bin /opt/homebrew/bin /sbin /usr/sbin; do
if [ -x "$d/$c" ]; then
ln -sf "$d/$c" "$tmp/bin/$c"
break
fi
done
if [ ! -x "$tmp/bin/$c" ]; then
echo "FAIL: could not stage $c in the stripped PATH; the check below would be vacuous" >&2
return 1
fi
done
printf '#!/bin/sh\nexit 91\n' >"$tmp/bin/unshare"
chmod +x "$tmp/bin/unshare"
if [ -e "$tmp/bin/mount" ]; then
echo "FAIL: mount leaked into the stripped PATH; this test would prove nothing" >&2
return 1
fi
rc=0
out=$(bash -c '
set -eu
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
COZY_REPORT_DIR='"$tmp"'/report
COZY_SNAPSHOT_NAME=kvm-smoke
PATH='"$tmp"'/bin
cozy_capture_sandbox_kvm_exits 1
' 2>&1) || rc=$?
printf '%s\n' "$out" >"$tmp/out"
[ "$rc" -ne 0 ]
marker="$tmp/report/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1/COLLECTION-FAILED.txt"
assert_file_contains 'mount is not on PATH' "$marker"
assert_file_contains 'not a reading that nesting cost it nothing' "$marker"
rm -rf "$tmp"
}
@test "a mount that was refused is not reported as a kernel running no guests" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_rc=32
mount_stderr='mount: /sys/kernel/debug: permission denied.'
stage "$tmp"
rc=0
run_capture || rc=$?
# The two readings this has to keep apart: a sandbox that would not let the
# capture at debugfs, and a kernel whose KVM is idle. Both leave no counters,
# and only the second is a statement about the run.
[ "$rc" -ne 0 ]
marker=$(marker_path)
assert_file_contains 'permission denied' "$marker"
assert_file_contains 'debugfs could not be mounted' "$marker"
assert_file_lacks_pattern 'no KVM' "$marker"
rm -rf "$tmp"
}
@test "a kernel with no kvm directory is a finding rather than an empty capture" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
# A hint on stderr beside the finding, so the arm that routes stderr by status
# is exercised for this code and not only for its sibling. The kernel answered
# here, so the complaint belongs in a file named as a warning; routed with the
# failures it would put read-error.log next to a capture that did not fail.
mount_stderr='mount: (hint) debugfs is already mounted here.'
stage "$tmp"
rm -rf "$tmp/debugfs/kvm"
rc=0
run_capture || rc=$?
# Answered to the caller and not only to the report, because one of the two
# callers runs on the passing path, whose report is the artifact nobody
# downloads, and this is the most consequential thing the instrument can say.
[ "$rc" -ne 0 ]
# Not a shortfall of the capture. The sandbox VMs are started with
# accel=kvm by hack/e2e-prepare-cluster.bats, so a kernel with no kvm
# directory under debugfs is a kernel that ran them some other way -- which
# would explain the slowness this whole line of measurement is chasing, and
# must not be filed as a collector that came up short.
capture=$(capture_path)
assert_file_contains 'NO-KVM-DIR' "$capture"
assert_file_contains 'accel=kvm' "$capture"
# And the legend must not ride on it. A file carrying a finding and no
# counters cannot be differenced against its sibling, and an instruction to do
# so would turn the sibling's whole cumulative total into a rate over the
# window -- the largest number this artifact could possibly produce, arrived
# at by following its own advice.
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
# And a finding is not also a truncated walk, nor a read that came back with
# nothing. All three sentences fit in this file and they contradict each
# other: one says the kernel answered and had no KVM, the second says the walk
# was cut off before it could tell, the third says the read never got that
# far. Only the first is true here, and the other two describe an instrument
# that failed rather than one that has something to report.
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'unavailable' "$capture"
# The probe's own status, read out of the capture rather than inferred from
# what the legend did or did not say. The collector decides four things from
# this code -- which file stderr lands in, whether the legend rides, whether
# the pairing instruction rides, what it returns -- so a probe that stopped
# emitting it would change all four, and a test that only watches the effects
# can be made blind by any one guard added upstream of them.
assert_file_contains '[capture exit code: 4]' "$capture"
dir="$COZY_REPORT_DIR/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1"
assert_file_contains 'debugfs is already mounted here' "$dir/READ-WARNINGS.txt"
[ ! -f "$dir/read-error.log" ]
rm -rf "$tmp"
}
@test "a kvm directory holding only per-VM counters is not reported as holding none" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The kernel-wide files gone and one VM directory left. Both levels are
# optional in their own way -- a kernel hosting no VM has no per-VM
# directories -- so the emptiness test has to be about the two together. Read
# as either-or it reports a capture that holds counters as one that holds
# none, and the arm that says so is the one a reader trusts over the numbers
# printed above it.
rm -f "$tmp"/debugfs/kvm/exits "$tmp"/debugfs/kvm/pf_taken "$tmp"/debugfs/kvm/tlb_flush
run_capture
capture=$(capture_path)
assert_file_contains '[vm 4242-13] exits 800' "$capture"
assert_file_lacks_pattern 'NO-COUNTER-FILES' "$capture"
rm -rf "$tmp"
}
@test "a kvm directory holding no counter is not filed as a reading" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm"
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
capture=$(capture_path)
assert_file_contains 'NO-COUNTER-FILES' "$capture"
# And the legend must not ride on it: telling a reader to subtract two files
# asserts that both hold a whole reading, and this one holds none.
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'unavailable' "$capture"
rm -rf "$tmp"
}
@test "a warning beside a finding is filed as a warning, not as a failed read" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_stderr='mount: (hint) debugfs is already mounted here.'
stage "$tmp"
# A kernel that answered and had no counters, while mount also wrote a hint.
# The status decides which file the complaint lands in, and the two findings
# are on the side that succeeded: routing them with the failures would put a
# file named read-error.log next to a capture that reports what the kernel
# said, which is the failure-shaped name this routing exists to keep off a
# reading that did not fail.
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm"
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
dir="$COZY_REPORT_DIR/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1"
assert_file_contains 'NO-COUNTER-FILES' "$dir/kvm-stats.txt"
assert_file_contains '[capture exit code: 5]' "$dir/kvm-stats.txt"
assert_file_contains 'debugfs is already mounted here' "$dir/READ-WARNINGS.txt"
[ ! -f "$dir/read-error.log" ]
rm -rf "$tmp"
}
@test "a capture holding no counter carries no legend about how to read counters" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The legend rides on any capture that HOLDS numbers, which is wider than the
# captures that hold a whole reading -- and the widening has a floor. Every
# line of it describes numbers: which layer they belong to, which of them may
# be differenced, what the per-VM tag does and does not identify. On a file
# with no numbers the most consequential of those lines invites a reader to
# difference a counter that is not there, which is the sibling's entire
# cumulative total read as a rate over the window.
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm"
run_capture || true
capture=$(capture_path)
assert_file_lacks_pattern 'a difference against the sibling sample' "$capture"
assert_file_lacks_pattern 'no ept_violation counter' "$capture"
assert_file_lacks_pattern 'per-VM and anonymous' "$capture"
rm -rf "$tmp"
}
@test "a kvm directory holding only kernel-wide counters is not reported as holding none" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The mirror of the per-VM case beside this one, and the production shape
# whenever the sandbox VMs are down: the kernel keeps its own totals and hosts
# no VM, so there is no per-VM directory at all. The emptiness test is about
# the two levels together, and read as either-or this direction reports a
# capture holding three counters as one holding none.
rm -rf "$tmp/debugfs/kvm/4242-13"
run_capture
capture=$(capture_path)
assert_file_contains '[global] exits 1200' "$capture"
assert_file_lacks_pattern 'NO-COUNTER-FILES' "$capture"
rm -rf "$tmp"
}
@test "a counter written without a trailing newline is read rather than passed over" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# `read` returns non-zero on a last line with no newline while still setting
# the variable, so a walk that trusted its status would drop the value it had
# just read. The kernel formats its own stat files through "%llu\n" and so
# does not produce this shape, which is exactly why nothing else here would
# exercise the branch that handles it.
# One at each level, because the fallback is written twice -- once per walk --
# and a fixture at one level leaves the other's copy free to be deleted with
# the suite still green.
printf '7' >"$tmp/debugfs/kvm/nested_run"
printf '5' >"$tmp/debugfs/kvm/4242-13/irq_exits"
run_capture
assert_file_contains '[global] nested_run 7' "$(capture_path)"
assert_file_contains '[vm 4242-13] irq_exits 5' "$(capture_path)"
rm -rf "$tmp"
}
@test "a per-VM directory holding no readable counter is not filed as a reading" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# A VM directory whose counters cannot be read is the shape a VM going away
# under the walk produces, and it is indistinguishable on disk from a VM that
# reported nothing. Counting directories rather than counters lets it satisfy
# the emptiness test: the capture then holds no number and still carries the
# instruction to difference it against its sibling, which would turn that
# sibling's whole cumulative total into a rate over the window.
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm/4242-13"
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
capture=$(capture_path)
assert_file_contains 'NO-COUNTER-FILES' "$capture"
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'unavailable' "$capture"
rm -rf "$tmp"
}
@test "a read killed before it reached a counter is not presented as a partial reading" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The walk writes a heading before it reads anything, so a read killed on its
# first blocked file leaves a file that is not empty and holds no number. That
# shape is staged here by walking a kvm directory with nothing in it and
# forcing the wrapper's status, since what the arm has to get right is the
# file, not how it came to look that way. Keyed on file size rather than on a
# counter, the capture would be labelled partial and the caller told a pair
# was collected.
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm"
timeout_rc_override=124
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
capture=$(capture_path)
assert_file_contains 'no counter at all' "$capture"
assert_file_lacks_pattern 'incomplete' "$capture"
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
rm -rf "$tmp"
}
@test "a read that returned nothing is reported to the caller, not only to the report" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# Killed before it produced a byte: the dominant shape of a bounded read that
# ran out. Both callers ask this function whether it collected anything, and
# one of them runs on the passing path where the report is the artifact nobody
# downloads -- so a capture that got nothing has to answer non-zero or the
# warning that says the pair is broken never prints.
unshare_rc_override=124
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
capture=$(capture_path)
assert_file_contains 'produced no counter at all' "$capture"
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
rm -rf "$tmp"
}
@test "a partial read is not reported to the caller as nothing collected" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The other half of the pair above, and the reason the status is decided by
# what reached the file rather than by the exit code alone: a walk cut off
# after it wrote counters left a usable capture, and a caller told that
# nothing was collected would contradict the artifact sitting beside it.
timeout_rc_override=124
rc=0
run_capture || rc=$?
[ "$rc" -eq 0 ]
assert_file_contains 'incomplete' "$(capture_path)"
rm -rf "$tmp"
}
@test "a capture that failed points at the error log written beside it" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_stderr='mount: (hint) debugfs is already mounted here.'
stage "$tmp"
# A read that was cut short while something was also written to stderr: the
# message belongs beside the capture rather than in the job log of a run that
# may be days old, and an arm that reports the shortfall without saying where
# the message went sends the reader looking there anyway. The node CPU time
# capture beside this one names its own error file for the same reason.
timeout_rc_override=124
run_capture || true
dir="$COZY_REPORT_DIR/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1"
assert_file_contains 'read-error.log' "$dir/kvm-stats.txt"
assert_file_contains 'debugfs is already mounted here' "$dir/read-error.log"
rm -rf "$tmp"
}
@test "a read that returned no counter but left a message points at where it went" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_stderr='mount: (hint) debugfs is already mounted here.'
stage "$tmp"
# The fourth corner of the pair of questions the arms answer: no counter
# reached the file, and something was written to stderr. The three other
# corners are staged elsewhere in this file, and without this one the arm that
# names the error log can be deleted with the suite still green -- leaving a
# reader told the capture is unavailable and not told that the reason is
# sitting in the file next to it.
rm -rf "$tmp/debugfs/kvm"
mkdir -p "$tmp/debugfs/kvm"
timeout_rc_override=124
rc=0
run_capture || rc=$?
[ "$rc" -ne 0 ]
capture=$(capture_path)
assert_file_contains 'no counter at all' "$capture"
assert_file_contains 'read-error.log' "$capture"
rm -rf "$tmp"
}
@test "a mount that refused without a word does not point at a message that is not there" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_rc=32
mount_stderr=
stage "$tmp"
rc=0
run_capture || rc=$?
# mount almost always writes a reason, which is why the marker was written to
# point at one. When it does not, the marker is a single line telling the
# reader to look above it at nothing, and a reader who follows that concludes
# the artifact lost the message rather than that there never was one.
[ "$rc" -ne 0 ]
marker=$(marker_path)
assert_file_contains 'without a word on either stream' "$marker"
assert_file_lacks_pattern 'the message above' "$marker"
rm -rf "$tmp"
}
@test "a read cut off part way is marked incomplete rather than presented whole" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# The walk wrote counters and then the wrapper killed it: the dominant shape
# of a bounded read that ran out, and the one where a whole-looking file is
# most misleading, since the difference against its sibling would understate
# every counter the walk never reached.
timeout_rc_override=124
run_capture || true
capture=$(capture_path)
assert_file_contains 'exits 1200' "$capture"
assert_file_contains 'incomplete' "$capture"
assert_file_lacks_pattern 'subtracting this file from its sibling' "$capture"
assert_file_contains '[capture exit code: 124]' "$capture"
# The lines that say how to read the numbers stay, because they do not depend
# on the walk having finished and this is exactly where a reader needs them:
# the counters above are real, and the naming trap and the anonymity of the
# per-VM split apply to them whether or not the walk reached the end. Only the
# instruction to difference two files asserts completeness.
assert_file_contains 'no ept_violation counter' "$capture"
assert_file_contains 'per-VM and anonymous' "$capture"
rm -rf "$tmp"
}
@test "a warning beside a healthy read is not filed as a failure" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
mount_stderr='mount: (hint) source is ignored for debugfs.'
stage "$tmp"
run_capture
# mount writes hints to stderr and exits zero, exactly as kubectl and talosctl
# do. Which file the complaint lands in is decided by the status, so a healthy
# capture keeps its warning instead of shipping a failure marker beside a
# reading that succeeded.
dir="$COZY_REPORT_DIR/snapshots/kvm-smoke/sandbox-kvm-exits/sample-1"
assert_file_contains 'source is ignored for debugfs' "$dir/READ-WARNINGS.txt"
[ ! -f "$dir/COLLECTION-FAILED.txt" ]
assert_file_contains 'exits 1200' "$dir/kvm-stats.txt"
rm -rf "$tmp"
}
@test "a report directory that cannot be created is said out loud" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
# A file where the directory has to go. Every write below it then fails, and
# without a check the capture is a silent no-op -- the empty directory this
# whole suite exists to refuse, produced by the one cause the markers cannot
# describe, because there is nowhere to write a marker.
mkdir -p "$COZY_REPORT_DIR/snapshots/kvm-smoke"
printf 'not a directory\n' >"$COZY_REPORT_DIR/snapshots/kvm-smoke/sandbox-kvm-exits"
rc=0
out=$( ( set +x; cozy_capture_sandbox_kvm_exits 1 ) 2>&1 ) || rc=$?
[ "$rc" -ne 0 ]
case "$out" in
*"could not be created"*) ;;
*)
printf 'expected the collector to say the report directory could not be created, got: %s\n' "$out" >&2
return 1
;;
esac
rm -rf "$tmp"
}
@test "the sample number decides which reading a file belongs to" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
tmp=$(mktemp -d)
timeout_calls="$tmp/timeout.calls"
unshare_calls="$tmp/unshare.calls"
stage "$tmp"
run_capture 2
# The pair is the instrument: one reading of a running total is an average
# over its whole life, not a rate over the failure. Two readings filed under
# the same name would leave the
# second overwriting the first and the artifact reading like a single sample.
assert_file_contains 'exits 1200' "$(capture_path 2)"
[ ! -f "$(capture_path 1)" ]
rm -rf "$tmp"
}