#!/usr/bin/env bats # Regression coverage for the tenant-worker network counter capture in # hack/e2e-chainsaw/_lib/run-kubernetes.sh. cozytest.sh ends an @test block at # the first bare closing brace, so command mocks stay at top level. # # What this capture answers, and why the guest cannot answer it: a worker that # misses the node-Ready deadline while its image pull crawls has two mechanisms # behind the same guest-visible symptom. The link is slow, or the pull keeps # restarting and discarding what it fetched. From inside the guest both read as # a progress line that barely moves. The host-side byte counters separate them, # because they count every byte that crossed into the Pod including the ones a # restart threw away. cAdvisor reads these inside the Pod's network namespace # and publishes one row per interface, and a bridge-bound worker has four -- # the number that answers is the renamed NIC's receive, while the interface # still named eth0 is a dummy holding the Pod IP and reads zero. A capture that # reports an unread kubelet as a quiet link would hand the next reader that same # ambiguity while looking like it had settled it, so these tests pin the # separation rather than the reading. # # The fixtures below are real kubelet output with the identifying labels # rewritten, not lines composed from the metric documentation. That choice is # the subject of two of the tests: the family carries no container name and is # spelled packets_dropped, and both are invisible to anyone writing the filter # from memory. load test_helper kubectl_calls=/dev/null kubectl_node_names= kubectl_list_rc=0 kubectl_list_stderr= kubectl_raw_rc=0 kubectl_raw_stderr= # One worker Pod's set as the kubelet serves it: eight families, one row each # per interface, every one of them carrying container="" and cAdvisor's own # sample timestamp as the third field. Trimmed to the labels the filter reads # plus the ones that make the shape recognisable; the id/name digests are # synthetic and the image is the upstream pause reference rather than whichever # mirror the cluster it was taken from happened to use. # # Two different provenances here, and the difference matters enough to state. # The row shape and its values are real kubelet output with the identifying # labels rewritten. The four INTERFACE NAMES are not measured: they are what # KubeVirt bridge binding produces in a worker Pod netns -- a dummy carrying the # original name and the Pod IP, the real NIC renamed with a -nic suffix, the # bridge, and the tap -- read from the generators in the version this repo # ships. A virt-launcher Pod was not available to sample, so the names are # derived rather than observed, and the counters attached to them are # illustrative of direction, not of magnitude. worker_pod='virt-launcher-kubernetes-test-latest-version-md0-abc12-xyz34' worker_labels_for() { printf 'container="",id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod0000000a_0000_0000_0000_00000000000b.slice/cri-containerd-0000000000000000000000000000000000000000000000000000000000000001.scope",image="registry.k8s.io/pause:3.10",interface="%s",name="0000000000000000000000000000000000000000000000000000000000000001",namespace="tenant-test",pod="%s"' "$1" "${worker_pod}" } worker_labels="$(worker_labels_for eth0)" kubectl_raw_output= # Built rather than pasted so the eight families cannot drift apart by a typo in # one of them, and so a test that needs one family can name it. worker_series() { local labels="$1" local rx_bytes="${2:-4.4040192e+07}" local rx_packets="${3:-31284}" printf 'container_network_receive_bytes_total{%s} %s 1786657157281\n' "${labels}" "${rx_bytes}" printf 'container_network_receive_packets_total{%s} %s 1786657157281\n' "${labels}" "${rx_packets}" printf 'container_network_receive_packets_dropped_total{%s} 0 1786657157281\n' "${labels}" printf 'container_network_receive_errors_total{%s} 0 1786657157281\n' "${labels}" printf 'container_network_transmit_bytes_total{%s} 1.820267529e+09 1786657157281\n' "${labels}" printf 'container_network_transmit_packets_total{%s} 932294 1786657157281\n' "${labels}" printf 'container_network_transmit_packets_dropped_total{%s} 285 1786657157281\n' "${labels}" printf 'container_network_transmit_errors_total{%s} 0 1786657157281\n' "${labels}" } # The four interfaces a bridge-bound worker Pod presents. eth0 is the dummy and # reports zero; eth0-nic is the NIC whose receive answers the question. Both # have to survive the filter, or the artifact holds a number the reader cannot # interpret -- or worse, only the zero. bridge_bound_worker() { worker_series "$(worker_labels_for eth0)" 0 0 worker_series "$(worker_labels_for eth0-nic)" 4.4040192e+07 31284 worker_series "$(worker_labels_for k6t-eth0)" 4.4040192e+07 31284 worker_series "$(worker_labels_for tap0)" 1.2e+05 900 } # The node's own interfaces arrive in the same families with both selector # labels empty. Kept in the default fixture rather than staged only where it is # asserted: it is what the wire actually carries, and a filter that lets it # through would report the sandbox node's uplink under a worker heading. node_series() { printf 'container_network_receive_bytes_total{container="",id="/",image="",interface="eth0",name="",namespace="",pod=""} 1.066300417297e+12 1786657157281\n' printf 'container_network_transmit_bytes_total{container="",id="/",image="",interface="cilium_host",name="",namespace="",pod=""} 27870 1786657157281\n' } timeout_calls=/dev/null timeout_fail_node= kubectl() { printf '%s\n' "$*" >>"${kubectl_calls}" if [ "${3:-}" = get ] && [ "${4:-}" = pods ]; then [ -z "${kubectl_list_stderr}" ] || printf '%s\n' "${kubectl_list_stderr}" >&2 [ "${kubectl_list_rc}" -eq 0 ] || return "${kubectl_list_rc}" [ -z "${kubectl_node_names}" ] || printf '%s\n' ${kubectl_node_names} return 0 fi if [ "${1:-}" = get ] && [ "${2:-}" = --raw ]; then [ -z "${kubectl_raw_stderr}" ] || printf '%s\n' "${kubectl_raw_stderr}" >&2 # Output before status, for the reason the sibling suite gives: a read cut # off part way has already written what it managed to read, and a stub that # returns first makes the branch labelling a partial capture unreachable in # tests while it stays the likeliest one in production. [ -z "${kubectl_raw_output}" ] || printf '%s\n' "${kubectl_raw_output}" return "${kubectl_raw_rc}" fi return 0 } timeout() { local command_rc=0 printf '%s\n' "$*" >>"${timeout_calls}" # Return 97 rather than running the command when the wrapper is not the # bounded form: a read that lost its ceiling must not pass as a read. [ "${1:-}" = -k ] || return 97 shift 3 "$@" || command_rc=$? if [ -n "${timeout_fail_node}" ]; then case " $* " in *"${timeout_fail_node}"*) return 124 ;; esac fi return "${command_rc}" } assert_file_contains() { local needle="$1" local file="$2" if [ ! -f "${file}" ]; then printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${needle}" >&2 return 1 fi case "$(cat "${file}")" in *"${needle}"*) return 0 ;; esac printf 'expected %s to contain: %s\n' "${file}" "${needle}" >&2 return 1 } assert_file_lacks_pattern() { local pattern="$1" local file="$2" # A missing file must fail rather than vacuously pass: a bare `! grep -q` # succeeds on an unreadable path, which is indistinguishable from "the file # exists and does not carry the label". if [ ! -f "${file}" ]; then printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 return 1 fi # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for # "I could not evaluate this"; folded into an `if`, both land in the passing # branch and the assertion "this is absent" is satisfied by the matcher giving # up. Every negative claim in this file rests on the three-way split. local _rc=0 awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? case "${_rc}" in 0) printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 return 1 ;; 1) return 0 ;; *) printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 return 1 ;; esac } # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being # redirected, and every assertion about those files would be reading the tracer. # Call through this wrapper so the sinks hold what production writes. run_capture() { local _rc=0 ( set +x; cozy_capture_tenant_worker_network_counters ) || _rc=$? return "${_rc}" } @test "the capture does not send a reader after a sibling sample it never takes" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" # This capture is read once. Its sibling under the same shared walk is read # twice, and the walk is where a stamp would naturally be written for both -- # which is how a file could end up carrying, on consecutive lines, this # capture's own note that no second reading exists and an instruction to # subtract one. Two lines of the same artifact contradicting each other is the # reading failure this tree is written against, and it costs a reader the time # it takes to go looking for a directory that was never created. assert_file_contains 'a second capture of the same stream is what turns them into one' "$out" assert_file_lacks_pattern 'sibling' "$out" assert_file_lacks_pattern 'other sample directory' "$out" # The stamp itself stays: when the read happened is true of every caller, and # it is what makes a rate computable at all if a second capture is ever added. assert_file_contains 'read attempted from' "$out" rm -rf "$tmp" } @test "a negative assertion fails when its own matcher cannot evaluate" { # The helper under every "must not appear" claim in this file. What rides on # it is the collector's contract stated negatively: that an unread kubelet was # not written up as a quiet link, that a node interface did not land under a # worker heading. A matcher that fails open turns each of those into a # sentence nobody checked. tmp=$(mktemp -d) printf 'anything\n' >"$tmp/subject" if assert_file_lacks_pattern '*(' "$tmp/subject" 2>/dev/null; then echo "FAIL: an unparseable pattern was reported as absent" >&2 rm -rf "$tmp" return 1 fi assert_file_lacks_pattern 'absent-string' "$tmp/subject" if assert_file_lacks_pattern 'anything' "$tmp/subject" 2>/dev/null; then echo "FAIL: a present pattern was reported as absent" >&2 rm -rf "$tmp" return 1 fi rm -rf "$tmp" } @test "the eight families of a worker Pod are captured, packets_dropped included" { # The positive control every negative claim below is measured against. It # names the eight families one by one rather than counting lines, because a # count passes while a family is silently missing from the filter -- and the # dropped-packet families are the ones a filter written from memory loses, # since the obvious spelling for them is *_drops_total, which does not exist. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(node_series; worker_series "$worker_labels")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" for family in \ container_network_receive_bytes_total \ container_network_receive_packets_total \ container_network_receive_packets_dropped_total \ container_network_receive_errors_total \ container_network_transmit_bytes_total \ container_network_transmit_packets_total \ container_network_transmit_packets_dropped_total \ container_network_transmit_errors_total; do assert_file_contains "${family}{" "$out" done # And the value, so a filter that kept the family name while dropping the row # it was on cannot pass: 285 is the transmit drop count in the fixture. assert_file_contains ' 285 ' "$out" rm -rf "$tmp" } @test "the filter does not select on the container label, which is always empty here" { # The trap this collector is likeliest to acquire on its next edit. The CPU # counters beside it carry a real container name, so a filter copied from # there and tightened with container="compute" matches nothing at all -- and # nothing at all reads as a worker that moved no bytes. Every row of this # family carries container="", measured against a live kubelet, so the guard # is that the fixture's empty-container rows survive. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'container=""' "$out" rm -rf "$tmp" } @test "node interfaces and foreign namespaces are dropped, workers are kept" { # The node's own uplink arrives in the same eight families with pod="" and # namespace="", so a filter on the family name alone reports the sandbox # node's traffic under a heading that says worker. A Pod of another namespace # and a non-worker Pod of this one are the other two rows that must not pass. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" other_ns=$(printf 'container="",interface="eth0",namespace="cozy-system",pod="virt-launcher-elsewhere"') other_pod=$(printf 'container="",interface="eth0",namespace="tenant-test",pod="kubernetes-test-latest-version-kube-apiserver-0"') kubectl_raw_output="$(node_series; worker_series "$worker_labels"; worker_series "$other_ns"; worker_series "$other_pod")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains "pod=\"${worker_pod}\"" "$out" assert_file_lacks_pattern 'pod=""' "$out" assert_file_lacks_pattern 'cozy-system' "$out" assert_file_lacks_pattern 'kube-apiserver' "$out" rm -rf "$tmp" } @test "a kubelet that never answered is reported as unknown, not as a link that moved nothing" { # The conflation this collector exists to prevent. Zero bytes captured is the # same bytes on disk whether the worker received nothing or the kubelet was # never read, and only the note tells them apart. Keyed on the exit status # rather than on stderr, because `timeout` kills its child without a word and # kubectl dies on SIGTERM the same way, so the dominant failure arrives # non-zero with an empty error log. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output= kubectl_raw_rc=1 kubectl_raw_stderr='Error from server (Forbidden): nodes "node-a" is forbidden' run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'is unknown' "$out" assert_file_contains 'the kubelet was not read' "$out" assert_file_lacks_pattern 'reported no network series' "$out" # The trailing note is gated on series having arrived, and only the "it # appears" direction was pinned. Ungated it would attach "these counters are # cumulative..." to a capture holding no counters, which reads as counters # that exist. The CPU sibling pins its own note both ways for this reason. assert_file_lacks_pattern 'cumulative' "$out" assert_file_contains 'forbidden' \ "$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.read-error.log" rm -rf "$tmp" } @test "a kubelet that answered without worker rows says which of the two shapes it saw" { # Two empty results with different fixes. A namespace present but carrying no # virt-launcher-* Pod is a filter problem -- what an upstream rename of the # prefix looks like from inside -- and is fixed by looking at what the prefix # is now. A node with none of this namespace on it is a scheduling reading and # is fixed by looking at where the workers went. Naming the wrong one costs # the reader the finding. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" renamed=$(printf 'container="",interface="eth0",namespace="tenant-test",pod="compute-launcher-md0-abc12-xyz34"') kubectl_raw_output="$(worker_series "$renamed")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'none of them from a worker Pod named virt-launcher-*' "$out" assert_file_lacks_pattern 'no network series for a tenant-test worker' "$out" # And the other shape, on a node carrying nothing of this namespace. rm -rf "$tmp/snapshots" kubectl_raw_output="$(node_series)" run_capture assert_file_contains 'reported no network series for a tenant-test worker on this node' "$out" assert_file_lacks_pattern 'none of them from a worker Pod' "$out" rm -rf "$tmp" } @test "a filter that could not read its input is not reported as a kubelet with nothing to say" { # grep exits 1 for "read it, no match" and 2 for "could not read" -- a full # scratch directory on the runner reaches the second. Folded together they # land in the arm asserting the kubelet answered and carried no worker series, # which is a claim about the cluster manufactured by a local failure. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" # grep 2 is what an unreadable input produces; the stub stands in for it # without needing a full disk. Scoped to the calls that read a FILE, which is # what the three filter stages do -- the node listing pipes through a grep of # its own, and failing that one instead would empty the walk and test nothing. grep() { local _last eval "_last=\${$#}" if [ -f "${_last}" ]; then return 2 fi command grep "$@" } run_capture unset -f grep out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'could not be read back on this runner' "$out" assert_file_lacks_pattern 'reported no network series' "$out" rm -rf "$tmp" } @test "the counters are labelled as totals since the sandbox started, not as a rate" { # The reading this capture is most likely to be misread as. Every family here # is cumulative from the Pod's sandbox coming up, so a drop count of 285 says # nothing about the minutes the node-join deadline covered -- a link that # dropped nothing during the failure and a link that dropped throughout carry # the same total if the earlier traffic was the same. Saying so in the file is # the whole mitigation: a rate needs a second capture of the same stream, and # nothing in the artifact would otherwise say the number is not one. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'cumulative' "$out" rm -rf "$tmp" } @test "a listing that never answered is not reported as a namespace with no workers" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_list_rc=1 kubectl_list_stderr='Unable to connect to the server' rc=0 run_capture || rc=$? [ "$rc" -ne 0 ] err="$tmp/snapshots/kubernetes-latest/tenant-network-counters/COLLECTION-FAILED.txt" assert_file_contains 'failed to list' "$err" assert_file_contains 'Unable to connect to the server' "$err" rm -rf "$tmp" } @test "a namespace with no scheduled worker says so instead of writing an empty tree" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names= rc=0 run_capture || rc=$? [ "$rc" -ne 0 ] assert_file_contains 'no kubelet to ask' \ "$tmp/snapshots/kubernetes-latest/tenant-network-counters/COLLECTION-FAILED.txt" rm -rf "$tmp" } @test "the node walk stops at its cap and says how many carried a worker" { # A cap that drops nodes silently answers for part of the cluster while # reading as though it answered for all of it. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a node-b node-c node-d" kubectl_raw_output="$(worker_series "$worker_labels")" run_capture dir="$tmp/snapshots/kubernetes-latest/tenant-network-counters" assert_file_contains '4 carried a worker in total' "$dir/COLLECTION-TRUNCATED.txt" [ ! -f "$dir/node-d.txt" ] rm -rf "$tmp" } @test "the read is bounded and the timeout-absent fallback still reads" { # Bounding with a binary that is not there turns every read into an exit 127 # and every note into "the kubelet refused" -- a missing local dependency # reported as the cluster failing. The sibling collectors guard the call with # command -v for that reason and this one is held to the same shape. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" run_capture # Bounded: the wrapper saw the raw read, in the -k form the stub demands. grep -q 'metrics/cadvisor' "$timeout_calls" out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'container_network_receive_bytes_total{' "$out" # And without `timeout` on PATH the read still happens rather than exiting 127. rm -rf "$tmp/snapshots" command() { if [ "${1:-}" = -v ] && [ "${2:-}" = timeout ]; then return 1 fi builtin command "$@" } run_capture unset -f command assert_file_contains 'container_network_receive_bytes_total{' "$out" assert_file_lacks_pattern 'is unknown' "$out" rm -rf "$tmp" } @test "a read cut short keeps what arrived and marks it incomplete" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(worker_series "$worker_labels")" timeout_fail_node=node-a run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'these counters are incomplete' "$out" assert_file_contains 'container_network_receive_bytes_total{' "$out" assert_file_contains '[exit 124:' "$out" rm -rf "$tmp" } @test "a node read whose status never arrived is not written up as a kubelet that answered" { # The default behind `rc=${rc:-137}`, and the only route to it. The status # comes back through a command substitution now, so a subshell killed outright # returns nothing at all -- and the value chosen for that case decides which # sentence the artifact carries. Zero is the one value meaning "the kubelet # answered", so defaulting there would take the single conclusion this # collector may never manufacture and manufacture it from a local failure. # # Written because the default had none: flipping it back to `:-0` left every # suite in this tree green while a killed read was reported as a quiet link. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" # Writes the sinks and returns nothing on stdout: what a killed substitution # leaves behind. Stubbed at the helper because no mock of kubectl can produce # an empty command substitution while the function still runs. _cozy_cadvisor_node_stream() { : >"$2"; : >"$3"; } run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" assert_file_contains 'the kubelet was not read' "$out" assert_file_contains '[exit 137:' "$out" assert_file_lacks_pattern 'reported no network series' "$out" # And the trailing note stays away from a capture that holds no counters: it # describes numbers that are not there, and beside an unknown it reads as # counters that exist. assert_file_lacks_pattern 'cumulative' "$out" rm -rf "$tmp" } @test "every interface of a bridge-bound worker survives the filter, dummy and NIC alike" { # The reading this capture is most easily inverted by. A bridge-bound worker # presents four interfaces in its Pod netns, and the one named eth0 is a dummy # that holds the Pod IP and carries no traffic -- so it reports zero, and it is # the row a reader reaches for first because of its name. The number that # answers is the renamed NIC's receive. # # The filter therefore carries no interface predicate, and this pins that: # keeping only one interface would either drop the answer or leave the reader # holding the zero. Which row means what is stated where the collector is # gated; what this test guarantees is that both are in the artifact to compare. . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) export COZY_REPORT_DIR="$tmp" COZY_SNAPSHOT_NAME=kubernetes-latest kubectl_calls="$tmp/kubectl.calls" timeout_calls="$tmp/timeout.calls" kubectl_node_names="node-a" kubectl_raw_output="$(node_series; bridge_bound_worker)" run_capture out="$tmp/snapshots/kubernetes-latest/tenant-network-counters/node-a.txt" for iface in eth0 eth0-nic k6t-eth0 tap0; do assert_file_contains "interface=\"${iface}\"" "$out" done # The dummy really is staged at zero and the NIC is not, so "both survive" is # a statement about two distinguishable rows rather than two copies. grep -q "^container_network_receive_bytes_total{.*interface=\"eth0\".*} 0 " "$out" grep -q "^container_network_receive_bytes_total{.*interface=\"eth0-nic\".*} 4.4040192e+07 " "$out" # And the node's own interfaces still do not reach a worker heading. assert_file_lacks_pattern 'pod=""' "$out" rm -rf "$tmp" }