mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 12:43:32 +09:00
Add a talos.registryMirrors passthrough to the kubernetes and kubernetes-nodes
charts that renders into the worker TalosConfigTemplate machine.registries.mirrors,
so an operator can point a registry such as ghcr.io at an in-cluster pull-through
mirror and a worker boot no longer depends on live public egress. Default {}
keeps the rendered worker reconcile Job byte-identical to before, content-hash
Job name included. Free-form mirror values are shell-escaped for the unquoted
apply heredoc, covered by an execution-level bats test.
Also add the e2e ghcr-mirror (registry:2 pull-through plus a Cilium egress
policy) and wire it into the kubernetes chainsaw suites, so CI exercises the
mirror path and takes the flaky public ghcr.io kubelet-image pull off live
egress.
Refs #3513 (addresses the egress variant of the tenant-worker node-join
failures; the dominant variant is CI-runner CPU starvation, tracked in the
issue).
Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
494 lines
29 KiB
Bash
494 lines
29 KiB
Bash
# shellcheck shell=bash
|
|
# Shared helper: choose the Talos Image Factory base URL for tenant Kubernetes
|
|
# e2e CRs.
|
|
#
|
|
# Prefer the in-sandbox caching mirror (hack/e2e-talos-image-cache.yaml, deployed
|
|
# by hack/e2e-install-cozystack.bats) when a worker's CDI importer can actually
|
|
# reach it, otherwise fall back to the chart default (the public factory.talos.dev)
|
|
# by emitting nothing. The mirror rides out the public factory's flaky/range-less
|
|
# egress that otherwise stalls worker DataVolume imports past the 12-minute
|
|
# node-join deadline — see cozystack/cozystack#3231. Falling back to the default
|
|
# means the mirror can only help, never make CI worse.
|
|
#
|
|
# Reachability is not assumed. Tenant namespaces run under a default-deny Cilium
|
|
# egress installed by the tenant chart: a worker's importer may reach the `world`
|
|
# entity (hence the public factory) and the tenant's own tree, but not an
|
|
# arbitrary kube-system Service. Without an explicit allow, the importer resolves
|
|
# talos-image-cache.kube-system.svc yet its TCP connect to the ClusterIP is
|
|
# silently dropped and the disk import never starts. This helper installs that
|
|
# allow (the CiliumClusterwideNetworkPolicy shipped in the mirror manifest) and
|
|
# then verifies the path end to end from a Pod that faces the exact same egress
|
|
# rules as a real importer, so the mirror is used only when it genuinely works.
|
|
#
|
|
# Root cause + fix are from cozystack/cozystack#3254 (@lexfrei), ported here to
|
|
# the Chainsaw layout (hack/e2e-chainsaw/_lib/).
|
|
|
|
TALOS_IMAGE_CACHE_SVC_URL="${TALOS_IMAGE_CACHE_SVC_URL:-http://talos-image-cache.kube-system.svc}"
|
|
_TALOS_IMAGE_FACTORY_DECISION_FILE="${_TALOS_IMAGE_FACTORY_DECISION_FILE:-/tmp/e2e-talos-image-factory-url}"
|
|
TALOS_IMAGE_CACHE_MANIFEST="${TALOS_IMAGE_CACHE_MANIFEST:-hack/e2e-talos-image-cache.yaml}"
|
|
# Tenant namespace the kubernetes-* CRs are created in (run-kubernetes.sh and the
|
|
# oidc suites all target tenant-test); the reachability probe runs here so it
|
|
# inherits that namespace's default-deny egress, exactly as a worker importer does.
|
|
TALOS_IMAGE_CACHE_PROBE_NS="${TALOS_IMAGE_CACHE_PROBE_NS:-tenant-test}"
|
|
TALOS_IMAGE_CACHE_PROBE_POD="${TALOS_IMAGE_CACHE_PROBE_POD:-talos-image-cache-probe}"
|
|
|
|
# _apply_talos_image_cache_egress_policy: install the CiliumClusterwideNetworkPolicy
|
|
# that lets tenant CDI importer Pods egress to the mirror. It lives in the mirror
|
|
# manifest but is applied here, not in hack/e2e-install-cozystack.bats, because that
|
|
# manifest is applied before Cozystack (and thus before Cilium's CRDs) exists.
|
|
# Idempotent; best-effort — a failure just leaves the probe below to fall back.
|
|
_apply_talos_image_cache_egress_policy() {
|
|
command -v yq >/dev/null 2>&1 || return 1
|
|
yq 'select(.kind == "CiliumClusterwideNetworkPolicy")' "$TALOS_IMAGE_CACHE_MANIFEST" 2>/dev/null \
|
|
| kubectl apply -f - >/dev/null 2>&1
|
|
}
|
|
|
|
# _talos_image_cache_probe_overrides: emit the kubectl --overrides JSON for the
|
|
# probe Pod. $1 is the container image, $2 the shell command it runs. The single
|
|
# container replaces the one `kubectl run` generates (merge patch semantics), and
|
|
# the security context satisfies the tenant namespace's restricted PSA.
|
|
#
|
|
# Pure string builder, kept separate so it can be unit-tested: an edit that
|
|
# unbalances a quote here would emit invalid JSON, every probe would fail, and CI
|
|
# would silently fall back to the public factory instead of erroring.
|
|
_talos_image_cache_probe_overrides() {
|
|
printf '{"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"c","image":"%s","command":["sh","-ec","%s"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}' \
|
|
"$1" "$2"
|
|
}
|
|
|
|
# _talos_image_cache_probe_succeeded: succeed iff the probe output ($1) carries the
|
|
# strict 206 marker.
|
|
#
|
|
# Gate strictly on 206. The old localhost check kept a reachable-but-range-less
|
|
# mirror and only warned; here any non-206 — unreachable (code=000) or a plain 200 —
|
|
# falls back to the public factory instead. serve.py always answers a valid range
|
|
# with 206, so the 200 case is unreachable in practice; the strict gate keeps the
|
|
# "use the mirror only when it is fully working" rule simple.
|
|
#
|
|
# Pure predicate, kept separate so it can be unit-tested: this single comparison is
|
|
# what decides whether tenant workers are pointed at the mirror at all.
|
|
_talos_image_cache_probe_succeeded() {
|
|
printf '%s' "$1" | grep -q 'code=206'
|
|
}
|
|
|
|
# _talos_image_cache_reachable_from_tenant: succeed iff a Pod that faces the exact
|
|
# same tenant egress restrictions and network path as a worker's CDI importer can
|
|
# reach the mirror Service ClusterIP with a byte-range (206) response.
|
|
#
|
|
# This is the load-bearing gate: a green result guarantees a real importer will
|
|
# reach the mirror. It runs a throwaway Pod in the tenant namespace, labelled
|
|
# cdi.kubevirt.io=importer (so the tenant egress default-deny plus the allow
|
|
# policy above apply to it just as they do to a real importer), and has it curl
|
|
# the ClusterIP Service for the seeded image with a Range header — exercising DNS,
|
|
# the egress policy, ClusterIP translation, and range support together, not merely
|
|
# that the server answers on localhost.
|
|
#
|
|
# cdi.kubevirt.io=importer is CDI's own importer-pod label. The probe and the
|
|
# egress policy's endpointSelector must both use it; if a future CDI renames it,
|
|
# they would still agree with each other but no longer with reality, and the probe
|
|
# would pass while real importers stayed blocked. It is correct as of the CDI
|
|
# version this platform ships (packages/system/kubevirt-cdi-operator).
|
|
_talos_image_cache_reachable_from_tenant() {
|
|
local pod rel img curl_cmd overrides out
|
|
pod=$(kubectl -n kube-system get pod \
|
|
-l app.kubernetes.io/name=talos-image-cache \
|
|
--field-selector=status.phase=Running \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || return 1
|
|
[ -n "$pod" ] || return 1
|
|
# Discover the seeded file (its presence also confirms the seed finished) and
|
|
# probe that exact path, so the probe requests what a worker actually requests.
|
|
rel=$(kubectl -n kube-system exec "$pod" -c serve -- sh -ec \
|
|
'f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1); [ -n "$f" ] || exit 1; printf "%s" "${f#/data/}"' \
|
|
2>/dev/null) || return 1
|
|
[ -n "$rel" ] || return 1
|
|
# Reuse the mirror's own digest-pinned image for the probe Pod. It is the same
|
|
# image many CI hooks share, so by this late stage of the suite it is usually
|
|
# already cached on whatever node the probe lands on and no pull is needed; if
|
|
# the probe does land on a node without it, the pull is bounded by the Pod's
|
|
# --pod-running-timeout below and a timeout just falls back to the public factory.
|
|
img=$(kubectl -n kube-system get deploy talos-image-cache \
|
|
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null) || return 1
|
|
[ -n "$img" ] || return 1
|
|
|
|
# --retry rides out the seconds Cilium needs to program the freshly-applied allow
|
|
# policy; a persistently unreachable ClusterIP still fails fast per attempt
|
|
# (connect-timeout) and returns a non-206, falling back to the public factory.
|
|
# The total budget is deliberately generous: this decision is cached once for the
|
|
# whole suite, so a single slow policy-programming window must not permanently
|
|
# disable the mirror. --max-time still bounds a hung attempt (a genuinely
|
|
# unreachable ClusterIP resolves well within it and falls back).
|
|
curl_cmd="curl -s -o /dev/null -w code=%{http_code} --retry 8 --retry-all-errors --retry-delay 5 --connect-timeout 5 --max-time 45 -r 0-0 ${TALOS_IMAGE_CACHE_SVC_URL}/${rel}"
|
|
overrides=$(_talos_image_cache_probe_overrides "$img" "$curl_cmd")
|
|
|
|
kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \
|
|
--ignore-not-found >/dev/null 2>&1
|
|
# No --rm: with --rm the Pod is gone the instant it exits, and --attach can lose
|
|
# the tail of a fast-completing Pod's stream, leaving $out empty — a false negative
|
|
# that would fall back to the flaky public factory this mirror exists to avoid.
|
|
# Keep the Pod so its logs can be re-read if the attach stream was lost, then
|
|
# delete it explicitly below.
|
|
out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" run "$TALOS_IMAGE_CACHE_PROBE_POD" \
|
|
--restart=Never --attach --pod-running-timeout=90s \
|
|
--image="$img" --labels='cdi.kubevirt.io=importer' \
|
|
--overrides="$overrides" 2>&1) || true
|
|
# --attach returns only after the container exits, so its logs are ready now;
|
|
# fall back to them when the attach stream carried no result marker.
|
|
if ! printf '%s' "$out" | grep -q 'code='; then
|
|
out=$(kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" logs "$TALOS_IMAGE_CACHE_PROBE_POD" 2>/dev/null || true)
|
|
fi
|
|
# Fire-and-forget: the result is already captured in $out, so nothing depends on
|
|
# the Pod actually being gone. The pre-run delete above must NOT do this — there,
|
|
# `kubectl run` would race a still-terminating Pod of the same name.
|
|
kubectl -n "$TALOS_IMAGE_CACHE_PROBE_NS" delete pod "$TALOS_IMAGE_CACHE_PROBE_POD" \
|
|
--ignore-not-found --wait=false >/dev/null 2>&1
|
|
_talos_image_cache_probe_succeeded "$out"
|
|
}
|
|
|
|
# _talos_image_cache_deploy_state: answer "is the mirror Deployment there?" as
|
|
# exactly one of present, absent or unknown.
|
|
#
|
|
# --ignore-not-found is what makes a three-way answer possible: a genuine absence
|
|
# becomes exit 0 with no output, which leaves a non-zero exit meaning only "the
|
|
# question was never answered" — connection refused, Unauthorized, timed out.
|
|
# A plain `get` collapses those two into one exit code, and the caller below then
|
|
# reads a blip as an absence, caches it, and sends every tenant worker in the
|
|
# suite to the public factory over the live egress the mirror exists to avoid.
|
|
# Same idiom as the teardown poll in hack/e2e-chainsaw/_lib/etcd-cleanup.sh.
|
|
#
|
|
# talos_image_cache_diagnose asks the same question and answers it its own way,
|
|
# and the two are separate on purpose. That one runs on the node-join failure
|
|
# path under a phase budget, where re-asking spends what the crust-gather
|
|
# snapshot after it needs, and it gives up after one attempt. This one runs on
|
|
# the happy path, where a re-ask costs a suite nothing and buys it the mirror.
|
|
#
|
|
# The re-ask is small, and each attempt has to be bounded twice over for the
|
|
# re-ask to be worth anything. kubectl's --request-timeout defaults to 0 — no
|
|
# deadline on the request at all — and the case this loop exists for is a
|
|
# connection that establishes and then stalls, which client-go's dial timeout
|
|
# does not bound. The flag on its own is still not a wall-clock bound: against
|
|
# an unreachable apiserver kubectl retries discovery several times before it
|
|
# gives up, so a call carrying only the flag costs a multiple of it, which
|
|
# hack/e2e-chainsaw/_lib/pod-label-census.sh measured and sized its caller's op
|
|
# timeout around. Asking three times would inherit that multiple, inside a
|
|
# Chainsaw op the rest of the tenant test has to fit in. So each attempt takes
|
|
# both bounds, the pairing the reads in run-kubernetes.sh use and explain beside
|
|
# its pod list: --request-timeout bounds the HTTP request, timeout bounds the
|
|
# client retrying against a wedged apiserver. A kill by timeout exits non-zero
|
|
# and is simply another attempt that did not answer. Past that, an apiserver
|
|
# that is down does not become reachable by asking longer, and a run whose
|
|
# apiserver is down has bigger problems than which factory it pulls from. A
|
|
# real absence answers on the first try and waits for nothing.
|
|
#
|
|
# The wrapper is skipped rather than depended on when `timeout` is not installed,
|
|
# for the reason _talos_image_cache_bounded_read gives below and one more that is
|
|
# specific to a loop: its exit 127 would otherwise count as an attempt that did
|
|
# not answer, three times over, and a missing local binary would pin the whole
|
|
# suite to the public factory while every message blamed the apiserver.
|
|
#
|
|
# The outer bound sits ABOVE the inner one rather than level with it, and that
|
|
# gap is the whole reason kubectl ever gets to explain itself. Measured against a
|
|
# stub apiserver, the three ways this call fails are not alike. A refused port
|
|
# answers in under a second and names the refusal. One that completes TCP but
|
|
# stalls in TLS gets kubectl's own handshake deadline, which is shorter than
|
|
# either bound and fires twice inside the window, so there is text even though
|
|
# the wrapper ends up killing the call. A blackholed address — SYN dropped, no
|
|
# RST, which is what a wedged node or a netpol looks like — produces nothing at
|
|
# all until some deadline expires, and with the two bounds equal the expiry that
|
|
# arrives is the wall-clock kill, at the same instant as the request deadline it
|
|
# beats to it. `--request-timeout` then cannot fire in any of the three, which
|
|
# makes it decoration. Ten seconds of headroom hands that case back to kubectl:
|
|
# the request deadline ends the attempt and says why, and the wrapper stays as
|
|
# the backstop for a client that overruns its own deadline. The cost is ten more
|
|
# seconds per attempt on a path where the apiserver is already unreachable,
|
|
# against a 50m op.
|
|
#
|
|
# kubectl's stderr is therefore deliberately not swallowed: it is the only thing
|
|
# that separates refused from unauthorized from stalled, and suppressing it would
|
|
# replace one silent failure with another. It is still not guaranteed — a kill
|
|
# can always land before the client has said anything — so the last attempt's
|
|
# exit status is named too, in the vocabulary the diagnostic reads below use for
|
|
# 124 and 137. Between the two the caller's warning always has something real to
|
|
# point at.
|
|
_TALOS_IMAGE_CACHE_QUERY_TRIES="${_TALOS_IMAGE_CACHE_QUERY_TRIES:-3}"
|
|
_TALOS_IMAGE_CACHE_QUERY_DELAY="${_TALOS_IMAGE_CACHE_QUERY_DELAY:-5}"
|
|
_talos_image_cache_deploy_state() {
|
|
local out rc try=1
|
|
while :; do
|
|
rc=0
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
out=$(timeout -k 5 40 kubectl -n kube-system get deploy talos-image-cache \
|
|
--request-timeout=30s --ignore-not-found -o name) || rc=$?
|
|
else
|
|
out=$(kubectl -n kube-system get deploy talos-image-cache \
|
|
--request-timeout=30s --ignore-not-found -o name) || rc=$?
|
|
fi
|
|
if [ "$rc" -eq 0 ]; then
|
|
if [ -n "$out" ]; then
|
|
printf 'present'
|
|
else
|
|
printf 'absent'
|
|
fi
|
|
return 0
|
|
fi
|
|
if [ "$try" -ge "$_TALOS_IMAGE_CACHE_QUERY_TRIES" ]; then
|
|
echo "WARNING: the talos-image-cache lookup gave up after ${_TALOS_IMAGE_CACHE_QUERY_TRIES} attempts; the last one exited ${rc} (124 = cut off at the wall-clock bound, 137 = SIGKILL after it, and both of those print nothing of their own; any other status is kubectl's, and its message is above)" >&2
|
|
printf 'unknown'
|
|
return 0
|
|
fi
|
|
try=$((try + 1))
|
|
sleep "$_TALOS_IMAGE_CACHE_QUERY_DELAY"
|
|
done
|
|
}
|
|
|
|
# resolve_talos_image_factory_url: print the imageFactoryURL to use, or an empty
|
|
# string to signal "use the chart default". A decision that was actually reached
|
|
# — the mirror is there, or it demonstrably is not — is taken once (with a
|
|
# bounded wait for the mirror to finish seeding) and cached in a /tmp file that
|
|
# persists across bats files (they all exec in the same sandbox container), so
|
|
# only the first tenant Kubernetes test pays the readiness wait. A lookup that
|
|
# never got an answer decides nothing and caches nothing; the next tenant test
|
|
# asks again.
|
|
resolve_talos_image_factory_url() {
|
|
if [ -f "$_TALOS_IMAGE_FACTORY_DECISION_FILE" ]; then
|
|
cat "$_TALOS_IMAGE_FACTORY_DECISION_FILE"
|
|
return 0
|
|
fi
|
|
local url="" state
|
|
state=$(_talos_image_cache_deploy_state)
|
|
if [ "$state" = unknown ]; then
|
|
# Nothing was established, so nothing is cached: this caller falls back for
|
|
# itself and the next tenant test asks again.
|
|
echo "WARNING: could not determine whether the talos-image-cache mirror is deployed (the lookup's own account of why is above) — this test falls back to public factory.talos.dev; nothing is cached, so the next tenant test asks again" >&2
|
|
return 0
|
|
fi
|
|
if [ "$state" = present ]; then
|
|
# Available only after the seed initContainer has fully fetched the image, so
|
|
# this is the signal that the mirror is warm.
|
|
if kubectl -n kube-system rollout status deploy/talos-image-cache --timeout=12m >/dev/null 2>&1; then
|
|
# Available proves only that the seed finished and the server answers on
|
|
# localhost — NOT that a worker's importer, penned in by the tenant egress
|
|
# default-deny, can reach the ClusterIP. Install the allow policy (Cilium is
|
|
# up by now, unlike at seed-deploy time) and verify the whole importer path
|
|
# end to end. Point tenants at the mirror only when that check passes;
|
|
# otherwise fall back to the public factory so the mirror can only help,
|
|
# never point workers at an unreachable ClusterIP and make CI worse.
|
|
_apply_talos_image_cache_egress_policy || true
|
|
if _talos_image_cache_reachable_from_tenant; then
|
|
url="$TALOS_IMAGE_CACHE_SVC_URL"
|
|
echo "talos-image-cache mirror reachable from tenant namespace (byte-range 206 verified) — tenant workers import from ${url}" >&2
|
|
else
|
|
echo "WARNING: talos-image-cache is Available but a tenant-scoped probe could not reach its ClusterIP with a 206 — tenant workers fall back to public factory.talos.dev" >&2
|
|
fi
|
|
else
|
|
echo "WARNING: talos-image-cache mirror not Available in time — tenant workers fall back to public factory.talos.dev" >&2
|
|
fi
|
|
else
|
|
echo "talos-image-cache mirror not deployed — tenant workers use public factory.talos.dev" >&2
|
|
fi
|
|
printf '%s' "$url" > "$_TALOS_IMAGE_FACTORY_DECISION_FILE"
|
|
printf '%s' "$url"
|
|
}
|
|
|
|
# One value, so the wall-clock bound and the per-request bound cannot drift apart
|
|
# into a pair where the inner one outlasts the outer. Overridable for the same
|
|
# reason the caller's bounds are: a test that turns the caller's budgets down and
|
|
# leaves these at the real twenty seconds waits out a bound it thought it had
|
|
# lowered. Not read from the caller's COZY_DIAG_* -- this file is sourced before
|
|
# those exist and is also used on its own.
|
|
#
|
|
# Spelled out at each call site rather than precomputed into one prefix string,
|
|
# because a prefix built here is frozen at source time while the paired
|
|
# `--request-timeout` beside it expands at call time. Overriding the knob after
|
|
# sourcing -- which is the only moment a test can do it, and how the caller's own
|
|
# bounds are overridden -- would then move the inner bound and leave the outer at
|
|
# twenty seconds: exactly the drift the single value exists to prevent, arriving
|
|
# through the mechanism meant to make it adjustable.
|
|
#
|
|
# _talos_image_cache_seconds <value> <default> <name> [positive]: echo <value> when
|
|
# it is bare integer seconds, else echo <default> and say why on stderr, naming the
|
|
# knob the caller passed. `positive` additionally rejects zero.
|
|
#
|
|
# The name and the flag are parameters rather than constants because this validates
|
|
# two knobs with different rules, and the first version hardcoded one knob's name
|
|
# into all three messages -- so a bad grace was reported as a bad timeout, with the
|
|
# grace's default quoted beside the timeout's name. A note that sends the reader to
|
|
# the wrong knob is the same defect class this file exists to remove.
|
|
#
|
|
# Its own copy of the check rather than a call to run-kubernetes.sh's: that file
|
|
# sources this one, so nothing of its is defined yet here, and this file is also used
|
|
# on its own. A suffix would reach `--request-timeout=${...}s` as `2ms` and make every
|
|
# dump fail instantly; a leading zero is all digits and reads as octal in arithmetic
|
|
# and decimal elsewhere; and zero disables `timeout` outright while
|
|
# `--request-timeout=0s` means no timeout to kubectl, which is the unbounded read this
|
|
# file just stopped having.
|
|
_talos_image_cache_seconds() {
|
|
if [ -n "${4:-}" ] && [ "${1}" = 0 ]; then
|
|
echo "» WARNING: ignoring ${3}='${1}' (zero disables the bound instead of tightening it); using ${2}" >&2
|
|
printf '%s\n' "${2}"
|
|
return 0
|
|
fi
|
|
case "${1}" in
|
|
'' | *[!0-9]*)
|
|
echo "» WARNING: ignoring ${3}='${1}' (a bare integer, no unit suffix); using ${2}" >&2
|
|
printf '%s\n' "${2}"
|
|
;;
|
|
0?*)
|
|
echo "» WARNING: ignoring ${3}='${1}' (a leading zero reads as octal in arithmetic and as decimal elsewhere); using ${2}" >&2
|
|
printf '%s\n' "${2}"
|
|
;;
|
|
*) printf '%s\n' "${1}" ;;
|
|
esac
|
|
}
|
|
# Named once, for the reason its twin's are: a default passed as a literal at every
|
|
# call site leaves the re-checks falling back to a stale number when it changes, and
|
|
# only on the error path.
|
|
_TALOS_IMAGE_CACHE_READ_TIMEOUT_DEFAULT=20
|
|
_TALOS_IMAGE_CACHE_READ_GRACE_DEFAULT=5
|
|
_TALOS_IMAGE_CACHE_READ_TIMEOUT=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_TIMEOUT:-$_TALOS_IMAGE_CACHE_READ_TIMEOUT_DEFAULT}" "$_TALOS_IMAGE_CACHE_READ_TIMEOUT_DEFAULT" _TALOS_IMAGE_CACHE_READ_TIMEOUT positive)
|
|
|
|
# Validated too. A suffix really is legal for `timeout -k`, but that was never the
|
|
# hazard: `timeout -k abc 20` exits 125 before running the command, so a non-numeric
|
|
# grace makes every dump report 125 and collects nothing. Zero stays legal -- it only
|
|
# skips the follow-up SIGKILL, which kubectl does not need.
|
|
_TALOS_IMAGE_CACHE_READ_GRACE=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_GRACE:-$_TALOS_IMAGE_CACHE_READ_GRACE_DEFAULT}" "$_TALOS_IMAGE_CACHE_READ_GRACE_DEFAULT" _TALOS_IMAGE_CACHE_READ_GRACE)
|
|
|
|
# _talos_image_cache_bounded_read <label> <command...>: one bounded diagnostic
|
|
# dump, with the outcome named whenever it does not finish.
|
|
#
|
|
# A local twin of run-kubernetes.sh's cozy_diag_read rather than a call to it,
|
|
# for the reason given above: this file is sourced from the top of that one, about
|
|
# a thousand lines before COZY_DIAG_* exists, and it is also used on its own.
|
|
#
|
|
# The note is what makes bounding these dumps safe rather than a new way to
|
|
# mislead. `timeout` prints nothing when it fires, so a cut-off dump would leave a
|
|
# bare section header and nothing else -- and an empty
|
|
# `get deploy,pod,svc,endpointslice` section reads as a cache with no Pod, no
|
|
# Service and no EndpointSlice. That is a finding a triager acts on, drawn from a
|
|
# read that never reached the apiserver, which is the same mistake the gate above
|
|
# was rewritten to stop making. Callers run inside a `{ … } >&2` group, so a plain
|
|
# echo lands beside the dump it describes.
|
|
#
|
|
# Always returns 0: this whole function is best-effort and never fails its caller.
|
|
_talos_image_cache_bounded_read() {
|
|
local label="$1"
|
|
shift
|
|
local rc=0
|
|
|
|
# Unbounded when `timeout` is absent rather than bounded into exit 127, which
|
|
# would drop every dump and blame the cluster for a missing local binary. The
|
|
# caller says so once; see cozy_diag_read for the same shape and reason.
|
|
# Re-checked here as well as at assignment: a value set after this file was sourced
|
|
# -- which is how a test lowers it -- never passed that check, and zero there would
|
|
# hand `timeout` a duration that disables it.
|
|
_TALOS_IMAGE_CACHE_READ_TIMEOUT=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_TIMEOUT-}" "$_TALOS_IMAGE_CACHE_READ_TIMEOUT_DEFAULT" _TALOS_IMAGE_CACHE_READ_TIMEOUT positive)
|
|
_TALOS_IMAGE_CACHE_READ_GRACE=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_GRACE-}" "$_TALOS_IMAGE_CACHE_READ_GRACE_DEFAULT" _TALOS_IMAGE_CACHE_READ_GRACE)
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
timeout -k "$_TALOS_IMAGE_CACHE_READ_GRACE" "$_TALOS_IMAGE_CACHE_READ_TIMEOUT" "$@" || rc=$?
|
|
else
|
|
"$@" || rc=$?
|
|
fi
|
|
case "$rc" in
|
|
0) ;;
|
|
124)
|
|
echo "=== ${label}: read did not finish within ${_TALOS_IMAGE_CACHE_READ_TIMEOUT}s and was cut off; whatever it had not printed is absent from this log, not absent from the cluster ==="
|
|
;;
|
|
137)
|
|
echo "=== ${label}: read was killed before it finished (SIGKILL); whatever it had not printed is absent from this log, not absent from the cluster ==="
|
|
;;
|
|
*)
|
|
echo "=== ${label}: read failed (exit ${rc}); what it would have shown was not observed ==="
|
|
;;
|
|
esac
|
|
return 0
|
|
}
|
|
|
|
# talos_image_cache_diagnose: on-demand diagnostic for the node-join failure path
|
|
# in run-kubernetes.sh (Chainsaw-branch addition, not in #3254). Re-probes the
|
|
# mirror from the tenant namespace — the same path a real importer faces — and
|
|
# dumps the cacher + egress-policy state, so an operator can tell "the egress
|
|
# allow / ClusterIP path is broken" apart from "the upstream factory was slow".
|
|
# All output to stderr; never fails the caller.
|
|
#
|
|
# Its own reads are bounded because its only caller is the node-join failure path
|
|
# in run-kubernetes.sh, which ends in the `exit 1` that triggers the tenant
|
|
# crust-gather snapshot: a read that hangs here holds the Chainsaw op until the
|
|
# op is killed and the snapshot after it is lost rather than truncated.
|
|
#
|
|
# The reachability re-probe is the exception, and the exception is wider than the
|
|
# curl budget it is easy to mistake it for. _talos_image_cache_reachable_from_tenant
|
|
# issues seven unbounded management-cluster calls around that curl -- the Running-pod
|
|
# jsonpath get, the `exec` that finds the seeded file, the deploy-image jsonpath
|
|
# get, the pre-run `delete pod` that waits for deletion, the `run --attach` whose
|
|
# --pod-running-timeout bounds the wait for Running and not the attach stream, and
|
|
# the `logs` re-read and the fire-and-forget `delete pod` after it, whose
|
|
# --wait=false drops the wait for deletion and not the DELETE request -- and any one of them can hang on a wedged apiserver exactly
|
|
# as the reads below could before they were bounded.
|
|
#
|
|
# They are left alone here rather than bounded, and the reason is not that they
|
|
# are safe. That function also runs on the happy path from
|
|
# resolve_talos_image_factory_url, where its non-zero return disables the mirror
|
|
# for the whole suite, so changing when it gives up is a decision about which
|
|
# image factory tenant workers use and does not belong in a diagnostic's commit.
|
|
# What covers them meanwhile is the caller's phase budget, and only partly: the
|
|
# gate stops the re-probe from being STARTED once the phase is out of budget, and
|
|
# nothing stops it running long once started. Closing that surface is its own
|
|
# change, tracked in cozystack/cozystack#3666.
|
|
talos_image_cache_diagnose() {
|
|
# The gate is bounded like everything under it, and its outcomes are three
|
|
# rather than two: deployed, absent, and a read that did not answer.
|
|
#
|
|
# --ignore-not-found is what separates the last two. Without it "absent" and
|
|
# "the call failed" are the same non-zero status -- `kubectl get` exits 1 for a
|
|
# refused connection, for Unauthorized and for an unrecognised kind exactly as
|
|
# it does for NotFound -- so a read that never reached the apiserver was
|
|
# announced as "not deployed", a claim about the cluster drawn from a call that
|
|
# failed to make one, and it retired the cache hypothesis on the one path that
|
|
# exists to test it. With the flag, absent is exit 0 with empty output and
|
|
# every other non-zero is unknown.
|
|
# Validated once here, before the first read composes a `--request-timeout` string
|
|
# from it. The re-check inside _talos_image_cache_bounded_read corrects the wall
|
|
# clock, but each call site interpolates the value itself, so the first dump would
|
|
# otherwise pair a corrected `timeout` with a stale `--request-timeout=0s`.
|
|
_TALOS_IMAGE_CACHE_READ_TIMEOUT=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_TIMEOUT-}" "$_TALOS_IMAGE_CACHE_READ_TIMEOUT_DEFAULT" _TALOS_IMAGE_CACHE_READ_TIMEOUT positive)
|
|
_TALOS_IMAGE_CACHE_READ_GRACE=$(_talos_image_cache_seconds "${_TALOS_IMAGE_CACHE_READ_GRACE-}" "$_TALOS_IMAGE_CACHE_READ_GRACE_DEFAULT" _TALOS_IMAGE_CACHE_READ_GRACE)
|
|
local gate_rc=0 gate_out
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
gate_out=$(timeout -k "$_TALOS_IMAGE_CACHE_READ_GRACE" "$_TALOS_IMAGE_CACHE_READ_TIMEOUT" kubectl -n kube-system get deploy talos-image-cache \
|
|
--ignore-not-found -o name \
|
|
"--request-timeout=${_TALOS_IMAGE_CACHE_READ_TIMEOUT}s") || gate_rc=$?
|
|
else
|
|
gate_out=$(kubectl -n kube-system get deploy talos-image-cache --ignore-not-found -o name \
|
|
"--request-timeout=${_TALOS_IMAGE_CACHE_READ_TIMEOUT}s") || gate_rc=$?
|
|
fi
|
|
if [ "$gate_rc" -ne 0 ]; then
|
|
echo "could not read whether talos-image-cache is deployed (exit $gate_rc); whether the mirror was in play is unknown, not no" >&2
|
|
return 0
|
|
fi
|
|
if [ -z "$gate_out" ]; then
|
|
echo "talos-image-cache not deployed — tenant workers used the public factory; nothing to diagnose" >&2
|
|
return 0
|
|
fi
|
|
# The whole block goes to stderr, which is also where the notes belong, so the
|
|
# reads below need no redirect of their own: fd1 is already stderr in here and
|
|
# kubectl's own stderr is inherited to the same place.
|
|
{
|
|
if _talos_image_cache_reachable_from_tenant; then
|
|
echo "talos-image-cache reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (byte-range 206)"
|
|
else
|
|
echo "talos-image-cache NOT reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (tenant egress allow missing/unprogrammed, or cache down)"
|
|
fi
|
|
echo "--- cache deploy/pod/svc/endpointslice ---"
|
|
_talos_image_cache_bounded_read 'cache deploy/pod/svc/endpointslice' \
|
|
kubectl -n kube-system get deploy,pod,svc,endpointslice \
|
|
-l app.kubernetes.io/name=talos-image-cache -o wide "--request-timeout=${_TALOS_IMAGE_CACHE_READ_TIMEOUT}s"
|
|
echo "--- importer egress allow policy ---"
|
|
_talos_image_cache_bounded_read 'importer egress allow policy' \
|
|
kubectl get ciliumclusterwidenetworkpolicy \
|
|
e2e-talos-image-cache-importer-egress -o wide "--request-timeout=${_TALOS_IMAGE_CACHE_READ_TIMEOUT}s"
|
|
echo "--- serve container log (tail) ---"
|
|
_talos_image_cache_bounded_read 'serve container log' \
|
|
kubectl -n kube-system logs \
|
|
-l app.kubernetes.io/name=talos-image-cache -c serve --tail=50 --prefix
|
|
} >&2
|
|
}
|